diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..ed3c8ce --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - i3 + - openblade-production diff --git a/.github/workflows/_emulator-boot-test.yml b/.github/workflows/_emulator-boot-test.yml new file mode 100644 index 0000000..5dce2fe --- /dev/null +++ b/.github/workflows/_emulator-boot-test.yml @@ -0,0 +1,112 @@ +# Reusable: boot the mock-backend emulator API on 127.0.0.1:8000, wait for +# /health, then run pytest against it. Callers: emulator-change-gates.yml, +# i3-emulator-compliance.yml. Check names become " / boot-test". +name: _emulator-boot-test + +on: + workflow_call: + inputs: + timing_profile: + description: I3_TIMING_PROFILE for both the server and pytest. + type: string + default: instant + pytest_args: + description: >- + Space-separated pytest arguments (paths, -m, -q ...). Word-split by + the shell on purpose; must not contain quotes. + type: string + required: true + followup_pytest_args: + description: Optional second pytest run against the same booted API. + type: string + default: "" + python_version: + type: string + default: "3.12" + emulator_profile: + description: EMULATOR_PROFILE for the booted server and pytest ("" = default shape). + type: string + default: "" + validate_contract: + description: Run tools/emulator_spec/validate_cross_repo_contract.py first. + type: boolean + default: true + db_name: + description: Basename of the throwaway sqlite DB under /tmp. + type: string + default: openblade-emulator + artifact_name: + description: Upload test-results/ + the API log under this name ("" = no upload). + type: string + default: "" + +permissions: + contents: read + +jobs: + boot-test: + name: boot-test + runs-on: ubuntu-latest + env: + I3_TEST_MODE: emulator + I3_TIMING_PROFILE: ${{ inputs.timing_profile }} + I3_AML_URL: http://127.0.0.1:8000 + PYTEST_ARGS: ${{ inputs.pytest_args }} + FOLLOWUP_PYTEST_ARGS: ${{ inputs.followup_pytest_args }} + DB_PATH: /tmp/${{ inputs.db_name }}.db + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + # Job-wide EMULATOR_PROFILE only when one was given: an empty value must + # mean "unset" (default shape) for every later step, not "profile named ''". + - name: Export EMULATOR_PROFILE + env: + PROFILE: ${{ inputs.emulator_profile }} + run: | + if [ -n "$PROFILE" ]; then echo "EMULATOR_PROFILE=$PROFILE" >> "$GITHUB_ENV"; fi + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ inputs.python_version }} + cache: "pip" + - run: pip install -e '.[dev]' + - if: ${{ inputs.validate_contract }} + run: python3 tools/emulator_spec/validate_cross_repo_contract.py + - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py + - name: Boot emulator API + run: | + rm -f "$DB_PATH" + OPENBLADE_BACKEND=mock OPENBLADE_DB_URL="sqlite:///$DB_PATH" \ + nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & + - name: Wait for API health + run: | + python - <<'PY' + import time + import urllib.request + for _ in range(60): + try: + with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: + if response.status == 200: + raise SystemExit(0) + except Exception: + time.sleep(1) + raise SystemExit("API did not become healthy in time") + PY + - run: mkdir -p test-results + - name: Run pytest against the booted emulator + run: | + # shellcheck disable=SC2086 # PYTEST_ARGS is a word list by contract + python -m pytest $PYTEST_ARGS + - name: Follow-up pytest run + if: ${{ inputs.followup_pytest_args != '' }} + run: | + # shellcheck disable=SC2086 # word list by contract + python -m pytest $FOLLOWUP_PYTEST_ARGS + - name: Upload artifacts + if: ${{ always() && inputs.artifact_name != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact_name }} + path: | + test-results/ + /tmp/openblade-api.log diff --git a/.github/workflows/auto-merge-trusted.yml b/.github/workflows/auto-merge-trusted.yml index 8318a4c..62b82ba 100644 --- a/.github/workflows/auto-merge-trusted.yml +++ b/.github/workflows/auto-merge-trusted.yml @@ -1,13 +1,15 @@ name: Auto-merge trusted PRs -on: +# dangerous-triggers is ignored on `on:` below: this workflow runs only the GitHub API via +# github-script on the trusted-author allowlist; never checks out or executes +# PR code, which is what makes workflow_run dangerous. +on: # zizmor: ignore[dangerous-triggers] -- see comment above workflow_run: workflows: ["CI"] types: [completed] permissions: - contents: write - pull-requests: write + contents: read jobs: merge-trusted-pr: @@ -15,9 +17,12 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - name: Merge trusted PRs targeting master - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const owner = context.repo.owner; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 100067d..fd986d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,10 @@ jobs: cicd: ${{ steps.filter.outputs.cicd }} wiki: ${{ steps.filter.outputs.wiki }} steps: - - uses: actions/checkout@v5 - - uses: dorny/paths-filter@v3 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3.0.4 id: filter with: filters: | @@ -99,8 +101,10 @@ jobs: # goes red, the fix is `ruff check --fix` / `ruff format` with the PINNED # version, never a version bump in the same change. steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -122,8 +126,10 @@ jobs: # `# type: ignore` comments left in openblade/ (down from 18) carries its # justification at the site. steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -136,8 +142,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend == 'true' || needs.changes.outputs.api == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -162,8 +170,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.api == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -185,8 +195,10 @@ jobs: env: I3_TIMING_PROFILE: instant steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -216,8 +228,10 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: "npm" @@ -234,17 +248,16 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.webflask == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" - run: pip install -e '.[dev]' - # BLOCKING mypy, scoped to this package alone. `mypy openblade` in - # backend-typecheck has to stay advisory because of ~300 pre-existing errors - # in the god-file modules; openblade/web_flask is clean under strict mode - # today, and this job is what keeps it that way instead of letting it join - # the backlog. + # BLOCKING mypy, scoped to this package alone, so a web_flask-only change + # is gated here too (backend-typecheck is path-filtered to backend paths). - run: mypy openblade/web_flask # Redundant with backend-tests (tests/unit covers this file) but kept so the # job is a complete gate on its own for a web_flask-only change. Seconds. @@ -260,8 +273,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.wiki == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -278,8 +293,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.cicd == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - run: python -m pip install --upgrade pip pyyaml yamllint @@ -297,6 +314,26 @@ jobs: - name: yamllint workflows (real YAML errors — see .yamllint) run: yamllint -c .yamllint .github/workflows/ + # Changed-path ownership: every changed openblade/** or tests/** path must map + # to a test category in tools/ci_ownership.toml, so a new module can't land + # outside every CI lane unnoticed. Always runs (not path-filtered). + ownership: + name: ownership + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Check changed paths have an owning test category + env: + # PRs diff against their target branch; pushes/dispatch against master. + BASE_REF: ${{ github.base_ref || 'master' }} + run: python tools/ci_ownership.py --base "origin/$BASE_REF" + # Single always-running aggregation gate. Make THIS the only required # branch-protection check (plus operability-gate). It understands a valid # `skipped` result from path-filtered jobs and fails on any real failure or @@ -305,6 +342,7 @@ jobs: ci-gate: name: ci-gate needs: + - ownership - backend-lint - backend-typecheck - backend-tests @@ -326,6 +364,7 @@ jobs: *) echo "FAIL $1=$2"; fail=1 ;; esac } + check ownership "${{ needs.ownership.result }}" check backend-tests "${{ needs.backend-tests.result }}" check api-aml-integration "${{ needs.api-aml-integration.result }}" check i3-smoke "${{ needs.i3-smoke.result }}" @@ -360,8 +399,10 @@ jobs: env: I3_TIMING_PROFILE: instant steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 8f71130..807f1f0 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -9,6 +9,9 @@ on: paths: - .github/workflows/copilot-setup-steps.yml +permissions: + contents: read + jobs: copilot-setup-steps: runs-on: ubuntu-latest @@ -18,4 +21,6 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + # zizmor: ignore[artipacked] -- the Copilot coding agent works in this + # checkout and may need its git credentials; nothing here uploads artifacts. + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 diff --git a/.github/workflows/emulator-change-gates.yml b/.github/workflows/emulator-change-gates.yml index 7727d0b..f04c975 100644 --- a/.github/workflows/emulator-change-gates.yml +++ b/.github/workflows/emulator-change-gates.yml @@ -50,53 +50,38 @@ permissions: contents: read jobs: - emulator-contract: - name: emulator-contract + # The cross-repo contract (openblade/emulator_contract/contract.json) pins this + # exact command to this workflow file, so it stays here as an explicit step in + # addition to the reusable workflow's own validation. + contract-metadata: + name: contract-metadata runs-on: ubuntu-latest - env: - I3_TIMING_PROFILE: instant - I3_TEST_MODE: emulator - I3_AML_URL: http://127.0.0.1:8000 steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - run: python3 tools/emulator_spec/validate_cross_repo_contract.py - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - # Boot BEFORE the suites: this ordering was wrong since the workflow was - # written, masked for months because the contract-validator step above - # failed first — the suites below dial 127.0.0.1:8000. - - run: | - rm -f /tmp/openblade-emulator-gates.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-emulator-gates.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: > - python -m pytest - tests/i3/test_14_emulator_profile.py - tests/i3/test_18_config_profiles.py - tests/i3/test_15_manual_matrix_contract.py - tests/i3/test_16_manual_matrix_generated_suites.py - tests/i3/test_17_matrix_endpoint_coverage.py - tests/integration/test_aml_emulator_latency*.py - tests/integration/test_aml_scope_lockdown.py - tests/integration/test_aml_library_spec_parity.py - tests/unit/test_emulator_client_config.py - -q --tb=short - - run: python -m pytest tests/i3/ -m i3 -q --tb=short + + emulator-contract: + name: emulator-contract + needs: contract-metadata + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + db_name: openblade-emulator-gates + pytest_args: >- + tests/i3/test_14_emulator_profile.py + tests/i3/test_18_config_profiles.py + tests/i3/test_15_manual_matrix_contract.py + tests/i3/test_16_manual_matrix_generated_suites.py + tests/i3/test_17_matrix_endpoint_coverage.py + tests/integration/test_aml_emulator_latency*.py + tests/integration/test_aml_scope_lockdown.py + tests/integration/test_aml_library_spec_parity.py + tests/unit/test_emulator_client_config.py + -q --tb=short + followup_pytest_args: tests/i3/ -m i3 -q --tb=short diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml new file mode 100644 index 0000000..6a1902f --- /dev/null +++ b/.github/workflows/hardware-destructive.yml @@ -0,0 +1,187 @@ +name: Hardware destructive + +# Destructive lane against the physical Scalar i3 (docs/runbooks/hardware-lanes.md). +# Rehearsed first on mhvtl (mhvtl-weekly.yml runs the same allowlist + snapshot steps). +# +# Guard rails, in order: +# 1. environment `hardware-destructive` — required reviewers approve every run +# (configured in repo settings; a scheduled run waits for approval too); +# 2. rig guard — without HARDWARE_RIG_AVAILABLE only a NOT RUN summary job +# runs, so the run concludes *success*; this lane must never be a required +# check or a badge (docs/runbooks/hardware-lanes.md); +# 3. allowlist — exits non-zero before ANY hardware step if a barcode is +# outside vars.SACRIFICIAL_BARCODES, or the list is empty; +# 4. before/after inventory + drive snapshot, diffed and uploaded; +# 5. /jobs/recovery must be clean at the end. +# LTFS format only ever goes through the two-phase dry-run + safety-token flow. + +on: + schedule: + # 05:43 UTC every Saturday. Odd minute on purpose. + - cron: "43 5 * * 6" + workflow_dispatch: + inputs: + barcodes: + description: "Comma-separated cartridges to use; must be a subset of vars.SACRIFICIAL_BARCODES (default: all of them)" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' + runs-on: ubuntu-latest + steps: + - name: Record that the lane did not run + run: | + { + echo "## Hardware destructive: NOT RUN (neutral)" + echo + echo "HARDWARE_RIG_AVAILABLE is not 'true', so the destructive sequence was skipped." + echo "This is not a pass. See docs/runbooks/hardware-lanes.md." + } >> "$GITHUB_STEP_SUMMARY" + + destructive: + name: hardware-destructive + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + environment: hardware-destructive + # Job-level (after the environment gate): a run waiting for approval must + # not hold the rig slot. One rig, one lock — never two rig jobs at once, and + # never cancel one mid-sequence (that can strand a cartridge in a drive). + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false + timeout-minutes: 240 + env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would turn every step into a no-op + # that still reports success. + OPENBLADE_HARDWARE_DRY_RUN: "false" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} + SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES }} + # Never interpolated into a script: read from the environment only. + BARCODES: ${{ inputs.barcodes || vars.SACRIFICIAL_BARCODES }} + I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} + I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: pip install -e '.[dev]' + + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present + run: | + set -euo pipefail + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" + exit 1 + fi + + - name: Allowlist — refuse any cartridge outside the sacrificial set + id: allowlist + run: | + set -euo pipefail + # Export the tool's normalized list, never the raw input. + normalized="$(python -m tools.hardware.allowlist --barcodes "$BARCODES")" + echo "OPENBLADE_SCRATCH_BARCODES=$normalized" >> "$GITHUB_ENV" + + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots reports + python -m tools.hardware.snapshot capture --out snapshots/before.json + + - name: Load / unload + run: python -m pytest tests/hardware/test_changer_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_changer_operations.xml + + # Every LTFS test formats via FormatService.dry_run -> confirm(token), the + # two-phase dry-run + safety-token flow; none invokes mkltfs directly. + - name: LTFS write / read / checksum + run: python -m pytest tests/hardware/test_ltfs_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_ltfs_operations.xml + + - name: Archive + restore + run: python -m pytest tests/hardware/test_archive_restore.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_archive_restore.xml + + - name: Sharded write + restore + run: python -m pytest tests/hardware/test_sharded_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_sharded_operations.xml + + - name: Safe failure injection + env: + OPENBLADE_FAULT_TESTS: enabled + run: python -m pytest tests/hardware/test_fault_recovery.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_fault_recovery.xml + + # Runs only if every test step succeeded (no `if: always()`): a red test + # step already fails the job; this catches the green-by-skipping case. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate reports/*.xml + + # Reboot persistence is a MANUAL step (the runner cannot power-cycle the + # appliance): see docs/runbooks/hardware-lanes.md "Reboot persistence". + + - name: Recovery report must be clean + # Only once the allowlist passed: before that, nothing was touched. + if: always() && steps.allowlist.outcome == 'success' + run: | + set -euo pipefail + # Starting the app runs startup recovery against the same catalog the + # suite used; /jobs/recovery then reports what it had to repair. + python - <<'PY' + import json + import sys + + from fastapi.testclient import TestClient + + from openblade.api.main import app + + with TestClient(app) as client: + response = client.get("/jobs/recovery") + response.raise_for_status() + report = response.json() + print(json.dumps(report, indent=2)) + dirty = {k: v for k, v in report.items() if v} + if dirty: + print("::error::Recovery report is not clean: " + ", ".join(sorted(dirty))) + sys.exit(1) + PY + + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: hardware-destructive-${{ github.run_id }} + path: | + snapshots/ + reports/ + if-no-files-found: warn diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index 29e4a63..5301d21 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -1,123 +1,131 @@ name: Quantum hardware smoke +# Dispatchable subset of hardware-nightly-readonly.yml: the same read-only +# suite (tests/hardware/readonly/), on demand, on the same rig. +# +# Why kept rather than deleted: the workflow name and the job id +# `register-and-validate` are kept so any branch-protection rule or status +# check that references them keeps resolving. The old ubuntu-latest job could +# not reach a physical library anyway; the real check needs the [self-hosted, i3] +# runner (docs/runbooks/hardware-lanes.md). Motion/control-plane inputs were +# removed on purpose: motion belongs to hardware-destructive.yml, behind the +# environment approval and the cartridge allowlist. + on: workflow_dispatch: inputs: - target_aml_url: - description: "AML API base URL for the physical Quantum library" - required: true - type: string - target_model: - description: "Hardware model under test" - required: true - default: "Scalar i3" - type: choice - options: - - Scalar i3 - - Scalar i6 - - Other - target_name: - description: "Display name when registering with OpenBlade" - required: true - default: "Quantum Library Under Test" - type: string - target_serial: - description: "Serial number (optional, used for idempotent registration)" - required: false - default: "" - type: string - openblade_api_url: - description: "Optional OpenBlade API base URL for library registration" + test_filter: + description: "Optional pytest -k expression to narrow the read-only suite" required: false default: "" type: string - include_motion_tests: - description: "Include changer/move command checks in command matrix" - required: true - default: false - type: boolean - include_control_plane_checks: - description: "Validate /api/libraries command compatibility (requires OpenBlade target URL)" - required: true - default: false - type: boolean - timing_profile: - description: "Timing profile for real hardware smoke tests" - required: true - default: hardware - type: choice - options: - - instant - - realistic - - hardware permissions: contents: read jobs: - register-and-validate: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' runs-on: ubuntu-latest + steps: + - name: Record that the smoke did not run + run: | + { + echo "## Quantum hardware smoke: NOT RUN (neutral)" + echo + echo "HARDWARE_RIG_AVAILABLE is not 'true', so register-and-validate was skipped." + echo "This is not a pass. See docs/runbooks/hardware-lanes.md." + } >> "$GITHUB_STEP_SUMMARY" + + register-and-validate: + name: register-and-validate + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + # Job-level, so only a job that actually holds the rig takes the lock. + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false + timeout-minutes: 30 env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would make the lane a no-op. + OPENBLADE_HARDWARE_DRY_RUN: "false" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} + OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} - OPENBLADE_ADMIN_USER: ${{ secrets.OPENBLADE_ADMIN_USER }} - OPENBLADE_ADMIN_PASSWORD: ${{ secrets.OPENBLADE_ADMIN_PASSWORD }} steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - cache: "pip" + - run: pip install -e '.[dev]' - - name: Validate required secrets + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present run: | set -euo pipefail - if [ -z "${I3_AML_USER}" ] || [ -z "${I3_AML_PASSWORD}" ]; then - echo "::error::Set QUANTUM_AML_USER and QUANTUM_AML_PASSWORD repository secrets." + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" exit 1 fi - - name: Register target library in OpenBlade - if: ${{ inputs.openblade_api_url != '' }} + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + python -m tools.hardware.snapshot capture --out snapshots/before.json + + - name: Run the read-only suite env: - TARGET_URL: ${{ inputs.target_aml_url }} - TARGET_NAME: ${{ inputs.target_name }} - TARGET_MODEL: ${{ inputs.target_model }} - TARGET_SERIAL: ${{ inputs.target_serial }} - OPENBLADE_URL: ${{ inputs.openblade_api_url }} + TEST_FILTER: ${{ inputs.test_filter }} run: | set -euo pipefail - if [ -z "${OPENBLADE_ADMIN_USER}" ] || [ -z "${OPENBLADE_ADMIN_PASSWORD}" ]; then - echo "::error::Set OPENBLADE_ADMIN_USER and OPENBLADE_ADMIN_PASSWORD secrets to register libraries." - exit 1 + args=(tests/hardware/readonly/ -m real_hardware -v --tb=short -rs --junitxml=readonly-results.xml) + if [ -n "$TEST_FILTER" ]; then + args+=(-k "$TEST_FILTER") fi - python3 tests/i3/register_target_library.py \ - --openblade-url "${OPENBLADE_URL}" \ - --target-aml-url "${TARGET_URL}" \ - --target-name "${TARGET_NAME}" \ - --target-model "${TARGET_MODEL}" \ - --target-serial "${TARGET_SERIAL}" \ - --username "${OPENBLADE_ADMIN_USER}" \ - --password "${OPENBLADE_ADMIN_PASSWORD}" + python -m pytest "${args[@]}" - - name: Run real hardware command matrix - env: - I3_TEST_MODE: real - I3_REAL_HARDWARE_ENABLED: "true" - I3_AML_URL: ${{ inputs.target_aml_url }} - I3_TIMING_PROFILE: ${{ inputs.timing_profile }} - I3_COMMAND_MATRIX_INCLUDE_MOTION: ${{ inputs.include_motion_tests }} - I3_COMMAND_MATRIX_INCLUDE_CONTROL_PLANE: ${{ inputs.include_control_plane_checks }} - I3_COMMAND_MATRIX_REPORT: /tmp/i3-command-matrix.json + # Runs only after the suite succeeded (no `if: always()`): catches a + # green-by-skipping run. A -k filter deselects (absent from the XML), + # it does not skip, so it cannot trip this gate. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate readonly-results.xml + + - name: Snapshot inventory + drive state (after) and diff + if: always() run: | set -euo pipefail - python3 -m pytest tests/i3/test_00_command_matrix.py -q --tb=short - cat /tmp/i3-command-matrix.json + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json - - name: Upload hardware command matrix report + - name: Upload results, differential report and snapshots if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: i3-command-matrix - path: /tmp/i3-command-matrix.json + name: hardware-smoke-${{ github.run_id }} + path: | + readonly-results.xml + differential-report.json + snapshots/ + if-no-files-found: warn diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml new file mode 100644 index 0000000..413020b --- /dev/null +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -0,0 +1,127 @@ +name: Hardware nightly read-only + +# Read-only lane against the physical Scalar i3 (docs/runbooks/hardware-lanes.md). +# Every test in tests/hardware/readonly/ is read-only by construction (an AST +# safety test in that directory rejects load/unload/move/format/mount calls). +# The before/after snapshot diff is the runtime proof that nothing moved. +# +# Without a rig (HARDWARE_RIG_AVAILABLE != 'true') the hardware job is SKIPPED +# and the no-rig job writes a NOT RUN summary. The run then concludes +# *success* even though nothing was verified, so this lane must never be a +# required check or a badge (docs/runbooks/hardware-lanes.md). + +on: + schedule: + # 02:17 UTC nightly. Odd minute: top-of-hour schedules queue behind everyone. + - cron: "17 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' + runs-on: ubuntu-latest + steps: + - name: Record that the lane did not run + run: | + { + echo "## Hardware nightly read-only: NOT RUN (neutral)" + echo + echo "No i3 rig is registered for this repository (repo variable HARDWARE_RIG_AVAILABLE is not 'true')," + echo "so the read-only hardware job was skipped. This is not a pass: nothing was verified against the appliance." + echo "See docs/runbooks/hardware-lanes.md to register the self-hosted runner." + } >> "$GITHUB_STEP_SUMMARY" + + readonly: + name: hardware-readonly + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + # Job-level, so only a job that actually holds the rig takes the lock. + # Shared with the destructive lane and the smoke subset: a read-only pass + # must never observe a destructive run mid-sequence. + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false + timeout-minutes: 60 + env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would make the lane a no-op. + OPENBLADE_HARDWARE_DRY_RUN: "false" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} + OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json + I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} + I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: pip install -e '.[dev]' + + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present + run: | + set -euo pipefail + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" + exit 1 + fi + + # No allowlist step: this lane names no cartridges and performs no + # motion. The snapshot diff below is what would catch a violation. + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + python -m tools.hardware.snapshot capture --out snapshots/before.json + + - name: Run the read-only suite + run: | + set -euo pipefail + python -m pytest tests/hardware/readonly/ -m real_hardware -v --tb=short -rs \ + --junitxml=readonly-results.xml + + # Runs only after the suite succeeded (no `if: always()`): catches a + # green-by-skipping run. Add `--expected-skip classname::name` only with + # a comment saying why that skip is legitimate on the rig. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate readonly-results.xml + + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload results, differential report and snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: hardware-readonly-${{ github.run_id }} + path: | + readonly-results.xml + differential-report.json + snapshots/ + if-no-files-found: warn diff --git a/.github/workflows/i3-emulator-compliance.yml b/.github/workflows/i3-emulator-compliance.yml index 1625bf1..b62db2f 100644 --- a/.github/workflows/i3-emulator-compliance.yml +++ b/.github/workflows/i3-emulator-compliance.yml @@ -52,7 +52,6 @@ permissions: jobs: i3-emulator-suite: name: i3-emulator-${{ matrix.suite }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -63,56 +62,21 @@ jobs: tests: "tests/i3/test_06_operations.py tests/i3/test_07_ltfs.py tests/i3/test_08_archive_cycle.py tests/i3/test_09_restore_cycle.py" - suite: system tests: "tests/i3/test_10_fault_scenarios.py tests/i3/test_11_diagnostics.py tests/i3/test_12_multi_library.py tests/i3/test_13_ui_scenarios.py" - env: - I3_TEST_MODE: emulator - I3_TIMING_PROFILE: instant - I3_AML_URL: http://127.0.0.1:8000 - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - - run: python3 tools/emulator_spec/validate_cross_repo_contract.py - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - - run: | - rm -f /tmp/openblade-i3-compliance.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-i3-compliance.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: mkdir -p test-results - - run: pytest ${{ matrix.tests }} -q --tb=short --junitxml=test-results/i3-${{ matrix.suite }}.xml - - name: Upload i3 suite artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: i3-emulator-suite-${{ matrix.suite }} - path: | - test-results/i3-${{ matrix.suite }}.xml - /tmp/openblade-api.log + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + db_name: openblade-i3-compliance + pytest_args: ${{ matrix.tests }} -q --tb=short --junitxml=test-results/i3-${{ matrix.suite }}.xml + artifact_name: i3-emulator-suite-${{ matrix.suite }} i3-config-profiles: # Boot the emulator once per supported i3 configuration and assert it behaves # compliantly for that shape — the emulated i3 must behave like a real i3 across # the range of valid configurations (drive count, LTO generation, slot/module - # scaling, partition count). EMULATOR_PROFILE is set at job level so both the - # booted server and the pytest process observe the same shape. + # scaling, partition count). EMULATOR_PROFILE reaches both the booted server + # and the pytest process via the reusable workflow's emulator_profile input. name: i3-config-${{ matrix.profile }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -122,64 +86,32 @@ jobs: - scalar-i3-100-3 - scalar-i3-50-6 - scalar-i3-50-3-lto9 - env: - I3_TEST_MODE: emulator - I3_TIMING_PROFILE: instant - I3_AML_URL: http://127.0.0.1:8000 - EMULATOR_PROFILE: ${{ matrix.profile }} - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - - run: | - rm -f /tmp/openblade-i3-profile.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-i3-profile.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: mkdir -p test-results - - name: Run the i3 suite against the ${{ matrix.profile }} emulator - run: > - pytest - tests/i3/test_01_auth.py - tests/i3/test_02_inventory.py - tests/i3/test_03_changer.py - tests/i3/test_04_drives.py - tests/i3/test_05_media.py - tests/i3/test_14_emulator_profile.py - tests/i3/test_18_config_profiles.py - -q --tb=short --junitxml=test-results/i3-config-${{ matrix.profile }}.xml - - name: Upload i3 config-profile artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: i3-config-profile-${{ matrix.profile }} - path: | - test-results/i3-config-${{ matrix.profile }}.xml - /tmp/openblade-api.log + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + emulator_profile: ${{ matrix.profile }} + validate_contract: false + db_name: openblade-i3-profile + pytest_args: >- + tests/i3/test_01_auth.py + tests/i3/test_02_inventory.py + tests/i3/test_03_changer.py + tests/i3/test_04_drives.py + tests/i3/test_05_media.py + tests/i3/test_14_emulator_profile.py + tests/i3/test_18_config_profiles.py + -q --tb=short --junitxml=test-results/i3-config-${{ matrix.profile }}.xml + artifact_name: i3-config-profile-${{ matrix.profile }} aml-compatibility-suite: name: aml-compatibility runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -198,7 +130,7 @@ jobs: --junitxml=test-results/aml-compatibility.xml - name: Upload AML compatibility artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: aml-compatibility-suite path: test-results/aml-compatibility.xml diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 571e088..97b46c1 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -98,9 +98,11 @@ jobs: # ever trips, look at the per-step durations before raising it. timeout-minutes: 45 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -160,7 +162,7 @@ jobs: - name: Restore cached LTFS build id: ltfs-cache if: ${{ inputs.skip_ltfs != true }} - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /tmp/ltfs-stage.tar key: ${{ steps.ltfs-key.outputs.key }} @@ -276,7 +278,7 @@ jobs: lsscsi -g || true ls -l /dev/sg* /dev/nst* /dev/st* /dev/sch* 2>&1 || true lsmod | grep mhvtl || true - ps ax | grep -E 'vtltape|vtllibrary' | grep -v grep || true + pgrep -af 'vtltape|vtllibrary' || true systemctl status 'vtltape@*' 'vtllibrary@*' --no-pager 2>&1 | head -60 || true sudo dmesg | tail -80 || true cat /etc/mhvtl/device.conf || true @@ -306,6 +308,31 @@ jobs: printf '%s\n' "$block" \ | sed -n 's/^export \([A-Z_][A-Z0-9_]*\)=\(.*\)$/\1=\2/p' >> "$GITHUB_ENV" + # Rehearsal of the destructive lane's guard rails (hardware-destructive.yml): + # the same allowlist gate and before/after snapshot, so a broken guard is + # found here on a disposable VM before it matters on the real i3. + # Without the repo variable, the sacrificial set defaults to the scratch + # cartridges mhvtl loads for formatting (scripts/mhvtl/env.sh). + - name: Allowlist — refuse any cartridge outside the sacrificial set + env: + SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES || 'OB0007L8,OB0008L8' }} + run: | + set -euo pipefail + # Same contract as the i3 lane: export the tool's normalized list. + normalized="$(python -m tools.hardware.allowlist --barcodes "$OPENBLADE_SCRATCH_BARCODES")" + echo "OPENBLADE_SCRATCH_BARCODES=$normalized" >> "$GITHUB_ENV" + + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + sudo env \ + OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ + OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ + OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ + "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/before.json + - name: Run the hardware suite against the rig timeout-minutes: 20 env: @@ -325,16 +352,20 @@ jobs: env | grep '^OPENBLADE_' | sort if [ "${{ steps.ltfs.outputs.available }}" = "true" ]; then - suites="tests/hardware/" + suites=(tests/hardware/) else # Documented cut: everything below needs mkltfs to have formatted # the scratch cartridges. Without it those suites fail on mount # rather than skip, which would make the canary permanently red for # a reason that is not a product defect. - suites="tests/hardware/test_device_discovery.py \ - tests/hardware/test_changer_operations.py \ - tests/hardware/test_drive_health.py \ - tests/hardware/test_catalog_integrity.py" + # The drive-identity bridge is a mount DRY (no mkltfs needed). It + # skips (reported via -rs) unless OPENBLADE_APPLIANCE_* point at an + # AML endpoint, which this rig does not provide. + suites=(tests/hardware/readonly/test_drive_identity_bridge.py + tests/hardware/test_device_discovery.py + tests/hardware/test_changer_operations.py + tests/hardware/test_drive_health.py + tests/hardware/test_catalog_integrity.py) fi # `sudo env VAR=...` rather than `sudo -E`: -E depends on the host's @@ -349,18 +380,42 @@ jobs: OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ OPENBLADE_SCRATCH_BARCODES="$OPENBLADE_SCRATCH_BARCODES" \ OPENBLADE_FAULT_TESTS="${OPENBLADE_FAULT_TESTS:-}" \ - "$(command -v python)" -m pytest $suites \ + "$(command -v python)" -m pytest "${suites[@]}" \ -m real_hardware -v --tb=short -rs \ --junitxml=/tmp/mhvtl-hardware-results.xml - name: Upload hardware results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: mhvtl-hardware-results path: /tmp/mhvtl-hardware-results.xml if-no-files-found: warn + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + sudo env \ + OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ + OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ + OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ + "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: mhvtl-snapshots + path: snapshots/ + if-no-files-found: warn + - name: Tear the rig down if: always() run: sudo scripts/mhvtl/teardown.sh || true diff --git a/.github/workflows/nas-protocols.yml b/.github/workflows/nas-protocols.yml new file mode 100644 index 0000000..246b80d --- /dev/null +++ b/.github/workflows/nas-protocols.yml @@ -0,0 +1,46 @@ +name: NAS protocols + +# Nightly / manual only. Deliberately NOT a pull_request or push check: the rig +# builds Samba + NFS-Ganesha containers and needs a privileged client container. +on: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" + +permissions: + contents: read + +jobs: + protocols: + name: smb-nfs-protocol-rig + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Start protocol rig + run: make protocols-up + - name: Run protocol tests + run: make test-protocols + - name: Dump rig logs + if: failure() + run: | + mkdir -p protocol-logs + docker compose -f deploy/nas-protocols/docker-compose.yml ps > protocol-logs/ps.txt 2>&1 || true + docker compose -f deploy/nas-protocols/docker-compose.yml logs --no-color > protocol-logs/compose.log 2>&1 || true + - name: Upload rig logs + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nas-protocols-logs + path: protocol-logs/ + if-no-files-found: warn + - name: Tear down protocol rig + if: always() + run: make protocols-down diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..7ab23d6 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,111 @@ +# Nightly lanes that are too slow or too noisy for the PR path. Not a required +# PR check. Every lane is blocking within this workflow (no continue-on-error): +# a red nightly is a real regression to triage the next morning. +name: nightly + +on: + schedule: + - cron: "41 3 * * *" # odd minute: top-of-hour schedules queue behind everyone + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: false + +jobs: + slow: + # Tests excluded from full-suite by `not slow`. + name: nightly-slow + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Slow-marked tests + run: pytest -m "slow and not real_hardware and not real_i3" -q --tb=short + + stress: + # Fault-injection suite plus any stress-marked test elsewhere. + name: nightly-stress + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Fault-injection suite + run: pytest tests/fault -q --tb=short + - name: Stress-marked tests outside tests/fault + run: | + # Exit 5 = no tests collected: legitimate while nothing outside + # tests/fault carries the marker. Any other non-zero exit fails. + rc=0 + pytest tests --ignore=tests/fault -m "stress and not real_hardware and not real_i3" -q --tb=short || rc=$? + if [ "$rc" -ne 0 ] && [ "$rc" -ne 5 ]; then exit "$rc"; fi + + fuzz: + # Property-based (hypothesis) suite. + name: nightly-fuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Property tests + run: pytest tests/property -q --tb=short + + rebuild: + # Catalog rebuild from tape (incl. the worker). + name: nightly-rebuild + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Catalog rebuild tests + run: pytest tests/unit/test_catalog_rebuild.py tests/unit/test_catalog_rebuild_worker.py -q --tb=short + + mutation: + # mutmut over the safety-critical modules; fails on new survivors vs mutation/baseline.txt. + name: nightly-mutation + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Install mutmut + run: pip install mutmut==3.8.0 + - name: Mutation run + run: bash tools/mutation_run.sh diff --git a/.github/workflows/operability.yml b/.github/workflows/operability.yml index 0b52fd3..f883731 100644 --- a/.github/workflows/operability.yml +++ b/.github/workflows/operability.yml @@ -27,8 +27,10 @@ jobs: name: operability-gate runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da266b7..981470d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,15 @@ name: Release -# Tag-triggered release: builds the controller image, pushes it to GHCR, and -# cuts a GitHub Release from the tag. Runs on `v*` tag pushes only -- it never +# Tag-triggered release: build once -> SBOM + vuln scan -> sign + provenance -> +# verify -> staging (+ suites) -> approval -> production, all by image DIGEST. +# Stage-by-stage rationale and the rollback rule: docs/runbooks/release.md. Runs on `v*` tag pushes only -- it never # runs on a branch push, so it cannot race the version-bump automation in # other workflows. # # NOTE: this workflow has never been exercised by an actual tag push (no `v*` # tag exists on this repo yet). The first real run is what proves the GHCR # login/push and release-notes generation actually work end to end; until -# then, treat it as reviewed-but-unverified. See docs/wiki/guides/releasing.md +# then, treat it as reviewed-but-unverified. See docs/runbooks/release.md # for what to check if that first run fails, and keep changes to this file # small and easy to bisect for exactly that reason. @@ -29,64 +30,264 @@ permissions: contents: read jobs: - release: - name: build-and-release + # 1. Build ONCE. Every later job consumes needs.build.outputs.digest, never a tag. + build: runs-on: ubuntu-latest permissions: - contents: write # to create the GitHub Release from the tag - packages: write # to push images to ghcr.io + contents: read + packages: write # push the image to ghcr.io + outputs: + digest: ${{ steps.push.outputs.digest }} + version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v5 - + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" - name: Derive version from the tag id: version run: | tag="${GITHUB_REF#refs/tags/}" - version="${tag#v}" echo "tag=${tag}" >> "$GITHUB_OUTPUT" - echo "version=${version}" >> "$GITHUB_OUTPUT" - + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" - name: Cross-check tag against pyproject [project] version run: | pyproject_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" if [ "$pyproject_version" != "${{ steps.version.outputs.version }}" ]; then - echo "::error::tag v${{ steps.version.outputs.version }} does not match pyproject version ${pyproject_version} -- bump pyproject.toml and re-tag" + echo "::error::tag does not match pyproject version ${pyproject_version} -- bump pyproject.toml and re-tag" exit 1 fi - - - name: Log in to GHCR - uses: docker/login-action@v3 + - name: Build wheel + run: python -m pip install build && python -m build --wheel --outdir dist/ + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel + path: dist/*.whl + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Controller image: the root Dockerfile (FastAPI/Typer app). This is the - # only image this workflow builds -- see the Dockerfile.web note below. - - name: Build and push controller image - uses: docker/build-push-action@v6 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build and push controller image (once) + id: push + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . file: Dockerfile push: true - tags: | - ghcr.io/amantux/openblade:${{ steps.version.outputs.version }} - ghcr.io/amantux/openblade:latest + tags: ghcr.io/amantux/openblade:${{ steps.version.outputs.version }} + + # 2 + 3. SBOMs (image spdx-json, wheel cyclonedx) and vuln scan; HIGH+ fails. + scan: + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel + path: dist/ + - name: Resolve wheel dependencies + run: | + python -m venv /tmp/rel + /tmp/rel/bin/pip install dist/*.whl + /tmp/rel/bin/pip freeze --exclude openblade > resolved-requirements.txt + python -m pip install pip-audit + - name: Wheel SBOM (CycloneDX) + run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip -f cyclonedx-json -o wheel-sbom.cdx.json || true + - name: pip-audit (fails on any known vulnerability) + run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip --strict + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Image SBOM (SPDX) + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 + with: + image: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} + format: spdx-json + output-file: image-sbom.spdx.json + upload-artifact: false + - name: Trivy image scan by digest (CRITICAL/HIGH fail the release) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} + severity: CRITICAL,HIGH + exit-code: "1" + trivyignores: .trivyignore + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sbom + path: | + image-sbom.spdx.json + wheel-sbom.cdx.json + + # 4. Keyless sign + SLSA provenance. id-token: write exists ONLY on this job. + sign: + needs: [build, scan] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # push signature + attestation to ghcr.io + id-token: write # OIDC for keyless signing + attestations: write # SLSA provenance + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: cosign sign by digest + run: cosign sign --yes "${IMAGE}@${{ needs.build.outputs.digest }}" + - name: SLSA build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ needs.build.outputs.digest }} + push-to-registry: true - # `Dockerfile.web` is not tracked on master as of this workflow's - # authoring (only `frontend/Dockerfile` exists, consumed by - # docker-compose's `web` service, not published standalone to GHCR). - # If/when a top-level `Dockerfile.web` is added for a standalone web - # image, add a matching build-push step here tagged - # `ghcr.io/amantux/openblade-web:{version,latest}` -- do not invent one - # now. + verify: + needs: [build, sign] + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: cosign verify (expected release identity) + run: >- + cosign verify + --certificate-oidc-issuer https://token.actions.githubusercontent.com + --certificate-identity-regexp '^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$' + "${IMAGE}@${{ needs.build.outputs.digest }}" - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + release: + needs: [build, verify] + runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub Release + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + path: assets/ + merge-multiple: true + - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: - tag_name: ${{ steps.version.outputs.tag }} - name: ${{ steps.version.outputs.tag }} + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} generate_release_notes: true + body: "Image: `ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }}`" + files: assets/* + + # 5 + 6. Staging: deploy the digest, read-only postcheck, suites, rollback on failure. + deploy-staging: + needs: [build, verify] + runs-on: ubuntu-latest + environment: staging + permissions: + contents: read + packages: read + env: + OPENBLADE_IMAGE: ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }} + OPENBLADE_ENV: staging + I3_TIMING_PROFILE: instant + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel + path: dist/ + - name: Install the released wheel (+ test extras) + run: pip install "$(ls dist/*.whl)[dev]" + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: precheck -> deploy -> postcheck + run: >- + python scripts/deploy.py + --deploy-cmd "docker compose -f deploy/staging/docker-compose.yml up -d --wait" + --base-url http://localhost:8000 + - name: Read-only appliance postcheck + id: postcheck + run: python scripts/deploy.py postcheck --base-url http://localhost:8000 --read-only-appliance-checks + - name: Suites (emulator smoke, NAS journey, rebuild, restore) + run: >- + pytest -q --tb=short + tests/i3/test_01_auth.py tests/i3/test_02_inventory.py tests/i3/test_03_changer.py + tests/i3/test_12_multi_library.py tests/i3/test_14_emulator_profile.py + tests/e2e/test_nas_journey.py tests/unit/test_catalog_rebuild.py + tests/unit/test_catalog_rebuild_worker.py tests/e2e/test_mock_archive_restore.py + - name: Rollback on postcheck failure (application code only) + if: failure() && steps.postcheck.outcome == 'failure' && vars.STAGING_LAST_GOOD_DIGEST != '' + run: >- + python scripts/deploy.py rollback --to "${{ vars.STAGING_LAST_GOOD_DIGEST }}" + --deploy-cmd "docker compose -f deploy/staging/docker-compose.yml up -d --wait" + + # Required reviewers on the `production` environment are the approval gate + # (repo Settings -> Environments); this job waits there before any step runs. + deploy-production: + needs: [build, deploy-staging] + runs-on: [self-hosted, openblade-production] + environment: production + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + OPENBLADE_IMAGE: ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }} + OPENBLADE_ENV: production + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: cosign verify (expected release identity) + run: >- + cosign verify + --certificate-oidc-issuer https://token.actions.githubusercontent.com + --certificate-identity-regexp '^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$' + "${IMAGE}@${{ needs.build.outputs.digest }}" + - name: precheck -> deploy -> postcheck (same digest as staging) + run: >- + python3 scripts/deploy.py + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" + --base-url http://localhost:8000 + - name: Read-only appliance postcheck + id: postcheck + run: python3 scripts/deploy.py postcheck --base-url http://localhost:8000 --read-only-appliance-checks + - name: Rollback on postcheck failure (application code only) + if: failure() && steps.postcheck.outcome == 'failure' && vars.PRODUCTION_LAST_GOOD_DIGEST != '' + run: >- + python3 scripts/deploy.py rollback --to "${{ vars.PRODUCTION_LAST_GOOD_DIGEST }}" + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml new file mode 100644 index 0000000..5903cb5 --- /dev/null +++ b/.github/workflows/workflow-lint.yml @@ -0,0 +1,50 @@ +# Static checks for the workflows themselves: actionlint (incl. shellcheck on +# every inline run: block), zizmor (security audit) and yamllint. +name: workflow-lint + +on: + pull_request: + paths: + - ".github/workflows/**" + - ".yamllint" + push: + branches: [master] + paths: + - ".github/workflows/**" + - ".yamllint" + workflow_dispatch: + +permissions: + contents: read + +env: + ACTIONLINT_VERSION: "1.7.12" + # sha256 of actionlint_1.7.12_linux_amd64.tar.gz from the release checksums.txt + ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + +jobs: + workflow-lint: + name: workflow-lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install zizmor and yamllint (pinned) + run: python -m pip install zizmor==1.30.1 yamllint==1.38.0 + - name: Install actionlint (pinned, checksum-verified) + run: | + tarball="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -sSfL -o "$RUNNER_TEMP/$tarball" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/$tarball" + echo "$ACTIONLINT_SHA256 $RUNNER_TEMP/$tarball" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/$tarball" -C "$RUNNER_TEMP" actionlint + - name: actionlint (+ shellcheck on run blocks; shellcheck is preinstalled on ubuntu runners) + run: '"$RUNNER_TEMP/actionlint" -color' + - name: zizmor + run: zizmor --no-progress --min-severity low .github/workflows/ + - name: yamllint + run: yamllint -c .yamllint .github/workflows/ diff --git a/.gitignore b/.gitignore index 5f0f2e7..79c411f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ Thumbs.db # this into the repo root by default (OPENBLADE_PERF_RESULTS_FILE), so every # run of the hardware suite would otherwise dirty the worktree. .openblade_perf_results.json +mutants/ diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000..1478d1b --- /dev/null +++ b/.trivyignore @@ -0,0 +1,3 @@ +# Trivy ignore list for the release image scan (.github/workflows/release.yml). +# Every entry MUST carry a written justification on the line above it, an owner, +# and a review-by date. See docs/runbooks/release.md "Vulnerability exceptions". diff --git a/CLAUDE.md b/CLAUDE.md index 1ec5b26..21b338f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ Definition of done = `make all` green + relevant frontend checks + any parity ga - `ruff check .` is expected to be **clean (0)**. If you add a `# noqa`, it needs a written reason at the site — bare noqa is not accepted here. - `ruff format` changes go in their own format-only commit, never mixed with - logic. Known exception still outstanding: `openblade/assistant/tools.py`. + logic. (The former `openblade/assistant/tools.py` exception is resolved; the tree is fully formatted.) - Background + the deferred `BLE`/`S` rule families: docs/decisions/2026-09-11-pin-ruff-toolchain.md. diff --git a/Dockerfile b/Dockerfile index 6cd0525..b0dc4ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ mtx \ gosu \ + fuse3 \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml README.md ./ diff --git a/Makefile b/Makefile index 7a15dd5..0c792eb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash .ONESHELL: -.PHONY: up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware +.PHONY: mutation up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration test-contract seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware protocols-up protocols-down test-protocols # Start all services up: @@ -65,6 +65,11 @@ test-unit: test-integration: python3 -m pytest tests/integration/ -q --tb=short +# Run the backend contract suite against every backend pairing +# (the real+real pairing skips unless OPENBLADE_REAL_HARDWARE_ENABLED=true) +test-contract: + python3 -m pytest tests/contract/ -m contract -q --tb=short + # Seed default library instances (library-1, library-2, library-3) seed-libraries: python3 -c $$'from openblade.catalog.db import get_session, init_db\nfrom openblade.catalog.repository import CatalogRepository\nfrom openblade.config import load_config\n\ndb_url = load_config().db_url\ninit_db(db_url)\nrepo = CatalogRepository(get_session())\nfor name, url in [("library-1", "http://localhost:8010"), ("library-2", "http://localhost:8011"), ("library-3", "http://localhost:8012")]:\n existing = repo.get_library_instance_by_name(name)\n if existing:\n print(f"Skipped {name} (already exists)")\n continue\n repo.create_library_instance(name=name, emulator_url=url, model="Scalar i3")\n print(f"Created {name}")' @@ -106,3 +111,20 @@ all: lint test build test-hardware: OPENBLADE_BACKEND=real OPENBLADE_REAL_HARDWARE_ENABLED=true python3 -m pytest tests/hardware/ -v + +# NAS protocol rig (SMB/NFS, containerised). Host ports: OB_SMB_PORT / OB_NFS_PORT. +PROTOCOLS_COMPOSE = docker compose -f deploy/nas-protocols/docker-compose.yml + +protocols-up: + python -m tests.e2e.protocols.rig + OB_SMB_PORT=$${OB_SMB_PORT:-14450} OB_NFS_PORT=$${OB_NFS_PORT:-20490} $(PROTOCOLS_COMPOSE) up -d --build --wait + +protocols-down: + $(PROTOCOLS_COMPOSE) down -v --remove-orphans + +test-protocols: + python -m pytest -m protocols tests/e2e/protocols -q + +# Mutation lane (nightly); see tools/mutation_run.sh and mutation/baseline.txt +mutation: + PYTHON=$${PYTHON:-python} MUTATION_TIMEOUT=$${MUTATION_TIMEOUT:-3600} tools/mutation_run.sh diff --git a/README.md b/README.md index 5755345..fd7630c 100644 --- a/README.md +++ b/README.md @@ -19,16 +19,36 @@ OpenBlade is a simulator-first DIY tape archive controller inspired by iBlade-st 5. **Continuous verification**: enforce compatibility and regression evidence in CI/CD before changes land on `master`. ## Layered CI/CD (targeted) -OpenBlade CI/CD is intentionally split by layer so each change runs only relevant checks: +OpenBlade CI/CD is split by layer so that each change runs only the checks that matter for it. A single required check, `ci-gate`, then aggregates the results. -| Layer | Primary workflow/jobs | Trigger scope | +Backend contract suite: `make test-contract` runs `tests/contract/` against every backend pairing (simulator, in-process AML emulator, and real hardware when explicitly enabled). A new backend must pass it before it may be selected via `OPENBLADE_BACKEND`; see docs/test-plan.md "Backend contract suite". + +**PR lanes (`CI` / `ci.yml`).** These lanes run on pull requests, and `detect-changes` path-filters them: + +| Layer | Jobs | Trigger scope | | --- | --- | --- | -| API + backend domain | `CI`: `backend-lint`, `backend-typecheck`, `backend-tests`, `api-aml-integration` | `openblade/**/*.py`, AML integration tests, backend config | -| Simulator/emulator parity | `CI`: `i3-smoke`; `i3-emulator-compliance`; `emulator-change-gates` | simulator, AML routes, emulator contract/tools, i3 tests, compose/runtime wiring | -| Frontend/UI | `CI`: `frontend-build-test` (React SPA), `web-flask-smoke` (Flask NAS UI) | `frontend/**`; `openblade/web_flask/**`, `Dockerfile.web`, `docker-compose.yml` | -| CI/CD policy layer | `CI`: `cicd-workflow-validate` | `.github/workflows/**` | +| API + backend domain | `backend-lint` (ruff check + format), `backend-typecheck` (`mypy openblade`, strict), `backend-tests`, `api-aml-integration` | `openblade/**/*.py`, AML integration tests, backend config | +| Simulator/emulator parity | `i3-smoke`; `i3-emulator-compliance`; `emulator-change-gates` | simulator, AML routes, emulator contract/tools, i3 tests, compose/runtime wiring | +| Frontend/UI | `frontend-build-test` (React SPA), `web-flask-smoke` (Flask NAS UI) | `frontend/**`; `openblade/web_flask/**`, `Dockerfile.web`, `docker-compose.yml` | +| CI/CD policy | `cicd-workflow-validate`; `workflow-lint` | `.github/workflows/**` | + +- **Blocking.** Lint, format and `backend-typecheck` all block merge. None of these jobs uses `continue-on-error`, and `ci-gate` fails on any required job whose result is not `success` or `skipped`. +- **Ownership check.** `tools/ci_ownership.py --base origin/master` takes the changed paths from `git diff --name-only origin/master...HEAD`. It maps them through the glob table in `tools/ci_ownership.toml` to these categories: `unit`, `integration`, `safety`, `i3`, `compat`, `frontend`, `docs-only` and `ci-only`. It prints its reasoning, and it fails if a changed `openblade/**` or `tests/**` path has no owning category. +- **Emulator boot.** The reusable workflow `.github/workflows/_emulator-boot-test.yml` (`workflow_call`; inputs include `timing_profile`, `pytest_args` and `python_version`) boots the emulator, waits for it and runs pytest. `emulator-change-gates.yml` and `i3-emulator-compliance.yml` call it instead of copying those steps. + +**Workflow hygiene (`workflow-lint.yml`).** This workflow runs `actionlint` (with shellcheck on inline `run:` blocks), `zizmor` and `yamllint` (`.yamllint`). Every action is pinned to a full commit SHA with a `# vX.Y.Z` comment. Workflows declare `permissions: contents: read` at the top level and grant write scopes per job. + +**Nightly (`nightly.yml`).** Runs on a cron schedule and through `workflow_dispatch`. It is not a required PR check, but each of its lanes fails red: + +| Lane | Selection | +| --- | --- | +| `slow` | `-m slow` | +| `stress` | `tests/fault` + `-m stress` | +| `fuzz` | `tests/property` (`-m fuzz`) | +| `rebuild` | `tests/unit/test_catalog_rebuild*` (`-m rebuild`) | +| `mutation` | `tools/mutation_run.sh` (mutmut) over the safety-critical modules; ratchet against `mutation/baseline.txt`, so new surviving mutants fail the lane. Run it locally with `make mutation`. | -This keeps checks up to date and targeted while preserving full coverage on workflow dispatch and on emulator-specific workflows. +**i3 timing profiles.** `I3_TIMING_PROFILE` selects a profile for both the emulator and pytest. The names `instant`, `realistic` and `hardware` still work. The new profiles are `normal`, `slow-robotics`, `busy-library`, `intermittent-drive`, `session-expiry`, `rebooting` and `degraded-media`. Profile delays go through a `Clock` protocol (`RealClock`/`VirtualClock` in `tests/i3/timing.py`), so profile tests run deterministically under `VirtualClock` without real sleeps. ## Quick start ```bash diff --git a/compatibility/README.md b/compatibility/README.md index c3d797a..3fe7b2a 100644 --- a/compatibility/README.md +++ b/compatibility/README.md @@ -54,8 +54,51 @@ exact body match (use for `captured` cases where the full body is authoritative) ## Adding a real appliance capture (the goal) -1. Capture the sanitized request/response on the rig (headers, status, body). -2. Save it here with `"source": "captured"` and the firmware. -3. Run the harness. If the emulator diverges, either the emulator is wrong (fix it) - or the capture reveals a real contract we hadn't modeled (fix the emulator to - match). The emulator is required to satisfy the corpus — not the reverse. +Use `tools/compat_capture.py` (replays each recipe's `request` block, fills +`expected` from the observed response, sanitizes, writes `.json`). + +On site, against the appliance: + +```bash +python tools/compat_capture.py capture --base-url https:// \ + --recipe compatibility/scalar-i3-341g/aml --out compatibility/scalar-i3-341g/aml/captured \ + --source captured --appliance-firmware --attest "" --login : +``` + +- `--source captured` is REFUSED (exit 2, nothing written) unless BOTH + `--appliance-firmware` and `--attest` are given; they are recorded as + `firmware` / `attested_by`. Never label emulator output `captured`. +- Redacted before writing (`sanitize()`, pure + unit-tested): session tokens + (Bearer/Basic, cookie values, `session`/`token`-like JSON keys and XML tags), + serials (`serial*` keys/tags), hosts/IPs in URLs and text, ISO/HTTP-date + timestamps, and request passwords (`` = the `--login` credential, + `` = any other). Review the diff anyway before committing. +- Only response keys the sanitizer left untouched go into + `expected.json_contains`; the full scrubbed response is kept under `observed`. +- `captured` cases are ENFORCED by `tests/compat/test_compatibility_corpus.py`: + if the emulator diverges, fix the emulator (or record a contract we hadn't + modeled). The emulator must satisfy the corpus — not the reverse. Flip the + `len(CAPTURED) == 0` guard in that file when the first capture lands. + +Self-capture loop (emulator, `source: self-captured`, regression guard only): +`--diff` replays existing cases and prints drift without writing (exit 1 on +drift); then capture into `//self-captured/`. Against a live +base URL, the module-global login limiter persists between cases — prefer a +fresh process per run. + +## iblade-rev-a + +`iblade-rev-a/` is a scaffold profile for the Rev A (2017) iBlade surface. All its +cases are `inferred`; see `iblade-rev-a/README.md`. The harness discovers it +automatically (it globs `compatibility/**/*.json`). + +## Request/expectation keys beyond `json` + +- `request.body` — raw request body string, sent verbatim (use for XML or malformed + payloads). When present, `request.json` is ignored. +- `request.content_type` — sets the `Content-Type` header (overrides `request.headers`). + Omit it entirely to test a missing Content-Type. +- `request.repeat` — send the request this many extra times before the asserted one + (e.g. to trip the login rate limiter). The harness clears the limiter around each case. +- `expected.headers` — `{name: substring}`; each header must be present and contain the + substring (`""` asserts presence only). diff --git a/compatibility/iblade-rev-a/README.md b/compatibility/iblade-rev-a/README.md new file mode 100644 index 0000000..4ed55d9 --- /dev/null +++ b/compatibility/iblade-rev-a/README.md @@ -0,0 +1,11 @@ +# iblade-rev-a — Scalar-LTFS / Windows iBlade (Rev A, Sep 2017) + +Scope: the iBlade management surface (`/iblade/*`) as described by the local +iBlade reference manual, Rev A (2017). This profile is a **scaffold**. + +- Every case here is `inferred`: none has been checked against an appliance or + quoted from the manual. They lock current emulator behaviour only. +- The login, logout and 401 cases currently exercise the shared AML auth routes + (`/aml/users/login`, `/aml/auth/logout`) because no iBlade-specific auth path + is modelled yet. Replace them with `/iblade/*` paths once confirmed. +- Add `captured` cases next to these files, following `../README.md`. diff --git a/compatibility/iblade-rev-a/iblade/auth-login-success.json b/compatibility/iblade-rev-a/iblade/auth-login-success.json new file mode 100644 index 0000000..c79f1c9 --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-login-success.json @@ -0,0 +1,10 @@ +{ + "id": "iblade-login-success", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "fidelity_notes": "Scaffold: assumes the iBlade surface shares the AML login contract at /aml/users/login. Unverified.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" } }, + "expected": { "status": 200, "json_contains": { "code": 0, "description": "OK" } } +} diff --git a/compatibility/iblade-rev-a/iblade/auth-logout.json b/compatibility/iblade-rev-a/iblade/auth-logout.json new file mode 100644 index 0000000..d71e75e --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-logout.json @@ -0,0 +1,11 @@ +{ + "id": "iblade-logout", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "auth": "admin", + "fidelity_notes": "Scaffold: logout via the shared AML auth route. Unverified.", + "request": { "method": "POST", "path": "/aml/auth/logout" }, + "expected": { "status": 200 } +} diff --git a/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json b/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json new file mode 100644 index 0000000..5f55130 --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json @@ -0,0 +1,10 @@ +{ + "id": "iblade-unauthenticated-401", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "fidelity_notes": "Scaffold: unauthenticated request is rejected with 401. Unverified.", + "request": { "method": "GET", "path": "/aml/physicalLibrary/elements" }, + "expected": { "status": 401 } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json b/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json new file mode 100644 index 0000000..9fac14e --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json @@ -0,0 +1,4 @@ +{ "id": "aml-auth-expired-cookie", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "An unknown/expired sessionID cookie returns 401 and a Set-Cookie that clears it. Inferred; unverified against an appliance.", + "request": { "method": "GET", "path": "/aml/physicalLibrary/elements", "headers": { "Cookie": "sessionID=expired-session" } }, + "expected": { "status": 401, "headers": { "Set-Cookie": "Max-Age=0" } } } diff --git a/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json b/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json new file mode 100644 index 0000000..9fee9b4 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-malformed-xml", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Body sent with Content-Type application/xml but is not valid XML (the harness serialises json=), so it exercises the malformed-XML 400. A true XXE (DOCTYPE/ENTITY) case needs raw-body support in the harness; covered in integration tests instead.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Content-Type": "application/xml" } }, + "expected": { "status": 400, "json_contains": { "code": 400, "summary": "Malformed XML login payload" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json new file mode 100644 index 0000000..72e8fc2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json @@ -0,0 +1,11 @@ +{ + "id": "aml-auth-login-mfa-pending", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "setup": { "mfa": { "user": "admin", "type": "totp" } }, + "fidelity_notes": "Login by a user with an enabled MFA record returns the documented login body plus Warning: 199 - \"MFA Required\"; MFA never changes the status or body, in strict mode too (strict 412 only for the documented default-password condition); the session is pending (401 everywhere except POST /users/login/mfa, GET /users/login/mfa/{type}/key and logout) until a valid code is posted. The challenge wire shape is not documented in the Rev D manual; certify by appliance capture.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" } }, + "expected": { "status": 200, "headers": { "Warning": "MFA Required" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json b/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json new file mode 100644 index 0000000..e3a4c13 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-missing-content-type", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Missing Content-Type on login returns 415. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login" }, + "expected": { "status": 415, "json_contains": { "code": 415, "description": "Error", "summary": "Unsupported or missing Content-Type" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json b/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json new file mode 100644 index 0000000..faf8077 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json @@ -0,0 +1,4 @@ +{ "id": "aml-auth-login-rate-limited", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Repeated failed logins from one client return 429 with Retry-After. Default limit 10/300s. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "json": {"name": "admin", "password": "wrong"}, "repeat": 10 }, + "expected": { "status": 429, "headers": { "Retry-After": "" } } } diff --git a/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json b/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json new file mode 100644 index 0000000..6b9b355 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-unsupported-accept", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Unsupported Accept on login returns 406. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Accept": "image/png" } }, + "expected": { "status": 406, "json_contains": { "code": 406, "summary": "Unsupported Accept media type" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json b/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json new file mode 100644 index 0000000..16a8a06 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-xml-response", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "XML response (root WSResultCode) when Accept names an XML type and not application/json. The harness only parses JSON bodies and cannot send a raw XML request body, so this checks status only; the XML request path and XXE rejection are covered in tests/integration/test_aml_auth_fidelity.py.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Accept": "application/xml" } }, + "expected": { "status": 200 } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json b/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json new file mode 100644 index 0000000..60ffd41 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json @@ -0,0 +1,5 @@ +{ "id": "aml-auth-login-xxe-body", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "XML login body with a DOCTYPE/external entity is rejected with 400. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "content_type": "application/xml", + "body": "]>&x;p" }, + "expected": { "status": 400 } } diff --git a/compatibility/scalar-i3-341g/aml/logout.json b/compatibility/scalar-i3-341g/aml/logout.json index be77cee..c4695f4 100644 --- a/compatibility/scalar-i3-341g/aml/logout.json +++ b/compatibility/scalar-i3-341g/aml/logout.json @@ -5,7 +5,7 @@ "firmware": "341G", "auth": "admin", "manual_ref": "Web Services Guide Rev D §2.5 (logout)", - "fidelity_notes": "Emulator supports BOTH cookie-session (sessionID) and Bearer, with logout at POST /aml/auth/logout AND RESTful DELETE /aml/users/login, clearing the session cookie — closer to the real contract than the review assumed. Session-expiry, MFA, max-sessions, and default-password states remain unmodeled (roadmap Phase 5); certify the exact logout path/verb against a capture.", + "fidelity_notes": "Emulator supports BOTH cookie-session (sessionID) and Bearer, with logout at POST /aml/auth/logout AND RESTful DELETE /aml/users/login, clearing the session cookie — closer to the real contract than the review assumed. Session expiry (sliding timeout), per-user max-sessions eviction, the default-password Warning, and an inferred MFA-pending session (logout stays reachable while pending) are now modelled; all are emulator-inferred, not appliance-captured. Certify the exact logout path/verb against a capture.", "request": { "method": "POST", "path": "/aml/auth/logout" }, "expected": { "status": 200 } } diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json new file mode 100644 index 0000000..7c099b9 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json @@ -0,0 +1,48 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-expired-cookie", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "headers": { + "content-length": "169", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; Max-Age=0; Path=/; HttpOnly; SameSite=lax", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Cookie": "sessionID=" + }, + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json new file mode 100644 index 0000000..8de538b --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "Malformed XML login payload" + }, + "status": 400 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-malformed-xml", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "Malformed XML login payload" + }, + "headers": { + "content-length": "103", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Content-Type": "application/xml" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json new file mode 100644 index 0000000..5439cd2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json @@ -0,0 +1,44 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 415, + "customCode": 0, + "description": "Error", + "summary": "Unsupported or missing Content-Type" + }, + "status": 415 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-missing-content-type", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 415, + "customCode": 0, + "description": "Error", + "summary": "Unsupported or missing Content-Type" + }, + "headers": { + "content-length": "111", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json new file mode 100644 index 0000000..92b8db2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json @@ -0,0 +1,50 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Review the error details and retry.", + "code": "AML_ERROR", + "customCode": null, + "description": "Too many login attempts. Try again in 5 minutes.", + "summary": "Too many login attempts. Try again in 5 minutes." + }, + "status": 429 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-rate-limited", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": "Review the error details and retry.", + "code": "AML_ERROR", + "customCode": null, + "description": "Too many login attempts. Try again in 5 minutes.", + "summary": "Too many login attempts. Try again in 5 minutes." + }, + "headers": { + "content-length": "211", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "retry-after": "297", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login", + "repeat": 10 + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json new file mode 100644 index 0000000..74438c5 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 406, + "customCode": 0, + "description": "Error", + "summary": "Unsupported Accept media type" + }, + "status": 406 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-unsupported-accept", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 406, + "customCode": 0, + "description": "Error", + "summary": "Unsupported Accept media type" + }, + "headers": { + "content-length": "105", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Accept": "image/png" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json new file mode 100644 index 0000000..af077a2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json @@ -0,0 +1,40 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/xml" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-xml-response", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": "0OKLogin successful0", + "headers": { + "content-length": "184", + "content-type": "application/xml", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; HttpOnly; Max-Age=86400; Path=/; SameSite=lax", + "vary": "Origin", + "warning": "Default Password Supplied", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Accept": "application/xml" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json new file mode 100644 index 0000000..e52c712 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json @@ -0,0 +1,46 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "XML DTD/entity declarations are not allowed" + }, + "status": 400 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-xxe-body", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "XML DTD/entity declarations are not allowed" + }, + "headers": { + "content-length": "119", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "body": "]>&x;", + "content_type": "application/xml", + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json new file mode 100644 index 0000000..9e59e8a --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json @@ -0,0 +1,48 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 401, + "customCode": 0, + "description": "Error", + "summary": "Invalid credentials" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-login-invalid-credentials", + "manual_ref": "Web Services Guide Rev D \u00a72.5", + "observed": { + "body": { + "action": null, + "code": 401, + "customCode": 0, + "description": "Error", + "summary": "Invalid credentials" + }, + "headers": { + "content-length": "95", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json new file mode 100644 index 0000000..0711bf5 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Login successful" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-login-success", + "manual_ref": "Web Services Guide Rev D \u00a72.5 (users/login)", + "observed": { + "body": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Login successful", + "token": "" + }, + "headers": { + "content-length": "134", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; HttpOnly; Max-Age=86400; Path=/; SameSite=lax", + "vary": "Origin", + "warning": "Default Password Supplied", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json new file mode 100644 index 0000000..738f910 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json @@ -0,0 +1,46 @@ +{ + "auth": "admin", + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Logged out admin" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-logout-authed", + "manual_ref": "Web Services Guide Rev D \u00a72.5 (logout)", + "observed": { + "body": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Logged out admin" + }, + "headers": { + "content-length": "87", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; expires=; Max-Age=0; Path=/; SameSite=lax", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "POST", + "path": "/aml/auth/logout" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json new file mode 100644 index 0000000..74cd745 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json @@ -0,0 +1,1513 @@ +{ + "auth": "admin", + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "elementList": { + "element": [ + { + "address": 1, + "barcode": "VOL001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 2, + "barcode": "VOL002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 3, + "barcode": "VOL003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 4, + "barcode": "VOL004L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 5, + "barcode": "VOL005L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 6, + "barcode": "VOL006L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 7, + "barcode": "VOL007L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 8, + "barcode": "VOL008L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 9, + "barcode": "VOL009L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 10, + "barcode": "VOL010L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 11, + "barcode": "VOL011L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 12, + "barcode": "VOL012L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 13, + "barcode": "VOL013L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 14, + "barcode": "VOL014L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 15, + "barcode": "VOL015L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 16, + "barcode": "VOL016L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 17, + "barcode": "VOL017L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 18, + "barcode": "VOL018L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 19, + "barcode": "VOL019L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 20, + "barcode": "VOL020L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 21, + "barcode": "VOL021L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 22, + "barcode": "VOL022L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 23, + "barcode": "VOL023L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 24, + "barcode": "VOL024L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 25, + "barcode": "VOL025L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 26, + "barcode": "VOL026L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 27, + "barcode": "VOL027L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 28, + "barcode": "VOL028L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 29, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 30, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 31, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 32, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 33, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 34, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 35, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 36, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 37, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 38, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 39, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 40, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 41, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 42, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 43, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 44, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 45, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 46, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 47, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 48, + "barcode": "CLN001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 49, + "barcode": "CLN002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 50, + "barcode": "CLN003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 0, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + } + ] + } + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-physical-elements-authed", + "manual_ref": "Web Services Guide Rev D, Table 212 (physicalLibrary/elements)", + "observed": { + "body": { + "elementList": { + "element": [ + { + "address": 1, + "barcode": "VOL001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 2, + "barcode": "VOL002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 3, + "barcode": "VOL003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 4, + "barcode": "VOL004L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 5, + "barcode": "VOL005L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 6, + "barcode": "VOL006L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 7, + "barcode": "VOL007L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 8, + "barcode": "VOL008L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 9, + "barcode": "VOL009L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 10, + "barcode": "VOL010L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 11, + "barcode": "VOL011L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 12, + "barcode": "VOL012L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 13, + "barcode": "VOL013L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 14, + "barcode": "VOL014L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 15, + "barcode": "VOL015L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 16, + "barcode": "VOL016L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 17, + "barcode": "VOL017L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 18, + "barcode": "VOL018L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 19, + "barcode": "VOL019L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 20, + "barcode": "VOL020L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 21, + "barcode": "VOL021L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 22, + "barcode": "VOL022L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 23, + "barcode": "VOL023L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 24, + "barcode": "VOL024L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 25, + "barcode": "VOL025L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 26, + "barcode": "VOL026L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 27, + "barcode": "VOL027L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 28, + "barcode": "VOL028L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 29, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 30, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 31, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 32, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 33, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 34, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 35, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 36, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 37, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 38, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 39, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 40, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 41, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 42, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 43, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 44, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 45, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 46, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 47, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 48, + "barcode": "CLN001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 49, + "barcode": "CLN002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 50, + "barcode": "CLN003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 0, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + } + ] + } + }, + "headers": { + "content-length": "7423", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json new file mode 100644 index 0000000..dbbbfcc --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json @@ -0,0 +1,44 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-physical-elements-requires-auth", + "manual_ref": "Web Services Guide Rev D, Table 212 (physicalLibrary/elements)", + "observed": { + "body": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "headers": { + "content-length": "169", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/deploy/emulator/docker-compose.standalone.yml b/deploy/emulator/docker-compose.standalone.yml index fe6eeb3..74c9f58 100644 --- a/deploy/emulator/docker-compose.standalone.yml +++ b/deploy/emulator/docker-compose.standalone.yml @@ -22,6 +22,7 @@ x-runtime-defaults: &runtime-defaults EMULATOR_OCCUPANCY_PERCENT: ${EMULATOR_OCCUPANCY_PERCENT:-60} EMULATOR_LATENCY_PROFILE: ${EMULATOR_LATENCY_PROFILE:-instant} OPENBLADE_SCALAR_API_ONLY: ${OPENBLADE_SCALAR_API_ONLY:-true} + OPENBLADE_EMULATOR_FAULT_PROFILE: ${OPENBLADE_EMULATOR_FAULT_PROFILE:-} x-ui-target-defaults: &ui-target-defaults EMULATOR_UI_TARGET_LIBRARY1_URL: ${EMULATOR_UI_TARGET_LIBRARY1_URL:-http://openblade-emulator-library-1:8010} diff --git a/deploy/nas-protocols/.gitignore b/deploy/nas-protocols/.gitignore new file mode 100644 index 0000000..dc3b76b --- /dev/null +++ b/deploy/nas-protocols/.gitignore @@ -0,0 +1 @@ +state/ diff --git a/deploy/nas-protocols/Dockerfile b/deploy/nas-protocols/Dockerfile new file mode 100644 index 0000000..4df915f --- /dev/null +++ b/deploy/nas-protocols/Dockerfile @@ -0,0 +1,21 @@ +# The NFS server role needs a Ganesha built with VFS POSIX-ACL mapping so NFSv4 +# ACLs (nfs4_setfacl) work; Debian bookworm's nfs-ganesha 4.3 is built without it. +# Fedora 41's nfs-ganesha 5.9 links libacl into libfsalvfs.so and exports +# fsal_acl_2_posix_acl. fedora:41 pinned by digest (docker inspect RepoDigests). +FROM fedora@sha256:f1a3fab47bcb3c3ddf3135d5ee7ba8b7b25f2e809a47440936212a3a50957f3d AS nfs-ganesha +RUN dnf -y install --setopt=install_weak_deps=False "nfs-ganesha-5.9*" "nfs-ganesha-vfs-5.9*" rpcbind \ + && dnf clean all +COPY hydrator_shim.py /usr/local/bin/hydrator_shim.py + +# samba and client roles: one pinned Debian base, the role chosen by build arg. +# debian:bookworm-slim pinned by digest (docker inspect RepoDigests). +# Last stage, so it stays the default build target. +FROM debian@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 AS debian-roles +ARG ROLE +RUN apt-get update \ + && if [ "$ROLE" = samba ]; then pkgs="samba smbclient"; \ + else pkgs="smbclient cifs-utils nfs-common nfs4-acl-tools python3 python3-pip attr procps"; fi \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $pkgs \ + && apt-get clean \ + && if [ "$ROLE" = client ]; then pip3 install --break-system-packages --no-cache-dir smbprotocol==1.15.0; fi +COPY hydrator_shim.py /usr/local/bin/hydrator_shim.py diff --git a/deploy/nas-protocols/docker-compose.yml b/deploy/nas-protocols/docker-compose.yml new file mode 100644 index 0000000..2232fdb --- /dev/null +++ b/deploy/nas-protocols/docker-compose.yml @@ -0,0 +1,35 @@ +# OpenBlade NAS protocol test rig (roadmap item 19). `make protocols-up`. +# The harness (tests/e2e/protocols/conftest.py) renders state/smb.conf and +# state/ganesha.conf through openblade.nas.samba / openblade.nas.nfs and +# populates state/export/ before bringing the rig up. +name: ob-nas-protocols +services: + samba: + build: { context: ., args: { ROLE: samba } } + command: ["smbd", "--foreground", "--no-process-group", "--debug-stdout", "-s", "/rig/smb.conf"] + volumes: ["./state:/rig"] + ports: ["${OB_SMB_PORT:-14450}:445"] + healthcheck: + test: ["CMD", "smbclient", "-N", "-L", "localhost", "-m", "SMB3"] + interval: 2s + retries: 30 + nfs-ganesha: + build: { context: ., target: nfs-ganesha, args: { ROLE: nfs } } + # Ganesha's VFS FSAL needs open_by_handle_at (CAP_DAC_READ_SEARCH) etc. + privileged: true + command: ["sh", "-c", "rpcbind && exec ganesha.nfsd -F -L STDOUT -p /tmp/ganesha.pid -f /rig/ganesha.conf"] + volumes: ["./state:/rig"] + ports: ["${OB_NFS_PORT:-20490}:2049"] + healthcheck: + test: ["CMD-SHELL", "grep -qi ':0801 ' /proc/net/tcp /proc/net/tcp6"] + interval: 2s + retries: 30 + client: + build: { context: ., args: { ROLE: client } } + # Kernel cifs/nfs4 mounts need CAP_SYS_ADMIN: privileged client container. + privileged: true + command: ["python3", "/usr/local/bin/hydrator_shim.py", "/rig/export/rw"] + volumes: ["./state:/rig"] + depends_on: + samba: { condition: service_healthy } + nfs-ganesha: { condition: service_healthy } diff --git a/deploy/nas-protocols/hydrator_shim.py b/deploy/nas-protocols/hydrator_shim.py new file mode 100644 index 0000000..d51a405 --- /dev/null +++ b/deploy/nas-protocols/hydrator_shim.py @@ -0,0 +1,77 @@ +"""Hydrator shim for the protocol rig (NOT the real hydrator). + +The real hydrator is the FUSE path (openblade/fuse): a read of an offline file +blocks inside the filesystem until tape recall completes. This rig exports a +plain directory, so an offline file is a stub whose content starts with +STUB_MAGIC followed by a JSON body {"state": "offline_on_tape", "mode": ...}. +A client asks for recall by creating `/.ob-control/.req`; then, +per the stub's mode: + "delay:" -> after seconds atomically replace the stub with the bytes + in `/.ob-tape/` (simulated tape recall), + "never" -> leave it offline (hydration timeout), + "tape_error" -> write `.ob-control/.err` = OFFLINE_TAPE_UNAVAILABLE. +Each recall is logged once to `.ob-control/hydration.log`; requests for a path +already in flight are coalesced (one hydration event per path). + +The stub format is rig-only. The catalog has NO on-disk form for +NasFileState.OFFLINE_ON_TAPE (openblade/nas/types.py): no stub, sidecar or +xattr. An offline file exists only as a catalog NasFileRecord. /virtual reports +it with the catalog size and status, and a read raises CartridgeOfflineError. +The FUSE mount shows it read-only (0444) at catalog size. A read returns EIO, or +with --hydrate blocks until restore. So this stub only stands in for "offline" +when there is no FUSE layer. Its content, unlike FUSE, does not keep the +catalog size. +""" + +import json +import os +import sys +import threading +import time +from pathlib import Path + +STUB_MAGIC = b"OPENBLADE-OFFLINE-STUB\n" + + +def _hydrate(root: Path, rel: str, mode: str, inflight: set[str]) -> None: + target = root / rel + if mode == "tape_error": + (root / ".ob-control" / f"{rel}.err").write_text("OFFLINE_TAPE_UNAVAILABLE") + elif mode.startswith("delay:") or mode == "immediate": + if mode.startswith("delay:"): + time.sleep(float(mode.split(":", 1)[1])) + tmp = target.with_name(f".{target.name}.hydrating") + tmp.write_bytes((root / ".ob-tape" / rel).read_bytes()) + os.replace(tmp, target) + with (root / ".ob-control" / "hydration.log").open("a") as log: + log.write(f"hydrated {rel}\n") + inflight.discard(rel) + + +def main(root: Path) -> None: + control = root / ".ob-control" + inflight: set[str] = set() + while True: + control.mkdir(parents=True, exist_ok=True) + for req in sorted(control.rglob("*.req")): + rel = str(req.relative_to(control))[: -len(".req")] + req.unlink(missing_ok=True) + data = (root / rel).read_bytes() if (root / rel).is_file() else b"" + if rel in inflight or not data.startswith(STUB_MAGIC): + continue + # The shim is the client container's main process: one malformed or + # partially written stub must never take the whole rig down. + try: + body = json.loads(data[len(STUB_MAGIC) :] or b"{}") + mode = str(body.get("mode", "immediate")) if isinstance(body, dict) else "immediate" + except ValueError: + mode = "immediate" + inflight.add(rel) + with (control / "hydration.log").open("a") as log: + log.write(f"recall {rel} mode={mode}\n") + threading.Thread(target=_hydrate, args=(root, rel, mode, inflight), daemon=True).start() + time.sleep(0.1) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml new file mode 100644 index 0000000..e1bc85f --- /dev/null +++ b/deploy/production/docker-compose.yml @@ -0,0 +1,18 @@ +# OpenBlade production stack -- ALWAYS pinned by digest, never by tag. +# The release workflow (and scripts/deploy.py rollback) set OPENBLADE_IMAGE to +# ghcr.io/amantux/openblade@sha256:<64 hex> +# Compose refuses to start if it is unset, so a mutable tag can't sneak in. +services: + openblade: + image: ${OPENBLADE_IMAGE:?set OPENBLADE_IMAGE to ghcr.io/amantux/openblade@sha256:} + ports: + - "8000:8000" + environment: + OPENBLADE_ENV: production + OPENBLADE_DB_URL: sqlite:////data/openblade.db + volumes: + - openblade-production-data:/data + restart: unless-stopped + +volumes: + openblade-production-data: diff --git a/deploy/staging/docker-compose.yml b/deploy/staging/docker-compose.yml new file mode 100644 index 0000000..28e15bf --- /dev/null +++ b/deploy/staging/docker-compose.yml @@ -0,0 +1,18 @@ +# OpenBlade staging stack -- ALWAYS pinned by digest, never by tag. +# The release workflow (and scripts/deploy.py rollback) set OPENBLADE_IMAGE to +# ghcr.io/amantux/openblade@sha256:<64 hex> +# Compose refuses to start if it is unset, so a mutable tag can't sneak in. +services: + openblade: + image: ${OPENBLADE_IMAGE:?set OPENBLADE_IMAGE to ghcr.io/amantux/openblade@sha256:} + ports: + - "8000:8000" + environment: + OPENBLADE_ENV: staging + OPENBLADE_DB_URL: sqlite:////data/openblade.db + volumes: + - openblade-staging-data:/data + restart: unless-stopped + +volumes: + openblade-staging-data: diff --git a/docs/architecture.md b/docs/architecture.md index 0353524..35c450c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,5 +51,17 @@ The catalog uses SQLite tables for volume groups, barcode assignments, file reco ## Job system Jobs are stored in-process with explicit state transitions. The queue additionally tracks changer and drive ownership so concurrent operations cannot claim the same hardware resources. +## Drive leases and recovery +Drive exclusion for tape I/O is a catalog row, not process memory. `DriveScheduler` is a façade over a `LeaseStore`: `CatalogLeaseStore` in production (one per `AppContext`, shared by the API and the CLI through the same `drive_leases` table) and `InMemoryLeaseStore` for unit tests. Each lease carries a monotonic `fencing_token`; job modules call `scheduler.verify(handle)` before a read-write mount, a write batch, and unmount/unload, and `heartbeat` after writes, so a lease that expired or was released behind a running job aborts it with `StaleLeaseError` instead of writing to a drive it no longer owns. On startup `recover_after_restart` fails (`failed_recoverable`, journalled as `recovered`) every `running` job whose lease expired without a heartbeat — a `running` job with a live lease may belong to another process and is left alone — releases orphaned leases, and reports expected-vs-observed drive contents. It also reports, without changing anything, the uncommitted `STAGING`/`VERIFYING` instances of every `failed_recoverable` job (`staged_instances`: barcode, tape path, shard index, state — what to reconcile on tape before retry) and `pending` jobs older than one `DEFAULT_LEASE_TTL` with no live lease (`stale_pending_job_ids`). It never moves media. The report is read-only at `GET /jobs/recovery`. Spec and rationale: `docs/decisions/2026-10-09-persistent-drive-leases.md`. + ## FUSE namespace The FUSE-oriented layer is intentionally thin: catalog entries define the namespace, and hydration delegates to restore workflows. This keeps the namespace authoritative and avoids bypassing safety or verification logic. + +## NAS tape-native planning models + +Cartridge lifecycle (`openblade/nas/state_machine.py`), LTO media/drive +compatibility and foreign-media detection (`openblade/nas/media.py`), and pure +capacity/replication/export/spanning planners (`openblade/nas/capacity.py`) +sit beside the existing archive and restore planners. They hold no I/O and no +persistence yet; see the "Tape-native domain models" section of +`docs/wiki/guides/fuse-and-nas.md` for the model list and tests. diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md new file mode 100644 index 0000000..1273934 --- /dev/null +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -0,0 +1,318 @@ +# Persistent drive leases with fencing, and job recovery on restart + +Status: accepted 2026-10-09. Roadmap items 7 and 8 (Phase 2), first slice. + +## Where the four working copies stand (so nobody re-merges them) + +| checkout | branch | relation to `origin/master` | +|---|---|---| +| `/root/ob2-final` | `feat/persistent-drive-leases` | integration branch for this work and the 2026-10-09 fleet (per-item status table in `docs/production-readiness-roadmap.md`) | +| `/root/ob-final` | `integrate/eight-items` | tree-identical to PR #44 squash, already on master | +| `/root/ob-int2` | `integrate/campaign-wiki-assistant` | fully contained in `ob-final` | +| `/root/OpenBlade` | `chore/py312-baseline-and-quantum-refs` | stale (37 behind); its 3 stray commits are already on master by content | + +Nothing is left to merge. All new work starts from `origin/master`. + +## Problem + +`DriveScheduler` is the only thing that stops two tape jobs from driving the +same physical drive, and it is constructed **per request** and held **in +memory**: + +- `openblade/api/routes_archive.py:196`, `routes_restore.py:159` and `:234`, + `openblade/cli/main.py:624`, `:698`, `:838` each do + `DriveScheduler(num_drives=len(context.library.inventory().drives))`. +- Two API requests (only the archive route has a process-wide lock), or the + CLI and the API against the same catalog, each get a scheduler that believes + every drive is free. +- After a crash or restart, nothing knows which drives a dead job held, and + catalog jobs stay `running` forever. + +The roadmap (items 7, 8) asks for DB-backed leases with a monotonic fencing +token and restart recovery. This note is the spec for that slice. + +## Decision + +### 1. Lease store in the catalog + +New table `drive_leases` (SQLAlchemy model in `openblade/catalog/models.py`, +created by `Base.metadata.create_all`; nothing to add to `_migrate_schema` +because it is a new table): + +| column | type | notes | +|---|---|---| +| `id` | str pk (uuid4) | lease id | +| `drive_id` | int, not null | scheduler lock key; `UNIQUE` while the lease is live (see below) | +| `job_id` | str, not null, indexed | owning catalog job | +| `barcode` | str(8), not null | cartridge the lease was taken for | +| `physical_drive_id` | int, nullable | mirrors `DriveHandle.physical_drive_id` | +| `fencing_token` | int, not null, unique | monotonic across the catalog | +| `acquired_at` | datetime utc | | +| `heartbeat_at` | datetime utc | updated by `heartbeat()` | +| `expires_at` | datetime utc | `heartbeat_at + ttl`; default ttl 15 min | +| `released_at` | datetime utc, nullable | null = live | + +Live-ness is `released_at IS NULL AND expires_at > now`. Enforce "one live +lease per drive" in code under a write transaction (SQLite `BEGIN IMMEDIATE` +via `connection.exec_driver_sql`, Postgres not a target here) — do **not** rely +on a partial unique index, SQLite support for those in SQLAlchemy is uneven. + +`fencing_token` = `COALESCE(MAX(fencing_token), 0) + 1` inside the same +transaction. It is never reused, even after release. + +Repository API (`CatalogRepository`, keep it small): + +```python +def acquire_drive_leases(self, *, job_id, barcodes: list[str], num_drives: int, ttl: timedelta) -> list[DriveLease] | None # None = not all free, caller waits +def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None +def release_leases(self, lease_ids: list[str]) -> None +def lease_is_live(self, lease_id: str, fencing_token: int) -> bool +def live_leases(self) -> list[DriveLease] +def release_leases_for_job(self, job_id: str) -> int +``` + +`DriveLease` is a frozen dataclass in `openblade/domain/models.py`. + +### 2. `DriveScheduler` becomes a façade over a lease store + +Keep the public shape every caller and ~all tests already use: +`acquire_drives(barcodes, timeout)`, `release_drives(handles)`, `status()`, +`available_count()`, `num_drives`. Add: + +- `DriveScheduler(num_drives, *, store: LeaseStore, job_id: str, ttl=...)`. +- `LeaseStore` is a `Protocol` with two implementations: + `InMemoryLeaseStore` (existing behaviour, used by unit tests that build a + scheduler directly) and `CatalogLeaseStore` (wraps the repository calls + above). This is the one new abstraction and it exists because tests and + production genuinely differ. +- `acquire_drives` keeps the all-or-nothing + timeout semantics; the wait loop + polls the store (0.25 s) instead of `Condition.wait` when the store is the + catalog one. +- `DriveHandle` gains `lease_id: str` and `fencing_token: int`. `drive_id` + stays immutable (existing test pins this). +- New `verify(handle)` → raises `StaleLeaseError` (new typed error in + `openblade/domain/errors.py`, subclass of the existing domain error base so + the jobs error sanitizer treats it as typed) when the lease is not live or + the token differs. +- New `heartbeat(handles)`. + +### 3. Fencing at the destructive edges + +A fenced-out job (`StaleLeaseError`) must **not** unmount or unload: the drive +may already hold the new lease-holder's mounted tape. It logs the drive as +"physical state unknown" and leaves it for reconciliation. Held leases are +kept alive by a daemon heartbeat thread in the scheduler (every `ttl/3`), so a +single long write cannot expire its own lease. + +In `openblade/jobs/sharded_archive.py`, `archive.py`, `restore.py`, +`tree_restore.py`: call `scheduler.verify(handle)` immediately before +`ltfs.mount(..., READ_WRITE)`, before each shard/file write batch, and before +unmount/unload. A stale lease aborts the job with `StaleLeaseError`; the error +path must still attempt the clean unmount/unload **and must not suppress its +errors** (existing rule). Call `heartbeat` after each file/shard write. + +### 4. One scheduler per process, bound to the catalog + +- `AppContext` (`openblade/bootstrap.py`) gets a `lease_store: LeaseStore` + built from the catalog repository. The six call sites above construct + `DriveScheduler(num_drives=..., store=context.lease_store, job_id=job.id)`. + The CLI paths that have no catalog job yet must create one first (they + already have `context.catalog`). +- Delete nothing else; `JobQueue` (in-memory, `bootstrap.py:607`) is not in + scope — note it in the PR as follow-up. + +### 5. Recovery on startup (item 8, recovery half only) + +New `openblade/jobs/recovery.py::recover_after_restart(catalog, library) -> RecoveryReport`, +called from bootstrap after `init_db` and before the API/CLI serves anything: + +1. **Only jobs whose lease has EXPIRED** (no heartbeat within the TTL) → + `failed_recoverable`. The catalog is shared by the API and the CLI, so a + `running` row is not evidence that a job is dead — it may be running in + another process. A running job with no lease at all is left alone. + (Adversarial review 2026-10-09 caught the original "fail every running + job" wording as a cross-process regression; it never shipped.) +2. Every live lease whose job is not `running` → released (`released_at = now`). +3. For each lease released in step 2, compare expected (`barcode` in + `physical`) with `library.inventory()`; log one line per mismatch at WARNING + and include it in the report. **Never** move media here — report only. +4. Report is logged at INFO and returned; expose it read-only as + `GET /jobs/recovery` (native surface, bearer-gated like the other native + routes; 404 under `OPENBLADE_SCALAR_API_ONLY`). + +Resuming a job from staged shards is **deferred** (needs the STAGING / +VERIFYING instance states from item 4) — say so in the PR, do not stub it. + +## Tests (each must fail with the guard removed — mutation-check the first two) + +- `tests/unit/test_drive_leases.py`: two schedulers over the same catalog + store cannot both acquire drive 0; second waits then `DriveBusyError`. + Fencing tokens strictly increase; a released lease's token is never reissued. +- Lease fencing (landed in `tests/integration/test_sharded_archive_atomicity.py` + to reuse its helpers): release a lease behind a running sharded archive → + `StaleLeaseError`, no `mark_instance_archived`, and the drive is left + untouched (no unmount, no unload). +- `tests/unit/test_recovery.py`: seed a `running` job + live lease, restart → + job `failed_recoverable`, lease released, mismatch reported when the + simulator's drive is empty. +- Existing `tests/unit/test_scheduler*.py` / `test_sharded_archive_atomicity.py` + stay green with `InMemoryLeaseStore`. +- `tests/integration/test_api.py` (or the nearest native route suite): two + overlapping archive requests against one catalog serialise on the lease, and + `GET /jobs/recovery` is bearer-gated and 404 in scalar-api-only mode. + +## Deviations recorded at implementation time + +- `DriveScheduler(store=...)` stays optional (defaults to `InMemoryLeaseStore`) + because ~40 existing tests construct it bare; every production call site + passes the catalog store. +- `archive.py` / `restore.py` (`run_archive_job` / `run_restore_job`) take no + scheduler and hold no lease today, so there is nothing to fence; giving them + leases is a follow-up. `tree_restore.py` only forwards the scheduler to the + sharded restore, which is fenced. +- The tree-restore route builds its scheduler in the worker thread over a + `CatalogLeaseStore(worker_catalog)` rather than `context.lease_store`: the + context's session is not thread-safe. Same table, same exclusion. +- Explicit heartbeats run once per write batch; the scheduler's background + heartbeat thread covers the gaps. + +## Follow-up landed 2026-10-09 (phase 2: staged commits, journal, recovery report) + +Item 4's states and the job journal — listed under Non-goals above for phase 1 — +landed as a foundation plus three parallel workstreams. The foundation (states, +`journal`/`job_journal`, staged-instance repository methods) and the recovery report +are on this branch; the sharded-archive wiring of those states and journal events and +the non-sharded leases come from the sibling `fleet/jobs-sharded` and +`fleet/jobs-leases` branches and are only true once those merge: + +- **States.** `FileInstanceState.STAGING` / `VERIFYING` mark a sharded-archive + instance that is written but not committed. Instances are created STAGING + before their first write (`create_staged_instance`), moved to VERIFYING after + every lane wrote and read back its shard, and moved to ARCHIVED in ONE + all-or-nothing `mark_instances_archived` call only after the manifest/commit + marker (`finalize_tape_generation`, marker written last), clean unmount and + inventory reconcile. Staged instances are never listed as archived or + restorable. +- **Journal.** `job_journal` rows (`CatalogRepository.journal` / + `job_journal`) record lease_acquired/released, shard_staged, verify_started, + verify_finished, committed, failed, fenced_out, physical_state_unknown (a + cleanup step that raised; the original lane error is kept), and `recovered` + (written by startup recovery for each job it fails). +- **Non-sharded leases.** `run_archive_job` / `run_restore_job` take a + `DriveScheduler` and acquire, verify and release their drive through it like + the sharded path, so non-sharded and sharded jobs exclude each other across + processes; a stale lease aborts without touching hardware. +- **Recovery report fields.** `RecoveryReport` / `GET /jobs/recovery` gain + `staged_instances` — `{job_id: [{instance_id, barcode, tape_path, + shard_index, state}]}` for every `failed_recoverable` job, i.e. what to + reconcile on tape before retry — and `stale_pending_job_ids`: `pending` jobs + created more than one `DEFAULT_LEASE_TTL` ago. Jobs mark themselves `running` + before they acquire a lease, so in practice this is an age signal: a job + legitimately queued longer than the TTL is reported too. Both are report-only: + no state change, no media movement. Resuming from staged shards remains + deferred. + +## Non-goals + +Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any +`/aml/*` or `/iblade/*` route, the emulator contract, Postgres. + +## Definition of done + +`make lint` clean, `.venv/bin/mypy` strict clean (CI blocks on both), +`tests/unit tests/integration tests/safety` green, frontend untouched, reviewer +subagent pass, one commit per logical change (models+repo, scheduler, fencing +at call sites, recovery+route, docs). + +## Reconciliation + +A failed unmount/unload journals `physical_state_unknown` (`{"op","barcode","drive"}`): +the drive may still hold a possibly LTFS-mounted tape. `openblade/jobs/reconcile.py` +derives the pending set across all jobs (`CatalogRepository.journal_events`): a drive is +pending while its latest `physical_state_unknown` has no later `drive_reconciled` entry. + +- `DriveScheduler.acquire_drives` excludes pending drives from its candidates and raises + `DriveUnreconciledError` only when too few reconciled drives remain. +- The restore / sharded-archive / sharded-restore load helpers refuse a pending drive. +- `recover_after_restart` lists pending drives in `RecoveryReport.pending_reconciliation` + (report only; nothing is auto-reconciled). +- `POST /jobs/recovery/reconcile/{drive_id}` takes a fresh inventory snapshot, refuses + (409) while the drive still reports a mount, else journals `drive_reconciled` + (`{"drive","observed_barcode"}`); 404 for an unknown drive. +- **Barcode-level exclusion.** A pending entry also blocks its *barcode*: + `acquire_drives` raises `DriveUnreconciledError` naming any requested barcode in + `barcodes_pending_reconciliation` (this covers entries with no drive id). +- **Fail closed on journal failure.** If `physical_state_unknown` itself cannot be + journaled, the drive's lease is KEPT (`DriveHandle.hold_unjournaled`) and + `release_drives` raises `JournalWriteError` (a `SafetyViolationError`) after + releasing every other drive. The message names the original lane failure so the + job's recorded error is not only "journal write failed". Recovery likewise keeps + live leases of failed jobs and reports them in `held_failed_lease_ids`. + JobQueue slot claims (`drive_id < 0`, `QUEUE-CLAIM:`/`CHANGER:` barcodes) are not + compared against inventory as drive mismatches. +- **Residual risk.** A held lease lives only as long as the process heartbeats it. + Once the process dies, the lease lapses at its TTL and, because the journal write + failed, no pending entry exists. The lease TTL is therefore the reconciliation + window for that case: an operator must reconcile before it expires. + +## JobQueue follow-up landed + +The `JobQueue` deferral above (and the "replacing `JobQueue`" non-goal) is now +done as a persistent façade, not a replacement: +`JobQueue(catalog, lease_store, *, ttl=DEFAULT_LEASE_TTL)` keeps jobs only in the +catalog `jobs` table (`run_job` persists RUNNING before the work and +COMPLETED/FAILED after, error text still via `safe_job_error`), and drive/changer +ownership is a lease in the shared lease table, so it holds across processes. +`LeaseStore.acquire` allocates the lowest free ids and cannot target one, so a +claim requests every free id up to the target in one `BEGIN IMMEDIATE` +transaction, keeps the target, releases the probe leases, and retries briefly +under contention. Changer ownership is a lease on the reserved pseudo-drive +`CHANGER_DRIVE_ID = 64` (negative ids are unreachable through `range(num_drives)`; +a large sentinel would insert one probe row per lower id on every claim). +Known costs: each claim of id *k* writes up to *k* released probe rows; claims +are not heartbeated, so they lapse after the TTL. A repository method that +leases one specific drive id would remove both the probes and the retry loop. +Covered by `tests/unit/test_job_queue_persistent.py` (two sessions, one file). + +## Capacity reservations + +Decisions (WS-capacity, fleet round 3): + +1. **Pool reservations are the source of truth.** The planner takes + `pool_reserved_bytes` (sum of unexpired `nas_reservations` for the pool) and adds + the `POOL_BUDGET_BLOCKER` when placed bytes exceed total tape capacity minus that + budget. `reserved_bytes` stays per barcode for explicit holds. +2. **The budget is server-derived.** When `pool` is set on `/nas/archive-plan` (and + always on the upload share-push), `capacity.server_budget_request` derives + `available_tapes` and `tape_capacities` from the library inventory + catalog + (`server_data_tapes`: `capacity_bytes - used_bytes`, excluding cleaning cartridges + and volume-group members). A body that also supplies `tape_capacities`, + `available_tapes` or `reserved_bytes` is rejected with 422 (never silently + overwritten); an unknown pool id is 404. An empty server-derived tape set blocks + the plan instead of skipping the budget check. Pool-less requests keep the + body-driven path but get no pool budget and no booking. +3. **Book at consumption, not at preview.** `capacity.plan_with_pool_budget` only + computes the budget; previews book nothing, so repeating a preview is never + refused. `book_plan_capacity` books `total_bytes * copies` under `owner=plan_id` + for 1h through the atomic `reserve_nas_capacity`, called from + `POST /nas/ingest/start` and the share-push before the job exists; a lost race or + exhausted budget raises `ReservationError` -> 409 and no job is created. +4. **Release.** `run_ingest_job` deletes the owner's rows when the job ends archived, + failed or cancelled, and also whenever the executor raises (the status may still + be non-terminal and nothing will resume the job). A failing release is logged + ("reservation release failed") and never replaces the executor's exception; the + row then expires by TTL. +5. **Scratch** is counted from the library inventory + (`InventoryService(library).snapshot()`, `count_scratch_tapes`): slots holding a + barcode that is not a volume-group member, not catalogued with `used_bytes > 0`, + and not a `CLN` cartridge. Blank tapes have no catalog row, so a catalog count + would report a library full of scratch as CRITICAL. +6. **Health**: a `scratch` component only when thresholds are configured; WARN is OK + with a message, CRITICAL is DEGRADED, never UNHEALTHY. Thresholds are managed by + `GET`/`PUT /nas/scratch-thresholds` (404 unset, 422 on `warn < min`). + +Reviewer risks still open: `book_plan_capacity` books against `plan.pool`, not the +`pool_id` of the start request, so a plan previewed for one pool and started into +another is budgeted against the first; the share-push registers its plan before +booking, so a 409 there leaves a registered plan with no job; and a started job that +never reaches a terminal state without raising holds its bytes for the full hour. diff --git a/docs/deployment.md b/docs/deployment.md index 0efab13..3b5307a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -65,10 +65,21 @@ Exit 0 only when the deploy is **promoted** (all three pass). OPENBLADE_ENV=production \ OPENBLADE_ADMIN_PASSWORD=... OPENBLADE_SERVICE_PASSWORD=... \ OPENBLADE_SERVICE_TOKEN=... OPENBLADE_DB_URL=sqlite:////data/openblade.db \ +OPENBLADE_IMAGE=ghcr.io/amantux/openblade@sha256:<64-hex digest from the release> \ python3 scripts/deploy.py \ - --deploy-cmd "docker compose up -d" \ + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" \ --base-url https://openblade.internal +# Then the read-only appliance postcheck (GET /inventory/ + GET /jobs/recovery on +# top of the live topology; no non-GET request is ever sent): +python3 scripts/deploy.py postcheck --base-url https://openblade.internal \ + --read-only-appliance-checks + +# Roll back APPLICATION CODE ONLY to the previous digest (tags are refused). +# Physical tape state is never rolled back -- review GET /jobs/recovery after. +python3 scripts/deploy.py rollback --to sha256: \ + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" + # Re-run pre/post checks only (no deploy), e.g. as a post-deploy smoke: python3 scripts/deploy.py --skip-deploy @@ -88,7 +99,10 @@ deploy fails the build. No production data or credentials are used. - `--base-url` postcheck probes endpoint reachability only (it cannot introspect the remote process's in-process wiring); the in-process postcheck additionally asserts the AppContext is fully wired. -- Rollback orchestration and blue/green promotion are out of scope here; pair this - gate with your container platform's rollout. On a failed postcheck, do not - promote — roll back to the last verified image and restore the catalog per - docs/disaster-recovery.md if needed. +- Images are always deployed by digest (`deploy/staging/` and + `deploy/production/` compose files refuse to start without `OPENBLADE_IMAGE`). + The tag-triggered release pipeline (build once, scan, sign, staging, approval, + production, rollback) is described in docs/runbooks/release.md; it has **not + yet been exercised on a real tag**. +- `rollback` redeploys application code only. It never touches tape state; if + the catalog needs restoring, follow docs/disaster-recovery.md. diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md index 0af4a9f..318dbd0 100644 --- a/docs/disaster-recovery.md +++ b/docs/disaster-recovery.md @@ -55,3 +55,18 @@ against a freshly-seeded DB — this is what CI runs, with no production data. - A restore drill against a **real production backup** artifact. CI only exercises the mechanism on seeded data; never restore production data in a test path. - Scheduled backup rotation + off-host replication. + +## Commit markers and uncommitted generations + +Each tape write generation ends with `/.openblade/commit-.json` +(schema, generation id, sha256 of `manifest.json` and `catalog-shard.json`, +written_at). `finalize_tape_generation()` is the only writer and enforces the order: +data -> `manifest.json` (+`.sha256`) -> `catalog-shard.json` (+`.sha256`) -> +`tape.json` (`current_generation`/`previous_generation` chain) -> commit marker. + +On rebuild, a v2 manifest whose marker is **missing** or whose hashes **do not +match** is an uncommitted generation (an interrupted write): its files are not +restored as archived and the plan warns `uncommitted generation : `. +Pre-v2 manifests carry no generation id and are treated as committed. Each +manifest lists `sibling_tapes`; a sibling not included in the rebuild request is +reported as `missing sibling tape ` — restore needs every sibling. diff --git a/docs/fuse.md b/docs/fuse.md index d2a53a0..2913a36 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -1,3 +1,14 @@ -# FUSE namespace +# FUSE data plane -OpenBlade's FUSE-facing layer is currently a lightweight namespace abstraction over the catalog. It is designed so future kernel FUSE work can reuse the same hydration and metadata contracts without bypassing archive safety rules. +Five layers, top to bottom, and what is real today: + +| Layer | Status | +|---|---| +| SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`); `on_open` returns an opaque handle token and `on_close(token)` releases only that handle's cache pin (double or unknown closes are no-ops), so one client cannot unpin another's file. | +| Namespace/metadata | **Real.** `CatalogFuseOperations` (`openblade/fuse/mount.py`): offline files report their real size; xattr `user.openblade.state` = `offline`/`hydrating`/`online`. | +| Hydration/cache | **Real.** `Hydrator` (`openblade/fuse/hydration.py`): one shared ticket per path, `OPENBLADE_FUSE_HYDRATE_TIMEOUT` (default 300 s) blocking wait, or `EAGAIN` with `blocking=False`; failure = `EIO`, file stays `offline`, reason in `last_error()`. `HydrationCache` (`cache.py`): optional byte budget with LRU eviction that never evicts a file with open handles; restores land under a temp name and are renamed only after the sha256 matches. | +| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue_batch` — **one restore job per tape batch** (`run_restore_batch`: one lease, one load, one read-only mount, one unmount, one unload per source barcode; per-file checksum verification with a temp name renamed only on a match; a failed unmount journals `physical_state_unknown` and skips the unload). Never a tape backend directly. A restarted hydrator re-attaches to a pending/running single-file restore job for the path (batch jobs are not yet re-attachable). Every `restore_batch`/`resume` call runs on a Hydrator worker thread over its **own** catalog Session (`catalog_factory`), never the caller's. | +| Placement/robotics | **Partial.** Requests are batched per tape barcode for `OPENBLADE_FUSE_BATCH_WINDOW_MS` (default 250 ms) and each batch is one restore job, so one physical load per tape per batch. | + +Unmount (`destroy`) cancels batches not yet started and waits for in-flight restores. +`openblade fuse mount --hydrate` builds this data plane (`build_data_plane` in `openblade/cli/fuse_and_health.py`) and passes it as `data_plane=`. Timeout and batch window come from the settings fields `fuse_hydrate_timeout_s` / `fuse_batch_window_ms` (env `OPENBLADE_FUSE_HYDRATE_TIMEOUT` / `OPENBLADE_FUSE_BATCH_WINDOW_MS`). The cache byte budget (`max_bytes`) is not yet configurable from the CLI, so the mount's cache is unbounded. diff --git a/docs/hardware-setup.md b/docs/hardware-setup.md index 989c8ed..edd58a5 100644 --- a/docs/hardware-setup.md +++ b/docs/hardware-setup.md @@ -8,6 +8,21 @@ 6. Keep the application in mock mode until read-only inventory and LTFS validation succeed end-to-end. 7. Document barcode conventions and slot maps before allowing write workflows. +## Write paths stay disabled until the drive-identity bridge has passed on the rig + +Do not enable any write path (format, archive, the destructive CI lane) on a +real library until `tests/hardware/readonly/test_drive_identity_bridge.py` has +passed **on that rig**. It walks the whole identity chain once, end to end: +AML element coordinate → drive serial → `sg_inq` on the host → the +`/dev/tape/by-id` node → an LTFS mount **dry run** (the mount is not performed; +the test asserts the resolved device path exists and the serial matches, then +stops). Until it passes, a write could land on a different physical drive than +the one the library reports — the failure mode the serial correlation above +exists to prevent. + +The bridge test runs in the nightly read-only lane. How the lanes are wired, +gated and approved: [runbooks/hardware-lanes.md](runbooks/hardware-lanes.md). + ## Always use the no-rewind device node (`/dev/nstN`, never `/dev/stN`) Linux exposes every tape drive twice: `/dev/stN` **rewinds the tape on every diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index c259393..9dcdf4d 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -26,6 +26,38 @@ Verified against the repo before writing (not asserted from memory): - `sharded_archive.py:234,251` calls `mark_instance_archived` per-shard inside the write loop → review #6 confirmed real. ✅ - Review #5 (scheduler `drive_id` mutation) **not reproduced**: `jobs/scheduler.py` is a plain allocator with no `_load_barcode`/mutation — re-investigate before "fixing." ⚠️ +## Status as of 2026-10-11 (fleet integration on `feat/persistent-drive-leases`) + +Verified against the tree, not asserted. "Landed" = implemented and tested in-repo; +"appliance-gated" = authored and runnable, cannot be certified without the rig. + +| # | Item | Status | Where | +|---|---|---|---| +| 1 | Relabel simplified endpoint | landed | `routes_aml_move_medium.py` | +| 2 | ScalarCoordinate / MoveClass IntFlag | landed (bit values appliance-gated) | `domain/scalar_coordinate.py` | +| 3 | Compatibility corpus + differential harness | scaffold + `tools/compat_capture.py` capture/sanitize/diff; 12 self-captured emulator regression cases (0 drift vs inferred set); 0 appliance captures — fidelity unverified | `compatibility/`, `tests/compat/` | +| 4 | Atomic sharded commit | landed: STAGING→VERIFYING→one ARCHIVED commit, job journal, no suppressed cleanup errors; `physical_state_unknown` now has a consumer (drive/barcode refusal, `reconcile_drive`, fail-closed when the journal write fails); resume = report-only | `jobs/sharded_archive.py`, `catalog/repository.py`, `jobs/recovery.py` | +| 5 | Protocol-agnostic backends + contract suite | landed (sim, in-process emulator, scsi+fake-ltfs over a stateful mtx fake, real gated) | `tests/contract/` | +| 6 | Scheduler lease regression | landed | `tests/unit/test_drive_leases.py` | +| 7 | Persistent drive leases | landed | `catalog/models.py::DriveLeaseRecord`, `jobs/scheduler.py` | +| 8 | Persistent, resumable jobs | landed (recovery + report, `JobQueue` persistent over catalog jobs + leases, `POST /jobs/recovery/reconcile/{drive}`); resume-from-staged not implemented | `jobs/recovery.py`, `jobs/queue.py`, `GET /jobs/recovery` | +| 9 | Reconstructible catalog | landed: manifest v2, commit marker last, generation pointer, sibling tapes, uncommitted-generation rebuild rule | `nas/ltfs_manifest.py`, `nas/archive_lifecycle.py`, `nas/catalog_rebuild.py` | +| 10 | Separate the axes | landed (models + manifest/catalog carry); erasure coding modelled, unsupported | `domain/protection.py` | +| 11 | Aggregation gate + ownership check | landed | `ci.yml::ci-gate`, `tools/ci_ownership.py` | +| 12 | Full suite on master + nightly lanes | landed | `ci.yml::full-suite`, `nightly.yml` | +| 13 | Consolidated emulator workflows | landed | `_emulator-boot-test.yml` | +| 14 | Lint/type ratchet | landed (both blocking, both at zero) | `ci.yml` | +| 15 | Workflow hardening | landed: SHA pins, per-job permissions, actionlint/zizmor/shellcheck/yamllint | `workflow-lint.yml` | +| 16 | Differential + mutation | mutation lane landed (mutmut 3.8.0, ratchet baseline for 6/6 modules as of 2026-10-10); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | +| 17 | Timing profiles + virtual clock | landed (10 profiles; the 4 fault profiles are injected by the emulator via `OPENBLADE_EMULATOR_FAULT_PROFILE`) | `tests/i3/timing.py` | +| 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-pending session modelled (inferred): enabled-MFA login → 200 + `Warning: 199 - "MFA Required"` (412 strict); 401 except MFA challenge/key/logout until a valid TOTP; certify by appliance capture | `routes_aml_auth.py`, `compatibility/` | +| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 33/33 pass locally incl. real SMB byte-range lock conflict; NFSv4 named-user DENY ACE enforced across uids (Fedora 41 Ganesha 5.9 stage, VFS ACL mapping; EVERYONE@ deny folds into mode bits)) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | +| 20 | Tape-native NAS models | landed; pool protection/replication, reservations (atomic reserve), export sets, vaults, VG replication and file spans persisted in the catalog; scratch-threshold health and per-barcode `reserved_bytes` mapping still TODO | `nas/state_machine.py`, `nas/media.py`, `nas/capacity.py` | +| 21 | FUSE data plane | landed: hydrator, cache (open-time integrity check, persisted LRU index, chunked pread), xattr state, gateway; see `docs/fuse.md` | `fuse/` | +| 22 | Self-hosted i3/LTFS lanes | appliance-gated, authored: read-only nightly + destructive with environment approval, allowlist, snapshots; reboot persistence still manual (soft-reboot automation drafted, held for operator decision) | `hardware-*.yml`, `tools/hardware/` | +| 23 | Drive-identity bridge | implemented + test authored; uncertified — skips on mhvtl (needs an AML endpoint with matching serials); certify on appliance | `hardware/correlation.py`, `tests/hardware/readonly/` | +| 24 | Release pipeline | landed (never exercised on a tag): build-once digest, SBOM, scans, cosign, staging→approval→production, rollback | `release.yml`, `docs/runbooks/release.md` | + --- ## Phase 0 — Stop the self-referential fidelity (the meta-fix) 🟢🟡 @@ -151,10 +183,14 @@ emulator. (GitHub Environment approval, allowlisted sacrificial barcodes, snapshot before/after, load/unload, LTFS format, write/read/checksum, reboot persistence, sharded write+restore, safe failure injection, guaranteed recovery). -23. 🔴 **#E Physical drive-identity bridge:** certify AML coordinate ↔ serial ↔ SCSI - inquiry ↔ `/dev/tape/by-id` ↔ LTFS mount on the rig before enabling any real - write path. Never rely on transient `/dev/nst0` ordering. (`scalar_http` currently - raises `NotImplementedError` here — correct until proven on metal.) +23. 🟡 **#E Physical drive-identity bridge:** AML coordinate ↔ serial ↔ SCSI + inquiry ↔ `/dev/tape/by-id` ↔ LTFS mount (dry) is implemented in + `openblade/hardware/correlation.py` (`resolve_tape_by_id`) and covered by + `tests/hardware/readonly/test_drive_identity_bridge.py`. Never rely on transient + `/dev/nst0` ordering. Not yet certified: the test needs an AML endpoint + (`OPENBLADE_APPLIANCE_*`), so on the mhvtl rig it skips (2026-10-11: discovery + passes, bridge skipped). Certify on the appliance before enabling any real write + path. ## Phase 8 — Release pipeline (P2) 🟢🟡 diff --git a/docs/reference/i3-contract-notes.md b/docs/reference/i3-contract-notes.md index 3e9d00e..eab43e8 100644 --- a/docs/reference/i3-contract-notes.md +++ b/docs/reference/i3-contract-notes.md @@ -56,3 +56,23 @@ The Web Services guide is a *control-plane* API spec; drive/robotics performance figures live in hardware datasheets, not here. The emulator's latency profiles therefore remain OpenBlade-chosen approximations, not doc-certified numbers — do not present them as vendor-specified. + +## AML authentication behaviours + +Only the items marked "manual" are documented in Rev D; everything else is +inferred (OpenBlade-chosen) and unverified against an appliance. + +| Behaviour | Source | +|---|---| +| JSON login/logout; `sessionID` cookie | manual §2.5 (users/login) | +| `Warning: Default Password Supplied` header on login with a default password (200 kept by default) | manual §2.5 (users/login) | +| 412 `{"code":412,"description":"Error","summary":"Default password must be changed"}` plus token, cookie and Warning header when `OPENBLADE_AML_STRICT_PRECONDITIONS=true` (default false) | inferred | +| Per-user session cap, `OPENBLADE_AML_MAX_SESSIONS` (default 5); a login beyond the cap evicts the user's oldest session, whose token then gets 401 | inferred | +| Login also accepts form and `application/xml` / `text/xml` bodies (`....`) | inferred | +| XML response (root ``) when Accept names an XML type and not `application/json`; login and `DELETE /aml/users/login` only | inferred | +| ``). +3. In the runner's `.env` file, set the rig's device map so the suites can + address it: `OPENBLADE_CHANGER_DEVICE`, `OPENBLADE_DRIVE_DEVICES`, and the + drive map described in [../hardware-setup.md](../hardware-setup.md). +4. Install host tools the suites call: `sg3-utils`, `lsscsi`, `mtx`, LTFS. +5. Add repo secrets `QUANTUM_AML_USER` / `QUANTUM_AML_PASSWORD` (AML login). + +## 2. Repository variables + +Repo → **Settings → Secrets and variables → Actions → Variables**: + +- **`HARDWARE_RIG_AVAILABLE`** = `true` — only once the runner above is online. + Every physical lane's hardware job has `if: vars.HARDWARE_RIG_AVAILABLE == 'true'`. +- **`SACRIFICIAL_BARCODES`** — comma-separated list of cartridges that may be + loaded, formatted and overwritten (e.g. `OB0007L8,OB0008L8`). Nothing else may + ever be touched by the destructive lane. +- **`OPENBLADE_APPLIANCE_URL`**, **`OPENBLADE_APPLIANCE_USER`** (variables), + **`OPENBLADE_APPLIANCE_PASSWORD`** (secret — put it on the `hardware-destructive` + environment too), optional **`OPENBLADE_APPLIANCE_INSECURE`** = `true` for a + self-signed appliance cert. `OPENBLADE_CHANGER_DEVICE` and + `OPENBLADE_DRIVE_DEVICES` are set in the runner host's environment (`.env` of + the runner service). Every rig job has a **Preflight** step that fails if any + of these is empty — otherwise the rig tests would skip and the lane would go + green without testing anything. As a second layer, every rig pytest step + writes JUnit XML and a final **Gate** step (`python -m tools.hardware.junit_gate`) + fails the job if any test skipped or nothing passed. An expected skip must be + passed explicitly (`--expected-skip classname::name`) with a comment saying why. + +## 3. Environment approval (settings-only — cannot be committed) + +Repo → **Settings → Environments → New environment** `hardware-destructive`: + +- **Required reviewers**: at least one person who can physically check the + library. Every run waits for approval, including the weekly scheduled one. +- Optionally restrict **Deployment branches** to `main`. + +Without the environment, GitHub creates it on first use with no protection — +so create it, with reviewers, before setting `HARDWARE_RIG_AVAILABLE`. + +## 4. The cartridge allowlist + +Before any hardware step, the destructive lane (and its mhvtl rehearsal) runs: + +```bash +python -m tools.hardware.allowlist --barcodes "$BARCODES" +``` + +`BARCODES` is the dispatch input `barcodes` (or all of `SACRIFICIAL_BARCODES` +on a scheduled run). The step exits non-zero — and the job stops — if the list +is empty or names any barcode outside `SACRIFICIAL_BARCODES`. The input is +passed through `env:`, never interpolated into the script. mhvtl falls back to +the scratch cartridges its rig loads (`OB0007L8,OB0008L8`) when the variable is +unset. + +Each lane also captures inventory + drive state before and after +(`tools.hardware.snapshot capture`), diffs them, and uploads `snapshots/` as an +artifact even on failure. On the read-only lanes the diff must be empty. + +## 5. The destructive sequence + +Load/unload → LTFS write/read/checksum → archive + restore → sharded write + +restore → safe fault injection → `/jobs/recovery` must report no interrupted +jobs, released leases or drive mismatches. + +LTFS format goes only through the two-phase flow (dry-run plan → explicit +barcode + one-time safety token). `test_mkltfs_formats_tape` is deselected in +this lane because it calls `mkltfs` directly. + +### Reboot persistence (manual) + +The runner cannot power-cycle the appliance. Once per quarter, after a green +destructive run: reboot the i3 from its web UI, wait for it to come Ready, +dispatch `hardware-library-smoke.yml`, and confirm the snapshot diff against +the pre-reboot `after.json` shows the same cartridges in the same slots. + +## 6. What "no rig" looks like — and the no-green-by-skipping rule + +Without `HARDWARE_RIG_AVAILABLE=true`, the hardware job shows as **Skipped** +and a small `no-rig-notice` job writes "NOT RUN (neutral)" to the run summary. +Because `no-rig-notice` succeeds, **the workflow run concludes `success`** — +GitHub shows it with a green check even though nothing touched the appliance. +That is why these lanes must **never be required status checks or README +badges**: a no-rig run would satisfy the check and colour the badge green. Read +the run summary (`NOT RUN`) or the rig job's own result, never the run +conclusion. + +**No lane may be reported or treated as green because it skipped.** A skipped +hardware job proves nothing about the appliance. Do not add it as a required +status check that a skip satisfies, do not quote a skipped run as evidence, and +do not "fix" a red lane by unsetting the variable. + +`actionlint` needs to know the custom label: add `.github/actionlint.yaml` with +`self-hosted-runner: { labels: [i3] }` if you lint these files locally. diff --git a/docs/runbooks/release.md b/docs/runbooks/release.md new file mode 100644 index 0000000..50537a5 --- /dev/null +++ b/docs/runbooks/release.md @@ -0,0 +1,60 @@ +# Release runbook + +> **Status: this pipeline has not yet been exercised on a real tag.** No `v*` tag +> has been pushed since `.github/workflows/release.yml` was written. Treat the +> first run as the test: watch every job, and expect to fix something. + +Trigger: pushing a `v*` tag whose version equals `[project].version` in +`pyproject.toml` (the build job refuses a mismatch). + +## Stages (all keyed on the image DIGEST, never a tag) + +| # | Job | What it does | Fails the release when | +|---|-----|--------------|------------------------| +| 1 | `build` | Builds the wheel (`python -m build`) and the container **once**; pushes `ghcr.io/amantux/openblade:` and outputs `digest`. Wheel uploaded as an artifact. | tag/pyproject mismatch, build error | +| 2 | `scan` | Image SBOM (anchore, SPDX JSON) + wheel SBOM (pip-audit CycloneDX); `pip-audit --strict` on the wheel's resolved deps; Trivy on `image@digest` with `severity: CRITICAL,HIGH`, `exit-code: 1`. | any known vuln in deps; any HIGH/CRITICAL in the image not in `.trivyignore` | +| 3 | `sign` | `cosign sign` (keyless, GitHub OIDC) of `image@digest`; SLSA provenance via `actions/attest-build-provenance`, pushed to the registry. | signing/attestation error | +| 4 | `verify` | `cosign verify` with issuer `token.actions.githubusercontent.com` and identity regex `^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$`. | signature missing or signed by any other identity | +| 5 | `release` | GitHub Release with the wheel, both SBOMs, and the digest in the body. | — | +| 6 | `deploy-staging` | Environment `staging`. `scripts/deploy.py` precheck → `docker compose -f deploy/staging/docker-compose.yml up -d --wait` → postcheck, then `postcheck --read-only-appliance-checks`, then the suites (i3 CI subset with `I3_TIMING_PROFILE=instant`, `tests/e2e/test_nas_journey.py`, `tests/unit/test_catalog_rebuild*.py`, `tests/e2e/test_mock_archive_restore.py`). | any check or suite fails | +| 7 | `deploy-production` | Environment `production` (**approval gate**). Re-runs `cosign verify`, deploys the **same digest**, read-only postcheck. | any check fails | + +The suites do not accept a base URL, so they run in-process on the runner from +the tagged checkout with the released wheel installed — not against the staging +container. The staging container is covered by the two postchecks. + +## Approval gate + +Configure in repo **Settings → Environments → production → Required reviewers** +(at least one person who did not push the tag). That setting cannot live in the +repo; without it `deploy-production` runs unattended. The job also needs a +self-hosted runner labelled `openblade-production` on the production host. + +## Rollback rule + +On a failed read-only postcheck, the job runs +`scripts/deploy.py rollback --to `. +Update that environment variable to the new digest after a release is accepted. +If it is unset, no automatic rollback happens and an operator must run it. + +- Rollback redeploys **application code only** (`OPENBLADE_IMAGE=@`). + It refuses tags and has no option that touches tape state. +- **Physical tape state is never rolled back.** Cartridges moved, loaded or + written by the bad version stay where they are. After any rollback, review + `GET /jobs/recovery` and reconcile before resuming operations; restore the + catalog per `docs/disaster-recovery.md` if needed. + +## Credentials + +No long-lived deploy secrets. GHCR uses the job's `GITHUB_TOKEN`; signing uses +GitHub OIDC (`id-token: write` exists only on the `sign` job). Deploy targets +must authenticate by OIDC too (or run on a self-hosted runner on the target +host, as production does). Do not add registry or cloud keys as repo secrets. +Top-level `permissions: contents: read`; write scopes are granted per job. +Every action is pinned to a commit SHA with a `# vX.Y.Z` comment. + +## Vulnerability exceptions (`.trivyignore`) + +An entry is allowed only with, on the line above it: why the finding does not +apply (or why no fix exists), who owns it, and a review-by date. No blanket +ignores. Remove the entry once a fixed base image or dependency ships. diff --git a/docs/sharding.md b/docs/sharding.md index bbfb1f3..cf8f745 100644 --- a/docs/sharding.md +++ b/docs/sharding.md @@ -129,3 +129,23 @@ POST /archive/sharded - Restore always uses `READ_ONLY` mounts, even on shard tapes - DriveScheduler never grants the same drive to two jobs - All drives are released (even on error) via try/finally + +## Protection is a separate axis + +Sharding/striping spreads data across tapes for throughput. It does **not** make +data durable: losing any one tape of a STRIPE set loses data. Durability comes only +from redundant copies. `openblade/domain/protection.py` models the axes +independently (`Placement`, `Striping`, `Protection`, `FailureDomain`, `RestoreQuorum`) +and `ProtectionPolicy.durability_class` reports the result: + +| Striping | Protection | `is_protected` | `durability_class` | +|---|---|---|---| +| NONE | NONE / REPLICATION(1) | false | `unprotected` | +| STRIPE / BLOCK_STRIPE | NONE / REPLICATION(1) | false | `performance-only` | +| any | REPLICATION(copies >= 2) | true | `protected` | +| any | ERASURE_CODE(k, m) | true (modelled only) | planner raises `UnsupportedProtectionError` | + +Legacy `NasPool.replication_factor` maps via `protection_from_pool()`. The policy is +recorded in `manifest.json` (`protection`, `placement`) and per dataset in +`catalog-shard.json`, so a rebuild can recover it. Pinned by +`test_striping_never_implies_protection`. diff --git a/docs/tech-debt.md b/docs/tech-debt.md index b7c5272..4dac5c8 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -1,57 +1,104 @@ # Tracked technical debt -Known, quantified debt that is deliberately not blocking CI, with the safe way to -pay it down. Advisory CI jobs (`continue-on-error: true`) still run and report; -they just don't fail the build or block trusted-PR auto-merge. +Current, verified state only. Every number below was produced by the command +shown next to it, run from the repo root with the pinned dev toolchain +(`ruff==0.15.22`, `mypy==2.3.0` from the `dev` extra) on 2026-10-09. If you +change a number, re-run the command and paste the new result; do not estimate. -## Lint / format backlog — PAID DOWN (2026-09-12) +## Lint and format: clean and blocking -**This section is history.** The backlog described here is cleared; see +| Check | Command | Result | +| --- | --- | --- | +| Lint | `ruff check .` | `All checks passed!` (0 findings) | +| Format | `ruff format --check .` | `365 files already formatted` | + +Both run in the `backend-lint` job of `.github/workflows/ci.yml`. That job has no +`continue-on-error`, and `ci-gate` both lists it in `needs:` and fails on any +result other than `success`/`skipped`, so a lint or format failure blocks merge. + +The older note that `openblade/assistant/tools.py` was the one file failing +`ruff format --check` is no longer true: +`ruff format --check openblade/assistant/tools.py` reports `1 file already formatted`. + +History of how the backlog was paid down: [`decisions/2026-09-11-pin-ruff-toolchain.md`](decisions/2026-09-11-pin-ruff-toolchain.md). -What it used to say: ~180 `ruff check` findings + ~160 files needing -`ruff format`, with `A002` (~68 findings) called out as un-fixable because -FastAPI path/query params named `id`/`type`/`filter` are the wire contract. - -What actually happened: `A002` was added to the `ignore` list as that entry -recommended, which is why the count fell from ~180 to **13**. Those 13 were then -hand-fixed (12 real fixes + 1 justified `# noqa: SIM222`), and the 176-file -format backlog was swept in a single format-only commit verified AST-identical. -`ruff` is now pinned **exactly** (`ruff==0.15.22` in the `dev` extra), so -`make lint`, the editor hook, and CI enforce the same rules on the same version -instead of whatever each happened to resolve. - -Two items remain, both deliberate and both scoped: - -- `openblade/assistant/tools.py` is the one file still failing - `ruff format --check` — the package has a live owner and was left - byte-identical. It needs one `ruff format` pass from that owner. -- 27 `# noqa` markers reference rules that are **not selected** (`BLE001` x23, - `S603`/`S404`/`S310` x4), so they are inert documentation. Enabling those - families was measured, not guessed: `BLE` = 44 findings, `S` = 3,748 (3,655 of - them `S101`, bare `assert` in tests, which would need a per-file ignore). - Do not delete those markers as dead code — they record intent for that work. - -Rule for new debt of this kind: pin the tool version at the same time you pin -the rule set. An unpinned linter makes "is the tree clean?" a question with a -different answer per machine. - -## Type-check backlog (advisory: `backend-typecheck`) - -- ~300 `mypy openblade` errors across 51 files, concentrated in the god-file - modules (`aml_state.py`, `routes_aml_*.py`). -- Pay down **per module**, not in one sweep — annotate one file, keep it green, - and (eventually) move it to a mypy per-file strict allowlist so it can't - regress. Brute-forcing all 300 at once is high-regression-risk and unreviewable. - -## Why the type-check backlog is still advisory - -Hand-fixing 300 type errors across wire-contract code in one sweep is a large, -unreviewable, high-regression change. `backend-typecheck` therefore stays -`continue-on-error: true`. The required checks (`backend-tests`, `i3-smoke`) and -the `operability-gate` remain hard gates. - -`backend-lint`'s `continue-on-error` no longer has a justification and should be -removed — the exact workflow diff is in -[`decisions/2026-09-11-pin-ruff-toolchain.md`](decisions/2026-09-11-pin-ruff-toolchain.md); -it needs an owner of `.github/workflows/**` to apply it. +## Type check: strict, zero errors, blocking + +| Check | Command (as in `ci.yml`) | Result | +| --- | --- | --- | +| Backend | `mypy openblade` (`backend-typecheck`) | `Success: no issues found in 170 source files` | +| Flask UI | `mypy openblade/web_flask` (`web-flask-smoke`) | `Success: no issues found in 4 source files` | + +`[tool.mypy]` sets `strict = true`. `backend-typecheck` has no +`continue-on-error`. On this branch it is a required dependency of `ci-gate`: +it is listed under `needs:`, and the gate's aggregate step checks its result. +The old "advisory, ~300 errors" entry is obsolete. + +## Remaining `# noqa` markers + +Inventory command (counts rule codes, which is the same as counting lines here, +because no marker lists more than one rule): + +```bash +grep -rhoE "# noqa: ?[A-Z0-9, ]+" --include=*.py openblade tests tools \ + | sed -E 's/# noqa: ?//' | tr ',' '\n' | tr -d ' ' | grep -v '^$' \ + | sort | uniq -c | sort -rn +``` + +| Rule | Count | +| --- | --- | +| `BLE001` (blind `except Exception`) | 65 | +| `B009` (`getattr` with a constant) | 7 | +| `SIM222` | 1 | +| `F401` | 1 | +| `A001` | 1 | +| **Total** | **75** (openblade 55, tests 20, tools 0) | + +There are no bare `# noqa` markers without a rule code (0 matches for +`# noqa` not followed by `:`). `BLE` is now in the selected rule set +(`[tool.ruff.lint] select`), so the `BLE001` markers are active suppressions, +not inert documentation as an earlier version of this file said. Each marker is +supposed to carry a written reason (project rule). Paying them down means +narrowing each `except Exception` to the typed errors the call can actually +raise. + +## `datetime.utcnow` (deprecated since Python 3.12) + +Paid down 2026-10-09: every call and SQLAlchemy column default now goes through +`openblade/domain/clock.py::naive_utcnow()` (same naive-UTC value, no +deprecation warning). `openblade/api/aml_state.py` keeps its own aware +`_utcnow()` helper. Verify with: + +```bash +grep -rE "datetime\.utcnow\b" --include=*.py openblade tests tools | grep -v domain/clock.py | wc -l # 0 +``` + +## Job queue (resolved 2026-10-10) + +`openblade/jobs/queue.py` is now a façade over the catalog `jobs` table and the +`LeaseStore` (`acquire_drive_lease_at`, changer = pseudo-drive `-1`); claims are +heartbeated during `run_job` and refuse drives pending reconciliation. Remaining +gap: `DriveScheduler` leases and queue claims share one drive-id space by design, +and no production caller uses `claim_drive`/`claim_changer` yet. + +## NAS persistence gaps + +Resolved: unexpired `nas_reservations` rows are a pool-level budget +(`ArchivePlanRequest.pool_reserved_bytes`) that every plan from `/nas/archive-plan` +and the upload share-push must fit under; the per-barcode `reserved_bytes` dict +stays for explicit holds. With `pool` set, the budget's tape set and capacities are +derived server-side (client-supplied capacity fields are 422, unknown pool 404). +Previews book nothing; ingest start books `owner=plan_id` with a 1h TTL (409 and no +job when the budget is exhausted), released when the job reaches a terminal state or +the executor raises, and otherwise by expiry. Open: the booking uses `plan.pool` +rather than the start request's pool, and a 409 on share-push leaves a registered +plan with no job. Scratch health counts inventory slots whose barcode is not a +volume-group member, not written (`used_bytes > 0`) and not `CLN`; the `scratch` +component appears only once thresholds are set and degrades, never fails readiness. +`nas_file_spans` has no `pool_id`, so identical relative paths in two pools share +spans. The protocol rig's NFS server is Fedora 41 nfs-ganesha 5.9 (VFS FSAL, built +with POSIX-ACL mapping); NFSv4→POSIX ACL mapping is lossy: POSIX has no DENY, so a +special-principal deny (e.g. `D::EVERYONE@:r`) is folded into the mode bits and strips +that permission from the owner too, the `n`/`N` named-attribute bits are dropped, and +removing a named-user ACE leaves a residual `A:::...` entry behind. \ No newline at end of file diff --git a/docs/test-plan.md b/docs/test-plan.md index 549dc48..ffbb53d 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -20,3 +20,52 @@ Verify injected mount, write, and capacity faults surface as typed errors. ## Safety regressions Prove the default config blocks real hardware, formatting requires confirmation, and unload-while-mounted is rejected. + +## Backend contract suite + +`tests/contract/` (`make test-contract`, marker `contract`) runs one behavioural +contract against every library/LTFS pairing, using only the `LibraryBackend` / +`LTFSBackend` Protocols: inventory shape, load/unload round-trip, double-load +refusal, barcode lookup consistency, RW/RO mount + write + read + checksum, the +unload-while-mounted safety gate, and archive → restore byte equality (sharded and +single-job) through `InventoryService`. + +| Pairing | Library | LTFS | Runs | +|---|---|---|---| +| `sim+sim` | `MockLibraryBackend` | `MockLTFSBackend` | always | +| `emulator+sim-ltfs` | `ScalarHttpLibraryBackend` over in-process ASGI (`openblade.api.main:app`) | `MockLTFSBackend` | always | +| `scsi+fake-ltfs` | `RealLibraryBackend` + `MtxChangerBackend` over a stateful fake `mtx`/`sg_inq` runner (`tests/fakes/mtx.py`) | `MockLTFSBackend` | always | +| `real+real` | `get_library()` | `get_ltfs()` | only with `OPENBLADE_REAL_HARDWARE_ENABLED=true` (marker `real_hardware`) | +| `compat-capture` | in-process `TestClient` (or `--base-url`) via `tools/compat_capture.py` | n/a | always (`tests/compat`); `self-captured` cases are regression-only, `captured` enforced | + +Known contract violations are `xfail(strict=True)` with the defect's file:line, so a +fix flips them to XPASS and fails the run until the marker is removed. + +**Rule: a new backend must pass this suite before it may be selected via +`OPENBLADE_BACKEND`.** Add it as a named pairing in `tests/contract/conftest.py`. + +## Protocol tests +Run the SMB and NFS protocol rig with `make protocols-up && make test-protocols && make protocols-down` (tests are marked `protocols`, live in `tests/e2e/protocols`, and skip with a capability-naming reason when the rig is unavailable). They are not part of the default suite; `.github/workflows/nas-protocols.yml` runs them nightly and on manual dispatch, and CI records the results. The rig uses a hydrator shim to simulate tape recall, so it does not cover the real FUSE hydrator. NFSv4 ACL enforcement is a real scenario: `test_nfs_acl_mapping` sets a named-user DENY ACE with `nfs4_setfacl` and asserts the server refuses that uid a read its mode bits allow. See `docs/wiki/guides/fuse-and-nas.md`. + +## CI lanes +Lane definitions live in the README section "Layered CI/CD". This section lists which tests each lane runs. + +- **PR (`ci.yml`, required through `ci-gate`).** + - `backend-lint`: `ruff check .` and `ruff format --check .`. + - `backend-typecheck`: `mypy openblade` under strict mode. It blocks merge. + - `backend-tests`: unit and integration tests. + - `api-aml-integration`. + - `i3-smoke`. + - `frontend-build-test`. + - `web-flask-smoke`. + - The ownership check `tools/ci_ownership.py --base origin/master`. It maps every changed path, through `tools/ci_ownership.toml`, to `unit`, `integration`, `safety`, `i3`, `compat`, `frontend`, `docs-only` or `ci-only`. It fails on any changed `openblade/**` or `tests/**` path that has no owner. If you add a new test directory or package, add its glob to the TOML table. +- **Emulator.** `emulator-change-gates.yml` and `i3-emulator-compliance.yml` both run `tests/i3` through the reusable `_emulator-boot-test.yml`, with an `I3_TIMING_PROFILE` input. + - The timing profiles are `instant`, `realistic`, `hardware`, `normal`, `slow-robotics`, `busy-library`, `intermittent-drive`, `session-expiry`, `rebooting` and `degraded-media`. + - `tests/i3/test_timing_profiles.py` checks each profile's effect under `VirtualClock`, with no real sleeping. +- **Nightly (`nightly.yml`, not a PR check).** Run the same selections locally with pytest. + - `slow`: `-m slow`. + - `stress`: `tests/fault` plus `-m stress`. + - `fuzz`: `tests/property`. + - `rebuild`: `tests/unit/test_catalog_rebuild*`. + - `mutation`: `make mutation` / `tools/mutation_run.sh`. It runs mutmut against the safety-critical modules using `tests/safety` plus the scheduler, drive-lease, recovery and scalar-coordinate unit tests. Surviving mutants are compared with `mutation/baseline.txt`, and any new survivor fails the lane. + - The pytest markers `stress`, `fuzz`, `rebuild` and `mutation` are registered in `pyproject.toml`. diff --git a/docs/wiki/guides/fuse-and-nas.md b/docs/wiki/guides/fuse-and-nas.md index 02bbad8..2050aee 100644 --- a/docs/wiki/guides/fuse-and-nas.md +++ b/docs/wiki/guides/fuse-and-nas.md @@ -73,15 +73,15 @@ Other limits, stated rather than hidden: |---|---| | `OPENBLADE_FUSE_MOUNT_POINT` | The config field `fuse_mount_point` exists, is **read by nothing**, and has no environment override. `openblade fuse mount` takes the mount point as an argument. | | A FUSE mount of the **NAS** namespace | **No.** Only the catalog namespace is mountable. | -| An SMB (Samba) server | **No.** `openblade/nas/samba.py` is a 15-line class whose only method renders a 3-line `smb.conf` stanza. It is referenced by nothing — not by the app, not by tests. | -| An NFS server | **No.** Same shape: `nfs.py` renders one `/etc/exports` line and is referenced by nothing. | +| An SMB (Samba) server | **Not in the image.** The protocol rig lives in `deploy/nas-protocols`; see it for status. | +| An NFS server | **Not in the image.** See `deploy/nas-protocols`. | | An SFTP listener | **Effectively no.** See below. | -| `libfuse` in the shipped image | **No.** The extra and `libfuse2` are a host-side install; the container does not ship them. | +| `libfuse` in the shipped image | `fuse3` is now apt-installed in the Dockerfile. The `fuse` Python extra (fusepy, libfuse2 API) and `/dev/fuse` + `SYS_ADMIN` at runtime are still the operator's job — unverified in a container. | The `nas_shares` table and the `/nas/shares` CRUD endpoints store share definitions as **metadata only**. Nothing renders or applies them. -The code itself says so — `openblade/nas/fuse_hook.py`: +Hydration for the **catalog** mount is now real (`openblade/fuse/hydration.py`, see `docs/fuse.md`). The NAS-pool hook below is still the old stub — `openblade/nas/fuse_hook.py`: > Stub for optional FUSE virtual filesystem integration. In v1, this is a no-op > stub that records access attempts and returns appropriate offline/hydrating @@ -205,6 +205,60 @@ policy-resolution helper for ingest and has nothing to do with mounting. --- +## Protocol test rig + +`deploy/nas-protocols/docker-compose.yml` brings up a throwaway SMB and NFS +stack so the protocol exports can be exercised by real clients. Services: + +- `samba` — serves shares from an `smb.conf` rendered by `openblade/nas/samba.py`. +- `nfs-ganesha` — serves exports from a config rendered by `openblade/nas/nfs.py`. +- `client` — a privileged container that kernel-mounts both protocols + internally (SMB via `cifs vers=3.0,cache=none,actimeo=0`, NFS via `nfs4`); + mounting needs privileges the CI runner's test process does not have. + +Both config files are rendered from `NasShare` records, so the rig tests the +renderers' output rather than hand-written configs. + +Run it: + +```bash +make protocols-up && make test-protocols && make protocols-down +``` + +`protocols-up` builds, starts and waits for healthy; `test-protocols` runs +`python -m pytest -m protocols tests/e2e/protocols -q`; `protocols-down` tears +down with `-v`. Host ports come from `OB_SMB_PORT` and `OB_NFS_PORT` +(Makefile defaults 14450 and 20490); the pytest fixture instead picks free +ports when those variables are unset. Tests +carry the `protocols` marker and skip, with a reason naming the missing +capability, when the rig is not available. + +**What the rig simulates.** The rig contains a hydrator shim, not the real +hydrator. The shim models a tape recall: a stub file becomes real content after +a delay, or never does for an offline cartridge. It exists to check how SMB and +NFS clients behave while a read blocks or times out. The real hydrator is the +FUSE path described above, and it is not exercised by the rig, so rig results +say nothing about actual recall from tape. The offline stub file is a rig-only +format: the catalog has no on-disk OFFLINE_ON_TAPE representation. + +**SMB caching semantics.** The shim (like an evictor) swaps files outside +smbd, so smbd cannot break client leases. The rendered `smb.conf` therefore +sets `oplocks = no`, `level2 oplocks = no` and `smb2 leases = no`; with leases +on, the Linux cifs client's deferred close (`closetimeo=1`) kept a cached +handle on the stub's old inode while a poller reopened it more often than once +a second, and the recall was never seen. Observed with leases off: a handle +opened before an eviction keeps reading the old bytes, and while that handle +stays open a reopen by path also returns the old bytes; after it closes, the +next open sees the new file. A Samba restart mid-recall is survived (the soft +mount reconnects and the poll completes). + +**CI.** `.github/workflows/nas-protocols.yml` runs the rig nightly and on manual +dispatch. It is not triggered by pull requests or pushes and is not a required +check. Per-scenario results are recorded by CI runs; on failure the workflow +uploads the compose logs as an artifact. + +--- + ## Doc corrections - `docs/fuse.md` calls this "a lightweight namespace abstraction over the @@ -229,6 +283,26 @@ policy-resolution helper for ingest and has nothing to do with mounting. --- +## Tape-native domain models + +Pure planning models (no I/O). Each is validated and covered by unit tests. + +| Model / function | Module | Tests | +|---|---|---| +| `TapeState` / `TapeEvent` / `transition()` -> `InvalidTransitionError`; `file_state_for_tape()` maps onto the existing `NasFileState` | `openblade/nas/state_machine.py` | `tests/unit/test_nas_tape_states.py` (every allowed edge, every forbidden edge) | +| `MediaGeneration`, `can_read` / `can_write` (LTO-8/9 break the N-2 read rule), `Cartridge`, `is_foreign`, `require_writable` | `openblade/nas/media.py` | `tests/unit/test_nas_media_compat.py` | +| `ArchivePlanRequest.cartridges` / `drive_generation` / `reserved_bytes`: the archive planner raises `ForeignMediaError` for un-adopted media without `tape.json` (never auto-formatted) and `IncompatibleMediaError` for an unwritable drive/media pairing; reserved bytes reduce per-tape capacity | `openblade/nas/planner.py` | `tests/unit/test_nas_media_compat.py` | +| `ScratchThresholds` + `evaluate_scratch` (OK/WARN/CRITICAL) | `openblade/nas/capacity.py` | `tests/unit/test_nas_domain_models.py` | +| `TapeFragmentation.needs_consolidation` (`live_bytes / written_bytes`) | same | same | +| `ReservationLedger.reserve/release` (TTL; expired ignored; over-reserve -> `ReservationError`) | same | same | +| `VolumeGroupReplication` (IN_SYNC/LAGGING/BROKEN), `merge_volume_groups`, `require_export_ready` | same | same | +| `FileSpan` / `SpanSegment`, `required_tapes`, `span_state` (missing tape -> `MISSING_TAPE`) | same | same | +| `ExportSet` (PREPARING/READY/EXPORTED/IMPORTED), `mark_export_ready`, `Vault`, `planning_available` (vaulted = offline) | same | same | +| `plan_drives` -> `required_drives`, `estimated_swaps` | same | same | +| `AggregationPolicy.aggregate_below_bytes` + `group_for_write` (grouping only, no container format) | same | same | + +No read-only HTTP view is exposed yet: there is no per-pool health hook to attach to. + ## Related - [The catalog](the-catalog.md) — the two independent catalogs diff --git a/docs/wiki/reference/api.md b/docs/wiki/reference/api.md index 115b313..fb4e7cf 100644 --- a/docs/wiki/reference/api.md +++ b/docs/wiki/reference/api.md @@ -6,7 +6,7 @@ Introspected from the OpenAPI schema of `openblade.api.main:app`. -**1178 operations** total: 198 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. +**1190 operations** total: 210 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. The application serves two surfaces from one ASGI app. Setting `OPENBLADE_SCALAR_API_ONLY=true` puts it in emulator-only mode, where the @@ -83,11 +83,13 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/restore/` | Enqueue Restore | - | `openblade__api__routes_restore__RestoreRequest` | `EnqueuedJobResponse` | | `POST` | `/restore/tree` | Restore Tree | - | `TreeRestoreApiRequest` | `TreeRestoreResponse` | -### `jobs` (2 operations) +### `jobs` (4 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | | `GET` | `/jobs/` | List Jobs | `library_id`? (query) | - | `openblade__api__routes_jobs__JobResponse[]` | +| `GET` | `/jobs/recovery` | Get Recovery Report | - | - | `RecoveryReportResponse` | +| `POST` | `/jobs/recovery/reconcile/{drive_id}` | Post Reconcile Drive | `drive_id` (path) | - | `ReconcileDriveResponse` | | `GET` | `/jobs/{job_id}` | Get Job | `job_id` (path) | - | `openblade__api__routes_jobs__JobResponse` | ### `catalog` (6 operations) @@ -112,7 +114,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/virtual/ls` | List Virtual Directory | `path`? (query) | - | `VirtualDirectoryListing` | | `GET` | `/virtual/stat` | Stat Virtual Path | `path` (query) | - | `VirtualFileEntry` | -### `NAS Config` (118 operations) +### `NAS Config` (128 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | @@ -135,6 +137,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/datasets/{dataset_id}/manifest` | Get Dataset Manifest | `dataset_id` (path) | - | `object` | | `GET` | `/nas/datasets/{dataset_id}/report` | Get Dataset Report | `dataset_id` (path) | - | `object` | | `POST` | `/nas/datasets/{dataset_id}/verify` | Verify Dataset | `dataset_id` (path) | - | `object` | +| `GET` | `/nas/export-sets` | List Export Sets | - | - | `object[]` | | `GET` | `/nas/fuse/log` | Get Fuse Log | - | - | `object[]` | | `POST` | `/nas/fuse/open` | Open Virtual File | - | `FuseOpenRequest` | `object` | | `POST` | `/nas/ingest/start` | Start Ingest | - | `StartIngestRequest` | `StartIngestResponse` | @@ -158,6 +161,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/pools/{pool_id}/browse` | Browse Pool | `pool_id` (path), `path`? (query) | - | `object` | | `GET` | `/nas/pools/{pool_id}/files/{file_path}` | Get Pool File Detail | `pool_id` (path), `file_path` (path) | - | `NasFileRecord` | | `POST` | `/nas/pools/{pool_id}/request-restore` | Request Restore | `pool_id` (path) | `RestorePlanRequest` | `NasRestoreJob` | +| `GET` | `/nas/reservations` | List Reservations | `pool_id`? (query) | - | `object[]` | | `POST` | `/nas/resolve-policy` | Resolve Policy | - | `ResolvePolicyRequest` | `EffectivePolicy` | | `GET` | `/nas/restore-jobs` | List Restore Jobs | - | - | `NasRestoreJob[]` | | `GET` | `/nas/restore-jobs/{job_id}` | Get Restore Job | `job_id` (path) | - | `NasRestoreJob` | @@ -168,6 +172,8 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/nas/restore-jobs/{job_id}/retry` | Retry Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/nas/restore-jobs/{job_id}/run` | Run Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/nas/restore-plan` | Restore Plan | - | `RestorePlanRequest` | `RestorePlan` | +| `GET` | `/nas/scratch-thresholds` | Get Scratch Thresholds | - | - | `ScratchThresholds` | +| `PUT` | `/nas/scratch-thresholds` | Put Scratch Thresholds | - | `ScratchThresholds` | `ScratchThresholds` | | `GET` | `/nas/shares` | List Shares | - | - | `NasShareDefinition[]` | | `POST` | `/nas/shares` | Create Or Update Share | - | `NasShareDefinition` | `NasShareDefinition` | | `GET` | `/nas/shares/{share_id}` | Get Share | `share_id` (path) | - | `NasShareDefinition` | @@ -175,6 +181,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/source-stream` | Get Source Stream Config | - | - | `SourceStreamConfig` | | `PUT` | `/nas/source-stream` | Update Source Stream Config | - | `SourceStreamConfig` | `SourceStreamConfig` | | `DELETE` | `/nas/source-stream` | Delete Source Stream Config | - | - | `object` | +| `GET` | `/nas/vaults` | List Vaults | - | - | `object[]` | | `POST` | `/storage/nas/archive-plan` | Archive Plan | - | `ArchivePlanRequest` | `ArchivePlan` | | `GET` | `/storage/nas/cache-drives` | List Cache Drives | - | - | `CacheDriveConfig[]` | | `POST` | `/storage/nas/cache-drives` | Create Or Update Cache Drive | - | `CacheDriveConfig` | `CacheDriveConfig` | @@ -194,6 +201,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/datasets/{dataset_id}/manifest` | Get Dataset Manifest | `dataset_id` (path) | - | `object` | | `GET` | `/storage/nas/datasets/{dataset_id}/report` | Get Dataset Report | `dataset_id` (path) | - | `object` | | `POST` | `/storage/nas/datasets/{dataset_id}/verify` | Verify Dataset | `dataset_id` (path) | - | `object` | +| `GET` | `/storage/nas/export-sets` | List Export Sets | - | - | `object[]` | | `GET` | `/storage/nas/fuse/log` | Get Fuse Log | - | - | `object[]` | | `POST` | `/storage/nas/fuse/open` | Open Virtual File | - | `FuseOpenRequest` | `object` | | `POST` | `/storage/nas/ingest/start` | Start Ingest | - | `StartIngestRequest` | `StartIngestResponse` | @@ -217,6 +225,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/pools/{pool_id}/browse` | Browse Pool | `pool_id` (path), `path`? (query) | - | `object` | | `GET` | `/storage/nas/pools/{pool_id}/files/{file_path}` | Get Pool File Detail | `pool_id` (path), `file_path` (path) | - | `NasFileRecord` | | `POST` | `/storage/nas/pools/{pool_id}/request-restore` | Request Restore | `pool_id` (path) | `RestorePlanRequest` | `NasRestoreJob` | +| `GET` | `/storage/nas/reservations` | List Reservations | `pool_id`? (query) | - | `object[]` | | `POST` | `/storage/nas/resolve-policy` | Resolve Policy | - | `ResolvePolicyRequest` | `EffectivePolicy` | | `GET` | `/storage/nas/restore-jobs` | List Restore Jobs | - | - | `NasRestoreJob[]` | | `GET` | `/storage/nas/restore-jobs/{job_id}` | Get Restore Job | `job_id` (path) | - | `NasRestoreJob` | @@ -227,6 +236,8 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/storage/nas/restore-jobs/{job_id}/retry` | Retry Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/storage/nas/restore-jobs/{job_id}/run` | Run Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/storage/nas/restore-plan` | Restore Plan | - | `RestorePlanRequest` | `RestorePlan` | +| `GET` | `/storage/nas/scratch-thresholds` | Get Scratch Thresholds | - | - | `ScratchThresholds` | +| `PUT` | `/storage/nas/scratch-thresholds` | Put Scratch Thresholds | - | `ScratchThresholds` | `ScratchThresholds` | | `GET` | `/storage/nas/shares` | List Shares | - | - | `NasShareDefinition[]` | | `POST` | `/storage/nas/shares` | Create Or Update Share | - | `NasShareDefinition` | `NasShareDefinition` | | `GET` | `/storage/nas/shares/{share_id}` | Get Share | `share_id` (path) | - | `NasShareDefinition` | @@ -234,6 +245,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/source-stream` | Get Source Stream Config | - | - | `SourceStreamConfig` | | `PUT` | `/storage/nas/source-stream` | Update Source Stream Config | - | `SourceStreamConfig` | `SourceStreamConfig` | | `DELETE` | `/storage/nas/source-stream` | Delete Source Stream Config | - | - | `object` | +| `GET` | `/storage/nas/vaults` | List Vaults | - | - | `object[]` | ### `upload-download` (6 operations) diff --git a/mutation/baseline.txt b/mutation/baseline.txt new file mode 100644 index 0000000..eb8cf2c --- /dev/null +++ b/mutation/baseline.txt @@ -0,0 +1,9 @@ +# Mutation ratchet: . +# Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-10. Only ever lower them. +# All 6 modules in [tool.mutmut].only_mutate are baselined (openblade/safety/* == import_guard.py). +openblade/domain/policies.py 2 +openblade/domain/scalar_coordinate.py 4 +openblade/jobs/recovery.py 32 +openblade/jobs/scheduler.py 87 +openblade/nas/manifest_validator.py 74 +openblade/safety/import_guard.py 20 diff --git a/openblade/api/aml_faults.py b/openblade/api/aml_faults.py new file mode 100644 index 0000000..e251f65 --- /dev/null +++ b/openblade/api/aml_faults.py @@ -0,0 +1,157 @@ +"""Env-driven AML wire-level fault injection for the emulator. + +``OPENBLADE_EMULATOR_FAULT_PROFILE`` is either a preset name (see ``PRESETS``) or a +JSON object with any of the keys ``load_fail_every`` (int), ``auth_ttl_s`` (float), +``reboot_window_s`` (float) and ``checksum_retry_every`` (int). Unset/empty means +no faults. Anything unparseable raises ``FaultProfileError``: a silent no-fault run +would be a false green for the scenario that asked for faults. + +The presets are the single source of truth for the i3 timing profiles +(``tests/i3/timing.py`` imports them). +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from dataclasses import dataclass, fields + +FAULT_PROFILE_ENV = "OPENBLADE_EMULATOR_FAULT_PROFILE" + + +class FaultProfileError(ValueError): + """``OPENBLADE_EMULATOR_FAULT_PROFILE`` could not be parsed.""" + + +@dataclass(frozen=True) +class FaultProfile: + """Fault knobs (0/None = off).""" + + load_fail_every: int = 0 + auth_ttl_s: float | None = None + reboot_window_s: float = 0.0 + checksum_retry_every: int = 0 + + @property + def active(self) -> bool: + return self != FaultProfile() + + +PRESETS: dict[str, FaultProfile] = { + "instant": FaultProfile(), + "realistic": FaultProfile(), + "hardware": FaultProfile(), + "normal": FaultProfile(), + "slow-robotics": FaultProfile(), + "busy-library": FaultProfile(), + "intermittent-drive": FaultProfile(load_fail_every=3), + "session-expiry": FaultProfile(auth_ttl_s=5.0), + "rebooting": FaultProfile(reboot_window_s=20.0), + "degraded-media": FaultProfile(checksum_retry_every=4), +} + +_INT_KEYS = ("load_fail_every", "checksum_retry_every") +_FLOAT_KEYS = ("auth_ttl_s", "reboot_window_s") + + +def parse_fault_profile(raw: str | None) -> FaultProfile: + """Strictly parse a preset name or JSON object; raise ``FaultProfileError``.""" + text = (raw or "").strip() + if not text: + return FaultProfile() + if not text.startswith("{"): + try: + return PRESETS[text.lower()] + except KeyError: + raise FaultProfileError( + f"{FAULT_PROFILE_ENV}: unknown preset {text!r}; expected one of " + f"{sorted(PRESETS)} or a JSON object" + ) from None + try: + data = json.loads(text) + except json.JSONDecodeError: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: invalid JSON") from None + if not isinstance(data, dict): + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: JSON must be an object") + known = {f.name for f in fields(FaultProfile)} + unknown = sorted(set(data) - known) + if unknown: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: unknown keys {unknown}") + values: dict[str, int | float] = {} + for key, value in data.items(): + # bool is an int subclass; reject it explicitly. + if isinstance(value, bool) or not isinstance(value, int | float) or value < 0: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: {key} must be a number >= 0") + if key in _INT_KEYS: + if not isinstance(value, int): + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: {key} must be an integer") + values[key] = value + else: + values[key] = float(value) + return FaultProfile( + load_fail_every=int(values.get("load_fail_every", 0)), + auth_ttl_s=float(values["auth_ttl_s"]) if "auth_ttl_s" in values else None, + reboot_window_s=float(values.get("reboot_window_s", 0.0)), + checksum_retry_every=int(values.get("checksum_retry_every", 0)), + ) + + +def load_fault_profile_from_env() -> FaultProfile: + return parse_fault_profile(os.environ.get(FAULT_PROFILE_ENV)) + + +class FaultState: + """Process-wide fault counters (thread-safe).""" + + def __init__(self, profile: FaultProfile) -> None: + self.profile = profile + self.started_at = time.monotonic() + self.loads = 0 + self.verifies = 0 + self._lock = threading.Lock() + + def in_reboot_window(self) -> bool: + window = self.profile.reboot_window_s + return bool(window) and time.monotonic() - self.started_at < window + + def reboot_retry_after_s(self) -> int: + remaining = self.profile.reboot_window_s - (time.monotonic() - self.started_at) + return max(1, int(remaining + 0.999)) + + def should_fail_load(self) -> bool: + every = self.profile.load_fail_every + if not every: + return False + with self._lock: + self.loads += 1 + return self.loads % every == 0 + + def should_retry_checksum(self) -> bool: + every = self.profile.checksum_retry_every + if not every: + return False + with self._lock: + self.verifies += 1 + return self.verifies % every == 0 + + +_state: FaultState | None = None +_state_lock = threading.Lock() + + +def get_fault_state() -> FaultState: + """Return the process-wide state, parsing the env on first use (may raise).""" + global _state + with _state_lock: + if _state is None: + _state = FaultState(load_fault_profile_from_env()) + return _state + + +def reset_fault_state() -> None: + """Drop the cached state so the next call re-reads the env (tests).""" + global _state + with _state_lock: + _state = None diff --git a/openblade/api/aml_latency.py b/openblade/api/aml_latency.py index 5dabf8d..0fa8cee 100644 --- a/openblade/api/aml_latency.py +++ b/openblade/api/aml_latency.py @@ -9,8 +9,9 @@ from pathlib import Path from typing import Any -from fastapi import Request +from fastapi import HTTPException, Request +from openblade.api.aml_faults import get_fault_state from openblade.api.aml_state import ( get_aml_emulator_latency_config, record_aml_emulator_latency_metric, @@ -128,18 +129,53 @@ def _resolve_profile_delay_ms( return max(0, delay_ms) -def resolve_request_latency_delay_seconds(method: str, path: str) -> float: - if not _is_latency_managed_path(path): - return 0.0 +def _resolve_operation_class(method: str, path: str) -> str: operation_class = _lookup_operation_class(method, path) if operation_class is None: operation_class = _fallback_operation_class(method, path) + return operation_class + + +def resolve_request_latency_delay_seconds(method: str, path: str) -> float: + if not _is_latency_managed_path(path): + return 0.0 + operation_class = _resolve_operation_class(method, path) config = get_aml_emulator_latency_config() return _resolve_profile_delay_ms(config, operation_class) / 1000.0 +def resolve_request_fault(method: str, path: str) -> HTTPException | None: + """Return the injected AML error for this request, if a fault profile fires. + + Driven by ``OPENBLADE_EMULATOR_FAULT_PROFILE`` (see ``aml_faults``). The caller + renders it through the normal AML error envelope. + """ + if not _is_latency_managed_path(path): + return None + state = get_fault_state() + if not state.profile.active: + return None + if state.in_reboot_window(): + return HTTPException( + status_code=503, + detail="Library is rebooting; retry later", + headers={"Retry-After": str(state.reboot_retry_after_s())}, + ) + if _resolve_operation_class(method, path) == "mount" and state.should_fail_load(): + return HTTPException(status_code=409, detail="Drive load failed; retry the operation") + return None + + async def apply_request_latency(request: Request) -> float: - delay = resolve_request_latency_delay_seconds(request.method, request.url.path) + method, path = request.method, request.url.path + delay = resolve_request_latency_delay_seconds(method, path) + if ( + delay > 0 + and _resolve_operation_class(method, path) == "diagnostic" + and get_fault_state().should_retry_checksum() + ): + # Simulated checksum retry: the verify pass runs a second time. + delay *= 2 if delay > 0: await asyncio.sleep(delay) return delay diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 6c1b2dd..0665544 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -19,6 +19,7 @@ import pyotp from sqlalchemy import select +from openblade.api.aml_faults import get_fault_state from openblade.catalog.db import get_session, init_db from openblade.catalog.models import AmlUser from openblade.simulator.i3_config import scalar_i3_active_config @@ -234,6 +235,9 @@ class SessionRecord: role: int created_at: datetime expires_at: datetime + # Inferred (not in the Rev D manual): a login by a user with an enabled MFA + # record is pending until POST /users/login/mfa succeeds. Never serialised. + mfa_pending: bool = False def as_dict(self) -> dict[str, Any]: return { @@ -2462,15 +2466,50 @@ def purge_expired_sessions() -> None: _STATE.sessions.pop(token, None) -def create_session(user: AmlUser) -> SessionRecord: +_DEFAULT_MAX_SESSIONS_PER_USER = 5 + + +def max_sessions_per_user() -> int: + """Concurrent-session cap per user (inferred; the Rev D manual documents no limit). + + Read from ``OPENBLADE_AML_MAX_SESSIONS`` at call time; unparseable or < 1 falls + back to the default. + """ + raw = os.environ.get("OPENBLADE_AML_MAX_SESSIONS", "") + try: + value = int(raw) + except ValueError: + return _DEFAULT_MAX_SESSIONS_PER_USER + return value if value >= 1 else _DEFAULT_MAX_SESSIONS_PER_USER + + +def _session_lifetime() -> timedelta: + # OPENBLADE_EMULATOR_FAULT_PROFILE auth_ttl_s (e.g. "session-expiry") overrides + # the configured timeout so expiry is observable within a test run. + ttl = get_fault_state().profile.auth_ttl_s + if ttl is not None: + return timedelta(seconds=ttl) + return timedelta(minutes=_STATE.session_timeout_minutes) + + +def create_session(user: AmlUser, *, mfa_pending: bool = False) -> SessionRecord: purge_expired_sessions() + # Evict the user's oldest sessions so the new login always succeeds: refusing + # would let abandoned sessions lock an operator (incl. admin) out until expiry. + own = sorted( + (record for record in _STATE.sessions.values() if record.user_name == user.name), + key=lambda record: record.created_at, + ) + for stale in own[: max(0, len(own) - max_sessions_per_user() + 1)]: + _STATE.sessions.pop(stale.token, None) now = _utcnow() record = SessionRecord( token=str(uuid4()), user_name=user.name, role=user.role, created_at=now, - expires_at=now + timedelta(minutes=_STATE.session_timeout_minutes), + expires_at=now + _session_lifetime(), + mfa_pending=mfa_pending, ) _STATE.sessions[record.token] = record return record @@ -2481,10 +2520,23 @@ def get_session_user(token: str) -> AmlUser | None: record = _STATE.sessions.get(token) if record is None: return None - record.expires_at = _utcnow() + timedelta(minutes=_STATE.session_timeout_minutes) + if get_fault_state().profile.auth_ttl_s is None: + # Sliding expiry; a fault-profile TTL is absolute from login instead. + record.expires_at = _utcnow() + timedelta(minutes=_STATE.session_timeout_minutes) return get_user(record.user_name) +def is_session_mfa_pending(token: str) -> bool: + record = _STATE.sessions.get(token) + return record is not None and record.mfa_pending + + +def clear_session_mfa_pending(token: str) -> None: + record = _STATE.sessions.get(token) + if record is not None: + record.mfa_pending = False + + def clear_session(token: str) -> None: _STATE.sessions.pop(token, None) @@ -2575,6 +2627,10 @@ def set_user_mfa(user_name: str, mfa_type: str, enabled: bool) -> dict[str, Any] return existing.as_dict() +def user_has_enabled_mfa(user_name: str) -> bool: + return any(record.enabled for record in _STATE.mfa_configs.get(user_name, {}).values()) + + def get_user_mfa(user_name: str, mfa_type: str) -> MFARecord | None: return _STATE.mfa_configs.setdefault(user_name, _default_user_mfa()).get(mfa_type) diff --git a/openblade/api/main.py b/openblade/api/main.py index f4bc959..b16ea43 100644 --- a/openblade/api/main.py +++ b/openblade/api/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import os import time from typing import Any @@ -58,9 +59,11 @@ routes_virtual_fs, routes_volume_groups, ) +from openblade.api.aml_faults import get_fault_state from openblade.api.aml_latency import ( apply_request_latency, capture_request_latency_metric, + resolve_request_fault, should_capture_latency_metrics, ) from openblade.api.api_auth import api_auth_middleware, log_api_auth_status @@ -186,6 +189,12 @@ async def apply_aml_emulator_latency( if not should_capture_latency_metrics(request.url.path): return await call_next(request) + # Env-driven fault profile (OPENBLADE_EMULATOR_FAULT_PROFILE): reboot-window 503 + # and intermittent load 409 are answered here, before routing, like the appliance. + fault = resolve_request_fault(request.method, request.url.path) + if fault is not None: + return await handle_http_exception(request, fault) + start = time.perf_counter() simulated_delay = await apply_request_latency(request) status_code = 500 @@ -279,6 +288,9 @@ async def initialize_aml_state() -> None: from openblade.api.aml_state import ensure_initialized log_api_auth_status() + # Parse the fault profile at boot so a typo fails loudly here, not as a 500 on + # the first request; this also anchors the reboot window at process start. + get_fault_state() context = get_context() ensure_initialized( context.config.db_url, @@ -370,10 +382,21 @@ async def handle_http_exception(request: Request, exc: HTTPException) -> Respons if not _is_aml_compatible_path(request.url.path): return await http_exception_handler(request, exc) return JSONResponse( - status_code=exc.status_code, content=_aml_error_payload(exc.status_code, exc.detail) + status_code=exc.status_code, + content=_aml_error_payload(exc.status_code, exc.detail), + headers=exc.headers, ) +@app.exception_handler(Exception) +async def handle_unexpected_exception(request: Request, exc: Exception) -> Response: + """Last-resort handler: AML paths get the error envelope, never the exception text.""" + if not _is_aml_compatible_path(request.url.path): + raise exc + logging.getLogger(__name__).exception("Unhandled error on %s", request.url.path) + return JSONResponse(status_code=500, content=_aml_error_payload(500, "Internal server error")) + + @app.exception_handler(RequestValidationError) async def handle_request_validation_error( request: Request, exc: RequestValidationError diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index 8b13efb..9d2bc7b 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -17,8 +17,18 @@ from openblade.catalog.db import get_catalog_repository from openblade.catalog.models import AmlUser from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.domain.models import MountHandle, MountMode from openblade.domain.wire import coerce_int +from openblade.nas.capacity import ( + ClientBudgetFieldsError, + PoolNotFoundError, + ReservationError, + ScratchThresholds, + book_plan_capacity, + plan_with_pool_budget, + server_budget_request, +) from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.catalog_rebuild_worker import SAFE_REBUILD_PREFLIGHT_ERROR, CatalogRebuildWorker from openblade.nas.catalog_shard import CatalogShardWriter @@ -38,7 +48,6 @@ from openblade.nas.ltfs_manifest import TapeMetadataWriter from openblade.nas.manifest_validator import ManifestValidator from openblade.nas.path_mapping import PathMappingService -from openblade.nas.planner import ArchivePlanner from openblade.nas.restore_planner import RestorePlan, RestorePlanner from openblade.nas.service import NasService from openblade.nas.sidecar import SidecarResolver @@ -220,7 +229,7 @@ def _manifest_payload(service: NasService, dataset_id: str) -> dict[str, object] ], "total_files": detail["file_count"], "total_bytes": detail["total_bytes"], - "generated_at": datetime.utcnow().isoformat() + "Z", + "generated_at": naive_utcnow().isoformat() + "Z", } @@ -237,7 +246,7 @@ def _report_payload(service: NasService, dataset_id: str) -> dict[str, object]: for record in records ], "checksums": {record.relative_path: record.checksum_sha256 for record in records}, - "generated_at": datetime.utcnow().isoformat() + "Z", + "generated_at": naive_utcnow().isoformat() + "Z", } @@ -878,6 +887,50 @@ async def get_pool(pool_id: str, service: NasService = Depends(get_nas_service)) return pool +@router.get("/reservations") +async def list_reservations( + pool_id: str | None = None, + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_reservations(pool_id) + + +@router.get("/scratch-thresholds", response_model=ScratchThresholds) +async def get_scratch_thresholds( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> ScratchThresholds: + stored = repo.get_scratch_thresholds() + if stored is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Scratch thresholds not configured" + ) + return ScratchThresholds.model_validate(stored) + + +@router.put("/scratch-thresholds", response_model=ScratchThresholds) +async def put_scratch_thresholds( + thresholds: ScratchThresholds, + repo: CatalogRepository = Depends(get_catalog_repository), +) -> ScratchThresholds: + # The body is validated by ScratchThresholds itself (warn < min -> 422). + repo.set_scratch_thresholds(thresholds.scratch_min, thresholds.scratch_warn) + return thresholds + + +@router.get("/export-sets") +async def list_export_sets( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_export_sets() + + +@router.get("/vaults") +async def list_vaults( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_vaults() + + @router.put("/pools/{pool_id}", response_model=NasPool) async def update_pool( pool_id: str, @@ -1112,6 +1165,14 @@ async def archive_plan( request: ArchivePlanRequest, service: NasService = Depends(get_nas_service), ) -> ArchivePlan: + """Plan an archive. + + With ``pool`` set, tapes and capacities are derived server-side from the library + inventory + catalog: body-supplied ``available_tapes``/``tape_capacities``/ + ``reserved_bytes``/``pool_reserved_bytes`` are refused (422) and an unknown pool + is 404. Without ``pool`` the body-supplied inputs drive a preview-only plan that + gets no pool budget and books nothing. + """ policy = None if request.policy_id is not None: policy = service.get_policy(request.policy_id) @@ -1141,7 +1202,15 @@ async def archive_plan( updates["max_parallelism"] = policy.max_parallelism request = request.model_copy(update=updates) - plan = ArchivePlanner().plan(request) + try: + request = server_budget_request(service.repository, request, get_context().library) + except ClientBudgetFieldsError as exc: + # Curated domain message naming only the refused field names. + raise HTTPException(status_code=422, detail=str(exc)) from exc + except PoolNotFoundError as exc: + # Curated domain message (pool id only). + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + plan = plan_with_pool_budget(service.repository, request, now=datetime.now(timezone.utc)) if policy is not None: plan.policy_name = policy.name plan.policy_type = policy.policy_type @@ -1181,6 +1250,13 @@ async def start_ingest( policy = service.get_policy(plan.policy_name) if policy is not None and not policy.auto_clean_before_archive: auto_clean_drives = False + try: + book_plan_capacity( + service.repository, plan, context.library, now=datetime.now(timezone.utc) + ) + except ReservationError as exc: + # Curated domain message (pool + plan id only); the pool budget is exhausted. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc if auto_clean_drives: cleaned_drives = _auto_clean_required_drives() job = start_ingest_job( diff --git a/openblade/api/routes_aml_auth.py b/openblade/api/routes_aml_auth.py index b2188b5..056f516 100644 --- a/openblade/api/routes_aml_auth.py +++ b/openblade/api/routes_aml_auth.py @@ -4,10 +4,12 @@ import collections import os +import re import string import time from collections.abc import Callable from typing import Any, TypeVar +from xml.etree import ElementTree import pyotp from fastapi import ( @@ -58,6 +60,7 @@ def _check_rate_limit(remote_ip: str) -> None: raise HTTPException( status_code=429, detail=f"Too many login attempts. Try again in {_LOGIN_WINDOW_SECONDS // 60} minutes.", + headers={"Retry-After": str(max(1, int(deque[0] - window_start) + 1) if deque else 1)}, ) deque.append(now) @@ -332,31 +335,122 @@ def _verify_totp(secret: str, code: str) -> bool: return pyotp.TOTP(secret).verify(code, valid_window=1) -async def require_auth(request: Request, context: AppContext = Depends(get_context)) -> AmlUser: - _ensure_state(context) +def _session_token(request: Request) -> str | None: + """Return the token that authenticates this request (cookie first, then Bearer).""" + session_id = request.cookies.get("sessionID") + if session_id and aml_state.get_session_user(session_id) is not None: + return session_id + auth = request.headers.get("authorization", "") + if auth.lower().startswith("bearer "): + token = auth.split(" ", 1)[1] + if aml_state.get_session_user(token) is not None: + return token + return None + + +def _authenticate(request: Request, *, allow_mfa_pending: bool) -> AmlUser: # Prefer session cookie, fall back to Bearer token for API clients/tests session_id = request.cookies.get("sessionID") - user = None - if session_id: - user = aml_state.get_session_user(session_id) - if user is None: - auth = request.headers.get("authorization", "") - if auth.lower().startswith("bearer "): - token = auth.split(" ", 1)[1] - user = aml_state.get_session_user(token) - if user is None: + token = _session_token(request) + user = aml_state.get_session_user(token) if token is not None else None + if user is not None and not allow_mfa_pending and aml_state.is_session_mfa_pending(token or ""): + # Inferred: an MFA-pending session authenticates nothing but the MFA + # challenge routes and logout; same 401 as unauthenticated, cookie kept. raise HTTPException(status_code=401, detail="Authentication required") + if user is None: + headers = None + if session_id: + # Expired/unknown cookie: tell the browser to drop it. + headers = {"Set-Cookie": 'sessionID=""; Max-Age=0; Path=/; HttpOnly; SameSite=lax'} + raise HTTPException(status_code=401, detail="Authentication required", headers=headers) return user +async def require_auth(request: Request, context: AppContext = Depends(get_context)) -> AmlUser: + _ensure_state(context) + return _authenticate(request, allow_mfa_pending=False) + + +async def require_auth_mfa_pending_ok( + request: Request, context: AppContext = Depends(get_context) +) -> AmlUser: + """Auth for MFA challenge routes and logout, which an MFA-pending session may use.""" + _ensure_state(context) + return _authenticate(request, allow_mfa_pending=True) + + +_XML_MEDIA_TYPES = frozenset({"application/xml", "text/xml"}) +_FORM_MEDIA_TYPES = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +_ACCEPTABLE_MEDIA_TYPES = frozenset({"*/*", "application/*", "text/*", "application/json"}) +_XML_FORBIDDEN_MARKUP = re.compile(rb" None: + super().__init__(summary) + self.status_code = status_code + self.summary = summary + + +def _media_type(header_value: str) -> str: + return header_value.split(";", 1)[0].strip().lower() + + +def _wants_xml(request: Request) -> bool: + """True when Accept names an XML type and not JSON (inferred; manual is JSON-only).""" + accepted = {_media_type(part) for part in request.headers.get("accept", "").split(",")} + return bool(accepted & _XML_MEDIA_TYPES) and "application/json" not in accepted + + +def _check_acceptable(request: Request) -> None: + raw = request.headers.get("accept", "").strip() + if not raw: + return + accepted = {_media_type(part) for part in raw.split(",")} + if not accepted & (_ACCEPTABLE_MEDIA_TYPES | _XML_MEDIA_TYPES): + raise _LoginNegotiationError(406, "Unsupported Accept media type") + + +def _parse_login_xml(body: bytes) -> dict[str, Any]: + # defusedxml is not installed; refuse any DTD/entity declaration outright so + # neither external entities (XXE) nor entity expansion can reach the parser. + if _XML_FORBIDDEN_MARKUP.search(body): + raise _LoginNegotiationError(400, "XML DTD/entity declarations are not allowed") + try: + root = ElementTree.fromstring(body) + except ElementTree.ParseError as exc: + raise _LoginNegotiationError(400, "Malformed XML login payload") from exc + return {child.tag: (child.text or "") for child in root} + + +def _negotiated(request: Request, status_code: int, body: dict[str, Any]) -> Response: + if not _wants_xml(request): + return JSONResponse(status_code=status_code, content=body) + root = ElementTree.Element("WSResultCode") + for key, value in body.items(): + if value is not None: + ElementTree.SubElement(root, key).text = str(value) + return Response( + content=ElementTree.tostring(root, encoding="unicode"), + status_code=status_code, + media_type="application/xml", + ) + + async def _parse_login_request(request: Request) -> LoginRequest: - content_type = request.headers.get("content-type", "") + media_type = _media_type(request.headers.get("content-type", "")) payload: dict[str, Any] - if "json" in content_type: + if media_type == "application/json": payload = await request.json() - else: + elif media_type in _XML_MEDIA_TYPES: + payload = _parse_login_xml(await request.body()) + elif media_type in _FORM_MEDIA_TYPES: form = await request.form() payload = dict(form) + else: + raise _LoginNegotiationError(415, "Unsupported or missing Content-Type") # Backwards compatible: accept 'username' as an alias for 'name' if isinstance(payload, dict) and "username" in payload and "name" not in payload: @@ -505,7 +599,11 @@ async def test_ldap( @no_auth async def login(request: Request, context: AppContext = Depends(get_context)) -> Response: _ensure_state(context) - payload = await _parse_login_request(request) + try: + _check_acceptable(request) + payload = await _parse_login_request(request) + except _LoginNegotiationError as exc: + return _ws_error(exc.status_code, exc.summary) user_name = _validate_user_name(payload.name) remote_address = _remote_address(request) or "unknown" @@ -528,12 +626,24 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> _clear_rate_limit(remote_address) if user.role == 2 and not aml_state.get_service_access().get("enabled", True): raise HTTPException(status_code=503, detail="Service access is disabled") - session_record = aml_state.create_session(user) + mfa_pending = aml_state.user_has_enabled_mfa(user.name) + session_record = aml_state.create_session(user, mfa_pending=mfa_pending) aml_state.record_login_activity(user.name, success=True, remote_address=remote_address) body = _ws_result("Login successful").model_dump() # Provide session token in response body for API clients/tests that expect it body["token"] = session_record.token - response = JSONResponse(content=body) + # Inferred: Rev D signals a default password only via the Warning header. With + # OPENBLADE_AML_STRICT_PRECONDITIONS=true the same login answers 412 instead of + # 200; the session is still issued so the client can change the password. + strict = os.environ.get("OPENBLADE_AML_STRICT_PRECONDITIONS", "false").lower() == "true" + # Inferred likewise for MFA: the challenge wire shape is not in Rev D, so a + # pending login is signalled ONLY by the Warning header below. MFA never + # changes the status or body, in strict mode too: there is no documented + # MFA summary text to return. + precondition_failed = strict and user.require_password_change + if precondition_failed: + body.update(code=412, description="Error", summary="Default password must be changed") + response = _negotiated(request, 412 if precondition_failed else 200, body) _is_production = os.environ.get("OPENBLADE_ENV", "development").lower() == "production" response.set_cookie( "sessionID", @@ -545,6 +655,8 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> ) if user.require_password_change: response.headers["Warning"] = "Default Password Supplied" + if mfa_pending: + response.headers.append("Warning", '199 - "MFA Required"') return response @@ -558,12 +670,16 @@ async def auth_login(request: Request, context: AppContext = Depends(get_context @router.delete("/users/login", response_model=WSResultCode) async def logout( request: Request, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> Response: _ensure_state(context) + try: + _check_acceptable(request) + except _LoginNegotiationError as exc: + return _ws_error(exc.status_code, exc.summary) aml_state.clear_session(request.cookies.get("sessionID", "")) - response = JSONResponse(content=_ws_result(f"Logged out {current_user.name}").model_dump()) + response = _negotiated(request, 200, _ws_result(f"Logged out {current_user.name}").model_dump()) response.delete_cookie("sessionID") return response @@ -572,7 +688,7 @@ async def logout( @router.post("/auth/logout", response_model=WSResultCode) async def auth_logout( request: Request, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> Response: return await logout(request, current_user, context) @@ -580,8 +696,9 @@ async def auth_logout( @router.post("/users/login/mfa", response_model=WSResultCode) async def validate_mfa( + request: Request, payload: MFAAuthentication, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> WSResultCode: _ensure_state(context) @@ -591,13 +708,14 @@ async def validate_mfa( code = _validate_mfa_code(payload.authenticationCode) if not _verify_totp(config.secret, code): raise HTTPException(status_code=401, detail="Invalid authentication code") + aml_state.clear_session_mfa_pending(_session_token(request) or "") return _ws_result(f"MFA validated for {payload.type}") @router.get("/users/login/mfa/{type}/key", response_model=MFASharedData) async def get_mfa_key( type: str, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> MFASharedData: _ensure_state(context) diff --git a/openblade/api/routes_archive.py b/openblade/api/routes_archive.py index fef50b7..c3019b7 100644 --- a/openblade/api/routes_archive.py +++ b/openblade/api/routes_archive.py @@ -12,9 +12,10 @@ from openblade.api import aml_state from openblade.bootstrap import AppContext, get_context -from openblade.domain.errors import FileNotFoundError +from openblade.domain.errors import FileNotFoundError, safe_job_error from openblade.jobs.archive import ArchiveRequest as ArchiveJobRequest from openblade.jobs.archive import run_archive_job +from openblade.jobs.inventory import InventoryService from openblade.jobs.scheduler import DriveScheduler from openblade.jobs.shard import ShardMode from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive @@ -140,17 +141,22 @@ async def enqueue_archive( context.ltfs, context.catalog, job.id, + scheduler=DriveScheduler( + num_drives=len(InventoryService(context.library).snapshot().drives), + store=context.lease_store, + job_id=job.id, + ), ) except Exception as exc: _cleanup_failed_archive(context, source_path, request.volume_group) - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", source_path=request.source_path, volume_group=request.volume_group, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) @@ -193,7 +199,11 @@ async def enqueue_sharded_archive( "block_size_mb": request.block_size_mb, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) try: run_sharded_archive( ShardedArchiveRequest( @@ -210,14 +220,14 @@ async def enqueue_sharded_archive( job.id, ) except Exception as exc: - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", source_path=request.source_path, volume_group=request.volume_group, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) diff --git a/openblade/api/routes_jobs.py b/openblade/api/routes_jobs.py index 4b0a217..3e14095 100644 --- a/openblade/api/routes_jobs.py +++ b/openblade/api/routes_jobs.py @@ -2,10 +2,14 @@ from __future__ import annotations +from datetime import datetime + from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from openblade.bootstrap import AppContext, get_context +from openblade.domain.errors import DriveUnreconciledError +from openblade.jobs.reconcile import UnknownDriveError, reconcile_drive router = APIRouter() @@ -45,6 +49,109 @@ async def list_jobs( return [job for job in jobs if job.library_id is None or job.library_id == library_id] +class DriveMismatchResponse(BaseModel): + lease_id: str + job_id: str + drive_id: int + expected_barcode: str + observed_barcode: str | None + + +class StagedInstanceResponse(BaseModel): + instance_id: str + barcode: str + tape_path: str + shard_index: int | None + state: str + + +class PendingReconciliationResponse(BaseModel): + drive_id: int + barcode: str | None + op: str + job_id: str + at: datetime + + +class RecoveryReportResponse(BaseModel): + interrupted_job_ids: list[str] + released_lease_ids: list[str] + mismatches: list[DriveMismatchResponse] + staged_instances: dict[str, list[StagedInstanceResponse]] + stale_pending_job_ids: list[str] + pending_reconciliation: list[PendingReconciliationResponse] + + +class ReconcileDriveResponse(BaseModel): + drive: int + observed_barcode: str | None + job_id: str + at: datetime + + +# Declared before /{job_id} so the literal path wins route matching. +@router.get("/recovery", response_model=RecoveryReportResponse) +async def get_recovery_report( + context: AppContext = Depends(get_context), +) -> RecoveryReportResponse: + """What startup recovery did: interrupted jobs, released leases, drive mismatches, + uncommitted staged instances per recoverable job, and stale pending jobs.""" + report = context.recovery_report + return RecoveryReportResponse( + interrupted_job_ids=list(report.interrupted_job_ids), + released_lease_ids=list(report.released_lease_ids), + mismatches=[ + DriveMismatchResponse( + lease_id=m.lease_id, + job_id=m.job_id, + drive_id=m.drive_id, + expected_barcode=m.expected_barcode, + observed_barcode=m.observed_barcode, + ) + for m in report.mismatches + ], + staged_instances={ + job_id: [ + StagedInstanceResponse( + instance_id=i.instance_id, + barcode=i.barcode, + tape_path=i.tape_path, + shard_index=i.shard_index, + state=i.state, + ) + for i in instances + ] + for job_id, instances in report.staged_instances.items() + }, + stale_pending_job_ids=list(report.stale_pending_job_ids), + pending_reconciliation=[ + PendingReconciliationResponse( + drive_id=p.drive_id, barcode=p.barcode, op=p.op, job_id=p.job_id, at=p.at + ) + for p in report.pending_reconciliation + ], + ) + + +@router.post("/recovery/reconcile/{drive_id}", response_model=ReconcileDriveResponse) +async def post_reconcile_drive( + drive_id: int, context: AppContext = Depends(get_context) +) -> ReconcileDriveResponse: + """Clear a drive's "physical state unknown" mark after checking it is unmounted. + + Bearer-token gated like every native route (api_auth middleware).""" + try: + done = reconcile_drive(context.catalog, context.library, drive_id) + except UnknownDriveError: + raise HTTPException(status_code=404, detail=f"Drive {drive_id} not found") from None + except DriveUnreconciledError as exc: + # Curated message from reconcile_drive (drive id, barcode, mount state). + raise HTTPException(status_code=409, detail=str(exc)) from None + return ReconcileDriveResponse( + drive=done.drive_id, observed_barcode=done.barcode, job_id=done.job_id, at=done.at + ) + + @router.get("/{job_id}", response_model=JobResponse) async def get_job(job_id: str, context: AppContext = Depends(get_context)) -> JobResponse: job = context.catalog.get_job(job_id) diff --git a/openblade/api/routes_restore.py b/openblade/api/routes_restore.py index 47c1ec1..d932a1d 100644 --- a/openblade/api/routes_restore.py +++ b/openblade/api/routes_restore.py @@ -17,9 +17,10 @@ from openblade.catalog.db import get_session from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import OpenBladeError, safe_job_error +from openblade.jobs.inventory import InventoryService from openblade.jobs.restore import RestoreRequest as RestoreJobRequest from openblade.jobs.restore import run_restore_job -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler from openblade.jobs.sharded_restore import ShardedRestoreRequest, run_sharded_restore from openblade.jobs.tree_restore import TreeRestoreRequest, TreeRestoreResult, run_tree_restore @@ -156,7 +157,11 @@ async def enqueue_restore( ) try: if use_sharded_restore: - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) run_sharded_restore( ShardedRestoreRequest(catalog_path=catalog_path, dest_path=Path(request.dest_path)), context.library, @@ -172,16 +177,21 @@ async def enqueue_restore( context.ltfs, context.catalog, job.id, + scheduler=DriveScheduler( + num_drives=len(InventoryService(context.library).snapshot().drives), + store=context.lease_store, + job_id=job.id, + ), ) except Exception as exc: - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", catalog_path=catalog_path, dest_path=request.dest_path, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) @@ -231,7 +241,7 @@ async def restore_tree( "dry_run": request.dry_run, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + num_drives = len(context.library.inventory().drives) tree_request = TreeRestoreRequest( catalog_prefix=request.catalog_prefix, dest_dir=Path(request.dest_dir), @@ -246,12 +256,20 @@ def _run() -> TreeRestoreResult: # target drive 0) and two callers fight over one drive. db_session = get_session() try: + worker_catalog = CatalogRepository(db_session) + # Lease store over the worker's own session: the context's store + # wraps the process-global Session, which is not thread-safe. + scheduler = DriveScheduler( + num_drives=num_drives, + store=CatalogLeaseStore(worker_catalog), + job_id=job.id, + ) with _ARCHIVE_REQUEST_LOCK: return run_tree_restore( tree_request, context.library, context.ltfs, - CatalogRepository(db_session), + worker_catalog, scheduler, job.id, ) diff --git a/openblade/api/routes_upload.py b/openblade/api/routes_upload.py index 541cfef..27bd8e9 100644 --- a/openblade/api/routes_upload.py +++ b/openblade/api/routes_upload.py @@ -53,15 +53,21 @@ def _safe_resolve(base: Path, file_id: str) -> Path: from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.wire import coerce_int +from openblade.nas.capacity import ( + ReservationError, + book_plan_capacity, + plan_with_pool_budget, + server_budget_request, +) from openblade.nas.ingest import ( get_ingest_job, register_archive_plan, run_ingest_job, start_ingest_job, ) -from openblade.nas.planner import ArchivePlanner from openblade.nas.service import NasService from openblade.nas.types import ( + ArchivePlan, ArchivePlanRequest, DatasetStatus, IngestMode, @@ -199,23 +205,16 @@ def _resolve_pool_for_share( return pool -def _available_data_tapes(repo: CatalogRepository) -> list[str]: - barcodes: set[str] = set() - # The physical library holds the writable media (scratch/loaded tapes); the - # catalog only tracks already-archived cartridges, so it can be empty even - # when tapes are available. Source from both. - try: - for slot in get_context().library.inventory().slots: - if slot.barcode is not None: - barcodes.add(str(slot.barcode)) - except Exception: # noqa: BLE001 - library unavailable falls back to the catalog - pass - for cartridge in repo.list_cartridges(): - barcode = cartridge.get("barcode") if isinstance(cartridge, dict) else None - value = str(barcode or "").strip() - if value: - barcodes.add(value) - return sorted(value for value in barcodes if not value.upper().startswith("CLN")) +def _plan_share_push(repo: CatalogRepository, request: ArchivePlanRequest) -> ArchivePlan: + """Plan a share push against the server-derived pool budget (booked at ingest start).""" + request = server_budget_request(repo, request, get_context().library) + plan = plan_with_pool_budget(repo, request, now=datetime.now(timezone.utc)) + if not plan.is_safe_to_enqueue: + raise HTTPException( + status_code=400, + detail=f"Archive plan is not safe to enqueue: {', '.join(plan.enqueue_blockers)}", + ) + return plan def _safe_workspace_path(workspace: Path, relative_path: str) -> Path: @@ -588,7 +587,6 @@ async def push_staged_files_to_share( file_sizes[candidate] = max(int(destination_path.stat().st_size), 0) staged_file_ids.append(file_id) - available_tapes = _available_data_tapes(repo) request_data = { "policy_id": policy.id, "policy_type": policy.policy_type, @@ -597,7 +595,6 @@ async def push_staged_files_to_share( "pool": pool_id, "files": relative_files, "file_sizes": file_sizes, - "available_tapes": available_tapes, "copies": max(int(policy.copies_required), 1), "verify_before_archive": bool(policy.verify_before_archive), "verify_after_archive": bool(policy.verify_after_archive), @@ -608,19 +605,19 @@ async def push_staged_files_to_share( if policy.shard_strategy is not None: request_data["shard_strategy"] = policy.shard_strategy - plan = ArchivePlanner().plan(ArchivePlanRequest.model_validate(request_data)) + plan = _plan_share_push(repo, ArchivePlanRequest.model_validate(request_data)) plan.policy_name = policy.name plan.policy_type = policy.policy_type - if not plan.is_safe_to_enqueue: - raise HTTPException( - status_code=400, - detail=f"Archive plan is not safe to enqueue: {', '.join(plan.enqueue_blockers)}", - ) plan = register_archive_plan(plan) dataset_name = request.dataset_name or ( f"share-push-{pool_id}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}" ) + try: + book_plan_capacity(repo, plan, get_context().library, now=datetime.now(timezone.utc)) + except ReservationError as exc: + # Curated domain message (pool + plan id only); the pool budget is exhausted. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc job = start_ingest_job( plan=plan, dataset_name=dataset_name, diff --git a/openblade/bootstrap.py b/openblade/bootstrap.py index 82c487e..c5b4dec 100644 --- a/openblade/bootstrap.py +++ b/openblade/bootstrap.py @@ -25,7 +25,9 @@ from openblade.jobs.format import FormatService from openblade.jobs.inventory import InventoryService, run_inventory_job from openblade.jobs.queue import JobQueue +from openblade.jobs.recovery import RecoveryReport, recover_after_restart from openblade.jobs.restore import RestoreService +from openblade.jobs.scheduler import CatalogLeaseStore, LeaseStore from openblade.jobs.worker import Worker from openblade.nas.service import NasService from openblade.nas.types import ( @@ -403,6 +405,8 @@ class AppContext: library: LibraryBackend ltfs: LTFSBackend catalog: CatalogRepository + lease_store: LeaseStore + recovery_report: RecoveryReport queue: JobQueue worker: Worker inventory_service: InventoryService @@ -593,6 +597,9 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: else: library, ltfs = _create_real_backends(active_config) catalog = CatalogRepository(get_session()) + # Before anything can claim a drive: fail jobs the previous process left + # running, release their leases, and report (never fix) drive mismatches. + recovery_report = recover_after_restart(catalog, library) _seed_nas_defaults(catalog) if active_config.backend == BackendMode.MOCK: _seed_library_defaults(catalog, active_config) @@ -604,13 +611,16 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: with suppress(Exception): _seed_demo_ltfs(catalog, ltfs) run_inventory_job(library, catalog) - queue = JobQueue() + lease_store = CatalogLeaseStore(catalog) + queue = JobQueue(catalog, lease_store) worker = Worker(queue) return AppContext( config=active_config, library=library, ltfs=ltfs, catalog=catalog, + lease_store=lease_store, + recovery_report=recovery_report, queue=queue, worker=worker, inventory_service=InventoryService(library), diff --git a/openblade/catalog/db.py b/openblade/catalog/db.py index 5976278..05ee776 100644 --- a/openblade/catalog/db.py +++ b/openblade/catalog/db.py @@ -113,6 +113,18 @@ def _migrate_schema(engine: Engine) -> None: library_columns = { column["name"] for column in inspector.get_columns("library_instances") } + if "nas_pools" in inspector.get_table_names(): + pool_columns = {column["name"] for column in inspector.get_columns("nas_pools")} + if "replication_factor" not in pool_columns: + connection.execute( + text("ALTER TABLE nas_pools ADD COLUMN replication_factor INTEGER DEFAULT 1") + ) + if "protection_json" not in pool_columns: + connection.execute(text("ALTER TABLE nas_pools ADD COLUMN protection_json TEXT")) + if "nas_datasets" in inspector.get_table_names(): + dataset_columns = {column["name"] for column in inspector.get_columns("nas_datasets")} + if "protection_json" not in dataset_columns: + connection.execute(text("ALTER TABLE nas_datasets ADD COLUMN protection_json TEXT")) if library_columns and "role" not in library_columns: with suppress(Exception): connection.execute( @@ -287,6 +299,20 @@ def _migrate_schema(engine: Engine) -> None: """ ) ) + # NAS domain tables (added after initial release); create_all(checkfirst) is idempotent. + Base.metadata.create_all( + engine, + tables=[ + Base.metadata.tables[name] + for name in ( + "nas_reservations", + "nas_export_sets", + "nas_vaults", + "nas_vg_replication", + "nas_file_spans", + ) + ], + ) def init_db(db_url: str = "sqlite:///./openblade.db") -> None: diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index cf51f58..b5ec24b 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -18,6 +18,8 @@ ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from openblade.domain.clock import naive_utcnow + class Base(DeclarativeBase): """Base class for catalog ORM models.""" @@ -34,9 +36,9 @@ class LibraryInstance(Base): role: Mapped[str] = mapped_column(String, default="primary") sort_order: Mapped[int] = mapped_column(Integer, default=0) enabled: Mapped[bool] = mapped_column(Boolean, default=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @@ -45,7 +47,7 @@ class VolumeGroup(Base): id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) name: Mapped[str] = mapped_column(String, unique=True, nullable=False, index=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) cartridges: Mapped[list[Cartridge]] = relationship( "Cartridge", back_populates="volume_group", cascade="save-update" @@ -74,7 +76,7 @@ class Cartridge(Base): used_bytes: Mapped[int] = mapped_column(Integer, default=0) state: Mapped[str] = mapped_column(String, default="in_slot") formatted: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) volume_group: Mapped[VolumeGroup | None] = relationship( "VolumeGroup", back_populates="cartridges" @@ -96,7 +98,7 @@ class FileRecord(Base): block_size: Mapped[int | None] = mapped_column(Integer, nullable=True) shard_profile: Mapped[str | None] = mapped_column(String(32), nullable=True) parent_id: Mapped[str | None] = mapped_column(String, nullable=True, index=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) volume_group: Mapped[VolumeGroup] = relationship("VolumeGroup", back_populates="file_records") instances: Mapped[list[FileInstance]] = relationship( @@ -119,7 +121,7 @@ class FileInstance(Base): state: Mapped[str] = mapped_column(String, default="pending") archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) checksum_verified: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) file_record: Mapped[FileRecord] = relationship("FileRecord", back_populates="instances") @@ -132,9 +134,9 @@ class Job(Base): state: Mapped[str] = mapped_column(String, default="pending") metadata_json: Mapped[str] = mapped_column(Text, default="{}") error: Mapped[str | None] = mapped_column(Text, nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @property @@ -180,9 +182,9 @@ class AmlUser(Base): password: Mapped[str] = mapped_column(String(64), nullable=False) role: Mapped[int] = mapped_column(Integer, nullable=False, default=1) require_password_change: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @property @@ -197,8 +199,8 @@ class RbacRole(Base): name: Mapped[str] = mapped_column(String, unique=True, nullable=False, index=True) description: Mapped[str] = mapped_column(Text, default="") permissions: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) - updated_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) class RbacUser(Base): @@ -213,8 +215,8 @@ class RbacUser(Base): is_active: Mapped[bool] = mapped_column(Boolean, default=True) is_admin: Mapped[bool] = mapped_column(Boolean, default=False) api_token_ids: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) - updated_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) last_login_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -227,7 +229,7 @@ class RbacApiToken(Base): token_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) permissions: Mapped[str] = mapped_column(Text, default="[]") expires_at: Mapped[str | None] = mapped_column(Text, nullable=True) - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) last_used_at: Mapped[str | None] = mapped_column(Text, nullable=True) revoked: Mapped[bool] = mapped_column(Boolean, default=False) @@ -244,7 +246,7 @@ class RbacAuditEvent(Base): outcome: Mapped[str] = mapped_column(String, default="") details: Mapped[str] = mapped_column(Text, default="{}") created_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat(), index=True + Text, default=lambda: naive_utcnow().isoformat(), index=True ) ip_address: Mapped[str | None] = mapped_column(String, nullable=True) @@ -256,9 +258,9 @@ class NasStoragePolicy(Base): name: Mapped[str] = mapped_column(String, nullable=False) policy_type: Mapped[str] = mapped_column(String(64), nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @@ -270,7 +272,7 @@ class NasCacheDrive(Base): root_path: Mapped[str] = mapped_column(String, nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") enabled: Mapped[bool] = mapped_column(Boolean, default=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) class NasConfig(Base): @@ -287,7 +289,7 @@ class NasShare(Base): name: Mapped[str] = mapped_column(String, nullable=False) share_type: Mapped[str] = mapped_column(String(32), nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) class NasPool(Base): @@ -305,6 +307,9 @@ class NasPool(Base): cache_target_id: Mapped[str | None] = mapped_column(String, nullable=True) restore_target_path: Mapped[str] = mapped_column(String, default="/openblade/restore") access_mode: Mapped[str] = mapped_column(String, default="read_only") + replication_factor: Mapped[int] = mapped_column(Integer, default=1) + # Serialized ProtectionPolicy.to_dict(); None means derive from replication_factor. + protection_json: Mapped[str | None] = mapped_column(Text, nullable=True) created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -327,6 +332,8 @@ class NasDataset(Base): status: Mapped[str] = mapped_column(String, default="pending") copies_completed: Mapped[int] = mapped_column(Integer, default=0) manifest_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # Serialized ProtectionPolicy.to_dict(); written by catalog rebuild, None if unknown. + protection_json: Mapped[str | None] = mapped_column(Text, nullable=True) created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -372,10 +379,10 @@ class PathMapping(Base): checksum: Mapped[str | None] = mapped_column(String, nullable=True) last_seen_at: Mapped[str | None] = mapped_column(String, nullable=True) created_at: Mapped[str] = mapped_column( - String, nullable=False, default=lambda: datetime.utcnow().isoformat() + "Z" + String, nullable=False, default=lambda: naive_utcnow().isoformat() + "Z" ) updated_at: Mapped[str] = mapped_column( - String, nullable=False, default=lambda: datetime.utcnow().isoformat() + "Z" + String, nullable=False, default=lambda: naive_utcnow().isoformat() + "Z" ) __table_args__ = (UniqueConstraint("logical_path", "pool_id", name="uq_path_pool"),) @@ -395,12 +402,8 @@ class CatalogRebuildRun(Base): datasets_recovered: Mapped[int] = mapped_column(Integer, default=0) path_mappings_recovered: Mapped[int] = mapped_column(Integer, default=0) error_summary: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) - updated_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") completed_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -414,9 +417,7 @@ class ManifestVersion(Base): sha256: Mapped[str] = mapped_column(String(64), nullable=False) file_count: Mapped[int] = mapped_column(Integer, default=0) is_current: Mapped[bool] = mapped_column(Boolean, default=False) - recorded_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) + recorded_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") class NasRestoreJob(Base): @@ -449,3 +450,91 @@ class NasRestoreJob(Base): created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) completed_at: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class DriveLeaseRecord(Base): + """Catalog-persisted drive lease (docs/decisions/2026-10-09-persistent-drive-leases.md). + + Live = ``released_at IS NULL AND expires_at > now``. "One live lease per + drive" is enforced in the repository under ``BEGIN IMMEDIATE``, not by a + partial unique index. Timestamps are naive UTC, like the rest of the catalog. + """ + + __tablename__ = "drive_leases" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + drive_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + job_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + barcode: Mapped[str] = mapped_column(String(8), nullable=False) + physical_drive_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + fencing_token: Mapped[int] = mapped_column(Integer, nullable=False, unique=True) + acquired_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + heartbeat_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + released_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + + +class JobJournalEntry(Base): + """Append-only per-job event journal (lease acquired, shard staged, commit, failure...). + + Created by ``Base.metadata.create_all`` like ``drive_leases`` -- a new table + needs no ``_migrate_schema`` step. ``detail_json`` is a JSON object; a + ``shard_staged`` row carries ``instance_id`` and is how staged instances are + linked back to their job (``file_instances`` has no job column). + """ + + __tablename__ = "job_journal" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + job_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=naive_utcnow) + event: Mapped[str] = mapped_column(String(64), nullable=False) + detail_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + @property + def detail(self) -> dict[str, Any]: + return cast("dict[str, Any]", json.loads(self.detail_json or "{}")) + + +class NasReservation(Base): + __tablename__ = "nas_reservations" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + pool_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + owner: Mapped[str] = mapped_column(String, nullable=False) + # ISO-8601 with offset (SQLite DateTime would drop the tzinfo). + expires_at: Mapped[str] = mapped_column(Text, nullable=False) + + +class NasExportSet(Base): + __tablename__ = "nas_export_sets" + + id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + state: Mapped[str] = mapped_column(String, nullable=False) + + +class NasVault(Base): + __tablename__ = "nas_vaults" + + id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + location: Mapped[str] = mapped_column(String, nullable=False) + + +class NasVgReplication(Base): + __tablename__ = "nas_vg_replication" + + vg_id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + replicas_required: Mapped[int] = mapped_column(Integer, default=1) + replicas_present: Mapped[int] = mapped_column(Integer, default=0) + + +class NasFileSpan(Base): + __tablename__ = "nas_file_spans" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + path: Mapped[str] = mapped_column(Text, nullable=False, index=True) + segments_json: Mapped[str] = mapped_column(Text, default="[]") diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 3126075..4e4008d 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -4,32 +4,44 @@ import json import threading -from collections.abc import Mapping, Sequence +import time +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime, timedelta from pathlib import PurePosixPath from typing import Any, Protocol, cast, runtime_checkable from uuid import uuid4 from sqlalchemy import func, select +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session, selectinload from openblade.catalog.models import ( + Base, Cartridge, CatalogRebuildRun, + DriveLeaseRecord, FileInstance, FileRecord, Job, + JobJournalEntry, LibraryInstance, ManifestVersion, NasCacheDrive, NasConfig, NasDataset, + NasExportSet, NasFileRecord, + NasFileSpan, NasPool, + NasReservation, NasRestoreJob, NasShare, NasStoragePolicy, + NasVault, + NasVgReplication, PathMapping, RbacApiToken, RbacAuditEvent, @@ -39,8 +51,9 @@ TapeOpLog, VolumeGroup, ) +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import FileNotFoundError -from openblade.domain.models import FileInstanceState +from openblade.domain.models import DriveLease, FileInstanceState from openblade.domain.policies import SafetyToken from openblade.nas.types import ( CacheDriveConfig, @@ -112,7 +125,7 @@ def _load_json_value(value: str | None, default: object) -> object: def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + return naive_utcnow().isoformat() @dataclass(frozen=True) @@ -124,6 +137,24 @@ class CatalogBrowseEntry: shard_count: int +@dataclass(frozen=True) +class StagedInstance: + """A sharded-archive instance not yet committed (STAGING or VERIFYING).""" + + instance_id: str + file_record_id: str + barcode: str + tape_path: str + shard_index: int | None + state: str + + +_STAGED_INSTANCE_STATES = frozenset( + {FileInstanceState.STAGING.value, FileInstanceState.VERIFYING.value} +) +_SHARD_STAGED_EVENT = "shard_staged" + + class CatalogRepository: def __init__(self, session: Session) -> None: self.session = session @@ -152,6 +183,8 @@ def _nas_pool_to_dict(self, row: NasPool) -> dict[str, object]: "cache_target_id": row.cache_target_id, "restore_target_path": row.restore_target_path, "access_mode": row.access_mode, + "replication_factor": row.replication_factor, + "protection": _load_json_value(row.protection_json, None), "created_at": row.created_at, "updated_at": row.updated_at, } @@ -648,7 +681,7 @@ def mark_instance_archived(self, instance_id: str, checksum_verified: bool = Tru if instance is None: raise FileNotFoundError(f"File instance {instance_id} not found") instance.state = FileInstanceState.ARCHIVED.value - instance.archived_at = datetime.utcnow() + instance.archived_at = naive_utcnow() instance.checksum_verified = checksum_verified self.session.commit() @@ -660,6 +693,180 @@ def mark_instance_failed(self, instance_id: str, error: str) -> None: instance.state = FileInstanceState.FAILED.value self.session.commit() + # -- Sharded-archive staging + job journal --------------------------------- + # Lifecycle: create_staged_instance (STAGING) -> mark_instances_verifying -> + # mark_instances_archived, the last being the job's ONE commit. Each bulk + # call validates every id before mutating anything, so it is all-or-nothing. + + def journal( + self, job_id: str, event: str, detail: Mapping[str, object] | None = None + ) -> JobJournalEntry: + """Append one event to ``job_id``'s journal and commit.""" + entry = JobJournalEntry( + job_id=job_id, + at=naive_utcnow(), + event=event, + detail_json=json.dumps(dict(detail or {}), sort_keys=True, default=str), + ) + self.session.add(entry) + self.session.commit() + return entry + + # journal_durable: one attempt plus three retries after these backoffs. + _JOURNAL_RETRY_DELAYS = (0.05, 0.1, 0.2) + + def journal_durable( + self, job_id: str, event: str, detail: Mapping[str, object] | None = None + ) -> None: + """Append a journal event on a FRESH session over the same bind. + + For safety evidence written on an error path: ``self.session`` may already + be poisoned (PendingRollbackError) by the failure being recorded, so it is + never touched. Retries ``OperationalError`` (e.g. SQLite "database is + locked") with 50/100/200 ms backoff, then re-raises the last error. + """ + payload = json.dumps(dict(detail or {}), sort_keys=True, default=str) + for delay in (*self._JOURNAL_RETRY_DELAYS, None): + try: + with self._lease_session() as session: + session.add( + JobJournalEntry( + job_id=job_id, at=naive_utcnow(), event=event, detail_json=payload + ) + ) + session.commit() + return + except OperationalError: + if delay is None: + raise + time.sleep(delay) + + def job_journal(self, job_id: str) -> list[JobJournalEntry]: + """Every journal entry for ``job_id``, oldest first (ties broken by id).""" + stmt = ( + select(JobJournalEntry) + .where(JobJournalEntry.job_id == job_id) + .order_by(JobJournalEntry.at, JobJournalEntry.id) + ) + return list(self.session.scalars(stmt)) + + def create_staged_instance( + self, + job_id: str, + file_record_id: str, + barcode: str, + tape_path: str, + *, + state: FileInstanceState = FileInstanceState.STAGING, + ) -> FileInstance: + """Create an uncommitted shard instance and its ``shard_staged`` journal row. + + Both rows land in one transaction; the journal row is the job linkage + that ``list_staged_instances`` reads. + """ + if state.value not in _STAGED_INSTANCE_STATES: + raise ValueError(f"staged instance state must be staging/verifying, not {state.value}") + record = self.session.get(FileRecord, file_record_id) + if record is None: + raise FileNotFoundError(f"File record {file_record_id} not found") + instance = FileInstance( + file_record_id=file_record_id, + barcode=barcode, + tape_path=str(PurePosixPath(tape_path)), + state=state.value, + ) + self.session.add(instance) + self.session.flush() + self.session.add( + JobJournalEntry( + job_id=job_id, + at=naive_utcnow(), + event=_SHARD_STAGED_EVENT, + detail_json=json.dumps( + { + "instance_id": instance.id, + "barcode": barcode, + "tape_path": instance.tape_path, + "shard_index": record.shard_index, + }, + sort_keys=True, + ), + ) + ) + self.session.commit() + self.session.refresh(instance) + return instance + + def list_staged_instances(self, job_id: str) -> list[StagedInstance]: + """Instances staged by ``job_id`` that are still STAGING or VERIFYING.""" + instance_ids = [ + str(entry.detail.get("instance_id")) + for entry in self.job_journal(job_id) + if entry.event == _SHARD_STAGED_EVENT + ] + if not instance_ids: + return [] + stmt = ( + select(FileInstance, FileRecord.shard_index) + .join(FileRecord, FileInstance.file_record_id == FileRecord.id) + .where( + FileInstance.id.in_(instance_ids), + FileInstance.state.in_(_STAGED_INSTANCE_STATES), + ) + ) + staged = [ + StagedInstance( + instance_id=instance.id, + file_record_id=instance.file_record_id, + barcode=instance.barcode, + tape_path=instance.tape_path, + shard_index=shard_index, + state=instance.state, + ) + for instance, shard_index in self.session.execute(stmt) + ] + staged.sort(key=lambda s: (s.shard_index is None, s.shard_index or 0, s.barcode)) + return staged + + def _load_instances_in( + self, instance_ids: Sequence[str], allowed_states: frozenset[str] + ) -> list[FileInstance]: + instances: list[FileInstance] = [] + for instance_id in dict.fromkeys(instance_ids): + instance = self.session.get(FileInstance, instance_id) + if instance is None: + raise FileNotFoundError(f"File instance {instance_id} not found") + if instance.state not in allowed_states: + raise ValueError( + f"File instance {instance_id} is {instance.state}, " + f"expected one of {sorted(allowed_states)}" + ) + instances.append(instance) + return instances + + def mark_instances_verifying(self, instance_ids: Sequence[str]) -> None: + """Move STAGING (or already VERIFYING) instances to VERIFYING in one commit.""" + instances = self._load_instances_in(instance_ids, _STAGED_INSTANCE_STATES) + for instance in instances: + instance.state = FileInstanceState.VERIFYING.value + self.session.commit() + + def mark_instances_archived( + self, instance_ids: Sequence[str], *, checksum_verified: bool = True + ) -> None: + """The job's single commit: every VERIFYING instance becomes ARCHIVED together. + + Raises (changing nothing) if any id is missing or not VERIFYING. + """ + verifying = frozenset({FileInstanceState.VERIFYING.value}) + instances = self._load_instances_in(instance_ids, verifying) + now = naive_utcnow() + for instance in instances: + instance.state = FileInstanceState.ARCHIVED.value + instance.archived_at = now + instance.checksum_verified = checksum_verified + self.session.commit() + def create_job(self, job_type: str, metadata: dict[str, object]) -> Job: job = Job(job_type=job_type, state="pending", metadata_json=json.dumps(metadata)) self.session.add(job) @@ -673,7 +880,7 @@ def update_job_state(self, job_id: str, state: str, error: str | None = None) -> raise FileNotFoundError(f"Job {job_id} not found") job.state = state job.error = error - job.updated_at = datetime.utcnow() + job.updated_at = naive_utcnow() self.session.commit() def get_job(self, job_id: str) -> Job | None: @@ -754,6 +961,8 @@ def delete_file_record_if_unarchived(self, path: str) -> None: return if any( instance.state in {FileInstanceState.ARCHIVED.value, FileInstanceState.VERIFIED.value} + # Staged shards must stay discoverable for resume / reconcile. + or instance.state in _STAGED_INSTANCE_STATES for instance in record.instances ): return @@ -778,7 +987,7 @@ def upsert_nas_policy(self, policy_id: str, data: dict[str, object]) -> NasStora policy.name = parsed.name policy.policy_type = parsed.policy_type policy.config_json = json.dumps(payload) - policy.updated_at = datetime.utcnow() + policy.updated_at = naive_utcnow() self.session.commit() self.session.refresh(policy) return policy @@ -905,6 +1114,12 @@ def upsert_nas_pool(self, config: dict[str, object]) -> dict[str, object]: row.cache_target_id = parsed.cache_target_id row.restore_target_path = parsed.restore_target_path row.access_mode = parsed.access_mode + row.replication_factor = parsed.replication_factor + row.protection_json = ( + json.dumps(parsed.protection.to_dict(), sort_keys=True) + if parsed.protection is not None + else None + ) row.created_at = row.created_at or parsed.created_at or _utcnow_iso() row.updated_at = parsed.updated_at or _utcnow_iso() self.session.commit() @@ -1125,6 +1340,206 @@ def delete_nas_restore_job(self, job_id: str) -> bool: self.session.commit() return True + # --- NAS domain persistence --- + + _SCRATCH_THRESHOLDS_KEY = "scratch_thresholds" + + def _delete_nas_row(self, model: type[Base], key: str) -> bool: + row = self.session.get(model, key) + if row is None: + return False + self.session.delete(row) + self.session.commit() + return True + + @staticmethod + def _nas_reservation_to_dict(row: NasReservation) -> dict[str, object]: + return { + "id": row.id, + "pool_id": row.pool_id, + "bytes": row.bytes, + "owner": row.owner, + "expires_at": datetime.fromisoformat(row.expires_at), + } + + def list_nas_reservations(self, pool_id: str | None = None) -> list[dict[str, object]]: + stmt = select(NasReservation).order_by(NasReservation.expires_at) + if pool_id is not None: + stmt = stmt.where(NasReservation.pool_id == pool_id) + rows = self.session.execute(stmt).scalars().all() + return [self._nas_reservation_to_dict(row) for row in rows] + + def reservations_for_pool(self, pool_id: str, *, now: datetime) -> list[dict[str, object]]: + """Unexpired reservations for ``pool_id`` (a reservation expiring at ``now`` is expired).""" + return [ + item + for item in self.list_nas_reservations(pool_id) + if cast(datetime, item["expires_at"]) > now + ] + + def upsert_nas_reservation(self, data: dict[str, object]) -> dict[str, object]: + expires_at = data["expires_at"] + if not isinstance(expires_at, datetime): + expires_at = datetime.fromisoformat(str(expires_at)) + reservation_id = str(data.get("id") or uuid.uuid4()) + row = self.session.get(NasReservation, reservation_id) + if row is None: + row = NasReservation(id=reservation_id) + self.session.add(row) + row.pool_id = str(data["pool_id"]) + row.bytes = int(cast(int, data["bytes"])) + row.owner = str(data["owner"]) + row.expires_at = expires_at.isoformat() + self.session.commit() + return self._nas_reservation_to_dict(row) + + def reserve_nas_capacity( + self, + pool_id: str, + nbytes: int, + owner: str, + expires_at: datetime, + *, + capacity_bytes: int, + now: datetime, + ) -> bool: + """Atomically insert a reservation only if it fits under ``capacity_bytes``. + + Check-and-insert run inside one ``BEGIN IMMEDIATE`` on a dedicated + connection, so two processes racing for the last bytes cannot both pass + the check (reviewer S6). Returns whether the reservation was inserted. + """ + with self._lock, self._lease_session() as session: + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + rows = session.scalars(select(NasReservation).where(NasReservation.pool_id == pool_id)) + reserved = sum( + int(row.bytes) + for row in rows + if cast(datetime, self._nas_reservation_to_dict(row)["expires_at"]) > now + ) + if nbytes > capacity_bytes - reserved: + session.rollback() + return False + session.add( + NasReservation( + id=str(uuid.uuid4()), + pool_id=pool_id, + bytes=nbytes, + owner=owner, + expires_at=expires_at.isoformat(), + ) + ) + session.commit() + return True + + def delete_nas_reservation(self, reservation_id: str) -> bool: + return self._delete_nas_row(NasReservation, reservation_id) + + def delete_nas_reservations_for_owner(self, owner: str) -> int: + """Delete every reservation booked by ``owner`` (a plan id); return how many.""" + rows = list( + self.session.execute( + select(NasReservation).where(NasReservation.owner == owner) + ).scalars() + ) + for row in rows: + self.session.delete(row) + if rows: + self.session.commit() + return len(rows) + + def list_nas_export_sets(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasExportSet).order_by(NasExportSet.id)).scalars() + return [ + {"id": r.id, "barcodes": _load_json_value(r.barcodes_json, []), "state": r.state} + for r in rows + ] + + def upsert_nas_export_set(self, data: dict[str, object]) -> dict[str, object]: + row = self.session.get(NasExportSet, str(data["id"])) or NasExportSet(id=str(data["id"])) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.state = str(data["state"]) + self.session.commit() + return {"id": row.id, "barcodes": json.loads(row.barcodes_json), "state": row.state} + + def delete_nas_export_set(self, export_set_id: str) -> bool: + return self._delete_nas_row(NasExportSet, export_set_id) + + def list_nas_vaults(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasVault).order_by(NasVault.id)).scalars() + return [ + {"id": r.id, "barcodes": _load_json_value(r.barcodes_json, []), "location": r.location} + for r in rows + ] + + def upsert_nas_vault(self, data: dict[str, object]) -> dict[str, object]: + row = self.session.get(NasVault, str(data["id"])) or NasVault(id=str(data["id"])) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.location = str(data["location"]) + self.session.commit() + return {"id": row.id, "barcodes": json.loads(row.barcodes_json), "location": row.location} + + def delete_nas_vault(self, vault_id: str) -> bool: + return self._delete_nas_row(NasVault, vault_id) + + @staticmethod + def _nas_vg_replication_to_dict(row: NasVgReplication) -> dict[str, object]: + return { + "vg_id": row.vg_id, + "barcodes": _load_json_value(row.barcodes_json, []), + "replicas_required": row.replicas_required, + "replicas_present": row.replicas_present, + } + + def list_nas_vg_replication(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasVgReplication).order_by(NasVgReplication.vg_id)) + return [self._nas_vg_replication_to_dict(row) for row in rows.scalars()] + + def upsert_nas_vg_replication(self, data: dict[str, object]) -> dict[str, object]: + vg_id = str(data["vg_id"]) + row = self.session.get(NasVgReplication, vg_id) or NasVgReplication(vg_id=vg_id) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.replicas_required = int(cast(int, data.get("replicas_required", 1))) + row.replicas_present = int(cast(int, data.get("replicas_present", 0))) + self.session.commit() + return self._nas_vg_replication_to_dict(row) + + def delete_nas_vg_replication(self, vg_id: str) -> bool: + return self._delete_nas_row(NasVgReplication, vg_id) + + def list_nas_file_spans(self, path: str | None = None) -> list[dict[str, object]]: + stmt = select(NasFileSpan).order_by(NasFileSpan.path, NasFileSpan.id) + if path is not None: + stmt = stmt.where(NasFileSpan.path == path) + return [ + {"id": r.id, "path": r.path, "segments": _load_json_value(r.segments_json, [])} + for r in self.session.execute(stmt).scalars() + ] + + def upsert_nas_file_span(self, data: dict[str, object]) -> dict[str, object]: + span_id = str(data.get("id") or uuid.uuid4()) + row = self.session.get(NasFileSpan, span_id) or NasFileSpan(id=span_id) + self.session.add(row) + row.path = str(data["path"]) + row.segments_json = json.dumps(list(cast(list[object], data.get("segments", [])))) + self.session.commit() + return {"id": row.id, "path": row.path, "segments": json.loads(row.segments_json)} + + def delete_nas_file_span(self, span_id: str) -> bool: + return self._delete_nas_row(NasFileSpan, span_id) + + def get_scratch_thresholds(self) -> dict[str, object] | None: + return self.get_nas_config(self._SCRATCH_THRESHOLDS_KEY) + + def set_scratch_thresholds(self, scratch_min: int, scratch_warn: int) -> dict[str, object]: + return self.set_nas_config( + self._SCRATCH_THRESHOLDS_KEY, + {"scratch_min": scratch_min, "scratch_warn": scratch_warn}, + ) + def create_rebuild_run(self, run: dict[str, object]) -> dict[str, object]: parsed = CatalogRebuildRunRecord.model_validate(run) row = self.session.get(CatalogRebuildRun, parsed.id) @@ -1771,3 +2186,213 @@ def delete_safety_token(self, token_value: str) -> None: if row is not None: self.session.delete(row) self.session.commit() + + # ── Drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md) ── + # + # Lease operations run on their OWN short-lived Session over the catalog's + # engine, never on the shared `self.session`: a lease poll from a worker + # thread must not be able to commit or roll back another thread's pending + # rows (review finding, 2026-10-09). On a file-backed catalog this is a + # separate connection, so `BEGIN IMMEDIATE` is a real cross-process lock. + + @contextmanager + def _lease_session(self) -> Iterator[Session]: + with Session(bind=self.session.get_bind(), expire_on_commit=False) as session: + yield session + + def acquire_drive_leases( + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), + ) -> list[DriveLease] | None: + """All-or-nothing: one live lease per barcode, or None if not enough free drives.""" + with self._lock, self._lease_session() as session: + # Verified against installed SQLAlchemy 2.1.4 / pysqlite: an explicit + # BEGIN IMMEDIATE on the session's connection is honoured and + # session.commit() commits it, so check-then-insert is atomic. + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + now = _lease_now() + busy = set( + session.scalars( + select(DriveLeaseRecord.drive_id).where( + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ) + ) + free = [d for d in range(num_drives) if d not in busy and d not in exclude] + if len(free) < len(barcodes): + session.rollback() + return None + token = int( + session.scalar(select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0))) + or 0 + ) + rows: list[DriveLeaseRecord] = [] + for drive_id, barcode in zip(free, barcodes, strict=False): + token += 1 + row = DriveLeaseRecord( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + fencing_token=token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + session.add(row) + rows.append(row) + session.commit() + return [_lease_from_row(row) for row in rows] + + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + """Lease one specific drive id (negative ids allowed), or None if it is held.""" + with self._lock, self._lease_session() as session: + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + now = _lease_now() + held = session.scalar( + select(func.count()) + .select_from(DriveLeaseRecord) + .where( + DriveLeaseRecord.drive_id == drive_id, + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ) + if held: + session.rollback() + return None + token = int( + session.scalar(select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0))) + or 0 + ) + row = DriveLeaseRecord( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + fencing_token=token + 1, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + session.add(row) + session.commit() + return _lease_from_row(row) + + def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None: + with self._lock, self._lease_session() as session: + now = _lease_now() + for row in session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.id.in_(lease_ids), + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ): + row.heartbeat_at = now + row.expires_at = now + ttl + session.commit() + + def set_lease_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + """Record where the cartridge actually is, so recovery reconciles the right drive.""" + with self._lock, self._lease_session() as session: + row = session.get(DriveLeaseRecord, lease_id) + if row is not None: + row.physical_drive_id = physical_drive_id + session.commit() + + def release_leases(self, lease_ids: list[str]) -> None: + with self._lock, self._lease_session() as session: + now = _lease_now() + for row in session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.id.in_(lease_ids), DriveLeaseRecord.released_at.is_(None) + ) + ): + row.released_at = now + session.commit() + + def lease_is_live(self, lease_id: str, fencing_token: int) -> bool: + with self._lock, self._lease_session() as session: + row = session.get(DriveLeaseRecord, lease_id) + return ( + row is not None + and row.fencing_token == fencing_token + and row.released_at is None + and row.expires_at > _lease_now() + ) + + def live_leases(self) -> list[DriveLease]: + with self._lock, self._lease_session() as session: + now = _lease_now() + rows = session.scalars( + select(DriveLeaseRecord) + .where(DriveLeaseRecord.released_at.is_(None), DriveLeaseRecord.expires_at > now) + .order_by(DriveLeaseRecord.fencing_token) + ) + return [_lease_from_row(row) for row in rows] + + def expired_leases(self) -> list[DriveLease]: + """Unreleased leases whose owner stopped heartbeating: the only cross-process + evidence that a job is dead rather than running elsewhere.""" + with self._lock, self._lease_session() as session: + now = _lease_now() + rows = session.scalars( + select(DriveLeaseRecord) + .where(DriveLeaseRecord.released_at.is_(None), DriveLeaseRecord.expires_at <= now) + .order_by(DriveLeaseRecord.fencing_token) + ) + return [_lease_from_row(row) for row in rows] + + def release_leases_for_job(self, job_id: str) -> int: + with self._lock, self._lease_session() as session: + now = _lease_now() + rows = list( + session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.job_id == job_id, DriveLeaseRecord.released_at.is_(None) + ) + ) + ) + for row in rows: + row.released_at = now + session.commit() + return len(rows) + + # -- Drive reconciliation (cross-job journal query) ------------------------ + + def journal_events(self, event: str, *, since: datetime | None = None) -> list[JobJournalEntry]: + """Every ``event`` journal entry across all jobs, oldest first (ties by id).""" + stmt = select(JobJournalEntry).where(JobJournalEntry.event == event) + if since is not None: + stmt = stmt.where(JobJournalEntry.at >= since) + stmt = stmt.order_by(JobJournalEntry.at, JobJournalEntry.id) + return list(self.session.scalars(stmt)) + + +def _lease_now() -> datetime: + """Naive UTC, matching how the catalog's DateTime columns store values.""" + return datetime.now(UTC).replace(tzinfo=None) + + +def _lease_from_row(row: DriveLeaseRecord) -> DriveLease: + return DriveLease( + id=row.id, + drive_id=row.drive_id, + job_id=row.job_id, + barcode=row.barcode, + physical_drive_id=row.physical_drive_id, + fencing_token=row.fencing_token, + acquired_at=row.acquired_at, + heartbeat_at=row.heartbeat_at, + expires_at=row.expires_at, + released_at=row.released_at, + ) diff --git a/openblade/cli/fuse_and_health.py b/openblade/cli/fuse_and_health.py index 463d05f..da23a01 100644 --- a/openblade/cli/fuse_and_health.py +++ b/openblade/cli/fuse_and_health.py @@ -7,22 +7,26 @@ """ import logging -from collections.abc import Callable from pathlib import Path +from typing import Any import typer from rich.console import Console from rich.table import Table +from sqlalchemy.orm import Session +from openblade.catalog.repository import CatalogRepository from openblade.config import load_config from openblade.domain.errors import RealHardwareDisabledError from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import Hydrator, JobRestoreEngine from openblade.fuse.mount import FuseUnavailableError, mount_catalog from openblade.hardware.discovery import discover_library from openblade.hardware.runner import SafeRunner from openblade.hardware.safety import require_real_hardware from openblade.hardware.sg import sg_inq from openblade.hardware.tapealert import TapeAlertSeverity, read_tape_alerts +from openblade.jobs.restore import RestoreService console = Console(highlight=False) @@ -36,6 +40,39 @@ } +def build_data_plane(context: Any, filesystem: CatalogFilesystem) -> Hydrator: + """Production FUSE data plane: batched restores into the verified cache. + + Worker threads build their own catalog + RestoreService over a private + Session (``JobRestoreEngine``); the CLI's session stays on this thread. + """ + config = context.config + bind = context.catalog.session.get_bind() + if config.fuse_cache_max_bytes is not None: + filesystem.cache.max_bytes = config.fuse_cache_max_bytes + + def _catalog() -> CatalogRepository: + return CatalogRepository(Session(bind=bind, expire_on_commit=False)) + + def _service(catalog: CatalogRepository) -> RestoreService: + return RestoreService(context.library, context.ltfs, catalog, context.queue) + + engine = JobRestoreEngine( + None, + context.catalog, + Path(config.restore_dir) / "fuse-hydrate", + resume_timeout_s=config.fuse_hydrate_timeout_s, + catalog_factory=_catalog, + restore_service_factory=_service, + ) + return Hydrator( + context.catalog, + filesystem.cache, + engine, + batch_window_s=config.fuse_batch_window_ms / 1000.0, + ) + + @fuse_app.command("mount") def fuse_mount( mountpoint: str = typer.Argument(..., help="Existing empty directory to mount on"), @@ -67,19 +104,7 @@ def fuse_mount( context = _get_context() filesystem = CatalogFilesystem(context.catalog, cache_dir=context.config.cache_dir) - def _hydrate_through_restore(catalog_path: str) -> bytes: - """Restore through the existing restore service, then cache the bytes.""" - record = context.catalog.get_file_record(catalog_path) - if record is None: - raise FileNotFoundError(catalog_path) - staging = Path(context.config.restore_dir) / "fuse-hydrate" / record.id - staging.parent.mkdir(parents=True, exist_ok=True) - context.restore_service.enqueue(catalog_path, staging) - data = staging.read_bytes() - filesystem.cache.store(record.checksum_sha256, data) - return data - - hydrator: Callable[[str], bytes] | None = _hydrate_through_restore if hydrate else None + data_plane = build_data_plane(context, filesystem) if hydrate else None console.print( f"[green]Mounting[/green] catalog at {mountpoint} " @@ -89,7 +114,7 @@ def _hydrate_through_restore(catalog_path: str) -> bytes: mount_catalog( filesystem, mountpoint, - hydrator=hydrator, + data_plane=data_plane, allow_other=allow_other, ) except (FuseUnavailableError, NotADirectoryError) as exc: diff --git a/openblade/cli/main.py b/openblade/cli/main.py index 4173d28..2d3ab5d 100644 --- a/openblade/cli/main.py +++ b/openblade/cli/main.py @@ -21,6 +21,7 @@ from openblade.cli.assist import assist as assist_command from openblade.cli.fuse_and_health import register as register_fuse_and_health from openblade.config import OpenBladeConfig, load_config +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import ( CartridgeNotFoundError, DriveCorrelationError, @@ -621,13 +622,17 @@ def restore_file( # Checked BEFORE the job row is created, so a typo does not leave an # orphaned `queued` restore job behind. raise typer.BadParameter(f"{catalog_path} is not in the catalog") - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) treat_as_dir = into_dir or dest.is_dir() dest_path = dest / PurePosixPath(catalog_path).name if treat_as_dir else dest dest_path.parent.mkdir(parents=True, exist_ok=True) job = context.catalog.create_job( "restore", {"catalog_path": catalog_path, "dest_path": str(dest_path)} ) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) sharded = bool(context.catalog.list_shard_records(record.id)) or (record.shard_count or 1) > 1 err_console.print( f"Restoring {catalog_path} -> {dest_path} ({'sharded' if sharded else 'single-instance'})" @@ -659,6 +664,7 @@ def restore_file( context.ltfs, context.catalog, job.id, + scheduler=scheduler, ) payload = { "jobId": job.id, @@ -695,11 +701,15 @@ def restore_tree( to stdout, and any failure exits non-zero with the curated error. """ context = _get_context() - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) job = context.catalog.create_job( "restore", {"catalog_prefix": catalog_prefix, "dest_dir": str(dest), "bulk": True}, ) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) def _progress(tick: TreeRestoreProgress) -> None: marker = "[red]FAIL[/red]" if tick.failed else "ok" @@ -835,7 +845,11 @@ def archive_sharded( "block_size_mb": block_size_mb, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) err_console.print( f"Sharded archive {source} -> {volume_group} " f"mode={shard_mode.value} lanes={','.join(lane_barcodes)}" @@ -1057,7 +1071,7 @@ def catalog_backup( """ context = _get_context() dest.mkdir(parents=True, exist_ok=True) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + timestamp = naive_utcnow().strftime("%Y%m%dT%H%M%SZ") gz_path = dest / f"openblade-{timestamp}.db.gz" workdir = Path(tempfile.mkdtemp(prefix="openblade-backup-")) try: diff --git a/openblade/config.py b/openblade/config.py index 67f0a39..bf135dd 100644 --- a/openblade/config.py +++ b/openblade/config.py @@ -39,6 +39,9 @@ class OpenBladeConfig: staging_dir: str = str(_DEFAULT_HOME / "staging") restore_dir: str = str(_DEFAULT_HOME / "restore") fuse_mount_point: str = str(_DEFAULT_HOME / "mount") + fuse_hydrate_timeout_s: float = 300.0 + fuse_batch_window_ms: int = 250 + fuse_cache_max_bytes: int | None = None # None = unbounded ltfs_mount_root: str = str(_DEFAULT_HOME / "ltfs") job_timeout_seconds: int = 3600 changer_timeout_seconds: int = 60 @@ -184,6 +187,17 @@ def load_config() -> OpenBladeConfig: staging_dir=os.environ.get("OPENBLADE_STAGING_DIR", str(_DEFAULT_HOME / "staging")), restore_dir=os.environ.get("OPENBLADE_RESTORE_DIR", str(_DEFAULT_HOME / "restore")), ltfs_mount_root=os.environ.get("OPENBLADE_LTFS_MOUNT_ROOT", str(_DEFAULT_HOME / "ltfs")), + fuse_hydrate_timeout_s=float( + os.environ.get("OPENBLADE_FUSE_HYDRATE_TIMEOUT", "").strip() or 300.0 + ), + fuse_batch_window_ms=int( + os.environ.get("OPENBLADE_FUSE_BATCH_WINDOW_MS", "").strip() or 250 + ), + fuse_cache_max_bytes=( + int(os.environ["OPENBLADE_FUSE_CACHE_MAX_BYTES"]) + if os.environ.get("OPENBLADE_FUSE_CACHE_MAX_BYTES", "").strip() + else None + ), hardware_dry_run=os.environ.get("OPENBLADE_HARDWARE_DRY_RUN", "false").lower() == "true", changer_device=os.environ.get("OPENBLADE_CHANGER_DEVICE") or None, drive_devices=drive_devices, diff --git a/openblade/deploy.py b/openblade/deploy.py index 14cdce7..3632dbb 100644 --- a/openblade/deploy.py +++ b/openblade/deploy.py @@ -11,6 +11,7 @@ from __future__ import annotations +import re from collections.abc import Callable from dataclasses import asdict, dataclass, field from enum import Enum @@ -20,6 +21,7 @@ class Stage(str, Enum): PRECHECK = "precheck" DEPLOY = "deploy" POSTCHECK = "postcheck" + ROLLBACK = "rollback" @dataclass @@ -74,3 +76,97 @@ def run_deploy_pipeline( report.results.append(post) report.promoted = post.ok return report + + +# --- Read-only appliance postcheck ------------------------------------------- + +# A probe takes (method, path) and returns an HTTP status code (same contract as +# openblade.topology). These are the appliance checks run after a deploy; every +# one is a GET so a postcheck can never move a cartridge or start a job. +READ_ONLY_APPLIANCE_PROBES: tuple[tuple[str, str], ...] = ( + ("GET", "/inventory/"), + ("GET", "/jobs/recovery"), +) +_READ_ONLY_METHODS = frozenset({"GET", "HEAD"}) + + +class ReadOnlyProbe: + """Wraps a probe so that only GET/HEAD requests are ever sent. + + A non-read-only request is never forwarded: it is recorded in ``skipped`` and + answered with 405 (which topology treats as "present, not verified"), so the + caller can report exactly which checks were not exercised. + """ + + def __init__(self, probe: Callable[[str, str], int]) -> None: + self._probe = probe + self.skipped: list[str] = [] + + def __call__(self, method: str, path: str) -> int: + if method.upper() not in _READ_ONLY_METHODS: + self.skipped.append(f"{method.upper()} {path}") + return 405 + return self._probe(method.upper(), path) + + +def run_read_only_appliance_checks(probe: Callable[[str, str], int]) -> StageResult: + """GET each appliance endpoint; any non-2xx status is a blocking finding.""" + guarded = probe if isinstance(probe, ReadOnlyProbe) else ReadOnlyProbe(probe) + failures = [ + f"{method} {path} -> {status}" + for method, path in READ_ONLY_APPLIANCE_PROBES + if not 200 <= (status := guarded(method, path)) < 300 + ] + ok = not failures + return StageResult( + Stage.POSTCHECK, + ok, + "read-only appliance checks OK" if ok else f"appliance checks failed: {failures}", + findings=failures, + ) + + +# --- Rollback ----------------------------------------------------------------- + +ROLLBACK_NOTICE = ( + "!!! ROLLBACK REDEPLOYS APPLICATION CODE ONLY !!!\n" + "Physical tape state is NEVER rolled back: cartridges moved, written or loaded\n" + "by the rolled-back version stay where they are. Review reconciliation at\n" + "GET /jobs/recovery before resuming operations." +) + +_DIGEST_RE = re.compile(r"^(?:(?P[a-z0-9][a-z0-9._/:-]*)@)?(?Psha256:[0-9a-f]{64})$") + + +class InvalidDigestError(ValueError): + """The rollback target is not an immutable ``sha256:`` digest reference.""" + + +def parse_image_digest(ref: str, default_image: str) -> str: + """Return ``image@sha256:...`` for a bare digest or full digest reference. + + Tags are refused: a rollback must name the exact bytes that ran before. + """ + m = _DIGEST_RE.match(ref.strip()) + if m is None: + raise InvalidDigestError( + "rollback target must be sha256:<64 hex> or @sha256:<64 hex>" + ) + return f"{m.group('image') or default_image}@{m.group('digest')}" + + +def run_rollback( + *, + image_ref: str, + redeploy: Callable[[str], StageResult], + notify: Callable[[str], None], +) -> StageResult: + """Redeploy ``image_ref`` (application code only), announcing the tape caveat. + + The notice is emitted before the redeploy and again after it, whatever the + outcome, so it cannot be missed in a long log. + """ + notify(ROLLBACK_NOTICE) + result = redeploy(image_ref) + notify(ROLLBACK_NOTICE) + return StageResult(Stage.ROLLBACK, result.ok, result.detail, findings=list(result.findings)) diff --git a/openblade/domain/clock.py b/openblade/domain/clock.py new file mode 100644 index 0000000..0796272 --- /dev/null +++ b/openblade/domain/clock.py @@ -0,0 +1,14 @@ +"""Naive-UTC timestamps for catalog rows. + +The catalog stores naive datetimes that mean UTC. ``datetime.utcnow()`` is +deprecated since Python 3.12; this keeps the same value without the warning. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def naive_utcnow() -> datetime: + """Current UTC time as a naive datetime (catalog convention).""" + return datetime.now(UTC).replace(tzinfo=None) diff --git a/openblade/domain/errors.py b/openblade/domain/errors.py index 65b4ed8..cb42aa3 100644 --- a/openblade/domain/errors.py +++ b/openblade/domain/errors.py @@ -46,10 +46,26 @@ class TapeMountedError(SafetyViolationError): """Cannot unload a mounted tape.""" +class DriveLeasedError(SafetyViolationError): + """Another job holds a live lease on this drive; it may have LTFS mounted.""" + + +class DriveUnreconciledError(SafetyViolationError): + """A drive's physical state is unknown after a failed unmount/unload; reconcile first.""" + + +class JournalWriteError(SafetyViolationError): + """A drive's physical_state_unknown could not be journaled; its lease was kept.""" + + class DriveBusyError(OpenBladeError): """Drive is busy.""" +class StaleLeaseError(OpenBladeError): + """A drive lease expired, was released, or was superseded (fencing token mismatch).""" + + class ChangerBusyError(OpenBladeError): """Changer is busy with another operation.""" diff --git a/openblade/domain/models.py b/openblade/domain/models.py index 9220239..7a4042b 100644 --- a/openblade/domain/models.py +++ b/openblade/domain/models.py @@ -65,6 +65,11 @@ class MountMode(str, Enum): class FileInstanceState(str, Enum): PENDING = "pending" + # Sharded-archive lifecycle: written but not yet verified (STAGING), read + # back and being verified (VERIFYING). Neither is restorable or "archived"; + # only the single commit at the end of a job moves them to ARCHIVED. + STAGING = "staging" + VERIFYING = "verifying" ARCHIVED = "archived" VERIFIED = "verified" FAILED = "failed" @@ -174,3 +179,19 @@ class OperationResult: success: bool message: str details: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class DriveLease: + """A catalog-backed reservation of one scheduler drive slot for one job.""" + + id: str + drive_id: int + job_id: str + barcode: str + physical_drive_id: int | None + fencing_token: int + acquired_at: datetime + heartbeat_at: datetime + expires_at: datetime + released_at: datetime | None = None diff --git a/openblade/domain/protection.py b/openblade/domain/protection.py new file mode 100644 index 0000000..e68e091 --- /dev/null +++ b/openblade/domain/protection.py @@ -0,0 +1,205 @@ +"""Protection axes for archived data (roadmap item 10). + +Placement, striping and protection are independent axes. Spreading data across +tapes (striping/sharding) is a *performance* choice; it never makes data more +durable on its own. Only redundant copies make a policy "protected". + +Pure value objects: no IO, no library access. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any + + +class UnsupportedProtectionError(ValueError): + """A protection scheme that is modelled but cannot be planned (e.g. erasure coding).""" + + +class InvalidProtectionPolicyError(ValueError): + """A protection policy whose axes contradict each other.""" + + +class Placement(StrEnum): + SINGLE_TAPE = "single_tape" + STRIPED = "striped" + + +class StripingMode(StrEnum): + NONE = "none" + STRIPE = "stripe" + BLOCK_STRIPE = "block_stripe" + + +class ProtectionKind(StrEnum): + NONE = "none" + REPLICATION = "replication" + ERASURE_CODE = "erasure_code" + + +class FailureDomain(StrEnum): + DRIVE = "drive" + TAPE = "tape" + LIBRARY = "library" + + +class DurabilityClass(StrEnum): + UNPROTECTED = "unprotected" + PERFORMANCE_ONLY = "performance-only" + PROTECTED = "protected" + + +@dataclass(frozen=True) +class Striping: + mode: StripingMode = StripingMode.NONE + block_size: int | None = None + + def __post_init__(self) -> None: + if self.mode is StripingMode.BLOCK_STRIPE: + if self.block_size is None or self.block_size <= 0: + raise InvalidProtectionPolicyError("block striping requires a positive block_size") + elif self.block_size is not None: + raise InvalidProtectionPolicyError("block_size is only valid for block striping") + + +@dataclass(frozen=True) +class Protection: + kind: ProtectionKind = ProtectionKind.NONE + copies: int = 1 + k: int = 0 + m: int = 0 + + def __post_init__(self) -> None: + if self.copies < 1: + raise InvalidProtectionPolicyError("copies must be >= 1") + if self.kind is ProtectionKind.ERASURE_CODE and (self.k < 1 or self.m < 1): + raise InvalidProtectionPolicyError("erasure coding requires k >= 1 and m >= 1") + + @classmethod + def none(cls) -> Protection: + return cls() + + @classmethod + def replication(cls, copies: int) -> Protection: + return cls(kind=ProtectionKind.REPLICATION, copies=copies) + + @classmethod + def erasure_code(cls, k: int, m: int) -> Protection: + return cls(kind=ProtectionKind.ERASURE_CODE, k=k, m=m) + + @property + def supported(self) -> bool: + """Erasure coding is modelled only; the planner cannot produce it.""" + return self.kind is not ProtectionKind.ERASURE_CODE + + @property + def redundant(self) -> bool: + if self.kind is ProtectionKind.REPLICATION: + return self.copies >= 2 + return self.kind is ProtectionKind.ERASURE_CODE + + +@dataclass(frozen=True) +class RestoreQuorum: + tapes_required: int + tapes_total: int + + def __post_init__(self) -> None: + if self.tapes_required < 1 or self.tapes_total < 1: + raise InvalidProtectionPolicyError("restore quorum counts must be >= 1") + if self.tapes_required > self.tapes_total: + raise InvalidProtectionPolicyError("restore quorum cannot exceed total tapes") + + +@dataclass(frozen=True) +class ProtectionPolicy: + placement: Placement = Placement.SINGLE_TAPE + striping: Striping = field(default_factory=Striping) + protection: Protection = field(default_factory=Protection) + failure_domain: FailureDomain = FailureDomain.TAPE + quorum: RestoreQuorum | None = None + + def __post_init__(self) -> None: + striped = self.striping.mode is not StripingMode.NONE + if striped != (self.placement is Placement.STRIPED): + raise InvalidProtectionPolicyError("striping mode must match STRIPED placement") + + @property + def is_protected(self) -> bool: + return self.protection.redundant + + @property + def durability_class(self) -> DurabilityClass: + if self.is_protected: + return DurabilityClass.PROTECTED + if self.striping.mode is not StripingMode.NONE: + return DurabilityClass.PERFORMANCE_ONLY + return DurabilityClass.UNPROTECTED + + def require_plannable(self) -> None: + """Raise if the planner cannot realise this policy.""" + if not self.protection.supported: + raise UnsupportedProtectionError( + f"protection '{self.protection.kind.value}' is modelled but not supported" + ) + + def to_dict(self) -> dict[str, Any]: + return { + "placement": self.placement.value, + "striping": {"mode": self.striping.mode.value, "block_size": self.striping.block_size}, + "protection": { + "kind": self.protection.kind.value, + "copies": self.protection.copies, + "k": self.protection.k, + "m": self.protection.m, + }, + "failure_domain": self.failure_domain.value, + "quorum": None + if self.quorum is None + else { + "tapes_required": self.quorum.tapes_required, + "tapes_total": self.quorum.tapes_total, + }, + "durability_class": self.durability_class.value, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> ProtectionPolicy: + striping = data.get("striping") or {} + protection = data.get("protection") or {} + quorum = data.get("quorum") + return cls( + placement=Placement(data.get("placement", Placement.SINGLE_TAPE.value)), + striping=Striping( + mode=StripingMode(striping.get("mode", StripingMode.NONE.value)), + block_size=striping.get("block_size"), + ), + protection=Protection( + kind=ProtectionKind(protection.get("kind", ProtectionKind.NONE.value)), + copies=int(protection.get("copies", 1)), + k=int(protection.get("k", 0)), + m=int(protection.get("m", 0)), + ), + failure_domain=FailureDomain(data.get("failure_domain", FailureDomain.TAPE.value)), + quorum=None + if quorum is None + else RestoreQuorum(int(quorum["tapes_required"]), int(quorum["tapes_total"])), + ) + + +def protection_from_pool(pool: Any) -> ProtectionPolicy: + """Map a legacy ``NasPool.replication_factor`` onto the protection axis. + + ``replication_factor`` stays the source of truth for existing pools; this only + expresses it as ``Protection.REPLICATION(copies=n)``. Placement is single-tape: + the legacy field says nothing about striping. An explicit ``pool.protection`` + policy, when set, wins over the legacy field. + """ + explicit = getattr(pool, "protection", None) + if isinstance(explicit, ProtectionPolicy): + return explicit + copies = max(1, int(getattr(pool, "replication_factor", 1) or 1)) + protection = Protection.replication(copies) if copies >= 2 else Protection.none() + return ProtectionPolicy(protection=protection) diff --git a/openblade/emulator_contract/README.md b/openblade/emulator_contract/README.md index 4da6e3a..eb21fb3 100644 --- a/openblade/emulator_contract/README.md +++ b/openblade/emulator_contract/README.md @@ -167,3 +167,22 @@ in pull requests and pushes to `master`, and also support manual dispatch: review before OpenBlade consumes it. - OpenBlade should fail fast on startup/test bring-up when contract expectations are not met. + +## Fault profiles + +`OPENBLADE_EMULATOR_FAULT_PROFILE` (runtime env; not a wire change) injects +AML wire-level faults. Value: a preset name or a JSON object with any of: + +| key | type | effect | +|---|---|---| +| `load_fail_every` | int | every Nth `mount`-class request returns 409 `AML_CONFLICT` | +| `auth_ttl_s` | float | sessions expire this many seconds after login (absolute, no sliding refresh) → 401 | +| `reboot_window_s` | float | every AML request returns 503 `AML_UNAVAILABLE` + `Retry-After` for this long after start | +| `checksum_retry_every` | int | every Nth `diagnostic`-class request takes twice its latency (simulated checksum retry) | + +Presets (`openblade/api/aml_faults.py`, shared with `tests/i3/timing.py`): +`intermittent-drive` (`load_fail_every=3`), `session-expiry` (`auth_ttl_s=5`), +`rebooting` (`reboot_window_s=20`), `degraded-media` (`checksum_retry_every=4`); +`instant`, `realistic`, `hardware`, `normal`, `slow-robotics`, `busy-library` = no faults. +Unset/empty = no faults. An unknown preset or malformed JSON raises +`FaultProfileError` (never a silent no-fault run). diff --git a/openblade/emulator_contract/contract.json b/openblade/emulator_contract/contract.json index 8f4009a..34e7b55 100644 --- a/openblade/emulator_contract/contract.json +++ b/openblade/emulator_contract/contract.json @@ -42,7 +42,8 @@ "EMULATOR_DRIVE_COUNT": "integer", "EMULATOR_OCCUPANCY_PERCENT": "integer", "EMULATOR_LATENCY_PROFILE": "instant|realistic|hardware|custom", - "OPENBLADE_SCALAR_API_ONLY": "true|false" + "OPENBLADE_SCALAR_API_ONLY": "true|false", + "OPENBLADE_EMULATOR_FAULT_PROFILE": "preset-name|json-object" }, "latency_profiles": ["instant", "realistic", "hardware", "custom"], "manual_matrix_file": "openblade/emulator_contract/quantum_i3_rev_h_matrix.json", diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 379c481..fed1b44 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -3,15 +3,249 @@ from __future__ import annotations import hashlib +import json +import logging +import os +import threading +from collections import OrderedDict from pathlib import Path +from openblade.domain.errors import OpenBladeError + +logger = logging.getLogger(__name__) + +SAMPLE_BYTES = 64 * 1024 +_PREAD_CAP = 1 << 30 # 1 GiB per os.pread call + + +class CacheEntryInUseError(RuntimeError): + """Raised when eviction is asked to drop a file that still has open handles.""" + + +class CacheError(OpenBladeError, ValueError): + """Typed cache failure for the protocol boundary (ValueError kept for old callers).""" + + +class CacheChecksumError(CacheError): + """Raised when staged bytes do not match the catalog checksum.""" + + +class CacheIntegrityError(CacheError): + """Raised when a cached entry no longer matches its integrity record (it is evicted).""" + + +def _sample_offset(checksum: str, size: int) -> int: + """Deterministic sample offset derived from the checksum; 0 when the file fits in one block.""" + if size <= SAMPLE_BYTES: + return 0 + # Seed from a digest of the checksum so any string (legacy non-hex records, + # test presets) yields a deterministic offset instead of a ValueError. + seed = int.from_bytes(hashlib.sha256(checksum.encode()).digest()[:8], "big") + return seed % max(1, size - SAMPLE_BYTES) + class HydrationCache: - def __init__(self, cache_dir: str) -> None: + """Checksum-addressed staging cache. + + ``max_bytes`` turns on a byte budget with LRU eviction. Entries with open + handles (``acquire``/``release`` refcount) are never evicted; the budget may + be exceeded rather than drop a file a reader is using. + """ + + def __init__(self, cache_dir: str, max_bytes: int | None = None) -> None: self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) + self.max_bytes = max_bytes + self._lock = threading.RLock() + self._lru: OrderedDict[str, int] = OrderedDict() + self._refs: dict[str, int] = {} + with self._lock: + # Not persisted here: a fresh cache stays empty on disk, and the scan + # re-finds any unindexed files on the next start. Evictions persist. + self._load_index() + self._enforce_budget() + + # -- persisted LRU index ------------------------------------------------- + @property + def index_path(self) -> Path: + return self.cache_dir / "index.json" + + def _load_index(self) -> None: + """Rebuild ``_lru`` from ``index.json`` plus any on-disk entries it misses. + + Entries whose file is gone are dropped; files not in the index are appended + (size from stat) so a restart always counts on-disk bytes against the budget. + """ + if self.index_path.exists(): + try: + raw = json.loads(self.index_path.read_text(encoding="utf-8")) + for checksum, size in raw: + if isinstance(checksum, str) and self.cache_key(checksum).is_file(): + self._lru[checksum] = int(size) + except (OSError, ValueError, TypeError): + logger.warning("fuse cache: index.json unreadable; rebuilding from disk scan") + self._lru.clear() + for path in sorted(self.cache_dir.glob("??/*")): + name = path.name + if name.startswith(".") or name.endswith(".meta.json") or not path.is_file(): + continue + if name not in self._lru: + self._lru[name] = path.stat().st_size + + def _persist_index(self) -> None: + tmp = self.index_path.with_name(f".index.json.{os.getpid()}.{threading.get_ident()}.part") + try: + tmp.write_text(json.dumps([[c, n] for c, n in self._lru.items()]), encoding="utf-8") + os.replace(tmp, self.index_path) + except OSError: + logger.warning("fuse cache: could not persist index.json") + finally: + tmp.unlink(missing_ok=True) + + # -- refcounting / LRU --------------------------------------------------- + def acquire(self, checksum: str) -> None: + with self._lock: + self._refs[checksum] = self._refs.get(checksum, 0) + 1 + if checksum in self._lru: + self._lru.move_to_end(checksum) + + def release(self, checksum: str) -> None: + with self._lock: + count = self._refs.get(checksum, 0) - 1 + if count <= 0: + self._refs.pop(checksum, None) + else: + self._refs[checksum] = count + self._enforce_budget() + + def open_count(self, checksum: str) -> int: + with self._lock: + return self._refs.get(checksum, 0) + + def used_bytes(self) -> int: + with self._lock: + return sum(self._lru.values()) + + def _track(self, checksum: str, size: int) -> None: + with self._lock: + self._lru[checksum] = size + self._lru.move_to_end(checksum) + self._enforce_budget() + self._persist_index() + + def _enforce_budget(self) -> None: + if self.max_bytes is None: + return + # The newest entry is exempt: it was just stored for a reader about to open it. + for checksum in list(self._lru)[:-1]: + if sum(self._lru.values()) <= self.max_bytes: + return + if self._refs.get(checksum, 0) > 0: + continue # never evict an open file + self.evict(checksum) + + def store_verified(self, checksum: str, staged: Path) -> Path: + """Move ``staged`` into the cache only if its sha256 matches. + + Bytes are written to a temp name beside the final key and renamed only + after the checksum verifies, so a failed restore never leaves a partial + file under the checksum name. + """ + final = self.cache_key(checksum) + final.parent.mkdir(parents=True, exist_ok=True) + tmp = final.with_name(f".{checksum}.{os.getpid()}.{threading.get_ident()}.part") + digest = hashlib.sha256() + try: + with staged.open("rb") as src, tmp.open("wb") as dst: + for chunk in iter(lambda: src.read(1 << 20), b""): + digest.update(chunk) + dst.write(chunk) + if digest.hexdigest() != checksum: + raise CacheChecksumError(f"staged bytes for {checksum} failed verification") + os.replace(tmp, final) + finally: + tmp.unlink(missing_ok=True) + self._write_meta(checksum) + self._track(checksum, final.stat().st_size) + return final + + # -- integrity records --------------------------------------------------- + def meta_key(self, checksum: str) -> Path: + path = self.cache_key(checksum) + return path.with_name(f"{path.name}.meta.json") + + def _sample_digest(self, path: Path, offset: int, length: int) -> str: + with path.open("rb") as handle: + handle.seek(offset) + return hashlib.sha256(handle.read(length)).hexdigest() + + def _write_meta(self, checksum: str) -> None: + """Record size, mtime_ns and a sampled block hash beside the cached file.""" + path = self.cache_key(checksum) + st = path.stat() + offset = _sample_offset(checksum, st.st_size) + length = min(SAMPLE_BYTES, st.st_size) + record = { + "size": st.st_size, + "mtime_ns": st.st_mtime_ns, + "sample": { + "offset": offset, + "len": length, + "sha256": self._sample_digest(path, offset, length), + }, + } + meta = self.meta_key(checksum) + tmp = meta.with_name(f".{meta.name}.{os.getpid()}.{threading.get_ident()}.part") + try: + tmp.write_text(json.dumps(record), encoding="utf-8") + os.replace(tmp, meta) + finally: + tmp.unlink(missing_ok=True) + + def _entry_matches(self, checksum: str) -> bool: + path = self.cache_key(checksum) + try: + record = json.loads(self.meta_key(checksum).read_text(encoding="utf-8")) + st = path.stat() + sample = record["sample"] + if st.st_size != record["size"] or st.st_mtime_ns != record["mtime_ns"]: + return False + offset, length = int(sample["offset"]), int(sample["len"]) + if (offset, length) != ( + _sample_offset(checksum, st.st_size), + min(SAMPLE_BYTES, st.st_size), + ): + return False + return bool(self._sample_digest(path, offset, length) == sample["sha256"]) + except (OSError, ValueError, KeyError, TypeError): + return False + + def verify_entry(self, checksum: str) -> bool: + """Cheap open-time check: size + mtime_ns + one sampled 64 KiB block. + + On mismatch or a missing/unreadable integrity record the entry (file and + sidecar) is dropped and ``CacheIntegrityError`` raised, so the caller can + fall through to re-hydration. + + This is a sampled check by design: bit-rot outside the sampled window on + a file whose size and mtime_ns are preserved is NOT detected here; only + a full re-hash (``store_verified``) catches that. + """ + with self._lock: + if self._entry_matches(checksum): + return True + self._lru.pop(checksum, None) + self.cache_key(checksum).unlink(missing_ok=True) + self.meta_key(checksum).unlink(missing_ok=True) + self._persist_index() + raise CacheIntegrityError("cached entry failed integrity verification and was evicted") def cache_key(self, checksum: str) -> Path: + # The key is one path component, so only a whole-component "." / ".." or an + # embedded separator/NUL can escape cache_dir; a ".." substring inside a + # longer token (truncated test fixtures use "...") is harmless. + if checksum in ("", ".", "..") or any(bad in checksum for bad in ("/", "\\", "\0")): + raise CacheError("checksum is not a safe cache path component") return self.cache_dir / checksum[:2] / checksum def is_cached(self, checksum: str) -> bool: @@ -22,6 +256,8 @@ def store(self, checksum: str, data: bytes) -> Path: path.parent.mkdir(parents=True, exist_ok=True) with path.open("wb") as handle: handle.write(data) + self._write_meta(checksum) + self._track(checksum, len(data)) return path def retrieve(self, checksum: str) -> bytes: @@ -33,10 +269,46 @@ def retrieve(self, checksum: str) -> bytes: actual = hashlib.sha256(data).hexdigest() if actual != checksum: path.unlink() - raise ValueError(f"Cache integrity failure: expected {checksum}, got {actual}") + raise CacheChecksumError(f"Cache integrity failure: expected {checksum}, got {actual}") return data - def evict(self, checksum: str) -> None: + def read_range(self, checksum: str, offset: int, length: int) -> bytes: + """``pread`` ``[offset, offset+length)`` of a cached file without re-hashing it. + + The whole file was hashed once by ``store_verified``/``store`` callers; per-range + reads must stay O(length), so integrity is not re-checked here (``retrieve`` does). + """ + if offset < 0 or length < 0: + raise CacheError(f"invalid range offset={offset} length={length}") path = self.cache_key(checksum) - if path.exists(): - path.unlink() + try: + fd = os.open(path, os.O_RDONLY) + except FileNotFoundError: + raise FileNotFoundError(f"Not in cache: {checksum}") from None + try: + # pread may return short (and Linux caps one call near 2 GiB): loop until + # ``length`` bytes or EOF, returning exactly the bytes available. + parts: list[bytes] = [] + remaining = length + pos = offset + while remaining > 0: + chunk = os.pread(fd, min(remaining, _PREAD_CAP), pos) + if not chunk: + break # EOF + parts.append(chunk) + pos += len(chunk) + remaining -= len(chunk) + return b"".join(parts) + finally: + os.close(fd) + + def evict(self, checksum: str) -> None: + with self._lock: + if self._refs.get(checksum, 0) > 0: + raise CacheEntryInUseError(f"{checksum} has open handles") + self._lru.pop(checksum, None) + path = self.cache_key(checksum) + if path.exists(): + path.unlink() + self.meta_key(checksum).unlink(missing_ok=True) + self._persist_index() diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 17e80ad..d2c2063 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -2,7 +2,29 @@ from __future__ import annotations -from dataclasses import dataclass +import json +import logging +import os +import threading +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, Protocol + +from sqlalchemy.exc import InvalidRequestError +from sqlalchemy.orm import Session + +from openblade.domain.errors import OpenBladeError +from openblade.fuse.cache import CacheChecksumError, HydrationCache +from openblade.jobs.restore import RestoreRequest + +logger = logging.getLogger(__name__) + +_UNEXPECTED = "restore failed unexpectedly" +_UNMOUNTING = "hydration cancelled by unmount" @dataclass @@ -23,3 +45,417 @@ def pending(self) -> list[HydrationRequest]: def clear(self) -> None: self._queue.clear() + + +# -- hydration data plane ---------------------------------------------------- +# +# Layering: the Hydrator owns tickets, batching and the cache hand-off; a +# RestoreEngine owns *how* bytes come back from tape. The production engine goes +# through RestoreService (the job machinery). Nothing here touches a tape +# backend, formats media, or moves cartridges directly. + +DEFAULT_HYDRATE_TIMEOUT_S = 300.0 +DEFAULT_BATCH_WINDOW_MS = 250 + + +def hydrate_timeout_from_env() -> float: + raw = os.environ.get("OPENBLADE_FUSE_HYDRATE_TIMEOUT", "").strip() + return float(raw) if raw else DEFAULT_HYDRATE_TIMEOUT_S + + +def batch_window_from_env() -> float: + raw = os.environ.get("OPENBLADE_FUSE_BATCH_WINDOW_MS", "").strip() + return (int(raw) if raw else DEFAULT_BATCH_WINDOW_MS) / 1000.0 + + +class HydrationState(str, Enum): + OFFLINE = "offline" + HYDRATING = "hydrating" + ONLINE = "online" + + +class HydrationFailedError(RuntimeError): + """A restore for hydration failed; the message is curated, never raw.""" + + +class HydrationTimeoutError(TimeoutError): + """The file did not come online within the wait budget.""" + + +@dataclass +class HydrationTicket: + catalog_path: str + checksum: str + tape_key: str + job_id: str | None = None + error: str | None = None + done: threading.Event = field(default_factory=threading.Event) + + +class RestoreEngine(Protocol): + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + """Restore every path (all on one tape) and return staged file paths.""" + + def find_active_job(self, catalog_path: str) -> str | None: + """Return the id of a pending/running restore job for the path, if any.""" + + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + """Wait for an existing job to finish and return the path's staged file.""" + + +class JobRestoreEngine: + """Production engine: one restore job per tape batch via ``RestoreService``. + + Files in one batch share a tape; ``RestoreService.enqueue_batch`` takes one + lease and loads, mounts, unmounts and unloads that cartridge once + (scheduler/lease policy is the jobs layer's concern, not ours). + """ + + def __init__( + self, + restore_service: Any, + catalog: Any, + staging_root: Path, + *, + poll_interval_s: float = 0.5, + resume_timeout_s: float | None = None, + catalog_factory: Callable[[], Any] | None = None, + restore_service_factory: Callable[[Any], Any] | None = None, + ) -> None: + self.poll_interval_s = poll_interval_s + self.resume_timeout_s = ( + hydrate_timeout_from_env() if resume_timeout_s is None else resume_timeout_s + ) + self.restore_service = restore_service + self.catalog = catalog + self.staging_root = staging_root + self.catalog_factory = catalog_factory + # Builds a RestoreService over the worker's own catalog so the restore + # job's writes never go through the caller's (other-thread) Session. + self.restore_service_factory = restore_service_factory + + @contextmanager + def _worker_catalog(self) -> Iterator[Any]: + """A catalog over a Session private to the calling (worker) thread. + + ``restore_batch``/``resume`` run on Hydrator worker threads; SQLAlchemy + Sessions are not thread-safe, so the caller's session is never used here. + """ + if self.catalog_factory is not None: + repo = self.catalog_factory() + else: + from openblade.catalog.repository import CatalogRepository # local: avoid import cycle + + bind = self.catalog.session.get_bind() + repo = CatalogRepository(Session(bind=bind, expire_on_commit=False)) + try: + yield repo + finally: + repo.session.close() + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + with self._worker_catalog() as catalog: + return self._restore_batch(catalog, tape_key, catalog_paths) + + def _restore_batch( + self, catalog: Any, tape_key: str, catalog_paths: list[str] + ) -> dict[str, Path]: + del tape_key + service = ( + self.restore_service + if self.restore_service_factory is None + else self.restore_service_factory(catalog) + ) + staged: dict[str, Path] = {} + for path in catalog_paths: + record = catalog.get_file_record(path) + if record is None: + raise HydrationFailedError(f"{path} is not in the catalog") + dest = self.staging_root / record.id + dest.parent.mkdir(parents=True, exist_ok=True) + staged[path] = dest + # ONE job for the whole tape group: the jobs layer loads/mounts once. + job, result = service.enqueue_batch( + [RestoreRequest(catalog_path=path, dest_path=dest) for path, dest in staged.items()] + ) + if job.state != "completed": + failed = [item.catalog_path for item in result.items if not item.ok] + raise HydrationFailedError( + f"restore job {job.id} ended {job.state}; failed: {', '.join(failed) or 'tape'}" + ) + return staged + + def find_active_job(self, catalog_path: str) -> str | None: + for job in self.catalog.list_jobs(): + if job.job_type != "restore" or job.state not in {"pending", "running"}: + continue + if _job_entry(json.loads(job.metadata_json or "{}"), catalog_path) is not None: + return str(job.id) + return None + + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + with self._worker_catalog() as catalog: + return self._resume(catalog, job_id, catalog_path) + + def _resume(self, catalog: Any, job_id: str, catalog_path: str | None = None) -> Path: + deadline = time.monotonic() + self.resume_timeout_s + while True: + job = catalog.get_job(job_id) + if job is not None: + # The job is advanced by another session; re-read the row instead + # of trusting this session's identity-map copy. + try: + catalog.session.refresh(job) + except InvalidRequestError: + job = None + if job is None: + raise HydrationFailedError(f"restore job {job_id} disappeared") + if job.state == "completed": + return _job_dest(job_id, json.loads(job.metadata_json or "{}"), catalog_path) + if job.state not in {"pending", "running"}: + raise HydrationFailedError(f"restore job {job_id} ended {job.state}") + if time.monotonic() >= deadline: + raise HydrationTimeoutError(f"restore job {job_id} did not finish in time") + time.sleep(self.poll_interval_s) + + +def _job_entry(metadata: Any, catalog_path: str | None) -> dict[str, Any] | None: + """The {catalog_path, dest_path} entry for ``catalog_path`` in single or batch job metadata. + + Single jobs store the entry at top level; batch jobs store ``{"batch": [entry, ...]}``. + With ``catalog_path=None`` a single job's top-level entry is returned. + """ + if not isinstance(metadata, dict): + return None + if "catalog_path" in metadata and catalog_path in {None, metadata["catalog_path"]}: + return metadata + batch = metadata.get("batch") + if catalog_path is not None and isinstance(batch, list): + for entry in batch: + if isinstance(entry, dict) and entry.get("catalog_path") == catalog_path: + return entry + return None + + +def _job_dest(job_id: str, metadata: Any, catalog_path: str | None) -> Path: + entry = _job_entry(metadata, catalog_path) + dest = None if entry is None else entry.get("dest_path") + if not isinstance(dest, str) or not dest: + raise HydrationFailedError( + f"restore job {job_id} has no staged destination for {catalog_path or 'its path'}" + ) + return Path(dest) + + +class Hydrator: + def __init__( + self, + catalog: Any, + cache: HydrationCache, + engine: RestoreEngine, + *, + batch_window_s: float | None = None, + ) -> None: + self.catalog = catalog + self.cache = cache + self.engine = engine + self.batch_window_s = batch_window_from_env() if batch_window_s is None else batch_window_s + self._lock = threading.Lock() + self._tickets: dict[str, HydrationTicket] = {} + self._pending: dict[str, list[HydrationTicket]] = {} + self._timers: dict[str, threading.Timer] = {} + self._workers: list[threading.Thread] = [] + self._errors: dict[str, str] = {} + self._closed = False + + def _tape_key(self, catalog_path: str, record: Any) -> str: + try: + _, instance = self.catalog.get_latest_instance_for_path(catalog_path) + return str(instance.barcode) + except FileNotFoundError: + return f"vg:{record.volume_group_id}" + + def status(self, catalog_path: str) -> HydrationState: + record = self.catalog.get_file_record(catalog_path) + if record is not None and self.cache.is_cached(record.checksum_sha256): + return HydrationState.ONLINE + with self._lock: + ticket = self._tickets.get(catalog_path) + if ticket is not None and not ticket.done.is_set(): + return HydrationState.HYDRATING + return HydrationState.OFFLINE + + def last_error(self, catalog_path: str) -> str | None: + with self._lock: + return self._errors.get(catalog_path) + + def request(self, catalog_path: str) -> HydrationTicket: + record = self.catalog.get_file_record(catalog_path) + if record is None: + raise FileNotFoundError(catalog_path) + with self._lock: + if self._closed: + raise HydrationFailedError("hydrator is shut down") + existing = self._tickets.get(catalog_path) + if existing is not None and not existing.done.is_set(): + return existing + ticket = HydrationTicket(catalog_path, record.checksum_sha256, "") + if self.cache.is_cached(record.checksum_sha256): + ticket.done.set() + return ticket + self._tickets[catalog_path] = ticket + # Restart-safety: a job already in flight for this path is re-attached, + # not duplicated. + try: + job_id = self.engine.find_active_job(catalog_path) + if job_id is None: + ticket.tape_key = self._tape_key(catalog_path, record) + except BaseException: + # Never leave a registered ticket that no worker will complete. + with self._lock: + if self._tickets.get(catalog_path) is ticket: + del self._tickets[catalog_path] + raise + if job_id is not None: + ticket.job_id = job_id + self._spawn(self._resume, ticket) + return ticket + with self._lock: + closed = self._closed + if not closed: + self._pending.setdefault(ticket.tape_key, []).append(ticket) + if ticket.tape_key not in self._timers: + timer = threading.Timer( + self.batch_window_s, self._flush, args=(ticket.tape_key,) + ) + timer.daemon = True + self._timers[ticket.tape_key] = timer + timer.start() + if closed: + # shutdown() ran after the _closed check above; never strand the ticket. + self._fail(ticket, _UNMOUNTING) + return ticket + + def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: + return self._pinned_read(ticket, timeout, lambda: self.cache.retrieve(ticket.checksum)) + + def wait_range( + self, ticket: HydrationTicket, offset: int, length: int, timeout: float | None = None + ) -> bytes: + """Like ``wait`` but read only ``[offset, offset+length)``: O(length), no re-hash.""" + return self._pinned_read( + ticket, timeout, lambda: self.cache.read_range(ticket.checksum, offset, length) + ) + + def _pinned_read( + self, ticket: HydrationTicket, timeout: float | None, read: Callable[[], bytes] + ) -> bytes: + # Each waiter pins the entry before the restore commits, so the budget + # cannot evict it (e.g. while a later file of the same batch is stored) + # before this waiter has read it. Released on every exit path. + self.cache.acquire(ticket.checksum) + try: + if not ticket.done.wait(timeout): + raise HydrationTimeoutError(f"{ticket.catalog_path} still hydrating") + if ticket.error is not None: + raise HydrationFailedError(ticket.error) + return read() + finally: + self.cache.release(ticket.checksum) + + def shutdown(self, timeout: float = 30.0) -> None: + """Cancel batches not yet started and wait for in-flight restores.""" + with self._lock: + self._closed = True + timers, self._timers = self._timers, {} + pending, self._pending = self._pending, {} + workers = list(self._workers) + for timer in timers.values(): + timer.cancel() + for tickets in pending.values(): + for ticket in tickets: + self._fail(ticket, _UNMOUNTING) + deadline = time.monotonic() + timeout + for worker in workers: + worker.join(max(0.0, deadline - time.monotonic())) + + # -- internals ----------------------------------------------------------- + def _spawn(self, target: Any, *args: Any) -> None: + worker = threading.Thread(target=target, args=args, daemon=True) + with self._lock: + self._workers = [w for w in self._workers if w.is_alive()] + self._workers.append(worker) + worker.start() + + def _flush(self, tape_key: str) -> None: + with self._lock: + self._timers.pop(tape_key, None) + batch = self._pending.pop(tape_key, []) + closed = self._closed + if not closed and batch: + self._workers.append(threading.current_thread()) + if closed: + for ticket in batch: + self._fail(ticket, _UNMOUNTING) + return + if not batch: + return + try: + staged = self.engine.restore_batch(tape_key, [t.catalog_path for t in batch]) + except (HydrationFailedError, OpenBladeError) as exc: + reason = str(exc) # typed errors carry a curated message + except OSError: + reason = "restore I/O error" + except Exception: + logger.exception("fuse: unexpected restore failure for tape %s", tape_key) + reason = _UNEXPECTED + else: + for ticket in batch: + self._commit(ticket, staged.get(ticket.catalog_path)) + return + for ticket in batch: + self._fail(ticket, reason) + + def _resume(self, ticket: HydrationTicket) -> None: + assert ticket.job_id is not None + try: + self._commit(ticket, self.engine.resume(ticket.job_id, ticket.catalog_path)) + except HydrationTimeoutError: + self._fail(ticket, f"restore job {ticket.job_id} timed out") + except (HydrationFailedError, OpenBladeError, OSError, KeyError): + self._fail(ticket, f"restore job {ticket.job_id} did not complete") + except Exception: + logger.exception("fuse: unexpected failure resuming job %s", ticket.job_id) + self._fail(ticket, _UNEXPECTED) + + def _commit(self, ticket: HydrationTicket, staged: Path | None) -> None: + if staged is None: + self._fail(ticket, "restore produced no file") + return + try: + self.cache.store_verified(ticket.checksum, staged) + except CacheChecksumError: + self._discard_staged(staged) + self._fail(ticket, "restored bytes failed checksum verification") + return + except OSError: + self._fail(ticket, "restore I/O error") + return + self._discard_staged(staged) + with self._lock: + self._errors.pop(ticket.catalog_path, None) + ticket.done.set() + + @staticmethod + def _discard_staged(staged: Path) -> None: + """Remove the staging copy once the cache holds (or rejected) its bytes.""" + try: + staged.unlink(missing_ok=True) + except OSError: + logger.warning("fuse: could not remove staging file %s", staged) + + def _fail(self, ticket: HydrationTicket, reason: str) -> None: + ticket.error = reason + with self._lock: + self._errors[ticket.catalog_path] = reason + ticket.done.set() diff --git a/openblade/fuse/mount.py b/openblade/fuse/mount.py index 4bf5e75..de1d586 100644 --- a/openblade/fuse/mount.py +++ b/openblade/fuse/mount.py @@ -34,7 +34,17 @@ from pathlib import PurePosixPath from typing import Any +from openblade.fuse.cache import CacheIntegrityError, HydrationCache from openblade.fuse.filesystem import CatalogFilesystem, VirtualDirEntry +from openblade.fuse.hydration import ( + HydrationFailedError, + HydrationState, + HydrationTimeoutError, + Hydrator, + hydrate_timeout_from_env, +) + +STATE_XATTR = "user.openblade.state" logger = logging.getLogger(__name__) @@ -100,11 +110,20 @@ def __init__( filesystem: CatalogFilesystem, *, hydrator: Callable[[str], bytes] | None = None, + data_plane: Hydrator | None = None, + blocking: bool = True, + hydrate_timeout: float | None = None, uid: int | None = None, gid: int | None = None, ) -> None: self.filesystem = filesystem self.hydrator = hydrator + self.data_plane = data_plane + self.blocking = blocking + self.hydrate_timeout = ( + hydrate_timeout_from_env() if hydrate_timeout is None else hydrate_timeout + ) + self._handle_checksums: dict[int, str] = {} self.uid = os.getuid() if uid is None else uid self.gid = os.getgid() if gid is None else gid self._lock = threading.Lock() @@ -146,6 +165,9 @@ def init(self, path: str) -> None: def destroy(self, path: str) -> None: del path + if self.data_plane is not None: + # Unmount: cancel batches not yet started, wait for in-flight restores. + self.data_plane.shutdown() with self._lock: self._open_files.clear() @@ -201,12 +223,22 @@ def releasedir(self, path: str, fh: int) -> int: return 0 def getxattr(self, path: str, name: str, position: int = 0) -> bytes: - del path, name, position - raise OSError(errno.ENODATA, os.strerror(errno.ENODATA)) + del position + entry = self._entry(path) + if entry.is_dir or name != STATE_XATTR: + raise OSError(errno.ENODATA, os.strerror(errno.ENODATA)) + return self._state(str(entry.path)).value.encode() def listxattr(self, path: str) -> list[str]: - del path - return [] + entry = self._entry(path) + return [] if entry.is_dir else [STATE_XATTR] + + def _state(self, catalog_path: str) -> HydrationState: + if self.data_plane is not None: + return self.data_plane.status(catalog_path) + if self.filesystem.is_hydrated(catalog_path): + return HydrationState.ONLINE + return HydrationState.OFFLINE # -- reads -------------------------------------------------------------- def open(self, path: str, flags: int) -> int: @@ -215,11 +247,24 @@ def open(self, path: str, flags: int) -> int: entry = self._entry(path) if entry.is_dir: raise OSError(errno.EISDIR, os.strerror(errno.EISDIR), path) - data = self._materialise(str(entry.path)) + # Pin before materialising so the entry cannot be evicted in between. + record = self.filesystem.catalog.get_file_record(str(entry.path)) + if record is not None: + _verify_cached_entry(self.filesystem.cache, record.checksum_sha256) + self.filesystem.cache.acquire(record.checksum_sha256) + try: + data = self._materialise(str(entry.path)) + except BaseException: + if record is not None: + self.filesystem.cache.release(record.checksum_sha256) + raise with self._lock: handle = self._next_handle self._next_handle += 1 self._open_files[handle] = data + if record is not None: + with self._lock: + self._handle_checksums[handle] = record.checksum_sha256 return handle def read(self, path: str, size: int, offset: int, fh: int) -> bytes: @@ -243,6 +288,9 @@ def release(self, path: str, fh: int) -> int: del path with self._lock: self._open_files.pop(fh, None) + checksum = self._handle_checksums.pop(fh, None) + if checksum is not None: + self.filesystem.cache.release(checksum) return 0 def _materialise(self, catalog_path: str) -> bytes: @@ -261,6 +309,8 @@ def _materialise(self, catalog_path: str) -> bytes: except (OSError, ValueError) as exc: logger.warning("fuse: cache read failed for %s: %s", catalog_path, exc) raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + if self.data_plane is not None: + return self._hydrate(catalog_path) if self.hydrator is None: logger.warning( "fuse: %s is not in the staging cache; returning EIO. " @@ -276,6 +326,18 @@ def _materialise(self, catalog_path: str) -> bytes: logger.warning("fuse: hydration failed for %s: %s", catalog_path, exc) raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + def _hydrate(self, catalog_path: str) -> bytes: + assert self.data_plane is not None + ticket = self.data_plane.request(catalog_path) + timeout = self.hydrate_timeout if self.blocking else 0.0 + try: + return self.data_plane.wait(ticket, timeout) + except HydrationTimeoutError: + raise OSError(errno.EAGAIN, os.strerror(errno.EAGAIN), catalog_path) from None + except (HydrationFailedError, OSError, ValueError): + logger.warning("fuse: hydration failed for %s", catalog_path) + raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + def _catalog_path(path: str) -> str: """Map a FUSE path onto a catalog path, rejecting traversal.""" @@ -290,6 +352,8 @@ def mount_catalog( mountpoint: str, *, hydrator: Callable[[str], bytes] | None = None, + data_plane: Hydrator | None = None, + blocking: bool = True, allow_other: bool = False, foreground: bool = True, ) -> None: @@ -301,7 +365,9 @@ def mount_catalog( fuse = load_fuse() if not os.path.isdir(mountpoint): raise NotADirectoryError(f"Mount point is not a directory: {mountpoint}") - operations = CatalogFuseOperations(filesystem, hydrator=hydrator) + operations = CatalogFuseOperations( + filesystem, hydrator=hydrator, data_plane=data_plane, blocking=blocking + ) fuse.FUSE( operations, mountpoint, @@ -312,3 +378,13 @@ def mount_catalog( fsname="openblade-catalog", subtype="openblade", ) + + +def _verify_cached_entry(cache: HydrationCache, checksum: str) -> None: + """Open-time integrity check; a failed entry is evicted so hydration refetches it.""" + if not cache.is_cached(checksum): + return + try: + cache.verify_entry(checksum) + except CacheIntegrityError: + logger.warning("fuse: cached entry %s failed integrity check; re-hydrating", checksum) diff --git a/openblade/hardware/correlation.py b/openblade/hardware/correlation.py index f5da8c3..f76b560 100644 --- a/openblade/hardware/correlation.py +++ b/openblade/hardware/correlation.py @@ -61,6 +61,7 @@ from __future__ import annotations import logging +import os import subprocess from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import dataclass @@ -445,3 +446,33 @@ def _refuse_on_mismatch( def _format_serials(live: Mapping[str, str]) -> str: return ", ".join(f"{device}={serial or ''}" for device, serial in sorted(live.items())) + + +TAPE_BY_ID_DIR = "/dev/tape/by-id" +"""udev's stable per-serial tape symlink directory.""" + + +def resolve_tape_by_id(serial: str, by_id_dir: str = TAPE_BY_ID_DIR) -> str | None: + """Return the ``/dev/tape/by-id`` entry whose name carries ``serial``, or ``None``. + + Read-only: lists one directory and never opens a device. Prefers the + non-rewinding ``-nst`` link when udev publishes both. The match is on the + normalized serial as a whole ``-``/``_``-delimited token, so serial ``123`` + cannot match ``...-1234-nst``. + """ + wanted = _normalize_serial(serial) + if not wanted: + return None + try: + names = sorted(os.listdir(by_id_dir)) + except OSError: + return None + matches = [ + name + for name in names + if wanted in {_normalize_serial(t) for t in name.replace("_", "-").split("-")} + ] + if not matches: + return None + preferred = [m for m in matches if m.endswith("-nst")] or matches + return os.path.join(by_id_dir, preferred[0]) diff --git a/openblade/hardware/library.py b/openblade/hardware/library.py index 9ec8b56..405f425 100644 --- a/openblade/hardware/library.py +++ b/openblade/hardware/library.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from openblade.config import OpenBladeConfig +from openblade.domain.errors import TapeMountedError from openblade.domain.models import ( Barcode, CartridgeState, @@ -103,6 +104,11 @@ def load(self, source_slot: int, drive_id: int) -> OperationResult: return result def unload(self, drive_id: int, target_slot: int) -> OperationResult: + # Never unload while LTFS is mounted or dirty. The changer cannot see + # host-side mount state, so this in-process record is the gate. + mount_state = self._mount_states.get(drive_id, MountState.UNMOUNTED) + if mount_state is not MountState.UNMOUNTED: + raise TapeMountedError(f"Drive {drive_id} cannot be unloaded while {mount_state.value}") result = self.changer.unload(drive_id, target_slot) if result.success: self._mount_states[drive_id] = MountState.UNMOUNTED diff --git a/openblade/hardware/scalar_http/library_backend.py b/openblade/hardware/scalar_http/library_backend.py index a1565ff..055cbf1 100644 --- a/openblade/hardware/scalar_http/library_backend.py +++ b/openblade/hardware/scalar_http/library_backend.py @@ -69,7 +69,7 @@ import httpx -from openblade.domain.errors import DriveCorrelationError, OpenBladeError +from openblade.domain.errors import DriveCorrelationError, OpenBladeError, TapeMountedError from openblade.domain.models import ( Barcode, CartridgeState, @@ -184,7 +184,7 @@ def inventory(self) -> LibraryInventory: drive_id=int(address), barcode=_barcode_or_none(element.get("barcode")), drive_state=_drive_state(element.get("state")), - mount_state=MountState.UNMOUNTED, + mount_state=self._mount_states.get(int(address), MountState.UNMOUNTED), ) ) return LibraryInventory( @@ -258,6 +258,12 @@ def load(self, source_slot: int, drive_id: int) -> OperationResult: ) def unload(self, drive_id: int, target_slot: int) -> OperationResult: + # Safety gate: the Web Services API has no notion of LTFS mount state, so + # the host-side record is the only thing standing between a mounted (or + # dirty) volume and a moveMedium that ejects it. + mount_state = self._mount_states.get(drive_id, MountState.UNMOUNTED) + if mount_state is not MountState.UNMOUNTED: + raise TapeMountedError(f"Drive {drive_id} cannot be unloaded while {mount_state.value}") # Real i3 unload uses moveClass=8 (bit field) with the drive source; the # target slot is sent as a hint (Web Services manual). See # docs/reference/i3-contract-notes.md. diff --git a/openblade/jobs/archive.py b/openblade/jobs/archive.py index ecc424a..d9040fb 100644 --- a/openblade/jobs/archive.py +++ b/openblade/jobs/archive.py @@ -12,9 +12,16 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.capacity import has_room_for -from openblade.domain.errors import ChecksumMismatchError, NoScratchMediaError, safe_job_error +from openblade.domain.errors import ( + ChecksumMismatchError, + NoScratchMediaError, + StaleLeaseError, + safe_job_error, +) from openblade.domain.models import JobType, MountMode +from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue +from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import TapeOperationFailedError, execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -158,37 +165,40 @@ def _load_if_needed( catalog: CatalogRepository, library: LibraryBackend, ltfs: LTFSBackend, - barcode: str, + handle: DriveHandle, job_id: str, ) -> tuple[int, int | None]: + barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: + # Already in a drive: record it as the PHYSICAL drive; the lease key + # (handle.drive_id) stays what the scheduler reserved. + if drive_id != handle.drive_id: + handle.physical_drive_id = drive_id return drive_id, None slot_id = library.find_slot_by_barcode(barcode) if slot_id is None: raise NoScratchMediaError(f"Barcode {barcode} not found in inventory") - for drive in library.inventory().drives: - if drive.barcode is not None: - continue - try: - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.LOAD, - barcode=barcode, - drive_id=drive.drive_id, - slot_id=slot_id, - requested_by="archive-job", - job_id=job_id, - ), - raise_on_failed=True, - ) - return drive.drive_id, slot_id - except TapeOperationFailedError: - continue - raise NoScratchMediaError("No available drives for archive load") + try: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.LOAD, + barcode=barcode, + drive_id=handle.drive_id, + slot_id=slot_id, + requested_by="archive-job", + job_id=job_id, + ), + raise_on_failed=True, + ) + except TapeOperationFailedError as exc: + raise NoScratchMediaError( + f"Leased drive {handle.drive_id} could not load {barcode}" + ) from exc + return handle.drive_id, slot_id def run_archive_job( @@ -197,8 +207,16 @@ def run_archive_job( ltfs: LTFSBackend, catalog: CatalogRepository, job_id: str, + *, + scheduler: DriveScheduler | None = None, ) -> ArchiveResult: - """Archive source files onto LTFS media and record them in the catalog.""" + """Archive source files onto LTFS media and record them in the catalog. + + Each tape's drive is taken through ``scheduler`` (a lease shared with every + other job on the same store). ``None`` falls back to a job-local in-memory + scheduler, which serialises nothing across jobs -- callers that can reach the + app's lease store must pass one. + """ catalog.update_job_state(job_id, "running") files = _iter_source_files(request.source_path) volume_group = catalog.get_volume_group(request.volume_group_name) @@ -215,6 +233,12 @@ def run_archive_job( catalog.update_job_state(job_id, "completed") return result + if scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + leases = scheduler + current_lease: DriveHandle | None = None current_barcode: str | None = None current_handle = None current_slot_id: int | None = None @@ -233,10 +257,13 @@ def finalize_current_mount() -> None: pending_instance_ids if current_handle is None or current_barcode is None or current_drive_id is None: return + assert current_lease is not None + leases.verify(current_lease) # fencing: before unmount ltfs.unmount(current_handle) for instance_id in pending_instance_ids: catalog.mark_instance_archived(instance_id, checksum_verified=True) pending_instance_ids = [] + leases.verify(current_lease) # fencing: before unload if current_slot_id is not None: execute_tape_request( catalog, @@ -263,6 +290,18 @@ def finalize_current_mount() -> None: current_handle = None current_slot_id = None current_drive_id = None + release_current_lease() + + def release_current_lease() -> None: + nonlocal current_lease + if current_lease is None: + return + lease = current_lease + current_lease = None + leases.release_drives([lease]) + catalog.journal( + job_id, "lease_released", {"drive_id": lease.drive_id, "barcode": lease.barcode} + ) try: for file_path in files: @@ -278,14 +317,22 @@ def finalize_current_mount() -> None: selected_barcode = _choose_tape(catalog, library, ltfs, volume_group.id, size_bytes) if selected_barcode != current_barcode: finalize_current_mount() + current_lease = leases.acquire_drives([selected_barcode])[0] + catalog.journal( + job_id, + "lease_acquired", + {"drive_id": current_lease.drive_id, "barcode": current_lease.barcode}, + ) current_drive_id, current_slot_id = _load_if_needed( catalog, library, ltfs, - selected_barcode, + current_lease, job_id, ) + leases.record_physical_drives([current_lease]) _mark_aml_drive_busy(selected_barcode, current_drive_id) + leases.verify(current_lease) # fencing: never mount RW on a stale lease current_handle = ltfs.mount(selected_barcode, MountMode.READ_WRITE) current_barcode = selected_barcode tapes_used.append(selected_barcode) @@ -310,6 +357,19 @@ def finalize_current_mount() -> None: bytes_archived += size_bytes files_archived += 1 finalize_current_mount() + except StaleLeaseError as exc: + # Fenced out: leave the hardware ALONE -- whoever holds the lease now may + # have their own tape in that drive. Pending instances stay unarchived. + lost = current_lease + logger.error( + "job %s lost its lease on drive %s (%s); leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + lost.physical if lost is not None else "?", + current_barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise except Exception as exc: if current_handle is not None: try: @@ -340,6 +400,8 @@ def finalize_current_mount() -> None: logger.exception("failed to reset archive AML drive state for job %s", job_id) catalog.update_job_state(job_id, "failed", safe_job_error(exc)) raise + finally: + release_current_lease() result = ArchiveResult( job_id=job_id, @@ -370,11 +432,23 @@ def __init__( ltfs: LTFSBackend, catalog: CatalogRepository, queue: JobQueue, + *, + lease_store: LeaseStore | None = None, ) -> None: self.library = library self.ltfs = ltfs self.catalog = catalog self.queue = queue + # Production passes the catalog-backed store so this job's drives are + # excluded from every other job, in this process or another one. + self.lease_store = lease_store if lease_store is not None else CatalogLeaseStore(catalog) + + def _scheduler(self, job_id: str) -> DriveScheduler: + return DriveScheduler( + num_drives=len(InventoryService(self.library).snapshot().drives), + store=self.lease_store, + job_id=job_id, + ) def enqueue(self, volume_group_name: str, source_path: Path) -> Job: job = self.catalog.create_job( @@ -388,6 +462,7 @@ def enqueue(self, volume_group_name: str, source_path: Path) -> Job: self.ltfs, self.catalog, job.id, + scheduler=self._scheduler(job.id), ) except Exception: for file_path in _iter_source_files(source_path): diff --git a/openblade/jobs/queue.py b/openblade/jobs/queue.py index 2040996..d8905a0 100644 --- a/openblade/jobs/queue.py +++ b/openblade/jobs/queue.py @@ -1,12 +1,14 @@ -"""In-process job queue with resource ownership tracking.""" +"""Job queue persisted in the catalog, with resource ownership tracking.""" +import json import logging import threading from collections.abc import Callable -from dataclasses import replace -from datetime import datetime, timezone +from datetime import timedelta, timezone from typing import TypeVar +from openblade.catalog import models as catalog_models +from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import ( ChangerBusyError, DriveOccupiedError, @@ -14,79 +16,164 @@ safe_job_error, ) from openblade.domain.models import Job, JobState, JobType +from openblade.jobs.reconcile import ensure_drive_reconciled +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL, LeaseStore ResultT = TypeVar("ResultT") logger = logging.getLogger(__name__) +# Changer ownership is a lease on this reserved pseudo-drive id. Real drives +# are 0..N-1, so a negative id can never collide with one. +CHANGER_DRIVE_ID = -1 +_CHANGER_KIND = "CHANGER" +_DRIVE_KIND = "QUEUE-CLAIM" + + +def _job_from_row(row: catalog_models.Job) -> Job: + return Job( + id=row.id, + job_type=JobType(row.job_type), + state=JobState(row.state), + created_at=row.created_at.replace(tzinfo=timezone.utc), + updated_at=row.updated_at.replace(tzinfo=timezone.utc), + error=row.error, + metadata=json.loads(row.metadata_json or "{}"), + ) + class JobQueue: - def __init__(self) -> None: + """Jobs live in the catalog ``jobs`` table, so every process sharing it sees them.""" + + def __init__( + self, + catalog: CatalogRepository, + lease_store: LeaseStore, + *, + ttl: timedelta = DEFAULT_LEASE_TTL, + ) -> None: + self._catalog = catalog + self._lease_store = lease_store + self._ttl = ttl + # Guards the shared catalog session only; cross-process ownership is the + # catalog's BEGIN IMMEDIATE inside lease acquisition. self._lock = threading.RLock() - self._jobs: dict[str, Job] = {} - self._drive_owners: dict[int, str] = {} - self._changer_owner: str | None = None def create_job(self, job_type: JobType, metadata: dict[str, object]) -> Job: - job = Job(job_type=job_type, metadata=dict(metadata)) with self._lock: - self._jobs[job.id] = job - return job + row = self._catalog.create_job(JobType(job_type).value, dict(metadata)) + return _job_from_row(row) def get_job(self, job_id: str) -> Job: with self._lock: - try: - return self._jobs[job_id] - except KeyError as exc: - raise JobNotFoundError(job_id) from exc - - def _set_job(self, job: Job) -> None: - with self._lock: - self._jobs[job.id] = job + row = self._catalog.get_job(job_id) + if row is None: + raise JobNotFoundError(job_id) + self._catalog.session.refresh(row) + return _job_from_row(row) def update_job( self, job_id: str, *, state: JobState | None = None, error: str | None = None ) -> Job: - job = self.get_job(job_id) - updated = replace( - job, - state=state or job.state, - error=error, - updated_at=datetime.now(timezone.utc), - ) - self._set_job(updated) - return updated + with self._lock: + job = self.get_job(job_id) + self._catalog.update_job_state(job_id, (state or job.state).value, error) + return self.get_job(job_id) + + def _owner(self, slot: int) -> str | None: + for lease in self._lease_store.live_leases(): + if lease.drive_id == slot: + return lease.job_id + return None + + def _claim_slot( + self, slot: int, job_id: str, kind: str, busy: Callable[[str], Exception] + ) -> None: + owner = self._owner(slot) + if owner == job_id: + return + if owner is None: + lease = self._lease_store.acquire_drive_lease_at( + job_id=job_id, drive_id=slot, barcode=f"{kind}:{slot}", ttl=self._ttl + ) + if lease is not None: + return + owner = self._owner(slot) + if owner == job_id: + return + raise busy(owner or "another job") + + def _release_slot(self, slot: int, job_id: str) -> None: + mine = [ + lease.id + for lease in self._lease_store.live_leases() + if lease.drive_id == slot and lease.job_id == job_id + ] + if mine: + self._lease_store.release(mine) def claim_drive(self, drive_id: int, job_id: str) -> None: - with self._lock: - owner = self._drive_owners.get(drive_id) - if owner is not None and owner != job_id: - raise DriveOccupiedError(f"Drive {drive_id} is already owned by job {owner}") - self._drive_owners[drive_id] = job_id + if drive_id < 0: + raise ValueError(f"drive id {drive_id} out of range") + # Same barrier as DriveScheduler.acquire: a drive whose physical state is + # unknown after a crash must not be handed out (DriveUnreconciledError). + ensure_drive_reconciled(self._catalog, drive_id) + self._claim_slot( + drive_id, + job_id, + _DRIVE_KIND, + lambda owner: DriveOccupiedError(f"Drive {drive_id} is already owned by job {owner}"), + ) def release_drive(self, drive_id: int, job_id: str) -> None: - with self._lock: - if self._drive_owners.get(drive_id) == job_id: - del self._drive_owners[drive_id] + self._release_slot(drive_id, job_id) def claim_changer(self, job_id: str) -> None: - with self._lock: - if self._changer_owner is not None and self._changer_owner != job_id: - raise ChangerBusyError(f"Changer owned by job {self._changer_owner}") - self._changer_owner = job_id + self._claim_slot( + CHANGER_DRIVE_ID, + job_id, + _CHANGER_KIND, + lambda owner: ChangerBusyError(f"Changer owned by job {owner}"), + ) def release_changer(self, job_id: str) -> None: + self._release_slot(CHANGER_DRIVE_ID, job_id) + + def heartbeat(self, job_id: str) -> None: + """Renew every live drive/changer lease held by ``job_id`` for another ``ttl``.""" with self._lock: - if self._changer_owner == job_id: - self._changer_owner = None + mine = [lease.id for lease in self._lease_store.live_leases() if lease.job_id == job_id] + if mine: + self._lease_store.heartbeat(mine, self._ttl) + + def _heartbeat_loop(self, job_id: str, stop: threading.Event) -> None: + interval = max(self._ttl.total_seconds() / 3.0, 0.05) + while not stop.wait(interval): + try: + self.heartbeat(job_id) + except Exception: # noqa: BLE001 - a missed beat is logged; the next one retries + logger.warning("job %s lease heartbeat failed", job_id, exc_info=True) def run_job(self, job: Job, func: Callable[[], ResultT]) -> tuple[Job, ResultT]: self.update_job(job.id, state=JobState.RUNNING) + # Keep this job's claims alive while func runs; otherwise a holder past the + # TTL silently loses its slot and another job can lease the same drive. + stop = threading.Event() + beat = threading.Thread( + target=self._heartbeat_loop, + args=(job.id, stop), + name="job-lease-heartbeat", + daemon=True, + ) + beat.start() try: result = func() except Exception as exc: logger.warning("job %s failed", job.id, exc_info=True) self.update_job(job.id, state=JobState.FAILED, error=safe_job_error(exc)) raise + finally: + stop.set() + beat.join() completed = self.update_job(job.id, state=JobState.COMPLETED, error=None) return completed, result diff --git a/openblade/jobs/reconcile.py b/openblade/jobs/reconcile.py new file mode 100644 index 0000000..42d4064 --- /dev/null +++ b/openblade/jobs/reconcile.py @@ -0,0 +1,148 @@ +"""Drives whose physical state is unknown after a failed unmount/unload. + +``record_physical_state_unknown`` journals ``physical_state_unknown`` when an +unmount/unload fails: the drive may still hold a (possibly LTFS-mounted) tape. +Until a later ``drive_reconciled`` entry names the same drive, no job may +acquire or load into it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from openblade.catalog.repository import CatalogRepository +from openblade.domain.backends import LibraryBackend +from openblade.domain.errors import DriveUnreconciledError +from openblade.domain.models import MountState +from openblade.jobs.inventory import InventoryService + +PHYSICAL_STATE_UNKNOWN = "physical_state_unknown" +DRIVE_RECONCILED = "drive_reconciled" + + +@dataclass(frozen=True) +class PendingReconciliation: + drive_id: int + barcode: str | None + op: str + job_id: str + at: datetime + + +def _drive_of(detail: dict[str, object]) -> int | None: + drive = detail.get("drive") + if drive is None or isinstance(drive, bool): + return None + try: + return int(str(drive)) + except ValueError: + return None + + +def drives_pending_reconciliation(catalog: CatalogRepository) -> dict[int, PendingReconciliation]: + """Drives whose latest ``physical_state_unknown`` has no later ``drive_reconciled``.""" + events = catalog.journal_events(PHYSICAL_STATE_UNKNOWN) + catalog.journal_events( + DRIVE_RECONCILED + ) + # Order by autoincrement id only: every writer inserts into one table, while + # `at` is the writer's wall clock and a clock step would reorder events. + events.sort(key=lambda entry: entry.id) + pending: dict[int, PendingReconciliation] = {} + for entry in events: + detail = entry.detail + drive = _drive_of(detail) + if drive is None: + continue + if entry.event == DRIVE_RECONCILED: + pending.pop(drive, None) + continue + barcode = detail.get("barcode") + pending[drive] = PendingReconciliation( + drive_id=drive, + barcode=None if barcode is None else str(barcode), + op=str(detail.get("op", "")), + job_id=entry.job_id, + at=entry.at, + ) + return pending + + +def barcodes_pending_reconciliation(catalog: CatalogRepository) -> set[str]: + """Barcodes whose cartridge's physical state is unknown. + + Covers events journaled with ``drive=None`` (which no drive-keyed entry can + exclude): such an entry is cleared only by a later ``drive_reconciled`` + naming the same barcode. Drive-keyed pending entries contribute their barcode. + """ + events = catalog.journal_events(PHYSICAL_STATE_UNKNOWN) + catalog.journal_events( + DRIVE_RECONCILED + ) + events.sort(key=lambda entry: entry.id) + driveless: set[str] = set() + for entry in events: + barcode = entry.detail.get("barcode") + if barcode is None: + continue + if entry.event == DRIVE_RECONCILED: + driveless.discard(str(barcode)) + elif _drive_of(entry.detail) is None: + driveless.add(str(barcode)) + keyed = { + item.barcode + for item in drives_pending_reconciliation(catalog).values() + if item.barcode is not None + } + return driveless | keyed + + +def ensure_drive_reconciled( + catalog: CatalogRepository, drive_id: int, barcode: str | None = None +) -> None: + """Refuse to use ``drive_id`` (or load ``barcode``) while its physical state is unknown.""" + if barcode is not None and barcode in barcodes_pending_reconciliation(catalog): + raise DriveUnreconciledError( + f"Barcode {barcode} awaits reconciliation after a failed unmount/unload; " + "reconcile it before reuse" + ) + item = drives_pending_reconciliation(catalog).get(drive_id) + if item is not None: + raise DriveUnreconciledError( + f"Drive {drive_id} (barcode {item.barcode or 'unknown'}) awaits reconciliation " + f"after a failed {item.op or 'operation'}; reconcile it before reuse" + ) + + +class UnknownDriveError(LookupError): + """The library reports no drive with this id.""" + + +def reconcile_drive( + catalog: CatalogRepository, + library: LibraryBackend, + drive_id: int, + *, + job_id: str = "recovery", +) -> PendingReconciliation: + """Mark ``drive_id`` reconciled against a fresh inventory snapshot. + + Refuses while the drive still reports an LTFS mount: the operator must + unmount first. Returns the ``drive_reconciled`` record (``op`` is + ``drive_reconciled``; ``barcode`` is the barcode observed in the drive). + """ + snapshot = InventoryService(library).snapshot() # SAFETY_003: via service + drive = next((d for d in snapshot.drives if d.drive_id == drive_id), None) + if drive is None: + raise UnknownDriveError(f"Drive {drive_id} not found") + observed = None if drive.barcode is None else str(drive.barcode) + if drive.mount_state != MountState.UNMOUNTED: + raise DriveUnreconciledError( + f"Drive {drive_id} (barcode {observed or 'unknown'}) is still " + f"{drive.mount_state.value}; unmount it before reconciling" + ) + entry = catalog.journal( + job_id, DRIVE_RECONCILED, {"drive": drive_id, "observed_barcode": observed} + ) + return PendingReconciliation( + drive_id=drive_id, barcode=observed, op=DRIVE_RECONCILED, job_id=job_id, at=entry.at + ) diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py new file mode 100644 index 0000000..b8162bb --- /dev/null +++ b/openblade/jobs/recovery.py @@ -0,0 +1,170 @@ +"""Startup recovery: reconcile catalog jobs and drive leases after a restart. + +Report-only for physical state: this module never moves media. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta + +from openblade.catalog.repository import CatalogRepository, StagedInstance +from openblade.domain.backends import LibraryBackend +from openblade.domain.models import DriveLease +from openblade.jobs.inventory import InventoryService +from openblade.jobs.reconcile import PendingReconciliation, drives_pending_reconciliation +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL + +# Synthetic barcodes JobQueue stores on its slot-claim leases (`f"{kind}:{slot}"`, +# kinds _DRIVE_KIND / _CHANGER_KIND in jobs/queue.py). They never name a cartridge. +_QUEUE_CLAIM_PREFIXES = ("QUEUE-CLAIM:", "CHANGER:") + + +def _is_queue_claim(lease: DriveLease) -> bool: + return lease.drive_id < 0 or lease.barcode.startswith(_QUEUE_CLAIM_PREFIXES) + + +logger = logging.getLogger(__name__) + +_NON_TERMINAL_JOB_STATES = frozenset({"pending", "running"}) +_FAILED_JOB_STATES = frozenset({"failed", "failed_recoverable"}) + +INTERRUPTED_ERROR = ( + "interrupted: drive lease expired without a heartbeat; physical state unknown — " + "reconcile before retry" +) + + +@dataclass(frozen=True) +class DriveMismatch: + lease_id: str + job_id: str + drive_id: int + expected_barcode: str + observed_barcode: str | None + + +@dataclass(frozen=True) +class RecoveryReport: + interrupted_job_ids: list[str] = field(default_factory=list) + released_lease_ids: list[str] = field(default_factory=list) + mismatches: list[DriveMismatch] = field(default_factory=list) + # Uncommitted (STAGING/VERIFYING) instances per `failed_recoverable` job: what an + # operator must reconcile on tape before retrying or discarding the job. + staged_instances: dict[str, list[StagedInstance]] = field(default_factory=dict) + # `pending` jobs older than one lease TTL with no live lease. Report only. + stale_pending_job_ids: list[str] = field(default_factory=list) + # Drives whose physical state is unknown after a failed unmount/unload. Report + # only: an operator reconciles each via POST /jobs/recovery/reconcile/{drive_id}. + pending_reconciliation: list[PendingReconciliation] = field(default_factory=list) + # Live leases of FAILED jobs, deliberately NOT released: a failed job releases + # its leases in `finally` unless its physical_state_unknown could not be + # journaled (fail closed). Report only; the lease lapses at its TTL. + held_failed_lease_ids: list[str] = field(default_factory=list) + + +def _is_older_than(created_at: datetime, ttl: timedelta, now: datetime) -> bool: + # Job.created_at is written as naive UTC; tolerate an aware value too. + if created_at.tzinfo is not None: + created_at = created_at.astimezone(UTC).replace(tzinfo=None) + return created_at < now - ttl + + +def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) -> RecoveryReport: + """Fail jobs whose leases expired, release orphaned leases, report drive mismatches. + + The catalog is shared by every process (API + CLI), so "state == running" is + NOT evidence that a job is dead — it may be running in another process. The + only cross-process evidence is a lease that stopped heartbeating. Jobs that + are running without a lease are left alone. + """ + interrupted: list[str] = [] + orphaned = [] + for lease in catalog.expired_leases(): + orphaned.append(lease) + owner = catalog.get_job(lease.job_id) + if owner is not None and owner.state == "running" and owner.id not in interrupted: + catalog.update_job_state(owner.id, "failed_recoverable", INTERRUPTED_ERROR) + interrupted.append(owner.id) + catalog.journal( + owner.id, "recovered", {"lease_id": lease.id, "error": INTERRUPTED_ERROR} + ) + # Bookkeeping: a live lease whose job already reached a terminal state. + live = catalog.live_leases() + held: list[DriveLease] = [] + leased_job_ids = {lease.job_id for lease in live} + for lease in live: + owner = catalog.get_job(lease.job_id) + # A pending or running owner may belong to another live process; its + # lease is only reclaimed once it expires (handled above). + if owner is not None and owner.state in _FAILED_JOB_STATES: + held.append(lease) + elif owner is None or owner.state not in _NON_TERMINAL_JOB_STATES: + orphaned.append(lease) + catalog.release_leases([lease.id for lease in orphaned]) + + observed: dict[int, str | None] = { + drive.drive_id: None if drive.barcode is None else str(drive.barcode) + for drive in InventoryService(library).snapshot().drives # SAFETY_003: via service + } + mismatches: list[DriveMismatch] = [] + for lease in orphaned: + if _is_queue_claim(lease): + continue # a JobQueue slot claim, not a tape in a drive: nothing to compare + physical = ( + lease.physical_drive_id if lease.physical_drive_id is not None else lease.drive_id + ) + seen = observed.get(physical) + if seen != lease.barcode: + mismatch = DriveMismatch(lease.id, lease.job_id, physical, lease.barcode, seen) + logger.warning( + "recovery: drive %s expected %s, observed %s (lease %s, job %s)", + physical, + lease.barcode, + seen, + lease.id, + lease.job_id, + ) + mismatches.append(mismatch) + + staged = { + job.id: catalog.list_staged_instances(job.id) + for job in catalog.list_jobs(state="failed_recoverable") + } + # Evidence is the lease set observed before this pass released anything. + now = datetime.now(UTC).replace(tzinfo=None) + stale_pending = [ + job.id + for job in catalog.list_jobs(state="pending") + if job.id not in leased_job_ids and _is_older_than(job.created_at, DEFAULT_LEASE_TTL, now) + ] + + report = RecoveryReport( + interrupted_job_ids=interrupted, + released_lease_ids=[lease.id for lease in orphaned], + mismatches=mismatches, + staged_instances=staged, + stale_pending_job_ids=stale_pending, + pending_reconciliation=list(drives_pending_reconciliation(catalog).values()), + held_failed_lease_ids=[lease.id for lease in held], + ) + for lease in held: + logger.error( + "recovery: keeping lease %s on drive %d (failed job %s); reconcile the drive", + lease.id, + lease.drive_id, + lease.job_id, + ) + logger.info( + "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es), " + "%d staged instance(s) awaiting reconcile, %d stale pending job(s), " + "%d drive(s) awaiting reconciliation", + len(report.interrupted_job_ids), + len(report.released_lease_ids), + len(report.mismatches), + sum(len(items) for items in staged.values()), + len(stale_pending), + len(report.pending_reconciliation), + ) + return report diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index 12a3ff6..b61d1da 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -2,21 +2,36 @@ from __future__ import annotations +import logging from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath from openblade.api import aml_state -from openblade.catalog.models import Job +from openblade.catalog.models import FileInstance, FileRecord, Job from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend -from openblade.domain.errors import CartridgeOfflineError, ChecksumMismatchError -from openblade.domain.models import JobType, MountMode +from openblade.domain.errors import ( + CartridgeOfflineError, + ChecksumMismatchError, + StaleLeaseError, + safe_job_error, +) +from openblade.domain.errors import ( + FileNotFoundError as CatalogPathNotFoundError, +) +from openblade.domain.models import JobType, MountHandle, MountMode +from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue +from openblade.jobs.reconcile import ensure_drive_reconciled +from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore +from openblade.jobs.sharded_archive import record_physical_state_unknown from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType +logger = logging.getLogger(__name__) + @dataclass class RestoreRequest: @@ -89,16 +104,23 @@ def _load_if_needed( catalog: CatalogRepository, library: LibraryBackend, ltfs: LTFSBackend, - barcode: str, + handle: DriveHandle, job_id: str, ) -> tuple[int, int | None]: + barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: + ensure_drive_reconciled(catalog, drive_id, barcode) + # Already in a drive: record it as the PHYSICAL drive; the lease key + # (handle.drive_id) stays what the scheduler reserved. + if drive_id != handle.drive_id: + handle.physical_drive_id = drive_id return drive_id, None slot_id = library.find_slot_by_barcode(barcode) if slot_id is None: raise CartridgeOfflineError(f"Cartridge {barcode} is offline") - drive_id = 0 + drive_id = handle.drive_id + ensure_drive_reconciled(catalog, drive_id, barcode) execute_tape_request( catalog, library, @@ -122,8 +144,16 @@ def run_restore_job( ltfs: LTFSBackend, catalog: CatalogRepository, job_id: str, + *, + scheduler: DriveScheduler | None = None, ) -> RestoreResult: - """Restore a cataloged file from tape to a local path.""" + """Restore a cataloged file from tape to a local path. + + The drive is taken through ``scheduler`` (a lease shared with every other job + on the same store). ``None`` falls back to a job-local in-memory scheduler, + which serialises nothing across jobs -- callers that can reach the app's + lease store must pass one. + """ catalog.update_job_state(job_id, "running") record, instance = catalog.get_latest_instance_for_path(request.catalog_path) cartridge = catalog.get_cartridge(instance.barcode) @@ -135,35 +165,77 @@ def run_restore_job( return RestoreResult( job_id=job_id, source_barcode=instance.barcode, checksum_verified=False ) - drive_id, slot_id = _load_if_needed(catalog, library, ltfs, instance.barcode, job_id) - _mark_aml_drive_busy(instance.barcode, drive_id) + if scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + handles = scheduler.acquire_drives([instance.barcode]) + handle = handles[0] + catalog.journal( + job_id, "lease_acquired", {"drive_id": handle.drive_id, "barcode": handle.barcode} + ) final_dest = ( request.dest_path / PurePosixPath(request.catalog_path).name if request.dest_path.exists() and request.dest_path.is_dir() else request.dest_path ) + drive_id: int | None = None + slot_id: int | None = None + fenced_out = False try: - handle = ltfs.mount(instance.barcode, MountMode.READ_ONLY) try: - ltfs.read_file(handle, PurePosixPath(instance.tape_path), final_dest) + drive_id, slot_id = _load_if_needed(catalog, library, ltfs, handle, job_id) + scheduler.record_physical_drives(handles) + _mark_aml_drive_busy(instance.barcode, drive_id) + scheduler.verify(handle) # fencing: never mount on a stale lease + mount_handle = ltfs.mount(instance.barcode, MountMode.READ_ONLY) + try: + ltfs.read_file(mount_handle, PurePosixPath(instance.tape_path), final_dest) + finally: + scheduler.verify(handle) # fencing: before unmount + ltfs.unmount(mount_handle) + except StaleLeaseError: + fenced_out = True + raise finally: - ltfs.unmount(handle) + if not fenced_out and drive_id is not None: + try: + scheduler.verify(handle) # fencing: before unload + except StaleLeaseError: + fenced_out = True + raise + if slot_id is not None: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=instance.barcode, + drive_id=drive_id, + slot_id=slot_id, + requested_by="restore-job", + job_id=job_id, + ), + ) + _mark_aml_drive_idle(instance.barcode, drive_id, slot_id) + except StaleLeaseError as exc: + # Fenced out: leave the hardware ALONE -- whoever holds the lease now may + # have their own tape in that drive. Physical state is unknown -> reconcile. + logger.error( + "job %s lost its lease on drive %d (%s); leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + handle.physical, + handle.barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise finally: - if slot_id is not None: - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=instance.barcode, - drive_id=drive_id, - slot_id=slot_id, - requested_by="restore-job", - job_id=job_id, - ), - ) - _mark_aml_drive_idle(instance.barcode, drive_id, slot_id) + scheduler.release_drives(handles) + catalog.journal( + job_id, "lease_released", {"drive_id": handle.drive_id, "barcode": handle.barcode} + ) actual_checksum = sha256sum(final_dest) if actual_checksum != record.checksum_sha256: quarantine = final_dest.with_name(f"{final_dest.name}.quarantine") @@ -174,6 +246,228 @@ def run_restore_job( return RestoreResult(job_id=job_id, source_barcode=instance.barcode, checksum_verified=True) +@dataclass +class RestoreItemResult: + catalog_path: str + ok: bool + error: str | None = None + + +@dataclass +class RestoreBatchResult: + job_id: str + items: list[RestoreItemResult] + tape_errors: list[str] + + @property + def ok(self) -> bool: + return not self.tape_errors and all(item.ok for item in self.items) + + +def _batch_dest(request: RestoreRequest) -> Path: + if request.dest_path.exists() and request.dest_path.is_dir(): + return request.dest_path / PurePosixPath(request.catalog_path).name + return request.dest_path + + +def _restore_one_verified( + ltfs: LTFSBackend, + mount_handle: MountHandle, + request: RestoreRequest, + record: FileRecord, + instance: FileInstance, +) -> RestoreItemResult: + """Read into a temp name; only a verified checksum renames it into place.""" + final_dest = _batch_dest(request) + temp = final_dest.with_name(f".{final_dest.name}.restoring") + try: + ltfs.read_file(mount_handle, PurePosixPath(instance.tape_path), temp) + except StaleLeaseError: + raise + except Exception as exc: # per-item: one unreadable file must not abort the tape + logger.exception("batch restore read failed for %s", request.catalog_path) + temp.unlink(missing_ok=True) + return RestoreItemResult(request.catalog_path, ok=False, error=safe_job_error(exc)) + if sha256sum(temp) != record.checksum_sha256: + temp.rename(final_dest.with_name(f"{final_dest.name}.quarantine")) + return RestoreItemResult( + request.catalog_path, ok=False, error=f"Checksum mismatch for {request.catalog_path}" + ) + temp.replace(final_dest) + return RestoreItemResult(request.catalog_path, ok=True) + + +def _restore_tape_group( + barcode: str, + group: list[tuple[RestoreRequest, FileRecord, FileInstance]], + library: LibraryBackend, + ltfs: LTFSBackend, + catalog: CatalogRepository, + job_id: str, + scheduler: DriveScheduler, +) -> tuple[list[RestoreItemResult], str | None]: + """One lease, one load, one RO mount, one unmount, one unload for ``group``.""" + results: list[RestoreItemResult] = [] + tape_error: str | None = None + handles = scheduler.acquire_drives([barcode]) + handle = handles[0] + catalog.journal(job_id, "lease_acquired", {"drive_id": handle.drive_id, "barcode": barcode}) + catalog.journal(job_id, "batch_tape_started", {"barcode": barcode, "files": len(group)}) + drive_id: int | None = None + slot_id: int | None = None + fenced_out = False + unmount_failed = False + try: + try: + drive_id, slot_id = _load_if_needed(catalog, library, ltfs, handle, job_id) + scheduler.record_physical_drives(handles) + _mark_aml_drive_busy(barcode, drive_id) + scheduler.verify(handle) # fencing: never mount on a stale lease + mount_handle = ltfs.mount(barcode, MountMode.READ_ONLY) + try: + for request, record, instance in group: + results.append( + _restore_one_verified(ltfs, mount_handle, request, record, instance) + ) + finally: + scheduler.verify(handle) # fencing: before unmount + try: + ltfs.unmount(mount_handle) + except Exception: # noqa: BLE001 - logged+journaled by record_physical_state_unknown + # Still (maybe) mounted: unloading now could strand a dirty + # index. Leave the cartridge in the drive for reconcile. + unmount_failed = True + # One payload shape ({op, barcode, drive}) for every producer. A + # journal failure flags the handle; release_drives keeps its lease. + record_physical_state_unknown( + catalog, job_id, "unmount", barcode, drive_id, handles=handles + ) + tape_error = f"Unmount of {barcode} failed; left in drive for reconcile" + except StaleLeaseError: + fenced_out = True + raise + except Exception as exc: # tape-level: load/mount failed -> fail this tape's items + logger.exception("batch restore failed on tape %s", barcode) + message = safe_job_error(exc) + done = {item.catalog_path for item in results} + results.extend( + RestoreItemResult(req.catalog_path, ok=False, error=message) + for req, _, _ in group + if req.catalog_path not in done + ) + finally: + if not fenced_out and not unmount_failed and drive_id is not None: + try: + scheduler.verify(handle) # fencing: before unload + except StaleLeaseError: + fenced_out = True + raise + if slot_id is not None: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=barcode, + drive_id=drive_id, + slot_id=slot_id, + requested_by="restore-job", + job_id=job_id, + ), + ) + _mark_aml_drive_idle(barcode, drive_id, slot_id) + finally: + scheduler.release_drives(handles) + catalog.journal(job_id, "lease_released", {"drive_id": handle.drive_id, "barcode": barcode}) + catalog.journal( + job_id, + "batch_tape_finished", + {"barcode": barcode, "ok": sum(1 for r in results if r.ok), "files": len(group)}, + ) + return results, tape_error + + +def run_restore_batch( + requests: list[RestoreRequest], + library: LibraryBackend, + ltfs: LTFSBackend, + catalog: CatalogRepository, + job_id: str, + *, + scheduler: DriveScheduler | None = None, +) -> RestoreBatchResult: + """Restore many cataloged files with ONE load/mount/unload per source tape. + + Requests are grouped by the barcode of each path's latest archived + instance. A path with no archived instance (or on an exported cartridge) + is a per-item failure, not a batch abort. A lost lease aborts the batch + and leaves the hardware alone, exactly as ``run_restore_job`` does. + """ + catalog.update_job_state(job_id, "running") + items: list[RestoreItemResult] = [] + tape_errors: list[str] = [] + groups: dict[str, list[tuple[RestoreRequest, FileRecord, FileInstance]]] = {} + for request in requests: + try: + record, instance = catalog.get_latest_instance_for_path(request.catalog_path) + except CatalogPathNotFoundError: + items.append( + RestoreItemResult( + request.catalog_path, + ok=False, + error=f"No archived instance for {request.catalog_path}", + ) + ) + continue + cartridge = catalog.get_cartridge(instance.barcode) + if cartridge is not None and cartridge.state == "exported": + items.append( + RestoreItemResult( + request.catalog_path, + ok=False, + error=f"Cartridge {instance.barcode} is offline", + ) + ) + continue + if request.dry_run: + items.append(RestoreItemResult(request.catalog_path, ok=True)) + continue + groups.setdefault(instance.barcode, []).append((request, record, instance)) + if groups and scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + for barcode, group in groups.items(): + assert scheduler is not None + try: + results, tape_error = _restore_tape_group( + barcode, group, library, ltfs, catalog, job_id, scheduler + ) + except StaleLeaseError as exc: + logger.error( + "batch job %s lost its lease on %s; leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise + items.extend(results) + if tape_error is not None: + tape_errors.append(tape_error) + result = RestoreBatchResult(job_id=job_id, items=items, tape_errors=tape_errors) + if result.ok: + catalog.update_job_state(job_id, "completed") + else: + failed = sum(1 for item in items if not item.ok) + detail = f"{failed} of {len(items)} files failed to restore" + if tape_errors: + detail = f"{detail}; {'; '.join(tape_errors)}" + catalog.update_job_state(job_id, "failed", detail) + return result + + class RestoreService: def __init__( self, @@ -181,11 +475,23 @@ def __init__( ltfs: LTFSBackend, catalog: CatalogRepository, queue: JobQueue, + *, + lease_store: LeaseStore | None = None, ) -> None: self.library = library self.ltfs = ltfs self.catalog = catalog self.queue = queue + # Catalog-backed by default so this job's drives are excluded from every + # other job, in this process or another one. + self.lease_store = lease_store if lease_store is not None else CatalogLeaseStore(catalog) + + def _scheduler(self, job_id: str) -> DriveScheduler: + return DriveScheduler( + num_drives=len(InventoryService(self.library).snapshot().drives), + store=self.lease_store, + job_id=job_id, + ) def enqueue(self, catalog_path: str, destination: Path) -> Job: job = self.catalog.create_job( @@ -198,7 +504,31 @@ def enqueue(self, catalog_path: str, destination: Path) -> Job: self.ltfs, self.catalog, job.id, + scheduler=self._scheduler(job.id), ) refreshed = self.catalog.get_job(job.id) assert refreshed is not None return refreshed + + def enqueue_batch(self, requests: list[RestoreRequest]) -> tuple[Job, RestoreBatchResult]: + """One restore job covering every request; one load per source tape.""" + job = self.catalog.create_job( + JobType.RESTORE.value, + { + "batch": [ + {"catalog_path": r.catalog_path, "dest_path": str(r.dest_path)} + for r in requests + ] + }, + ) + result = run_restore_batch( + requests, + self.library, + self.ltfs, + self.catalog, + job.id, + scheduler=self._scheduler(job.id), + ) + refreshed = self.catalog.get_job(job.id) + assert refreshed is not None + return refreshed, result diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 1f13d89..0ec60e3 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -3,10 +3,24 @@ from __future__ import annotations import logging +import sys import threading -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Protocol +from uuid import uuid4 -from openblade.domain.errors import DriveBusyError +from openblade.domain.errors import ( + DriveBusyError, + DriveUnreconciledError, + JournalWriteError, + StaleLeaseError, + safe_job_error, +) +from openblade.domain.models import DriveLease + +if TYPE_CHECKING: + from openblade.catalog.repository import CatalogRepository logger = logging.getLogger(__name__) @@ -22,7 +36,14 @@ class DriveHandle: # one (e.g. the tape was already loaded elsewhere). Physical tape ops target # `physical`; the scheduler lock still tracks `drive_id`. physical_drive_id: int | None = None + # Lease identity + fencing token (docs/decisions/2026-10-09-persistent-drive-leases.md). + lease_id: str = "" + fencing_token: int = 0 _released: bool = field(default=False, init=False, repr=False) + # Set when this drive's physical_state_unknown could not be journaled: + # release_drives() then KEEPS the lease (fail closed) so no other job can + # LOAD/UNLOAD a drive that may still have LTFS mounted. + hold_unjournaled: bool = field(default=False, init=False, repr=False) @property def physical(self) -> int: @@ -30,20 +51,249 @@ def physical(self) -> int: return self.physical_drive_id if self.physical_drive_id is not None else self.drive_id +DEFAULT_LEASE_TTL = timedelta(minutes=15) + + +class LeaseStore(Protocol): + """Where drive leases live. In memory for unit tests, the catalog in production.""" + + poll_interval: float + + def acquire( + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), + ) -> list[DriveLease] | None: ... + + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + """Lease one specific drive id, or None if a live lease already holds it.""" + ... + + def unreconciled_drives(self) -> dict[int, str | None]: + """Drives awaiting reconciliation (physical state unknown) -> last barcode.""" + ... + + def unreconciled_barcodes(self) -> set[str]: + """Barcodes whose cartridge may still sit in a drive awaiting reconciliation.""" + ... + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: ... + + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: ... + + def release(self, lease_ids: list[str]) -> None: ... + + def is_live(self, lease_id: str, fencing_token: int) -> bool: ... + + def live_leases(self) -> list[DriveLease]: ... + + +class InMemoryLeaseStore: + """Process-local lease store (the pre-catalog behaviour).""" + + poll_interval = 1.0 + + def __init__(self) -> None: + # Drive -> barcode for drives whose physical state is unknown (tests set this). + self.unreconciled: dict[int, str | None] = {} + # Barcodes pending reconciliation (tests set this). + self.unreconciled_barcode_set: set[str] = set() + self._lock = threading.Lock() + self._leases: dict[str, DriveLease] = {} + self._last_token = 0 + + def acquire( + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), + ) -> list[DriveLease] | None: + with self._lock: + now = _now() + busy = {lease.drive_id for lease in self._leases.values() if _live(lease, now)} + free = [d for d in range(num_drives) if d not in busy and d not in exclude] + if len(free) < len(barcodes): + return None + acquired: list[DriveLease] = [] + for drive_id, barcode in zip(free, barcodes, strict=False): + self._last_token += 1 + lease = DriveLease( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + physical_drive_id=None, + fencing_token=self._last_token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + self._leases[lease.id] = lease + acquired.append(lease) + return acquired + + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + with self._lock: + now = _now() + if any( + lease.drive_id == drive_id and _live(lease, now) for lease in self._leases.values() + ): + return None + self._last_token += 1 + lease = DriveLease( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + physical_drive_id=None, + fencing_token=self._last_token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + self._leases[lease.id] = lease + return lease + + def unreconciled_drives(self) -> dict[int, str | None]: + return dict(self.unreconciled) + + def unreconciled_barcodes(self) -> set[str]: + return set(self.unreconciled_barcode_set) + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: + with self._lock: + now = _now() + for lease_id in lease_ids: + lease = self._leases.get(lease_id) + if lease is not None and _live(lease, now): + self._leases[lease_id] = replace(lease, heartbeat_at=now, expires_at=now + ttl) + + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + with self._lock: + lease = self._leases.get(lease_id) + if lease is not None: + self._leases[lease_id] = replace(lease, physical_drive_id=physical_drive_id) + + def release(self, lease_ids: list[str]) -> None: + with self._lock: + now = _now() + for lease_id in lease_ids: + lease = self._leases.get(lease_id) + if lease is not None and lease.released_at is None: + self._leases[lease_id] = replace(lease, released_at=now) + + def is_live(self, lease_id: str, fencing_token: int) -> bool: + with self._lock: + lease = self._leases.get(lease_id) + return ( + lease is not None and lease.fencing_token == fencing_token and _live(lease, _now()) + ) + + def live_leases(self) -> list[DriveLease]: + with self._lock: + now = _now() + return [lease for lease in self._leases.values() if _live(lease, now)] + + +class CatalogLeaseStore: + """Lease store shared by every process using the same catalog.""" + + poll_interval = 0.25 + + def __init__(self, repository: CatalogRepository) -> None: + self._repo = repository + + def acquire( + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), + ) -> list[DriveLease] | None: + return self._repo.acquire_drive_leases( + job_id=job_id, barcodes=barcodes, num_drives=num_drives, ttl=ttl, exclude=exclude + ) + + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + return self._repo.acquire_drive_lease_at( + job_id=job_id, drive_id=drive_id, barcode=barcode, ttl=ttl + ) + + def unreconciled_drives(self) -> dict[int, str | None]: + from openblade.jobs.reconcile import drives_pending_reconciliation # local: no cycle + + return { + drive_id: item.barcode + for drive_id, item in drives_pending_reconciliation(self._repo).items() + } + + def unreconciled_barcodes(self) -> set[str]: + from openblade.jobs.reconcile import barcodes_pending_reconciliation # local: no cycle + + return barcodes_pending_reconciliation(self._repo) + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: + self._repo.heartbeat_leases(lease_ids, ttl) + + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + self._repo.set_lease_physical_drive(lease_id, physical_drive_id) + + def release(self, lease_ids: list[str]) -> None: + self._repo.release_leases(lease_ids) + + def is_live(self, lease_id: str, fencing_token: int) -> bool: + return self._repo.lease_is_live(lease_id, fencing_token) + + def live_leases(self) -> list[DriveLease]: + return self._repo.live_leases() + + class DriveScheduler: """ Atomically allocates 1..N drives for parallel tape operations. Rules: - - A drive can be held by at most one job at a time. + - A drive can be held by at most one job at a time (across every scheduler + sharing the same lease store). - acquire_drives() waits (with timeout) until all requested drives are free. - - release_drives() marks drives as free and notifies waiting jobs. + - release_drives() releases the leases and wakes local waiters. + - verify() raises StaleLeaseError once a lease is no longer live (fencing). """ - def __init__(self, num_drives: int) -> None: + def __init__( + self, + num_drives: int, + *, + store: LeaseStore | None = None, + job_id: str = "local", + ttl: timedelta = DEFAULT_LEASE_TTL, + ) -> None: self._num_drives = num_drives + self._store: LeaseStore = store if store is not None else InMemoryLeaseStore() + self._job_id = job_id + self._ttl = ttl self._lock = threading.Condition(threading.Lock()) - self._held: dict[int, str | None] = {i: None for i in range(num_drives)} + # Leases are kept alive by a daemon thread for as long as any handle is + # held, so a single long tape write (real LTO: minutes to hours per file) + # cannot outlive its own TTL and be fenced out after the data is down. + self._held: list[DriveHandle] = [] + self._stop_heartbeat = threading.Event() + self._heartbeat_thread: threading.Thread | None = None @property def num_drives(self) -> int: @@ -51,8 +301,8 @@ def num_drives(self) -> int: def available_count(self) -> int: """Number of currently free drives.""" - with self._lock: - return sum(1 for value in self._held.values() if value is None) + held = {lease.drive_id for lease in self._store.live_leases()} + return sum(1 for drive_id in range(self._num_drives) if drive_id not in held) def acquire_drives( self, @@ -77,39 +327,150 @@ def acquire_drives( with self._lock: deadline = _monotonic() + timeout while True: - free_drives = [drive_id for drive_id, value in self._held.items() if value is None] - if len(free_drives) >= len(barcodes): + # A drive whose physical state is unknown (failed unmount/unload) + # may still hold a mounted tape: never a candidate until reconciled. + # A cartridge awaiting reconciliation may still be mounted in some + # drive: refuse it outright rather than move it. + pending = sorted(set(barcodes) & self._store.unreconciled_barcodes()) + if pending: + raise DriveUnreconciledError( + f"Tape(s) {', '.join(pending)} awaiting reconciliation after a " + "failed unmount/unload; reconcile before reuse" + ) + unreconciled = self._store.unreconciled_drives() + excluded = frozenset(d for d in unreconciled if 0 <= d < self._num_drives) + if self._num_drives - len(excluded) < len(barcodes): + named = ", ".join( + f"drive {d} (barcode {unreconciled[d] or 'unknown'})" + for d in sorted(excluded) + ) + raise DriveUnreconciledError( + f"Not enough reconciled drives for {len(barcodes)} tape(s): {named} " + "awaiting reconciliation after a failed unmount/unload" + ) + leases = self._store.acquire( + job_id=self._job_id, + barcodes=barcodes, + num_drives=self._num_drives, + ttl=self._ttl, + exclude=excluded, + ) + if leases is not None: break remaining = deadline - _monotonic() if remaining <= 0: raise DriveBusyError(f"Timed out waiting for {len(barcodes)} free drives") - self._lock.wait(timeout=min(remaining, 1.0)) + self._lock.wait(timeout=min(remaining, self._store.poll_interval)) + + handles: list[DriveHandle] = [] + for lease in leases: + handles.append( + DriveHandle( + drive_id=lease.drive_id, + barcode=lease.barcode, + lease_id=lease.id, + fencing_token=lease.fencing_token, + ) + ) + logger.info( + "Allocated drive %d for barcode %s (token %d)", + lease.drive_id, + lease.barcode, + lease.fencing_token, + ) + with self._lock: + self._held.extend(handles) + self._ensure_heartbeat_thread() + return handles - handles: list[DriveHandle] = [] - free_iter = iter(free_drives) - for barcode in barcodes: - drive_id = next(free_iter) - self._held[drive_id] = barcode - handles.append(DriveHandle(drive_id=drive_id, barcode=barcode)) - logger.info("Allocated drive %d for barcode %s", drive_id, barcode) + def _ensure_heartbeat_thread(self) -> None: + # Caller holds self._lock. + if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive(): + return + self._stop_heartbeat.clear() + self._heartbeat_thread = threading.Thread( + target=self._heartbeat_loop, name="drive-lease-heartbeat", daemon=True + ) + self._heartbeat_thread.start() - return handles + def _heartbeat_loop(self) -> None: + interval = max(self._ttl.total_seconds() / 3.0, 0.05) + while not self._stop_heartbeat.wait(interval): + with self._lock: + handles = [handle for handle in self._held if not handle._released] + if not handles: + return + try: + self.heartbeat(handles) + except Exception: # noqa: BLE001 - a missed beat is logged; the next verify() fences + logger.warning("drive lease heartbeat failed", exc_info=True) def release_drives(self, handles: list[DriveHandle]) -> None: """Release drives and notify waiting jobs.""" + for handle in handles: + if handle.hold_unjournaled and not handle._released: + logger.error( + "Keeping lease on drive %d (%s): physical_state_unknown was not " + "journaled; reconcile the drive before reuse", + handle.drive_id, + handle.barcode, + ) + pending = [h for h in handles if not h._released and not h.hold_unjournaled] + self._store.release([handle.lease_id for handle in pending]) with self._lock: - for handle in handles: - if handle._released: - continue - self._held[handle.drive_id] = None + for handle in pending: handle._released = True logger.info("Released drive %d (was %s)", handle.drive_id, handle.barcode) + self._held = [handle for handle in self._held if not handle._released] + if not self._held: + self._stop_heartbeat.set() self._lock.notify_all() + kept = [h.drive_id for h in handles if h.hold_unjournaled and not h._released] + if kept: + message = f"physical_state_unknown not journaled for drive(s) {kept}; leases kept" + # Called from a `finally` that is unwinding the lane's real failure: + # this raise replaces it (it survives only as __context__), so log it + # and name it in the message the job records. + original = sys.exception() + if isinstance(original, Exception) and not isinstance(original, JournalWriteError): + logger.error("Lane failed before lease release", exc_info=original) + message += f"; original failure: {safe_job_error(original)}" + # Raised only after every other drive was released (collect, then raise). + raise JournalWriteError(message) + + def verify(self, handle: DriveHandle) -> None: + """Raise StaleLeaseError unless the handle's lease is still live with its token.""" + if handle._released or not self._store.is_live(handle.lease_id, handle.fencing_token): + raise StaleLeaseError( + f"Lease on drive {handle.drive_id} for {handle.barcode} is no longer live" + ) + + def heartbeat(self, handles: list[DriveHandle]) -> None: + """Extend the leases behind these handles by the scheduler's ttl.""" + self._store.heartbeat([handle.lease_id for handle in handles], self._ttl) + + def record_physical_drives(self, handles: list[DriveHandle]) -> None: + """Persist where each cartridge actually ended up (when it differs from the + reserved drive) so restart recovery reconciles the right drive.""" + for handle in handles: + if handle.physical_drive_id is not None: + self._store.set_physical_drive(handle.lease_id, handle.physical_drive_id) def status(self) -> dict[int, str | None]: """Return copy of drive allocation status.""" - with self._lock: - return dict(self._held) + status: dict[int, str | None] = {i: None for i in range(self._num_drives)} + for lease in self._store.live_leases(): + if lease.drive_id in status: + status[lease.drive_id] = lease.barcode + return status + + +def _now() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +def _live(lease: DriveLease, now: datetime) -> bool: + return lease.released_at is None and lease.expires_at > now def _monotonic() -> float: diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 9ee07b8..a8e9029 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -7,6 +7,7 @@ import os import shutil import uuid +from collections.abc import Sequence from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -14,8 +15,15 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.capacity import has_room_for -from openblade.domain.errors import TapeFullError, safe_job_error +from openblade.domain.errors import ( + JournalWriteError, + StaleLeaseError, + TapeFullError, + safe_job_error, +) from openblade.domain.models import MountHandle, MountMode +from openblade.jobs.inventory import InventoryService +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import ( DEFAULT_BLOCK_SIZE, @@ -214,7 +222,7 @@ def run_sharded_archive( files_archived += 1 except Exception as exc: # noqa: BLE001 logger.exception("Shard archive failed for %s", source_file) - errors.append(safe_job_error(exc)) + errors.append(_job_error(exc)) else: files_archived, bytes_archived = _archive_stripe( files, @@ -274,16 +282,24 @@ def _clean_unmount_and_unload( caller does NOT mark the batch archived. """ failures: list[str] = [] + still_mounted: set[str] = set() for barcode, mount in list(mounts.items()): try: ltfs.unmount(mount) mounts.pop(barcode, None) except Exception as exc: # noqa: BLE001 - aggregated and re-raised below - failures.append(f"unmount {barcode}: {exc}") + logger.exception("job %s: unmount of %s failed", job_id, barcode) + failures.append(f"unmount {barcode}: {type(exc).__name__}") + still_mounted.add(barcode) for handle in handles: slot_id = loaded_slots.get(handle.physical) if slot_id is None: continue + if handle.barcode in still_mounted: + # Never unload while LTFS is mounted or dirty: a failed unmount leaves + # the drive in an unknown state, which is reported, not forced. + failures.append(f"unload {handle.barcode}: skipped, unmount failed") + continue try: # raise_on_failed=True is essential: execute_tape_request otherwise # SWALLOWS an unload failure (returns a FAILED record) and the batch would @@ -304,7 +320,8 @@ def _clean_unmount_and_unload( ) loaded_slots.pop(handle.physical, None) except Exception as exc: # noqa: BLE001 - aggregated and re-raised below - failures.append(f"unload {handle.barcode}: {exc}") + logger.exception("job %s: unload of %s failed", job_id, handle.barcode) + failures.append(f"unload {handle.barcode}: {type(exc).__name__}") if failures: raise TapePhysicalStateError( "shards written but physical state is unknown (reconcile required): " @@ -312,6 +329,178 @@ def _clean_unmount_and_unload( ) +def _log_fenced_out(handles: list[DriveHandle], job_id: str) -> None: + for handle in handles: + logger.error( + "job %s lost its lease on drive %d (%s); leaving the drive untouched — " + "physical state unknown, reconcile before reuse", + job_id, + handle.physical, + handle.barcode, + ) + + +PHYSICAL_STATE_UNKNOWN = "physical state unknown — reconcile before reuse" + + +def _job_error(exc: Exception) -> str: + """jobs.error text: the typed physical-state message, else safe_job_error.""" + if isinstance(exc, TapePhysicalStateError): + return PHYSICAL_STATE_UNKNOWN + return safe_job_error(exc) + + +def _journal_failure( + catalog: CatalogRepository, job_id: str, event: str, detail: dict[str, object] +) -> None: + """Journal on an error path without letting a broken session mask the real error.""" + try: + catalog.journal(job_id, event, detail) + except Exception: # noqa: BLE001 - evidence only; the original exception is what the caller reports + logger.exception("job %s: could not journal %s; rolling the session back", job_id, event) + with suppress(Exception): + catalog.session.rollback() + + +def record_physical_state_unknown( + catalog: CatalogRepository, + job_id: str, + op: str, + barcode: str, + drive: int | None, + *, + handles: Sequence[DriveHandle], +) -> None: + """Log + journal a cleanup op that raised; the drive must be reconciled before reuse. + + Fail closed: the row is written via ``journal_durable`` (fresh session, retried). + If that still fails, the matching handle is flagged ``hold_unjournaled`` so + ``release_drives`` keeps its lease and then raises ``JournalWriteError`` -- + after every other drive was cleaned and released. With no matching handle + there is no lease to keep, so the error is raised here. + """ + logger.exception( + "job %s: %s of %s (drive %s) failed during cleanup; %s", + job_id, + op, + barcode, + drive, + PHYSICAL_STATE_UNKNOWN, + ) + try: + catalog.journal_durable( + job_id, "physical_state_unknown", {"op": op, "barcode": barcode, "drive": drive} + ) + except Exception: # noqa: BLE001 - any journal failure must fail closed, never pass + matched = [ + h + for h in handles + if h.barcode == barcode or (drive is not None and drive in (h.drive_id, h.physical)) + ] + logger.exception( + "job %s: could not journal physical_state_unknown for %s (drive %s); keeping its lease", + job_id, + barcode, + drive, + ) + if not matched: + raise JournalWriteError( + f"physical_state_unknown for {barcode} could not be journaled" + ) from None + for handle in matched: + handle.hold_unjournaled = True + + +def _best_effort_unmount_and_unload( + catalog: CatalogRepository, + library: LibraryBackend, + ltfs: LTFSBackend, + mounts: dict[str, MountHandle], + handles: list[DriveHandle], + loaded_slots: dict[int, int | None], + job_id: str, + errors: list[str] | None, +) -> None: + """Error-path cleanup for a job that still owns its drives: unmount, then unload. + + Every op is attempted even if an earlier one raised. A failure is logged and + journaled as ``physical_state_unknown``; it becomes the job error only when + ``errors`` is empty (the original lane error wins). ``errors=None`` means an + exception is already propagating and will become the job error. + """ + drive_by_barcode = {handle.barcode: handle.physical for handle in handles} + failed = False + still_mounted: set[str] = set() + for barcode, mount in mounts.items(): + try: + ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues + failed = True + still_mounted.add(barcode) + record_physical_state_unknown( + catalog, + job_id, + "unmount", + barcode, + drive_by_barcode.get(barcode), + handles=handles, + ) + for handle in handles: + slot_id = loaded_slots.get(handle.physical) + if slot_id is None: + continue + if handle.barcode in still_mounted: + # Never unload while LTFS is mounted or dirty; the unmount failure above + # is already journaled as physical_state_unknown for this drive. + continue + try: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=handle.barcode, + drive_id=handle.physical, + slot_id=slot_id, + requested_by="sharded-archive", + job_id=job_id, + ), + ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues + failed = True + record_physical_state_unknown( + catalog, job_id, "unload", handle.barcode, handle.physical, handles=handles + ) + if failed and errors is not None and not errors: + errors.append(PHYSICAL_STATE_UNKNOWN) + + +def _mark_verifying(catalog: CatalogRepository, job_id: str, staged_ids: list[str]) -> None: + catalog.journal(job_id, "verify_started", {"instances": len(staged_ids)}) + catalog.mark_instances_verifying(staged_ids) + catalog.journal(job_id, "verify_finished", {"instances": len(staged_ids)}) + + +def _commit_staged( + catalog: CatalogRepository, + library: LibraryBackend, + job_id: str, + handles: list[DriveHandle], + staged_ids: list[str], +) -> None: + """The atomic commit: inventory reconciled, then ONE mark_instances_archived.""" + inventory = InventoryService(library).snapshot() # SAFETY_003: via service + seen = {str(slot.barcode) for slot in inventory.slots if slot.barcode is not None} + seen |= {str(drive.barcode) for drive in inventory.drives if drive.barcode is not None} + missing = sorted({handle.barcode for handle in handles} - seen) + if missing: + logger.error("job %s: barcodes %s not in any slot or drive after unload", job_id, missing) + raise TapePhysicalStateError(f"inventory does not show {', '.join(missing)}") + catalog.mark_instances_archived(staged_ids) + catalog.journal(job_id, "committed", {"instances": len(staged_ids)}) + + def _archive_stripe( files: list[Path], request: ShardedArchiveRequest, @@ -342,85 +531,107 @@ def _archive_stripe( for batch in batches: batch_barcodes = list(dict.fromkeys(barcode for _, barcode in batch)) handles = scheduler.acquire_drives(batch_barcodes) + catalog.journal(job_id, "lease_acquired", {"barcodes": batch_barcodes}) mounts: dict[str, MountHandle] = {} loaded_slots: dict[int, int | None] = {} - # (main_instance_id, shard_instance_id, size_bytes) for shards written+verified - # this batch. Nothing here is marked archived until the WHOLE batch has verified - # and every tape has cleanly unmounted — the atomic commit boundary. - staged: list[tuple[str, str, int]] = [] + # Instances are created STAGING before their first write and stay invisible + # until the WHOLE batch has verified, every tape cleanly unmounted and the + # inventory reconciled — then ONE mark_instances_archived (atomic commit). + staged_ids: list[str] = [] + batch_bytes = 0 + fenced_out = False try: for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: + scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) def _write_one( source_file: Path, barcode: str, mount: MountHandle, - ) -> tuple[Path, str, str, str, int]: - tape_path = _stripe_tape_path(source_file, request.source_path) + tape_path: str, + checksum: str, + ) -> None: _require_lane_room(ltfs, barcode, source_file.stat().st_size) - checksum = compute_checksum(source_file) ltfs.write_file(mount, source_file, PurePosixPath(tape_path)) stat = ltfs.stat(mount, PurePosixPath(tape_path)) if stat.checksum_sha256 != checksum: raise ValueError(f"Checksum mismatch: {source_file.name}") - return source_file, barcode, tape_path, checksum, source_file.stat().st_size + # Stage every instance (main thread: the catalog session is not + # thread-safe) BEFORE the first write, so a crash mid-write leaves + # STAGING rows that recovery can find, never ARCHIVED ones. + planned: list[tuple[Path, str, str, str]] = [] + for source_file, barcode in batch: + tape_path = _stripe_tape_path(source_file, request.source_path) + checksum = compute_checksum(source_file) + size_bytes = source_file.stat().st_size + file_record = catalog.create_file_record( + path=str(source_file), + size_bytes=size_bytes, + checksum=checksum, + vg_id=vg_id, + shard_count=1, + shard_index=None, + block_size=None, + shard_profile=_archive_profile(request.mode), + parent_id=None, + ) + instance = catalog.create_staged_instance( + job_id, file_record.id, barcode, tape_path + ) + shard_record = catalog.create_file_record( + path=_shard_record_path(source_file, 0), + size_bytes=size_bytes, + checksum=checksum, + vg_id=vg_id, + shard_count=1, + shard_index=0, + block_size=None, + shard_profile=_archive_profile(request.mode), + parent_id=file_record.id, + ) + shard_instance = catalog.create_staged_instance( + job_id, shard_record.id, barcode, tape_path + ) + staged_ids.extend((instance.id, shard_instance.id)) + batch_bytes += size_bytes + planned.append((source_file, barcode, tape_path, checksum)) + shard_group_ids.append(str(uuid.uuid4())) + + for handle in handles: + scheduler.verify(handle) # fencing: before the shard write batch with concurrent.futures.ThreadPoolExecutor(max_workers=len(batch)) as pool: futures = [ - pool.submit(_write_one, source_file, barcode, mounts[barcode]) - for source_file, barcode in batch + pool.submit(_write_one, source_file, barcode, mounts[barcode], path, csum) + for source_file, barcode, path, csum in planned ] for future in concurrent.futures.as_completed(futures): - source_file, barcode, tape_path, checksum, size_bytes = future.result() - file_record = catalog.create_file_record( - path=str(source_file), - size_bytes=size_bytes, - checksum=checksum, - vg_id=vg_id, - shard_count=1, - shard_index=None, - block_size=None, - shard_profile=_archive_profile(request.mode), - parent_id=None, - ) - instance = catalog.create_file_instance( - file_record_id=file_record.id, - barcode=barcode, - tape_path=tape_path, - ) - shard_record = catalog.create_file_record( - path=_shard_record_path(source_file, 0), - size_bytes=size_bytes, - checksum=checksum, - vg_id=vg_id, - shard_count=1, - shard_index=0, - block_size=None, - shard_profile=_archive_profile(request.mode), - parent_id=file_record.id, - ) - shard_instance = catalog.create_file_instance( - file_record_id=shard_record.id, - barcode=barcode, - tape_path=tape_path, - ) - # Created PENDING; not archived yet. - staged.append((instance.id, shard_instance.id, size_bytes)) - shard_group_ids.append(str(uuid.uuid4())) + future.result() + # Every lane wrote and read back (stat checksum) its shard. + _mark_verifying(catalog, job_id, staged_ids) # Every shard in the batch wrote and checksum-verified. Cleanly unmount and # unload BEFORE commit; a failed unmount raises and blocks the commit. + scheduler.heartbeat(handles) + for handle in handles: + scheduler.verify(handle) # fencing: before unmount/unload + commit _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) - - # COMMIT: only now is the batch durable. - for instance_id, shard_instance_id, size_bytes in staged: - catalog.mark_instance_archived(instance_id) - catalog.mark_instance_archived(shard_instance_id) - files_archived += 1 - bytes_archived += size_bytes + _commit_staged(catalog, library, job_id, handles, staged_ids) + files_archived += len(planned) + bytes_archived += batch_bytes + except StaleLeaseError: + # Fenced out: abort the whole job and leave the hardware ALONE. The + # drive may already be loaded + mounted by whoever holds the lease now, + # so unmount/unload here could eject their tape mid-write. Physical + # state is unknown -> reconcile (recovery report / inventory). + fenced_out = True + _journal_failure(catalog, job_id, "fenced_out", {"barcodes": batch_barcodes}) + raise except Exception as exc: # noqa: BLE001 # Staged instances remain PENDING -> not exposed as archived; resumable. # This branch had no logging at all, so a batch that failed left no @@ -432,29 +643,27 @@ def _write_one( exc, exc_info=True, ) - errors.append(safe_job_error(exc)) + _journal_failure(catalog, job_id, "failed", {"error": type(exc).__name__}) + errors.append(_job_error(exc)) finally: - for mount in mounts.values(): - with suppress(Exception): - ltfs.unmount(mount) - for handle in handles: - slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=handle.barcode, - drive_id=handle.physical, - slot_id=slot_id, - requested_by="sharded-archive", - job_id=job_id, - ), - ) - scheduler.release_drives(handles) + if fenced_out: + _log_fenced_out(handles, job_id) + else: + _best_effort_unmount_and_unload( + catalog, library, ltfs, mounts, handles, loaded_slots, job_id, errors + ) + try: + scheduler.release_drives(handles) + except JournalWriteError as release_exc: + # The lane error was caught above, so it is not in flight here and + # would be lost with `errors`: carry it into the raised message. + if not errors: + raise + summary = _summarize_errors(errors) + logger.error("Lane failed before lease release: %s", summary) + raise JournalWriteError( + f"{release_exc}; original failure: {summary}" + ) from release_exc return files_archived, bytes_archived @@ -479,15 +688,20 @@ def _archive_block_stripe( ) shard_group_ids.append(plan.shard_group_id) handles = scheduler.acquire_drives(request.lane_barcodes) + catalog.journal(job_id, "lease_acquired", {"barcodes": list(request.lane_barcodes)}) mounts: dict[str, MountHandle] = {} loaded_slots: dict[int, int | None] = {} shard_dir = scratch_dir / plan.shard_group_id shard_dir.mkdir(parents=True, exist_ok=True) + fenced_out = False try: for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: + scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) shard_tmp_files: list[Path] = [Path()] * len(plan.shards) @@ -507,28 +721,7 @@ def _archive_block_stripe( shard_index = future_map[future] shard_tmp_files[shard_index] = future.result() - def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: - _require_lane_room(ltfs, spec.barcode, shard_tmp.stat().st_size) - checksum = compute_checksum(shard_tmp) - mount = mounts[spec.barcode] - ltfs.write_file(mount, shard_tmp, PurePosixPath(spec.tape_path)) - stat = ltfs.stat(mount, PurePosixPath(spec.tape_path)) - if stat.checksum_sha256 != checksum: - raise ValueError(f"Shard {spec.shard_index} checksum mismatch") - return spec.shard_index, checksum, shard_tmp.stat().st_size - - shard_checksums: list[str] = [""] * len(plan.shards) - shard_sizes: list[int] = [0] * len(plan.shards) - with concurrent.futures.ThreadPoolExecutor(max_workers=len(plan.shards)) as pool: - futures = [ - pool.submit(_write_shard, spec, shard_tmp_files[spec.shard_index]) - for spec in plan.shards - ] - for shard_future in concurrent.futures.as_completed(futures): - shard_index, checksum, shard_size = shard_future.result() - shard_checksums[shard_index] = checksum - shard_sizes[shard_index] = shard_size - + shard_checksums = [compute_checksum(path) for path in shard_tmp_files] file_record = catalog.create_file_record( path=str(source_file), size_bytes=plan.file_size, @@ -540,18 +733,17 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: shard_profile=_archive_profile(request.mode), parent_id=None, ) - # All shards verified above. Create records/instances PENDING; do not mark - # archived until every tape has cleanly unmounted (atomic commit boundary). + # Stage records/instances STAGING before the first shard write (main thread: + # the catalog session is not thread-safe). Nothing is ARCHIVED until every + # lane verified, every tape cleanly unmounted and inventory reconciled. staged: list[str] = [] for spec in plan.shards: - instance = catalog.create_file_instance( - file_record_id=file_record.id, - barcode=spec.barcode, - tape_path=spec.tape_path, + instance = catalog.create_staged_instance( + job_id, file_record.id, spec.barcode, spec.tape_path ) shard_record = catalog.create_file_record( path=_shard_record_path(source_file, spec.shard_index), - size_bytes=shard_sizes[spec.shard_index], + size_bytes=shard_tmp_files[spec.shard_index].stat().st_size, checksum=shard_checksums[spec.shard_index], vg_id=vg_id, shard_count=len(plan.shards), @@ -560,42 +752,58 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: shard_profile=_archive_profile(request.mode), parent_id=file_record.id, ) - shard_instance = catalog.create_file_instance( - file_record_id=shard_record.id, - barcode=spec.barcode, - tape_path=spec.tape_path, + shard_instance = catalog.create_staged_instance( + job_id, shard_record.id, spec.barcode, spec.tape_path ) staged.extend((instance.id, shard_instance.id)) + def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: + _require_lane_room(ltfs, spec.barcode, shard_tmp.stat().st_size) + checksum = shard_checksums[spec.shard_index] + mount = mounts[spec.barcode] + ltfs.write_file(mount, shard_tmp, PurePosixPath(spec.tape_path)) + stat = ltfs.stat(mount, PurePosixPath(spec.tape_path)) + if stat.checksum_sha256 != checksum: + raise ValueError(f"Shard {spec.shard_index} checksum mismatch") + return spec.shard_index, checksum, shard_tmp.stat().st_size + + for handle in handles: + scheduler.verify(handle) # fencing: before the shard write batch + with concurrent.futures.ThreadPoolExecutor(max_workers=len(plan.shards)) as pool: + futures = [ + pool.submit(_write_shard, spec, shard_tmp_files[spec.shard_index]) + for spec in plan.shards + ] + for shard_future in concurrent.futures.as_completed(futures): + shard_future.result() + # Every lane wrote and read back (stat checksum) its shard. + _mark_verifying(catalog, job_id, staged) + + scheduler.heartbeat(handles) + for handle in handles: + scheduler.verify(handle) # fencing: before unmount/unload + commit # Clean unmount/unload BEFORE commit; a failed unmount raises and blocks it. _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) - - # COMMIT: block-striped file is durable only when all shards are down. - for instance_id in staged: - catalog.mark_instance_archived(instance_id) + _commit_staged(catalog, library, job_id, handles, staged) + except StaleLeaseError: + # Fenced out: abort and leave the hardware alone (see _archive_stripe). + fenced_out = True + _journal_failure(catalog, job_id, "fenced_out", {"barcodes": list(request.lane_barcodes)}) + raise + except Exception as exc: + _journal_failure(catalog, job_id, "failed", {"error": type(exc).__name__}) + raise finally: - for mount in mounts.values(): - with suppress(Exception): - ltfs.unmount(mount) - for handle in handles: - slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=handle.barcode, - drive_id=handle.physical, - slot_id=slot_id, - requested_by="sharded-archive", - job_id=job_id, - ), - ) - scheduler.release_drives(handles) - shutil.rmtree(shard_dir, ignore_errors=True) + if fenced_out: + _log_fenced_out(handles, job_id) + else: + _best_effort_unmount_and_unload( + catalog, library, ltfs, mounts, handles, loaded_slots, job_id, None + ) + try: + scheduler.release_drives(handles) # may raise JournalWriteError + finally: + shutil.rmtree(shard_dir, ignore_errors=True) def _load_barcode( @@ -607,6 +815,7 @@ def _load_barcode( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: + ensure_drive_reconciled(catalog, loaded_drive_id, handle.barcode) # Cartridge is already in a drive. Record it as the PHYSICAL drive without # touching the scheduler lock key (handle.drive_id) — mutating that key would # leak the reserved drive and free a drive the scheduler never held. @@ -625,6 +834,7 @@ def _load_barcode( ) if slot_id is None: raise ValueError(f"Barcode {handle.barcode} not found in any slot") + ensure_drive_reconciled(catalog, handle.drive_id, handle.barcode) execute_tape_request( catalog, library, diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index 9cf4786..0c99040 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -6,7 +6,6 @@ import logging import shutil import uuid -from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -15,8 +14,10 @@ from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.errors import CartridgeOfflineError, ChecksumMismatchError, FileNotFoundError from openblade.domain.models import MountMode +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import DEFAULT_BLOCK_SIZE, compute_checksum, reassemble_block_stripe +from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, record_physical_state_unknown from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -135,6 +136,21 @@ def run_sharded_restore( shutil.rmtree(scratch_dir, ignore_errors=True) +def _cleanup_failed( + catalog: CatalogRepository, + job_id: str, + op: str, + barcode: str, + drive: int | None, + handles: list[DriveHandle], +) -> None: + """Journal a failed cleanup op; it is the job error only if none exists yet.""" + record_physical_state_unknown(catalog, job_id, op, barcode, drive, handles=handles) + job = catalog.get_job(job_id) + if job is not None and not job.error: + catalog.update_job_state(job_id, "failed", error=PHYSICAL_STATE_UNKNOWN) + + def _restore_single( instance: Any, file_record: Any, @@ -157,6 +173,8 @@ def _restore_single( drive_id = handle.drive_id slot_id: int | None = None + cleanup_failed = False + still_mounted = False try: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) mount = ltfs.mount(instance.barcode, MountMode.READ_ONLY) @@ -175,10 +193,17 @@ def _restore_single( raise ChecksumMismatchError(error) shutil.copy2(tmp_dest, request.dest_path) finally: - ltfs.unmount(mount) + try: + ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + still_mounted = True + _cleanup_failed(catalog, job_id, "unmount", instance.barcode, drive_id, handles) finally: - if slot_id is not None: - with suppress(Exception): + # Never unload while LTFS is mounted or dirty: a failed unmount is + # journaled above and the drive is left for reconciliation. + if slot_id is not None and not still_mounted: + try: execute_tape_request( catalog, library, @@ -192,9 +217,13 @@ def _restore_single( job_id=job_id, ), ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + _cleanup_failed(catalog, job_id, "unload", instance.barcode, drive_id, handles) scheduler.release_drives(handles) - catalog.update_job_state(job_id, "completed") + if not cleanup_failed: + catalog.update_job_state(job_id, "completed") return ShardedRestoreResult( job_id=job_id, source_barcodes=[instance.barcode], @@ -228,10 +257,13 @@ def _restore_sharded( mounts: dict[str, Any] = {} loaded_slots: dict[int, int | None] = {} + cleanup_failed = False try: for handle in handles: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_ONLY) def _read_shard(index: int, instance: Any) -> Path: @@ -262,13 +294,22 @@ def _read_shard(index: int, instance: Any) -> Path: catalog.update_job_state(job_id, "failed", error=error) raise ChecksumMismatchError(error) finally: - for mount in mounts.values(): - with suppress(Exception): + drive_by_barcode = {handle.barcode: handle.physical for handle in handles} + still_mounted_barcodes: set[str] = set() + for barcode, mount in mounts.items(): + try: ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + still_mounted_barcodes.add(barcode) + _cleanup_failed( + catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode), handles + ) for handle in handles: slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): + # Never unload while LTFS is mounted or dirty (failed unmount above). + if slot_id is not None and handle.barcode not in still_mounted_barcodes: + try: execute_tape_request( catalog, library, @@ -282,9 +323,15 @@ def _read_shard(index: int, instance: Any) -> Path: job_id=job_id, ), ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + _cleanup_failed( + catalog, job_id, "unload", handle.barcode, handle.physical, handles + ) scheduler.release_drives(handles) - catalog.update_job_state(job_id, "completed") + if not cleanup_failed: + catalog.update_job_state(job_id, "completed") return ShardedRestoreResult( job_id=job_id, source_barcodes=barcodes, @@ -302,6 +349,7 @@ def _ensure_loaded( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: + ensure_drive_reconciled(catalog, loaded_drive_id, handle.barcode) # Record the physical drive without mutating the scheduler lock key # (handle.drive_id); mutating it leaks the reserved drive on release. if loaded_drive_id != handle.drive_id: @@ -311,6 +359,7 @@ def _ensure_loaded( slot_id = library.find_slot_by_barcode(handle.barcode) if slot_id is None: raise CartridgeOfflineError(f"Barcode {handle.barcode} not found in library") + ensure_drive_reconciled(catalog, handle.drive_id, handle.barcode) execute_tape_request( catalog, library, diff --git a/openblade/jobs/tree_restore.py b/openblade/jobs/tree_restore.py index b8baf1d..e4d16fa 100644 --- a/openblade/jobs/tree_restore.py +++ b/openblade/jobs/tree_restore.py @@ -304,6 +304,7 @@ def run_tree_restore( ltfs, catalog, child.id, + scheduler=scheduler, ) verified = plain_result.checksum_verified barcodes = [plain_result.source_barcode] diff --git a/openblade/jobs/verify.py b/openblade/jobs/verify.py index 84488dc..ec5ee9c 100644 --- a/openblade/jobs/verify.py +++ b/openblade/jobs/verify.py @@ -8,10 +8,14 @@ from openblade.catalog.models import FileRecord from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend -from openblade.domain.models import MountMode +from openblade.domain.models import FileInstanceState, MountMode from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType +_UNCOMMITTED_STATES = frozenset( + {FileInstanceState.STAGING.value, FileInstanceState.VERIFYING.value} +) + def sha256sum(path: Path) -> str: digest = hashlib.sha256() @@ -63,6 +67,10 @@ def run_verify_job( handle = ltfs.mount(barcode, MountMode.READ_ONLY) try: for instance in catalog.list_instances_for_barcode(barcode): + if instance.state in _UNCOMMITTED_STATES: + # A staged shard from an interrupted archive is not a + # durability claim yet; recovery owns it, not verify. + continue record = catalog.session.get(FileRecord, instance.file_record_id) if record is None: continue diff --git a/openblade/nas/archive_lifecycle.py b/openblade/nas/archive_lifecycle.py index c6b3ae5..aebf1c1 100644 --- a/openblade/nas/archive_lifecycle.py +++ b/openblade/nas/archive_lifecycle.py @@ -15,6 +15,7 @@ CatalogShardWriter, ) from openblade.nas.ltfs_manifest import ( + CommitMarker, ManifestFileEntry, ManifestJson, TapeJson, @@ -492,6 +493,58 @@ def _resolve_tape_path(self, barcode: str, file_record_id: str) -> str: return "" +def sibling_tapes_from_shard(shard: CatalogShard) -> list[str]: + """Other barcodes needed to restore every file of this generation (from shard records).""" + barcodes: list[str | None] = [] + for dataset in shard.datasets: + barcodes.extend(dataset.tape_set) + barcodes.extend(dataset.shard_set) + return sorted(set(_ordered_unique(barcodes)) - {shard.barcode}) + + +def finalize_tape_generation( + writer: TapeMetadataWriter, + shard_writer: CatalogShardWriter, + barcode: str, + *, + generation_id: str, + manifest: ManifestJson, + shard: CatalogShard, + tape_json: TapeJson, +) -> CommitMarker: + """Write a generation's metadata in the one durable order, commit marker LAST. + + Precondition: all file data for this generation is already on tape. Order: + manifest.json (+.sha256) -> catalog-shard.json (+.sha256) -> tape.json -> commit marker. + A crash before the marker leaves the generation uncommitted; rebuild ignores it. + """ + manifest = manifest.model_copy( + update={"generation_id": generation_id, "sibling_tapes": sibling_tapes_from_shard(shard)} + ) + manifest_sha = writer.write_manifest(barcode, manifest) + writer.write_manifest_checksum(barcode, manifest_sha) + shard_sha = shard_writer.write_shard(barcode, shard) + previous = tape_json.current_generation + writer.write_tape_json( + barcode, + tape_json.model_copy( + update={ + "current_generation": generation_id, + "previous_generation": previous if previous != generation_id else None, + "last_openblade_write_at": _utcnow_iso(), + } + ), + ) + marker = CommitMarker( + generation_id=generation_id, + manifest_sha256=manifest_sha, + catalog_shard_sha256=shard_sha, + written_at=_utcnow_iso(), + ) + writer.write_commit_marker(barcode, marker) + return marker + + def _ordered_unique(values: list[str | None]) -> list[str]: ordered: list[str] = [] seen: set[str] = set() diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py new file mode 100644 index 0000000..4580f55 --- /dev/null +++ b/openblade/nas/capacity.py @@ -0,0 +1,492 @@ +"""Tape-native NAS planning models: scratch thresholds, fragmentation, +capacity reservations, VG replication, file spanning, export sets/vaults, +drive planning, and small-file aggregation. + +Pure models and functions; no I/O. Planners compose these. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterable, Sequence +from datetime import datetime, timedelta +from enum import Enum +from typing import TYPE_CHECKING, cast + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from openblade.jobs.inventory import InventoryService +from openblade.nas.planner import ArchivePlanner +from openblade.nas.types import ArchivePlan, ArchivePlanRequest, NasFileState + +if TYPE_CHECKING: + from openblade.catalog.repository import CatalogRepository + from openblade.domain.backends import LibraryBackend + +# --- 2. scratch thresholds ------------------------------------------------- + + +class ScratchHealth(str, Enum): + OK = "ok" + WARN = "warn" + CRITICAL = "critical" + + +class ScratchThresholds(BaseModel): + model_config = ConfigDict(frozen=True) + + scratch_min: int = Field(ge=0) + scratch_warn: int = Field(ge=0) + + @model_validator(mode="after") + def _warn_above_min(self) -> ScratchThresholds: + if self.scratch_warn < self.scratch_min: + raise ValueError("scratch_warn must be >= scratch_min") + return self + + +def evaluate_scratch(scratch_count: int, thresholds: ScratchThresholds) -> ScratchHealth: + if scratch_count < thresholds.scratch_min: + return ScratchHealth.CRITICAL + if scratch_count < thresholds.scratch_warn: + return ScratchHealth.WARN + return ScratchHealth.OK + + +# --- 6. fragmentation ------------------------------------------------------ + +DEFAULT_CONSOLIDATION_RATIO = 0.5 + + +class TapeFragmentation(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + live_bytes: int = Field(ge=0) + written_bytes: int = Field(ge=0) + + @model_validator(mode="after") + def _live_le_written(self) -> TapeFragmentation: + if self.live_bytes > self.written_bytes: + raise ValueError("live_bytes cannot exceed written_bytes") + return self + + @property + def live_ratio(self) -> float: + return 1.0 if self.written_bytes == 0 else self.live_bytes / self.written_bytes + + def needs_consolidation(self, threshold: float = DEFAULT_CONSOLIDATION_RATIO) -> bool: + return self.written_bytes > 0 and self.live_ratio < threshold + + +# --- 7. capacity reservations --------------------------------------------- + + +logger = logging.getLogger(__name__) + + +class ReservationError(ValueError): + """A reservation cannot be satisfied from the pool's free capacity.""" + + +# No lease-TTL setting is in scope where plans are built, so a booked plan holds +# its pool bytes for one hour; expiry is the release path for plans never started. +PLAN_RESERVATION_TTL = timedelta(hours=1) + + +class PoolNotFoundError(LookupError): + """A pool-budgeted plan names a pool id the catalog does not know.""" + + +class ClientBudgetFieldsError(ValueError): + """A pool-budgeted request carried capacity inputs only the server may supply.""" + + +# With ``pool`` set these come from the library + catalog; a client value is refused +# (never silently overwritten) so an inflated budget cannot be smuggled in. +SERVER_BUDGET_FIELDS = ( + "available_tapes", + "tape_capacities", + "reserved_bytes", + "pool_reserved_bytes", +) + + +def server_data_tapes( + repository: CatalogRepository, library: LibraryBackend | None +) -> dict[str, int]: + """Writable data tapes (barcode -> usable bytes) from library inventory + catalog. + + The library holds blank/scratch media the catalog has never seen, so slots count + at the planner's default capacity; catalogued cartridges use their recorded + ``capacity_bytes - used_bytes``. Cleaning cartridges and volume-group members + (already assigned elsewhere) are excluded. + """ + capacities: dict[str, int] = {} + if library is not None: + try: + for slot in InventoryService(library).snapshot().slots: + if slot.barcode is not None: + capacities[str(slot.barcode)] = ArchivePlanner.DEFAULT_TAPE_CAPACITY + except Exception as exc: # noqa: BLE001 - library offline: fall back to the catalog alone + logger.warning("library inventory unavailable for pool budget", exc_info=exc) + assigned: set[str] = set() + for cartridge in repository.list_cartridges(): + barcode = str(cartridge.barcode or "").strip() + if not barcode: + continue + if cartridge.volume_group_id is not None: + assigned.add(barcode) + capacities[barcode] = max(0, int(cartridge.capacity_bytes) - int(cartridge.used_bytes)) + return { + b: capacities[b] + for b in sorted(capacities) + if b not in assigned and not b.upper().startswith("CLN") + } + + +def count_scratch_tapes(repository: CatalogRepository, library: LibraryBackend) -> int: + """Blank tapes in library slots: not in any volume group, not catalogued as written. + + The catalog only holds catalogued/archived cartridges, so scratch stock must be + counted from the inventory snapshot. Cleaning cartridges are excluded. + """ + excluded = { + str(c.barcode).strip() + for c in repository.list_cartridges() + if c.barcode and (c.volume_group_id is not None or int(c.used_bytes) > 0) + } + barcodes = { + str(slot.barcode) + for slot in InventoryService(library).snapshot().slots + if slot.barcode is not None + } + return sum(1 for b in barcodes if b not in excluded and not b.upper().startswith("CLN")) + + +def server_budget_request( + repository: CatalogRepository, request: ArchivePlanRequest, library: LibraryBackend | None +) -> ArchivePlanRequest: + """Replace a pool request's capacity inputs with server-derived ones. + + Pool-less requests are returned unchanged (they get no pool budget). Raises + ``ClientBudgetFieldsError`` if the body supplied any ``SERVER_BUDGET_FIELDS`` and + ``PoolNotFoundError`` for an unknown pool id. + """ + if request.pool is None: + return request + supplied = [name for name in SERVER_BUDGET_FIELDS if name in request.model_fields_set] + if supplied: + raise ClientBudgetFieldsError( + f"{', '.join(supplied)} cannot be supplied when pool is set; the server derives them" + ) + if repository.get_nas_pool(request.pool) is None: + raise PoolNotFoundError(f"Pool {request.pool} not found") + tapes = server_data_tapes(repository, library) + return request.model_copy(update={"available_tapes": list(tapes), "tape_capacities": tapes}) + + +def plan_with_pool_budget( + repository: CatalogRepository, + request: ArchivePlanRequest, + *, + now: datetime, + planner: ArchivePlanner | None = None, +) -> ArchivePlan: + """Plan against the pool's reservation budget without booking anything. + + Unexpired ``NasReservation`` rows for ``request.pool`` are the source of truth: + their sum becomes ``pool_reserved_bytes``. Previews never consume budget; the + plan's bytes are booked only when ingest starts (``book_plan_capacity``). + Requests without a pool are planned unchanged. + """ + planner = planner or ArchivePlanner() + pool_id = request.pool + if pool_id is None: + return planner.plan(request) + reserved = sum( + int(cast(int, row["bytes"])) for row in repository.reservations_for_pool(pool_id, now=now) + ) + return planner.plan(request.model_copy(update={"pool_reserved_bytes": reserved})) + + +def book_plan_capacity( + repository: CatalogRepository, + plan: ArchivePlan, + library: LibraryBackend | None, + *, + now: datetime, +) -> None: + """Book a pool plan's bytes at ingest start (owner=plan_id, TTL 1h). + + The capacity ceiling is server-derived (``server_data_tapes``), never taken from + the stored request. Pool-less plans book nothing. Raises ``ReservationError`` + when the atomic booking is refused, so the caller must not create the job. + """ + pool_id = plan.pool + if pool_id is None: + return + booked = repository.reserve_nas_capacity( + pool_id, + plan.total_bytes * plan.copies_required, + plan.plan_id, + now + PLAN_RESERVATION_TTL, + capacity_bytes=sum(server_data_tapes(repository, library).values()), + now=now, + ) + if not booked: + raise ReservationError(f"pool {pool_id} cannot reserve capacity for plan {plan.plan_id}") + + +class CapacityReservation(BaseModel): + model_config = ConfigDict(frozen=True) + + pool_id: str = Field(min_length=1) + bytes: int = Field(gt=0) + owner: str = Field(min_length=1) + expires_at: datetime + + def active(self, now: datetime) -> bool: + return now < self.expires_at + + +class ReservationLedger: + """Ledger of capacity reservations per pool. + + Pure in-memory by default; constructed with ``catalog`` it reads and writes + through the persisted ``nas_reservations`` table so reservations survive restarts. + """ + + def __init__(self, catalog: CatalogRepository | None = None) -> None: + self._items: list[CapacityReservation] = [] + self._catalog = catalog + + def _active(self, pool_id: str, now: datetime) -> list[CapacityReservation]: + if self._catalog is None: + return [r for r in self._items if r.pool_id == pool_id and r.active(now)] + return [ + CapacityReservation.model_validate(item) + for item in self._catalog.reservations_for_pool(pool_id, now=now) + ] + + def reserved(self, pool_id: str, now: datetime) -> int: + return sum(r.bytes for r in self._active(pool_id, now)) + + def available(self, pool_id: str, free_bytes: int, now: datetime) -> int: + return max(0, free_bytes - self.reserved(pool_id, now)) + + def reserve( + self, + pool_id: str, + nbytes: int, + owner: str, + ttl: timedelta, + *, + free_bytes: int, + now: datetime, + ) -> CapacityReservation: + res = CapacityReservation(pool_id=pool_id, bytes=nbytes, owner=owner, expires_at=now + ttl) + if self._catalog is not None: + # Check-and-insert is one BEGIN IMMEDIATE transaction in the catalog, + # so concurrent processes cannot both claim the last bytes. + if not self._catalog.reserve_nas_capacity( + pool_id, nbytes, owner, res.expires_at, capacity_bytes=free_bytes, now=now + ): + raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") + return res + if nbytes > self.available(pool_id, free_bytes, now): + raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") + self._items.append(res) + return res + + def release(self, pool_id: str, owner: str) -> int: + if self._catalog is not None: + removed = 0 + for item in self._catalog.list_nas_reservations(pool_id): + if item["owner"] == owner and self._catalog.delete_nas_reservation(str(item["id"])): + removed += 1 + return removed + before = len(self._items) + self._items = [r for r in self._items if not (r.pool_id == pool_id and r.owner == owner)] + return before - len(self._items) + + +# --- 5. VG replication ----------------------------------------------------- + + +class ReplicationState(str, Enum): + IN_SYNC = "in_sync" + LAGGING = "lagging" + BROKEN = "broken" + + +class ExportPrepError(ValueError): + """Export preparation refused (e.g. replicas not in sync).""" + + +class VolumeGroupReplication(BaseModel): + model_config = ConfigDict(frozen=True) + + vg_id: str = Field(min_length=1) + barcodes: tuple[str, ...] = () + replicas_required: int = Field(ge=1) + replicas_present: int = Field(ge=0) + + @property + def state(self) -> ReplicationState: + if self.replicas_present >= self.replicas_required: + return ReplicationState.IN_SYNC + if self.replicas_present == 0: + return ReplicationState.BROKEN + return ReplicationState.LAGGING + + def replica_write_completed(self) -> VolumeGroupReplication: + return self.model_copy(update={"replicas_present": self.replicas_present + 1}) + + +def merge_volume_groups( + a: VolumeGroupReplication, b: VolumeGroupReplication, new_id: str +) -> VolumeGroupReplication: + """Merged VG needs the stricter requirement and has only the replicas both share.""" + return VolumeGroupReplication( + vg_id=new_id, + barcodes=a.barcodes + tuple(x for x in b.barcodes if x not in a.barcodes), + replicas_required=max(a.replicas_required, b.replicas_required), + replicas_present=min(a.replicas_present, b.replicas_present), + ) + + +def require_export_ready(vg: VolumeGroupReplication) -> None: + if vg.state is not ReplicationState.IN_SYNC: + raise ExportPrepError(f"volume group {vg.vg_id} is {vg.state.value}; cannot export-prep") + + +# --- 8. file spanning ------------------------------------------------------ + + +class SpanSegment(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + offset: int = Field(ge=0) + length: int = Field(gt=0) + + +class FileSpan(BaseModel): + model_config = ConfigDict(frozen=True) + + path: str = Field(min_length=1) + segments: tuple[SpanSegment, ...] = Field(min_length=1) + + @property + def barcodes(self) -> list[str]: + return list(dict.fromkeys(s.barcode for s in self.segments)) + + +def span_state(span: FileSpan, available: set[str]) -> NasFileState: + if all(b in available for b in span.barcodes): + return NasFileState.OFFLINE_ON_TAPE + return NasFileState.MISSING_TAPE + + +def required_tapes(spans: Iterable[FileSpan]) -> list[str]: + return list(dict.fromkeys(b for s in spans for b in s.barcodes)) + + +# --- 9. export sets + vault ------------------------------------------------ + + +class ExportSetState(str, Enum): + PREPARING = "preparing" + READY = "ready" + EXPORTED = "exported" + IMPORTED = "imported" + + +class ExportSet(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(min_length=1) + barcodes: tuple[str, ...] = Field(min_length=1) + state: ExportSetState = ExportSetState.PREPARING + + +class Vault(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(min_length=1) + barcodes: frozenset[str] = frozenset() + location: str = Field(min_length=1) + + +def mark_export_ready(export_set: ExportSet, groups: Sequence[VolumeGroupReplication]) -> ExportSet: + """PREPARING → READY only if every member tape's VG replication is IN_SYNC.""" + if export_set.state is not ExportSetState.PREPARING: + raise ExportPrepError(f"export set {export_set.id} is {export_set.state.value}") + for barcode in export_set.barcodes: + owners = [g for g in groups if barcode in g.barcodes] + if not owners: + raise ExportPrepError(f"tape {barcode} has no known replication state") + for g in owners: + require_export_ready(g) + return export_set.model_copy(update={"state": ExportSetState.READY}) + + +def planning_available(library_barcodes: Iterable[str], vaults: Iterable[Vault]) -> set[str]: + """Barcodes usable for planning: anything in a vault is OFFLINE.""" + vaulted = {b for v in vaults for b in v.barcodes} + return {b for b in library_barcodes if b not in vaulted} + + +# --- 10. required drives / estimated swaps -------------------------------- + + +class DrivePlan(BaseModel): + model_config = ConfigDict(frozen=True) + + tapes: tuple[str, ...] + required_drives: int + estimated_swaps: int + + +def plan_drives(spans: Iterable[FileSpan], available_drives: int) -> DrivePlan: + if available_drives < 1: + raise ValueError("available_drives must be >= 1") + tapes = required_tapes(spans) + used = min(len(tapes), available_drives) + return DrivePlan( + tapes=tuple(tapes), + required_drives=used, + estimated_swaps=max(0, len(tapes) - available_drives), + ) + + +# --- 11. small-file aggregation ------------------------------------------- + + +class AggregationPolicy(BaseModel): + model_config = ConfigDict(frozen=True) + + aggregate_below_bytes: int = Field(ge=0) + max_container_bytes: int = Field(gt=0) + + +def group_for_write( + files: Sequence[tuple[str, int]], policy: AggregationPolicy +) -> list[list[tuple[str, int]]]: + """Group files into write units: small files share containers, large stand alone.""" + units: list[list[tuple[str, int]]] = [] + current: list[tuple[str, int]] = [] + current_size = 0 + for path, size in files: + if size >= policy.aggregate_below_bytes: + units.append([(path, size)]) + continue + if current and current_size + size > policy.max_container_bytes: + units.append(current) + current, current_size = [], 0 + current.append((path, size)) + current_size += size + if current: + units.append(current) + return units diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index 9795f89..77df65d 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -2,17 +2,21 @@ from __future__ import annotations +import hashlib +import json import logging -from datetime import datetime from typing import cast from uuid import uuid4 +from openblade.catalog.models import NasDataset as NasDatasetRow from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow +from openblade.domain.protection import ProtectionPolicy from openblade.nas.catalog_shard import ( CatalogShard, CatalogShardWriter, ) -from openblade.nas.ltfs_manifest import TapeMetadataWriter +from openblade.nas.ltfs_manifest import ManifestJson, TapeMetadataWriter from openblade.nas.manifest_validator import ManifestValidator from openblade.nas.path_mapping import PathMappingService from openblade.nas.types import ( @@ -20,25 +24,31 @@ DatasetStatus, IngestMode, ManifestVersionRecord, + MissingSibling, NasDataset, NasFileRecord, NasFileState, RebuildPlanRequest, RebuildPlanResult, RebuildRunStatus, + UncommittedGeneration, ) logger = logging.getLogger(__name__) def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" # The catalog rows below come back as `dict[str, object]` (serialized from the # same pydantic models that are rebuilt from them), so every column read needs a # type. These are `cast` only -- no runtime narrowing, no new failure mode: # validation stays exactly where it was, in the model construction one line down. +def _sha256_text(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def _text_column(value: object) -> str | None: return cast("str | None", value) @@ -82,6 +92,8 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: barcodes_missing_shard: list[str] = [] barcodes_invalid: list[str] = [] warnings: list[str] = [] + uncommitted_generations: list[UncommittedGeneration] = [] + missing_siblings: list[MissingSibling] = [] estimated_files = 0 estimated_datasets = 0 estimated_path_mappings = 0 @@ -116,11 +128,49 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: warnings.append(f"{barcode}: catalog-shard.json missing") continue + uncommitted = self._uncommitted_reason(barcode, manifest) + if uncommitted is not None: + # Never restore an uncommitted generation's files as archived. + # _uncommitted_reason only returns a reason when generation_id is set. + uncommitted_generations.append( + UncommittedGeneration( + barcode=barcode, + generation_id=manifest.generation_id or "", + reason=uncommitted, + ) + ) + continue + requested = set(request.barcodes) + for sibling in manifest.sibling_tapes: + if sibling not in requested: + missing_siblings.append( + MissingSibling( + barcode=sibling, + required_by_barcode=barcode, + generation_id=manifest.generation_id or None, + ) + ) + barcodes_to_scan.append(barcode) estimated_files += len(shard.files) estimated_datasets += len(shard.datasets) estimated_path_mappings += len(self.shard_writer.shard_to_path_mappings(shard)) + # One operator-facing line per category; the structured lists carry the detail. + if uncommitted_generations: + warnings.append( + f"{len(uncommitted_generations)} uncommitted generation(s) skipped: " + + ", ".join( + f"{u.barcode} ({u.generation_id}: {u.reason})" for u in uncommitted_generations + ) + ) + if missing_siblings: + warnings.append( + f"{len(missing_siblings)} sibling tape(s) missing from request: " + + ", ".join( + f"{m.barcode} (required by {m.required_by_barcode})" for m in missing_siblings + ) + ) safe_to_enqueue = len(barcodes_invalid) == 0 and len(barcodes_to_scan) > 0 run_id = "" if not request.dry_run: @@ -155,10 +205,37 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: estimated_files=estimated_files, estimated_datasets=estimated_datasets, estimated_path_mappings=estimated_path_mappings, + uncommitted_generations=uncommitted_generations, + missing_siblings=missing_siblings, warnings=warnings, safe_to_enqueue=safe_to_enqueue, ) + def _uncommitted_reason(self, barcode: str, manifest: ManifestJson) -> str | None: + """Why this tape's generation is uncommitted, or None when its commit marker checks out. + + Pre-v2 manifests carry no generation_id and predate commit markers; they are + accepted as committed so existing tapes stay rebuildable. + """ + if not manifest.generation_id: + return None + marker = self.metadata_writer.read_commit_marker(barcode, manifest.generation_id) + if marker is None: + return "commit marker missing" + # Hash the raw on-tape bytes, never a re-dump of the parsed model: a re-dump + # changes whenever the model gains a defaulted field, which would silently mark + # every older generation uncommitted. Markers written by the current writer hash + # these same bytes, so no compatibility shim is needed. + manifest_raw = self.metadata_writer._read_text( + barcode, self.metadata_writer._metadata_path("manifest.json") + ) + shard_raw = self.metadata_writer._read_text(barcode, self.shard_writer.SHARD_METADATA_PATH) + if manifest_raw is None or marker.manifest_sha256 != _sha256_text(manifest_raw): + return "commit marker manifest hash mismatch" + if shard_raw is None or marker.catalog_shard_sha256 != _sha256_text(shard_raw): + return "commit marker catalog-shard hash mismatch" + return None + def execute_rebuild_run(self, run_id: str) -> CatalogRebuildRunRecord: """ Execute a planned rebuild run and persist progress in the catalog database. @@ -276,10 +353,34 @@ def _recover_datasets(self, barcode: str, shard: CatalogShard) -> int: updated_at=_utcnow_iso(), ) self.repo.upsert_nas_dataset(dataset.model_dump(mode="json")) + self._recover_dataset_protection(barcode, entry.dataset_id, entry.protection) if is_new: count += 1 return count + def _recover_dataset_protection( + self, barcode: str, dataset_id: str, protection: dict[str, object] | None + ) -> None: + """Write the shard's protection/placement policy back onto the dataset row. + + Pre-v2 shards carry no policy; the existing row value is kept. A malformed + policy is logged and skipped so one bad shard entry cannot abort the rebuild. + """ + if protection is None: + return + try: + policy = ProtectionPolicy.from_dict(dict(protection)) + except (AttributeError, KeyError, TypeError, ValueError): + logger.warning( + "Skipping malformed protection policy for dataset %s on %s", dataset_id, barcode + ) + return + row = self.repo.session.get(NasDatasetRow, dataset_id) + if row is None: + return + row.protection_json = json.dumps(policy.to_dict(), sort_keys=True) + self.repo.session.commit() + def _recover_files(self, barcode: str, shard: CatalogShard) -> int: """Upsert file records from shard; count only files not already in the catalog.""" count = 0 diff --git a/openblade/nas/catalog_rebuild_worker.py b/openblade/nas/catalog_rebuild_worker.py index c680b51..7c35824 100644 --- a/openblade/nas/catalog_rebuild_worker.py +++ b/openblade/nas/catalog_rebuild_worker.py @@ -2,13 +2,13 @@ from __future__ import annotations -from datetime import datetime from typing import Any from uuid import uuid4 import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.types import CatalogRebuildRunRecord, RebuildPlanRequest, RebuildRunStatus @@ -17,7 +17,7 @@ def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" class CatalogRebuildWorker: diff --git a/openblade/nas/catalog_shard.py b/openblade/nas/catalog_shard.py index dd6730f..09d1518 100644 --- a/openblade/nas/catalog_shard.py +++ b/openblade/nas/catalog_shard.py @@ -1,10 +1,11 @@ from __future__ import annotations import json -from datetime import datetime +from typing import Any from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from openblade.domain.clock import naive_utcnow from openblade.nas.ltfs_manifest import TapeMetadataWriter from openblade.nas.types import NasFileState, PathMappingRecord @@ -38,6 +39,8 @@ class CatalogShardDatasetEntry(BaseModel): total_bytes: int = 0 tape_set: list[str] = Field(default_factory=list) shard_set: list[str] = Field(default_factory=list) + # Serialized openblade.domain.protection.ProtectionPolicy; None on pre-v2 shards. + protection: dict[str, Any] | None = None class CatalogShard(BaseModel): @@ -91,7 +94,7 @@ def build_shard( barcode=barcode, openblade_tape_id=tape_id, volume_group=volume_group, - generated_at=datetime.utcnow().isoformat() + "Z", + generated_at=naive_utcnow().isoformat() + "Z", files=files, datasets=datasets, ) diff --git a/openblade/nas/export_render.py b/openblade/nas/export_render.py new file mode 100644 index 0000000..e908c38 --- /dev/null +++ b/openblade/nas/export_render.py @@ -0,0 +1,49 @@ +"""Map catalog ``NasShare`` records onto Samba/NFS export definitions. + +``NasShare.config_json`` keys honoured: ``read_only`` (bool, default True), +``case_sensitive`` (bool, default False), ``client`` (NFS client spec, default +``*``), ``export_id`` (int) and ``pseudo`` (NFSv4 pseudo path). +""" + +import json +from pathlib import Path +from typing import Any + +from openblade.catalog.models import NasShare +from openblade.nas.nfs import NfsExport +from openblade.nas.samba import SambaShare + + +class InvalidShareConfigError(ValueError): + """``NasShare.config_json`` is not a JSON object.""" + + +def _config(share: NasShare) -> dict[str, Any]: + try: + raw = json.loads(share.config_json or "{}") + except json.JSONDecodeError as exc: + raise InvalidShareConfigError(f"share {share.name!r}: config_json is not JSON") from exc + if not isinstance(raw, dict): + raise InvalidShareConfigError(f"share {share.name!r}: config_json must be an object") + return raw + + +def samba_share_from(share: NasShare) -> SambaShare: + cfg = _config(share) + return SambaShare( + name=share.name, + path=Path(share.path), + read_only=bool(cfg.get("read_only", True)), + case_sensitive=bool(cfg.get("case_sensitive", False)), + ) + + +def nfs_export_from(share: NasShare, export_id: int = 1) -> NfsExport: + cfg = _config(share) + return NfsExport( + path=Path(share.path), + client=str(cfg.get("client", "*")), + read_only=bool(cfg.get("read_only", True)), + export_id=int(cfg.get("export_id", export_id)), + pseudo=cfg.get("pseudo"), + ) diff --git a/openblade/nas/fuse_hook.py b/openblade/nas/fuse_hook.py index 16c7d54..2f469e3 100644 --- a/openblade/nas/fuse_hook.py +++ b/openblade/nas/fuse_hook.py @@ -2,9 +2,9 @@ from __future__ import annotations -from datetime import datetime from typing import Any +from openblade.domain.clock import naive_utcnow from openblade.nas.service import NasService from openblade.nas.types import NasFileState @@ -23,7 +23,7 @@ def __init__(self, service: NasService): def _log_access(self, *, pool_id: str, logical_path: str, state: str, action: str) -> None: self._access_log.append( { - "timestamp": datetime.utcnow().isoformat() + "Z", + "timestamp": naive_utcnow().isoformat() + "Z", "pool_id": pool_id, "path": logical_path, "state": state, diff --git a/openblade/nas/health_service.py b/openblade/nas/health_service.py index bd12815..a6dd1d2 100644 --- a/openblade/nas/health_service.py +++ b/openblade/nas/health_service.py @@ -4,7 +4,6 @@ import time from collections.abc import Callable -from datetime import datetime from typing import Any import structlog @@ -12,6 +11,13 @@ from openblade.catalog.models import Cartridge, NasDataset, NasFileRecord, PathMapping from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow +from openblade.nas.capacity import ( + ScratchHealth, + ScratchThresholds, + count_scratch_tapes, + evaluate_scratch, +) from openblade.nas.types import ( CatalogStatusResponse, ComponentHealth, @@ -31,7 +37,7 @@ def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" class HealthService: @@ -51,6 +57,9 @@ def check_health(self) -> HealthResponse: self._check_library_component(), self._check_ltfs_component(), ] + scratch = self._check_scratch_component() + if scratch is not None: + components.append(scratch) status = max(components, key=lambda component: _STATUS_PRIORITY[component.status]).status return HealthResponse( status=status, @@ -59,6 +68,34 @@ def check_health(self) -> HealthResponse: version=get_version_info()["version"], ) + def _check_scratch_component(self) -> ComponentHealth | None: + """Scratch-stock component; absent when no thresholds are configured. + + CRITICAL maps to DEGRADED, never UNHEALTHY: tape stock must not fail a probe. + """ + try: + stored = self.repo.get_scratch_thresholds() + if stored is None: + return None + thresholds = ScratchThresholds.model_validate(stored) + count = count_scratch_tapes(self.repo, self.library) + except Exception: # noqa: BLE001 - health must not raise; DB/library failures are reported by their own components + logger.warning("scratch health probe failed", exc_info=True) + return None + level = evaluate_scratch(count, thresholds) + if level is ScratchHealth.CRITICAL: + return ComponentHealth( + name="scratch", + status=HealthStatus.DEGRADED, + message=f"scratch count {count} below min {thresholds.scratch_min}", + ) + message = ( + f"scratch count {count} below warn {thresholds.scratch_warn}" + if level is ScratchHealth.WARN + else "" + ) + return ComponentHealth(name="scratch", status=HealthStatus.OK, message=message) + def check_ready(self) -> ReadyResponse: """Return ready=True only when the database is reachable and the library is connected.""" checked_at = _utcnow_iso() diff --git a/openblade/nas/ingest.py b/openblade/nas/ingest.py index 9f1ca42..6227de9 100644 --- a/openblade/nas/ingest.py +++ b/openblade/nas/ingest.py @@ -793,7 +793,29 @@ def run_ingest_job( ltfs=ltfs, config=nas_service.get_source_stream_config(), ) - return executor.run() + raised = True + try: + result = executor.run() + raised = False + return result + finally: + # The plan's pool booking (owner=plan_id) is held only while the job can still + # write; any terminal state frees it. run() normally converts failures into + # FAILED, but if it raises anyway (e.g. _mark_dataset_failed hit a broken + # session) the status can still be ARCHIVING and nothing will ever resume this + # job, so an escaping exception releases too. A non-terminal *return* keeps it. + if raised or job.status in _TERMINAL_JOB_STATES: + try: + nas_service.repository.delete_nas_reservations_for_owner(job.plan.plan_id) + except Exception as exc: # noqa: BLE001 - a failed release (e.g. session in PendingRollbackError) must not replace the executor's exception; the booking expires by TTL + logger.warning( + "reservation release failed for plan %s", job.plan.plan_id, exc_info=exc + ) + + +_TERMINAL_JOB_STATES = frozenset( + {DatasetStatus.ARCHIVED, DatasetStatus.FAILED, DatasetStatus.CANCELLED} +) def _build_archive_lifecycle_manager( diff --git a/openblade/nas/ltfs_manifest.py b/openblade/nas/ltfs_manifest.py index dd51f20..fcb5a48 100644 --- a/openblade/nas/ltfs_manifest.py +++ b/openblade/nas/ltfs_manifest.py @@ -2,12 +2,23 @@ import hashlib import json -from datetime import datetime from pathlib import PurePosixPath from typing import Any from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from openblade.domain.clock import naive_utcnow + +MANIFEST_SCHEMA_V1 = "openblade.manifest.v1" +MANIFEST_SCHEMA_V2 = "openblade.manifest.v2" +MANIFEST_SCHEMA = MANIFEST_SCHEMA_V2 +SUPPORTED_MANIFEST_SCHEMAS = frozenset({MANIFEST_SCHEMA_V1, MANIFEST_SCHEMA_V2}) +COMMIT_MARKER_SCHEMA = "openblade.commit.v1" + + +class UnsupportedManifestSchemaError(ValueError): + """manifest.json declares a schema this build does not understand (e.g. a newer one).""" + class TapeJson(BaseModel): schema_: str = Field(default="openblade.tape.v1", alias="schema") @@ -23,6 +34,9 @@ class TapeJson(BaseModel): state: str = "active" generation: str = "LTO-8" notes: str = "" + # `generation` above is the LTO media generation; the write-generation chain is below. + current_generation: str | None = None + previous_generation: str | None = None model_config = ConfigDict(populate_by_name=True) @@ -41,7 +55,7 @@ class ManifestFileEntry(BaseModel): class ManifestJson(BaseModel): - schema_: str = Field(default="openblade.manifest.v1", alias="schema") + schema_: str = Field(default=MANIFEST_SCHEMA, alias="schema") barcode: str openblade_tape_id: str volume_group: str = "" @@ -52,6 +66,11 @@ class ManifestJson(BaseModel): file_count: int = 0 total_logical_bytes: int = 0 files: list[ManifestFileEntry] = Field(default_factory=list) + # v2 fields; a v1 manifest parses with these defaults. + generation_id: str = "" + sibling_tapes: list[str] = Field(default_factory=list) + placement: str = "" + protection: dict[str, Any] | None = None model_config = ConfigDict(populate_by_name=True) @@ -88,6 +107,30 @@ class ShardSetManifest(BaseModel): model_config = ConfigDict(populate_by_name=True) +class CommitMarker(BaseModel): + """/.openblade/commit-.json — written LAST; its absence means uncommitted.""" + + schema_: str = Field(default=COMMIT_MARKER_SCHEMA, alias="schema") + generation_id: str + manifest_sha256: str + catalog_shard_sha256: str + written_at: str + + model_config = ConfigDict(populate_by_name=True) + + +def parse_manifest(payload: dict[str, Any]) -> ManifestJson: + """Validate a manifest payload, rejecting schemas newer than this build understands. + + Raises UnsupportedManifestSchemaError for an unknown schema and pydantic's + ValidationError for a malformed supported one. + """ + schema = str(payload.get("schema", MANIFEST_SCHEMA_V1)) + if schema not in SUPPORTED_MANIFEST_SCHEMAS: + raise UnsupportedManifestSchemaError(f"unsupported manifest schema: {schema}") + return ManifestJson.model_validate(payload) + + class ChecksumEntry(BaseModel): checksum: str path: str @@ -120,12 +163,17 @@ def write_tape_json(self, barcode: str, tape_json: TapeJson) -> None: ) def write_manifest(self, barcode: str, manifest: ManifestJson) -> str: - """Write /.openblade/manifest.json and return its sha256 checksum.""" + """Write /.openblade/manifest.json and return the sha256 of the exact bytes written. + + The checksum covers the on-tape bytes, not a re-dump of the model, so a commit + marker stays verifiable after ``ManifestJson`` gains new defaulted fields. + """ self.ensure_openblade_dirs(barcode) - payload = manifest.model_dump(by_alias=True) - checksum = self.compute_json_checksum(payload) - self._write_json(barcode, self._metadata_path("manifest.json"), payload) - return checksum + content = json.dumps( + manifest.model_dump(by_alias=True), sort_keys=True, separators=(",", ":") + ) + self._write_text(barcode, self._metadata_path("manifest.json"), content) + return hashlib.sha256(content.encode("utf-8")).hexdigest() def write_manifest_checksum(self, barcode: str, checksum: str) -> None: """Write /.openblade/manifest.sha256 for tamper-detection.""" @@ -166,6 +214,23 @@ def write_shard_set_manifest(self, barcode: str, shard_set: ShardSetManifest) -> shard_set.model_dump(by_alias=True), ) + def write_commit_marker(self, barcode: str, marker: CommitMarker) -> None: + """Write /.openblade/commit-.json. Callers must write it last.""" + self.ensure_openblade_dirs(barcode) + name = f"commit-{self._safe_name(marker.generation_id)}.json" + self._write_json(barcode, self._metadata_path(name), marker.model_dump(by_alias=True)) + + def read_commit_marker(self, barcode: str, generation_id: str) -> CommitMarker | None: + """Read a generation's commit marker; None when missing or corrupt.""" + name = f"commit-{self._safe_name(generation_id)}.json" + try: + payload = self._read_json(barcode, self._metadata_path(name)) + if payload is None: + return None + return CommitMarker.model_validate(payload) + except (json.JSONDecodeError, ValidationError): + return None + def ensure_openblade_dirs(self, barcode: str) -> None: """Ensure all reserved /.openblade/ subdirectories exist in the backend.""" for directory in self.RESERVED_DIRS: @@ -193,8 +258,8 @@ def read_manifest(self, barcode: str) -> ManifestJson | None: payload = self._read_json(barcode, self._metadata_path("manifest.json")) if payload is None: return None - return ManifestJson.model_validate(payload) - except (json.JSONDecodeError, ValidationError): + return parse_manifest(payload) + except (json.JSONDecodeError, ValidationError, UnsupportedManifestSchemaError): return None def initialize_tape( @@ -308,4 +373,4 @@ def _safe_name(value: str) -> str: @staticmethod def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/openblade/nas/manifest_validator.py b/openblade/nas/manifest_validator.py index 91b5be5..35c9201 100644 --- a/openblade/nas/manifest_validator.py +++ b/openblade/nas/manifest_validator.py @@ -1,12 +1,18 @@ from __future__ import annotations import json -from datetime import datetime from pydantic import BaseModel, Field, ValidationError +from openblade.domain.clock import naive_utcnow from openblade.nas.catalog_shard import CatalogShard, CatalogShardWriter -from openblade.nas.ltfs_manifest import ManifestJson, TapeMetadataWriter +from openblade.nas.ltfs_manifest import ( + MANIFEST_SCHEMA, + ManifestJson, + TapeMetadataWriter, + UnsupportedManifestSchemaError, + parse_manifest, +) class ManifestValidationResult(BaseModel): @@ -60,7 +66,7 @@ class ManifestValidator: MANIFEST_CHECKSUM_PATH = "/.openblade/manifest.sha256" CATALOG_SHARD_PATH = "/.openblade/catalog-shard.json" CATALOG_SHARD_CHECKSUM_PATH = "/.openblade/catalog-shard.sha256" - MANIFEST_SCHEMA = "openblade.manifest.v1" + MANIFEST_SCHEMA = MANIFEST_SCHEMA CATALOG_SHARD_SCHEMA = "openblade.catalog_shard.v1" def __init__( @@ -83,15 +89,16 @@ def validate_manifest(self, barcode: str) -> ManifestValidationResult: result.schema_version = str(payload.get("schema", "")) try: - manifest = ManifestJson.model_validate(payload) + manifest = parse_manifest(payload) + except UnsupportedManifestSchemaError: + result.errors.append(f"manifest schema mismatch: {result.schema_version}") + return result except ValidationError: result.errors.append("manifest.json failed schema validation") return result result.file_count = manifest.file_count result.total_bytes = manifest.total_logical_bytes - if result.schema_version != self.MANIFEST_SCHEMA: - result.errors.append(f"manifest schema mismatch: {result.schema_version}") raw_files = payload.get("files", []) if not isinstance(raw_files, list): @@ -230,7 +237,7 @@ def begin_write(self, barcode: str, manifest: ManifestJson) -> str: """ self.writer.ensure_openblade_dirs(barcode) self.writer._ensure_dir(barcode, self.VERSIONS_DIR) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + timestamp = naive_utcnow().strftime("%Y%m%dT%H%M%SZ") temp_path = f"{self.VERSIONS_DIR}/manifest.{timestamp}.tmp" self.writer._write_json(barcode, temp_path, manifest.model_dump(by_alias=True)) return temp_path diff --git a/openblade/nas/media.py b/openblade/nas/media.py new file mode 100644 index 0000000..ecdc41f --- /dev/null +++ b/openblade/nas/media.py @@ -0,0 +1,93 @@ +"""LTO media generations, drive compatibility, and foreign-media detection.""" + +from __future__ import annotations + +from enum import IntEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class MediaGeneration(IntEnum): + UNKNOWN = 0 + LTO5 = 5 + LTO6 = 6 + LTO7 = 7 + LTO8 = 8 + LTO9 = 9 + + +# LTO Program (lto.org) compatibility rules: +# - LTO-5, LTO-6, LTO-7 drives follow the classic rule: read back two generations +# (N-2) and write back one generation (N-1). +# - LTO-8 broke the N-2 read rule: an LTO-8 drive reads and writes LTO-7 and +# LTO-8 media only (no LTO-6 read). +# - LTO-9 reads and writes LTO-8 and LTO-9 media only. +# (LTO-7 Type M / M8 media is not modelled; it is reported as LTO7.) +_READ: dict[MediaGeneration, frozenset[MediaGeneration]] = { + # LTO-5 also reads LTO-3/LTO-4, which are outside the modelled range. + MediaGeneration.LTO5: frozenset({MediaGeneration.LTO5}), + MediaGeneration.LTO6: frozenset({MediaGeneration.LTO5, MediaGeneration.LTO6}), + MediaGeneration.LTO7: frozenset( + {MediaGeneration.LTO5, MediaGeneration.LTO6, MediaGeneration.LTO7} + ), + MediaGeneration.LTO8: frozenset({MediaGeneration.LTO7, MediaGeneration.LTO8}), + MediaGeneration.LTO9: frozenset({MediaGeneration.LTO8, MediaGeneration.LTO9}), +} +_WRITE: dict[MediaGeneration, frozenset[MediaGeneration]] = { + MediaGeneration.LTO5: frozenset({MediaGeneration.LTO5}), + MediaGeneration.LTO6: frozenset({MediaGeneration.LTO5, MediaGeneration.LTO6}), + MediaGeneration.LTO7: frozenset({MediaGeneration.LTO6, MediaGeneration.LTO7}), + MediaGeneration.LTO8: frozenset({MediaGeneration.LTO7, MediaGeneration.LTO8}), + MediaGeneration.LTO9: frozenset({MediaGeneration.LTO8, MediaGeneration.LTO9}), +} + + +def can_read(drive_gen: MediaGeneration, media_gen: MediaGeneration) -> bool: + return media_gen in _READ.get(drive_gen, frozenset()) + + +def can_write(drive_gen: MediaGeneration, media_gen: MediaGeneration) -> bool: + return media_gen in _WRITE.get(drive_gen, frozenset()) + + +class IncompatibleMediaError(ValueError): + """Drive generation cannot perform the requested operation on the media.""" + + def __init__(self, drive_gen: MediaGeneration, media_gen: MediaGeneration, op: str) -> None: + super().__init__(f"{drive_gen.name} drive cannot {op} {media_gen.name} media") + self.drive_gen = drive_gen + self.media_gen = media_gen + self.op = op + + +class ForeignMediaError(ValueError): + """Media not formatted by OpenBlade; must be explicitly adopted first.""" + + def __init__(self, barcode: str) -> None: + super().__init__(f"cartridge {barcode} is foreign media; adopt it before use") + self.barcode = barcode + + +class Cartridge(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + generation: MediaGeneration = MediaGeneration.UNKNOWN + has_tape_json: bool = False + adopted: bool = False + + +def is_foreign(cartridge: Cartridge) -> bool: + """True for media OpenBlade did not format (no tape.json) and that was not adopted. + + Foreign media is read-only for planning and is never auto-formatted. + """ + return not cartridge.has_tape_json and not cartridge.adopted + + +def require_writable(cartridge: Cartridge, drive_gen: MediaGeneration) -> None: + """Planner guard for a tape write: refuses foreign or incompatible media.""" + if is_foreign(cartridge): + raise ForeignMediaError(cartridge.barcode) + if not can_write(drive_gen, cartridge.generation): + raise IncompatibleMediaError(drive_gen, cartridge.generation, "write") diff --git a/openblade/nas/nfs.py b/openblade/nas/nfs.py index 55188d2..60ff0f6 100644 --- a/openblade/nas/nfs.py +++ b/openblade/nas/nfs.py @@ -1,5 +1,6 @@ """NFS export configuration rendering.""" +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -9,7 +10,52 @@ class NfsExport: path: Path client: str = "*" read_only: bool = True + export_id: int = 1 + pseudo: str | None = None def render(self) -> str: mode = "ro" if self.read_only else "rw" return f"{self.path} {self.client}({mode},sync,no_subtree_check)" + + def render_ganesha(self) -> str: + """Render this export as an NFS-Ganesha ``EXPORT`` block.""" + access = "RO" if self.read_only else "RW" + pseudo = self.pseudo or str(self.path) + return ( + "EXPORT {\n" + f" Export_Id = {self.export_id};\n" + f' Path = "{self.path}";\n' + f' Pseudo = "{pseudo}";\n' + " Protocols = 4;\n" + " Transports = TCP;\n" + f" Access_Type = {access};\n" + " Squash = No_Root_Squash;\n" + " SecType = sys;\n" + # Ganesha parses Clients as a bare client-list token; a quoted + # "*" is rejected ("Expected a client") and drops the whole EXPORT. + f" CLIENT {{ Clients = {self.client}; Access_Type = {access}; }}\n" + " FSAL { Name = VFS; }\n" + "}\n" + ) + + +def render_exports(exports: Iterable[NfsExport]) -> str: + """Render a kernel-nfsd ``/etc/exports`` file.""" + return "".join(export.render() + "\n" for export in exports) + + +def render_ganesha_conf(exports: Iterable[NfsExport]) -> str: + """Render a complete NFS-Ganesha config (NFSv4 only, VFS FSAL).""" + header = ( + "NFS_CORE_PARAM {\n" + " Protocols = 4;\n" + " NFS_Port = 2049;\n" + " Enable_NLM = false;\n" + " Enable_RQUOTA = false;\n" + "}\n" + "NFSV4 {\n" + " Grace_Period = 5;\n" + " Lease_Lifetime = 5;\n" + "}\n" + ) + return header + "".join(export.render_ganesha() for export in exports) diff --git a/openblade/nas/planner.py b/openblade/nas/planner.py index 2a4b1ee..e2d3906 100644 --- a/openblade/nas/planner.py +++ b/openblade/nas/planner.py @@ -5,6 +5,7 @@ import posixpath from datetime import datetime, timezone +from openblade.nas.media import ForeignMediaError, is_foreign, require_writable from openblade.nas.types import ( ArchivePlan, ArchivePlanRequest, @@ -15,6 +16,8 @@ TapeAssignment, ) +POOL_BUDGET_BLOCKER = "Insufficient pool capacity after reservations" + class ArchivePlanner: DEFAULT_TAPE_CAPACITY = 12_000_000_000_000 @@ -42,6 +45,7 @@ def plan(self, request: ArchivePlanRequest) -> ArchivePlan: created_at=datetime.now(timezone.utc).isoformat(), ) + self._check_media(request) self._apply_common_checks(request, plan) if not plan.is_safe_to_enqueue: return plan @@ -348,6 +352,21 @@ def _apply_common_checks(self, request: ArchivePlanRequest, plan: ArchivePlan) - "Requested copies exceed number of available tapes", field="copies", ) + self._check_pool_budget(request, plan) + + def _check_pool_budget(self, request: ArchivePlanRequest, plan: ArchivePlan) -> None: + """Refuse a plan whose placed bytes exceed capacity left after pool reservations.""" + if request.pool is None: + return + # An empty server-derived tape set yields budget 0 -> blocked, never skipped. + budget = self.pool_capacity_bytes(request) - request.pool_reserved_bytes + placed = plan.total_bytes * plan.copies_required + if placed > budget: + self._block_plan(plan, POOL_BUDGET_BLOCKER) + + def pool_capacity_bytes(self, request: ArchivePlanRequest) -> int: + """Summed usable capacity of the request's tapes (per-barcode holds already removed).""" + return sum(self._tape_capacity(request, barcode) for barcode in request.available_tapes) def _ordered_tapes(self, request: ArchivePlanRequest) -> list[str]: preferred = [ @@ -362,7 +381,23 @@ def _single_tape_capacity(self, request: ArchivePlanRequest) -> int: return self.DEFAULT_TAPE_CAPACITY def _tape_capacity(self, request: ArchivePlanRequest, barcode: str) -> int: - return request.tape_capacities.get(barcode, self.DEFAULT_TAPE_CAPACITY) + capacity = request.tape_capacities.get(barcode, self.DEFAULT_TAPE_CAPACITY) + return max(0, capacity - request.reserved_bytes.get(barcode, 0)) + + def _check_media(self, request: ArchivePlanRequest) -> None: + """Refuse foreign (un-adopted) or drive-incompatible target cartridges. + + Raises ForeignMediaError / IncompatibleMediaError; foreign media is never + auto-formatted. + """ + for barcode in request.available_tapes: + cartridge = request.cartridges.get(barcode) + if cartridge is None: + continue + if is_foreign(cartridge): + raise ForeignMediaError(barcode) + if request.drive_generation is not None: + require_writable(cartridge, request.drive_generation) def _file_size(self, request: ArchivePlanRequest, path: str) -> int: return request.file_sizes.get(path, 0) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 515bb4d..c8bce57 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -10,15 +10,23 @@ import hashlib import hmac import importlib.util +import logging import os import posixpath import secrets import socket +import threading from dataclasses import dataclass, field from datetime import datetime from enum import Enum from typing import Any +from openblade.domain.clock import naive_utcnow +from openblade.fuse.cache import CacheEntryInUseError, CacheIntegrityError +from openblade.fuse.hydration import HydrationFailedError, HydrationState, Hydrator + +logger = logging.getLogger(__name__) + class GatewayStatus(str, Enum): RUNNING = "running" @@ -90,7 +98,7 @@ class GatewayCredential: _hashed_password: str allowed_paths: list[str] = field(default_factory=lambda: [InboxPath.GENERAL.value]) enabled: bool = True - created_at: datetime = field(default_factory=datetime.utcnow) + created_at: datetime = field(default_factory=naive_utcnow) last_seen_at: datetime | None = None @classmethod @@ -127,7 +135,7 @@ class GatewayUpload: requested_path: str routed_path: str bytes_uploaded: int - uploaded_at: datetime = field(default_factory=datetime.utcnow) + uploaded_at: datetime = field(default_factory=naive_utcnow) @dataclass @@ -143,6 +151,14 @@ class GatewaySession: uploads: list[GatewayUpload] = field(default_factory=list) +@dataclass(frozen=True) +class GatewayHandle: + """An open handle: opaque ``token`` for ``on_close`` plus the state at open.""" + + token: str + state: HydrationState + + class ProtocolGateway: """In-process protocol gateway manager for emulated SFTP/SCP ingest.""" @@ -150,6 +166,8 @@ def __init__(self) -> None: self._credentials: dict[str, GatewayCredential] = {} self._sessions: list[GatewaySession] = [] self._last_error: str | None = None + self._handles: dict[str, str] = {} # open-handle token -> pinned checksum + self._handles_lock = threading.Lock() self._bind_host = os.environ.get("OPENBLADE_SFTP_HOST", "0.0.0.0") self._bind_port = int(os.environ.get("OPENBLADE_SFTP_PORT", "2222")) self._max_sessions = int(os.environ.get("OPENBLADE_SFTP_MAX_SESSIONS", "10")) @@ -211,7 +229,7 @@ def authenticate(self, username: str, password: str) -> GatewayCredential | None if not cred or not cred.enabled: return None if cred.verify_password(password): - cred.last_seen_at = datetime.utcnow() + cred.last_seen_at = naive_utcnow() return cred return None @@ -252,7 +270,7 @@ def open_session(self, username: str, remote_addr: str) -> GatewaySession: session_id=secrets.token_hex(16), username=username, remote_addr=remote_addr, - connected_at=datetime.utcnow(), + connected_at=naive_utcnow(), ) self._sessions.append(session) return session @@ -284,7 +302,7 @@ def close_session( session = self._get_session(session_id) if session is None: return - session.disconnected_at = datetime.utcnow() + session.disconnected_at = naive_utcnow() session.bytes_uploaded = max(session.bytes_uploaded, bytes_uploaded) session.files_uploaded = max(session.files_uploaded, files_uploaded) session.errors = errors @@ -334,7 +352,7 @@ def stop(self) -> None: """Mark gateway as stopped, closing any active sessions.""" if self._status is GatewayStatus.DISABLED: return - now = datetime.utcnow() + now = naive_utcnow() for session in self._sessions: if session.disconnected_at is None: with contextlib.suppress(Exception): @@ -367,6 +385,78 @@ def _get_session(self, session_id: str) -> GatewaySession | None: return session return None + # -- hydration entry points (single path for the SMB/NFS rig) ------------ + def attach_hydrator(self, hydrator: Hydrator | None) -> None: + self._hydrator = hydrator + + def _require_hydrator(self) -> Hydrator: + hydrator: Hydrator | None = getattr(self, "_hydrator", None) + if hydrator is None: + raise HydrationFailedError("no hydrator attached to the protocol gateway") + return hydrator + + def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> GatewayHandle: + """Pin the file and start hydration; optionally wait ``timeout`` seconds. + + Every successful ``on_open`` takes exactly one pin, whatever the state, + and returns an opaque handle token; ``on_close(token)`` releases exactly + that pin, so a double close cannot drop another handle's pin. + """ + hydrator = self._require_hydrator() + record = hydrator.catalog.get_file_record(catalog_path) + if record is not None and hydrator.cache.is_cached(record.checksum_sha256): + # Open-time integrity check; a failed entry is evicted so request() re-hydrates. + try: + hydrator.cache.verify_entry(record.checksum_sha256) + except CacheIntegrityError: + logger.warning( + "gateway: cached entry %s failed integrity check; re-hydrating", + record.checksum_sha256, + ) + ticket = hydrator.request(catalog_path) + hydrator.cache.acquire(ticket.checksum) + try: + if timeout: + hydrator.wait(ticket, timeout) + state = hydrator.status(catalog_path) + except BaseException: + hydrator.cache.release(ticket.checksum) + raise + token = secrets.token_urlsafe(16) + with self._handles_lock: + self._handles[token] = ticket.checksum + return GatewayHandle(token=token, state=state) + + def on_read_range( + self, catalog_path: str, offset: int, length: int, *, timeout: float | None = None + ) -> bytes: + hydrator = self._require_hydrator() + return hydrator.wait_range(hydrator.request(catalog_path), offset, length, timeout) + + def on_close(self, token: str) -> None: + """Release the pin taken by the ``on_open`` that returned ``token``. + + Unknown or already-closed tokens are a no-op: SMB/NFS rigs close + handles whose open failed, and a double close must not unpin others. + """ + hydrator = self._require_hydrator() + with self._handles_lock: + checksum = self._handles.pop(token, None) + if checksum is not None: + hydrator.cache.release(checksum) + + def on_evict(self, catalog_path: str) -> bool: + """Drop the cached copy; False when the file still has open handles.""" + hydrator = self._require_hydrator() + record = hydrator.catalog.get_file_record(catalog_path) + if record is None: + return False + try: + hydrator.cache.evict(record.checksum_sha256) + except CacheEntryInUseError: + return False + return True + _gateway = ProtocolGateway() diff --git a/openblade/nas/rbac_service.py b/openblade/nas/rbac_service.py index 1b4c5ec..3a1f9ad 100644 --- a/openblade/nas/rbac_service.py +++ b/openblade/nas/rbac_service.py @@ -10,6 +10,7 @@ import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.types import ( CreateTokenRequest, CreateTokenResult, @@ -279,4 +280,4 @@ def _sanitize_details(self, value: Any) -> Any: return value def _utcnow_iso(self) -> str: - return datetime.utcnow().isoformat() + return naive_utcnow().isoformat() diff --git a/openblade/nas/restore_planner.py b/openblade/nas/restore_planner.py index b53c918..e108927 100644 --- a/openblade/nas/restore_planner.py +++ b/openblade/nas/restore_planner.py @@ -46,6 +46,7 @@ def plan(self, request: RestorePlanRequest) -> RestorePlan: requested_paths = [self._normalize_path(path) for path in request.paths] records = self._resolve_records(request.pool_id, requested_paths) + span_barcodes = self._span_barcodes_by_path() batches_by_tape: dict[str, list[str]] = defaultdict(list) required_counts: dict[str, int] = defaultdict(int) @@ -75,6 +76,14 @@ def plan(self, request: RestorePlanRequest) -> RestorePlan: ): required_counts[record.tape_barcode] += 1 batches_by_tape[record.tape_barcode].append(logical_path) + # A file spanning several tapes needs every segment's tape loaded. + for barcode in span_barcodes.get(logical_path, []): + if ( + barcode != record.tape_barcode + and logical_path not in batches_by_tape[barcode] + ): + required_counts[barcode] += 1 + batches_by_tape[barcode].append(logical_path) elif state is NasFileState.MISSING_TAPE: missing_tapes.add(record.tape_barcode or "") elif state is NasFileState.EXPORTED and record.tape_barcode: @@ -180,6 +189,20 @@ def _build_tape_load_order( return sorted(required_counts, key=lambda barcode: (-required_counts[barcode], barcode)) + def _span_barcodes_by_path(self) -> dict[str, list[str]]: + """Persisted multi-tape spans, keyed by normalized path, barcodes in segment order.""" + spans: dict[str, list[str]] = {} + for span in self.service.repository.list_nas_file_spans(): + segments = span.get("segments") + if not isinstance(segments, list): + continue + barcodes = spans.setdefault(self._normalize_path(str(span["path"])), []) + for segment in segments: + barcode = segment.get("barcode") if isinstance(segment, dict) else None + if isinstance(barcode, str) and barcode and barcode not in barcodes: + barcodes.append(barcode) + return spans + def _normalize_path(self, path: str) -> str: normalized = str(PurePosixPath("/" + str(path or "").lstrip("/"))).lstrip("/") return "" if normalized == "." else normalized diff --git a/openblade/nas/samba.py b/openblade/nas/samba.py index feb8930..b3e7d3e 100644 --- a/openblade/nas/samba.py +++ b/openblade/nas/samba.py @@ -1,5 +1,6 @@ """Samba share configuration rendering.""" +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -9,7 +10,47 @@ class SambaShare: name: str path: Path read_only: bool = True + case_sensitive: bool = False + guest_ok: bool = True def render(self) -> str: writable = "no" if self.read_only else "yes" - return f"[{self.name}]\n path = {self.path}\n writeable = {writable}\n" + lines = [ + f"[{self.name}]", + f" path = {self.path}", + f" writeable = {writable}", + f" case sensitive = {'yes' if self.case_sensitive else 'no'}", + f" guest ok = {'yes' if self.guest_ok else 'no'}", + " browseable = yes", + ] + return "\n".join(lines) + "\n" + + +def render_smb_conf(shares: Iterable[SambaShare], *, force_user: str = "root") -> str: + """Render a complete ``smb.conf`` for ``shares``. + + The global section serves guest SMB2/3 only (no SMB1) with POSIX locking on + so byte-range locks are visible across protocols. + + Oplocks and SMB2 leases are off: the hydrator replaces offline stubs with + recalled bytes behind smbd's back, and smbd cannot break a client lease + for a change it did not make. A client holding a handle lease keeps its + (deferred-close) handle on the stub's old inode and never sees the recall. + """ + header = ( + "[global]\n" + " workgroup = OPENBLADE\n" + " server role = standalone server\n" + " server min protocol = SMB2_10\n" + " map to guest = Bad User\n" + f" guest account = {force_user}\n" + " load printers = no\n" + " disable spoolss = yes\n" + " posix locking = yes\n" + " oplocks = no\n" + " level2 oplocks = no\n" + " smb2 leases = no\n" + " unix extensions = no\n" + " log file = /var/log/samba/%m.log\n" + ) + return header + "".join("\n" + share.render() for share in shares) diff --git a/openblade/nas/state_machine.py b/openblade/nas/state_machine.py new file mode 100644 index 0000000..6178185 --- /dev/null +++ b/openblade/nas/state_machine.py @@ -0,0 +1,95 @@ +"""Tape/cartridge lifecycle state machine for the tape-native NAS. + +This models the *cartridge* lifecycle. Per-file states live in +``openblade.nas.types.NasFileState`` and are deliberately not duplicated here: +a file is ``OFFLINE_ON_TAPE``/``MISSING_TAPE``/``EXPORTED`` as a consequence of +the cartridge state it sits on (see :func:`file_state_for_tape`). +""" + +from __future__ import annotations + +from enum import Enum + +from openblade.nas.types import NasFileState + + +class TapeState(str, Enum): + ONLINE = "online" + OFFLINE = "offline" + DEGRADED = "degraded" + HYDRATING = "hydrating" + EXPORTING = "exporting" + EXPORTED = "exported" + SEQUESTERED = "sequestered" + REPAIR_REQUIRED = "repair_required" + + +class TapeEvent(str, Enum): + GO_OFFLINE = "go_offline" + BRING_ONLINE = "bring_online" + ERROR_DETECTED = "error_detected" + HYDRATE_START = "hydrate_start" + HYDRATE_DONE = "hydrate_done" + EXPORT_START = "export_start" + EXPORT_DONE = "export_done" + EXPORT_CANCEL = "export_cancel" + IMPORT = "import" + SEQUESTER = "sequester" + OPERATOR_REPAIR = "operator_repair" + OPERATOR_RELEASE = "operator_release" + REPAIR_DONE = "repair_done" + + +class InvalidTransitionError(ValueError): + """Raised when an event is not allowed from the current tape state.""" + + def __init__(self, state: TapeState, event: TapeEvent) -> None: + super().__init__(f"transition {event.value!r} not allowed from {state.value!r}") + self.state = state + self.event = event + + +_S = TapeState +_E = TapeEvent + +# Explicit, exhaustive transition table. Anything absent is forbidden. +TRANSITIONS: dict[tuple[TapeState, TapeEvent], TapeState] = { + (_S.ONLINE, _E.GO_OFFLINE): _S.OFFLINE, + (_S.ONLINE, _E.ERROR_DETECTED): _S.DEGRADED, + (_S.ONLINE, _E.HYDRATE_START): _S.HYDRATING, + (_S.ONLINE, _E.EXPORT_START): _S.EXPORTING, + (_S.ONLINE, _E.SEQUESTER): _S.SEQUESTERED, + (_S.OFFLINE, _E.BRING_ONLINE): _S.ONLINE, + (_S.OFFLINE, _E.SEQUESTER): _S.SEQUESTERED, + (_S.DEGRADED, _E.OPERATOR_REPAIR): _S.REPAIR_REQUIRED, + (_S.DEGRADED, _E.SEQUESTER): _S.SEQUESTERED, + (_S.DEGRADED, _E.GO_OFFLINE): _S.OFFLINE, + (_S.HYDRATING, _E.HYDRATE_DONE): _S.ONLINE, + (_S.HYDRATING, _E.ERROR_DETECTED): _S.DEGRADED, + (_S.EXPORTING, _E.EXPORT_DONE): _S.EXPORTED, + (_S.EXPORTING, _E.EXPORT_CANCEL): _S.ONLINE, + (_S.EXPORTED, _E.IMPORT): _S.OFFLINE, + # Sequestered media only leaves quarantine through an explicit operator action. + (_S.SEQUESTERED, _E.OPERATOR_REPAIR): _S.REPAIR_REQUIRED, + (_S.SEQUESTERED, _E.OPERATOR_RELEASE): _S.ONLINE, + (_S.REPAIR_REQUIRED, _E.REPAIR_DONE): _S.ONLINE, + (_S.REPAIR_REQUIRED, _E.SEQUESTER): _S.SEQUESTERED, +} + + +def transition(state: TapeState, event: TapeEvent) -> TapeState: + try: + return TRANSITIONS[(state, event)] + except KeyError: + raise InvalidTransitionError(state, event) from None + + +def file_state_for_tape(state: TapeState) -> NasFileState: + """Planning view of a file whose only copy is on a tape in ``state``.""" + if state is TapeState.EXPORTED: + return NasFileState.EXPORTED + if state is TapeState.HYDRATING: + return NasFileState.HYDRATING + if state in (TapeState.SEQUESTERED, TapeState.REPAIR_REQUIRED): + return NasFileState.MISSING_TAPE + return NasFileState.OFFLINE_ON_TAPE diff --git a/openblade/nas/tape_orchestrator.py b/openblade/nas/tape_orchestrator.py index dd88064..f423618 100644 --- a/openblade/nas/tape_orchestrator.py +++ b/openblade/nas/tape_orchestrator.py @@ -4,7 +4,6 @@ import hashlib import threading -from datetime import datetime from pathlib import PurePosixPath from typing import Any, cast from uuid import uuid4 @@ -12,8 +11,10 @@ import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import ( ChecksumMismatchError, + DriveLeasedError, ExportRefusedError, ImportExportSlotError, MailslotUnsupportedError, @@ -237,11 +238,31 @@ def _unload(self, request: TapeOpRequest) -> dict[str, Any]: if drive_id is None: raise ValueError(f"Barcode {request.barcode} is not loaded in a drive") slot_id = request.slot_id if request.slot_id is not None else self._find_empty_slot() + self._refuse_if_leased_by_another_job(drive_id, request.job_id) result = self.library.unload(drive_id, slot_id) return self._operation_result( result, {"barcode": request.barcode, "drive_id": drive_id, "slot_id": slot_id} ) + def _refuse_if_leased_by_another_job(self, drive_id: int, job_id: str | None) -> None: + """Cross-process half of "never unload while mounted". + + Mount state is tracked per process, so a drive another job (possibly in + another process) still holds a live lease on may have LTFS mounted there. + The lease is the only shared evidence; an unload waits for it to be + released or to expire (recovery then reports the drive for reconciliation). + """ + if self.repo is None: + return + for lease in self.repo.live_leases(): + physical = ( + lease.physical_drive_id if lease.physical_drive_id is not None else lease.drive_id + ) + if physical == drive_id and lease.job_id != job_id: + raise DriveLeasedError( + f"Drive {drive_id} is leased by job {lease.job_id}; refusing to unload" + ) + def _format(self, request: TapeOpRequest) -> dict[str, Any]: confirmation = request.extras.get("format_confirmation") if not isinstance(confirmation, FormatConfirmation): @@ -670,4 +691,4 @@ def _required_int(extras: dict[str, Any], key: str, op_name: str) -> int: def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 97d5225..9f51d71 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -3,13 +3,16 @@ from __future__ import annotations import uuid -from datetime import datetime from enum import Enum from typing import Any, Literal from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator +from openblade.domain.clock import naive_utcnow +from openblade.domain.protection import ProtectionPolicy +from openblade.nas.media import Cartridge, MediaGeneration + class EffectivePolicySource(str, Enum): SYSTEM_DEFAULT = "system_default" @@ -448,6 +451,13 @@ class ArchivePlanRequest(BaseModel): shard_size_bytes: int | None = Field(default=None, ge=1) shard_strategy: ShardStrategy | None = None max_parallelism: int = Field(default=1, ge=1, le=16) + # Tape-native planning inputs (all optional; empty = previous behaviour). + cartridges: dict[str, Cartridge] = Field(default_factory=dict) + drive_generation: MediaGeneration | None = None + reserved_bytes: dict[str, int] = Field(default_factory=dict) + # Pool-level budget held by other jobs' NasReservation rows (the source of + # truth); subtracted from the summed tape capacity, not spread per barcode. + pool_reserved_bytes: int = Field(default=0, ge=0) class CacheDriveConfig(BaseModel): @@ -524,6 +534,8 @@ class NasPool(BaseModel): virtual_mount_enabled: bool = True hydration_behavior: HydrationBehavior = HydrationBehavior.QUEUE replication_factor: int = Field(default=1, ge=1, le=4) + # Explicit protection/placement policy; None means derive from replication_factor. + protection: ProtectionPolicy | None = None backup_order_mode: Literal["sequential", "parallel"] = "sequential" cache_target_id: str | None = None restore_target_path: str = "/openblade/restore" @@ -683,6 +695,22 @@ class RebuildPlanRequest(BaseModel): dry_run: bool = False +class UncommittedGeneration(BaseModel): + """A tape whose generation has no valid commit marker; it is never rebuilt.""" + + barcode: str + generation_id: str + reason: str + + +class MissingSibling(BaseModel): + """A sibling tape a planned tape's generation needs but the request omitted.""" + + barcode: str + required_by_barcode: str + generation_id: str | None = None + + class RebuildPlanResult(BaseModel): run_id: str dry_run: bool @@ -693,6 +721,8 @@ class RebuildPlanResult(BaseModel): estimated_files: int = 0 estimated_datasets: int = 0 estimated_path_mappings: int = 0 + uncommitted_generations: list[UncommittedGeneration] = Field(default_factory=list) + missing_siblings: list[MissingSibling] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list) safe_to_enqueue: bool @@ -860,8 +890,8 @@ class PathMappingRecord(BaseModel): size: int = 0 checksum: str = "" last_seen_at: str = "" - created_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") - updated_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") + created_at: str = Field(default_factory=lambda: naive_utcnow().isoformat() + "Z") + updated_at: str = Field(default_factory=lambda: naive_utcnow().isoformat() + "Z") @field_validator("logical_path", mode="before") @classmethod diff --git a/openblade/nas/virtual_fs.py b/openblade/nas/virtual_fs.py index 62212f9..7804151 100644 --- a/openblade/nas/virtual_fs.py +++ b/openblade/nas/virtual_fs.py @@ -6,13 +6,13 @@ from __future__ import annotations -from datetime import datetime from secrets import token_hex from threading import RLock import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.types import ( HydrationJob, HydrationRequest, @@ -405,4 +405,4 @@ def _split_path(self, path: str) -> list[str]: return [segment for segment in path.split("/") if segment] def _utcnow_iso(self) -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/pyproject.toml b/pyproject.toml index 5cf6af1..f276bef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ plugins = [] warn_unused_configs = true [tool.pytest.ini_options] -addopts = "-ra" +addopts = "-ra --strict-markers" asyncio_default_fixture_loop_scope = "function" testpaths = ["tests"] markers = [ @@ -134,4 +134,45 @@ markers = [ "slow: long-running simulator stress tests", "i3: Quantum i3 emulator and protocol tests", "real_i3: tests requiring a physical Quantum Scalar i3", + "contract: behavioural contract suite run against every backend pairing", + "protocols: containerised SMB/NFS protocol rig tests (make test-protocols)", + "stress: fault-injection / stress tests (nightly stress lane)", + "fuzz: property-based / fuzz tests (nightly fuzz lane)", + "rebuild: catalog rebuild tests (nightly rebuild lane)", + "mutation: tests used as the kill-set for the mutation lane", +] + +[tool.mutmut] +# mutmut 3.x (verified against installed 3.8.0 configuration.py). Scope: the +# safety-critical modules; tools/mutation_run.sh selects a subset per run. +source_paths = ["openblade"] +only_mutate = [ + "openblade/domain/policies.py", + "openblade/safety/*", + "openblade/jobs/scheduler.py", + "openblade/jobs/recovery.py", + "openblade/domain/scalar_coordinate.py", + "openblade/nas/manifest_validator.py", +] +# "-s": mutmut runs several pytest sessions in one process. openblade.bootstrap +# binds structlog to the sys.stderr seen at import (pytest's capture file in +# session 1), which is closed by session 2 -> "I/O operation on closed file" in +# the clean-test run. No capture keeps the stream valid across sessions. +pytest_add_cli_args = ["-p", "no:cacheprovider", "-q", "-s"] +# tests/safety/test_assistant_read_only.py is excluded: it does not exercise the +# mutated modules and stalled >60s locally (2026-10-09), which blocks mutmut stats. +pytest_add_cli_args_test_selection = [ + "tests/safety/test_import_guard.py", + "tests/safety/test_safety_regression.py", + "tests/unit/test_scheduler.py", + "tests/unit/test_drive_leases.py", + "tests/unit/test_recovery.py", + "tests/unit/test_scalar_coordinate.py", + "tests/unit/test_manifest_validator.py", + "tests/unit/test_import_guard.py", + "tests/unit/test_safety_gates.py", + "tests/unit/test_hardware_safety.py", + "tests/unit/test_capacity_policy.py", + "tests/unit/test_policies_guards.py", + "tests/unit/test_recovery_guards.py", ] diff --git a/scripts/deploy.py b/scripts/deploy.py index e63d355..4996b01 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -20,9 +20,19 @@ import sys import urllib.error import urllib.request +from dataclasses import asdict from openblade.config_validation import is_deployable, validate_config -from openblade.deploy import Stage, StageResult, run_deploy_pipeline +from openblade.deploy import ( + InvalidDigestError, + ReadOnlyProbe, + Stage, + StageResult, + parse_image_digest, + run_deploy_pipeline, + run_read_only_appliance_checks, + run_rollback, +) from openblade.topology import is_healthy_topology, verify_topology @@ -34,12 +44,12 @@ def _precheck() -> StageResult: return StageResult(Stage.PRECHECK, ok, detail, findings=blocking) -def _deploy(cmd: list[str] | None) -> StageResult: +def _deploy(cmd: list[str] | None, env: dict[str, str] | None = None) -> StageResult: if not cmd: return StageResult(Stage.DEPLOY, True, "skipped (no --deploy-cmd)") # `cmd` is already an argv list (shlex.split of --deploy-cmd) and no shell=True: # an operator string can therefore never become shell syntax. - completed = subprocess.run(cmd, check=False) + completed = subprocess.run(cmd, check=False, env=env) ok = completed.returncode == 0 return StageResult(Stage.DEPLOY, ok, f"`{' '.join(cmd)}` exited {completed.returncode}") @@ -66,8 +76,8 @@ def _postcheck_in_process() -> StageResult: ) -def _postcheck_live(base_url: str) -> StageResult: - def probe(method: str, path: str) -> int: +def _postcheck_live(base_url: str, *, read_only_appliance_checks: bool = False) -> StageResult: + def raw_probe(method: str, path: str) -> int: req = urllib.request.Request(base_url.rstrip("/") + path, method=method) try: # The URL is operator-supplied (--postcheck-url), so the scheme is not @@ -84,15 +94,19 @@ def probe(method: str, path: str) -> int: # satisfied by _AllWired and only endpoint reachability is meaningful. Pass no # emulator URLs so the fleet check surfaces its honest "unverified" warning # rather than falsely reporting the remote fleet as configured. + # In read-only appliance mode every request goes through ReadOnlyProbe, so a + # non-GET topology probe (e.g. the login POST) is never sent; it is reported. + probe = ReadOnlyProbe(raw_probe) if read_only_appliance_checks else raw_probe findings = verify_topology(probe=probe, context=_AllWired(), emulator_urls=[]) ok = is_healthy_topology(findings) blocking = [f.code for f in findings if f.severity == "blocking"] - return StageResult( - Stage.POSTCHECK, - ok, - "live topology OK" if ok else f"blocking: {blocking}", - findings=blocking, - ) + detail = "live topology OK" if ok else f"blocking: {blocking}" + if isinstance(probe, ReadOnlyProbe): + appliance = run_read_only_appliance_checks(probe) + ok = ok and appliance.ok + blocking += appliance.findings + detail += f"; {appliance.detail}; not sent (non-GET): {probe.skipped}" + return StageResult(Stage.POSTCHECK, ok, detail, findings=blocking) class _AllWired: @@ -102,7 +116,68 @@ def __getattr__(self, _name: str) -> object: return self +def _print_report(results: list[StageResult], as_json: bool) -> None: + if as_json: + print(json.dumps([asdict(r) | {"stage": r.stage.value} for r in results], indent=2)) + return + for r in results: + print(f" {'✓' if r.ok else '✗'} {r.stage.value}: {r.detail}") + + +def postcheck_main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(prog="deploy.py postcheck") + ap.add_argument("--base-url", required=True, help="the deployed appliance to verify") + ap.add_argument( + "--read-only-appliance-checks", + action="store_true", + help="also GET /inventory and /jobs/recovery; never sends a non-GET request", + ) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + result = _postcheck_live( + args.base_url, read_only_appliance_checks=args.read_only_appliance_checks + ) + _print_report([result], args.json) + return 0 if result.ok else 1 + + +def rollback_main(argv: list[str]) -> int: + # Deliberately no option that touches tape state: rollback redeploys the + # application image only (asserted by tests/unit/test_deploy_release.py). + ap = argparse.ArgumentParser(prog="deploy.py rollback") + ap.add_argument( + "--to", required=True, help="previous image digest: sha256: or @sha256:" + ) + ap.add_argument( + "--image", default="ghcr.io/amantux/openblade", help="image name for a bare digest" + ) + ap.add_argument( + "--deploy-cmd", + required=True, + help="redeploy command; receives OPENBLADE_IMAGE=@ in its environment", + ) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + try: + image_ref = parse_image_digest(args.to, args.image) + except InvalidDigestError as exc: + print(f"rollback refused: {exc}", file=sys.stderr) + return 2 + cmd = shlex.split(args.deploy_cmd) + result = run_rollback( + image_ref=image_ref, + redeploy=lambda ref: _deploy(cmd, env=os.environ | {"OPENBLADE_IMAGE": ref}), + notify=lambda msg: print(msg, file=sys.stderr), + ) + _print_report([result], args.json) + return 0 if result.ok else 1 + + def main(argv: list[str]) -> int: + if argv and argv[0] == "postcheck": + return postcheck_main(argv[1:]) + if argv and argv[0] == "rollback": + return rollback_main(argv[1:]) ap = argparse.ArgumentParser() ap.add_argument("--deploy-cmd", help="deploy command, e.g. 'docker compose up -d'") ap.add_argument("--skip-deploy", action="store_true", help="run pre/post checks only") diff --git a/scripts/mvp_e2e_demo.py b/scripts/mvp_e2e_demo.py index 7c2dc61..20746d6 100644 --- a/scripts/mvp_e2e_demo.py +++ b/scripts/mvp_e2e_demo.py @@ -278,7 +278,7 @@ def policy_driven_nas_flow() -> None: from fastapi.testclient import TestClient from openblade.api.main import app - from openblade.bootstrap import create_context, get_context, reset_context + from openblade.bootstrap import create_context, reset_context from openblade.config import OpenBladeConfig from openblade.nas.ingest import clear_ingest_state @@ -347,11 +347,8 @@ def policy_driven_nas_flow() -> None: ) ok("policy + pool + cache-drive configured", "policy=nas-sharded -> pool=archive-pool") - tapes = sorted( - str(s.barcode) - for s in get_context().library.inventory().slots - if s.barcode is not None and not str(s.barcode).upper().startswith("CLN") - ) + # With ``pool`` set the server derives the tape set and capacities itself; + # client-supplied ``available_tapes`` is rejected (422) by design. plan = client.post( "/nas/archive-plan", json={ @@ -360,7 +357,6 @@ def policy_driven_nas_flow() -> None: "pool": "archive-pool", "files": relative_paths, "file_sizes": file_sizes, - "available_tapes": tapes[:3], "copies": 1, "max_parallelism": 2, }, diff --git a/tests/compat/test_compat_capture.py b/tests/compat/test_compat_capture.py new file mode 100644 index 0000000..ebbd30a --- /dev/null +++ b/tests/compat/test_compat_capture.py @@ -0,0 +1,178 @@ +"""Unit tests for tools/compat_capture.py: sanitizer rules, provenance refusal, determinism.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + +import httpx +import pytest + +_PATH = Path(__file__).resolve().parents[2] / "tools" / "compat_capture.py" +_spec = importlib.util.spec_from_file_location("compat_capture", _PATH) +assert _spec and _spec.loader +cc = importlib.util.module_from_spec(_spec) +sys.modules["compat_capture"] = cc +_spec.loader.exec_module(cc) + +SECRET = "s3cr3t-T0KEN-abcdef0123456789" + + +def _dirty_case() -> dict[str, Any]: + return { + "id": "x", + "request": { + "method": "POST", + "path": "/aml/users/login", + "headers": {"Authorization": f"Bearer {SECRET}", "Cookie": f"sessionID={SECRET}"}, + "json": {"name": "admin", "password": "hunter2-plain", "newPassword": "pw2"}, + }, + "observed": { + "headers": { + "set-cookie": f"sessionID={SECRET}; expires=Sun, 11 Oct 2026 03:23:02 GMT; Path=/" + }, + "body": { + "sessionId": SECRET, + "access_token": SECRET, + "hardwareSerialNumber": "QTM-SN-998877", + "partnerBladeSerial": "PB-1234", + "link": "https://i3-lab.example.com:8443/aml/x?y=1", + "peer": "connect to 10.20.30.40:3260 ok", + "when": "2026-10-11T09:15:02.123Z", + "httpDate": "Sun, 11 Oct 2026 09:15:02 GMT", + "nested": [{"session_token": SECRET, "count": 3, "ok": True}], + "xml": f"0{SECRET}SN9", + }, + }, + } + + +def test_sanitize_scrubs_every_secret_class() -> None: + clean = cc.sanitize(_dirty_case()) + text = json.dumps(clean) + for leaked in (SECRET, "hunter2-plain", "pw2", "QTM-SN-998877", "PB-1234", "03:23:02"): + assert leaked not in text, f"leaked {leaked!r}" + for leaked in ("i3-lab.example.com", "10.20.30.40", "2026-10-11T09", "09:15:02 GMT"): + assert leaked not in text, f"leaked {leaked!r}" + body = clean["observed"]["body"] + assert clean["request"]["json"]["password"] == cc.REDACTED + assert clean["request"]["json"]["name"] == "admin" + assert clean["request"]["headers"]["Authorization"] == cc.TOKEN + assert body["link"] == f"https://{cc.HOST}/aml/x?y=1" + assert body["when"] == cc.TIMESTAMP + assert body["nested"][0] == {"session_token": cc.TOKEN, "count": 3, "ok": True} + # Cookie attributes survive; only the value is replaced. + assert clean["observed"]["headers"]["set-cookie"] == ( + f"sessionID={cc.TOKEN}; expires={cc.TIMESTAMP}; Path=/" + ) + assert "0" in body["xml"] and "SN9" not in body["xml"] + + +def test_sanitize_is_pure_and_idempotent() -> None: + dirty = _dirty_case() + before = json.dumps(dirty, sort_keys=True) + once = cc.sanitize(dirty) + assert json.dumps(dirty, sort_keys=True) == before, "input was mutated" + assert cc.sanitize(once) == once + + +def _resp(body: dict[str, Any], status: int = 200) -> httpx.Response: + return httpx.Response( + status, + json=body, + headers={"set-cookie": f"sessionID={SECRET}; Path=/", "date": "Sun, 11 Oct 2026"}, + request=httpx.Request("GET", "http://appliance.local/aml/x"), + ) + + +RECIPE = {"id": "r1", "profile": "scalar-i3-341g", "source": "inferred", "request": {}} + + +def test_build_case_is_deterministic_and_enforces_only_unsanitized_keys() -> None: + body = {"status": "ok", "sessionId": SECRET, "serialNumber": "SN1"} + a = cc.dumps_case(cc.build_case(RECIPE, _resp(body), source="self-captured")) + b = cc.dumps_case(cc.build_case(RECIPE, _resp(body), source="self-captured")) + assert a == b + case = json.loads(a) + assert SECRET not in a and "SN1" not in a + assert case["source"] == "self-captured" + assert case["expected"]["json_contains"] == {"status": "ok"} + assert "date" not in case["observed"]["headers"] + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"appliance_firmware": "341G"}, {"attest": "Jane Operator"}], + ids=["neither", "firmware-only", "attest-only"], +) +def test_captured_without_attestation_is_refused(kwargs: dict[str, str]) -> None: + with pytest.raises(cc.CaptureRefusedError): + cc.build_case(RECIPE, _resp({}), source="captured", **kwargs) + + +def test_captured_with_attestation_records_provenance() -> None: + case = cc.build_case( + RECIPE, _resp({}), source="captured", appliance_firmware="341G", attest="Jane Operator" + ) + assert (case["source"], case["firmware"], case["attested_by"]) == ( + "captured", + "341G", + "Jane Operator", + ) + + +def test_cli_refusal_writes_nothing(tmp_path: Path) -> None: + recipe = tmp_path / "r.json" + recipe.write_text(json.dumps({**RECIPE, "request": {"method": "GET", "path": "/x"}})) + out = tmp_path / "out" + + def factory() -> httpx.Client: + return httpx.Client( + base_url="http://t", transport=httpx.MockTransport(lambda r: httpx.Response(200)) + ) + + argv = ["capture", "--base-url", "http://t", "--recipe", str(recipe), "--out", str(out)] + assert cc.main([*argv, "--source", "captured"], factory=factory) == 2 + assert not out.exists() + assert cc.main(argv, factory=factory) == 0 + assert (out / "r1.json").exists() + + +def test_wrong_password_stays_distinguishable_after_redaction() -> None: + recipe = {**RECIPE, "request": {"json": {"name": "a", "password": "wrong"}}} + bad = cc.build_case(recipe, _resp({}), source="self-captured", login="a:right") + recipe["request"]["json"]["password"] = "right" + good = cc.build_case(recipe, _resp({}), source="self-captured", login="a:right") + assert bad["request"]["json"]["password"] == cc.REDACTED_INVALID + assert good["request"]["json"]["password"] == cc.REDACTED + + +@pytest.mark.parametrize( + ("dirty", "secret"), + [ + # Secret-class key holding a list: every leaf is masked, not just scalars. + ({"x-auth-tokens": ["eyJhbGciOi.secret"]}, "eyJhbGciOi.secret"), + ({"serials": {"drives": ["SN-LEAK-1"]}}, "SN-LEAK-1"), + ({"password": ["hunter2-list"]}, "hunter2-list"), + # XML attribute whose name hits a key rule. + ({"body": 'x'}, "S3CR3T"), + # XML element that carries attributes. + ({"body": 'S3CR3T2'}, "S3CR3T2"), + # Basic credentials in a key that is not token-class. + ({"note": "Basic dXNlcjpwYXNz"}, "dXNlcjpwYXNz"), + ], +) +def test_sanitize_scrubs_nested_xml_attr_and_basic_shapes( + dirty: dict[str, Any], secret: str +) -> None: + clean = cc.sanitize(dirty) + assert secret not in json.dumps(clean) + assert cc.sanitize(clean) == clean # still idempotent + + +def test_sanitize_xml_keeps_non_secret_attributes() -> None: + clean = cc.sanitize({"body": 'x'}) + assert clean["body"] == f'{cc.TOKEN}' diff --git a/tests/compat/test_compatibility_corpus.py b/tests/compat/test_compatibility_corpus.py index a1be904..b8ac436 100644 --- a/tests/compat/test_compatibility_corpus.py +++ b/tests/compat/test_compatibility_corpus.py @@ -14,11 +14,13 @@ from __future__ import annotations import json +from collections.abc import Iterator from pathlib import Path import pytest from fastapi.testclient import TestClient +from openblade.api import routes_aml_auth from openblade.api.main import app from openblade.bootstrap import create_context, reset_context from openblade.config import OpenBladeConfig @@ -32,13 +34,23 @@ def _load_cases() -> list[dict]: CASES = _load_cases() CAPTURED = [c for c in CASES if c.get("source") == "captured"] -INFERRED = [c for c in CASES if c.get("source") == "inferred"] +# self-captured = recorded from the OpenBlade emulator by tools/compat_capture.py; +# treated exactly like inferred (regression guard, never fidelity evidence). +INFERRED = [c for c in CASES if c.get("source") in {"inferred", "self-captured"}] +SELF_CAPTURED = [c for c in CASES if c.get("source") == "self-captured"] +# compat_capture redacts request passwords; replay them with the emulator's default. +_REDACTED = "" +_EMULATOR_PASSWORD = "password" +_PLACEHOLDERS = {_REDACTED: _EMULATOR_PASSWORD, "": "wrong-password"} @pytest.fixture() -def client(tmp_path: Path) -> TestClient: +def client(tmp_path: Path) -> Iterator[TestClient]: reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'compat.db'}"))) - return TestClient(app) + # The login limiter is module-global; isolate cases that use request.repeat. + routes_aml_auth._login_attempts.clear() + yield TestClient(app) + routes_aml_auth._login_attempts.clear() def _authenticate(client: TestClient) -> None: @@ -49,13 +61,60 @@ def _authenticate(client: TestClient) -> None: assert resp.status_code == 200, f"corpus auth setup failed: {resp.status_code}" +def _enable_mfa(client: TestClient, user: str, mfa_type: str) -> None: + # Same documented route pair as tests/i3/test_aml_mfa_pending.py: fetch the + # user's key, then PUT /aml/users/mfa with a valid code to enable it. + import pyotp + + login = client.post("/aml/users/login", json={"name": user, "password": _EMULATOR_PASSWORD}) + assert login.status_code == 200, f"corpus mfa setup login failed: {login.status_code}" + bearer = {"Authorization": f"Bearer {login.json()['token']}"} + key = client.get(f"/aml/users/login/mfa/{mfa_type}/key", headers=bearer) + assert key.status_code == 200, f"corpus mfa setup key failed: {key.status_code}" + code = pyotp.TOTP(key.json()["key"]).now() + enabled = client.put( + "/aml/users/mfa", + json={"type": mfa_type, "enabled": True, "authenticationCode": code}, + headers=bearer, + ) + assert enabled.status_code == 200, f"corpus mfa setup enable failed: {enabled.status_code}" + client.cookies.clear() + + def _run(client: TestClient, case: dict): + # Optional "setup": {"mfa": {"user": ..., "type": "totp"}} enables MFA first. + mfa = (case.get("setup") or {}).get("mfa") + if mfa: + _enable_mfa(client, mfa["user"], mfa["type"]) if case.get("auth") == "admin": _authenticate(client) req = case["request"] - return client.request( - req["method"], req["path"], json=req.get("json"), headers=req.get("headers") - ) + headers = dict(req.get("headers") or {}) + if "content_type" in req: + headers["Content-Type"] = req["content_type"] + kwargs: dict[str, object] = {"headers": headers or None} + if "body" in req: + kwargs["content"] = req["body"] + else: + payload = req.get("json") + if isinstance(payload, dict): + payload = { + k: _PLACEHOLDERS.get(v, v) if isinstance(v, str) else v for k, v in payload.items() + } + kwargs["json"] = payload + # request.repeat: send the request this many extra times first (e.g. to trip a limiter). + for _ in range(int(req.get("repeat", 0))): + client.request(req["method"], req["path"], **kwargs) + return client.request(req["method"], req["path"], **kwargs) + + +def _header_problems(case: dict, resp) -> list[str]: + problems = [] + for name, needle in case["expected"].get("headers", {}).items(): + value = resp.headers.get(name) + if value is None or needle not in value: + problems.append(f"header {name}={value!r} lacks {needle!r}") + return problems def _subset_matches(expected: dict, actual: dict) -> list[str]: @@ -71,7 +130,7 @@ def _subset_matches(expected: dict, actual: dict) -> list[str]: def test_corpus_is_non_empty_and_wellformed() -> None: assert CASES, "no compatibility cases found" for case in CASES: - assert case.get("source") in {"captured", "inferred"}, case.get("id") + assert case.get("source") in {"captured", "inferred", "self-captured"}, case.get("id") assert case.get("profile") and "request" in case and "expected" in case @@ -82,6 +141,7 @@ def test_inferred_case_behaviour_is_stable(case: dict, client: TestClient) -> No assert resp.status_code == case["expected"]["status"], ( f"{case['id']}: status {resp.status_code} != {case['expected']['status']}" ) + assert not _header_problems(case, resp), f"{case['id']}: {_header_problems(case, resp)}" if "json_contains" in case["expected"]: problems = _subset_matches(case["expected"]["json_contains"], resp.json()) assert not problems, f"{case['id']}: {problems}" @@ -110,7 +170,10 @@ def test_captured_case_matches_appliance(case, client: TestClient) -> None: def test_fidelity_coverage_is_reported(capsys) -> None: # Make the gap visible in test output rather than silently green. with capsys.disabled(): - print(f"\n[compatibility corpus] captured={len(CAPTURED)} inferred={len(INFERRED)}") + print( + f"\n[compatibility corpus] captured={len(CAPTURED)} " + f"inferred={len(INFERRED) - len(SELF_CAPTURED)} self-captured={len(SELF_CAPTURED)}" + ) if not CAPTURED: print( "[compatibility corpus] WARNING: i3 wire fidelity is UNVERIFIED (no appliance captures)." diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/contract/conftest.py b/tests/contract/conftest.py new file mode 100644 index 0000000..1b35bf5 --- /dev/null +++ b/tests/contract/conftest.py @@ -0,0 +1,185 @@ +"""Backend pairings for the behavioural contract suite. + +Every contract test receives a ``BackendPair`` and talks to it ONLY through the +``LibraryBackend`` / ``LTFSBackend`` Protocols. A new backend must pass this suite +before it may be selected via ``OPENBLADE_BACKEND`` (docs/test-plan.md). +""" + +from __future__ import annotations + +import inspect +import os +from collections.abc import Callable, Generator, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.backends import LibraryBackend, LTFSBackend +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.inventory import InventoryService +from openblade.jobs.scheduler import DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend + +CAPACITY = 50 * 1024 * 1024 +SIM_BARCODES = ["CTR001L8", "CTR002L8", "CTR003L8"] + + +@dataclass +class BackendPair: + name: str + library: LibraryBackend + ltfs: LTFSBackend + num_drives: int + barcodes: list[str] + + def format_all(self) -> None: + for barcode in self.barcodes: + self.ltfs.format( + barcode, + FormatConfirmation( + expected_barcode=barcode, + safety_token=SafetyToken.generate("format", barcode), + ), + ) + + def unload_everything(self) -> None: + """Return every loaded cartridge to its home slot (emulator state is shared).""" + snapshot = InventoryService(self.library).snapshot() + empty_slots = [slot.slot_id for slot in snapshot.slots if not slot.occupied] + for drive in snapshot.drives: + if drive.barcode is not None and empty_slots: + self.library.unload(drive.drive_id, empty_slots.pop(0)) + + +def _sim_pair() -> Iterator[BackendPair]: + library = MockLibraryBackend(num_slots=20, num_drives=3) + for slot_id, barcode in enumerate(SIM_BARCODES, start=1): + library.add_cartridge(slot_id, barcode) + ltfs = MockLTFSBackend(library, capacity_bytes=CAPACITY) + yield BackendPair("sim+sim", library, ltfs, 3, list(SIM_BARCODES)) + + +def _emulator_pair(tmp_path: Path) -> Iterator[BackendPair]: + os.environ.setdefault("OPENBLADE_BACKEND", "mock") + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.hardware.scalar_http.library_backend import ScalarHttpLibraryBackend + from openblade.hardware.scalar_http.session import ScalarHttpSession + + with TestClient(app) as client: + library = ScalarHttpLibraryBackend( + ScalarHttpSession(client, username="admin", password="password") + ) + snapshot = InventoryService(library).snapshot() + barcodes = [slot.barcode.value for slot in snapshot.slots if slot.barcode][:3] + # The simulator LTFS is typed against MockLibraryBackend but only consumes + # the LibraryBackend surface; the cast is the pairing under test. + ltfs = MockLTFSBackend(cast(MockLibraryBackend, library), capacity_bytes=CAPACITY) + pair = BackendPair("emulator+sim-ltfs", library, ltfs, len(snapshot.drives), barcodes) + pair.unload_everything() + try: + yield pair + finally: + pair.unload_everything() + + +def _scsi_fake_ltfs_pair() -> Iterator[BackendPair]: + """The production SCSI backend (``RealLibraryBackend`` + ``MtxChangerBackend``) + driven by a stateful fake ``mtx``: real argv building, status parsing and the + unload-while-mounted gate, with no /dev access.""" + from openblade.config import BackendMode, OpenBladeConfig + from openblade.hardware.discovery import LibraryDiscovery + from openblade.hardware.library import RealLibraryBackend + from tests.fakes.mtx import StatefulMtxRunner + + devices = tuple(f"/dev/nst{i}" for i in range(3)) + config = OpenBladeConfig( + backend=BackendMode.REAL, + real_hardware_enabled=True, + hardware_dry_run=False, + changer_device="/dev/sg3", + drive_devices=devices, + drive_serial_map=tuple((f"FAKESER{i}", i) for i in range(len(devices))), + ) + runner = StatefulMtxRunner( + "/dev/sg3", + barcodes=dict(enumerate(SIM_BARCODES, start=1)), + drive_serials={device: f"FAKESER{i}" for i, device in enumerate(devices)}, + ) + library = RealLibraryBackend( + config=config, + runner=runner, + discovery=LibraryDiscovery(changers=[], drives=[], sg_map={}), + ) + # MockLTFSBackend consumes only get_all_barcodes / find_drive_by_barcode / + # set_drive_mount_state, which RealLibraryBackend implements; the latter is + # what arms RealLibraryBackend.unload's TapeMountedError gate. + ltfs = MockLTFSBackend(cast(MockLibraryBackend, library), capacity_bytes=CAPACITY) + yield BackendPair("scsi+fake-ltfs", library, ltfs, len(devices), list(SIM_BARCODES)) + + +def _real_pair() -> Iterator[BackendPair]: + from openblade.bootstrap import get_library, get_ltfs + + library = get_library() + snapshot = InventoryService(library).snapshot() + barcodes = [slot.barcode.value for slot in snapshot.slots if slot.barcode][:1] + yield BackendPair("real+real", library, get_ltfs(), len(snapshot.drives), barcodes) + + +_REAL_ENABLED = os.environ.get("OPENBLADE_REAL_HARDWARE_ENABLED", "false").lower() == "true" + +PAIRINGS = [ + pytest.param("sim+sim", id="sim+sim"), + pytest.param("emulator+sim-ltfs", id="emulator+sim-ltfs"), + pytest.param("scsi+fake-ltfs", id="scsi+fake-ltfs"), + pytest.param( + "real+real", + id="real+real", + marks=[ + pytest.mark.real_hardware, + pytest.mark.skipif( + not _REAL_ENABLED, reason="OPENBLADE_REAL_HARDWARE_ENABLED is not true" + ), + ], + ), +] + + +@pytest.fixture(params=PAIRINGS) +def backend_pair(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[BackendPair]: + name = request.param + if name == "sim+sim": + yield from _sim_pair() + elif name == "emulator+sim-ltfs": + yield from _emulator_pair(tmp_path) + elif name == "scsi+fake-ltfs": + yield from _scsi_fake_ltfs_pair() + else: + yield from _real_pair() + + +@pytest.fixture +def catalog(backend_pair: BackendPair, tmp_path: Path) -> CatalogRepository: + # Depends on backend_pair so the emulator app's own init_db runs first. + init_db(f"sqlite:///{tmp_path / 'contract-catalog.db'}") + return CatalogRepository(get_session()) + + +def call_with_scheduler( + fn: Callable[..., Any], *args: Any, scheduler: DriveScheduler, **kwargs: Any +) -> Any: + """Call a job runner whether or not its signature takes ``scheduler``. + + ``run_archive_job`` / ``run_restore_job`` are gaining a ``scheduler=`` kwarg in a + parallel workstream; this keeps the contract test valid across both shapes. + """ + if "scheduler" in inspect.signature(fn).parameters: + kwargs["scheduler"] = scheduler + return fn(*args, **kwargs) diff --git a/tests/contract/test_archive_restore_contract.py b/tests/contract/test_archive_restore_contract.py new file mode 100644 index 0000000..f324175 --- /dev/null +++ b/tests/contract/test_archive_restore_contract.py @@ -0,0 +1,99 @@ +"""Archive → restore byte equality through the job layer, for every backend pairing.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openblade.catalog.repository import CatalogRepository +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_job +from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.shard import ShardMode +from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive +from openblade.jobs.sharded_restore import ShardedRestoreRequest, run_sharded_restore +from tests.contract.conftest import BackendPair, call_with_scheduler + +pytestmark = pytest.mark.contract + + +def _source_files(root: Path, count: int = 4) -> list[Path]: + root.mkdir() + files = [] + for index in range(count): + path = root / f"file_{index}.bin" + path.write_bytes(bytes((index + value) % 256 for value in range(700 + index))) + files.append(path) + return files + + +def test_sharded_archive_then_restore_is_byte_identical( + backend_pair: BackendPair, catalog: CatalogRepository, tmp_path: Path +) -> None: + backend_pair.format_all() + lanes = backend_pair.barcodes[: min(len(backend_pair.barcodes), backend_pair.num_drives)] + scheduler = DriveScheduler(num_drives=backend_pair.num_drives) + files = _source_files(tmp_path / "src") + request = ShardedArchiveRequest( + source_path=tmp_path / "src", + volume_group_name="contract", + lane_barcodes=lanes, + mode=ShardMode.STRIPE, + ) + job = catalog.create_job("archive", {}) + + result = run_sharded_archive( + request, backend_pair.library, backend_pair.ltfs, catalog, scheduler, job.id + ) + + assert result.errors == [] + for path in files: + dest = tmp_path / f"restored_{path.name}" + restore_job = catalog.create_job("restore", {}) + restored = run_sharded_restore( + ShardedRestoreRequest(catalog_path=str(path), dest_path=dest), + backend_pair.library, + backend_pair.ltfs, + catalog, + scheduler, + restore_job.id, + ) + assert restored.error is None + assert restored.checksum_verified + assert dest.read_bytes() == path.read_bytes() + + +def test_archive_job_then_restore_job_is_byte_identical( + backend_pair: BackendPair, catalog: CatalogRepository, tmp_path: Path +) -> None: + backend_pair.format_all() + scheduler = DriveScheduler(num_drives=backend_pair.num_drives) + files = _source_files(tmp_path / "src", count=2) + job = catalog.create_job("archive", {}) + + result = call_with_scheduler( + run_archive_job, + ArchiveRequest(source_path=tmp_path / "src", volume_group_name="contract"), + backend_pair.library, + backend_pair.ltfs, + catalog, + job_id=job.id, + scheduler=scheduler, + ) + + assert result.errors == [] + for path in files: + dest = tmp_path / f"restored_{path.name}" + restore_job = catalog.create_job("restore", {}) + restored = call_with_scheduler( + run_restore_job, + RestoreRequest(catalog_path=f"/contract/{path.name}", dest_path=dest), + backend_pair.library, + backend_pair.ltfs, + catalog, + job_id=restore_job.id, + scheduler=scheduler, + ) + assert restored.error is None + assert dest.read_bytes() == path.read_bytes() diff --git a/tests/contract/test_library_contract.py b/tests/contract/test_library_contract.py new file mode 100644 index 0000000..da99261 --- /dev/null +++ b/tests/contract/test_library_contract.py @@ -0,0 +1,76 @@ +"""LibraryBackend behavioural contract, run against every backend pairing.""" + +from __future__ import annotations + +import pytest + +from openblade.domain.backends import LibraryBackend, LTFSBackend +from openblade.domain.errors import OpenBladeError +from openblade.jobs.inventory import InventoryService +from tests.contract.conftest import BackendPair + +pytestmark = pytest.mark.contract + + +def _home_slot(library: LibraryBackend, barcode: str) -> int: + slot = library.find_slot_by_barcode(barcode) + assert slot is not None, f"{barcode} not found in any slot" + return slot + + +def test_backends_satisfy_runtime_protocols(backend_pair: BackendPair) -> None: + assert isinstance(backend_pair.library, LibraryBackend) + assert isinstance(backend_pair.ltfs, LTFSBackend) + + +def test_inventory_shape_is_consistent(backend_pair: BackendPair) -> None: + snapshot = InventoryService(backend_pair.library).snapshot() + + slot_ids = [slot.slot_id for slot in snapshot.slots] + drive_ids = [drive.drive_id for drive in snapshot.drives] + assert slot_ids and drive_ids + assert len(set(slot_ids)) == len(slot_ids) + assert len(set(drive_ids)) == len(drive_ids) + for slot in snapshot.slots: + assert slot.occupied == (slot.barcode is not None) + slot_barcodes = {slot.barcode.value for slot in snapshot.slots if slot.barcode} + assert set(backend_pair.barcodes) <= slot_barcodes + + +def test_load_unload_round_trip_keeps_barcode_lookups_consistent( + backend_pair: BackendPair, +) -> None: + library = backend_pair.library + barcode = backend_pair.barcodes[0] + slot = _home_slot(library, barcode) + drive_id = InventoryService(library).snapshot().drives[0].drive_id + + assert library.load(slot, drive_id).success + assert library.find_drive_by_barcode(barcode) == drive_id + assert library.find_slot_by_barcode(barcode) is None + loaded = library.get_drive(drive_id).barcode + assert loaded is not None and loaded.value == barcode + + assert library.unload(drive_id, slot).success + assert library.find_drive_by_barcode(barcode) is None + assert library.find_slot_by_barcode(barcode) == slot + assert library.get_drive(drive_id).barcode is None + + +def test_double_load_into_occupied_drive_is_an_error(backend_pair: BackendPair) -> None: + library = backend_pair.library + first, second = backend_pair.barcodes[0], backend_pair.barcodes[1] + first_slot, second_slot = _home_slot(library, first), _home_slot(library, second) + drive_id = InventoryService(library).snapshot().drives[0].drive_id + assert library.load(first_slot, drive_id).success + + try: + result = library.load(second_slot, drive_id) + except OpenBladeError: + pass # a typed domain error is an acceptable refusal + else: + assert not result.success, "loading into an occupied drive must fail" + + assert library.find_drive_by_barcode(first) == drive_id + assert library.find_slot_by_barcode(second) == second_slot + assert library.unload(drive_id, first_slot).success diff --git a/tests/contract/test_ltfs_contract.py b/tests/contract/test_ltfs_contract.py new file mode 100644 index 0000000..c95211c --- /dev/null +++ b/tests/contract/test_ltfs_contract.py @@ -0,0 +1,63 @@ +"""LTFSBackend behavioural contract (mount/write/read/checksum + unload safety gate).""" + +from __future__ import annotations + +import hashlib +from pathlib import Path, PurePosixPath + +import pytest + +from openblade.domain.errors import TapeMountedError +from openblade.domain.models import MountMode +from openblade.jobs.inventory import InventoryService +from tests.contract.conftest import BackendPair + +pytestmark = pytest.mark.contract + + +def _load_first(pair: BackendPair) -> tuple[str, int, int]: + barcode = pair.barcodes[0] + slot = pair.library.find_slot_by_barcode(barcode) + assert slot is not None + drive_id = InventoryService(pair.library).snapshot().drives[0].drive_id + assert pair.library.load(slot, drive_id).success + return barcode, slot, drive_id + + +def test_mount_rw_write_then_ro_read_round_trips_checksum( + backend_pair: BackendPair, tmp_path: Path +) -> None: + backend_pair.format_all() + barcode, slot, drive_id = _load_first(backend_pair) + payload = bytes(range(256)) * 64 + source = tmp_path / "in.bin" + source.write_bytes(payload) + dest = PurePosixPath("/contract/in.bin") + + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_WRITE) + backend_pair.ltfs.write_file(handle, source, dest) + stat = backend_pair.ltfs.stat(handle, dest) + assert stat.size_bytes == len(payload) + assert stat.checksum_sha256 == hashlib.sha256(payload).hexdigest() + assert backend_pair.ltfs.unmount(handle).success + + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_ONLY) + restored = tmp_path / "out.bin" + assert backend_pair.ltfs.read_file(handle, dest, restored).success + assert backend_pair.ltfs.unmount(handle).success + + assert hashlib.sha256(restored.read_bytes()).digest() == hashlib.sha256(payload).digest() + assert backend_pair.library.unload(drive_id, slot).success + + +def test_unload_while_mounted_is_refused(backend_pair: BackendPair) -> None: + backend_pair.format_all() + barcode, slot, drive_id = _load_first(backend_pair) + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_WRITE) + + with pytest.raises(TapeMountedError): + backend_pair.library.unload(drive_id, slot) + + assert backend_pair.library.find_drive_by_barcode(barcode) == drive_id + assert backend_pair.ltfs.unmount(handle).success + assert backend_pair.library.unload(drive_id, slot).success diff --git a/tests/e2e/protocols/__init__.py b/tests/e2e/protocols/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/protocols/conftest.py b/tests/e2e/protocols/conftest.py new file mode 100644 index 0000000..d603f7a --- /dev/null +++ b/tests/e2e/protocols/conftest.py @@ -0,0 +1,89 @@ +"""Fixtures for the containerised SMB/NFS protocol rig (marker: protocols). + +Path taken for NFS (and SMB file I/O): KERNEL mounts (cifs vers=3.0, nfs4) +inside the privileged `client` container; checks run there via +`docker compose exec`. Without docker every test here SKIPS. +""" + +import shutil +import subprocess +from collections.abc import Iterator + +import pytest + +# Before any test module imports scenarios, so its bare asserts show values. +pytest.register_assert_rewrite("tests.e2e.protocols.scenarios") + +from tests.e2e.protocols import rig # noqa: E402 -- must follow register_assert_rewrite + +MOUNTS = { + "smb": "mount -t cifs //samba/{s} /mnt/smb/{s} -o guest,vers=3.0,cache=none,actimeo=0", + "nfs": "mount -t nfs4 -o port=2049,noac,lookupcache=none,timeo=50 nfs-ganesha:/{s} /mnt/nfs/{s}", +} + + +class Rig: + def __init__(self, env: dict[str, str]) -> None: + self.env = env + + def compose(self, *args: str, timeout: float = 120) -> subprocess.CompletedProcess[str]: + return subprocess.run( # argv list, shell=False + [*rig.COMPOSE, *args], env=self.env, capture_output=True, text=True, timeout=timeout + ) + + def sh(self, script: str, timeout: float = 60) -> subprocess.CompletedProcess[str]: + """Run a shell snippet inside the client container.""" + return self.compose("exec", "-T", "client", "sh", "-c", script, timeout=timeout) + + def py(self, code: str, timeout: float = 60) -> subprocess.CompletedProcess[str]: + return self.compose("exec", "-T", "client", "python3", "-c", code, timeout=timeout) + + def mount(self, proto: str) -> None: + for share in ("ro", "rw"): + cmd = MOUNTS[proto].format(s=share) + res = self.sh( + f"mkdir -p /mnt/{proto}/{share}; mountpoint -q /mnt/{proto}/{share} || {cmd}" + ) + if res.returncode != 0: + pytest.skip( + f"kernel {proto} mount refused in client container: {res.stderr.strip()[:200]}" + ) + + +def _docker_ok() -> bool: + if shutil.which("docker") is None: + return False + try: + info = subprocess.run(["docker", "info"], capture_output=True, timeout=20) # argv list + except (OSError, subprocess.TimeoutExpired): + return False + return info.returncode == 0 + + +@pytest.fixture(scope="session") +def protocol_rig() -> Iterator[Rig]: + if not _docker_ok(): + pytest.skip("docker daemon not available: protocol rig cannot start") + env = rig.compose_env() + handle = Rig(env) + running = handle.compose("ps", "--status", "running", "-q", "client") + if not running.stdout.strip(): + rig.prepare() + up = handle.compose("up", "-d", "--build", "--wait", timeout=600) + if up.returncode != 0: + pytest.skip( + f"protocol rig failed to start (docker compose up): {up.stderr.strip()[-300:]}" + ) + # Always reset the offline stubs in place: a reused rig still carries the + # previous session's hydrated files and recall log. + restaged = handle.compose( + "exec", "-T", "client", "python3", "-c", rig.RESTAGE, *rig.restage_snippet_args() + ) + assert "restaged" in restaged.stdout, restaged.stderr + yield handle + + +@pytest.fixture +def mounted(protocol_rig: Rig, proto: str) -> Rig: + protocol_rig.mount(proto) + return protocol_rig diff --git a/tests/e2e/protocols/rig.py b/tests/e2e/protocols/rig.py new file mode 100644 index 0000000..81b2ec4 --- /dev/null +++ b/tests/e2e/protocols/rig.py @@ -0,0 +1,146 @@ +"""Protocol rig harness: render configs from NasShare records, stage the export. + +`python -m tests.e2e.protocols.rig` prepares deploy/nas-protocols/state/ (used +by `make protocols-up`). Offline files are stubs handled by the rig's hydrator +shim (deploy/nas-protocols/hydrator_shim.py); the real hydrator is FUSE. +""" + +import json +import os +import socket +import subprocess +from pathlib import Path + +from openblade.catalog.models import NasShare +from openblade.nas.export_render import nfs_export_from, samba_share_from +from openblade.nas.nfs import render_ganesha_conf +from openblade.nas.samba import render_smb_conf + +ROOT = Path(__file__).resolve().parents[3] +RIG = ROOT / "deploy" / "nas-protocols" +COMPOSE = ["docker", "compose", "-f", str(RIG / "docker-compose.yml")] +STATE = RIG / "state" +STUB_MAGIC = b"OPENBLADE-OFFLINE-STUB\n" +PAYLOAD = bytes(range(256)) * 4096 # 1 MiB, position-identifiable +DEEP_PARTS = ["d" * 200] * 19 # ~3.8 KiB relative path, under PATH_MAX 4096 +LONG_NAME = "n" * 251 + ".bin" # 255 bytes +UNICODE_NAME = "réсумé-日本語-🎞.txt" +# (scenario, stub mode) per protocol; each test gets its own stub. +OFFLINE = { + "blocking": "delay:2", + "timeout": "never", + "tape_error": "tape_error", + "concurrent": "delay:2", + "evict": "delay:0", + "restart": "delay:4", +} + +SHARES = [ + NasShare( + path="/rig/export/ro", + name="ro", + share_type="smb+nfs", + config_json=json.dumps({"pseudo": "/ro"}), + ), + NasShare( + path="/rig/export/rw", + name="rw", + share_type="smb+nfs", + config_json=json.dumps({"read_only": False, "case_sensitive": True, "pseudo": "/rw"}), + ), +] + + +RESTAGE = """ +import json, os, sys, glob +root = '/rig/export/rw' +offline = json.loads(sys.argv[1]); magic = sys.argv[2].encode() +ctl = os.path.join(root, '.ob-control') +for proto in ('smb', 'nfs'): + for scenario, mode in offline.items(): + rel = f'offline/{proto}-{scenario}.bin' + tmp = os.path.join(root, rel + '.restage') + with open(tmp, 'wb') as f: + f.write(magic + json.dumps({'state': 'offline_on_tape', 'mode': mode}).encode()) + os.chmod(tmp, 0o666) + os.replace(tmp, os.path.join(root, rel)) +for stale in glob.glob(os.path.join(ctl, 'offline', '*.req')) + glob.glob(os.path.join(ctl, 'offline', '*.err')): + os.unlink(stale) +open(os.path.join(ctl, 'hydration.log'), 'w').close() +print('restaged') +""" + + +def restage_snippet_args() -> list[str]: + """argv for RESTAGE: reset every offline stub in place (new inode, same path). + + Used when a running rig is reused so an earlier run's hydrated files, stale + recall requests and log lines cannot leak into the next session. + """ + return [json.dumps(OFFLINE), STUB_MAGIC.decode()] + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def compose_env() -> dict[str, str]: + env = dict(os.environ) + env.setdefault("OB_SMB_PORT", str(free_port())) + env.setdefault("OB_NFS_PORT", str(free_port())) + return env + + +def prepare() -> None: + """Render configs through openblade.nas and stage a fresh export tree.""" + export = STATE / "export" + if export.exists(): # container-created files are root-owned: clean in-container + subprocess.run( # argv list, shell=False + [ + "docker", + "run", + "--rm", + "-v", + f"{STATE}:/s", + "debian:bookworm-slim", + "rm", + "-rf", + "/s/export", + ], + check=True, + ) + STATE.mkdir(parents=True, exist_ok=True) + (STATE / "smb.conf").write_text(render_smb_conf(samba_share_from(s) for s in SHARES)) + exports = [nfs_export_from(s, export_id=i + 1) for i, s in enumerate(SHARES)] + (STATE / "ganesha.conf").write_text(render_ganesha_conf(exports)) + ro, rw = export / "ro", export / "rw" + deep = ro.joinpath(*DEEP_PARTS) + deep.mkdir(parents=True) + (deep / "leaf.txt").write_text("deep") + (rw / "offline").mkdir(parents=True) + (rw / ".ob-tape" / "offline").mkdir(parents=True) + # Clients drop recall requests here; must exist and be guest-writable before + # the shim starts (it only creates .ob-control itself, root-owned 0755). + (rw / ".ob-control" / "offline").mkdir(parents=True) + (ro / "readme.txt").write_text("hello openblade\n") + (ro / "data.bin").write_bytes(PAYLOAD) + (ro / UNICODE_NAME).write_text("unicode") + (ro / LONG_NAME).write_text("long") + (ro / "mode640.txt").write_text("acl") + (ro / "mode640.txt").chmod(0o640) + for proto in ("smb", "nfs"): + for scenario, mode in OFFLINE.items(): + rel = f"offline/{proto}-{scenario}.bin" + (rw / rel).write_bytes( + STUB_MAGIC + json.dumps({"state": "offline_on_tape", "mode": mode}).encode() + ) + (rw / ".ob-tape" / rel).write_bytes(PAYLOAD) + os.chmod(export, 0o777) + for path in [rw, *rw.rglob("*")]: + path.chmod(0o777 if path.is_dir() else 0o666) + + +if __name__ == "__main__": + prepare() diff --git a/tests/e2e/protocols/scenarios.py b/tests/e2e/protocols/scenarios.py new file mode 100644 index 0000000..788bedf --- /dev/null +++ b/tests/e2e/protocols/scenarios.py @@ -0,0 +1,151 @@ +"""Scenarios shared by test_smb.py and test_nfs.py (each binds the `proto` fixture). + +All file I/O runs inside the client container against the kernel mount at +/mnt//{ro,rw}. Hydration uses the rig shim (not FUSE): a client requests +recall by creating .ob-control/.req through the share, then polls. +""" + +import time + +import pytest + +from tests.e2e.protocols import rig +from tests.e2e.protocols.conftest import Rig + +pytestmark = pytest.mark.protocols + +POLL = """ +import sys, time +p, deadline = sys.argv[1], time.time() + float(sys.argv[2]) +try: + open(p.replace('/offline/', '/.ob-control/offline/') + '.req', 'w').close() +except FileNotFoundError: + pass # SMB: shim unlinked a concurrent client's .req mid-create; recall already queued +while time.time() < deadline: + data = open(p, 'rb').read() + if not data.startswith(b'OPENBLADE-OFFLINE-STUB'): + print('HYDRATED', len(data)); sys.exit(0) + time.sleep(0.2) +print('HYDRATION_TIMEOUT'); sys.exit(3) +""" + + +def _poll(r: Rig, proto: str, scenario: str, timeout: float) -> tuple[int, str]: + path = f"/mnt/{proto}/rw/offline/{proto}-{scenario}.bin" + res = r.compose( + "exec", "-T", "client", "python3", "-c", POLL, path, str(timeout), timeout=timeout + 30 + ) + return res.returncode, res.stdout + + +def _hydration_log(r: Rig) -> str: + return (rig.STATE / "export" / "rw" / ".ob-control" / "hydration.log").read_text() + + +def test_listing_shows_staged_files(mounted: Rig, proto: str) -> None: + res = mounted.sh(f"ls /mnt/{proto}/ro") + assert {"readme.txt", "data.bin", rig.UNICODE_NAME} <= set(res.stdout.split("\n")) + + +def test_open_read_seek_partial_range(mounted: Rig, proto: str) -> None: + code = ( + f"f=open('/mnt/{proto}/ro/data.bin','rb'); f.seek(1000); print(f.read(4).hex()); f.close()" + ) + assert mounted.py(code).stdout.strip() == rig.PAYLOAD[1000:1004].hex() + + +def test_rename_delete_on_rw_share_succeeds(mounted: Rig, proto: str) -> None: + d = f"/mnt/{proto}/rw" + res = mounted.sh( + f"echo x > {d}/a.txt && mv {d}/a.txt {d}/b.txt && rm {d}/b.txt && ! test -e {d}/b.txt" + ) + assert res.returncode == 0, res.stderr + + +def test_write_on_ro_share_is_refused(mounted: Rig, proto: str) -> None: + res = mounted.sh(f"mv /mnt/{proto}/ro/readme.txt /mnt/{proto}/ro/x.txt") + assert res.returncode != 0 + assert mounted.sh(f"test -e /mnt/{proto}/ro/readme.txt").returncode == 0 + + +def test_unicode_long_name_and_deep_path_readable(mounted: Rig, proto: str) -> None: + deep = "/".join(rig.DEEP_PARTS) + res = mounted.sh( + f"cat '/mnt/{proto}/ro/{rig.UNICODE_NAME}' /mnt/{proto}/ro/{rig.LONG_NAME} /mnt/{proto}/ro/{deep}/leaf.txt" + ) + assert res.stdout == "unicodelongdeep", res.stderr + + +def test_sparse_write_past_end_keeps_apparent_size(mounted: Rig, proto: str) -> None: + name = f"sparse-{proto}.bin" + mounted.py(f"f=open('/mnt/{proto}/rw/{name}','wb'); f.seek(64<<20); f.write(b'z'); f.close()") + st = (rig.STATE / "export" / "rw" / name).stat() + assert st.st_size == (64 << 20) + 1 + assert st.st_blocks * 512 < st.st_size // 2, f"server allocated {st.st_blocks * 512} bytes" + + +def test_interrupted_reader_then_reread_ok(mounted: Rig, proto: str) -> None: + reader = f"while true; do cat /mnt/{proto}/ro/data.bin > /dev/null; done" + mounted.sh( + f"nohup sh -c '{reader}' >/dev/null 2>&1 & sleep 1; pkill -9 -f 'data.bin > /dev/null' ; true" + ) + code = f"import hashlib;print(hashlib.sha256(open('/mnt/{proto}/ro/data.bin','rb').read()).hexdigest())" + import hashlib + + assert mounted.py(code).stdout.strip() == hashlib.sha256(rig.PAYLOAD).hexdigest() + + +def test_hydration_blocks_until_shim_recalls(mounted: Rig, proto: str) -> None: + start = time.monotonic() + rc, out = _poll(mounted, proto, "blocking", 20) + assert rc == 0 and f"HYDRATED {len(rig.PAYLOAD)}" in out + assert time.monotonic() - start >= 2 # stub mode delay:2 + + +def test_hydration_never_recalled_times_out(mounted: Rig, proto: str) -> None: + rc, out = _poll(mounted, proto, "timeout", 3) + assert (rc, out.strip()) == (3, "HYDRATION_TIMEOUT") + + +def test_offline_tape_error_is_reported(mounted: Rig, proto: str) -> None: + rc, _ = _poll(mounted, proto, "tape_error", 3) + err = rig.STATE / "export" / "rw" / ".ob-control" / "offline" / f"{proto}-tape_error.bin.err" + assert rc == 3 and err.read_text() == "OFFLINE_TAPE_UNAVAILABLE" + + +def test_concurrent_opens_trigger_one_hydration(mounted: Rig, proto: str) -> None: + path = f"/mnt/{proto}/rw/offline/{proto}-concurrent.bin" + script = f'for i in 1 2 3 4 5 6 7 8; do python3 -c "$P" {path} 20 & done; wait' + res = mounted.compose("exec", "-T", "-e", f"P={POLL}", "client", "sh", "-c", script, timeout=60) + assert res.stdout.count("HYDRATED") == 8 + assert _hydration_log(mounted).count(f"recall offline/{proto}-concurrent.bin") == 1 + + +def test_eviction_while_open_keeps_old_bytes_or_estale(mounted: Rig, proto: str) -> None: + """Shim-style atomic replace (new inode) while a handle is open.""" + rel = f"offline/{proto}-evict.bin" + assert _poll(mounted, proto, "evict", 20)[0] == 0 + code = f""" +import os, time, errno +f = open('/mnt/{proto}/rw/{rel}', 'rb'); first = f.read(4) +srv = '/rig/export/rw/{rel}' # server-side eviction, as the real evictor would +open(srv + '.tmp', 'wb').write(b'OPENBLADE-OFFLINE-STUB\\n{{"state": "offline_on_tape", "mode": "immediate"}}'); os.replace(srv + '.tmp', srv) +time.sleep(1) +try: + f.seek(0); print('OLD' if f.read(4) == first else 'NEW') +except OSError as e: + print(errno.errorcode[e.errno]) +""" + proc = mounted.compose("exec", "-T", "client", "python3", "-c", code, timeout=30) + assert proc.stdout.strip() in {"OLD", "ESTALE"}, proc.stdout + proc.stderr + + +def test_server_restart_during_hydration_completes(mounted: Rig, proto: str) -> None: + service = "samba" if proto == "smb" else "nfs-ganesha" + path = f"/mnt/{proto}/rw/offline/{proto}-restart.bin" + mounted.compose( + "exec", "-T", "client", "touch", path.replace("/offline/", "/.ob-control/offline/") + ".req" + ) + assert mounted.compose("restart", service, timeout=90).returncode == 0 + rc, out = _poll(mounted, proto, "restart", 60) + assert rc == 0 and "HYDRATED" in out diff --git a/tests/e2e/protocols/test_nfs.py b/tests/e2e/protocols/test_nfs.py new file mode 100644 index 0000000..a06b49c --- /dev/null +++ b/tests/e2e/protocols/test_nfs.py @@ -0,0 +1,58 @@ +"""NFSv4 (NFS-Ganesha, kernel nfs4 client) protocol scenarios.""" + +import pytest + +from tests.e2e.protocols.conftest import Rig +from tests.e2e.protocols.scenarios import * # noqa: F403 -- re-collect the shared scenarios bound to proto=nfs + +pytestmark = pytest.mark.protocols + + +@pytest.fixture +def proto() -> str: + return "nfs" + + +def test_nfs_is_case_sensitive(mounted: Rig) -> None: + assert mounted.sh("cat /mnt/nfs/ro/README.TXT").returncode != 0 + + +def test_nfs_posix_lock_granted_by_server(mounted: Rig) -> None: + code = "import fcntl;f=open('/mnt/nfs/rw/lock.bin','wb');fcntl.lockf(f,fcntl.LOCK_EX|fcntl.LOCK_NB,10);print('LOCKED')" + assert mounted.py(code).stdout.strip() == "LOCKED" + + +def test_nfs_acl_mapping(mounted: Rig) -> None: + """A real NFSv4 ACL is enforced server-side, beyond what the mode bits say. + + The rig's Ganesha (Fedora 41 nfs-ganesha 5.9, VFS FSAL) maps NFSv4 ACLs onto POSIX + ACLs. A DENY ACE for a named user (``nobody``, uid 65534 -- numeric principals, the + kernel client's default without idmapping) is stored as a ``user:65534:---`` POSIX + entry, so ``nobody`` is refused a read that mode 644 alone would allow, while the + owner keeps reading. The mapping is lossy (POSIX has no DENY and no + ``n``/``N`` named-attribute bits), so only the deny ACE itself is asserted to + round-trip, not the whole ACL text. Mode-bit enforcement for a second uid is kept. + """ + f = "/mnt/nfs/rw/acl.bin" + deny_ace = "D::65534:r" + assert mounted.sh(f"rm -f {f} && echo hi > {f} && chmod 644 {f}").returncode == 0 + assert mounted.sh(f"runuser -u nobody -- cat {f}").returncode == 0 + + set_ = mounted.sh(f"nfs4_setfacl -a {deny_ace} {f}") + assert set_.returncode == 0, set_.stdout + set_.stderr + assert deny_ace in mounted.sh(f"nfs4_getfacl {f}").stdout.splitlines() + assert mounted.sh(f"stat -c %a {f}").stdout.strip() == "644" + assert mounted.sh(f"runuser -u nobody -- cat {f}").returncode != 0 + owner = mounted.sh(f"cat {f}") + assert owner.returncode == 0 and owner.stdout.strip() == "hi", owner.stderr + + unset = mounted.sh(f"nfs4_setfacl -x {deny_ace} {f}") + assert unset.returncode == 0, unset.stdout + unset.stderr + assert deny_ace not in mounted.sh(f"nfs4_getfacl {f}").stdout.splitlines() + readable = mounted.sh(f"runuser -u nobody -- cat {f}") + assert readable.returncode == 0 and readable.stdout.strip() == "hi", readable.stderr + + assert mounted.sh(f"rm -f {f} && echo hi > {f}").returncode == 0 + assert mounted.sh(f"chmod 600 {f} && runuser -u nobody -- cat {f}").returncode != 0 + allowed = mounted.sh(f"chmod 644 {f} && runuser -u nobody -- cat {f}") + assert allowed.returncode == 0 and allowed.stdout.strip() == "hi", allowed.stderr diff --git a/tests/e2e/protocols/test_smb.py b/tests/e2e/protocols/test_smb.py new file mode 100644 index 0000000..ebd84d3 --- /dev/null +++ b/tests/e2e/protocols/test_smb.py @@ -0,0 +1,100 @@ +"""SMB (Samba, kernel cifs vers=3.0) protocol scenarios.""" + +import pytest + +from tests.e2e.protocols.conftest import Rig +from tests.e2e.protocols.scenarios import * # noqa: F403 -- re-collect the shared scenarios bound to proto=smb + +pytestmark = pytest.mark.protocols + + +@pytest.fixture +def proto() -> str: + return "smb" + + +def test_smb_case_insensitive_default_share(mounted: Rig) -> None: + assert mounted.sh("cat /mnt/smb/ro/README.TXT").stdout == "hello openblade\n" + + +def test_smb_case_sensitive_share_keeps_distinct_names(mounted: Rig) -> None: + res = mounted.sh( + "cd /mnt/smb/rw && echo a > Case.txt && echo b > case.txt && cat Case.txt case.txt" + ) + assert res.stdout == "a\nb\n", res.stderr + + +def test_smb_acl_maps_posix_mode(mounted: Rig) -> None: + res = mounted.sh("smbcacls -N //samba/ro mode640.txt") + assert res.returncode == 0, res.stderr + # smbcacls prints "ACL::ALLOWED//". Mode 0640 must map + # to owner=RW, group=R, Everyone=no rights. The owner's name is the server's + # netbios name (container id), so match on the OWNER: line rather than a literal. + lines = res.stdout.splitlines() + owner = next(line.split(":", 1)[1] for line in lines if line.startswith("OWNER:")) + assert f"ACL:{owner}:ALLOWED/0x0/RW" in lines, res.stdout + assert "ACL:Unix Group\\root:ALLOWED/0x0/R" in lines, res.stdout + assert "ACL:Everyone:ALLOWED/0x0/" in lines, res.stdout + + +# smbprotocol 1.15.0 has the SMB2_LOCK command code but no LOCK request/Open.lock(), so the +# snippet builds the [MS-SMB2] 2.2.26 request itself. Guest sessions cannot sign, hence +# require_signing=False and require_secure_negotiate=False. +_LOCK_SNIPPET = r""" +import struct +from collections import OrderedDict +import smbclient +from smbprotocol.exceptions import SMBResponseException +from smbprotocol.header import Commands +from smbprotocol.structure import BytesField, IntField, Structure + +class LockReq(Structure): # [MS-SMB2] 2.2.26; smbprotocol 1.15.0 has no LOCK request + COMMAND = Commands.SMB2_LOCK + def __init__(self): + self.fields = OrderedDict([ + ("structure_size", IntField(size=2, default=48)), + ("lock_count", IntField(size=2, default=1)), + ("lock_sequence", IntField(size=4)), + ("file_id", BytesField(size=16)), + ("locks", BytesField(size=24)), + ]) + super().__init__() + +def lock(f, flags): + o = getattr(f, "raw", f).fd + req = LockReq(); req["file_id"] = o.file_id + req["locks"] = struct.pack(" dict: + # Guest sessions cannot sign; smbclient requires signing by default. + cache: dict = {} + smbclient.register_session("samba", username="guest", password="", auth_protocol="ntlm", require_signing=False, connection_cache=cache) + return cache + +smbclient.ClientConfig(require_secure_negotiate=False) # needs a signing key guests lack +ca, cb = session(), session() +with smbclient.open_file(p, "wb", connection_cache=ca) as f: + f.write(b"0" * 100) +a = smbclient.open_file(p, "rb", share_access="rw", connection_cache=ca) +b = smbclient.open_file(p, "rb", share_access="rw", connection_cache=cb) +assert getattr(a, "raw", a).fd.tree_connect.session.session_id != getattr(b, "raw", b).fd.tree_connect.session.session_id +print(lock(a, EXCL_NOW), lock(b, EXCL_NOW), lock(a, UNLOCK), lock(b, EXCL_NOW), lock(b, UNLOCK)) +""" + + +def test_smb_byte_range_lock_conflicts_across_sessions(mounted: Rig) -> None: + """A conflicting exclusive lock from a second SMB session is refused until released.""" + res = mounted.py(_LOCK_SNIPPET) + assert res.returncode == 0, res.stderr[-800:] + # a locks, b conflicts (STATUS_LOCK_NOT_GRANTED), a unlocks, b now locks, b unlocks. + assert res.stdout.split() == ["GRANTED", "0xc0000055", "GRANTED", "GRANTED", "GRANTED"], ( + res.stdout + ) diff --git a/tests/fakes/__init__.py b/tests/fakes/__init__.py new file mode 100644 index 0000000..5ddd376 --- /dev/null +++ b/tests/fakes/__init__.py @@ -0,0 +1 @@ +"""Stateful fakes for external tools used by the hardware backends.""" diff --git a/tests/fakes/mtx.py b/tests/fakes/mtx.py new file mode 100644 index 0000000..14c0354 --- /dev/null +++ b/tests/fakes/mtx.py @@ -0,0 +1,153 @@ +"""A stateful ``mtx`` fake so the SCSI backend can be exercised without a changer. + +``StatefulMtxRunner`` replaces :class:`SafeRunner` and answers ``mtx -f ...`` +argv the way the real tool does: ``status`` renders the text format that +``parse_mtx_status`` consumes (byte-shaped like ``SAMPLE_MTX_THREE_DRIVES``), and +``load`` / ``unload`` / ``transfer`` mutate the element state or fail with an +mtx-like ``Request Sense`` message. Element numbering follows mtx: drives are +0-based Data Transfer Elements, storage slots are 1-based, and the import/export +slots follow the storage slots. +""" + +from __future__ import annotations + +from openblade.hardware.runner import CommandResult, SafeRunner + +_ILLEGAL_REQUEST = "mtx: Request Sense: Sense Key=Illegal Request" + + +class StatefulMtxRunner(SafeRunner): + """SafeRunner that simulates one SCSI medium changer driven by ``mtx``.""" + + def __init__( + self, + device: str, + *, + drives: int = 3, + slots: int = 50, + ie_slots: int = 2, + barcodes: dict[int, str], + drive_serials: dict[str, str] | None = None, + ) -> None: + super().__init__(dry_run=False) + self.device = device + self.drive_count = drives + self.slot_count = slots + self.ie_count = ie_slots + total = slots + ie_slots + for slot in barcodes: + if not 1 <= slot <= total: + raise ValueError(f"slot {slot} is outside 1..{total}") + self.slots: dict[int, str | None] = {s: barcodes.get(s) for s in range(1, total + 1)} + # Per drive: (barcode, source slot) when full, None when empty. + self.drives: dict[int, tuple[str, int] | None] = dict.fromkeys(range(drives)) + # ``RealLibraryBackend`` correlates drives at construction with + # ``sg_inq ``; answer it for the tape devices named here. + self.drive_serials = dict(drive_serials or {}) + self.calls: list[list[str]] = [] + + def run( + self, + args: list[str], + timeout: int | None = None, + redact_args: list[int] | None = None, + ) -> CommandResult: + self.calls.append(list(args)) + if len(args) == 2 and args[0] == "sg_inq" and args[1] in self.drive_serials: + return self._result(args, 0, stdout=_sg_inq_output(self.drive_serials[args[1]])) + if len(args) < 4 or args[:3] != ["mtx", "-f", self.device]: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + verb, operands = args[3], args[4:] + if verb == "status" and not operands: + return self._result(args, 0, stdout=self.render_status()) + numbers = _ints(operands) + if numbers is None or len(numbers) != 2: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + if verb == "load": + error = self._load(*numbers) + elif verb == "unload": + error = self._unload(*numbers) + elif verb == "transfer": + error = self._transfer(*numbers) + else: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + if error is not None: + return self._result(args, 1, stderr=f"{_ILLEGAL_REQUEST} ({error})") + return self._result(args, 0) + + def render_status(self) -> str: + lines = [ + f"Storage Changer {self.device}:{self.drive_count} Drives, " + f"{self.slot_count} Slots ( {self.ie_count} Import/Export )" + ] + for drive_id, content in self.drives.items(): + if content is None: + lines.append(f"Data Transfer Element {drive_id}:Empty") + else: + barcode, source = content + lines.append( + f"Data Transfer Element {drive_id}:Full (Storage Element {source} Loaded)" + f":VolumeTag = {barcode}" + ) + for slot_id, slot_barcode in self.slots.items(): + flag = " IMPORT/EXPORT" if slot_id > self.slot_count else "" + state = "Empty" if slot_barcode is None else f"Full :VolumeTag={slot_barcode}" + lines.append(f" Storage Element {slot_id}{flag}:{state}") + return "\n" + "\n".join(lines) + "\n" + + def _load(self, slot: int, drive: int) -> str | None: + if slot not in self.slots or drive not in self.drives: + return "invalid element address" + barcode = self.slots[slot] + if barcode is None: + return f"source element {slot} is empty" + if self.drives[drive] is not None: + return f"drive {drive} is full" + self.slots[slot] = None + self.drives[drive] = (barcode, slot) + return None + + def _unload(self, slot: int, drive: int) -> str | None: + if slot not in self.slots or drive not in self.drives: + return "invalid element address" + content = self.drives[drive] + if content is None: + return f"drive {drive} is empty" + if self.slots[slot] is not None: + return f"destination element {slot} is full" + self.slots[slot] = content[0] + self.drives[drive] = None + return None + + def _transfer(self, source: int, target: int) -> str | None: + if source not in self.slots or target not in self.slots: + return "invalid element address" + if self.slots[source] is None: + return f"source element {source} is empty" + if self.slots[target] is not None: + return f"destination element {target} is full" + self.slots[target], self.slots[source] = self.slots[source], None + return None + + @staticmethod + def _result(args: list[str], code: int, *, stdout: str = "", stderr: str = "") -> CommandResult: + return CommandResult( + args=list(args), returncode=code, stdout=stdout, stderr=stderr, elapsed_seconds=0.0 + ) + + +def _sg_inq_output(serial: str) -> str: + return ( + "standard INQUIRY:\n" + " PQual=0 PDT=1 RMB=1 LU_CONG=0 version=0x06 [SPC-4]\n" + " Vendor identification: IBM\n" + " Product identification: ULTRIUM-TD8\n" + f" Unit serial number: {serial}\n" + ) + + +def _ints(values: list[str]) -> list[int] | None: + try: + return [int(value) for value in values] + except ValueError: + return None diff --git a/tests/fault/test_fault_injection.py b/tests/fault/test_fault_injection.py index af02852..a00a9d4 100644 --- a/tests/fault/test_fault_injection.py +++ b/tests/fault/test_fault_injection.py @@ -22,6 +22,8 @@ from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend +pytestmark = pytest.mark.stress + def _confirmation(barcode: str): from openblade.domain.policies import FormatConfirmation, SafetyToken diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 05df03c..a023897 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -57,13 +57,18 @@ def drive_devices(): return [d.strip() for d in devs.split(",") if d.strip()] +def split_barcode_env(raw: str) -> list[str]: + """Split a comma-separated barcode env value (the form tools.hardware.allowlist prints).""" + return [b.strip() for b in raw.split(",") if b.strip()] + + @pytest.fixture def scratch_barcodes(): _skip_if_no_real_hardware() barcodes_str = os.environ.get("OPENBLADE_SCRATCH_BARCODES", "") if not barcodes_str: pytest.skip("No scratch barcodes configured (OPENBLADE_SCRATCH_BARCODES)") - return [b.strip() for b in barcodes_str.split(",") if b.strip()] + return split_barcode_env(barcodes_str) @pytest.fixture diff --git a/tests/hardware/readonly/__init__.py b/tests/hardware/readonly/__init__.py new file mode 100644 index 0000000..6af84bf --- /dev/null +++ b/tests/hardware/readonly/__init__.py @@ -0,0 +1 @@ +"""Read-only real-hardware tests (see test_readonly_ast_safety.py).""" diff --git a/tests/hardware/readonly/conftest.py b/tests/hardware/readonly/conftest.py new file mode 100644 index 0000000..22c71d1 --- /dev/null +++ b/tests/hardware/readonly/conftest.py @@ -0,0 +1,191 @@ +"""Fixtures for the read-only real-hardware lane. + +Gating mirrors tests/hardware/conftest.py: every test here requests +``real_hardware_guard`` (skips unless OPENBLADE_BACKEND=real and +OPENBLADE_REAL_HARDWARE_ENABLED=true). The appliance's AML web services are +reached at OPENBLADE_APPLIANCE_URL with OPENBLADE_APPLIANCE_USER / +OPENBLADE_APPLIANCE_PASSWORD; tests needing them skip with that reason when unset. +""" + +from __future__ import annotations + +import json +import os +import re +import ssl +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any + +import pytest + +from openblade.config import load_config +from openblade.domain.policies import RealHardwareGuard +from openblade.hardware.runner import CommandResult, SafeRunner +from openblade.hardware.safety import require_real_hardware + +CORPUS_DIR = Path(__file__).resolve().parents[3] / "compatibility" + + +def read_corpus() -> list[dict[str, Any]]: + return [json.loads(p.read_text()) for p in sorted(CORPUS_DIR.rglob("*.json"))] + + +class ReadOnlyViolation(RuntimeError): + """A read-only-lane test attempted an HTTP request or command that could mutate the rig.""" + + +# The only non-GET requests the read-only lane may send: the session handshake. +AUTH_REQUESTS = frozenset({("POST", "/aml/users/login"), ("POST", "/aml/auth/logout")}) + + +def check_appliance_request(method: str, path: str) -> None: + """Raise ReadOnlyViolation unless ``method path`` is a GET or the login/logout call. + + The method is matched exactly (HTTP methods are case-sensitive), so ``"get"`` + is refused rather than normalised. + """ + verb = method + if not path.startswith("/"): + raise ReadOnlyViolation(f"appliance path must be absolute, got {path!r}") + if verb == "GET" or (verb, path) in AUTH_REQUESTS: + return + raise ReadOnlyViolation(f"read-only lane refuses {verb} {path}") + + +_DEVICE = re.compile(r"/dev/[A-Za-z0-9_./:+-]+") +_LOG_PAGE = re.compile(r"(0x)?[0-9A-Fa-f]{1,2}") + + +def _is_device(arg: str) -> bool: + return _DEVICE.fullmatch(arg) is not None and ".." not in arg + + +def _is_path(arg: str) -> bool: + return arg.startswith("/") and ".." not in arg + + +def is_allowed_argv(argv: list[str]) -> bool: + """Exact argv shapes the read-only lane may execute; anything else is refused. + + Positions are fixed so a flag can't be smuggled in (``sg_logs -R`` resets log + pages; ``mtx -f dev load`` moves a cartridge). + """ + match argv: + case ["sg_inq", dev]: + return _is_device(dev) + case ["sg_logs", "-p", page, dev]: + return _LOG_PAGE.fullmatch(page) is not None and _is_device(dev) + case ["mtx", "-f", dev, "status"]: + return _is_device(dev) + case ["ls", *paths] if paths: + return all(_is_path(p) for p in paths) + return False + + +class ReadOnlyRunner(SafeRunner): + """SafeRunner that only executes the read-only argv shapes in ``is_allowed_argv``.""" + + def run( + self, + args: list[str], + timeout: int | None = None, + redact_args: list[int] | None = None, + ) -> CommandResult: + if isinstance(args, str) or not is_allowed_argv(list(args)): + raise ReadOnlyViolation(f"read-only lane refuses command {args!r}") + return super().run(args, timeout=timeout, redact_args=redact_args) + + +@dataclass +class ApplianceResponse: + status: int + body: Any + + +@dataclass +class ApplianceClient: + """Minimal cookie-session HTTP client for the real appliance's AML API.""" + + base_url: str + user: str + password: str + insecure: bool = False + _opener: urllib.request.OpenerDirector = field(init=False) + + def __post_init__(self) -> None: + context = ssl.create_default_context() + if self.insecure: + # Opt-in via OPENBLADE_APPLIANCE_INSECURE=true for a rig whose + # appliance only has its factory self-signed certificate. + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + self._opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(CookieJar()), + urllib.request.HTTPSHandler(context=context), + ) + + def request(self, method: str, path: str, json_body: Any = None) -> ApplianceResponse: + check_appliance_request(method, path) + data = None if json_body is None else json.dumps(json_body).encode() + req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config, scheme checked in fixture + self.base_url.rstrip("/") + path, + data=data, + method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with self._opener.open(req, timeout=30) as resp: + status, raw = resp.status, resp.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + try: + body: Any = json.loads(raw) if raw else None + except ValueError: + body = None + return ApplianceResponse(status=status, body=body) + + def login(self) -> ApplianceResponse: + return self.request( + "POST", "/aml/users/login", {"name": self.user, "password": self.password} + ) + + def logout(self) -> ApplianceResponse: + return self.request("POST", "/aml/auth/logout") + + +@pytest.fixture +def appliance(real_hardware_guard) -> ApplianceClient: + url = os.environ.get("OPENBLADE_APPLIANCE_URL", "") + user = os.environ.get("OPENBLADE_APPLIANCE_USER", "") + password = os.environ.get("OPENBLADE_APPLIANCE_PASSWORD", "") + if not (url and user and password): + pytest.skip( + "Appliance AML API not configured: set OPENBLADE_APPLIANCE_URL, " + "OPENBLADE_APPLIANCE_USER and OPENBLADE_APPLIANCE_PASSWORD" + ) + if not url.startswith(("https://", "http://")): + pytest.fail(f"OPENBLADE_APPLIANCE_URL must be http(s)://, got scheme of {url[:8]!r}") + insecure = os.environ.get("OPENBLADE_APPLIANCE_INSECURE") == "true" + return ApplianceClient(base_url=url, user=user, password=password, insecure=insecure) + + +@pytest.fixture +def authed_appliance(appliance: ApplianceClient): + resp = appliance.login() + assert resp.status == 200, f"appliance login failed: HTTP {resp.status}" + yield appliance + appliance.logout() + + +@pytest.fixture +def readonly_runner() -> ReadOnlyRunner: + return ReadOnlyRunner(dry_run=False) + + +@pytest.fixture +def hardware_guard(real_hardware_guard) -> RealHardwareGuard: + return require_real_hardware(load_config()) diff --git a/tests/hardware/readonly/test_drive_identity_bridge.py b/tests/hardware/readonly/test_drive_identity_bridge.py new file mode 100644 index 0000000..6ffb9da --- /dev/null +++ b/tests/hardware/readonly/test_drive_identity_bridge.py @@ -0,0 +1,59 @@ +"""AML coordinate -> serial -> sg_inq -> /dev/tape/by-id -> LTFS mount DRY. + +Resolves the device LTFS *would* mount for each drive and stops: asserts the +by-id path exists and its sg_inq serial matches the library's, then performs +no mount. +""" + +from __future__ import annotations + +import os +from typing import Any + +import pytest + +from openblade.domain.scalar_coordinate import ScalarCoordinate +from openblade.hardware.correlation import resolve_tape_by_id +from openblade.hardware.sg import sg_inq +from tests.hardware.readonly.conftest import ReadOnlyRunner + +from .conftest import ApplianceClient + +pytestmark = [pytest.mark.real_hardware, pytest.mark.usefixtures("real_hardware_guard")] + +_DRIVE_ELEMENT_TYPE = 4 # SMC Data Transfer Element + + +def _drive_elements(node: Any) -> list[dict[str, Any]]: + if isinstance(node, dict): + coord = node.get("coordinate") + if isinstance(coord, dict) and node.get("serialNumber"): + return [node] + return [e for v in node.values() for e in _drive_elements(v)] + if isinstance(node, list): + return [e for v in node for e in _drive_elements(v)] + return [] + + +def test_drive_coordinate_resolves_to_matching_by_id_device( + authed_appliance: ApplianceClient, hardware_guard +) -> None: + resp = authed_appliance.request("GET", "/aml/physicalLibrary/elements") + assert resp.status == 200 + drives = [ + e + for e in _drive_elements(resp.body) + if ScalarCoordinate.from_dict(e["coordinate"]).element_type == _DRIVE_ELEMENT_TYPE + ] + # On the rig (real_hardware_guard already skipped us off-rig) a library with + # no resolvable drive is a failure, not a pass-by-skip: the bridge is unproven. + if not drives: + pytest.fail("Appliance reported no drive elements with coordinate + serialNumber") + for element in drives: + serial = str(element["serialNumber"]) + by_id = resolve_tape_by_id(serial) + assert by_id is not None, f"no /dev/tape/by-id entry for drive serial {serial}" + assert os.path.exists(by_id) + inquiry = sg_inq(os.path.realpath(by_id), ReadOnlyRunner(dry_run=False), hardware_guard) + assert inquiry.serial.strip().casefold() == serial.strip().casefold() + # DRY: the LTFS mount target is resolved; no mount is performed. diff --git a/tests/hardware/readonly/test_readonly_ast_safety.py b/tests/hardware/readonly/test_readonly_ast_safety.py new file mode 100644 index 0000000..8a35367 --- /dev/null +++ b/tests/hardware/readonly/test_readonly_ast_safety.py @@ -0,0 +1,187 @@ +"""Plain-CI guard: tests/hardware/readonly/ must be read-only by construction. + +Fails if any module there (recursively) imports, calls or merely references a +load/unload/move/format/mount API, fetches one via ``getattr(x, "")``, +puts a mutating verb into an argv list literal, or sends a non-GET request +other than the login/logout handshake. Resolution helpers stay allowed, matched +by prefix/suffix only ("resolve_*", "plan_*", "dry_run_*", "*_dry_run_plan"). +Not rig-gated. The runtime guards in conftest.py (ReadOnlyViolation) back this +up for anything a static scan can't see. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +READONLY_DIR = Path(__file__).resolve().parent + +FORBIDDEN_EXACT = frozenset( + { + "load", "unload", "move", "move_medium", "exchange", "transfer", "eject", + "format", "mkltfs", "mount", "unmount", "umount", "ltfs", "erase", + "run_inventory_job", "inventory", + } +) # fmt: skip +FORBIDDEN_PREFIXES = ("load_", "unload_", "move_", "format_", "mount_", "erase_", "mkltfs_") +FORBIDDEN_SUBSTRINGS = ("_load", "_unload", "_move", "_format", "_mount", "_erase") +ALLOWED_PREFIXES = ("resolve_", "plan_", "dry_run_") +ALLOWED_SUFFIXES = ("_dry_run_plan",) +# Config loading reads a file, not a cartridge; exempt by exact name only. +ALLOWED_EXACT = frozenset({"load_config"}) +# Whole argv items that make a command mutate media (mtx load, mkltfs, ltfs, ...). +FORBIDDEN_ARGV_WORDS = frozenset( + { + "load", "unload", "mkltfs", "ltfs", "mount", "umount", "unmount", "transfer", + "move", "format", "eject", "erase", "exchange", + } +) # fmt: skip +WRITE_VERBS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) +AUTH_PATHS = frozenset({"/aml/users/login", "/aml/auth/logout"}) + + +def is_forbidden_name(name: str) -> bool: + lowered = name.lower() + if ( + lowered in ALLOWED_EXACT + or lowered.startswith(ALLOWED_PREFIXES) + or lowered.endswith(ALLOWED_SUFFIXES) + ): + return False + return ( + lowered in FORBIDDEN_EXACT + or lowered.startswith(FORBIDDEN_PREFIXES) + or any(s in lowered for s in FORBIDDEN_SUBSTRINGS) + ) + + +def _str_const(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _write_request_hit(node: ast.Call) -> str | None: + """Name a non-GET HTTP call unless it is a literal login/logout path.""" + func = node.func + if not isinstance(func, ast.Attribute): + return None + args = list(node.args) + if func.attr.upper() in WRITE_VERBS: + verb = func.attr.upper() + elif func.attr == "request" and args and _str_const(args[0]) is not None: + verb = str(_str_const(args[0])).upper() + args = args[1:] + if verb not in WRITE_VERBS: + return None + else: + return None + path = _str_const(args[0]) if args else None + if verb == "POST" and path in AUTH_PATHS: + return None + return f"{verb} {path if path is not None else ''}" + + +def find_forbidden(source: str) -> list[str]: + """Return ``line:what`` for every mutating reference in ``source``.""" + hits: list[str] = [] + for node in ast.walk(ast.parse(source)): + line = getattr(node, "lineno", 0) + if isinstance(node, ast.ImportFrom | ast.Import): + for alias in node.names: + if is_forbidden_name(alias.name.rsplit(".", 1)[-1]): + hits.append(f"{line}:{alias.name}") + elif isinstance(node, ast.Attribute): + # Any reference, not just a call: `op = lib.load; op(1)` is caught here. + # Bare names are checked only as call targets (below): a forbidden bare + # name can only arrive via an import (flagged above) or a local binding + # of a flagged value, and flagging every Name trips on locals such as + # `inventory = InventoryService(...).snapshot()`. + if _str_const(node.value) is not None: + continue # "...".format(...) is string formatting, not a tape API + if is_forbidden_name(node.attr): + hits.append(f"{line}:{node.attr}") + elif isinstance(node, ast.List | ast.Tuple): + for elt in node.elts: + word = _str_const(elt) + if word is not None and word.lower() in FORBIDDEN_ARGV_WORDS: + hits.append(f"{line}:argv {word!r}") + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and is_forbidden_name(node.func.id): + hits.append(f"{line}:{node.func.id}") + if isinstance(node.func, ast.Name) and node.func.id in {"getattr", "setattr"}: + attr = _str_const(node.args[1]) if len(node.args) > 1 else None + if attr is not None and is_forbidden_name(attr): + hits.append(f"{line}:getattr {attr!r}") + write = _write_request_hit(node) + if write is not None: + hits.append(f"{line}:{write}") + return hits + + +def _readonly_modules() -> list[Path]: + # The scanner scans its own source too: its snippets are whole-statement + # strings, never argv-shaped single words, so they don't misfire. + return sorted(READONLY_DIR.rglob("*.py")) + + +def test_readonly_dir_has_modules_to_scan() -> None: + assert len(_readonly_modules()) >= 3 + + +@pytest.mark.parametrize("path", _readonly_modules(), ids=lambda p: p.name) +def test_readonly_module_calls_no_mutating_api(path: Path) -> None: + assert find_forbidden(path.read_text()) == [] + + +@pytest.mark.parametrize( + "snippet", + [ + "library.load(1, 'A00001L8')", + "backend.unload(0)", + "changer.move_medium(src, dst)", + "ltfs.format_tape('A', '/dev/nst0')", + "ltfs.format('A', confirmation)", + "mount_ltfs('/dev/nst0', '/mnt')", + "from openblade.hardware.ltfs import ltfs_mount", + "from openblade.jobs.inventory import run_inventory_job", + "library.inventory()", + "svc.load_tape(1)", + # B3 bypasses: each must be flagged. + "client.post('/aml/media/move', json={'src': 1})", + "requests.put(url + '/format')", + "runner.run(['mtx', '-f', d, 'load', '1', '0'])", + "subprocess.run(['mkltfs', '--force'])", + "getattr(lib, 'load')(1)", + "op = lib.load\nop(1)", + "lib.load_plan_now(1)", + "backend.unload_dry_run(0)", + "client.request('DELETE', '/aml/users/login')", + "client.delete('/aml/partitions/1')", + "appliance.request('POST', '/aml/physicalLibrary/elements/move', {})", + ], +) +def test_scanner_flags_forbidden_snippet(snippet: str) -> None: + assert find_forbidden(snippet) + + +@pytest.mark.parametrize( + "snippet", + [ + "resolve_tape_by_id('X')", + "LTFSCommandBackend.format_dry_run_plan('A', '/dev/nst0')", + "'{}'.format(1)", + "InventoryService(lib).snapshot()", + "sg_inq(dev, runner, guard)", + "from openblade.config import load_config", + "runner.run(['mtx', '-f', d, 'status'])", + "client.get('/aml/partitions')", + "client.request('GET', '/aml/physicalLibrary/elements')", + "client.request('POST', '/aml/users/login', body)", + "emulator.post('/aml/auth/logout')", + ], +) +def test_scanner_allows_read_only_snippet(snippet: str) -> None: + assert find_forbidden(snippet) == [] diff --git a/tests/hardware/readonly/test_readonly_rig.py b/tests/hardware/readonly/test_readonly_rig.py new file mode 100644 index 0000000..8346db9 --- /dev/null +++ b/tests/hardware/readonly/test_readonly_rig.py @@ -0,0 +1,133 @@ +"""Read-only checks against the real i3 rig (nightly lane). + +Nothing here loads, moves, formats or mounts anything — enforced by +test_readonly_ast_safety.py. Skipped without the rig (see conftest gating). +""" + +from __future__ import annotations + +import json +import os +import tempfile +import warnings +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +from openblade.domain.scalar_coordinate import ScalarCoordinate +from openblade.hardware.sg import sg_inq +from openblade.hardware.tapealert import read_tape_alerts +from openblade.jobs.inventory import InventoryService + +from .conftest import CORPUS_DIR, ApplianceClient, ReadOnlyRunner, read_corpus + +pytestmark = [pytest.mark.real_hardware, pytest.mark.usefixtures("real_hardware_guard")] + + +def test_appliance_login_then_logout_succeeds(appliance: ApplianceClient) -> None: + assert appliance.login().status == 200 + assert appliance.logout().status in {200, 204} + + +def test_changer_firmware_and_identity_are_discoverable(changer_device, hardware_guard) -> None: + inquiry = sg_inq(changer_device, ReadOnlyRunner(dry_run=False), hardware_guard) + assert inquiry.vendor and inquiry.product and inquiry.revision + + +def test_inventory_snapshot_reports_slots_and_drives(real_library_backend) -> None: + inventory = InventoryService(real_library_backend).snapshot() + assert inventory.slots and inventory.drives + + +def test_element_coordinates_round_trip(authed_appliance: ApplianceClient) -> None: + resp = authed_appliance.request("GET", "/aml/physicalLibrary/elements") + assert resp.status == 200 + flat = _find_coordinates(resp.body) + assert flat, "appliance returned no element coordinates" + for raw in flat: + assert ScalarCoordinate.from_dict(raw).to_dict() == {k: raw[k] for k in _COORD_KEYS} + + +def test_partitions_are_visible(authed_appliance: ApplianceClient) -> None: + resp = authed_appliance.request("GET", "/aml/partitions") + assert resp.status == 200 and resp.body + + +def test_tapealert_health_reads_on_every_drive(drive_devices, hardware_guard) -> None: + for device in drive_devices: + report = read_tape_alerts(device, ReadOnlyRunner(dry_run=False), hardware_guard) + assert report.device + + +_COORD_KEYS = ("frame", "rack", "section", "column", "row", "type") + + +def _find_coordinates(node: Any) -> list[dict[str, Any]]: + if isinstance(node, dict): + if all(k in node for k in _COORD_KEYS): + return [node] + return [c for v in node.values() for c in _find_coordinates(v)] + if isinstance(node, list): + return [c for v in node for c in _find_coordinates(v)] + return [] + + +def _report_path() -> Path: + default = Path(tempfile.gettempdir()) / "differential-report.json" + return Path(os.environ.get("OPENBLADE_DIFFERENTIAL_REPORT", str(default))) + + +# The differential must compare SOMETHING: an empty/mis-filtered corpus would +# otherwise produce zero divergences and pass vacuously. +MIN_DIFFERENTIAL_CASES = 1 + + +def _is_read_only(case: dict[str, Any]) -> bool: + req = case["request"] + return req["method"] == "GET" or req["path"] in {"/aml/users/login", "/aml/auth/logout"} + + +def test_emulator_matches_appliance_on_compatibility_corpus( + appliance: ApplianceClient, tmp_path: Path +) -> None: + from openblade.api.main import app + from openblade.bootstrap import create_context, reset_context + from openblade.config import OpenBladeConfig + + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'diff.db'}"))) + emulator = TestClient(app) + divergences: list[dict[str, Any]] = [] + cases = [c for c in read_corpus() if _is_read_only(c)] + assert len(cases) >= MIN_DIFFERENTIAL_CASES, ( + f"differential ran {len(cases)} read-only corpus case(s) from {CORPUS_DIR}; " + f"need >= {MIN_DIFFERENTIAL_CASES} (refusing to pass vacuously)" + ) + for case in cases: + req = case["request"] + if case.get("auth") == "admin": + emulator.post("/aml/users/login", json={"name": "admin", "password": "password"}) + appliance.login() + # Credentials in corpus cases are emulator defaults; the appliance gets the rig's. + body = req.get("json") + if req["path"] == "/aml/users/login" and body and body.get("password") == "password": + body = {"name": appliance.user, "password": appliance.password} + emu = emulator.request(req["method"], req["path"], json=req.get("json")) + real = appliance.request(req["method"], req["path"], body) + if emu.status_code != real.status: + divergences.append( + { + "id": case["id"], + "source": case["source"], + "path": req["path"], + "emulator_status": emu.status_code, + "appliance_status": real.status, + } + ) + _report_path().write_text(json.dumps({"divergences": divergences}, indent=2)) + captured = [d for d in divergences if d["source"] == "captured"] + for d in divergences: + if d["source"] != "captured": + warnings.warn(f"inferred case diverges from appliance: {d}", stacklevel=1) + assert not captured, f"captured-case divergence (see {_report_path()}): {captured}" diff --git a/tests/hardware/readonly/test_readonly_runtime_guards.py b/tests/hardware/readonly/test_readonly_runtime_guards.py new file mode 100644 index 0000000..3257527 --- /dev/null +++ b/tests/hardware/readonly/test_readonly_runtime_guards.py @@ -0,0 +1,156 @@ +"""Plain-CI unit tests for the read-only lane's runtime guards (no rig needed). + +ApplianceClient.request and ReadOnlyRunner.run must refuse anything that could +mutate the rig with ReadOnlyViolation *before* touching the network or a +subprocess. The transport and subprocess.run are replaced so nothing executes. +""" + +from __future__ import annotations + +import io +import json +import subprocess +from typing import Any + +import pytest + +from .conftest import ApplianceClient, ReadOnlyRunner, ReadOnlyViolation + + +class _FakeResponse(io.BytesIO): + status = 200 + + +@pytest.fixture +def sent() -> list[tuple[str, str]]: + return [] + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, sent: list[tuple[str, str]]) -> ApplianceClient: + appliance = ApplianceClient(base_url="https://rig.invalid", user="u", password="p") + + def fake_open(req: Any, timeout: int = 0) -> _FakeResponse: + sent.append((req.get_method(), req.full_url)) + return _FakeResponse(json.dumps({"ok": True}).encode()) + + monkeypatch.setattr(appliance._opener, "open", fake_open) + return appliance + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("POST", "/aml/media/move"), + ("PUT", "/aml/partitions/1"), + ("PATCH", "/aml/partitions/1"), + ("DELETE", "/aml/partitions/1"), + ("DELETE", "/aml/users/login"), + ("POST", "/aml/users/login/../media/move"), + ("post", "/aml/physicalLibrary/elements/move"), + ("GET", "aml/partitions"), + ("get", "/aml/partitions"), + ], +) +def test_appliance_request_mutating_call_raises_before_sending( + client: ApplianceClient, sent: list[tuple[str, str]], method: str, path: str +) -> None: + with pytest.raises(ReadOnlyViolation): + client.request(method, path) + + assert sent == [] + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("GET", "/aml/partitions"), + ("POST", "/aml/users/login"), + ("POST", "/aml/auth/logout"), + ], +) +def test_appliance_request_read_or_auth_call_is_sent( + client: ApplianceClient, sent: list[tuple[str, str]], method: str, path: str +) -> None: + resp = client.request(method, path) + + assert resp.status == 200 + assert sent == [(method, "https://rig.invalid" + path)] + + +def test_appliance_login_and_logout_pass_the_guard( + client: ApplianceClient, sent: list[tuple[str, str]] +) -> None: + client.login() + client.logout() + + assert [m for m, _ in sent] == ["POST", "POST"] + + +@pytest.fixture +def executed(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_run(args: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + # SafeRunner calls subprocess.run via the module, so this intercepts it. + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + +# Commands are space-joined strings (split at call time) rather than list +# literals so test_readonly_ast_safety's argv scan of this file stays clean. +@pytest.mark.parametrize( + "command", + [ + "mtx -f /dev/sg0 load 1 0", + "mtx -f /dev/sg0 unload 1 0", + "mtx -f /dev/sg0 transfer 1 2", + "mtx -f /dev/sg0 status load", + "mkltfs --force -d /dev/nst0", + "ltfs /mnt/tape -o devname=/dev/nst0", + "mount /dev/nst0 /mnt", + "sg_logs -R /dev/sg1", + "sg_logs -p 0x2e -R /dev/sg1", + "sg_inq --page=0x83 /dev/sg1", + "sg_inq /etc/passwd", + "mt -f /dev/nst0 erase", + "ls -R /dev", + "ls", + ], +) +def test_readonly_runner_forbidden_argv_raises_before_executing( + executed: list[list[str]], command: str +) -> None: + with pytest.raises(ReadOnlyViolation): + ReadOnlyRunner(dry_run=False).run(command.split()) + + assert executed == [] + + +def test_readonly_runner_string_command_raises(executed: list[list[str]]) -> None: + with pytest.raises(ReadOnlyViolation): + ReadOnlyRunner(dry_run=False).run("mtx -f /dev/sg0 status") # type: ignore[arg-type] + + assert executed == [] + + +# Commands are space-joined strings (split at call time) rather than list +# literals so test_readonly_ast_safety's argv scan of this file stays clean. +@pytest.mark.parametrize( + "command", + [ + "mtx -f /dev/sg0 status", + "sg_inq /dev/sg1", + "sg_inq /dev/tape/by-id/scsi-3500e09e0bb562001-nst", + "sg_logs -p 0x2e /dev/sg1", + "ls /dev/tape/by-id", + ], +) +def test_readonly_runner_allowed_argv_executes(executed: list[list[str]], command: str) -> None: + result = ReadOnlyRunner(dry_run=False).run(command.split()) + + assert result.returncode == 0 + assert executed == [command.split()] diff --git a/tests/hardware/test_allowlist.py b/tests/hardware/test_allowlist.py new file mode 100644 index 0000000..4d40274 --- /dev/null +++ b/tests/hardware/test_allowlist.py @@ -0,0 +1,90 @@ +"""Unit tests for tools/hardware/allowlist.py — run without the rig.""" + +from __future__ import annotations + +import pytest + +from tools.hardware.allowlist import ( + EXIT_OK, + EXIT_VIOLATION, + AllowlistError, + check_subset, + format_barcodes, + main, + parse_barcodes, +) + + +def test_parse_barcodes_splits_commas_and_whitespace() -> None: + assert parse_barcodes(" a00001l8, A00002L8\nA00003L8 a00001L8 ") == [ + "A00001L8", + "A00002L8", + "A00003L8", + ] + + +def test_check_subset_accepts_subset() -> None: + assert check_subset(["A1"], ["A1", "A2"]) == ["A1"] + + +def test_check_subset_outside_allowlist_names_offenders() -> None: + with pytest.raises(AllowlistError, match="PROD9, PROD8"): + check_subset(["A1", "PROD9", "PROD8"], ["A1", "A2"]) + + +def test_check_subset_empty_allowlist_fails_closed() -> None: + with pytest.raises(AllowlistError, match="fail closed"): + check_subset(["A1"], []) + + +def test_check_subset_empty_request_refused() -> None: + with pytest.raises(AllowlistError, match="No barcodes"): + check_subset([], ["A1"]) + + +def test_main_offender_exits_2_with_message(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1,A2") + assert main(["--barcodes", "A1 PROD9"]) == EXIT_VIOLATION + assert "PROD9" in capsys.readouterr().err + + +def test_main_falls_back_to_env_barcodes(monkeypatch) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1 A2") + monkeypatch.setenv("BARCODES", "a2") + assert main([]) == EXIT_OK + + +def test_main_unset_allowlist_exits_2(monkeypatch) -> None: + monkeypatch.delenv("SACRIFICIAL_BARCODES", raising=False) + assert main(["--barcodes", "A1"]) == EXIT_VIOLATION + + +def test_main_success_prints_only_normalized_list_on_stdout(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1 A2 A3") + assert main(["--barcodes", " a2, A1\na2 "]) == EXIT_OK + out, err = capsys.readouterr() + assert out == "A2,A1\n" + assert "OK" in err + + +def test_main_refusal_prints_nothing_on_stdout(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1") + assert main(["--barcodes", "PROD9"]) == EXIT_VIOLATION + assert capsys.readouterr().out == "" + + +def test_allowlist_output_round_trips_through_conftest_parser(monkeypatch, capsys) -> None: + from tests.hardware.conftest import split_barcode_env + + monkeypatch.setenv("SACRIFICIAL_BARCODES", "TST001L8,TST002L8,TST003L8") + assert main(["--barcodes", "tst003l8 TST001L8,tst003L8"]) == EXIT_OK + printed = capsys.readouterr().out.strip() + assert split_barcode_env(printed) == ["TST003L8", "TST001L8"] + assert format_barcodes(split_barcode_env(printed)) == printed + + +def test_conftest_parser_still_accepts_spaced_comma_lists() -> None: + from tests.hardware.conftest import split_barcode_env + + assert split_barcode_env(" A1 , A2,,") == ["A1", "A2"] + assert split_barcode_env("") == [] diff --git a/tests/hardware/test_differential_corpus.py b/tests/hardware/test_differential_corpus.py new file mode 100644 index 0000000..ebe1aea --- /dev/null +++ b/tests/hardware/test_differential_corpus.py @@ -0,0 +1,16 @@ +"""Off-rig guard for the read-only differential: its corpus filter must select cases. + +The on-rig differential asserts a minimum case count too, but only on the rig; +this catches an emptied or mis-filtered corpus in ordinary CI first. +""" + +from __future__ import annotations + +from tests.hardware.readonly.conftest import read_corpus +from tests.hardware.readonly.test_readonly_rig import MIN_DIFFERENTIAL_CASES, _is_read_only + + +def test_differential_corpus_has_minimum_read_only_cases() -> None: + cases = [c for c in read_corpus() if _is_read_only(c)] + assert MIN_DIFFERENTIAL_CASES > 0 + assert len(cases) >= MIN_DIFFERENTIAL_CASES diff --git a/tests/hardware/test_junit_gate.py b/tests/hardware/test_junit_gate.py new file mode 100644 index 0000000..d55d511 --- /dev/null +++ b/tests/hardware/test_junit_gate.py @@ -0,0 +1,49 @@ +"""Unit tests for the skip-as-pass gate (no rig needed).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools.hardware import junit_gate + + +def _report(tmp_path: Path, body: str, name: str = "r.xml") -> Path: + path = tmp_path / name + path.write_text(f'{body}') + return path + + +PASS = '' +SKIP = '' +FAIL = '' + + +def test_all_passed_is_green(tmp_path: Path) -> None: + assert junit_gate.main([str(_report(tmp_path, PASS))]) == 0 + + +@pytest.mark.parametrize("body", [SKIP, PASS + SKIP, "", FAIL]) +def test_skip_or_nothing_passed_is_red(tmp_path: Path, body: str) -> None: + assert junit_gate.main([str(_report(tmp_path, body))]) == 1 + + +def test_expected_skip_is_allowed(tmp_path: Path) -> None: + path = _report(tmp_path, PASS + SKIP) + assert junit_gate.main([str(path), "--expected-skip", "t.m::test_skip"]) == 0 + + +def test_expected_skip_does_not_mask_zero_passed(tmp_path: Path) -> None: + path = _report(tmp_path, SKIP) + assert junit_gate.main([str(path), "--expected-skip", "t.m::test_skip"]) == 1 + + +def test_missing_report_is_red(tmp_path: Path) -> None: + assert junit_gate.main([str(tmp_path / "absent.xml")]) == 1 + + +def test_multiple_reports_aggregate(tmp_path: Path) -> None: + a = _report(tmp_path, PASS, "a.xml") + b = _report(tmp_path, SKIP, "b.xml") + assert junit_gate.main([str(a), str(b)]) == 1 diff --git a/tests/hardware/test_ltfs_operations.py b/tests/hardware/test_ltfs_operations.py index e644561..df06c5d 100644 --- a/tests/hardware/test_ltfs_operations.py +++ b/tests/hardware/test_ltfs_operations.py @@ -42,23 +42,37 @@ def _unload_barcode(changer_device: str, runner, slot_id: int): assert result.returncode == 0, result.stderr -def _format_tape(runner, drive_device: str, barcode: str): - # --tape-serial takes exactly 6 alphanumeric characters. Real LTO barcodes - # are 6-8 (typically 8: six characters plus a two-character media-type - # suffix such as "L8"), so passing a barcode there fails every time with - # "LTFS15029E Tape serial must be 6 characters." The barcode belongs in - # --volume-name, which is unbounded - and that is what - # openblade.hardware.ltfs.LTFSCommandBackend.format_tape uses. - result = runner.run( - [ - "mkltfs", - f"--device={resolve_sg_device(drive_device)}", - f"--volume-name={barcode}", - "--force", - ], - timeout=LTFS_TIMEOUT, - ) - assert result.returncode == 0, result.stderr +def _format_tape(app_context, drive_device: str, barcode: str): + """Format ``barcode`` through the product's two-phase flow - never mkltfs directly. + + Phase 1 is ``FormatService.dry_run``: it checks the barcode against live + inventory, persists a one-time SafetyToken and returns the plan. Phase 2 is + ``FormatService.confirm`` with exactly that token, which re-validates it + against the stored row before LTFSCommandBackend.format_tape runs mkltfs. + The cartridge is already in drive 0 (``_load_barcode``), and the orchestrator + formats a loaded cartridge in place without moving it. + """ + service = app_context.format_service + # The tests mount ``drive_device``; make sure the service formats the same + # drive, otherwise every later mount would be testing an unformatted tape. + library = app_context.library + drive_id = library.find_drive_by_barcode(barcode) + assert drive_id is not None, f"{barcode} is not loaded in a drive the app can see" + drive_device_for = getattr(library, "drive_device", None) + if drive_device_for is not None: + assert resolve_sg_device(drive_device_for(drive_id)) == resolve_sg_device(drive_device), ( + f"App would format {barcode} in drive {drive_id}, not {drive_device}" + ) + + plan, token = service.dry_run(barcode) + assert plan.operation == "format" + assert plan.is_destructive + assert plan.affected_barcodes == [barcode] + assert token.operation == "format" + assert token.target_barcode == barcode + + result = service.confirm(barcode, token.token) + assert result.success, result.message return result @@ -98,12 +112,13 @@ def test_mkltfs_formats_tape( drive_devices, runner, scratch_barcode, + real_app_context, ): """Requires: a scratch tape that is safe to format and load into drive 0.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - result = _format_tape(runner, drive_devices[0], scratch_barcode) - assert result.returncode == 0 + result = _format_tape(real_app_context, drive_devices[0], scratch_barcode) + assert result.success finally: _unload_barcode(changer_device, runner, slot_id) @@ -114,12 +129,13 @@ def test_ltfs_mount_formatted( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) result = _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) assert result.returncode == 0 assert tmp_mount_dir.exists() @@ -137,6 +153,7 @@ def test_ltfs_write_small_file( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -144,7 +161,7 @@ def test_ltfs_write_small_file( slot_id = _load_barcode(changer_device, runner, scratch_barcode) payload = tmp_path / "small.bin" try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) _write_payload(payload) target = tmp_mount_dir / payload.name @@ -165,6 +182,7 @@ def test_ltfs_read_small_file( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -174,7 +192,7 @@ def test_ltfs_read_small_file( restored = tmp_path / "restored.bin" try: expected_sha = _write_payload(source) - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) target = tmp_mount_dir / source.name target.write_bytes(source.read_bytes()) @@ -195,12 +213,13 @@ def test_ltfs_unmount_clean( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) result = _unmount_ltfs(runner, tmp_mount_dir) assert result.returncode == 0 @@ -214,6 +233,7 @@ def test_ltfs_remount_persistence( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -221,7 +241,7 @@ def test_ltfs_remount_persistence( slot_id = _load_barcode(changer_device, runner, scratch_barcode) payload = tmp_path / "persist.bin" try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) expected_sha = _write_payload(payload) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) target = tmp_mount_dir / payload.name @@ -245,12 +265,13 @@ def test_ltfs_capacity_reporting( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) result = runner.run(["df", "-h", str(tmp_mount_dir)], timeout=30) assert result.returncode == 0, result.stderr diff --git a/tests/hardware/test_snapshot.py b/tests/hardware/test_snapshot.py new file mode 100644 index 0000000..1de119e --- /dev/null +++ b/tests/hardware/test_snapshot.py @@ -0,0 +1,92 @@ +"""Unit tests for tools/hardware/snapshot.py — simulator backend, no rig.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.config import BackendMode, OpenBladeConfig +from tools.hardware.snapshot import ( + EXIT_BAD_INPUT, + EXIT_CHANGED, + EXIT_REFUSED, + capture_snapshot, + diff_snapshots, + dry_run_refusal, + main, +) + + +def _snap(slots, drives, changer="idle"): + return {"library_id": "L", "slots": slots, "drives": drives, "changer_state": changer} + + +def test_capture_against_simulator_is_json_serializable(tmp_path: Path) -> None: + library = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 's.db'}")).library + snap = capture_snapshot(library) + assert snap["slots"] and "slot_id" in snap["slots"][0] + assert json.loads(json.dumps(snap)) == snap + + +def test_diff_identical_snapshots_reports_unchanged() -> None: + s = _snap([{"slot_id": 1, "barcode": "A"}], [{"drive_id": 0, "barcode": None}]) + assert diff_snapshots(s, s)["changed"] is False + + +def test_diff_reports_moved_cartridge_per_element() -> None: + before = _snap([{"slot_id": 1, "barcode": "A"}], [{"drive_id": 0, "barcode": None}]) + after = _snap([{"slot_id": 1, "barcode": None}], [{"drive_id": 0, "barcode": "A"}]) + result = diff_snapshots(before, after) + assert result["changed"] is True + assert result["slots"][0]["slot_id"] == "1" and result["slots"][0]["after"]["barcode"] is None + assert result["drives"][0]["after"]["barcode"] == "A" + + +def test_diff_cli_writes_file_and_rejects_bad_input(tmp_path: Path) -> None: + good = tmp_path / "a.json" + good.write_text(json.dumps(_snap([], []))) + out = tmp_path / "d.json" + assert main(["diff", str(good), str(good), "--out", str(out)]) == 0 + assert json.loads(out.read_text())["changed"] is False + bad = tmp_path / "bad.json" + bad.write_text("not json") + assert main(["diff", str(good), str(bad), "--out", str(out)]) == EXIT_BAD_INPUT + + +def test_diff_cli_changed_snapshots_exit_1_after_writing(tmp_path: Path) -> None: + a, b, out = tmp_path / "a.json", tmp_path / "b.json", tmp_path / "d.json" + a.write_text(json.dumps(_snap([{"slot_id": 1, "barcode": "A"}], []))) + b.write_text(json.dumps(_snap([{"slot_id": 1, "barcode": None}], []))) + assert main(["diff", str(a), str(b), "--out", str(out)]) == EXIT_CHANGED + assert json.loads(out.read_text())["changed"] is True + + +def test_diff_detects_library_id_change_alone() -> None: + before = _snap([{"slot_id": 1, "barcode": "A"}], []) + after = dict(before, library_id="OTHER") + result = diff_snapshots(before, after) + assert result["changed"] is True + assert result["library_id"] == {"before": "L", "after": "OTHER"} + + +def test_dry_run_refusal_only_for_real_backend_in_dry_run() -> None: + assert dry_run_refusal(OpenBladeConfig(backend=BackendMode.REAL, hardware_dry_run=True)) + assert dry_run_refusal(OpenBladeConfig(backend=BackendMode.REAL)) is None + assert dry_run_refusal(OpenBladeConfig(hardware_dry_run=True)) is None + + +def test_capture_cli_refuses_real_dry_run_with_exit_2(monkeypatch, tmp_path: Path, capsys) -> None: + import tools.hardware.snapshot as snapshot + + def _must_not_build(_config): + raise AssertionError("capture touched the backend despite dry-run") + + monkeypatch.setattr(snapshot, "_build_library", _must_not_build) + monkeypatch.setenv("OPENBLADE_BACKEND", "real") + monkeypatch.setenv("OPENBLADE_HARDWARE_DRY_RUN", "true") + out = tmp_path / "before.json" + assert main(["capture", "--out", str(out)]) == EXIT_REFUSED == 2 + assert not out.exists() + captured = capsys.readouterr() + assert "snapshot: REFUSED:" in captured.err and captured.out == "" diff --git a/tests/i3/conftest.py b/tests/i3/conftest.py index 136b6ad..44bf166 100644 --- a/tests/i3/conftest.py +++ b/tests/i3/conftest.py @@ -6,7 +6,13 @@ I3_AML_URL base URL for AML API (default: http://localhost:8000) I3_AML_USER AML username (default: admin) I3_AML_PASSWORD AML password (default: password) - I3_TIMING_PROFILE instant | realistic | hardware + I3_TIMING_PROFILE instant | realistic | hardware | + I3_FAULTS_SERVER_SIDE 1 = emulator injects faults (default when I3_TEST_MODE=emulator) + +Server-side faults: the emulator must be started with +OPENBLADE_EMULATOR_FAULT_PROFILE set to the SAME name as I3_TIMING_PROFILE +(presets in openblade.api.aml_faults). No AML endpoint exposes the fault +profile name, so the client cannot assert the two match -- keep them in sync. I3_REAL_HARDWARE_ENABLED safety gate — must be "true" to run real-i3 tests OPENBLADE_API_TOKEN native-REST bearer token, if the target enforces one OPENBLADE_API_TOKEN_FILE file holding that token (takes precedence) diff --git a/tests/i3/test_aml_mfa_pending.py b/tests/i3/test_aml_mfa_pending.py new file mode 100644 index 0000000..5ba25ee --- /dev/null +++ b/tests/i3/test_aml_mfa_pending.py @@ -0,0 +1,70 @@ +"""MFA-required pending session (emulator only; inferred, not in the Rev D manual). + +A login by a user with an enabled MFA record answers the documented login body +plus ``Warning: 199 - "MFA Required"``; until ``POST /aml/users/login/mfa`` +succeeds the session is refused (401) everywhere except the MFA challenge routes +and logout. The challenge wire shape must be certified by appliance capture. +""" + +from __future__ import annotations + +import httpx +import pyotp +import pytest + +pytestmark = pytest.mark.i3 + +MFA_WARNING = '199 - "MFA Required"' + + +def _bearer(client: httpx.Client, user: str, password: str) -> dict[str, str]: + resp = client.post("/aml/users/login", json={"name": user, "password": password}) + assert resp.status_code in (200, 412), resp.text + return {"Authorization": f"Bearer {resp.json()['token']}"} + + +def test_mfa_pending_session_gated_until_totp( + i3_mode: str, i3_base_url: str, i3_credentials: tuple[str, str] +) -> None: + if i3_mode != "emulator": + pytest.skip("enabling MFA on a real appliance account is out of scope") + user, password = i3_credentials + with httpx.Client(base_url=i3_base_url, timeout=30.0) as client: + setup = _bearer(client, user, password) + secret = client.get("/aml/users/login/mfa/totp/key", headers=setup).json()["key"] + totp = pyotp.TOTP(secret) + enabled = client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": True, "authenticationCode": totp.now()}, + headers=setup, + ) + assert enabled.status_code == 200, enabled.text + client.cookies.clear() + pending: dict[str, str] = {} + try: + login = client.post("/aml/users/login", json={"name": user, "password": password}) + assert login.status_code in (200, 412) + assert MFA_WARNING in login.headers.get_list("Warning") + pending = {"Authorization": f"Bearer {login.json()['token']}"} + client.cookies.clear() + assert client.get("/aml/physicalLibrary/elements", headers=pending).status_code == 401 + ok = client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": totp.now()}, + headers=pending, + ) + assert ok.status_code == 200, ok.text + assert client.get("/aml/physicalLibrary/elements", headers=pending).status_code == 200 + finally: + if pending: + client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": totp.now()}, + headers=pending, + ) + client.cookies.clear() + client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": False, "authenticationCode": totp.now()}, + headers=pending or setup, + ) diff --git a/tests/i3/test_timing_profiles.py b/tests/i3/test_timing_profiles.py new file mode 100644 index 0000000..409f78b --- /dev/null +++ b/tests/i3/test_timing_profiles.py @@ -0,0 +1,262 @@ +"""Unit tests for tests/i3/timing.py profiles. + +Runs entirely under ``VirtualClock`` -- no emulator, no real sleeping. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Iterator + +import pytest + +from tests.i3.timing import ( + PROFILE_NAMES, + ProfileName, + ProfileRuntime, + RealClock, + RetryableOpError, + SessionExpiredError, + VirtualClock, + emulator_env, + get_clock, + get_profile_name, + use_clock, + wait_for_op, +) + +_MAX_WALL_S = 1.0 + + +@pytest.fixture +def vclock(monkeypatch: pytest.MonkeyPatch) -> Iterator[VirtualClock]: + monkeypatch.delenv("I3_TIMING_PROFILE", raising=False) + monkeypatch.delenv("I3_TEST_MODE", raising=False) + monkeypatch.setenv("I3_FAULTS_SERVER_SIDE", "0") + clock = VirtualClock() + started = time.monotonic() + with use_clock(clock): + yield clock + assert time.monotonic() - started < _MAX_WALL_S, "profile test slept for real" + + +def _runtime(name: ProfileName, clock: VirtualClock) -> ProfileRuntime: + return ProfileRuntime(name, clock=clock) + + +# --- Clock ------------------------------------------------------------------ + + +def test_virtual_clock_sleep_advances_and_records() -> None: + clock = VirtualClock(start=10.0) + + clock.sleep(2.5) + clock.advance(1.0) + + assert clock.monotonic() == 13.5 + assert clock.sleeps == [2.5] + + +def test_virtual_clock_negative_duration_raises() -> None: + clock = VirtualClock() + + with pytest.raises(ValueError): + clock.sleep(-1.0) + with pytest.raises(ValueError): + clock.advance(-1.0) + + +def test_use_clock_restores_previous_clock() -> None: + before = get_clock() + + with use_clock(VirtualClock()) as installed: + assert get_clock() is installed + + assert get_clock() is before + assert isinstance(before, RealClock) + + +# --- Profile selection -------------------------------------------------------- + + +@pytest.mark.parametrize("name", PROFILE_NAMES) +def test_profile_name_from_env_resolves(name: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", f" {name.upper()} ") + + assert get_profile_name() == name + + +def test_unknown_profile_falls_back_to_mode_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", "bogus") + monkeypatch.setenv("I3_TEST_MODE", "real") + + assert get_profile_name() == "hardware" + + +# --- Delays through the clock ------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "expected"), [("instant", []), ("realistic", [3.0]), ("hardware", [35.0])] +) +def test_wait_for_op_legacy_profiles_sleep_via_clock( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch, name: str, expected: list[float] +) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", name) + + wait_for_op("tape_load") + + assert vclock.sleeps == expected + + +def test_wait_for_op_hardware_rewind_does_not_really_sleep( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", "hardware") + + wait_for_op("rewind", multiplier=2.0) + + assert vclock.monotonic() == 180.0 + + +def test_slow_robotics_move_is_three_times_normal(vclock: VirtualClock) -> None: + normal, slow = _runtime("normal", vclock), _runtime("slow-robotics", vclock) + + normal.perform("move") + slow.perform("move") + + assert vclock.sleeps == [1.5, 4.5] + + +def test_busy_library_adds_queue_wait_to_mechanical_ops_only(vclock: VirtualClock) -> None: + busy = _runtime("busy-library", vclock) + + busy.perform("tape_load") + busy.perform("auth") + + assert vclock.sleeps == [5.0, 0.1] + + +# --- Faults ----------------------------------------------------------------- + + +def _load_outcomes(runtime: ProfileRuntime, count: int) -> list[int | None]: + outcomes: list[int | None] = [] + for _ in range(count): + try: + runtime.perform("tape_load") + outcomes.append(None) + except RetryableOpError as exc: + outcomes.append(exc.status) + return outcomes + + +def test_intermittent_drive_every_third_load_fails_retryably(vclock: VirtualClock) -> None: + runtime = _runtime("intermittent-drive", vclock) + + outcomes = _load_outcomes(runtime, 6) + + assert outcomes == [None, None, 409, None, None, 409] + + +def test_normal_profile_loads_never_fail(vclock: VirtualClock) -> None: + assert _load_outcomes(_runtime("normal", vclock), 6) == [None] * 6 + + +def test_session_expiry_rejects_session_after_ttl(vclock: VirtualClock) -> None: + runtime = _runtime("session-expiry", vclock) + issued = runtime.issue_session() + + vclock.advance(4.9) + runtime.check_session(issued) + vclock.advance(0.1) + + with pytest.raises(SessionExpiredError): + runtime.check_session(issued) + + +def test_normal_profile_session_survives_an_hour(vclock: VirtualClock) -> None: + runtime = _runtime("normal", vclock) + issued = runtime.issue_session() + + vclock.advance(3600.0) + + runtime.check_session(issued) + + +def test_rebooting_returns_503_until_window_closes(vclock: VirtualClock) -> None: + runtime = _runtime("rebooting", vclock) + statuses: list[int] = [] + + while True: + try: + runtime.perform("inventory") + break + except RetryableOpError as exc: + statuses.append(exc.status) + vclock.sleep(5.0) + + assert statuses == [503, 503, 503, 503] + assert vclock.monotonic() == 22.0 + + +def test_degraded_media_verify_slow_with_checksum_retry(vclock: VirtualClock) -> None: + runtime = _runtime("degraded-media", vclock) + + for _ in range(4): + runtime.perform("verify") + runtime.perform("read") + + assert runtime.checksum_retries == 1 + assert vclock.sleeps == [9.0, 9.0, 9.0, 9.0, 9.0, 6.0] + + +def test_normal_profile_verify_has_no_checksum_retry(vclock: VirtualClock) -> None: + runtime = _runtime("normal", vclock) + + for _ in range(8): + runtime.perform("verify") + + assert runtime.checksum_retries == 0 + + +# --- Emulator env wiring ------------------------------------------------------ + + +@pytest.mark.parametrize("name", ["instant", "realistic", "hardware"]) +def test_emulator_env_legacy_profiles_unchanged(name: ProfileName) -> None: + assert emulator_env(name) == { + "I3_TIMING_PROFILE": name, + "OPENBLADE_EMULATOR_LATENCY_PROFILE": name, + } + + +def test_emulator_env_latency_override_accepted_by_emulator_hook( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from openblade.api.aml_state import _env_emulator_latency_profile_ms + + env = emulator_env("slow-robotics") + monkeypatch.setenv( + "OPENBLADE_EMULATOR_LATENCY_PROFILE_MS", env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"] + ) + + parsed = _env_emulator_latency_profile_ms() + + assert env["OPENBLADE_EMULATOR_LATENCY_PROFILE"] == "realistic" + assert parsed == json.loads(env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"]) + assert parsed is not None and parsed["move"]["realistic"] == 4500 + + +def test_profile_runtime_server_side_faults_skips_client_injection( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("I3_FAULTS_SERVER_SIDE", "1") + runtime = ProfileRuntime("intermittent-drive", clock=vclock) + + for _ in range(6): + runtime.perform("tape_load") + runtime.check_session(runtime.issue_session() - 3600.0) + + assert runtime.loads == 6 diff --git a/tests/i3/timing.py b/tests/i3/timing.py index 05de605..cdfde29 100644 --- a/tests/i3/timing.py +++ b/tests/i3/timing.py @@ -1,20 +1,47 @@ """ Timing profiles for Quantum i3 test suite. -Three profiles: +Legacy profiles (names kept stable for CI and existing callers): instant — zero delays, for fast CI runs realistic — short delays that feel like real hardware (default in emulator mode) hardware — real Quantum i3 mechanical tolerances (used with real i3 target) +Scenario profiles (realistic baseline plus one stressor each): + normal — alias of realistic delays, no faults + slow-robotics — robot/drive mechanics 3x slower + busy-library — every mechanical op waits behind a contended robot queue + intermittent-drive — every Nth tape load fails with a RetryableOpError + session-expiry — auth sessions expire after a short TTL + rebooting — a window of HTTP 503 at the start of the run + degraded-media — read/verify 3x slower + periodic checksum retry + +All sleeping goes through a ``Clock`` (``RealClock`` by default). Unit tests +install a ``VirtualClock`` with ``use_clock()`` so profile behaviour can be +asserted deterministically without real sleeping. + +Delays are applied client-side by ``wait_for_op``. Faults (load failures, +session expiry, 503 window, checksum retries) are applied client-side by a +``ProfileRuntime`` unless ``faults_server_side()`` -- then the emulator injects +them via ``OPENBLADE_EMULATOR_FAULT_PROFILE`` (presets in ``openblade.api.aml_faults``); +``emulator_env()`` exports the env vars the emulator DOES honour +(``I3_TIMING_PROFILE``, ``OPENBLADE_EMULATOR_LATENCY_PROFILE``, +``OPENBLADE_EMULATOR_LATENCY_PROFILE_MS``) so its own latency matches. + Usage: from tests.i3.timing import get_profile, wait_for_op, assert_within_tolerance """ from __future__ import annotations +import json import os import time -from typing import Literal +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Literal, Protocol, get_args + +from openblade.api.aml_faults import PRESETS as FAULT_PRESETS OpType = Literal[ "tape_load", @@ -26,9 +53,96 @@ "unmount", "inventory", "auth", + "read", + "verify", +] + +LegacyProfileName = Literal["instant", "realistic", "hardware"] +ProfileName = Literal[ + "instant", + "realistic", + "hardware", + "normal", + "slow-robotics", + "busy-library", + "intermittent-drive", + "session-expiry", + "rebooting", + "degraded-media", ] +PROFILE_NAMES: tuple[ProfileName, ...] = get_args(ProfileName) + + +# --------------------------------------------------------------------------- +# Clock +# --------------------------------------------------------------------------- + + +class Clock(Protocol): + """Time source used by every profile sleep.""" + + def monotonic(self) -> float: ... + + def sleep(self, seconds: float) -> None: ... + + +class RealClock: + """Wall-clock implementation backed by :mod:`time`.""" + + def monotonic(self) -> float: + return time.monotonic() + + def sleep(self, seconds: float) -> None: + if seconds > 0: + time.sleep(seconds) + + +class VirtualClock: + """Deterministic clock: ``sleep`` advances virtual time instantly.""" + + def __init__(self, start: float = 0.0) -> None: + self._now = start + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self._now + + def sleep(self, seconds: float) -> None: + if seconds < 0: + raise ValueError("cannot sleep a negative duration") + self.sleeps.append(seconds) + self._now += seconds + + def advance(self, seconds: float) -> None: + """Move virtual time forward without recording a sleep.""" + if seconds < 0: + raise ValueError("cannot advance a negative duration") + self._now += seconds + + +_clock: Clock = RealClock() + + +def get_clock() -> Clock: + return _clock + + +def set_clock(clock: Clock) -> Clock: + """Install ``clock`` globally and return the previous one.""" + global _clock + previous = _clock + _clock = clock + return previous + + +@contextmanager +def use_clock(clock: Clock) -> Iterator[Clock]: + previous = set_clock(clock) + try: + yield clock + finally: + set_clock(previous) -ProfileName = Literal["instant", "realistic", "hardware"] TIMING_PROFILES: dict[ProfileName, dict[OpType, float]] = { "instant": { @@ -41,6 +155,8 @@ "unmount": 0.0, "inventory": 0.0, "auth": 0.0, + "read": 0.0, + "verify": 0.0, }, "realistic": { "tape_load": 3.0, @@ -52,6 +168,8 @@ "unmount": 1.5, "inventory": 2.0, "auth": 0.1, + "read": 2.0, + "verify": 3.0, }, "hardware": { "tape_load": 35.0, @@ -63,9 +181,185 @@ "unmount": 10.0, "inventory": 45.0, "auth": 0.5, + "read": 60.0, + "verify": 120.0, }, } +_MECHANICAL_OPS: tuple[OpType, ...] = ( + "tape_load", + "tape_unload", + "move", + "mount", + "unmount", + "inventory", +) +_REALISTIC = TIMING_PROFILES["realistic"] +_BUSY_QUEUE_WAIT_S = 2.0 + + +def _scaled(ops: tuple[OpType, ...], factor: float) -> dict[OpType, float]: + return {op: (v * factor if op in ops else v) for op, v in _REALISTIC.items()} + + +TIMING_PROFILES.update( + { + "normal": dict(_REALISTIC), + "slow-robotics": _scaled(_MECHANICAL_OPS, 3.0), + "busy-library": { + op: (v + _BUSY_QUEUE_WAIT_S if op in _MECHANICAL_OPS else v) + for op, v in _REALISTIC.items() + }, + "intermittent-drive": dict(_REALISTIC), + "session-expiry": dict(_REALISTIC), + "rebooting": dict(_REALISTIC), + "degraded-media": _scaled(("read", "verify"), 3.0), + } +) + + +@dataclass(frozen=True) +class ProfileFaults: + """Client-side fault knobs for a scenario profile (0/None = off).""" + + base: LegacyProfileName = "realistic" + load_fail_every: int = 0 + auth_ttl_s: float | None = None + reboot_window_s: float = 0.0 + checksum_retry_every: int = 0 + + +_FAULT_BASES: dict[ProfileName, LegacyProfileName] = { + "instant": "instant", + "hardware": "hardware", +} + +# Fault knobs come from the emulator's presets (single source of truth). +PROFILE_FAULTS: dict[ProfileName, ProfileFaults] = { + name: ProfileFaults( + base=_FAULT_BASES.get(name, "realistic"), + load_fail_every=FAULT_PRESETS[name].load_fail_every, + auth_ttl_s=FAULT_PRESETS[name].auth_ttl_s, + reboot_window_s=FAULT_PRESETS[name].reboot_window_s, + checksum_retry_every=FAULT_PRESETS[name].checksum_retry_every, + ) + for name in TIMING_PROFILES +} + + +def faults_server_side() -> bool: + """True when the emulator injects the faults (``OPENBLADE_EMULATOR_FAULT_PROFILE``). + + ``I3_FAULTS_SERVER_SIDE`` wins; otherwise ON when ``I3_TEST_MODE`` is emulator (its default). + """ + raw = os.environ.get("I3_FAULTS_SERVER_SIDE") + if raw is not None and raw.strip(): + return raw.strip().lower() in {"1", "true", "yes", "on"} + return os.environ.get("I3_TEST_MODE", "emulator").strip().lower() == "emulator" + + +class RetryableOpError(Exception): + """A transient failure the caller is expected to retry.""" + + def __init__(self, op_type: OpType, status: int, message: str) -> None: + super().__init__(message) + self.op_type = op_type + self.status = status + + +class SessionExpiredError(Exception): + """The session issued by ``ProfileRuntime.issue_session`` has expired (HTTP 401).""" + + +@dataclass +class ProfileRuntime: + """Stateful, per-test application of a profile's delays and faults.""" + + name: ProfileName + clock: Clock = field(default_factory=get_clock) + loads: int = 0 + verifies: int = 0 + checksum_retries: int = 0 + started_at: float = field(init=False) + + def __post_init__(self) -> None: + self.started_at = self.clock.monotonic() + + @property + def faults(self) -> ProfileFaults: + if faults_server_side(): + return ProfileFaults(base=PROFILE_FAULTS[self.name].base) + return PROFILE_FAULTS[self.name] + + def _sleep_for(self, op_type: OpType, multiplier: float = 1.0) -> None: + self.clock.sleep(TIMING_PROFILES[self.name].get(op_type, 0.0) * multiplier) + + def perform(self, op_type: OpType, multiplier: float = 1.0) -> None: + """Apply delay + faults for one operation; raise on an injected fault.""" + window = self.faults.reboot_window_s + if window and self.clock.monotonic() - self.started_at < window: + raise RetryableOpError(op_type, 503, "library rebooting (503)") + self._sleep_for(op_type, multiplier) + if op_type == "tape_load": + self.loads += 1 + every = self.faults.load_fail_every + if every and self.loads % every == 0: + raise RetryableOpError(op_type, 409, f"drive load {self.loads} failed (retryable)") + if op_type == "verify": + self.verifies += 1 + every = self.faults.checksum_retry_every + if every and self.verifies % every == 0: + self.checksum_retries += 1 + self._sleep_for("verify", multiplier) + + def issue_session(self) -> float: + """Return the clock time at which a freshly issued session was created.""" + self._sleep_for("auth") + return self.clock.monotonic() + + def check_session(self, issued_at: float) -> None: + ttl = self.faults.auth_ttl_s + if ttl is not None and self.clock.monotonic() - issued_at >= ttl: + raise SessionExpiredError(f"session expired after {ttl:.1f}s") + + +_EMULATOR_LATENCY_OPS: tuple[OpType, ...] = ( + "auth", + "inventory", + "mount", + "unmount", + "move", + "format", +) + + +def emulator_env(name: ProfileName) -> dict[str, str]: + """Env vars to start the emulator with so its latency matches ``name``. + + Uses only hooks the emulator already reads: ``I3_TIMING_PROFILE`` + (simulator load delay; legacy names only), ``OPENBLADE_EMULATOR_LATENCY_PROFILE`` + and ``OPENBLADE_EMULATOR_LATENCY_PROFILE_MS`` (``openblade/api/aml_state.py``). + Faults have no env hook and stay client-side in ``ProfileRuntime``. + """ + base = PROFILE_FAULTS[name].base + env: dict[str, str] = {"I3_TIMING_PROFILE": base, "OPENBLADE_EMULATOR_LATENCY_PROFILE": base} + if name not in get_args(LegacyProfileName): + delays = TIMING_PROFILES[name] + hardware = TIMING_PROFILES["hardware"] + env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"] = json.dumps( + { + op: { + "instant": 0, + "realistic": round(delays[op] * 1000), + "hardware": round(hardware[op] * 1000), + } + for op in _EMULATOR_LATENCY_OPS + }, + sort_keys=True, + ) + return env + + # Tolerance multipliers for hardware mode (real i3 timing can vary) HARDWARE_TOLERANCE: dict[OpType, float] = { "tape_load": 0.4, # ±40% — mechanical variation @@ -83,8 +377,9 @@ def get_profile_name() -> ProfileName: """Read profile from env. Defaults to 'instant' for emulator, 'hardware' for real.""" env = os.environ.get("I3_TIMING_PROFILE", "").strip().lower() - if env in ("instant", "realistic", "hardware"): - return env # type: ignore[return-value] + for name in PROFILE_NAMES: + if env == name: + return name mode = os.environ.get("I3_TEST_MODE", "emulator").strip().lower() return "hardware" if mode == "real" else "instant" @@ -97,12 +392,14 @@ def get_profile() -> dict[OpType, float]: def wait_for_op(op_type: OpType, multiplier: float = 1.0) -> None: """Sleep for the appropriate time for this operation. - In instant mode this is a no-op. In realistic/hardware modes the sleep - simulates the mechanical delay an operator would observe on a real i3. + In instant mode this is a no-op. Otherwise the sleep simulates the + mechanical delay an operator would observe on a real i3. The sleep goes + through the active ``Clock`` (see ``use_clock``). Faults are NOT applied + here -- use ``ProfileRuntime.perform`` for fault-bearing profiles. """ delay = get_profile().get(op_type, 0.0) * multiplier if delay > 0: - time.sleep(delay) + get_clock().sleep(delay) def assert_within_tolerance( diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py new file mode 100644 index 0000000..b7af767 --- /dev/null +++ b/tests/integration/test_aml_auth_fidelity.py @@ -0,0 +1,423 @@ +"""AML auth fidelity (roadmap item 18): sessions, preconditions, negotiation, envelopes. + +Every behaviour here is *inferred* unless noted: the Rev D manual documents JSON +login/logout, the ``Warning: Default Password Supplied`` header and the +``sessionID`` cookie, but no session cap, 412, XML, 415/406 or 429 semantics. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from openblade.api import aml_state, routes_aml_auth +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig + +CREDS = {"name": "admin", "password": "password"} +LEAK = "postgres://u:SECRET@db/leak" + + +@pytest.fixture() +def client(tmp_path: Path) -> Iterator[TestClient]: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'fid.db'}")) + reset_context(context) + routes_aml_auth._login_attempts.clear() + yield TestClient(app, raise_server_exceptions=False) + routes_aml_auth._login_attempts.clear() + + +def _login(client: TestClient) -> str: + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200, response.text + return str(response.json()["token"]) + + +# --- 1. max sessions per user (oldest evicted) --------------------------------- + + +def test_session_cap_evicts_oldest(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENBLADE_AML_MAX_SESSIONS", "3") + tokens = [_login(client) for _ in range(4)] + client.cookies.clear() + first = client.get("/aml/users", headers={"Authorization": f"Bearer {tokens[0]}"}) + assert first.status_code == 401 + for token in tokens[1:]: + ok = client.get("/aml/users", headers={"Authorization": f"Bearer {token}"}) + assert ok.status_code == 200 + + +def test_session_cap_default_is_five(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENBLADE_AML_MAX_SESSIONS", raising=False) + assert aml_state.max_sessions_per_user() == 5 + monkeypatch.setenv("OPENBLADE_AML_MAX_SESSIONS", "bogus") + assert aml_state.max_sessions_per_user() == 5 + + +# --- 2. 412 behind OPENBLADE_AML_STRICT_PRECONDITIONS -------------------------- + + +def test_default_password_lenient_is_200_with_warning( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENBLADE_AML_STRICT_PRECONDITIONS", raising=False) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + assert response.headers["Warning"] == "Default Password Supplied" + + +def test_default_password_strict_is_412_and_session_usable( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 412 + body = response.json() + assert body["code"] == 412 and body["summary"] == "Default password must be changed" + assert response.headers["Warning"] == "Default Password Supplied" + assert client.get("/aml/users").status_code == 200 + + +# --- 3. content negotiation ---------------------------------------------------- + +XML_LOGIN = b"adminpassword" + + +def test_xml_login_and_logout_round_trip(client: TestClient) -> None: + xml = {"Content-Type": "application/xml", "Accept": "application/xml"} + response = client.post("/aml/users/login", content=XML_LOGIN, headers=xml) + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/xml") + assert "Login successful" in response.text + out = client.delete("/aml/users/login", headers={"Accept": "text/xml"}) + assert out.status_code == 200 + assert out.text.startswith("") + + +def test_text_xml_request_gets_json_by_default(client: TestClient) -> None: + response = client.post( + "/aml/users/login", content=XML_LOGIN, headers={"Content-Type": "text/xml"} + ) + assert response.status_code == 200 + assert response.json()["summary"] == "Login successful" + + +@pytest.mark.parametrize( + "payload", + [ + b']>' + b"admin&x;", + b']>' + b"admin&b;", + ], + ids=["xxe", "entity-expansion"], +) +def test_xml_dtd_and_entities_rejected(client: TestClient, payload: bytes) -> None: + response = client.post( + "/aml/users/login", content=payload, headers={"Content-Type": "application/xml"} + ) + assert response.status_code == 400 + assert response.json()["summary"] == "XML DTD/entity declarations are not allowed" + assert "root:" not in response.text + + +def test_malformed_xml_is_400(client: TestClient) -> None: + response = client.post( + "/aml/users/login", content=b"", headers={"Content-Type": "text/xml"} + ) + assert response.status_code == 400 + assert response.json()["summary"] == "Malformed XML login payload" + + +@pytest.mark.parametrize("content_type", [None, "text/plain", "application/yaml", ";;"]) +def test_unsupported_content_type_is_415(client: TestClient, content_type: str | None) -> None: + headers = {"Content-Type": content_type} if content_type else {} + response = client.post("/aml/users/login", content=b"name=admin", headers=headers) + assert response.status_code == 415 + body = response.json() + assert body["code"] == 415 + assert body["summary"] == "Unsupported or missing Content-Type" + + +def test_unsupported_accept_is_406(client: TestClient) -> None: + response = client.post("/aml/users/login", json=CREDS, headers={"Accept": "image/png"}) + assert response.status_code == 406 + assert response.json()["code"] == 406 + token = _login(client) + out = client.delete( + "/aml/users/login", + headers={"Accept": "image/png", "Authorization": f"Bearer {token}"}, + ) + assert out.status_code == 406 + + +# --- 5. expiry + rate limit + constant-time compare ---------------------------- + + +def _expire_all_sessions() -> None: + for record in aml_state._STATE.sessions.values(): + record.expires_at = record.expires_at - timedelta(days=2) + + +_HEADERS_DROPPED = ( + "openblade/api/main.py handle_http_exception drops HTTPException.headers on /aml " + "paths; cross-cutting fix requested (pass headers=exc.headers)" +) + + +def test_expired_cookie_is_401(client: TestClient) -> None: + _login(client) + _expire_all_sessions() + assert client.get("/aml/users").status_code == 401 + + +def test_expired_cookie_is_cleared(client: TestClient) -> None: + _login(client) + _expire_all_sessions() + response = client.get("/aml/users") + set_cookie = response.headers.get("set-cookie", "") + assert set_cookie.startswith("sessionID=") and "Max-Age=0" in set_cookie + + +def test_expired_bearer_is_401(client: TestClient) -> None: + token = _login(client) + client.cookies.clear() + _expire_all_sessions() + response = client.get("/aml/users", headers={"Authorization": f"Bearer {token}"}) + assert response.status_code == 401 + assert "set-cookie" not in response.headers + + +def test_eleventh_login_attempt_is_429_with_retry_after( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 10) + bad = {"name": "admin", "password": "wrong-password"} + for _ in range(10): + assert client.post("/aml/users/login", json=bad).status_code == 401 + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 429 + + +def test_rate_limited_login_carries_retry_after( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 0) + response = client.post("/aml/users/login", json=CREDS) + assert int(response.headers["Retry-After"]) >= 1 + + +def test_password_compare_is_constant_time() -> None: + import inspect + + assert "compare_digest" in inspect.getsource(aml_state.verify_password) + assert "compare_digest" in inspect.getsource(aml_state.authenticate_ldap_user) + + +# --- 4. error-code envelope matrix --------------------------------------------- + + +def _trigger(status: int, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> object: + if status == 401: + return client.get("/aml/users") + if status == 403: + monkeypatch.setattr(aml_state, "get_login_mode", lambda: 2) + return client.post("/aml/users/login", json=CREDS) + if status == 404: + _login(client) + return client.get("/aml/users/no-such-user") + if status == 412: + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + return client.post("/aml/users/login", json=CREDS) + if status == 429: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 0) + return client.post("/aml/users/login", json=CREDS) + if status == 500: + + def boom(*_a: object, **_k: object) -> None: + raise RuntimeError(LEAK) + + monkeypatch.setattr(aml_state, "verify_credentials", boom) + return client.post("/aml/users/login", json=CREDS) + assert status == 503 + monkeypatch.setattr(aml_state, "get_service_access", lambda: {"enabled": False}) + real = aml_state.verify_credentials + + def as_service(name: str, password: str) -> object: + user = real(name, password) + assert user is not None + return SimpleNamespace( + name=user.name, role=2, require_password_change=user.require_password_change + ) + + monkeypatch.setattr(aml_state, "verify_credentials", as_service) + return client.post("/aml/users/login", json=CREDS) + + +_NO_500_ENVELOPE = ( + "openblade/api/main.py has no catch-all Exception handler for /aml paths, so an " + "unhandled error is Starlette's plain-text 500; cross-cutting fix requested" +) + + +def test_unhandled_error_is_500_without_leak( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + response = _trigger(500, client, monkeypatch) + assert response.status_code == 500 # type: ignore[attr-defined] + text = response.text # type: ignore[attr-defined] + assert LEAK not in text and "Traceback" not in text and "RuntimeError" not in text + + +@pytest.mark.parametrize( + "status", + [ + 401, + 403, + 404, + 412, + 429, + 500, + 503, + ], +) +def test_error_envelope_shape( + status: int, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + response = _trigger(status, client, monkeypatch) + assert response.status_code == status, response.text # type: ignore[attr-defined] + text = response.text # type: ignore[attr-defined] + body = response.json() # type: ignore[attr-defined] + assert isinstance(body, dict) + assert "Traceback" not in text and LEAK not in text and "RuntimeError" not in text + message = body.get("summary", body.get("detail", body.get("message"))) + assert isinstance(message, str) and message + + +# --- MFA-required pending session (inferred; no Rev D challenge wire shape) ----- + +MFA_WARNING = '199 - "MFA Required"' + + +def _enable_totp(client: TestClient) -> str: + """Enable TOTP for admin through the documented routes; return the secret.""" + import pyotp + + token = _login(client) + auth = {"Authorization": f"Bearer {token}"} + secret = client.get("/aml/users/login/mfa/totp/key", headers=auth).json()["key"] + enabled = client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": True, "authenticationCode": pyotp.TOTP(secret).now()}, + headers=auth, + ) + assert enabled.status_code == 200, enabled.text + client.cookies.clear() + return str(secret) + + +def test_mfa_login_lenient_is_200_with_warning_and_pending( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENBLADE_AML_STRICT_PRECONDITIONS", raising=False) + _enable_totp(client) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + assert MFA_WARNING in response.headers.get_list("Warning") + assert "Default Password Supplied" in response.headers.get_list("Warning") + # No new JSON fields: same keys as an ordinary (non-pending) login body. + assert set(response.json()) == { + "action", + "code", + "customCode", + "description", + "summary", + "token", + } + assert aml_state._STATE.sessions[response.json()["token"]].mfa_pending is True + + +def test_mfa_login_strict_default_password_keeps_documented_412( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + # MFA never invents a status or summary: the 412 is the default-password one. + _enable_totp(client) + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 412 + body = response.json() + assert body["code"] == 412 and body["summary"] == "Default password must be changed" + assert MFA_WARNING in response.headers.get_list("Warning") + assert aml_state._STATE.sessions[body["token"]].mfa_pending is True + + +def test_mfa_login_strict_without_other_precondition_is_200_with_warning( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _enable_totp(client) + aml_state.update_user(CREDS["name"], password=CREDS["password"]) # clears default-pw flag + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + body = response.json() + assert body["code"] == 0 and body["summary"] == "Login successful" + assert response.headers.get_list("Warning") == [MFA_WARNING] + assert aml_state._STATE.sessions[body["token"]].mfa_pending is True + + +def test_login_without_enabled_mfa_is_not_pending(client: TestClient) -> None: + token = _login(client) + assert aml_state._STATE.sessions[token].mfa_pending is False + + +def _pending_login(client: TestClient) -> tuple[str, dict[str, str]]: + secret = _enable_totp(client) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + client.cookies.clear() + return secret, {"Authorization": f"Bearer {response.json()['token']}"} + + +def test_mfa_pending_session_is_401_until_valid_code(client: TestClient) -> None: + import pyotp + + secret, auth = _pending_login(client) + assert client.get("/aml/users", headers=auth).status_code == 401 + assert client.get("/aml/physicalLibrary/elements", headers=auth).status_code == 401 + bad = client.post( + "/aml/users/login/mfa", json={"type": "totp", "authenticationCode": "000001"}, headers=auth + ) + assert bad.status_code == 401 + assert client.get("/aml/users", headers=auth).status_code == 401 # still pending + good = client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": pyotp.TOTP(secret).now()}, + headers=auth, + ) + assert good.status_code == 200, good.text + assert client.get("/aml/users", headers=auth).status_code == 200 + + +def test_mfa_pending_cookie_session_is_gated_without_dropping_cookie( + client: TestClient, +) -> None: + _enable_totp(client) + assert client.post("/aml/users/login", json=CREDS).status_code == 200 + blocked = client.get("/aml/users") + assert blocked.status_code == 401 + assert "set-cookie" not in blocked.headers + + +def test_mfa_pending_session_can_fetch_key_route_and_logout(client: TestClient) -> None: + _, auth = _pending_login(client) + # Key route is reachable (403 because TOTP is already confirmed, not 401). + assert client.get("/aml/users/login/mfa/totp/key", headers=auth).status_code == 403 + client.cookies.set("sessionID", auth["Authorization"].split(" ", 1)[1]) + assert client.delete("/aml/users/login").status_code == 200 + assert client.get("/aml/users", headers=auth).status_code == 401 diff --git a/tests/integration/test_aml_emulator_faults.py b/tests/integration/test_aml_emulator_faults.py new file mode 100644 index 0000000..31c2252 --- /dev/null +++ b/tests/integration/test_aml_emulator_faults.py @@ -0,0 +1,102 @@ +"""OPENBLADE_EMULATOR_FAULT_PROFILE faults are observable on the AML wire.""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import datetime, timedelta +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from openblade.api import aml_faults, aml_state +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig + + +def _client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, profile: str | None) -> TestClient: + if profile is None: + monkeypatch.delenv(aml_faults.FAULT_PROFILE_ENV, raising=False) + else: + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, profile) + aml_faults.reset_fault_state() + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'faults.db'}")) + reset_context(context) + return TestClient(app) + + +def _login(client: TestClient) -> None: + resp = client.post("/aml/users/login", json={"name": "admin", "password": "password"}) + assert resp.status_code == 200 + + +@pytest.fixture(autouse=True) +def _reset_faults() -> Iterator[None]: + yield + aml_faults.reset_fault_state() + + +def test_no_fault_profile_mount_requests_never_409( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, None) + _login(client) + + statuses = [client.post("/aml/operations/mount", json={}).status_code for _ in range(6)] + + assert 409 not in statuses + + +def test_intermittent_drive_every_third_load_returns_409( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, "intermittent-drive") + _login(client) + + responses = [client.post("/aml/operations/mount", json={}) for _ in range(3)] + + assert [r.status_code == 409 for r in responses] == [False, False, True] + assert responses[2].json()["code"] == "AML_CONFLICT" + + +def test_rebooting_returns_503_with_retry_after( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, "rebooting") + + response = client.get("/aml/system") + + assert response.status_code == 503 + assert int(response.headers["Retry-After"]) >= 1 + + +def test_session_expiry_profile_expires_session_after_ttl( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, '{"auth_ttl_s": 5}') + start = datetime(2026, 1, 1, 12, 0, 0) + now = [start] + monkeypatch.setattr(aml_state, "_utcnow", lambda: now[0]) + _login(client) + assert client.get("/aml/system").status_code == 200 + + now[0] = start + timedelta(seconds=6) + response = client.get("/aml/system") + + assert response.status_code == 401 + + +def test_no_fault_profile_session_outlives_five_seconds( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, None) + start = datetime(2026, 1, 1, 12, 0, 0) + now = [start] + monkeypatch.setattr(aml_state, "_utcnow", lambda: now[0]) + _login(client) + + now[0] = start + timedelta(seconds=6) + response = client.get("/aml/system") + + assert response.status_code == 200 diff --git a/tests/integration/test_archive_aml_bridge.py b/tests/integration/test_archive_aml_bridge.py index 73ed689..c98fcc7 100644 --- a/tests/integration/test_archive_aml_bridge.py +++ b/tests/integration/test_archive_aml_bridge.py @@ -168,7 +168,11 @@ def _boom(*args, **kwargs): monkeypatch.setattr(context.ltfs, "write_file", original_write_file) jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "archive bridge boom") + aml_job = next( + job + for job in jobs + if job["result"] == "Job failed (RuntimeError); see server logs for detail" + ) assert aml_job["type"] == "archive" assert aml_job["status"] == "failed" @@ -179,7 +183,9 @@ def _boom(*args, **kwargs): if event["component"] == "archive" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "archive bridge boom" + assert ( + failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + ) drive = ( client.get("/aml/drive/DRV-001").json().get("drive") @@ -225,7 +231,11 @@ def _boom(*args, **kwargs): assert response.status_code == 500 jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "restore bridge boom") + aml_job = next( + job + for job in jobs + if job["result"] == "Job failed (RuntimeError); see server logs for detail" + ) assert aml_job["type"] == "restore" assert aml_job["status"] == "failed" @@ -236,7 +246,9 @@ def _boom(*args, **kwargs): if event["component"] == "restore" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "restore bridge boom" + assert ( + failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + ) assert failure_event["details"]["catalogPath"] == "/photos/restore.txt" assert archived_file.read_text() == "restore aml bridge" diff --git a/tests/integration/test_drive_lease_serialisation.py b/tests/integration/test_drive_lease_serialisation.py new file mode 100644 index 0000000..c81bb97 --- /dev/null +++ b/tests/integration/test_drive_lease_serialisation.py @@ -0,0 +1,85 @@ +"""Two overlapping archive requests against one catalog serialise on the drive lease. + +The archive route's in-process lock cannot see a second process (the CLI, another +worker). The overlap here is a lease held through a *separate* catalog session on +the same SQLite file -- exactly what another process would hold -- while +``POST /archive/sharded`` runs. The request must wait for that lease, not drive +the same tape concurrently. +""" + +from __future__ import annotations + +import threading +from datetime import timedelta +from pathlib import Path + +from fastapi.testclient import TestClient +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from openblade.api.main import app +from openblade.catalog.db import get_session +from openblade.catalog.models import DriveLeaseRecord +from openblade.catalog.repository import CatalogRepository +from openblade.jobs.scheduler import CatalogLeaseStore +from tests.integration.test_sharded_archive_cli_api_parity import ( + LANES, + build_context, + seed_source, +) + +client = TestClient(app) + + +def test_overlapping_archive_waits_for_a_lease_held_by_another_session(tmp_path: Path) -> None: + context = build_context(tmp_path / "leases.db") + context.catalog.create_volume_group("shards") + source = tmp_path / "src" + seed_source(source) + + other_session = get_session() + try: + other = CatalogLeaseStore(CatalogRepository(other_session)) + held = other.acquire( + job_id="other-process", + barcodes=list(LANES), + num_drives=2, + ttl=timedelta(minutes=15), + ) + assert held is not None + + finished = threading.Event() + responses: list[int] = [] + + def _post() -> None: + response = client.post( + "/archive/sharded", + json={ + "source_path": str(source), + "volume_group": "shards", + "lane_barcodes": LANES, + "mode": "stripe", + }, + ) + responses.append(response.status_code) + finished.set() + + worker = threading.Thread(target=_post) + worker.start() + # While the other session holds both drives, the request must not finish. + assert not finished.wait(1.0), "archive ran while another job held the drives" + other.release([lease.id for lease in held]) + worker.join(timeout=60) + assert finished.is_set() + assert responses == [202] + + # The request ran under its own, strictly newer leases, now released. + assert other.live_leases() == [] + assert max(lease.fencing_token for lease in held) < _max_token(other_session) + finally: + other_session.close() + + +def _max_token(session: Session) -> int: + value = session.scalar(select(func.max(DriveLeaseRecord.fencing_token))) + return int(value or 0) diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py new file mode 100644 index 0000000..a9943e4 --- /dev/null +++ b/tests/integration/test_fuse_data_plane.py @@ -0,0 +1,397 @@ +"""FUSE data plane: hydration, cache, batching, safety (no kernel FUSE needed).""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import threading +import time +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import CartridgeOfflineError +from openblade.fuse.cache import CacheEntryInUseError, HydrationCache +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import ( + HydrationFailedError, + HydrationTimeoutError, + Hydrator, + JobRestoreEngine, +) +from openblade.fuse.mount import STATE_XATTR, CatalogFuseOperations + +FILES = { + "/a.bin": (b"alpha" * 100, "TAPE01"), + "/b.bin": (b"bravo" * 50, "TAPE01"), + "/c.bin": (b"charlie", "TAPE02"), +} + + +class FakeEngine: + """Stands in for the restore job path; records one call per tape batch.""" + + def __init__(self, staging: Path, *, corrupt: bool = False) -> None: + self.staging = staging + self.corrupt = corrupt + self.batches: list[tuple[str, list[str]]] = [] + self.gate = threading.Event() + self.gate.set() + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + self.gate.wait(10) + self.batches.append((tape_key, sorted(catalog_paths))) + out = {} + for path in catalog_paths: + dest = self.staging / path.strip("/") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(b"half" if self.corrupt else FILES[path][0]) + out[path] = dest + return out + + def find_active_job(self, catalog_path: str) -> str | None: + return None + + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + raise HydrationFailedError(job_id) + + +@pytest.fixture +def catalog(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + for path, (data, barcode) in FILES.items(): + rec = repo.create_file_record(path, len(data), hashlib.sha256(data).hexdigest(), group.id) + inst = repo.create_file_instance(rec.id, barcode, path) + repo.mark_instance_archived(inst.id) + return repo + + +def _plane(catalog: CatalogRepository, tmp_path: Path, **kw: object): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "staging", corrupt=bool(kw.pop("corrupt", False))) + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.05) + ops = CatalogFuseOperations(fs, data_plane=hyd, hydrate_timeout=5.0, **kw) # type: ignore[arg-type] + return ops, hyd, engine + + +def test_offline_read_hydrates_and_bytes_match(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, _, _ = _plane(catalog, tmp_path) + assert ops.getattr("/a.bin")["st_size"] == len(FILES["/a.bin"][0]) # real size while offline + assert ops.getxattr("/a.bin", STATE_XATTR) == b"offline" + assert ops.listxattr("/a.bin") == [STATE_XATTR] + fh = ops.open("/a.bin", os.O_RDONLY) + assert ops.read("/a.bin", 10_000, 0, fh) == FILES["/a.bin"][0] + assert ops.getxattr("/a.bin", STATE_XATTR) == b"online" + + +def test_concurrent_requests_share_one_ticket(catalog, tmp_path): # type: ignore[no-untyped-def] + _, hyd, engine = _plane(catalog, tmp_path) + engine.gate.clear() + t1, t2 = hyd.request("/a.bin"), hyd.request("/a.bin") + assert t1 is t2 + assert hyd.status("/a.bin").value == "hydrating" + engine.gate.set() + hyd.wait(t1, 5) + assert len(engine.batches) == 1 + + +def test_nonblocking_open_returns_eagain_then_succeeds(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path, blocking=False) + engine.gate.clear() + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EAGAIN + engine.gate.set() + hyd.wait(hyd.request("/a.bin"), 5) + assert ops.open("/a.bin", os.O_RDONLY) > 0 + + +def test_blocking_open_times_out_with_eagain(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path) + ops.hydrate_timeout = 0.2 + engine.gate.clear() + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EAGAIN + engine.gate.set() + hyd.shutdown() + + +def test_eviction_never_drops_an_open_file(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, _ = _plane(catalog, tmp_path) + fh = ops.open("/a.bin", os.O_RDONLY) + checksum = catalog.get_file_record("/a.bin").checksum_sha256 # type: ignore[union-attr] + with pytest.raises(CacheEntryInUseError): + hyd.cache.evict(checksum) + hyd.cache.max_bytes = 1 # budget pressure must skip the open file too + ops.open("/c.bin", os.O_RDONLY) + assert hyd.cache.is_cached(checksum) + ops.release("/a.bin", fh) + assert not hyd.cache.is_cached(checksum) # released -> evictable under budget + + +def test_failed_restore_leaves_file_offline_without_partial(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, _ = _plane(catalog, tmp_path, corrupt=True) + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EIO + assert ops.getxattr("/a.bin", STATE_XATTR) == b"offline" + assert hyd.last_error("/a.bin") == "restored bytes failed checksum verification" + leftovers = [p for p in (tmp_path / "cache").rglob("*") if p.is_file()] + assert leftovers == [] # no half file under the checksum name or a temp name + + +def test_same_tape_batching(catalog, tmp_path): # type: ignore[no-untyped-def] + _, hyd, engine = _plane(catalog, tmp_path) + tickets = [hyd.request(p) for p in ("/a.bin", "/b.bin", "/c.bin")] + for t in tickets: + hyd.wait(t, 5) + assert sorted(engine.batches) == [("TAPE01", ["/a.bin", "/b.bin"]), ("TAPE02", ["/c.bin"])] + + +def test_restart_rederives_ticket_from_job_state(catalog, tmp_path): # type: ignore[no-untyped-def] + dest = tmp_path / "staged" / "a" + job = catalog.create_job("restore", {"catalog_path": "/a.bin", "dest_path": str(dest)}) + cache = HydrationCache(str(tmp_path / "cache")) + hyd = Hydrator(catalog, cache, JobRestoreEngine(None, catalog, tmp_path), batch_window_s=0.05) + ticket = hyd.request("/a.bin") + assert ticket.job_id == job.id # re-attached, not a second restore + dest.parent.mkdir(parents=True) + dest.write_bytes(FILES["/a.bin"][0]) + job.state = "completed" + catalog.session.commit() + assert hyd.wait(ticket, 5) == FILES["/a.bin"][0] + + +def test_unmount_cancels_pending_batches(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path) + hyd.batch_window_s = 30 + ticket = hyd.request("/a.bin") + ops.destroy("/") + assert ticket.done.is_set() and ticket.error == "hydration cancelled by unmount" + assert engine.batches == [] + + +def test_gateway_close_of_unpinned_open_does_not_unpin_other_handle(catalog, tmp_path): # type: ignore[no-untyped-def] + from openblade.nas.protocol_gateway import ProtocolGateway + + _, hyd, engine = _plane(catalog, tmp_path) + gw = ProtocolGateway() + gw.attach_hydrator(hyd) + checksum = catalog.get_file_record("/a.bin").checksum_sha256 # type: ignore[union-attr] + engine.gate.clear() + handle_b = gw.on_open("/a.bin") # client B opens mid-hydration + assert handle_b.state.value == "hydrating" + engine.gate.set() + hyd.wait(hyd.request("/a.bin"), 5) + assert gw.on_open("/a.bin").state.value == "online" # client A opens the online file + + gw.on_close(handle_b.token) # client B closes; A still holds the file open + gw.on_close(handle_b.token) # double close (rig retry) must not drop A's pin + gw.on_close("never-opened") # close of a failed open is a no-op + hyd.cache.max_bytes = 1 + gw.on_open("/c.bin", timeout=5) # budget pressure + + assert hyd.cache.is_cached(checksum) + + +class _RaisingEngine(FakeEngine): + def __init__(self, staging: Path, exc: Exception) -> None: + super().__init__(staging) + self.exc = exc + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + raise self.exc + + +@pytest.mark.parametrize( + ("exc", "expected"), + [ + (CartridgeOfflineError("cartridge TAPE01 is offline"), "cartridge TAPE01 is offline"), + (RuntimeError("dsn=postgres://u:secret@db"), "restore failed unexpectedly"), + ], +) +def test_untyped_engine_error_fails_ticket_and_file_goes_offline(catalog, tmp_path, exc, expected): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + hyd = Hydrator(catalog, fs.cache, _RaisingEngine(tmp_path / "s", exc), batch_window_s=0.01) + ticket = hyd.request("/a.bin") + assert ticket.done.wait(3), "worker died without completing the ticket" + assert ticket.error == expected + assert "secret" not in (hyd.last_error("/a.bin") or "") + assert hyd.status("/a.bin").value == "offline" + + +def test_request_unregisters_ticket_when_job_lookup_raises(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s") + engine.find_active_job = lambda p: (_ for _ in ()).throw(RuntimeError("db down")) # type: ignore[method-assign] + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + with pytest.raises(RuntimeError): + hyd.request("/a.bin") + assert hyd.status("/a.bin").value == "offline" + + +def test_resume_sees_job_completed_by_another_session(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"catalog_path": "/a.bin"}) + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=3.0 + ) + assert catalog.get_job(job.id).state == "pending" # load into the identity map + other = CatalogRepository(get_session()) + row = other.get_job(job.id) + row.metadata_json = json.dumps({"catalog_path": "/a.bin", "dest_path": str(tmp_path / "x")}) + row.state = "completed" + other.session.commit() + assert engine.resume(str(job.id)) == tmp_path / "x" + + +def test_resume_gives_up_at_deadline_with_typed_error(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"catalog_path": "/a.bin"}) + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=0.2 + ) + with pytest.raises(HydrationTimeoutError): + engine.resume(str(job.id)) + + +def test_batch_member_is_not_evicted_before_its_waiter_reads_it(catalog, tmp_path): # type: ignore[no-untyped-def] + class SlowReader(HydrationCache): + def retrieve(self, checksum: str) -> bytes: + threading.Event().wait(0.3) # let the rest of the batch commit first + return super().retrieve(checksum) + + cache = SlowReader(str(tmp_path / "cache"), max_bytes=600) # a (500) + b (250) > 600 + engine = FakeEngine(tmp_path / "s") + engine.gate.clear() + hyd = Hydrator(catalog, cache, engine, batch_window_s=0.05) + tickets = [hyd.request("/a.bin"), hyd.request("/b.bin")] + results: dict[str, object] = {} + + def waiter(t): # type: ignore[no-untyped-def] + try: + results[t.catalog_path] = hyd.wait(t, 5) + except Exception as exc: # noqa: BLE001 - the test records whatever the waiter saw + results[t.catalog_path] = exc + + threads = [threading.Thread(target=waiter, args=(t,)) for t in tickets] + for th in threads: + th.start() + threading.Event().wait(0.2) + engine.gate.set() + for th in threads: + th.join(10) + assert results == {"/a.bin": FILES["/a.bin"][0], "/b.bin": FILES["/b.bin"][0]} + assert cache.open_count(tickets[0].checksum) == 0 # waiter pins are released + assert cache.open_count(tickets[1].checksum) == 0 + + +def test_request_racing_shutdown_fails_ticket_instead_of_stranding_it(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + hyd = Hydrator(catalog, fs.cache, FakeEngine(tmp_path / "s"), batch_window_s=0.01) + real_tape_key = hyd._tape_key + + def shutdown_mid_request(path, record): # type: ignore[no-untyped-def] + hyd.shutdown(timeout=1) # lands after request() passed its _closed check + return real_tape_key(path, record) + + hyd._tape_key = shutdown_mid_request # type: ignore[method-assign] + ticket = hyd.request("/a.bin") + assert ticket.done.wait(2), "ticket stranded by shutdown" + assert ticket.error == "hydration cancelled by unmount" + + +def test_shutdown_waits_on_one_overall_deadline(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s") + engine.gate.clear() + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + hyd.request("/a.bin") # TAPE01 + hyd.request("/c.bin") # TAPE02 -> a second blocked worker + threading.Event().wait(0.2) + started = time.monotonic() + hyd.shutdown(timeout=0.4) + elapsed = time.monotonic() - started + engine.gate.set() + assert elapsed < 0.6, f"shutdown took {elapsed:.2f}s for a 0.4s budget" + + +@pytest.mark.parametrize("corrupt", [False, True]) +def test_staging_copy_is_removed_after_verified_store(catalog, tmp_path, corrupt): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s", corrupt=corrupt) + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + ticket = hyd.request("/c.bin") + assert ticket.done.wait(3) + assert (ticket.error is not None) is corrupt + assert not (tmp_path / "s" / "c.bin").exists(), "staging copy leaked" + + +def test_restarted_engine_finds_and_resumes_batch_job(catalog, tmp_path): # type: ignore[no-untyped-def] + """Regression: batch jobs store {"batch": [...]}; a restarted hydrator must see them.""" + other = CatalogRepository(get_session()) + job = other.create_job( + "restore", + { + "batch": [ + {"catalog_path": "/a.bin", "dest_path": str(tmp_path / "a")}, + {"catalog_path": "/b.bin", "dest_path": str(tmp_path / "b")}, + ] + }, + ) + other.session.commit() + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=3.0 + ) + enqueued: list[object] = [] + engine.restore_batch = lambda *a: enqueued.append(a) or {} # type: ignore[method-assign,func-returns-value] + + assert engine.find_active_job("/b.bin") == str(job.id) + assert engine.find_active_job("/c.bin") is None + row = other.get_job(job.id) + row.state = "completed" + other.session.commit() + assert engine.resume(str(job.id), "/b.bin") == tmp_path / "b" + assert enqueued == [] + + +def test_resume_job_without_destination_raises_typed_error(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"something": "else"}) + job.state = "completed" + catalog.session.commit() + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=1.0 + ) + with pytest.raises(HydrationFailedError): + engine.resume(str(job.id), "/a.bin") + + +def test_read_range_is_o_length_and_never_rehashes(tmp_path, monkeypatch): # type: ignore[no-untyped-def] + """Regression: range reads used to re-read and re-hash the whole file per call.""" + import openblade.fuse.cache as cache_mod + from openblade.fuse.cache import CacheError + + data = os.urandom(4 * 1024 * 1024) + checksum = hashlib.sha256(data).hexdigest() + cache = HydrationCache(str(tmp_path / "cache")) + cache.store(checksum, data) + calls: list[int] = [] + real = cache_mod.hashlib.sha256 + + def counting(*a, **k): # type: ignore[no-untyped-def] + calls.append(1) + return real(*a, **k) + + monkeypatch.setattr(cache_mod.hashlib, "sha256", counting) + chunk = 64 * 1024 + out = b"".join(cache.read_range(checksum, off, chunk) for off in range(0, len(data), chunk)) + + assert out == data + assert calls == [] + with pytest.raises(CacheError): + cache.read_range(checksum, -1, 10) diff --git a/tests/integration/test_nas_i3_end_to_end_archive_recall.py b/tests/integration/test_nas_i3_end_to_end_archive_recall.py index df07fd5..869ecae 100644 --- a/tests/integration/test_nas_i3_end_to_end_archive_recall.py +++ b/tests/integration/test_nas_i3_end_to_end_archive_recall.py @@ -207,7 +207,7 @@ def test_nas_i3_end_to_end_archive_recall_hash_verified(iteration: int, tmp_path "pool": "archive-pool", "files": relative_paths, "file_sizes": file_sizes, - "available_tapes": tape_candidates[:2], + # With `pool` set the server derives the tape set; sending it is a 422. "copies": 1, "verify_before_archive": True, "verify_after_archive": True, diff --git a/tests/integration/test_sharded_archive_atomicity.py b/tests/integration/test_sharded_archive_atomicity.py index e2814a7..0758161 100644 --- a/tests/integration/test_sharded_archive_atomicity.py +++ b/tests/integration/test_sharded_archive_atomicity.py @@ -9,13 +9,15 @@ from pathlib import Path +import pytest from sqlalchemy import func, select from openblade.catalog.db import get_session, init_db from openblade.catalog.models import FileInstance from openblade.catalog.repository import _ARCHIVED_INSTANCE_STATES, CatalogRepository +from openblade.domain.errors import StaleLeaseError from openblade.domain.policies import FormatConfirmation, SafetyToken -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore from openblade.jobs.shard import ShardMode from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive from openblade.simulator.library import MockLibraryBackend @@ -74,7 +76,9 @@ def test_load_helpers_do_not_mutate_scheduler_lock_key() -> None: assert slot is not None library.load(slot, 1) # cartridge physically in drive 1 handle = DriveHandle(drive_id=0, barcode=BARCODES[0]) # scheduler reserved drive 0 - physical, _slot = helper(None, library, ltfs, handle, "job") # already-loaded fast path + physical, _slot = helper( + _catalog(), library, ltfs, handle, "job" + ) # already-loaded fast path assert handle.drive_id == 0, f"{helper.__name__} mutated the scheduler lock key" assert handle.physical == 1 and physical == 1 @@ -168,3 +172,39 @@ def test_unload_failure_blocks_commit(tmp_path: Path, monkeypatch) -> None: assert result.errors # stuck unload surfaced, not swallowed assert result.files_archived == 0 assert _archived_count(catalog) == 0 # tape stuck in drive -> batch must not be durable + + +def test_stale_lease_aborts_archive_commits_nothing_and_leaves_the_drive_alone( + tmp_path: Path, monkeypatch +) -> None: + # Spec docs/decisions/2026-10-09-persistent-drive-leases.md §3: a lease + # released behind a running job fences it out before unmount + commit, and + # the job must NOT touch the drive afterwards — whoever holds the lease now + # may already have their own tape loaded and mounted there. + library, ltfs = _setup() + catalog = _catalog() + store = InMemoryLeaseStore() + scheduler = DriveScheduler(num_drives=2, store=store) + real_write, real_unmount = ltfs.write_file, ltfs.unmount + unmounted: list[object] = [] + + def write_then_lose_lease(*args, **kwargs): + result = real_write(*args, **kwargs) + store.release([lease.id for lease in store.live_leases()]) + return result + + def tracking_unmount(mount): + unmounted.append(mount) + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "write_file", write_then_lose_lease) + monkeypatch.setattr(ltfs, "unmount", tracking_unmount) + job = catalog.create_job("archive", {}) + + with pytest.raises(StaleLeaseError): + run_sharded_archive(_request(_source(tmp_path)), library, ltfs, catalog, scheduler, job.id) + + assert _archived_count(catalog) == 0 # no mark_instance_archived + assert unmounted == [] # fenced out: no unmount + loaded = {str(drive.barcode) for drive in library.inventory().drives if drive.barcode} + assert loaded == set(BARCODES) # ...and no unload: the drives are left as they were diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py new file mode 100644 index 0000000..c52652b --- /dev/null +++ b/tests/integration/test_staged_sharded_archive.py @@ -0,0 +1,192 @@ +"""Staged (STAGING -> VERIFYING -> one bulk ARCHIVED) sharded archive + cleanup journaling.""" + +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import select +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session + +from openblade.catalog.models import FileInstance +from openblade.domain.errors import JournalWriteError +from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore +from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, run_sharded_archive +from tests.integration.test_sharded_archive_atomicity import ( + BARCODES, + _catalog, + _request, + _setup, + _source, +) + + +def _states(catalog: Any) -> set[str]: + return {str(i.state) for i in catalog.session.scalars(select(FileInstance)).all()} + + +def _events(catalog: Any, job_id: str) -> list[str]: + return [entry.event for entry in catalog.job_journal(job_id)] + + +def test_second_lane_write_failure_stages_but_never_archives(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + real_write, real_unmount = ltfs.write_file, ltfs.unmount + calls = {"write": 0} + unmounted: list[Any] = [] + lock = threading.Lock() + + def write(*args: Any, **kwargs: Any) -> Any: + with lock: + calls["write"] += 1 + nth = calls["write"] + if nth == 2: + raise OSError("lane 2 write failed") + return real_write(*args, **kwargs) + + def unmount(mount: Any) -> Any: + unmounted.append(mount) + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "write_file", write) + monkeypatch.setattr(ltfs, "unmount", unmount) + job = catalog.create_job("archive", {}) + result = run_sharded_archive( + _request(_source(tmp_path)), library, ltfs, catalog, DriveScheduler(num_drives=2), job.id + ) + + assert result.errors and result.files_archived == 0 + states = _states(catalog) + assert states and states <= {"staging", "verifying"}, states # nothing ARCHIVED + events = _events(catalog, job.id) + assert "failed" in events and "committed" not in events + assert len(unmounted) == 2 # both lanes unmounted on the error path + assert all(d.barcode is None for d in library.inventory().drives) # and unloaded + + +def test_unmount_raise_in_cleanup_is_journaled_and_other_drive_unloaded( + tmp_path: Path, monkeypatch +) -> None: + library, ltfs = _setup() + catalog = _catalog() + real_unmount = ltfs.unmount + + def unmount(mount: Any) -> Any: + if str(mount.barcode) == BARCODES[0]: + raise RuntimeError("stuck") + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "unmount", unmount) + real_unload = library.unload + unload_requests: list[int] = [] + + def unload(drive_id: int, target_slot: int) -> Any: + unload_requests.append(drive_id) + return real_unload(drive_id, target_slot) + + monkeypatch.setattr(library, "unload", unload) + job = catalog.create_job("archive", {}) + run_sharded_archive( + _request(_source(tmp_path)), library, ltfs, catalog, DriveScheduler(num_drives=2), job.id + ) + + stored = catalog.get_job(job.id) + assert stored is not None and PHYSICAL_STATE_UNKNOWN in str(stored.error) + rows = [e for e in catalog.job_journal(job.id) if e.event == "physical_state_unknown"] + assert rows and rows[0].detail["barcode"] == BARCODES[0] + drives = {str(d.barcode): d.drive_id for d in library.inventory().drives if d.barcode} + in_drives = set(drives) + assert BARCODES[1] not in in_drives # the OTHER drive was still unloaded + # Never unload while LTFS is mounted: the stuck drive must not even be asked. + assert BARCODES[0] in in_drives + assert drives[BARCODES[0]] not in unload_requests + assert "archived" not in _states(catalog) + + +def _stuck_unmount(ltfs: Any, monkeypatch: Any) -> None: + real_unmount = ltfs.unmount + + def unmount(mount: Any) -> Any: + if str(mount.barcode) == BARCODES[0]: + raise RuntimeError("stuck") + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "unmount", unmount) + + +def test_journal_locked_once_is_retried_and_written(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + _stuck_unmount(ltfs, monkeypatch) + real_add = Session.add + calls = {"n": 0} + + def flaky_add(self: Session, instance: object, *args: Any, **kwargs: Any) -> None: + is_psu = getattr(instance, "event", None) == "physical_state_unknown" + if is_psu and calls["n"] == 0: + calls["n"] += 1 + raise OperationalError("INSERT", {}, Exception("database is locked")) + real_add(self, instance, *args, **kwargs) + + monkeypatch.setattr(Session, "add", flaky_add) + store = InMemoryLeaseStore() + job = catalog.create_job("archive", {}) + run_sharded_archive( + _request(_source(tmp_path)), + library, + ltfs, + catalog, + DriveScheduler(num_drives=2, store=store), + job.id, + ) + assert calls["n"] == 1 + rows = [e for e in catalog.job_journal(job.id) if e.event == "physical_state_unknown"] + assert rows and rows[0].detail["barcode"] == BARCODES[0] + assert store.live_leases() == [] # journaled -> released as before + + +def test_journal_failure_keeps_the_lease_fail_closed(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + _stuck_unmount(ltfs, monkeypatch) + real_unload = library.unload + unloaded: list[int] = [] + + def unload(drive_id: int, target_slot: int) -> Any: + unloaded.append(drive_id) + return real_unload(drive_id, target_slot) + + monkeypatch.setattr(library, "unload", unload) + + def broken(*_args: Any, **_kwargs: Any) -> None: + raise OperationalError("INSERT", {}, Exception("database is locked")) + + monkeypatch.setattr(catalog, "journal_durable", broken) + store = InMemoryLeaseStore() + job = catalog.create_job("archive", {}) + with pytest.raises(JournalWriteError) as caught: + run_sharded_archive( + _request(_source(tmp_path)), + library, + ltfs, + catalog, + DriveScheduler(num_drives=2, store=store, job_id=job.id), + job.id, + ) + # The job's recorded error still names the lane failure, not only the journal. + assert "original failure: " in str(caught.value) + in_drives = {str(d.barcode) for d in library.inventory().drives if d.barcode} + assert BARCODES[1] not in in_drives # the other drive was still cleaned + assert unloaded # ...by a real UNLOAD + live = store.live_leases() + assert [lease.barcode for lease in live] == [BARCODES[0]] + # A second job must NOT be handed the drive that may still have LTFS mounted. + second = DriveScheduler(num_drives=2, store=store, job_id="second") + with pytest.raises(Exception): # noqa: B017,PT011 - any refusal; success is the bug + second.acquire_drives(["SECOND1", "SECOND2"], timeout=0.2) + got = second.acquire_drives(["SECOND1"], timeout=0.2) + assert got[0].drive_id != live[0].drive_id diff --git a/tests/integration/test_three_drive_concurrency.py b/tests/integration/test_three_drive_concurrency.py index ed3c7e4..c7391d2 100644 --- a/tests/integration/test_three_drive_concurrency.py +++ b/tests/integration/test_three_drive_concurrency.py @@ -10,14 +10,17 @@ import threading import time from collections.abc import Callable +from pathlib import Path import pytest +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import ChangerBusyError, DriveOccupiedError from openblade.domain.models import JobType, MountMode from openblade.domain.policies import FormatConfirmation, SafetyToken from openblade.jobs.queue import JobQueue -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend from openblade.simulator.scenarios import scalar_i3_default @@ -25,6 +28,13 @@ BARCODES = ["TRI001L8", "TRI002L8", "TRI003L8"] +@pytest.fixture +def queue(tmp_path: Path) -> JobQueue: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + catalog = CatalogRepository(get_session()) + return JobQueue(catalog, CatalogLeaseStore(catalog)) + + @pytest.fixture def three_drive_library() -> tuple[MockLibraryBackend, MockLTFSBackend]: library = MockLibraryBackend(num_slots=10, num_drives=3) @@ -55,13 +65,18 @@ def test_simulator_drive_count_is_configurable() -> None: def test_three_jobs_run_on_three_drives_with_disjoint_ownership( three_drive_library: tuple[MockLibraryBackend, MockLTFSBackend], + queue: JobQueue, ) -> None: library, ltfs = three_drive_library scheduler = DriveScheduler(num_drives=3) - queue = JobQueue() jobs = [queue.create_job(JobType.ARCHIVE, {"barcode": barcode}) for barcode in BARCODES] started = threading.Barrier(len(jobs)) + # Rendezvous reached by each job only while it HOLDS its drive lease. Without + # it, a fast job can finish and release drive 0 before a descheduled job even + # acquires, and the scheduler then (correctly) hands drive 0 out again — so + # "three jobs, three drives" is only asserted when all three overlap. + all_holding = threading.Barrier(len(jobs)) ownership: list[tuple[str, int]] = [] errors: list[BaseException] = [] ownership_lock = threading.Lock() @@ -75,6 +90,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: with ownership_lock: ownership.append((job_id, handle.drive_id)) try: + all_holding.wait(timeout=10) # One robot, three drives: the changer serializes, the I/O does not. _load_with_retry(library, slot_id, handle.drive_id) mount = ltfs.mount(barcode, MountMode.READ_WRITE) @@ -97,6 +113,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: for thread in threads: thread.join(timeout=15) + assert not any(thread.is_alive() for thread in threads) assert errors == [] assert len(ownership) == 3 # Every job got its own drive, and all three drives were used. @@ -109,8 +126,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: assert ltfs.read_bytes(barcode, f"/{barcode}.bin") == b"x" * 1024 -def test_job_queue_refuses_to_share_a_drive_between_jobs() -> None: - queue = JobQueue() +def test_job_queue_refuses_to_share_a_drive_between_jobs(queue: JobQueue) -> None: first = queue.create_job(JobType.ARCHIVE, {}) second = queue.create_job(JobType.ARCHIVE, {}) queue.claim_drive(2, first.id) diff --git a/tests/property/test_robot_transitions.py b/tests/property/test_robot_transitions.py index 62fe713..d04a1ab 100644 --- a/tests/property/test_robot_transitions.py +++ b/tests/property/test_robot_transitions.py @@ -2,6 +2,7 @@ from pathlib import Path, PurePosixPath +import pytest from hypothesis import given, settings from hypothesis import strategies as st @@ -10,6 +11,8 @@ from openblade.domain.policies import FormatConfirmation, SafetyToken from openblade.simulator.scenarios import two_drive_library +pytestmark = pytest.mark.fuzz + @given(st.lists(st.integers(min_value=1, max_value=20), min_size=0, max_size=50)) @settings(max_examples=50) diff --git a/tests/test_nas_protocol_render.py b/tests/test_nas_protocol_render.py new file mode 100644 index 0000000..e33de63 --- /dev/null +++ b/tests/test_nas_protocol_render.py @@ -0,0 +1,60 @@ +"""Unit tests for the SMB/NFS config renderers (no docker; not marked protocols).""" + +from pathlib import Path + +from openblade.nas.nfs import NfsExport, render_exports, render_ganesha_conf +from openblade.nas.samba import SambaShare, render_smb_conf + + +def test_ganesha_client_list_is_unquoted() -> None: + # Regression: Ganesha rejects `Clients = "*"` ("Expected a client") and then + # silently drops the whole EXPORT, so NFS mounts fail with ENOENT. + block = NfsExport(Path("/rig/export/ro"), pseudo="/ro").render_ganesha() + + assert "Clients = *;" in block + assert '"*"' not in block + + +def test_ganesha_conf_renders_one_export_per_share_with_access_type() -> None: + conf = render_ganesha_conf( + [ + NfsExport(Path("/e/ro"), export_id=1, pseudo="/ro"), + NfsExport(Path("/e/rw"), export_id=2, pseudo="/rw", read_only=False), + ] + ) + + assert conf.count("EXPORT {") == 2 + assert 'Pseudo = "/ro";' in conf and "Access_Type = RO;" in conf + assert 'Pseudo = "/rw";' in conf and "Access_Type = RW;" in conf + assert "Protocols = 4;" in conf + + +def test_ganesha_pseudo_defaults_to_path() -> None: + assert 'Pseudo = "/data/x";' in NfsExport(Path("/data/x")).render_ganesha() + + +def test_kernel_exports_line_reflects_read_only() -> None: + out = render_exports([NfsExport(Path("/a")), NfsExport(Path("/b"), read_only=False)]) + + assert out == "/a *(ro,sync,no_subtree_check)\n/b *(rw,sync,no_subtree_check)\n" + + +def test_smb_conf_maps_share_flags() -> None: + conf = render_smb_conf( + [ + SambaShare("ro", Path("/e/ro")), + SambaShare("rw", Path("/e/rw"), read_only=False, case_sensitive=True), + ] + ) + + ro, rw = conf.split("[ro]", 1)[1].split("[rw]", 1) + assert "writeable = no" in ro and "case sensitive = no" in ro + assert "writeable = yes" in rw and "case sensitive = yes" in rw + + +def test_smb_conf_disables_client_caching_of_hydrating_files() -> None: + conf = render_smb_conf([SambaShare("rw", Path("/e/rw"), read_only=False)]) + globals_ = conf.split("[rw]", 1)[0] + + for option in ("oplocks = no", "level2 oplocks = no", "smb2 leases = no"): + assert option in globals_ diff --git a/tests/unit/test_aml_faults.py b/tests/unit/test_aml_faults.py new file mode 100644 index 0000000..c05d7ab --- /dev/null +++ b/tests/unit/test_aml_faults.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from openblade.api import aml_faults +from openblade.api.aml_faults import ( + FaultProfile, + FaultProfileError, + FaultState, + parse_fault_profile, +) + + +def test_unset_means_no_faults() -> None: + assert parse_fault_profile(None) == FaultProfile() + assert parse_fault_profile(" ") == FaultProfile() + assert not parse_fault_profile("instant").active + + +def test_presets_resolve() -> None: + assert parse_fault_profile("intermittent-drive").load_fail_every == 3 + assert parse_fault_profile("Session-Expiry").auth_ttl_s == 5.0 + assert parse_fault_profile("rebooting").reboot_window_s == 20.0 + assert parse_fault_profile("degraded-media").checksum_retry_every == 4 + + +def test_json_object() -> None: + profile = parse_fault_profile('{"load_fail_every": 2, "auth_ttl_s": 1}') + assert profile == FaultProfile(load_fail_every=2, auth_ttl_s=1.0) + + +@pytest.mark.parametrize( + "raw", + [ + "no-such-profile", + "{not json", + "[1]", + '{"bogus": 1}', + '{"load_fail_every": 1.5}', + '{"load_fail_every": true}', + '{"auth_ttl_s": -1}', + '{"reboot_window_s": "20"}', + ], +) +def test_strict_parse_rejects(raw: str) -> None: + with pytest.raises(FaultProfileError): + parse_fault_profile(raw) + + +def test_counters() -> None: + state = FaultState(FaultProfile(load_fail_every=3, checksum_retry_every=2)) + assert [state.should_fail_load() for _ in range(6)] == [False, False, True] * 2 + assert [state.should_retry_checksum() for _ in range(4)] == [False, True] * 2 + assert not state.in_reboot_window() + + +def test_reboot_window(monkeypatch: pytest.MonkeyPatch) -> None: + now = [100.0] + monkeypatch.setattr(aml_faults.time, "monotonic", lambda: now[0]) + state = FaultState(FaultProfile(reboot_window_s=20.0)) + assert state.in_reboot_window() + assert state.reboot_retry_after_s() == 20 + now[0] = 120.0 + assert not state.in_reboot_window() + + +def test_env_state_cached_and_reset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, "rebooting") + aml_faults.reset_fault_state() + try: + assert aml_faults.get_fault_state().profile.reboot_window_s == 20.0 + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, "bad") + assert aml_faults.get_fault_state().profile.reboot_window_s == 20.0 + aml_faults.reset_fault_state() + with pytest.raises(FaultProfileError): + aml_faults.get_fault_state() + finally: + aml_faults.reset_fault_state() diff --git a/tests/unit/test_capacity_budget_api.py b/tests/unit/test_capacity_budget_api.py new file mode 100644 index 0000000..d644dc5 --- /dev/null +++ b/tests/unit/test_capacity_budget_api.py @@ -0,0 +1,97 @@ +"""API-level guards: a pool-budgeted plan's capacity inputs come from the server only.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig +from openblade.nas.planner import POOL_BUDGET_BLOCKER + +client = TestClient(app) + + +@pytest.fixture(autouse=True) +def _context(tmp_path: Path) -> None: + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'b.db'}"))) + assert client.post("/nas/pools", json={"id": "pool-1", "name": "P1"}).status_code in {200, 201} + + +def _body(size: int, **extra: object) -> dict[str, object]: + return {"files": ["a.bin"], "file_sizes": {"a.bin": size}, "pool": "pool-1", **extra} + + +@pytest.mark.parametrize( + "field", + [ + {"tape_capacities": {"T1": 10**18}}, + {"available_tapes": ["T1"]}, + {"reserved_bytes": {"T1": 0}}, + {"pool_reserved_bytes": 0}, + ], +) +def test_client_capacity_inputs_with_pool_are_refused(field: dict[str, object]) -> None: + response = client.post("/nas/archive-plan", json=_body(1, **field)) + assert response.status_code == 422 + assert "server derives" in response.json()["detail"] + + +def test_unknown_pool_is_404() -> None: + response = client.post("/nas/archive-plan", json={**_body(1), "pool": "nope"}) + assert response.status_code == 404 + + +def test_honest_over_budget_is_blocked_by_server_capacity() -> None: + plan = client.post("/nas/archive-plan", json=_body(10**18)).json() + assert POOL_BUDGET_BLOCKER in plan["enqueue_blockers"] + + +def test_poolless_request_keeps_body_inputs_and_no_budget() -> None: + body = { + "files": ["a.bin"], + "file_sizes": {"a.bin": 10}, + "available_tapes": ["T1"], + "tape_capacities": {"T1": 100}, + } + plan = client.post("/nas/archive-plan", json=body).json() + assert POOL_BUDGET_BLOCKER not in plan["enqueue_blockers"] + + +def test_preview_books_nothing_but_second_start_is_409(monkeypatch: pytest.MonkeyPatch) -> None: + from datetime import UTC, datetime, timedelta + + from openblade.bootstrap import get_context + from openblade.nas.capacity import server_data_tapes + + # Arrange: keep started jobs running (no release) and leave 1500 bytes of pool budget. + monkeypatch.setattr("openblade.api.nas_config.run_ingest_job", lambda *a, **k: None) + context = get_context() + repo = context.catalog + capacity = sum(server_data_tapes(repo, context.library).values()) + now = datetime.now(UTC) + assert repo.reserve_nas_capacity( + "pool-1", + capacity - 1_500, + "other-job", + now + timedelta(hours=1), + capacity_bytes=capacity, + now=now, + ) + + # Act: two previews of 1000 bytes each, then start both. + first = client.post("/nas/archive-plan", json=_body(1_000, ingest_mode="source_stream")).json() + second = client.post("/nas/archive-plan", json=_body(1_000, ingest_mode="source_stream")).json() + start = {"dataset_name": "ds", "pool_id": "pool-1", "auto_clean_drives": False} + first_start = client.post("/nas/ingest/start", json={**start, "plan_id": first["plan_id"]}) + second_start = client.post("/nas/ingest/start", json={**start, "plan_id": second["plan_id"]}) + + # Assert: previews never consumed budget; the second START lost to the first booking. + assert first["is_safe_to_enqueue"] and second["is_safe_to_enqueue"] + assert first_start.status_code == 200, first_start.text + assert second_start.status_code == 409 + owners = {r["owner"] for r in repo.reservations_for_pool("pool-1", now=now)} + assert owners == {"other-job", first["plan_id"]} diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py new file mode 100644 index 0000000..9ccd8bd --- /dev/null +++ b/tests/unit/test_capacity_reservations.py @@ -0,0 +1,401 @@ +"""Pool reservations are a budget the archive planner must respect.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from openblade.catalog.repository import CatalogRepository +from openblade.nas.capacity import ( + PLAN_RESERVATION_TTL, + ReservationError, + book_plan_capacity, + count_scratch_tapes, + plan_with_pool_budget, +) +from openblade.nas.planner import POOL_BUDGET_BLOCKER, ArchivePlanner +from openblade.nas.types import ArchivePlanRequest + +TAPE_CAPACITY = 1_000 + + +def _request( + pool_reserved_bytes: int, *, copies: int = 1, pool: str | None = "pool-1" +) -> ArchivePlanRequest: + return ArchivePlanRequest( + pool=pool, + files=["/a", "/b"], + file_sizes={"/a": 300, "/b": 300}, + available_tapes=["T00001L8", "T00002L8"], + tape_capacities={"T00001L8": TAPE_CAPACITY, "T00002L8": TAPE_CAPACITY}, + copies=copies, + pool_reserved_bytes=pool_reserved_bytes, + ) + + +def _server_request() -> ArchivePlanRequest: + """A pool request as the share-push route builds it: no client capacity inputs.""" + return ArchivePlanRequest( + files=["a.bin", "b.bin"], file_sizes={"a.bin": 300, "b.bin": 300}, pool="pool-1" + ) + + +def test_plan_fits_exactly_within_pool_budget() -> None: + # 2 tapes x 1000 = 2000; 600 placed -> exactly 1400 may be reserved by others. + plan = ArchivePlanner().plan(_request(1_400)) + assert POOL_BUDGET_BLOCKER not in plan.enqueue_blockers + assert plan.is_safe_to_enqueue + + +def test_same_plan_refused_when_budget_shrinks_by_one_byte() -> None: + plan = ArchivePlanner().plan(_request(1_401)) + assert POOL_BUDGET_BLOCKER in plan.enqueue_blockers + assert not plan.is_safe_to_enqueue + + +def test_budget_counts_every_copy() -> None: + # Two copies place 1200 bytes; 801 reserved leaves only 1199. + assert POOL_BUDGET_BLOCKER in ArchivePlanner().plan(_request(801, copies=2)).enqueue_blockers + assert ( + POOL_BUDGET_BLOCKER not in ArchivePlanner().plan(_request(800, copies=2)).enqueue_blockers + ) + + +def test_default_budget_is_unchanged_behaviour() -> None: + request = _request(0) + assert request.pool_reserved_bytes == 0 + assert ArchivePlanner().plan(request).is_safe_to_enqueue + + +NOW = datetime(2026, 10, 11, 12, 0, tzinfo=UTC) + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + from tests.unit.test_nas_restore_planner import make_nas_service + + return make_nas_service(tmp_path).repository + + +@pytest.fixture +def two_tapes(monkeypatch: pytest.MonkeyPatch) -> None: + """Server-derived pool capacity of 2000 bytes for ``book_plan_capacity``.""" + monkeypatch.setattr( + "openblade.nas.capacity.server_data_tapes", + lambda repo, library: {"T00001L8": TAPE_CAPACITY, "T00002L8": TAPE_CAPACITY}, + ) + + +@pytest.mark.usefixtures("two_tapes") +def test_preview_books_nothing_but_second_start_is_refused(repo: CatalogRepository) -> None: + # Arrange: pool of 2000 bytes; another job already holds 800. + assert repo.reserve_nas_capacity( + "pool-1", 800, "other-job", NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + ) + + # Act: 600- and 601-byte previews both fit (nothing booked); only one START can book. + first = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW) + second_request = _request(0, pool="pool-1").model_copy( + update={"file_sizes": {"/a": 300, "/b": 301}} + ) + second = plan_with_pool_budget(repo, second_request, now=NOW) + book_plan_capacity(repo, first, None, now=NOW) + + # Assert + assert first.is_safe_to_enqueue and second.is_safe_to_enqueue + with pytest.raises(ReservationError): + book_plan_capacity(repo, second, None, now=NOW) + assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)} == { + "other-job", + first.plan_id, + } + + +@pytest.mark.usefixtures("two_tapes") +def test_booking_expires_after_ttl(repo: CatalogRepository) -> None: + first = plan_with_pool_budget(repo, _request(0, pool="pool-1", copies=3), now=NOW) + book_plan_capacity(repo, first, None, now=NOW) + + later = plan_with_pool_budget( + repo, _request(0, pool="pool-1", copies=3), now=NOW + PLAN_RESERVATION_TTL + ) + book_plan_capacity(repo, later, None, now=NOW + PLAN_RESERVATION_TTL) + + assert later.is_safe_to_enqueue + + +def test_expired_reservation_is_ignored(repo: CatalogRepository) -> None: + assert repo.reserve_nas_capacity( + "pool-1", 2_000, "stale-job", NOW + timedelta(seconds=1), capacity_bytes=2_000, now=NOW + ) + + plan = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW + timedelta(minutes=1)) + + assert plan.is_safe_to_enqueue + + +@pytest.mark.usefixtures("two_tapes") +def test_lost_booking_race_raises_typed_error(repo: CatalogRepository) -> None: + # Simulate a concurrent writer landing between the budget read and the booking. + plan = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW) + + class RacingRepo: + def reserve_nas_capacity(self, *args: object, **kwargs: object) -> bool: + return False + + with pytest.raises(ReservationError): + book_plan_capacity(RacingRepo(), plan, None, now=NOW) # type: ignore[arg-type] + + +def _scratch_service(tmp_path: Path): # type: ignore[no-untyped-def] + from tests.unit.test_health_service import make_health_service + + service, _ = make_health_service(tmp_path, "scratch.db") + return service, count_scratch_tapes(service.repo, service.library) + + +def test_scratch_counts_blank_inventory_minus_assigned_and_written(tmp_path: Path) -> None: + from openblade.catalog.models import Cartridge + from openblade.jobs.inventory import InventoryService + + # Arrange: N blank tapes from the simulator inventory; none are catalog rows yet. + service, blank = _scratch_service(tmp_path) + slots = InventoryService(service.library).snapshot().slots + barcodes = sorted( + str(s.barcode) for s in slots if s.barcode and not str(s.barcode).startswith("CLN") + ) + assert blank == len(barcodes) >= 2 + + # Act + Assert: a volume-group member is no longer scratch... + group = service.repo.create_volume_group("vg-scratch") + service.repo.add_barcode_to_volume_group(group.id, barcodes[0]) + assert count_scratch_tapes(service.repo, service.library) == blank - 1 + # ...nor is a tape catalogued as written. + written = service.repo.session.query(Cartridge).filter_by(barcode=barcodes[1]).one_or_none() + if written is None: + service.repo.session.add(Cartridge(barcode=barcodes[1], used_bytes=10)) + else: + written.used_bytes = 10 + service.repo.session.commit() + assert count_scratch_tapes(service.repo, service.library) == blank - 2 + + +def _scratch_component(tmp_path: Path, thresholds: tuple[int, int] | None): # type: ignore[no-untyped-def] + service, base = _scratch_service(tmp_path) + if thresholds is not None: + service.repo.set_scratch_thresholds(base + thresholds[0], base + thresholds[1]) + result = service.check_health() + return result, next((c for c in result.components if c.name == "scratch"), None) + + +def test_scratch_component_absent_when_thresholds_unset(tmp_path: Path) -> None: + _, component = _scratch_component(tmp_path, None) + assert component is None + + +def test_scratch_warn_is_ok_with_message(tmp_path: Path) -> None: + _, component = _scratch_component(tmp_path, (-1, 2)) + assert component is not None + assert component.status.value == "ok" + assert component.message.startswith("scratch count ") + assert " below warn " in component.message + + +def test_scratch_critical_is_degraded_not_unhealthy(tmp_path: Path) -> None: + from openblade.nas.types import HealthStatus + + _, component = _scratch_component(tmp_path, (1, 3)) + assert component is not None + assert component.status is HealthStatus.DEGRADED + + +def test_upload_share_push_plan_refused_when_pool_budget_consumed( + repo: CatalogRepository, +) -> None: + from fastapi import HTTPException + + from openblade.api.routes_upload import _plan_share_push + from openblade.bootstrap import get_context + from openblade.nas.capacity import server_data_tapes + + # Arrange: another job holds all but 100 bytes of the SERVER-derived capacity. + repo.upsert_nas_pool({"id": "pool-1", "name": "P1"}) + capacity = sum(server_data_tapes(repo, get_context().library).values()) + now = datetime.now(UTC) + assert repo.reserve_nas_capacity( + "pool-1", + capacity - 100, + "other-job", + now + timedelta(hours=1), + capacity_bytes=capacity, + now=now, + ) + + # Act: a body-free push (the route never sends capacity inputs) needing 600 bytes. + with pytest.raises(HTTPException) as exc_info: + _plan_share_push(repo, _server_request()) + + # Assert: refused on the budget, and nothing new was booked. + assert exc_info.value.status_code == 400 + assert POOL_BUDGET_BLOCKER in str(exc_info.value.detail) + assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=now)] == ["other-job"] + + +def test_upload_share_push_plan_books_nothing_until_start(repo: CatalogRepository) -> None: + from openblade.api.routes_upload import _plan_share_push + + repo.upsert_nas_pool({"id": "pool-1", "name": "P1"}) + plan = _plan_share_push(repo, _server_request()) + + assert plan.is_safe_to_enqueue + assert repo.reservations_for_pool("pool-1", now=datetime.now(UTC)) == [] + + +def test_finished_job_releases_its_plan_reservation(tmp_path: Path) -> None: + from openblade.nas.ingest import register_archive_plan, start_ingest_job + from tests.unit.test_nas_ingest import _make_plan, _run_job, _setup_service + + # Arrange: the plan's own booking fills the 2000-byte pool; another job holds none. + service, cache_root = _setup_service(tmp_path) + repo = service.repository + now = datetime.now(UTC) + plan = register_archive_plan(_make_plan(cache_root)) + assert repo.reserve_nas_capacity( + "pool-1", 2_000, plan.plan_id, now + PLAN_RESERVATION_TTL, capacity_bytes=2_000, now=now + ) + assert POOL_BUDGET_BLOCKER in ( + plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=now).enqueue_blockers + ) + job = start_ingest_job( + plan=plan, + dataset_name="dataset-a", + pool_id="pool-1", + nas_service=service, + cache_drive_id="cache-1", + ) + + # Act + finished = _run_job(service, job.job_id) + + # Assert: terminal job freed the booking, so the same plan now fits. + assert finished.status.value in {"archived", "failed", "cancelled"} + assert repo.reservations_for_pool("pool-1", now=now) == [] + assert plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=now).is_safe_to_enqueue + + +def test_delete_reservations_for_owner_keeps_other_owners(repo: CatalogRepository) -> None: + for owner in ("plan-a", "plan-a", "plan-b"): + assert repo.reserve_nas_capacity( + "pool-1", 10, owner, NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + ) + + assert repo.delete_nas_reservations_for_owner("plan-a") == 2 + assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)] == ["plan-b"] + + +# --- S3: releasing the booking must never mask the executor's exception --------- + + +def _raising_ingest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def] + """A started cache-drive ingest whose executor raises past run()'s own handler.""" + from openblade.bootstrap import create_context, reset_context + from openblade.config import OpenBladeConfig + from openblade.nas import ingest as ingest_mod + from openblade.nas.service import NasService + from openblade.nas.types import ( + ArchivePlan, + CacheDriveConfig, + IngestMode, + NasPool, + TapeAssignment, + ) + + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 's3.db'}")) + reset_context(context) + ingest_mod.clear_ingest_state() + service = NasService(context.catalog) + cache_root = tmp_path / "cache" + (cache_root / "dataset").mkdir(parents=True) + (cache_root / "dataset" / "a.txt").write_bytes(b"alpha") + service.upsert_pool(NasPool(id="pool-1", name="pool-1")) + service.upsert_cache_drive( + CacheDriveConfig( + id="cache-1", name="c", root_path=str(cache_root), max_bytes=10**6, min_free_bytes=0 + ) + ) + plan = ingest_mod.register_archive_plan( + ArchivePlan( + plan_id="plan-s3", + ingest_mode=IngestMode.CACHE_DRIVE, + source_path=str(cache_root / "dataset"), + pool="pool-1", + volume_group="vg-1", + files=[str(cache_root / "dataset" / "a.txt")], + total_files=1, + total_bytes=5, + tape_assignments=[ + TapeAssignment(barcode="VOL001L9", files=["a.txt"], estimated_bytes=5) + ], + ) + ) + job = ingest_mod.start_ingest_job( + plan=plan, + dataset_name="ds", + pool_id="pool-1", + nas_service=service, + cache_drive_id="cache-1", + ) + + def executor_boom(self: object) -> object: + raise RuntimeError("executor exploded") + + # Raises past run()'s boundary without reaching a terminal status, like + # _mark_dataset_failed failing on a session in PendingRollbackError. + monkeypatch.setattr(ingest_mod.CacheDriveIngest, "run", executor_boom) + + def run() -> object: + return ingest_mod.run_ingest_job( + job.job_id, + nas_service=service, + library=context.library, + ltfs=context.ltfs, + cache_drive_id="cache-1", + ) + + return service, run + + +def test_failed_release_does_not_mask_executor_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service, run = _raising_ingest(tmp_path, monkeypatch) + + def release_boom(owner: str) -> int: + raise ValueError("session is in PendingRollbackError") + + monkeypatch.setattr(service.repository, "delete_nas_reservations_for_owner", release_boom) + + with ( + caplog.at_level("WARNING", logger="openblade.nas.ingest"), + pytest.raises(RuntimeError, match="executor exploded"), + ): + run() + assert "reservation release failed for plan plan-s3" in caplog.text + + +def test_executor_exception_releases_booking_even_if_status_not_terminal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, run = _raising_ingest(tmp_path, monkeypatch) + released: list[str] = [] + monkeypatch.setattr( + service.repository, + "delete_nas_reservations_for_owner", + lambda owner: released.append(owner), + ) + + with pytest.raises(RuntimeError, match="executor exploded"): + run() + assert released == ["plan-s3"] diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index e2d0f87..3dbd2c8 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -32,6 +32,8 @@ RebuildRunStatus, ) +pytestmark = pytest.mark.rebuild + client = TestClient(app) @@ -82,6 +84,7 @@ def _seed_tape( include_shard: bool = True, files_per_tape: int = 2, create_versions: int = 0, + protection: dict[str, object] | None = None, ) -> dict[str, object]: metadata_writer: TapeMetadataWriter = rebuild_env["metadata_writer"] shard_writer: CatalogShardWriter = rebuild_env["shard_writer"] @@ -165,6 +168,7 @@ def _seed_tape( total_bytes=sum(item.size for item in shard_files), tape_set=[barcode], shard_set=[], + protection=protection, ) ], files=shard_files, @@ -406,3 +410,164 @@ def test_api_get_run_returns_record(rebuild_env: dict[str, object]) -> None: assert response.status_code == 200 assert response.json()["id"] == run_id assert response.json()["status"] == RebuildRunStatus.PLANNED.value + + +def _stamp_generation( + rebuild_env: dict[str, object], barcode: str, *, marker: str, siblings: list[str] | None = None +) -> None: + from openblade.nas.ltfs_manifest import CommitMarker + + planner = rebuild_env["planner"] + writer = planner.metadata_writer + manifest = writer.read_manifest(barcode).model_copy( + update={"generation_id": "gen-7", "sibling_tapes": siblings or []} + ) + manifest_sha = writer.write_manifest(barcode, manifest) + writer.write_manifest_checksum(barcode, manifest_sha) + shard_sha = writer._read_text(barcode, planner.shard_writer.SHARD_CHECKSUM_METADATA_PATH) + if marker == "missing": + return + writer.write_commit_marker( + barcode, + CommitMarker( + generation_id="gen-7", + manifest_sha256=manifest_sha if marker == "ok" else "0" * 64, + catalog_shard_sha256=shard_sha.strip(), + written_at="2026-10-09T00:00:00Z", + ), + ) + + +@pytest.mark.parametrize( + ("marker", "reason"), + [("missing", "commit marker missing"), ("bad", "commit marker manifest hash mismatch")], +) +def test_plan_rebuild_uncommitted_generation_not_scanned( + rebuild_env: dict[str, object], marker: str, reason: str +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker=marker) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.barcodes_to_scan == [] + assert [u.model_dump() for u in result.uncommitted_generations] == [ + {"barcode": barcode, "generation_id": "gen-7", "reason": reason} + ] + assert result.missing_siblings == [] + assert len(result.warnings) == 1 + assert barcode in result.warnings[0] + assert reason in result.warnings[0] + + +def test_plan_rebuild_committed_generation_reports_missing_sibling( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker="ok", siblings=["ZZ9999L8"]) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.barcodes_to_scan == [barcode] + assert [m.model_dump() for m in result.missing_siblings] == [ + {"barcode": "ZZ9999L8", "required_by_barcode": barcode, "generation_id": "gen-7"} + ] + assert result.uncommitted_generations == [] + assert len(result.warnings) == 1 + assert "ZZ9999L8" in result.warnings[0] + + +def _dataset_protection_json(rebuild_env: dict[str, object], dataset_id: str) -> str | None: + from openblade.catalog.models import NasDataset as NasDatasetRow + + repo = rebuild_env["planner"].repo + repo.session.expire_all() + row = repo.session.get(NasDatasetRow, dataset_id) + assert row is not None + return row.protection_json + + +def test_execute_rebuild_recovers_dataset_protection_policy( + rebuild_env: dict[str, object], +) -> None: + import json + + from openblade.domain.protection import Protection, ProtectionPolicy + + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + policy = ProtectionPolicy(protection=Protection.replication(3)) + _seed_tape(rebuild_env, barcode, protection=policy.to_dict()) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + stored = _dataset_protection_json(rebuild_env, f"dataset-{barcode}") + assert stored is not None + assert ProtectionPolicy.from_dict(json.loads(stored)) == policy + + +def test_execute_rebuild_pre_v2_shard_leaves_protection_unset( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + assert _dataset_protection_json(rebuild_env, f"dataset-{barcode}") is None + + +def test_execute_rebuild_skips_malformed_protection_policy( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode, protection={"placement": "bogus"}) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + result = rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + assert result.datasets_recovered == 1 + assert _dataset_protection_json(rebuild_env, f"dataset-{barcode}") is None + + +def test_commit_marker_survives_new_defaulted_manifest_field( + rebuild_env: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: the marker hashes on-tape bytes, so a newer ManifestJson with an extra + defaulted field (which changes model_dump()) still sees older generations as committed.""" + from openblade.nas.ltfs_manifest import ManifestJson + + class FutureManifestJson(ManifestJson): + future_field: str = "added-later" + + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker="ok") + writer = rebuild_env["planner"].metadata_writer + original_read = writer.read_manifest + + def read_future(bc: str) -> ManifestJson | None: + parsed = original_read(bc) + return None if parsed is None else FutureManifestJson.model_validate(parsed.model_dump()) + + monkeypatch.setattr(writer, "read_manifest", read_future) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.uncommitted_generations == [] + assert result.barcodes_to_scan == [barcode] diff --git a/tests/unit/test_catalog_rebuild_worker.py b/tests/unit/test_catalog_rebuild_worker.py index 032789a..7dbd213 100644 --- a/tests/unit/test_catalog_rebuild_worker.py +++ b/tests/unit/test_catalog_rebuild_worker.py @@ -39,6 +39,8 @@ RebuildRunStatus, ) +pytestmark = pytest.mark.rebuild + client = TestClient(app) diff --git a/tests/unit/test_ci_ownership.py b/tests/unit/test_ci_ownership.py new file mode 100644 index 0000000..85cf402 --- /dev/null +++ b/tests/unit/test_ci_ownership.py @@ -0,0 +1,89 @@ +"""Tests for tools/ci_ownership.py (changed-path -> CI category mapper).""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "ci_ownership", ROOT / "tools" / "ci_ownership.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_ownership"] = module + spec.loader.exec_module(module) + return module + + +ci = _load() +RULES = ci.load_rules(ROOT / "tools" / "ci_ownership.toml") + + +def test_scheduler_change_selects_safety_and_unit() -> None: + sel = ci.select(["openblade/jobs/scheduler.py"], RULES) + assert {"unit", "safety"} <= sel.categories + assert sel.unowned == () + + +def test_docs_only_change_selects_only_docs() -> None: + sel = ci.select(["docs/test-plan.md", "README.md"], RULES) + assert sel.categories == frozenset({"docs-only"}) + assert sel.unowned == () + + +def test_unowned_openblade_path_is_reported() -> None: + sel = ci.select(["openblade/brand_new_pkg/mod.py", "tests/unit/test_x.py"], RULES) + assert sel.unowned == ("openblade/brand_new_pkg/mod.py",) + + +def test_unowned_tests_path_is_reported() -> None: + sel = ci.select(["tests/new_lane/test_y.py"], RULES) + assert sel.unowned == ("tests/new_lane/test_y.py",) + + +def test_unmatched_path_outside_enforced_trees_is_not_a_failure() -> None: + sel = ci.select(["scripts/whatever.sh"], RULES) + assert sel.unowned == () + assert sel.categories == frozenset() + + +def test_main_exits_1_on_unowned_path(capsys: pytest.CaptureFixture[str]) -> None: + assert ci.main(["--paths", "openblade/brand_new_pkg/mod.py"]) == 1 + captured = capsys.readouterr() + assert "UNOWNED" in captured.out + assert "FAIL" in captured.err + + +def test_main_exits_0_and_prints_reasoning(capsys: pytest.CaptureFixture[str]) -> None: + assert ci.main(["--paths", "openblade/domain/policies.py"]) == 0 + out = capsys.readouterr().out + assert "openblade/domain/* -> unit,safety" in out + assert "selected categories:" in out + + +def test_invalid_category_in_table_is_rejected(tmp_path: Path) -> None: + table = tmp_path / "t.toml" + table.write_text('[[rule]]\nglob = "x/*"\ncategories = ["bogus"]\n', encoding="utf-8") + with pytest.raises(ValueError, match="bogus"): + ci.load_rules(table) + + +def test_every_tracked_file_is_owned() -> None: + files = subprocess.run( + ["git", "ls-files", "openblade", "tests"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.split() + assert files + assert ci.select(files, RULES).unowned == () diff --git a/tests/unit/test_deploy_release.py b/tests/unit/test_deploy_release.py new file mode 100644 index 0000000..61737bc --- /dev/null +++ b/tests/unit/test_deploy_release.py @@ -0,0 +1,151 @@ +"""Release-pipeline deploy stages: read-only appliance postcheck and rollback. + +Safety invariants under test: +- the read-only postcheck never sends a non-GET request to the appliance; +- rollback redeploys application code only, has no option that could touch tape + state, refuses a mutable tag, and always prints the tape-state notice. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from openblade.deploy import ( + READ_ONLY_APPLIANCE_PROBES, + ROLLBACK_NOTICE, + InvalidDigestError, + ReadOnlyProbe, + Stage, + StageResult, + parse_image_digest, + run_read_only_appliance_checks, + run_rollback, +) + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) +import deploy as deploy_cli # noqa: E402 - scripts/ is not a package; path set just above + +DIGEST = "sha256:" + "a" * 64 + + +def _recording_probe(status: int = 200) -> tuple[list[tuple[str, str]], ReadOnlyProbe]: + sent: list[tuple[str, str]] = [] + + def probe(method: str, path: str) -> int: + sent.append((method, path)) + return status + + return sent, ReadOnlyProbe(probe) + + +def test_appliance_checks_are_all_gets() -> None: + assert {m for m, _ in READ_ONLY_APPLIANCE_PROBES} == {"GET"} + assert {p for _, p in READ_ONLY_APPLIANCE_PROBES} == {"/inventory/", "/jobs/recovery"} + + +def test_appliance_checks_pass_on_2xx() -> None: + sent, probe = _recording_probe(200) + result = run_read_only_appliance_checks(probe) + assert result.ok and result.stage is Stage.POSTCHECK + assert sent == list(READ_ONLY_APPLIANCE_PROBES) + + +@pytest.mark.parametrize("status", [404, 500, 503, 307]) +def test_appliance_checks_fail_on_non_2xx(status: int) -> None: + _, probe = _recording_probe(status) + result = run_read_only_appliance_checks(probe) + assert not result.ok + assert len(result.findings) == 2 + + +@pytest.mark.parametrize("method", ["POST", "PUT", "PATCH", "DELETE", "post"]) +def test_read_only_probe_never_forwards_mutating_requests(method: str) -> None: + sent, probe = _recording_probe() + assert probe(method, "/aml/users/login") == 405 + assert sent == [] + assert probe.skipped == [f"{method.upper()} /aml/users/login"] + + +def test_live_postcheck_read_only_mode_sends_only_gets(monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[str] = [] + + class _Resp: + status = 200 + + def __enter__(self) -> _Resp: + return self + + def __exit__(self, *_: object) -> None: + return None + + def fake_urlopen(req: object, timeout: float) -> _Resp: + sent.append(req.get_method()) # type: ignore[attr-defined] + return _Resp() + + monkeypatch.setattr(deploy_cli.urllib.request, "urlopen", fake_urlopen) + result = deploy_cli._postcheck_live("http://appliance", read_only_appliance_checks=True) + assert sent and set(sent) == {"GET"} + assert "not sent (non-GET)" in result.detail + + +@pytest.mark.parametrize( + ("ref", "expected"), + [ + (DIGEST, f"ghcr.io/amantux/openblade@{DIGEST}"), + (f"ghcr.io/other/img@{DIGEST}", f"ghcr.io/other/img@{DIGEST}"), + ], +) +def test_parse_image_digest_accepts_digests(ref: str, expected: str) -> None: + assert parse_image_digest(ref, "ghcr.io/amantux/openblade") == expected + + +@pytest.mark.parametrize("ref", ["latest", "ghcr.io/amantux/openblade:0.5.0", "sha256:abc", ""]) +def test_parse_image_digest_refuses_tags(ref: str) -> None: + with pytest.raises(InvalidDigestError): + parse_image_digest(ref, "ghcr.io/amantux/openblade") + + +def test_rollback_prints_notice_and_redeploys_image_only() -> None: + notices: list[str] = [] + redeployed: list[str] = [] + + def redeploy(ref: str) -> StageResult: + redeployed.append(ref) + return StageResult(Stage.DEPLOY, False, "exited 1") + + result = run_rollback(image_ref=f"img@{DIGEST}", redeploy=redeploy, notify=notices.append) + assert redeployed == [f"img@{DIGEST}"] + assert result.stage is Stage.ROLLBACK and not result.ok + assert notices == [ROLLBACK_NOTICE, ROLLBACK_NOTICE] + assert "tape state is NEVER rolled back" in ROLLBACK_NOTICE + assert "/jobs/recovery" in ROLLBACK_NOTICE + + +def test_rollback_cli_has_no_tape_state_option(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + deploy_cli.rollback_main(["--help"]) + help_text = capsys.readouterr().out.lower() + for forbidden in ("tape", "cartridge", "inventory", "unload", "move", "slot", "drive"): + assert f"--{forbidden}" not in help_text + assert set(help_text.split()) & {"--to", "--image", "--deploy-cmd", "--json"} == { + "--to", + "--image", + "--deploy-cmd", + "--json", + } + + +def test_rollback_cli_refuses_tag(capsys: pytest.CaptureFixture[str]) -> None: + rc = deploy_cli.main(["rollback", "--to", "latest", "--deploy-cmd", "true"]) + assert rc == 2 + assert "rollback refused" in capsys.readouterr().err + + +def test_rollback_cli_passes_digest_and_prints_notice(capsys: pytest.CaptureFixture[str]) -> None: + cmd = f"{sys.executable} -c \"import os,sys; sys.exit(0 if os.environ['OPENBLADE_IMAGE'].endswith('{DIGEST}') else 3)\"" + rc = deploy_cli.main(["rollback", "--to", DIGEST, "--deploy-cmd", cmd]) + assert rc == 0 + assert capsys.readouterr().err.count("Physical tape state is NEVER rolled back") == 2 diff --git a/tests/unit/test_drive_leases.py b/tests/unit/test_drive_leases.py new file mode 100644 index 0000000..1e4f732 --- /dev/null +++ b/tests/unit/test_drive_leases.py @@ -0,0 +1,152 @@ +"""Catalog-backed drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md).""" + +from __future__ import annotations + +import time +from datetime import timedelta +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.models import DriveLease +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler, InMemoryLeaseStore + + +def _two_repos(tmp_path: Path) -> tuple[CatalogRepository, CatalogRepository]: + # A file DB with two independent sessions models two processes (CLI + API) + # sharing one catalog; an in-memory StaticPool DB would share one connection. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()), CatalogRepository(get_session()) + + +def test_two_schedulers_on_one_catalog_cannot_both_hold_drive_0(tmp_path: Path) -> None: + repo_a, repo_b = _two_repos(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="job-a") + sched_b = DriveScheduler(1, store=CatalogLeaseStore(repo_b), job_id="job-b") + + held = sched_a.acquire_drives(["AAA001L9"], timeout=1.0) + assert held[0].drive_id == 0 + + with pytest.raises(DriveBusyError): + sched_b.acquire_drives(["BBB001L9"], timeout=0.6) + assert sched_b.status() == {0: "AAA001L9"} + + sched_a.release_drives(held) + got = sched_b.acquire_drives(["BBB001L9"], timeout=1.0) + assert got[0].drive_id == 0 + assert got[0].fencing_token > held[0].fencing_token + + +def test_fencing_tokens_strictly_increase_and_are_never_reissued(tmp_path: Path) -> None: + repo, _ = _two_repos(tmp_path) + sched = DriveScheduler(2, store=CatalogLeaseStore(repo), job_id="job-1") + + seen: list[int] = [] + for _ in range(3): + handles = sched.acquire_drives(["AAA001L9", "AAA002L9"], timeout=1.0) + seen.extend(h.fencing_token for h in handles) + sched.release_drives(handles) + + assert seen == sorted(seen) + assert len(set(seen)) == len(seen) + + +def test_verify_raises_stale_lease_after_release_or_expiry(tmp_path: Path) -> None: + repo, other = _two_repos(tmp_path) + sched = DriveScheduler(2, store=CatalogLeaseStore(repo), job_id="job-1") + (handle,) = sched.acquire_drives(["AAA001L9"], timeout=1.0) + sched.verify(handle) # live + sched.heartbeat([handle]) + sched.verify(handle) + + assert other.release_leases_for_job("job-1") == 1 + with pytest.raises(StaleLeaseError): + sched.verify(handle) + + expired = DriveScheduler( + 2, store=CatalogLeaseStore(repo), job_id="job-2", ttl=timedelta(seconds=-1) + ) + (dead,) = expired.acquire_drives(["AAA002L9"], timeout=1.0) + with pytest.raises(StaleLeaseError): + expired.verify(dead) + + +def test_held_leases_are_kept_alive_without_explicit_heartbeats() -> None: + # A single long tape write must not outlive its own lease: the scheduler + # heartbeats in the background for as long as a handle is held. + store = InMemoryLeaseStore() + scheduler = DriveScheduler(num_drives=1, store=store, ttl=timedelta(seconds=0.3)) + handles = scheduler.acquire_drives(["MCK00001"]) + time.sleep(0.8) # > 2 TTLs, with no heartbeat() call from the "job" + scheduler.verify(handles[0]) # still live + scheduler.release_drives(handles) + time.sleep(0.4) + assert store.live_leases() == [] # released, and the thread did not revive it + + +TTL = timedelta(seconds=60) + + +def test_excluded_drive_is_skipped_in_one_call(tmp_path: Path) -> None: + repo, _ = _two_repos(tmp_path) + leases = repo.acquire_drive_leases( + job_id="j", barcodes=["A00001L9"], num_drives=2, ttl=TTL, exclude=frozenset({0}) + ) + assert leases is not None and [lease.drive_id for lease in leases] == [1] + assert ( + CatalogLeaseStore(repo).acquire( + job_id="k", barcodes=["B"], num_drives=2, ttl=TTL, exclude=frozenset({0}) + ) + is None + ) + + +@pytest.mark.parametrize("drive_id", [0, -1]) +def test_acquire_drive_lease_at_free_then_held(tmp_path: Path, drive_id: int) -> None: + repo, other = _two_repos(tmp_path) + lease = repo.acquire_drive_lease_at(job_id="j", drive_id=drive_id, barcode="A", ttl=TTL) + assert lease is not None and lease.drive_id == drive_id + assert other.acquire_drive_lease_at(job_id="k", drive_id=drive_id, barcode="B", ttl=TTL) is None + repo.release_leases([lease.id]) + assert other.acquire_drive_lease_at(job_id="k", drive_id=drive_id, barcode="B", ttl=TTL) + + +def test_in_memory_acquire_drive_lease_at_is_exclusive() -> None: + store = InMemoryLeaseStore() + assert store.acquire_drive_lease_at(job_id="j", drive_id=-1, barcode="A", ttl=TTL) + assert store.acquire_drive_lease_at(job_id="k", drive_id=-1, barcode="B", ttl=TTL) is None + + +def test_two_sessions_racing_for_one_drive_id_exactly_one_wins(tmp_path: Path) -> None: + import threading + + repo_a, repo_b = _two_repos(tmp_path) + for _ in range(20): + barrier = threading.Barrier(2) + results: list[DriveLease | None] = [] + + def grab( + repo: CatalogRepository, + job: str, + barrier: threading.Barrier = barrier, + results: list[DriveLease | None] = results, + ) -> None: + barrier.wait() + results.append( + repo.acquire_drive_lease_at(job_id=job, drive_id=3, barcode=job, ttl=TTL) + ) + + threads = [ + threading.Thread(target=grab, args=(repo_a, "a")), + threading.Thread(target=grab, args=(repo_b, "b")), + ] + for t in threads: + t.start() + for t in threads: + t.join() + winners = [r for r in results if r is not None] + assert len(winners) == 1 + repo_a.release_leases([winners[0].id]) diff --git a/tests/unit/test_fake_mtx.py b/tests/unit/test_fake_mtx.py new file mode 100644 index 0000000..6300631 --- /dev/null +++ b/tests/unit/test_fake_mtx.py @@ -0,0 +1,97 @@ +"""The stateful mtx fake must speak the format ``parse_mtx_status`` consumes.""" + +from __future__ import annotations + +from openblade.hardware.mtx import parse_mtx_status +from tests.fakes.mtx import StatefulMtxRunner + +DEV = "/dev/sg3" + + +def _runner() -> StatefulMtxRunner: + return StatefulMtxRunner(DEV, barcodes={1: "VOL001L8", 2: "VOL002L8", 52: "VOL052L8"}) + + +def _status(runner: StatefulMtxRunner) -> str: + result = runner.run(["mtx", "-f", DEV, "status"]) + assert result.returncode == 0 + return result.stdout + + +def test_fake_mtx_status_round_trips_through_parser() -> None: + status = parse_mtx_status(_status(_runner())) + + assert (status.drive_count, status.slot_count) == (3, 50) + assert len(status.slots) == 50 + assert {s.slot_id: s.barcode for s in status.slots if s.occupied} == { + 1: "VOL001L8", + 2: "VOL002L8", + } + assert [(s.slot_id, s.barcode) for s in status.import_export_slots] == [ + (51, None), + (52, "VOL052L8"), + ] + + +def test_fake_mtx_load_reports_drive_barcode_and_source_slot() -> None: + runner = _runner() + + assert runner.run(["mtx", "-f", DEV, "load", "2", "1"]).returncode == 0 + + drive = next(d for d in parse_mtx_status(_status(runner)).drives if d.drive_id == 1) + assert (drive.loaded, drive.barcode, drive.source_slot) == (True, "VOL002L8", 2) + + +def test_fake_mtx_unload_and_transfer_move_cartridges() -> None: + runner = _runner() + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + assert runner.run(["mtx", "-f", DEV, "unload", "5", "0"]).returncode == 0 + assert runner.run(["mtx", "-f", DEV, "transfer", "5", "6"]).returncode == 0 + + full = {s.slot_id: s.barcode for s in parse_mtx_status(_status(runner)).slots if s.occupied} + assert full == {2: "VOL002L8", 6: "VOL001L8"} + + +def test_fake_mtx_load_from_empty_slot_fails_with_request_sense() -> None: + result = _runner().run(["mtx", "-f", DEV, "load", "3", "0"]) + + assert result.returncode == 1 + assert result.stderr.startswith("mtx: Request Sense:") + + +def test_fake_mtx_load_into_full_drive_fails_without_moving_tape() -> None: + runner = _runner() + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + result = runner.run(["mtx", "-f", DEV, "load", "2", "0"]) + + assert result.returncode == 1 + assert runner.slots[2] == "VOL002L8" + + +def test_fake_mtx_unload_empty_drive_or_into_full_slot_fails() -> None: + runner = _runner() + assert runner.run(["mtx", "-f", DEV, "unload", "5", "0"]).returncode == 1 + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + assert runner.run(["mtx", "-f", DEV, "unload", "2", "0"]).returncode == 1 + + +def test_fake_mtx_unknown_argv_returns_2() -> None: + runner = _runner() + + for argv in ( + ["mtx", "-f", DEV, "inquiry"], + ["mtx", "-f", "/dev/other", "status"], + ["mtx", "-f", DEV, "load", "x", "0"], + ["sg_inq", DEV], + ): + assert runner.run(argv).returncode == 2, argv + + +def test_fake_mtx_answers_sg_inq_only_for_declared_drives() -> None: + runner = StatefulMtxRunner(DEV, barcodes={}, drive_serials={"/dev/nst0": "SER0"}) + + assert "Unit serial number: SER0" in runner.run(["sg_inq", "/dev/nst0"]).stdout + assert runner.run(["sg_inq", "/dev/nst1"]).returncode == 2 diff --git a/tests/unit/test_fuse_cache_integrity.py b/tests/unit/test_fuse_cache_integrity.py new file mode 100644 index 0000000..311929d --- /dev/null +++ b/tests/unit/test_fuse_cache_integrity.py @@ -0,0 +1,212 @@ +"""FUSE hydration cache integrity: open-time verify, persisted LRU index, chunked pread.""" + +from __future__ import annotations + +import hashlib +import os +import threading +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.fuse import cache as cache_mod +from openblade.fuse.cache import SAMPLE_BYTES, CacheError, CacheIntegrityError, HydrationCache +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import HydrationFailedError, Hydrator +from openblade.fuse.mount import CatalogFuseOperations + +BIG = bytes(range(256)) * 1024 # 256 KiB, larger than one sample block +BIG_SUM = hashlib.sha256(BIG).hexdigest() + + +def _rewrite_keep_stat(path: Path, mutate: bytes) -> None: + """Replace file contents but restore mtime_ns, so only the sample hash can notice.""" + st = path.stat() + path.write_bytes(mutate) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns)) + + +def _flip_sampled_byte(cache: HydrationCache, checksum: str) -> None: + path = cache.cache_key(checksum) + data = bytearray(path.read_bytes()) + offset = cache_mod._sample_offset(checksum, len(data)) + data[offset + SAMPLE_BYTES // 2] ^= 0xFF + _rewrite_keep_stat(path, bytes(data)) + + +def test_verify_entry_passes_for_intact_entry(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + assert cache.verify_entry(BIG_SUM) is True + assert cache.is_cached(BIG_SUM) + + +def test_flipped_byte_in_sample_is_detected_and_evicted(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + _flip_sampled_byte(cache, BIG_SUM) + + with pytest.raises(CacheIntegrityError) as exc: + cache.verify_entry(BIG_SUM) + assert str(tmp_path) not in str(exc.value) # curated message, no paths + assert not cache.is_cached(BIG_SUM) + assert not cache.meta_key(BIG_SUM).exists() + assert cache.used_bytes() == 0 + + +def test_truncated_entry_is_detected_and_evicted(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + _rewrite_keep_stat(cache.cache_key(BIG_SUM), BIG[:1000]) + + with pytest.raises(CacheIntegrityError): + cache.verify_entry(BIG_SUM) + assert not cache.is_cached(BIG_SUM) + assert cache.used_bytes() == 0 + + +def test_missing_sidecar_is_treated_as_corrupt(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + cache.meta_key(BIG_SUM).unlink() + with pytest.raises(CacheIntegrityError): + cache.verify_entry(BIG_SUM) + assert not cache.is_cached(BIG_SUM) + + +# -- (c) mount open re-hydrates a corrupted entry instead of EIO --------------- + + +class _Engine: + def __init__(self, staging: Path) -> None: + self.staging = staging + self.batches: list[list[str]] = [] + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + self.batches.append(sorted(catalog_paths)) + out = {} + for path in catalog_paths: + dest = self.staging / path.strip("/") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(BIG) + out[path] = dest + return out + + def find_active_job(self, catalog_path: str) -> str | None: + return None + + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + raise HydrationFailedError(job_id) + + +def test_mount_open_rehydrates_corrupted_entry(tmp_path: Path) -> None: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + rec = repo.create_file_record("/big.bin", len(BIG), BIG_SUM, group.id) + inst = repo.create_file_instance(rec.id, "TAPE01", "/big.bin") + repo.mark_instance_archived(inst.id) + fs = CatalogFilesystem(repo, cache_dir=str(tmp_path / "cache")) + engine = _Engine(tmp_path / "staging") + hyd = Hydrator(repo, fs.cache, engine, batch_window_s=0.01) # type: ignore[arg-type] + ops = CatalogFuseOperations(fs, data_plane=hyd, hydrate_timeout=5.0) + + fh = ops.open("/big.bin", os.O_RDONLY) + ops.release("/big.bin", fh) + assert len(engine.batches) == 1 + + _flip_sampled_byte(fs.cache, BIG_SUM) + + fh = ops.open("/big.bin", os.O_RDONLY) # must not raise EIO + assert ops.read("/big.bin", len(BIG), 0, fh) == BIG + ops.release("/big.bin", fh) + assert len(engine.batches) == 2 # went back through the data plane + + +# -- (d) restart keeps the byte budget honest --------------------------------- + + +def _payloads() -> list[tuple[str, bytes]]: + out = [] + for i in range(3): + data = bytes([i]) * 1000 + out.append((hashlib.sha256(data).hexdigest(), data)) + return out + + +def _on_disk_bytes(cache: HydrationCache) -> int: + return sum(cache.cache_key(c).stat().st_size for c, _ in _payloads() if cache.is_cached(c)) + + +@pytest.mark.parametrize("drop_index", [False, True]) +def test_restart_counts_on_disk_bytes_and_enforces_budget(tmp_path: Path, drop_index: bool) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + if drop_index: + first.index_path.unlink() # scan must still find every file + + reopened = HydrationCache(str(tmp_path)) + assert reopened.used_bytes() == _on_disk_bytes(reopened) == 3000 + + budgeted = HydrationCache(str(tmp_path), max_bytes=1500) + assert budgeted.used_bytes() == _on_disk_bytes(budgeted) <= 1500 + assert budgeted.used_bytes() > 0 + + +def test_corrupt_index_is_rebuilt_from_scan(tmp_path: Path) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + first.index_path.write_text("{not json", encoding="utf-8") + assert HydrationCache(str(tmp_path)).used_bytes() == 3000 + + +def test_index_drops_entries_whose_file_is_gone(tmp_path: Path) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + first.cache_key(_payloads()[0][0]).unlink() + assert HydrationCache(str(tmp_path)).used_bytes() == 2000 + + +# -- (e) read_range survives short preads ------------------------------------- + + +def test_read_range_loops_over_short_preads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + real_pread = os.pread + calls = threading.local() + calls.n = 0 + + def short_pread(fd: int, length: int, offset: int) -> bytes: + calls.n += 1 + return real_pread(fd, min(length, 7), offset) + + monkeypatch.setattr(cache_mod.os, "pread", short_pread) + assert cache.read_range(BIG_SUM, 3, 100) == BIG[3:103] + assert calls.n > 1 + # Past EOF: exactly the bytes available. + assert cache.read_range(BIG_SUM, len(BIG) - 10, 100) == BIG[-10:] + + +def test_non_hex_checksum_large_entry_stores_and_verifies(tmp_path: Path) -> None: + """Regression S5: a non-hex checksum on a >64 KiB file must not raise ValueError.""" + cache = HydrationCache(str(tmp_path)) + data = b"x" * (200 * 1024) + path = cache.store("preset", data) + assert path.read_bytes() == data + assert cache.verify_entry("preset") is True + + +@pytest.mark.parametrize("checksum", ["../x", "", ".", "..", "a/b", "a\\b", "a\0b"]) +def test_unsafe_checksum_path_component_is_rejected(tmp_path: Path, checksum: str) -> None: + cache = HydrationCache(str(tmp_path)) + with pytest.raises(CacheError): + cache.store(checksum, b"") + assert not (tmp_path.parent / "x").exists() diff --git a/tests/unit/test_fuse_cli_wiring.py b/tests/unit/test_fuse_cli_wiring.py new file mode 100644 index 0000000..7cbee13 --- /dev/null +++ b/tests/unit/test_fuse_cli_wiring.py @@ -0,0 +1,106 @@ +"""`openblade fuse mount --hydrate` builds the production data plane.""" + +from __future__ import annotations + +import threading +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from typer.testing import CliRunner + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.cli import fuse_and_health +from openblade.config import load_config +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import Hydrator + + +def _context(tmp_path: Path, **config_overrides: Any) -> SimpleNamespace: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + config = replace( + load_config(), + cache_dir=str(tmp_path / "cache"), + restore_dir=str(tmp_path / "restore"), + **config_overrides, + ) + return SimpleNamespace( + config=config, + catalog=CatalogRepository(get_session()), + library=object(), + ltfs=object(), + queue=object(), + ) + + +def test_build_data_plane_reads_timeouts_from_settings(tmp_path: Path) -> None: + ctx = _context(tmp_path, fuse_hydrate_timeout_s=7.5, fuse_batch_window_ms=40) + + plane = fuse_and_health.build_data_plane( + ctx, CatalogFilesystem(ctx.catalog, ctx.config.cache_dir) + ) + + assert isinstance(plane, Hydrator) + assert plane.batch_window_s == pytest.approx(0.04) + assert plane.engine.resume_timeout_s == 7.5 # type: ignore[attr-defined] + + +def test_worker_restore_never_uses_the_callers_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + group = ctx.catalog.create_volume_group("g") + ctx.catalog.create_file_record("/a.bin", 3, "0" * 64, group.id) + plane = fuse_and_health.build_data_plane( + ctx, CatalogFilesystem(ctx.catalog, ctx.config.cache_dir) + ) + seen: list[Any] = [] + + class StubService: + def __init__(self, library: Any, ltfs: Any, catalog: Any, queue: Any) -> None: + seen.append(catalog) + + def enqueue(self, path: str, dest: Path) -> Any: + return SimpleNamespace(id="j1", state="completed") + + def enqueue_batch(self, requests: list[Any]) -> Any: + return SimpleNamespace(id="j1", state="completed"), SimpleNamespace(items=[]) + + monkeypatch.setattr(fuse_and_health, "RestoreService", StubService) + worker = threading.Thread(target=plane.engine.restore_batch, args=("T1", ["/a.bin"])) + worker.start() + worker.join(5) + + assert len(seen) == 1 + assert seen[0].session is not ctx.catalog.session + + +def test_mount_with_hydrate_passes_data_plane( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + calls: dict[str, Any] = {} + monkeypatch.setattr("openblade.cli.main._get_context", lambda: ctx) + monkeypatch.setattr(fuse_and_health, "mount_catalog", lambda fs, mp, **kw: calls.update(kw)) + + result = CliRunner().invoke(fuse_and_health.fuse_app, [str(tmp_path), "--hydrate"]) + + assert result.exit_code == 0, result.output + assert isinstance(calls["data_plane"], Hydrator) + + +def test_mount_without_hydrate_has_no_data_plane( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + calls: dict[str, Any] = {} + monkeypatch.setattr("openblade.cli.main._get_context", lambda: ctx) + monkeypatch.setattr(fuse_and_health, "mount_catalog", lambda fs, mp, **kw: calls.update(kw)) + + result = CliRunner().invoke(fuse_and_health.fuse_app, [str(tmp_path)]) + + assert result.exit_code == 0, result.output + assert calls["data_plane"] is None diff --git a/tests/unit/test_health_service.py b/tests/unit/test_health_service.py index 3d98d87..dad1731 100644 --- a/tests/unit/test_health_service.py +++ b/tests/unit/test_health_service.py @@ -72,6 +72,7 @@ def test_check_health_all_ok(tmp_path) -> None: result = service.check_health() assert result.status is HealthStatus.OK + # Thresholds unset -> no scratch component, so the set stays exact. assert {component.name for component in result.components} == {"database", "library", "ltfs"} diff --git a/tests/unit/test_job_error_sanitization.py b/tests/unit/test_job_error_sanitization.py index 7b9a572..34702a4 100644 --- a/tests/unit/test_job_error_sanitization.py +++ b/tests/unit/test_job_error_sanitization.py @@ -37,10 +37,15 @@ class DemoError(OpenBladeError): ) -def test_run_job_records_sanitized_error() -> None: +def test_run_job_records_sanitized_error(tmp_path) -> None: + from openblade.catalog.db import get_session, init_db + from openblade.catalog.repository import CatalogRepository from openblade.jobs.queue import JobQueue + from openblade.jobs.scheduler import CatalogLeaseStore - queue = JobQueue() + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + catalog = CatalogRepository(get_session()) + queue = JobQueue(catalog, CatalogLeaseStore(catalog)) job = queue.create_job("archive", {}) def boom(): diff --git a/tests/unit/test_job_queue_persistent.py b/tests/unit/test_job_queue_persistent.py new file mode 100644 index 0000000..fc3a354 --- /dev/null +++ b/tests/unit/test_job_queue_persistent.py @@ -0,0 +1,170 @@ +"""JobQueue state is shared by every process using the same catalog file.""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from datetime import timedelta +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import ChangerBusyError, DriveOccupiedError, DriveUnreconciledError +from openblade.domain.models import JobState, JobType +from openblade.jobs.queue import JobQueue +from openblade.jobs.reconcile import PHYSICAL_STATE_UNKNOWN +from openblade.jobs.scheduler import CatalogLeaseStore + + +@pytest.fixture +def queues(tmp_path: Path) -> tuple[JobQueue, JobQueue]: + # Two sessions on one file DB model two processes sharing one catalog. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + return JobQueue(repo_a, CatalogLeaseStore(repo_a)), JobQueue(repo_b, CatalogLeaseStore(repo_b)) + + +def test_job_created_by_one_process_is_visible_to_another( + queues: tuple[JobQueue, JobQueue], +) -> None: + a, b = queues + job = a.create_job(JobType.ARCHIVE, {"barcode": "TAPE01L8"}) + seen = b.get_job(job.id) + assert seen.job_type is JobType.ARCHIVE + assert seen.state is JobState.PENDING + assert seen.metadata == {"barcode": "TAPE01L8"} + + +def test_drive_claim_is_exclusive_across_processes(queues: tuple[JobQueue, JobQueue]) -> None: + a, b = queues + job_a = a.create_job(JobType.ARCHIVE, {}) + job_b = b.create_job(JobType.ARCHIVE, {}) + a.claim_drive(0, job_a.id) + with pytest.raises(DriveOccupiedError): + b.claim_drive(0, job_b.id) + b.claim_drive(1, job_b.id) + with pytest.raises(DriveOccupiedError): + a.claim_drive(1, job_a.id) + a.release_drive(0, job_a.id) + b.claim_drive(0, job_b.id) + + +def test_changer_claim_is_exclusive_across_processes(queues: tuple[JobQueue, JobQueue]) -> None: + a, b = queues + job_a = a.create_job(JobType.ARCHIVE, {}) + job_b = b.create_job(JobType.ARCHIVE, {}) + a.claim_changer(job_a.id) + with pytest.raises(ChangerBusyError): + b.claim_changer(job_b.id) + # The changer pseudo-drive does not occupy a real drive. + b.claim_drive(0, job_b.id) + a.release_changer(job_a.id) + b.claim_changer(job_b.id) + + +def test_run_job_transitions_are_visible_from_another_process( + queues: tuple[JobQueue, JobQueue], +) -> None: + a, b = queues + job = a.create_job(JobType.ARCHIVE, {}) + running = threading.Event() + proceed = threading.Event() + + def func() -> str: + running.set() + assert proceed.wait(timeout=5) + return "done" + + results: list[str] = [] + worker = threading.Thread(target=lambda: results.append(a.run_job(job, func)[1])) + worker.start() + assert running.wait(timeout=5) + assert b.get_job(job.id).state is JobState.RUNNING + proceed.set() + worker.join(timeout=5) + assert results == ["done"] + assert b.get_job(job.id).state is JobState.COMPLETED + + +@pytest.mark.parametrize("target", ["drive", "changer"]) +def test_racing_claims_past_the_owner_precheck_exactly_one_wins( + queues: tuple[JobQueue, JobQueue], monkeypatch: pytest.MonkeyPatch, target: str +) -> None: + # Both claimers pass the "nobody owns it" pre-check before either acquires, + # so exclusivity rests solely on the store's targeted acquisition. + barrier = threading.Barrier(2) + for queue in queues: + original = queue._owner + calls = {"n": 0} + + def gated( + slot: int, + original: Callable[[int], str | None] = original, + calls: dict[str, int] = calls, + ) -> str | None: + calls["n"] += 1 + result = original(slot) + if calls["n"] == 1: + barrier.wait(timeout=5) + return result + + monkeypatch.setattr(queue, "_owner", gated) + + outcomes: list[str] = [] + + def claim(queue: JobQueue) -> None: + job = queue.create_job(JobType.ARCHIVE, {}) + try: + if target == "drive": + queue.claim_drive(2, job.id) + else: + queue.claim_changer(job.id) + outcomes.append("won") + except (DriveOccupiedError, ChangerBusyError): + outcomes.append("busy") + + threads = [threading.Thread(target=claim, args=(q,)) for q in queues] + for t in threads: + t.start() + for t in threads: + t.join() + assert sorted(outcomes) == ["busy", "won"] + + +def test_claim_drive_refuses_unreconciled_drive(tmp_path: Path) -> None: + """Regression S3: the public claim API honours the reconciliation barrier.""" + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo = CatalogRepository(get_session()) + queue = JobQueue(repo, CatalogLeaseStore(repo)) + repo.journal( + "crashed-job", PHYSICAL_STATE_UNKNOWN, {"op": "unload", "barcode": "T1L8", "drive": 2} + ) + job = queue.create_job(JobType.ARCHIVE, {}) + with pytest.raises(DriveUnreconciledError): + queue.claim_drive(2, job.id) + assert all(lease.drive_id != 2 for lease in CatalogLeaseStore(repo).live_leases()) + queue.claim_drive(1, job.id) # other drives are unaffected + + +def test_run_job_heartbeats_claims_past_the_ttl(tmp_path: Path) -> None: + """Regression S3: a holder running longer than the TTL keeps its drive claim.""" + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + queue = JobQueue(repo_a, CatalogLeaseStore(repo_a), ttl=timedelta(seconds=2)) + observer = CatalogLeaseStore(repo_b) + job = queue.create_job(JobType.ARCHIVE, {}) + queue.claim_drive(0, job.id) + seen: list[bool] = [] + + def work() -> None: + for _ in range(10): + time.sleep(0.5) + repo_b.session.expire_all() + seen.append(any(lease.job_id == job.id for lease in observer.live_leases())) + + queue.run_job(job, work) + assert len(seen) == 10 + assert all(seen), seen diff --git a/tests/unit/test_ltfs_manifest.py b/tests/unit/test_ltfs_manifest.py index aaac9a4..57016d8 100644 --- a/tests/unit/test_ltfs_manifest.py +++ b/tests/unit/test_ltfs_manifest.py @@ -2,6 +2,17 @@ import json +import pytest + +from openblade.domain.protection import ( + Placement, + Protection, + ProtectionPolicy, + Striping, + StripingMode, +) +from openblade.nas.archive_lifecycle import finalize_tape_generation +from openblade.nas.catalog_shard import CatalogShard, CatalogShardDatasetEntry, CatalogShardWriter from openblade.nas.ltfs_manifest import ( ChecksumEntry, ManifestFileEntry, @@ -10,6 +21,8 @@ TapeJson, TapeMetadataWriter, TapeSetManifest, + UnsupportedManifestSchemaError, + parse_manifest, ) from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend @@ -281,3 +294,122 @@ def test_metadata_path_rejects_traversal_attempt() -> None: with pytest.raises(ValueError): writer._metadata_path("../etc/passwd") + + +class RecordingMetadataBackend(FakeMetadataBackend): + def __init__(self) -> None: + super().__init__() + self.write_order: list[str] = [] + + def write_bytes(self, path: str, content: bytes) -> None: + self.write_order.append(path.rsplit("/", 1)[-1]) + super().write_bytes(path, content) + + +def _generation_inputs(barcode: str = "OB0001L8") -> tuple[ManifestJson, CatalogShard, TapeJson]: + manifest = ManifestJson(barcode=barcode, openblade_tape_id="tape-1") + shard = CatalogShard( + barcode=barcode, + openblade_tape_id="tape-1", + datasets=[ + CatalogShardDatasetEntry( + dataset_id="ds-1", shard_set=[barcode, "OB0002L8"], tape_set=["OB0003L8"] + ) + ], + ) + tape = TapeJson( + openblade_tape_id="tape-1", + barcode=barcode, + created_at="2026-01-01T00:00:00Z", + last_openblade_write_at="2026-01-01T00:00:00Z", + current_generation="gen-1", + ) + return manifest, shard, tape + + +def test_finalize_tape_generation_writes_commit_marker_last() -> None: + backend = RecordingMetadataBackend() + writer = TapeMetadataWriter(backend) + manifest, shard, tape = _generation_inputs() + + finalize_tape_generation( + writer, + CatalogShardWriter(writer), + "OB0001L8", + generation_id="gen-2", + manifest=manifest, + shard=shard, + tape_json=tape, + ) + + order = [name for name in backend.write_order if not name.startswith(".")] + assert order == [ + "manifest.json", + "manifest.sha256", + "catalog-shard.json", + "catalog-shard.sha256", + "tape.json", + "commit-gen-2.json", + ] + + +def test_finalize_tape_generation_chains_generations_and_records_siblings() -> None: + _, writer = _writer() + manifest, shard, tape = _generation_inputs() + + marker = finalize_tape_generation( + writer, + CatalogShardWriter(writer), + "OB0001L8", + generation_id="gen-2", + manifest=manifest, + shard=shard, + tape_json=tape, + ) + + stored_tape = writer.read_tape_json("OB0001L8") + stored_manifest = writer.read_manifest("OB0001L8") + assert stored_tape is not None and stored_manifest is not None + assert (stored_tape.current_generation, stored_tape.previous_generation) == ("gen-2", "gen-1") + assert stored_manifest.sibling_tapes == ["OB0002L8", "OB0003L8"] + assert writer.read_commit_marker("OB0001L8", "gen-2") == marker + assert marker.manifest_sha256 == writer.compute_json_checksum( + stored_manifest.model_dump(by_alias=True) + ) + + +def test_v1_manifest_parses_with_v2_defaults() -> None: + manifest = parse_manifest( + {"schema": "openblade.manifest.v1", "barcode": "OB0001L8", "openblade_tape_id": "t"} + ) + assert manifest.sibling_tapes == [] + assert manifest.protection is None + assert manifest.generation_id == "" + + +def test_newer_manifest_schema_is_rejected_with_typed_error() -> None: + with pytest.raises(UnsupportedManifestSchemaError): + parse_manifest( + {"schema": "openblade.manifest.v3", "barcode": "OB0001L8", "openblade_tape_id": "t"} + ) + + +def test_manifest_round_trip_preserves_protection_policy() -> None: + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(StripingMode.STRIPE), + protection=Protection.replication(2), + ) + _, writer = _writer() + manifest = ManifestJson( + barcode="OB0001L8", + openblade_tape_id="t", + placement=policy.placement.value, + protection=policy.to_dict(), + ) + + writer.write_manifest("OB0001L8", manifest) + stored = writer.read_manifest("OB0001L8") + + assert stored is not None and stored.protection is not None + assert ProtectionPolicy.from_dict(stored.protection) == policy diff --git a/tests/unit/test_manifest_validator.py b/tests/unit/test_manifest_validator.py index 1b8bca2..3ed93f6 100644 --- a/tests/unit/test_manifest_validator.py +++ b/tests/unit/test_manifest_validator.py @@ -158,7 +158,7 @@ def test_validate_manifest_sets_schema_version_from_content() -> None: result = validator.validate_manifest(manifest.barcode) - assert result.schema_version == "openblade.manifest.v1" + assert result.schema_version == "openblade.manifest.v2" def test_validate_manifest_sets_file_count_from_manifest() -> None: diff --git a/tests/unit/test_nas_domain_models.py b/tests/unit/test_nas_domain_models.py new file mode 100644 index 0000000..c4a1a61 --- /dev/null +++ b/tests/unit/test_nas_domain_models.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from pydantic import ValidationError + +from openblade.nas.capacity import ( + AggregationPolicy, + ExportPrepError, + ExportSet, + ExportSetState, + FileSpan, + ReplicationState, + ReservationError, + ReservationLedger, + ScratchHealth, + ScratchThresholds, + SpanSegment, + TapeFragmentation, + Vault, + VolumeGroupReplication, + evaluate_scratch, + group_for_write, + mark_export_ready, + merge_volume_groups, + plan_drives, + planning_available, + require_export_ready, + required_tapes, + span_state, +) +from openblade.nas.types import NasFileState + +NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +# --- scratch thresholds +def test_scratch_thresholds_evaluate_ok_warn_critical() -> None: + t = ScratchThresholds(scratch_min=2, scratch_warn=5) + assert evaluate_scratch(1, t) is ScratchHealth.CRITICAL + assert evaluate_scratch(2, t) is ScratchHealth.WARN + assert evaluate_scratch(5, t) is ScratchHealth.OK + + +def test_scratch_thresholds_warn_below_min_rejected() -> None: + with pytest.raises(ValidationError): + ScratchThresholds(scratch_min=5, scratch_warn=2) + + +# --- fragmentation +def test_fragmentation_below_threshold_needs_consolidation() -> None: + assert TapeFragmentation(barcode="T1", live_bytes=40, written_bytes=100).needs_consolidation() + assert not TapeFragmentation( + barcode="T1", live_bytes=60, written_bytes=100 + ).needs_consolidation() + assert not TapeFragmentation(barcode="T1", live_bytes=0, written_bytes=0).needs_consolidation() + + +def test_fragmentation_live_above_written_rejected() -> None: + with pytest.raises(ValidationError): + TapeFragmentation(barcode="T1", live_bytes=2, written_bytes=1) + + +# --- reservations +def test_reservation_counts_against_free_capacity() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 60, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + assert ledger.available("p1", 100, NOW) == 40 + assert ledger.available("p2", 100, NOW) == 100 + + +def test_reservation_over_reserve_fails_typed() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 60, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + with pytest.raises(ReservationError): + ledger.reserve("p1", 41, "job-b", timedelta(minutes=5), free_bytes=100, now=NOW) + + +def test_reservation_expired_is_ignored() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 100, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + later = NOW + timedelta(minutes=6) + assert ledger.available("p1", 100, later) == 100 + ledger.reserve("p1", 100, "job-b", timedelta(minutes=5), free_bytes=100, now=later) + + +def test_reservation_release_frees_capacity() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 100, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + assert ledger.release("p1", "job-a") == 1 + assert ledger.available("p1", 100, NOW) == 100 + + +# --- VG replication +def _vg(vg_id: str, req: int, present: int, *barcodes: str) -> VolumeGroupReplication: + return VolumeGroupReplication( + vg_id=vg_id, barcodes=barcodes, replicas_required=req, replicas_present=present + ) + + +def test_replication_state_assignment() -> None: + assert _vg("a", 2, 2).state is ReplicationState.IN_SYNC + assert _vg("a", 2, 1).state is ReplicationState.LAGGING + assert _vg("a", 2, 0).state is ReplicationState.BROKEN + + +def test_replication_merge_re_evaluates_state() -> None: + merged = merge_volume_groups(_vg("a", 1, 1, "T1"), _vg("b", 2, 2, "T2"), "ab") + assert merged.replicas_required == 2 + assert merged.state is ReplicationState.LAGGING + assert merged.barcodes == ("T1", "T2") + + +def test_replication_lagging_vg_cannot_export_prep() -> None: + with pytest.raises(ExportPrepError): + require_export_ready(_vg("a", 2, 1)) + + +def test_replication_broken_repairs_only_after_replica_write() -> None: + vg = _vg("a", 1, 0) + assert vg.state is ReplicationState.BROKEN + assert vg.replica_write_completed().state is ReplicationState.IN_SYNC + + +# --- spanning +def _span(path: str, *barcodes: str) -> FileSpan: + segs = tuple(SpanSegment(barcode=b, offset=0, length=10) for b in barcodes) + return FileSpan(path=path, segments=segs) + + +def test_span_required_tapes_deduplicated_in_order() -> None: + assert required_tapes([_span("/a", "T1", "T2"), _span("/b", "T2", "T3")]) == ["T1", "T2", "T3"] + + +def test_span_on_missing_tape_is_missing_tape() -> None: + span = _span("/a", "T1", "T2") + assert span_state(span, {"T1", "T2"}) is NasFileState.OFFLINE_ON_TAPE + assert span_state(span, {"T1"}) is NasFileState.MISSING_TAPE + + +def test_span_requires_segments() -> None: + with pytest.raises(ValidationError): + FileSpan(path="/a", segments=()) + + +# --- export sets + vault +def test_export_set_ready_requires_all_members_in_sync() -> None: + es = ExportSet(id="e1", barcodes=("T1", "T2")) + groups = [_vg("a", 1, 1, "T1"), _vg("b", 2, 1, "T2")] + with pytest.raises(ExportPrepError): + mark_export_ready(es, groups) + groups[1] = groups[1].replica_write_completed() + assert mark_export_ready(es, groups).state is ExportSetState.READY + + +def test_export_set_unknown_member_refused() -> None: + with pytest.raises(ExportPrepError): + mark_export_ready(ExportSet(id="e1", barcodes=("T9",)), [_vg("a", 1, 1, "T1")]) + + +def test_vaulted_barcode_is_offline_for_planning() -> None: + vault = Vault(id="v1", barcodes=frozenset({"T2"}), location="offsite") + avail = planning_available(["T1", "T2"], [vault]) + assert avail == {"T1"} + assert span_state(_span("/a", "T2"), avail) is NasFileState.MISSING_TAPE + + +# --- drive planning +def test_plan_drives_one_drive_three_tapes() -> None: + plan = plan_drives([_span("/a", "T1", "T2"), _span("/b", "T3")], available_drives=1) + assert plan.required_drives == 1 + assert plan.estimated_swaps == 2 + + +def test_plan_drives_enough_drives_no_swaps() -> None: + plan = plan_drives([_span("/a", "T1", "T2", "T3")], available_drives=4) + assert (plan.required_drives, plan.estimated_swaps) == (3, 0) + + +# --- aggregation +def test_aggregation_groups_small_files_into_containers() -> None: + policy = AggregationPolicy(aggregate_below_bytes=10, max_container_bytes=12) + files = [("/s1", 5), ("/big", 50), ("/s2", 5), ("/s3", 5)] + assert group_for_write(files, policy) == [ + [("/big", 50)], + [("/s1", 5), ("/s2", 5)], + [("/s3", 5)], + ] diff --git a/tests/unit/test_nas_media_compat.py b/tests/unit/test_nas_media_compat.py new file mode 100644 index 0000000..51dabb8 --- /dev/null +++ b/tests/unit/test_nas_media_compat.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import pytest + +from openblade.nas.media import ( + Cartridge, + ForeignMediaError, + IncompatibleMediaError, + MediaGeneration, + can_read, + can_write, + is_foreign, + require_writable, +) +from openblade.nas.planner import ArchivePlanner +from openblade.nas.types import ArchivePlanRequest, PolicyType + +G = MediaGeneration + + +@pytest.mark.parametrize( + ("drive", "readable", "writable"), + [ + (G.LTO6, {G.LTO5, G.LTO6}, {G.LTO5, G.LTO6}), + (G.LTO7, {G.LTO5, G.LTO6, G.LTO7}, {G.LTO6, G.LTO7}), + # LTO-8 and LTO-9 break the N-2 read rule. + (G.LTO8, {G.LTO7, G.LTO8}, {G.LTO7, G.LTO8}), + (G.LTO9, {G.LTO8, G.LTO9}, {G.LTO8, G.LTO9}), + ], +) +def test_compat_matrix(drive: G, readable: set[G], writable: set[G]) -> None: + assert {m for m in G if can_read(drive, m)} == readable + assert {m for m in G if can_write(drive, m)} == writable + + +def test_lto8_cannot_read_lto6_and_lto9_cannot_read_lto7() -> None: + assert not can_read(G.LTO8, G.LTO6) + assert not can_read(G.LTO9, G.LTO7) + + +def test_unknown_media_never_compatible() -> None: + assert not any(can_read(d, G.UNKNOWN) or can_write(d, G.UNKNOWN) for d in G) + + +def test_foreign_detection() -> None: + assert is_foreign(Cartridge(barcode="F00001L8")) + assert not is_foreign(Cartridge(barcode="F00001L8", adopted=True)) + assert not is_foreign(Cartridge(barcode="A00001L8", has_tape_json=True)) + + +def test_require_writable_typed_errors() -> None: + with pytest.raises(ForeignMediaError): + require_writable(Cartridge(barcode="F1", generation=G.LTO8), G.LTO8) + with pytest.raises(IncompatibleMediaError): + require_writable(Cartridge(barcode="A1", generation=G.LTO6, has_tape_json=True), G.LTO8) + + +def _request(cart: Cartridge, drive: G | None = None) -> ArchivePlanRequest: + return ArchivePlanRequest( + files=["/a"], + file_sizes={"/a": 10}, + available_tapes=[cart.barcode], + cartridges={cart.barcode: cart}, + drive_generation=drive, + ) + + +def test_archive_planner_refuses_foreign_media_without_adopt() -> None: + foreign = Cartridge(barcode="F00001L8", generation=G.LTO8) + with pytest.raises(ForeignMediaError): + ArchivePlanner().plan(_request(foreign)) + adopted = foreign.model_copy(update={"adopted": True}) + ArchivePlanner().plan(_request(adopted)) + + +def test_archive_planner_refuses_unwritable_pairing() -> None: + old = Cartridge(barcode="A00001L6", generation=G.LTO6, has_tape_json=True) + with pytest.raises(IncompatibleMediaError): + ArchivePlanner().plan(_request(old, G.LTO8)) + ArchivePlanner().plan(_request(old, G.LTO7)) + + +def test_archive_plan_reserved_bytes_block_plan_that_would_otherwise_fit() -> None: + def req(reserved: int) -> ArchivePlanRequest: + return ArchivePlanRequest( + policy_type=PolicyType.CRITICAL_SEQUENTIAL, + files=["/a"], + file_sizes={"/a": 80}, + available_tapes=["T1"], + tape_capacities={"T1": 100}, + reserved_bytes={"T1": reserved}, + ) + + assert ArchivePlanner().plan(req(0)).is_safe_to_enqueue + assert not ArchivePlanner().plan(req(30)).is_safe_to_enqueue diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py new file mode 100644 index 0000000..f28fa60 --- /dev/null +++ b/tests/unit/test_nas_persistence.py @@ -0,0 +1,259 @@ +"""Persistence of the NAS domain models (pools, reservations, spans, export sets, vaults).""" + +from __future__ import annotations + +import threading +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, inspect, text + +from openblade.catalog.db import _migrate_schema, get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.nas.capacity import ReservationError, ReservationLedger +from openblade.nas.restore_planner import RestorePlanner +from openblade.nas.types import RestorePlanRequest +from tests.unit.test_nas_restore_planner import make_nas_service, seed_dataset, seed_file, seed_pool + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()) + + +_OLD_NAS_POOLS_DDL = """ +CREATE TABLE nas_pools ( + id VARCHAR PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + description TEXT, + volume_group_ids TEXT, + default_policy_id VARCHAR, + default_ingest_mode VARCHAR, + mount_path VARCHAR, + virtual_mount_enabled BOOLEAN, + hydration_behavior VARCHAR, + cache_target_id VARCHAR, + restore_target_path VARCHAR, + access_mode VARCHAR, + created_at TEXT, + updated_at TEXT +) +""" + +_NEW_TABLES = { + "nas_reservations", + "nas_export_sets", + "nas_vaults", + "nas_vg_replication", + "nas_file_spans", +} + + +def test_migration_upgrades_pre_change_db_idempotently(tmp_path: Path) -> None: + engine = create_engine(f"sqlite:///{tmp_path / 'old.db'}") + with engine.begin() as connection: + connection.execute(text(_OLD_NAS_POOLS_DDL)) + connection.execute(text("INSERT INTO nas_pools (id, name) VALUES ('p1', 'legacy')")) + + _migrate_schema(engine) + _migrate_schema(engine) # second run must be a no-op, not an error + + inspector = inspect(engine) + pool_columns = {column["name"] for column in inspector.get_columns("nas_pools")} + assert {"replication_factor", "protection_json"} <= pool_columns + assert set(inspector.get_table_names()) >= _NEW_TABLES + with engine.connect() as connection: + row = connection.execute( + text("SELECT replication_factor, protection_json FROM nas_pools WHERE id = 'p1'") + ).one() + assert row == (1, None) + + +def test_pool_round_trips_replication_and_protection(repo: CatalogRepository) -> None: + saved = repo.upsert_nas_pool({"id": "p1", "name": "pool-a", "replication_factor": 2}) + assert saved["replication_factor"] == 2 + assert saved["protection"] is None + loaded = repo.get_nas_pool("p1") + assert loaded is not None + assert loaded["replication_factor"] == 2 + # Round trip: the dict form must be accepted back by upsert unchanged. + again = repo.upsert_nas_pool(dict(loaded)) + assert again["replication_factor"] == 2 + + +def test_pool_protection_is_serialised_like_dataset_protection(repo: CatalogRepository) -> None: + import json + + from openblade.domain.protection import ProtectionPolicy + + policy = ProtectionPolicy() + saved = repo.upsert_nas_pool({"id": "p2", "name": "pool-b", "protection": policy}) + assert saved["protection"] == json.loads(json.dumps(policy.to_dict(), sort_keys=True)) + again = repo.upsert_nas_pool(dict(saved)) + assert again["protection"] == saved["protection"] + + +_NOW = datetime(2026, 1, 1, 12, 0, tzinfo=UTC) + + +def test_reservations_round_trip_and_expiry_filter(repo: CatalogRepository) -> None: + live = repo.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 100, "owner": "job-1", "expires_at": _NOW + timedelta(hours=1)} + ) + repo.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 50, "owner": "job-2", "expires_at": _NOW} + ) + repo.upsert_nas_reservation( + {"pool_id": "p2", "bytes": 7, "owner": "job-3", "expires_at": _NOW + timedelta(days=1)} + ) + assert live["expires_at"] == _NOW + timedelta(hours=1) + assert len(repo.list_nas_reservations("p1")) == 2 + active = repo.reservations_for_pool("p1", now=_NOW) + assert [item["owner"] for item in active] == ["job-1"] + assert repo.delete_nas_reservation(str(live["id"])) is True + assert repo.reservations_for_pool("p1", now=_NOW) == [] + assert repo.delete_nas_reservation(str(live["id"])) is False + + +def test_export_sets_vaults_and_vg_replication_round_trip(repo: CatalogRepository) -> None: + repo.upsert_nas_export_set({"id": "e1", "barcodes": ["A00001L9"], "state": "pending"}) + repo.upsert_nas_export_set({"id": "e1", "barcodes": ["A00001L9", "A00002L9"], "state": "out"}) + assert repo.list_nas_export_sets() == [ + {"id": "e1", "barcodes": ["A00001L9", "A00002L9"], "state": "out"} + ] + repo.upsert_nas_vault({"id": "v1", "barcodes": ["A00003L9"], "location": "offsite"}) + assert repo.list_nas_vaults() == [{"id": "v1", "barcodes": ["A00003L9"], "location": "offsite"}] + repo.upsert_nas_vg_replication( + {"vg_id": "vg1", "barcodes": ["A"], "replicas_required": 2, "replicas_present": 1} + ) + assert repo.list_nas_vg_replication() == [ + {"vg_id": "vg1", "barcodes": ["A"], "replicas_required": 2, "replicas_present": 1} + ] + assert repo.delete_nas_export_set("e1") + assert repo.delete_nas_vault("v1") + assert repo.delete_nas_vg_replication("vg1") + assert repo.list_nas_export_sets() == repo.list_nas_vaults() == [] + assert repo.list_nas_vg_replication() == [] + + +def test_file_spans_and_scratch_thresholds_round_trip(repo: CatalogRepository) -> None: + segments = [ + {"barcode": "A00001L9", "offset": 0, "length": 10}, + {"barcode": "A00002L9", "offset": 0, "length": 5}, + ] + span = repo.upsert_nas_file_span({"path": "/big.bin", "segments": segments}) + repo.upsert_nas_file_span({"path": "/other.bin", "segments": segments[:1]}) + assert repo.list_nas_file_spans("/big.bin") == [ + {"id": span["id"], "path": "/big.bin", "segments": segments} + ] + assert len(repo.list_nas_file_spans()) == 2 + assert repo.delete_nas_file_span(str(span["id"])) + assert repo.get_scratch_thresholds() is None + repo.set_scratch_thresholds(2, 5) + assert repo.get_scratch_thresholds() == {"scratch_min": 2, "scratch_warn": 5} + + +def test_ledger_reservation_survives_new_ledger_instance(repo: CatalogRepository) -> None: + ReservationLedger(repo).reserve( + "p1", 400, "job-1", timedelta(hours=1), free_bytes=1_000, now=_NOW + ) + fresh = ReservationLedger(repo) + assert fresh.reserved("p1", _NOW) == 400 + assert fresh.available("p1", 1_000, _NOW) == 600 + with pytest.raises(ReservationError): + fresh.reserve("p1", 700, "job-2", timedelta(hours=1), free_bytes=1_000, now=_NOW) + assert fresh.reserved("p1", _NOW + timedelta(hours=2)) == 0 # expired + assert fresh.release("p1", "job-1") == 1 + assert ReservationLedger(repo).reserved("p1", _NOW) == 0 + + +def test_in_memory_ledger_does_not_touch_catalog(repo: CatalogRepository) -> None: + ReservationLedger().reserve("p1", 10, "job", timedelta(hours=1), free_bytes=100, now=_NOW) + assert repo.list_nas_reservations() == [] + + +def test_spanning_file_requires_every_segment_tape(tmp_path: Path) -> None: + service = make_nas_service(tmp_path) + pool = seed_pool(service) + dataset = seed_dataset(service, pool_id=pool.id) + seed_file( + service, + dataset_id=dataset.id, + pool_id=pool.id, + relative_path="video/big.mkv", + tape_barcode="VOL001L9", + ) + service.repository.upsert_nas_file_span( + { + "path": "video/big.mkv", + "segments": [ + {"barcode": "VOL001L9", "offset": 0, "length": 6}, + {"barcode": "VOL002L9", "offset": 0, "length": 4}, + ], + } + ) + + plan = RestorePlanner(service).plan(RestorePlanRequest(pool_id=pool.id)) + + assert set(plan.required_tapes) == {"VOL001L9", "VOL002L9"} + assert set(plan.tape_load_order) == {"VOL001L9", "VOL002L9"} + assert plan.batches_by_tape["VOL002L9"] == ["video/big.mkv"] + + +def test_api_exposes_pool_protection_and_persisted_lists(tmp_path: Path) -> None: + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.bootstrap import create_context, get_context, reset_context + from openblade.config import OpenBladeConfig + + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'api.db'}"))) + catalog = get_context().catalog + catalog.upsert_nas_pool({"id": "p1", "name": "pool-a", "replication_factor": 2}) + catalog.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 5, "owner": "job", "expires_at": _NOW} + ) + catalog.upsert_nas_export_set({"id": "e1", "barcodes": ["A"], "state": "out"}) + catalog.upsert_nas_vault({"id": "v1", "barcodes": ["B"], "location": "offsite"}) + client = TestClient(app) + + pool = client.get("/nas/pools/p1").json() + assert pool["replication_factor"] == 2 + assert "protection" in pool + reservations = client.get("/nas/reservations", params={"pool_id": "p1"}).json() + assert [item["owner"] for item in reservations] == ["job"] + assert client.get("/nas/reservations", params={"pool_id": "other"}).json() == [] + assert client.get("/nas/export-sets").json()[0]["id"] == "e1" + assert client.get("/nas/vaults").json()[0]["location"] == "offsite" + + +def test_concurrent_reservations_across_processes_never_overcommit(tmp_path: Path) -> None: + """Regression S6: 20 racers over two catalog sessions each want 60% -> exactly one wins.""" + init_db(f"sqlite:///{tmp_path / 'race.db'}") + ledgers = [ReservationLedger(CatalogRepository(get_session())) for _ in range(2)] + start = threading.Barrier(20) + won: list[str] = [] + errors: list[BaseException] = [] + + def racer(i: int) -> None: + start.wait() + try: + ledgers[i % 2].reserve( + "p1", 600, f"owner-{i}", timedelta(minutes=5), free_bytes=1000, now=_NOW + ) + won.append(f"owner-{i}") + except ReservationError: + pass + except BaseException as exc: # noqa: BLE001 - surfaced by the assertion below + errors.append(exc) + + threads = [threading.Thread(target=racer, args=(i,)) for i in range(20)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert errors == [] + assert len(won) == 1 + assert ledgers[0].reserved("p1", _NOW) == 600 diff --git a/tests/unit/test_nas_tape_states.py b/tests/unit/test_nas_tape_states.py new file mode 100644 index 0000000..73a24a5 --- /dev/null +++ b/tests/unit/test_nas_tape_states.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import itertools + +import pytest + +from openblade.nas.state_machine import ( + TRANSITIONS, + InvalidTransitionError, + TapeEvent, + TapeState, + file_state_for_tape, + transition, +) +from openblade.nas.types import NasFileState + +S = TapeState +E = TapeEvent + + +@pytest.mark.parametrize(("pair", "target"), list(TRANSITIONS.items())) +def test_every_allowed_transition(pair: tuple[TapeState, TapeEvent], target: TapeState) -> None: + assert transition(*pair) is target + + +_FORBIDDEN = [p for p in itertools.product(S, E) if p not in TRANSITIONS] + + +@pytest.mark.parametrize(("state", "event"), _FORBIDDEN) +def test_every_forbidden_transition_raises_typed(state: TapeState, event: TapeEvent) -> None: + with pytest.raises(InvalidTransitionError) as exc: + transition(state, event) + assert exc.value.state is state and exc.value.event is event + + +def test_exported_cannot_come_online_without_import() -> None: + with pytest.raises(InvalidTransitionError): + transition(S.EXPORTED, E.BRING_ONLINE) + assert transition(transition(S.EXPORTED, E.IMPORT), E.BRING_ONLINE) is S.ONLINE + + +def test_exporting_cannot_jump_online_except_by_cancel() -> None: + for event in E: + if event in (E.EXPORT_DONE, E.EXPORT_CANCEL): + continue + with pytest.raises(InvalidTransitionError): + transition(S.EXPORTING, event) + + +def test_sequestered_leaves_only_via_operator() -> None: + exits = {e for (s, e) in TRANSITIONS if s is S.SEQUESTERED} + assert exits == {E.OPERATOR_REPAIR, E.OPERATOR_RELEASE} + assert {TRANSITIONS[(S.SEQUESTERED, e)] for e in exits} == {S.REPAIR_REQUIRED, S.ONLINE} + + +def test_degraded_never_returns_online_without_repair() -> None: + assert (S.DEGRADED, E.BRING_ONLINE) not in TRANSITIONS + assert (S.DEGRADED, E.REPAIR_DONE) not in TRANSITIONS + + +def test_file_state_projection_reuses_nas_file_state() -> None: + assert file_state_for_tape(S.EXPORTED) is NasFileState.EXPORTED + assert file_state_for_tape(S.SEQUESTERED) is NasFileState.MISSING_TAPE + assert file_state_for_tape(S.REPAIR_REQUIRED) is NasFileState.MISSING_TAPE + assert file_state_for_tape(S.HYDRATING) is NasFileState.HYDRATING + assert file_state_for_tape(S.OFFLINE) is NasFileState.OFFLINE_ON_TAPE diff --git a/tests/unit/test_nonsharded_leases.py b/tests/unit/test_nonsharded_leases.py new file mode 100644 index 0000000..1aa41bf --- /dev/null +++ b/tests/unit/test_nonsharded_leases.py @@ -0,0 +1,201 @@ +"""Non-sharded run_archive_job / run_restore_job take drives through DriveScheduler leases.""" + +from __future__ import annotations + +import functools +from pathlib import Path, PurePosixPath +from typing import Any + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_job +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend +from openblade.simulator.scenarios import one_drive_twenty_slots_five_cartridges + + +def _stack( + tmp_path: Path, +) -> tuple[CatalogRepository, CatalogRepository, MockLibraryBackend, MockLTFSBackend]: + # Two sessions on one file DB model two processes sharing one lease store. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + library, ltfs = one_drive_twenty_slots_five_cartridges() + barcode = str(library.inventory().slots[0].barcode) + library.load(1, 0) + ltfs.format(barcode, FormatConfirmation(barcode, SafetyToken.generate("format", barcode))) + library.unload(0, 1) + group = repo_a.create_volume_group("photos") + repo_a.add_barcode_to_volume_group(group.id, barcode) + return repo_a, repo_b, library, ltfs + + +def _archive( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + source: Path, + scheduler: DriveScheduler, +) -> None: + job = repo.create_job("archive", {"source_path": str(source), "volume_group": "photos"}) + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name="photos"), + library, + ltfs, + repo, + job.id, + scheduler=scheduler, + ) + + +def test_archive_holding_drive_0_blocks_restore_until_released(tmp_path: Path) -> None: + repo_a, repo_b, library, ltfs = _stack(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="archive") + first = tmp_path / "a.txt" + first.write_text("first") + _archive(repo_a, library, ltfs, first, sched_a) + + sched_b = DriveScheduler(1, store=CatalogLeaseStore(repo_b), job_id="restore") + sched_b.acquire_drives = functools.partial(sched_b.acquire_drives, timeout=0.2) # type: ignore[method-assign] + restore_job = repo_b.create_job("restore", {}) + blocked: list[BaseException] = [] + real_write = ltfs.write_file + + def write_then_contend(handle: Any, src: Path, dest: PurePosixPath) -> Any: + result = real_write(handle, src, dest) + with pytest.raises(DriveBusyError) as caught: + run_restore_job( + RestoreRequest("/photos/a.txt", tmp_path / "blocked.txt"), + library, + ltfs, + repo_b, + restore_job.id, + scheduler=sched_b, + ) + blocked.append(caught.value) + return result + + ltfs.write_file = write_then_contend # type: ignore[method-assign] + second = tmp_path / "b.txt" + second.write_text("second") + _archive(repo_a, library, ltfs, second, sched_a) + ltfs.write_file = real_write # type: ignore[method-assign] + assert len(blocked) == 1 + assert not (tmp_path / "blocked.txt").exists() + + # Released after the archive finished: the same restore now gets drive 0. + result = run_restore_job( + RestoreRequest("/photos/a.txt", tmp_path / "restored.txt"), + library, + ltfs, + repo_b, + repo_b.create_job("restore", {}).id, + scheduler=sched_b, + ) + assert result.checksum_verified + assert (tmp_path / "restored.txt").read_text() == "first" + events = [entry.event for entry in repo_a.job_journal(repo_a.list_jobs()[1].id)] + assert events.count("lease_acquired") == 1 + assert events.count("lease_released") == 1 + + +def test_stale_lease_aborts_archive_without_unmount_or_unload(tmp_path: Path) -> None: + repo_a, repo_b, library, ltfs = _stack(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="archive") + other_store = CatalogLeaseStore(repo_b) + calls: list[str] = [] + real_write, real_unmount, real_unload = ltfs.write_file, ltfs.unmount, library.unload + + def write_then_lose_lease(handle: Any, src: Path, dest: PurePosixPath) -> Any: + result = real_write(handle, src, dest) + # Another process reclaims the lease (e.g. TTL expiry) mid-job. + other_store.release([lease.id for lease in other_store.live_leases()]) + return result + + def record_unmount(handle: Any) -> None: + calls.append("unmount") + real_unmount(handle) + + def record_unload(*args: Any, **kwargs: Any) -> Any: + calls.append("unload") + return real_unload(*args, **kwargs) + + ltfs.write_file = write_then_lose_lease # type: ignore[method-assign] + ltfs.unmount = record_unmount # type: ignore[method-assign] + library.unload = record_unload # type: ignore[method-assign] + source = tmp_path / "a.txt" + source.write_text("payload") + job = repo_a.create_job("archive", {"source_path": str(source), "volume_group": "photos"}) + with pytest.raises(StaleLeaseError): + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name="photos"), + library, + ltfs, + repo_a, + job.id, + scheduler=sched_a, + ) + assert calls == [] + refreshed = repo_a.get_job(job.id) + assert refreshed is not None and refreshed.state == "failed" + + +def test_archive_service_holds_a_catalog_lease_while_writing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The service path (assistant, API) must exclude other jobs, not just run_*_job.""" + from openblade.jobs.archive import ArchiveService + from openblade.jobs.queue import JobQueue + + repo_a, repo_b, library, ltfs = _stack(tmp_path) + source = tmp_path / "src" + source.mkdir() + (source / "a.bin").write_bytes(b"x" * 64) + service = ArchiveService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))) + seen: list[list[str]] = [] + real_write = ltfs.write_file + + def write(*args: Any, **kwargs: Any) -> Any: + seen.append([lease.job_id for lease in repo_b.live_leases()]) + return real_write(*args, **kwargs) + + monkeypatch.setattr(ltfs, "write_file", write) + job = service.enqueue("photos", source) + assert job.state == "completed" + assert seen and all(owners == [job.id] for owners in seen) # seen from "another process" + assert repo_b.live_leases() == [] + + +def test_restore_service_holds_a_catalog_lease_while_reading( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from openblade.jobs.archive import ArchiveService + from openblade.jobs.queue import JobQueue + from openblade.jobs.restore import RestoreService + + repo_a, repo_b, library, ltfs = _stack(tmp_path) + source = tmp_path / "src" + source.mkdir() + (source / "a.bin").write_bytes(b"y" * 64) + ArchiveService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))).enqueue( + "photos", source + ) + service = RestoreService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))) + seen: list[list[str]] = [] + real_read = ltfs.read_file + + def read(*args: Any, **kwargs: Any) -> Any: + seen.append([lease.job_id for lease in repo_b.live_leases()]) + return real_read(*args, **kwargs) + + monkeypatch.setattr(ltfs, "read_file", read) + job = service.enqueue("/photos/a.bin", tmp_path / "out.bin") + assert job.state == "completed" + assert seen and all(owners == [job.id] for owners in seen) + assert repo_b.live_leases() == [] diff --git a/tests/unit/test_policies_guards.py b/tests/unit/test_policies_guards.py new file mode 100644 index 0000000..c6f4a55 --- /dev/null +++ b/tests/unit/test_policies_guards.py @@ -0,0 +1,68 @@ +"""Pin the safety-token / format-confirmation / real-hardware guards in policies.py. + +Written to kill surviving mutmut mutants: every assertion here fails when the +corresponding guard, field or curated operator message is mutated. +""" + +from __future__ import annotations + +import pytest + +from openblade.domain import policies +from openblade.domain.errors import ( + BarcodeMismatchError, + RealHardwareDisabledError, + SafetyViolationError, +) +from openblade.domain.policies import FormatConfirmation, RealHardwareGuard, SafetyToken + + +def test_generate_binds_operation_barcode_and_a_32_byte_token() -> None: + tok = SafetyToken.generate("format", "ABC123L6") + assert tok.operation == "format" + assert tok.target_barcode == "ABC123L6" + assert isinstance(tok.token, str) + # secrets.token_urlsafe(32) -> 32 bytes base64url without padding = 43 chars. + assert len(tok.token) == 43 + assert tok.token != SafetyToken.generate("format", "ABC123L6").token + + +def test_token_is_invalid_at_exact_expiry_instant(monkeypatch: pytest.MonkeyPatch) -> None: + tok = SafetyToken.generate("format", "ABC123L6", ttl_seconds=10) + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at) + assert tok.is_valid() is False + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at - 0.001) + assert tok.is_valid() is True + + +def test_expired_token_raises_curated_message(monkeypatch: pytest.MonkeyPatch) -> None: + tok = SafetyToken.generate("format", "ABC123L6", ttl_seconds=10) + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at + 1) + with pytest.raises(SafetyViolationError) as exc: + tok.validate() + assert str(exc.value) == "Safety token has expired" + + +def test_barcode_mismatch_names_both_barcodes() -> None: + conf = FormatConfirmation("ABC123L6", SafetyToken.generate("format", "ABC123L6")) + conf.validate("ABC123L6") + with pytest.raises(BarcodeMismatchError) as exc: + conf.validate("XYZ999L6") + assert str(exc.value) == "Expected barcode 'ABC123L6' but got 'XYZ999L6'" + + +@pytest.mark.parametrize(("backend", "enabled"), [("sim", True), ("real", False)]) +def test_real_hardware_disabled_message_names_both_settings(backend: str, enabled: bool) -> None: + with pytest.raises(RealHardwareDisabledError) as exc: + RealHardwareGuard(backend, enabled, "operator ok").validate() + assert str(exc.value) == ( + "Real hardware operations require OPENBLADE_BACKEND=real and " + "OPENBLADE_REAL_HARDWARE_ENABLED=true" + ) + + +def test_real_hardware_requires_operator_acknowledgment() -> None: + RealHardwareGuard("real", True, "operator ok").validate() + with pytest.raises(RealHardwareDisabledError) as exc: + RealHardwareGuard("real", True, "").validate() + assert str(exc.value) == "Operator acknowledgment is required" diff --git a/tests/unit/test_protection_axes.py b/tests/unit/test_protection_axes.py new file mode 100644 index 0000000..9b915b3 --- /dev/null +++ b/tests/unit/test_protection_axes.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from openblade.domain.protection import ( + DurabilityClass, + FailureDomain, + InvalidProtectionPolicyError, + Placement, + Protection, + ProtectionPolicy, + RestoreQuorum, + Striping, + StripingMode, + UnsupportedProtectionError, + protection_from_pool, +) + +_STRIPINGS = [ + (Placement.SINGLE_TAPE, Striping()), + (Placement.STRIPED, Striping(StripingMode.STRIPE)), + (Placement.STRIPED, Striping(StripingMode.BLOCK_STRIPE, block_size=1 << 20)), +] +_PROTECTIONS = [ + (Protection.none(), False), + (Protection.replication(1), False), + (Protection.replication(2), True), + (Protection.replication(3), True), + (Protection.erasure_code(4, 2), True), +] + + +@pytest.mark.parametrize("domain", list(FailureDomain)) +@pytest.mark.parametrize(("placement", "striping"), _STRIPINGS) +@pytest.mark.parametrize(("protection", "expected"), _PROTECTIONS) +def test_is_protected_for_every_combination( + domain: FailureDomain, + placement: Placement, + striping: Striping, + protection: Protection, + expected: bool, +) -> None: + policy = ProtectionPolicy( + placement=placement, striping=striping, protection=protection, failure_domain=domain + ) + assert policy.is_protected is expected + assert (policy.durability_class is DurabilityClass.PROTECTED) is expected + + +@pytest.mark.parametrize("mode", [StripingMode.STRIPE, StripingMode.BLOCK_STRIPE]) +@pytest.mark.parametrize("copies", [1]) +def test_striping_never_implies_protection(mode: StripingMode, copies: int) -> None: + block = 4096 if mode is StripingMode.BLOCK_STRIPE else None + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(mode, block_size=block), + protection=Protection.replication(copies), + ) + assert policy.is_protected is False + assert policy.durability_class is DurabilityClass.PERFORMANCE_ONLY + + +def test_quorum_cannot_exceed_total() -> None: + with pytest.raises(InvalidProtectionPolicyError): + RestoreQuorum(tapes_required=3, tapes_total=2) + assert RestoreQuorum(2, 2).tapes_required == 2 + + +def test_striping_must_match_placement() -> None: + with pytest.raises(InvalidProtectionPolicyError): + ProtectionPolicy(placement=Placement.SINGLE_TAPE, striping=Striping(StripingMode.STRIPE)) + with pytest.raises(InvalidProtectionPolicyError): + Striping(StripingMode.BLOCK_STRIPE) + + +def test_erasure_code_is_modelled_but_not_plannable() -> None: + policy = ProtectionPolicy(protection=Protection.erasure_code(4, 2)) + assert policy.protection.supported is False + with pytest.raises(UnsupportedProtectionError): + policy.require_plannable() + ProtectionPolicy(protection=Protection.replication(2)).require_plannable() + + +@pytest.mark.parametrize(("factor", "copies", "protected"), [(1, 1, False), (2, 2, True)]) +def test_protection_from_pool(factor: int, copies: int, protected: bool) -> None: + policy = protection_from_pool(SimpleNamespace(replication_factor=factor)) + assert policy.protection.copies == copies + assert policy.is_protected is protected + + +def test_protection_from_pool_prefers_explicit_policy() -> None: + from openblade.nas.types import NasPool + + explicit = ProtectionPolicy(protection=Protection.replication(3)) + pool = NasPool(name="p", replication_factor=1, protection=explicit) + assert protection_from_pool(pool) == explicit + # Unset field falls back to the legacy replication_factor. + assert protection_from_pool(NasPool(name="q", replication_factor=2)).protection.copies == 2 + + +def test_policy_dict_round_trip() -> None: + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(StripingMode.BLOCK_STRIPE, block_size=8192), + protection=Protection.replication(2), + failure_domain=FailureDomain.LIBRARY, + quorum=RestoreQuorum(2, 4), + ) + assert ProtectionPolicy.from_dict(policy.to_dict()) == policy diff --git a/tests/unit/test_protocol_gateway.py b/tests/unit/test_protocol_gateway.py index 8e2f382..24dcfcf 100644 --- a/tests/unit/test_protocol_gateway.py +++ b/tests/unit/test_protocol_gateway.py @@ -242,3 +242,32 @@ def test_sessions_api_includes_upload_audit( assert resp.status_code == 200 payload = resp.json() assert payload[0]["uploads"][0]["requested_path"] == "/openblade/inbox/upload.iso" + + +def test_on_open_evicts_corrupt_cache_entry_and_rehydrates( + gw: ProtocolGateway, tmp_path: Path +) -> None: + from openblade.catalog.db import get_session, init_db + from openblade.catalog.repository import CatalogRepository + from openblade.fuse.cache import HydrationCache + from openblade.fuse.hydration import Hydrator + from tests.unit.test_fuse_cache_integrity import BIG, BIG_SUM, _Engine, _flip_sampled_byte + + init_db(f"sqlite:///{tmp_path / 'gw-cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + rec = repo.create_file_record("/big.bin", len(BIG), BIG_SUM, group.id) + inst = repo.create_file_instance(rec.id, "TAPE01", "/big.bin") + repo.mark_instance_archived(inst.id) + cache = HydrationCache(str(tmp_path / "cache")) + cache.store(BIG_SUM, BIG) + cache.verify_entry(BIG_SUM) # a verified, intact entry + engine = _Engine(tmp_path / "staging") + gw.attach_hydrator(Hydrator(repo, cache, engine, batch_window_s=0.01)) # type: ignore[arg-type] # _Engine is a duck-typed test double + + _flip_sampled_byte(cache, BIG_SUM) + handle = gw.on_open("/big.bin", timeout=5.0) + + assert len(engine.batches) == 1 # a hydration was requested, not the bad bytes served + assert cache.cache_key(BIG_SUM).read_bytes() == BIG + gw.on_close(handle.token) diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py new file mode 100644 index 0000000..14bb0c3 --- /dev/null +++ b/tests/unit/test_reconcile.py @@ -0,0 +1,197 @@ +"""Drives with unknown physical state (failed unmount/unload) must be reconciled before reuse.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from openblade.api import routes_jobs +from openblade.bootstrap import AppContext, create_context, get_context +from openblade.config import OpenBladeConfig +from openblade.domain.errors import ( + DriveUnreconciledError, + JournalWriteError, + TapeFullError, + safe_job_error, +) +from openblade.domain.models import MountState +from openblade.jobs.reconcile import ( + DRIVE_RECONCILED, + PHYSICAL_STATE_UNKNOWN, + drives_pending_reconciliation, + reconcile_drive, +) +from openblade.jobs.restore import _load_if_needed +from openblade.jobs.scheduler import ( + CatalogLeaseStore, + DriveHandle, + DriveScheduler, + InMemoryLeaseStore, +) +from openblade.simulator.library import MockLibraryBackend + + +def _context(tmp_path: Path) -> AppContext: + return create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + + +def _mark_unknown(context: AppContext, drive: int | None, barcode: str = "MCK00001") -> None: + context.catalog.journal( + "job-1", PHYSICAL_STATE_UNKNOWN, {"op": "unload", "barcode": barcode, "drive": drive} + ) + + +def test_pending_set_tracks_unknown_and_reconciled(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, 1) + _mark_unknown(context, None) # no drive named: ignored + pending = drives_pending_reconciliation(context.catalog) + assert set(pending) == {1} + assert pending[1].barcode == "MCK00001" + assert pending[1].op == "unload" + assert pending[1].job_id == "job-1" + context.catalog.journal("recovery", DRIVE_RECONCILED, {"drive": 1}) + assert drives_pending_reconciliation(context.catalog) == {} + _mark_unknown(context, 1) # a later failure makes it pending again + assert set(drives_pending_reconciliation(context.catalog)) == {1} + + +def test_scheduler_skips_unreconciled_drive_and_refuses_when_none_left() -> None: + store = InMemoryLeaseStore() + store.unreconciled = {0: "MCK00001"} + scheduler = DriveScheduler(2, store=store) + [handle] = scheduler.acquire_drives(["MCK00002"], timeout=0.1) + assert handle.drive_id == 1 + scheduler.release_drives([handle]) + + store.unreconciled = {0: "MCK00001", 1: None} + with pytest.raises(DriveUnreconciledError, match="drive 0 .*MCK00001"): + scheduler.acquire_drives(["MCK00002"], timeout=0.1) + + +def test_load_if_needed_refuses_unreconciled_drive(tmp_path: Path) -> None: + context = _context(tmp_path) + barcode = next(str(slot.barcode) for slot in context.library.inventory().slots if slot.barcode) + _mark_unknown(context, 0, barcode="MCK99999") + handle = DriveHandle(drive_id=0, barcode=barcode) + with pytest.raises(DriveUnreconciledError, match="Drive 0"): + _load_if_needed(context.catalog, context.library, context.ltfs, handle, "job-2") + assert context.library.find_drive_by_barcode(barcode) is None # nothing was loaded + + +def test_reconcile_drive_refuses_while_mounted(tmp_path: Path) -> None: + context = _context(tmp_path) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + _mark_unknown(context, 1) + library._drives[1].mount_state = MountState.MOUNTED_RW + with pytest.raises(DriveUnreconciledError, match="still mounted_rw"): + reconcile_drive(context.catalog, library, 1) + assert set(drives_pending_reconciliation(context.catalog)) == {1} + + library._drives[1].mount_state = MountState.UNMOUNTED + done = reconcile_drive(context.catalog, library, 1) + assert done.drive_id == 1 + assert done.op == DRIVE_RECONCILED + assert drives_pending_reconciliation(context.catalog) == {} + + +def test_reconcile_api_round_trip(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, 1) + app = FastAPI() + app.include_router(routes_jobs.router, prefix="/jobs") + app.dependency_overrides[get_context] = lambda: context + client = TestClient(app) + + context.library._drives[1].mount_state = MountState.MOUNTED_RO # type: ignore[attr-defined] + conflict = client.post("/jobs/recovery/reconcile/1") + assert conflict.status_code == 409 + assert "Drive 1" in conflict.json()["detail"] + + context.library._drives[1].mount_state = MountState.UNMOUNTED # type: ignore[attr-defined] + ok = client.post("/jobs/recovery/reconcile/1") + assert ok.status_code == 200 + assert ok.json()["drive"] == 1 + assert drives_pending_reconciliation(context.catalog) == {} + + assert client.post("/jobs/recovery/reconcile/99").status_code == 404 + + +def test_pending_orders_by_id_not_wall_clock(tmp_path: Path) -> None: + from datetime import timedelta + + from openblade.jobs.reconcile import drives_pending_reconciliation + + catalog = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'c.db'}")).catalog + job = catalog.create_job("archive", {}) + catalog.journal(job.id, "drive_reconciled", {"drive": 1, "barcode": "MCK00001"}) + later = catalog.journal( + job.id, "physical_state_unknown", {"op": "unmount", "barcode": "MCK00001", "drive": 1} + ) + later.at = later.at - timedelta(hours=1) # writer clock stepped back + catalog.session.commit() + + assert 1 in drives_pending_reconciliation(catalog) + + +def test_driveless_event_still_blocks_a_load_of_that_barcode(tmp_path: Path) -> None: + from openblade.jobs.reconcile import barcodes_pending_reconciliation, ensure_drive_reconciled + + catalog = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'c.db'}")).catalog + job = catalog.create_job("archive", {}) + catalog.journal( + job.id, "physical_state_unknown", {"op": "unmount", "barcode": "MCK00002", "drive": None} + ) + assert barcodes_pending_reconciliation(catalog) == {"MCK00002"} + with pytest.raises(DriveUnreconciledError, match="MCK00002"): + ensure_drive_reconciled(catalog, 0, "MCK00002") + ensure_drive_reconciled(catalog, 0, "MCK00003") # other barcodes unaffected + catalog.journal(job.id, "drive_reconciled", {"drive": 0, "barcode": "MCK00002"}) + assert barcodes_pending_reconciliation(catalog) == set() + + +def test_scheduler_refuses_barcode_pending_reconciliation() -> None: + store = InMemoryLeaseStore() + store.unreconciled_barcode_set = {"MCK00001"} + scheduler = DriveScheduler(3, store=store) + with pytest.raises(DriveUnreconciledError, match="MCK00001"): + scheduler.acquire_drives(["MCK00002", "MCK00001"], timeout=0.1) + # Other barcodes are unaffected, and no lease leaked from the refusal. + handles = scheduler.acquire_drives(["MCK00002", "MCK00003", "MCK00004"], timeout=0.1) + assert len(handles) == 3 + scheduler.release_drives(handles) + + +def test_catalog_store_reports_driveless_pending_barcode(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, None, barcode="MCK00007") + scheduler = DriveScheduler(2, store=CatalogLeaseStore(context.catalog)) + with pytest.raises(DriveUnreconciledError, match="MCK00007"): + scheduler.acquire_drives(["MCK00007"], timeout=0.1) + + +def test_release_failure_in_finally_names_the_original_lane_error() -> None: + scheduler = DriveScheduler(2, store=InMemoryLeaseStore()) + [handle] = scheduler.acquire_drives(["MCK00001"], timeout=0.1) + handle.hold_unjournaled = True + with pytest.raises(JournalWriteError) as caught: + try: + raise TapeFullError("tape MCK00001 is full") + finally: + scheduler.release_drives([handle]) + recorded = safe_job_error(caught.value) # what the job queue stores + assert "not journaled" in recorded + assert "original failure: tape MCK00001 is full" in recorded + assert isinstance(caught.value.__context__, TapeFullError) + + +def test_release_failure_without_lane_error_keeps_plain_message() -> None: + scheduler = DriveScheduler(2, store=InMemoryLeaseStore()) + [handle] = scheduler.acquire_drives(["MCK00001"], timeout=0.1) + handle.hold_unjournaled = True + with pytest.raises(JournalWriteError) as caught: + scheduler.release_drives([handle]) + assert "original failure" not in str(caught.value) diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py new file mode 100644 index 0000000..a61c2ec --- /dev/null +++ b/tests/unit/test_recovery.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.catalog.repository import CatalogRepository +from openblade.config import OpenBladeConfig +from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL +from openblade.simulator.library import MockLibraryBackend + + +def _lease(catalog: CatalogRepository, job_id: str, barcode: str, *, ttl: timedelta) -> str: + leases = catalog.acquire_drive_leases(job_id=job_id, barcodes=[barcode], num_drives=2, ttl=ttl) + assert leases is not None + return leases[0].id + + +def test_recovery_fails_only_jobs_whose_lease_expired(tmp_path: Path) -> None: + # Two `running` jobs share the catalog. One stopped heartbeating (dead + # process); the other holds a live lease — it may be running in ANOTHER + # process right now, and recovery must not touch it. + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + dead = catalog.create_job("archive", {}) + catalog.update_job_state(dead.id, "running") + dead_lease = _lease(catalog, dead.id, "MCK00001", ttl=timedelta(seconds=-1)) + alive = catalog.create_job("archive", {}) + catalog.update_job_state(alive.id, "running") + alive_lease = _lease(catalog, alive.id, "MCK00002", ttl=timedelta(minutes=15)) + empty_library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, empty_library) + + assert report.interrupted_job_ids == [dead.id] + assert report.released_lease_ids == [dead_lease] + refreshed = catalog.get_job(dead.id) + assert refreshed is not None + assert refreshed.state == "failed_recoverable" + assert refreshed.error == INTERRUPTED_ERROR + assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ + ("MCK00001", None) + ] + untouched = catalog.get_job(alive.id) + assert untouched is not None + assert untouched.state == "running" + assert [lease.id for lease in catalog.live_leases()] == [alive_lease] + + +def test_recovery_releases_live_lease_of_a_finished_job(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + done = catalog.create_job("archive", {}) + catalog.update_job_state(done.id, "completed") + lease_id = _lease(catalog, done.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.interrupted_job_ids == [] + assert report.released_lease_ids == [lease_id] + assert catalog.live_leases() == [] + + +def test_restart_runs_recovery_and_exposes_it_read_only(tmp_path: Path) -> None: + # A job left `running` by a previous process is failed on the next start, + # and the report is visible on the native surface only. + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.bootstrap import reset_context + + db_url = f"sqlite:///{tmp_path / 'restart.db'}" + first = create_context(OpenBladeConfig(db_url=db_url)) + job = first.catalog.create_job("archive", {}) + first.catalog.update_job_state(job.id, "running") + _lease(first.catalog, job.id, "MCK00001", ttl=timedelta(seconds=-1)) + + second = create_context(OpenBladeConfig(db_url=db_url)) + reset_context(second) + assert second.recovery_report.interrupted_job_ids == [job.id] + + response = TestClient(app).get("/jobs/recovery") + assert response.status_code == 200 + body = response.json() + assert body["interrupted_job_ids"] == [job.id] + assert body["staged_instances"] == {job.id: []} + assert body["stale_pending_job_ids"] == [] + + strict = create_context(OpenBladeConfig(db_url=db_url, scalar_api_only=True)) + reset_context(strict) + assert TestClient(app).get("/jobs/recovery").status_code == 404 + + +def test_recovery_lists_staged_instances_of_an_interrupted_job(tmp_path: Path) -> None: + # A sharded archive died after staging shards but before the commit marker: + # the operator needs to know exactly which tape paths are uncommitted. + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + _lease(catalog, job.id, "AAA001L9", ttl=timedelta(seconds=-1)) + vg = catalog.create_volume_group("vg-staged") + record = catalog.create_file_record("/data/big.bin.shard0", 10, "0" * 64, vg.id, shard_index=0) + staged = catalog.create_staged_instance(job.id, record.id, "AAA001L9", "/big.bin.shard0") + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.interrupted_job_ids == [job.id] + [listed] = report.staged_instances[job.id] + assert (listed.instance_id, listed.barcode, listed.tape_path, listed.shard_index) == ( + staged.id, + "AAA001L9", + "/big.bin.shard0", + 0, + ) + assert listed.state == "staging" + assert "recovered" in [entry.event for entry in catalog.job_journal(job.id)] + + +def test_recovery_reports_stale_pending_jobs_without_touching_them(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + stale = catalog.create_job("archive", {}) + fresh = catalog.create_job("archive", {}) + old_but_leased = catalog.create_job("archive", {}) + long_ago = datetime.now(UTC).replace(tzinfo=None) - DEFAULT_LEASE_TTL - timedelta(minutes=1) + for job in (stale, old_but_leased): + job.created_at = long_ago + catalog.session.commit() + _lease(catalog, old_but_leased.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.stale_pending_job_ids == [stale.id] + for job_id in (stale.id, fresh.id): + refreshed = catalog.get_job(job_id) + assert refreshed is not None + assert refreshed.state == "pending" + + +def test_recovery_keeps_live_lease_of_a_failed_job(tmp_path: Path) -> None: + # Fail closed: a failed job only leaves a live lease behind when it could not + # journal physical_state_unknown; auto-releasing it would hand the drive out. + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + failed = catalog.create_job("archive", {}) + catalog.update_job_state(failed.id, "failed", error="physical state unknown") + lease_id = _lease(catalog, failed.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.released_lease_ids == [] + assert report.held_failed_lease_ids == [lease_id] + assert [lease.id for lease in catalog.live_leases()] == [lease_id] + + +def test_recovery_ignores_queue_claim_leases_but_reports_real_mismatches(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + expired = timedelta(seconds=-1) + # JobQueue's slot claims (jobs/queue.py): the changer at -1 and drive 1. + for drive_id, barcode in ((-1, "CHANGER:-1"), (1, "QUEUE-CLAIM:1")): + claim = catalog.acquire_drive_lease_at( + job_id=job.id, drive_id=drive_id, barcode=barcode, ttl=expired + ) + assert claim is not None + _lease(catalog, job.id, "MCK00001", ttl=expired) # a real tape lease + empty_library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, empty_library) + + assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ + ("MCK00001", None) + ] diff --git a/tests/unit/test_recovery_guards.py b/tests/unit/test_recovery_guards.py new file mode 100644 index 0000000..f16011c --- /dev/null +++ b/tests/unit/test_recovery_guards.py @@ -0,0 +1,89 @@ +"""Pin recovery guards that mutation testing showed were unasserted. + +Each test fails when the corresponding guard in openblade/jobs/recovery.py is +mutated (and->or on the owner check, the once-per-job dedupe, the journal +payload, the stale-age boundary and its timezone normalisation). +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.catalog.repository import CatalogRepository +from openblade.config import OpenBladeConfig +from openblade.jobs import recovery +from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart +from openblade.simulator.library import MockLibraryBackend + +EXPIRED = timedelta(seconds=-1) + + +def _catalog(tmp_path: Path) -> CatalogRepository: + return create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")).catalog + + +def _library() -> MockLibraryBackend: + return MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + +def test_recovery_expired_lease_of_pending_job_leaves_job_untouched(tmp_path: Path) -> None: + catalog = _catalog(tmp_path) + job = catalog.create_job("archive", {}) + leases = catalog.acquire_drive_leases( + job_id=job.id, barcodes=["MCK00001"], num_drives=2, ttl=EXPIRED + ) + assert leases is not None + + report = recover_after_restart(catalog, _library()) + + assert report.interrupted_job_ids == [] + refreshed = catalog.get_job(job.id) + assert refreshed is not None + assert refreshed.state == "pending" + assert [e.event for e in catalog.job_journal(job.id)].count("recovered") == 0 + + +def test_recovery_job_with_two_expired_leases_is_failed_once(tmp_path: Path) -> None: + catalog = _catalog(tmp_path) + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + leases = catalog.acquire_drive_leases( + job_id=job.id, barcodes=["MCK00001", "MCK00002"], num_drives=2, ttl=EXPIRED + ) + assert leases is not None + assert len(leases) == 2 + + report = recover_after_restart(catalog, _library()) + + assert report.interrupted_job_ids == [job.id] + recovered = [e for e in catalog.job_journal(job.id) if e.event == "recovered"] + assert len(recovered) == 1 + assert json.loads(recovered[0].detail_json) == { + "lease_id": leases[0].id, + "error": INTERRUPTED_ERROR, + } + + +def test_is_older_than_exact_ttl_boundary_is_not_stale() -> None: + now = datetime(2026, 10, 10, 12, 0, 0) + ttl = timedelta(minutes=15) + + assert recovery._is_older_than(now - ttl, ttl, now) is False + assert recovery._is_older_than(now - ttl - timedelta(microseconds=1), ttl, now) is True + + +def test_is_older_than_normalises_aware_timestamp_to_utc() -> None: + now = datetime(2026, 10, 10, 12, 0, 0) + ttl = timedelta(minutes=15) + plus_five = timezone(timedelta(hours=5)) + # 16:50+05:00 == 11:50 UTC: 10 min old -> fresh. Read naively it is in the future. + fresh = datetime(2026, 10, 10, 16, 50, tzinfo=plus_five) + # 16:40+05:00 == 11:40 UTC: 20 min old -> stale. Read naively (16:40) it is fresh. + stale = datetime(2026, 10, 10, 16, 40, tzinfo=plus_five) + + assert recovery._is_older_than(fresh, ttl, now) is False + assert recovery._is_older_than(stale, ttl, now) is True + assert recovery._is_older_than(datetime(2026, 10, 10, 11, 50, tzinfo=UTC), ttl, now) is False diff --git a/tests/unit/test_restore_batch.py b/tests/unit/test_restore_batch.py new file mode 100644 index 0000000..c963b59 --- /dev/null +++ b/tests/unit/test_restore_batch.py @@ -0,0 +1,185 @@ +"""run_restore_batch: one lease/load/mount/unmount/unload per source tape.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_batch +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend +from openblade.simulator.scenarios import one_drive_twenty_slots_five_cartridges + + +class _Recorder: + """Counts library load/unload calls without changing their behaviour.""" + + def __init__(self, library: MockLibraryBackend) -> None: + self.loads = 0 + self.unloads = 0 + real_load, real_unload = library.load, library.unload + + def load(*args: Any, **kwargs: Any) -> Any: + self.loads += 1 + return real_load(*args, **kwargs) + + def unload(*args: Any, **kwargs: Any) -> Any: + self.unloads += 1 + return real_unload(*args, **kwargs) + + library.load = load # type: ignore[method-assign] + library.unload = unload # type: ignore[method-assign] + + +def _stack( + tmp_path: Path, groups: list[str] +) -> tuple[CatalogRepository, MockLibraryBackend, MockLTFSBackend]: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo = CatalogRepository(get_session()) + library, ltfs = one_drive_twenty_slots_five_cartridges() + slots = library.inventory().slots + for index, name in enumerate(groups): + slot = slots[index] + barcode = str(slot.barcode) + library.load(slot.slot_id, 0) + ltfs.format(barcode, FormatConfirmation(barcode, SafetyToken.generate("format", barcode))) + library.unload(0, slot.slot_id) + group = repo.create_volume_group(name) + repo.add_barcode_to_volume_group(group.id, barcode) + return repo, library, ltfs + + +def _archive( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + source: Path, + group: str, +) -> None: + job = repo.create_job("archive", {"source_path": str(source), "volume_group": group}) + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name=group), + library, + ltfs, + repo, + job.id, + scheduler=DriveScheduler(1, store=CatalogLeaseStore(repo), job_id=job.id), + ) + + +def _write(tmp_path: Path, name: str) -> Path: + path = tmp_path / "src" / name + path.parent.mkdir(exist_ok=True) + path.write_text(f"payload of {name}") + return path + + +def _run( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + requests: list[RestoreRequest], +) -> tuple[str, Any]: + job = repo.create_job("restore", {}) + scheduler = DriveScheduler(1, store=CatalogLeaseStore(repo), job_id=job.id) + result = run_restore_batch(requests, library, ltfs, repo, job.id, scheduler=scheduler) + return job.id, result + + +def test_restore_batch_two_files_one_tape_loads_and_unloads_once(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + for name in ("a.txt", "b.txt"): + _archive(repo, library, ltfs, _write(tmp_path, name), "photos") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + job_id, result = _run( + repo, + library, + ltfs, + [RestoreRequest(f"/photos/{n}", out / n) for n in ("a.txt", "b.txt")], + ) + + assert (recorder.loads, recorder.unloads) == (1, 1) + assert result.ok and [i.ok for i in result.items] == [True, True] + assert (out / "a.txt").read_text() == "payload of a.txt" + assert (out / "b.txt").read_text() == "payload of b.txt" + assert repo.get_job(job_id).state == "completed" # type: ignore[union-attr] + + +def test_restore_batch_two_tapes_loads_and_unloads_each_once(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos", "video"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + _archive(repo, library, ltfs, _write(tmp_path, "c.txt"), "video") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + job_id, result = _run( + repo, + library, + ltfs, + [ + RestoreRequest("/photos/a.txt", out / "a.txt"), + RestoreRequest("/video/c.txt", out / "c.txt"), + ], + ) + + assert (recorder.loads, recorder.unloads) == (2, 2) + assert result.ok + events = [e.event for e in repo.job_journal(job_id)] + assert events.count("batch_tape_started") == 2 + assert events.count("batch_tape_finished") == 2 + + +def test_restore_batch_unknown_path_is_item_failure_not_abort(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + out = tmp_path / "out" + out.mkdir() + + job_id, result = _run( + repo, + library, + ltfs, + [ + RestoreRequest("/photos/missing.txt", out / "missing.txt"), + RestoreRequest("/photos/a.txt", out / "a.txt"), + ], + ) + + by_path = {i.catalog_path: i for i in result.items} + assert not by_path["/photos/missing.txt"].ok + assert by_path["/photos/a.txt"].ok and (out / "a.txt").exists() + assert repo.get_job(job_id).state == "failed" # type: ignore[union-attr] + + +def test_restore_batch_unmount_failure_skips_unload_and_journals( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + def broken_unmount(handle: Any) -> Any: + raise OSError("simulated unmount failure") + + monkeypatch.setattr(ltfs, "unmount", broken_unmount) + + job_id, result = _run(repo, library, ltfs, [RestoreRequest("/photos/a.txt", out / "a.txt")]) + + assert recorder.loads == 1 + assert recorder.unloads == 0 # never unload a (maybe) still-mounted tape + assert not result.ok + assert "physical_state_unknown" in [e.event for e in repo.job_journal(job_id)] + assert repo.get_job(job_id).state == "failed" # type: ignore[union-attr] diff --git a/tests/unit/test_staged_catalog.py b/tests/unit/test_staged_catalog.py new file mode 100644 index 0000000..be44f58 --- /dev/null +++ b/tests/unit/test_staged_catalog.py @@ -0,0 +1,132 @@ +"""Staged (STAGING/VERIFYING) instances and the job journal in the catalog. + +Follow-up to docs/decisions/2026-10-09-persistent-drive-leases.md: a sharded +archive stages its instances and exposes them as ARCHIVED only in one commit. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.models import FileInstance +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import FileNotFoundError +from openblade.domain.models import FileInstanceState + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()) + + +def _stage_two_shards(repo: CatalogRepository, job_id: str = "job-1") -> list[FileInstance]: + vg = repo.create_volume_group("vg-staged") + instances = [] + for index, barcode in enumerate(["AAA001L9", "BBB001L9"]): + record = repo.create_file_record( + f"/data/big.bin.shard{index}", 10, "0" * 64, vg.id, shard_index=index + ) + instances.append( + repo.create_staged_instance(job_id, record.id, barcode, f"/big.bin.shard{index}") + ) + return instances + + +def _states(repo: CatalogRepository, instances: list[FileInstance]) -> list[str]: + repo.session.expire_all() + return [repo.session.get(FileInstance, i.id).state for i in instances] # type: ignore[union-attr] + + +def test_journal_round_trip_is_ordered_and_scoped_to_the_job(repo: CatalogRepository) -> None: + repo.journal("job-1", "lease_acquired", {"drive_id": 0, "barcode": "AAA001L9"}) + repo.journal("job-2", "lease_acquired") + repo.journal("job-1", "failure", {"error": "LaneWriteError"}) + + entries = repo.job_journal("job-1") + + assert [e.event for e in entries] == ["lease_acquired", "failure"] + assert entries[0].detail == {"drive_id": 0, "barcode": "AAA001L9"} + assert entries[1].detail == {"error": "LaneWriteError"} + assert repo.job_journal("job-2")[0].detail == {} + assert repo.job_journal("nope") == [] + + +def test_create_staged_instance_journals_and_lists_by_job(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + other_job_instance = repo.create_staged_instance( + "job-2", instances[0].file_record_id, "CCC001L9", "/other" + ) + + staged = repo.list_staged_instances("job-1") + + assert [(s.barcode, s.tape_path, s.shard_index, s.state) for s in staged] == [ + ("AAA001L9", "/big.bin.shard0", 0, "staging"), + ("BBB001L9", "/big.bin.shard1", 1, "staging"), + ] + assert [s.instance_id for s in repo.list_staged_instances("job-2")] == [other_job_instance.id] + assert [e.event for e in repo.job_journal("job-1")] == ["shard_staged", "shard_staged"] + assert repo.list_staged_instances("no-such-job") == [] + + +def test_create_staged_instance_refuses_a_committed_state(repo: CatalogRepository) -> None: + vg = repo.create_volume_group("vg-x") + record = repo.create_file_record("/x", 1, "0" * 64, vg.id) + with pytest.raises(ValueError): + repo.create_staged_instance( + "job-1", record.id, "AAA001L9", "/x", state=FileInstanceState.ARCHIVED + ) + assert repo.list_instances_for_barcode("AAA001L9") == [] + + +def test_bulk_commit_archives_every_instance_together(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + ids = [i.id for i in instances] + + repo.mark_instances_verifying(ids) + assert _states(repo, instances) == ["verifying", "verifying"] + assert [s.state for s in repo.list_staged_instances("job-1")] == ["verifying"] * 2 + + repo.mark_instances_archived(ids) + + assert _states(repo, instances) == ["archived", "archived"] + assert all(repo.session.get(FileInstance, i).archived_at is not None for i in ids) # type: ignore[union-attr] + assert repo.list_staged_instances("job-1") == [] + + +def test_bulk_commit_is_all_or_nothing(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + repo.mark_instances_verifying([instances[0].id]) + + # One shard never reached VERIFYING: nothing may become ARCHIVED. + with pytest.raises(ValueError): + repo.mark_instances_archived([i.id for i in instances]) + assert _states(repo, instances) == ["verifying", "staging"] + + with pytest.raises(FileNotFoundError): + repo.mark_instances_archived([instances[0].id, "missing-id"]) + assert _states(repo, instances) == ["verifying", "staging"] + + +def test_staged_instances_are_never_listed_as_archived_or_restorable( + repo: CatalogRepository, +) -> None: + instances = _stage_two_shards(repo) + repo.mark_instances_verifying([instances[1].id]) # one STAGING, one VERIFYING + + assert repo.list_catalog_tape_barcodes() == [] + assert repo.list_ltfs_entries() == [] + for index in (0, 1): + with pytest.raises(FileNotFoundError): + repo.get_latest_instance_for_path(f"/data/big.bin.shard{index}") + + # Failure cleanup must not drop the records staged shards hang off. + repo.delete_file_record_if_unarchived("/data/big.bin.shard0") + assert repo.get_file_record("/data/big.bin.shard0") is not None + + repo.mark_instances_verifying([i.id for i in instances]) + repo.mark_instances_archived([i.id for i in instances]) + assert repo.list_catalog_tape_barcodes() == ["AAA001L9", "BBB001L9"] diff --git a/tests/unit/test_unload_gates.py b/tests/unit/test_unload_gates.py new file mode 100644 index 0000000..07cf44e --- /dev/null +++ b/tests/unit/test_unload_gates.py @@ -0,0 +1,120 @@ +"""Never unload while LTFS is mounted — both halves of the gate. + +In-process: the real-hardware library adapter refuses while its mount record is +not UNMOUNTED. Cross-process: the tape orchestrator refuses while another job +holds a live catalog lease on the drive (mount state is per process, the lease +is the only shared evidence). +""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import TapeMountedError +from openblade.domain.models import MountState, OperationResult +from openblade.hardware.library import RealLibraryBackend +from openblade.nas.tape_orchestrator import TapeOperationOrchestrator +from openblade.nas.types import TapeOpRequest, TapeOpStatus, TapeOpType +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend + + +class _RecordingChanger: + def __init__(self) -> None: + self.unloads: list[tuple[int, int]] = [] + + def unload(self, drive_id: int, target_slot: int) -> OperationResult: + self.unloads.append((drive_id, target_slot)) + return OperationResult(success=True, message="unloaded") + + +def _hardware_library() -> tuple[RealLibraryBackend, _RecordingChanger]: + # Bypass the guarded constructor: this test is about the unload gate only. + library = RealLibraryBackend.__new__(RealLibraryBackend) + changer = _RecordingChanger() + object.__setattr__(library, "changer", changer) + object.__setattr__(library, "_mount_states", {}) + return library, changer + + +def test_hardware_library_refuses_unload_while_mounted() -> None: + library, changer = _hardware_library() + library._mount_states[0] = MountState.MOUNTED_RW + with pytest.raises(TapeMountedError): + library.unload(0, 1) + assert changer.unloads == [] + + +def test_hardware_library_refuses_unload_while_dirty() -> None: + library, changer = _hardware_library() + library._mount_states[0] = MountState.DIRTY + with pytest.raises(TapeMountedError): + library.unload(0, 1) + assert changer.unloads == [] + + +def test_hardware_library_unloads_when_unmounted() -> None: + library, changer = _hardware_library() + assert library.unload(0, 1).success + assert changer.unloads == [(0, 1)] + + +def _orchestrator() -> tuple[CatalogRepository, MockLibraryBackend, TapeOperationOrchestrator]: + init_db("sqlite:///:memory:") + repo = CatalogRepository(get_session()) + library = MockLibraryBackend(num_slots=4, num_drives=1) + library.seed_slots(["OBX001L9"]) + ltfs = MockLTFSBackend(library) + library.load(1, 0) + return repo, library, TapeOperationOrchestrator(repo, library, ltfs) + + +def test_orchestrator_refuses_unload_while_another_job_holds_a_live_lease() -> None: + repo, library, orchestrator = _orchestrator() + owner = repo.create_job("archive", {}) + leases = repo.acquire_drive_leases( + job_id=owner.id, barcodes=["OBX001L9"], num_drives=1, ttl=timedelta(minutes=15) + ) + assert leases is not None + other = repo.create_job("restore", {}) + record = orchestrator.execute( + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode="OBX001L9", + drive_id=0, + slot_id=1, + requested_by="test", + job_id=other.id, + ) + ) + assert record.status is TapeOpStatus.FAILED + assert record.error == "Tape unload operation failed" # curated; cause is logged + assert library.find_drive_by_barcode("OBX001L9") == 0 # still in the drive + + +def test_orchestrator_allows_unload_for_the_lease_owner_and_after_release() -> None: + repo, library, orchestrator = _orchestrator() + owner = repo.create_job("archive", {}) + leases = repo.acquire_drive_leases( + job_id=owner.id, barcodes=["OBX001L9"], num_drives=1, ttl=timedelta(minutes=15) + ) + assert leases is not None + request = TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode="OBX001L9", + drive_id=0, + slot_id=1, + requested_by="test", + job_id=owner.id, + ) + orchestrator.execute(request) + assert library.find_drive_by_barcode("OBX001L9") is None + library.load(1, 0) + repo.release_leases([lease.id for lease in leases]) + other = repo.create_job("restore", {}) + orchestrator.execute(request.model_copy(update={"job_id": other.id})) + assert library.find_drive_by_barcode("OBX001L9") is None diff --git a/tools/ci_ownership.py b/tools/ci_ownership.py new file mode 100644 index 0000000..1745dbd --- /dev/null +++ b/tools/ci_ownership.py @@ -0,0 +1,121 @@ +"""Map changed paths to the CI test categories that own them. + +Usage (CI calls it this way, before any dependency is installed, so this +module is stdlib-only):: + + python tools/ci_ownership.py --base origin/master + python tools/ci_ownership.py --paths openblade/jobs/scheduler.py docs/x.md + +Prints the per-path selection reasoning and the selected category set. Exits +1 when any changed ``openblade/**`` or ``tests/**`` path has no owning +category in ``tools/ci_ownership.toml``. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tomllib +from collections.abc import Sequence +from dataclasses import dataclass +from fnmatch import fnmatchcase +from pathlib import Path + +CATEGORIES = frozenset( + {"unit", "integration", "safety", "i3", "compat", "frontend", "docs-only", "ci-only"} +) +ENFORCED_PREFIXES = ("openblade/", "tests/") +DEFAULT_TABLE = Path(__file__).with_name("ci_ownership.toml") + + +@dataclass(frozen=True) +class Rule: + glob: str + categories: tuple[str, ...] + + +@dataclass(frozen=True) +class Selection: + owners: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] + """path -> ((matching glob, its categories), ...)""" + categories: frozenset[str] + unowned: tuple[str, ...] + """Enforced (openblade/ or tests/) paths that no rule matched.""" + + +def load_rules(table: Path) -> list[Rule]: + data = tomllib.loads(table.read_text(encoding="utf-8")) + rules: list[Rule] = [] + for raw in data.get("rule", []): + cats = tuple(raw["categories"]) + bad = set(cats) - CATEGORIES + if bad or not cats: + raise ValueError(f"rule {raw['glob']!r}: invalid categories {sorted(bad) or '[]'}") + rules.append(Rule(glob=raw["glob"], categories=cats)) + return rules + + +def select(paths: Sequence[str], rules: Sequence[Rule]) -> Selection: + """Pure mapping: changed paths -> owning categories + unowned enforced paths.""" + owners: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = {} + categories: set[str] = set() + unowned: list[str] = [] + for path in paths: + matches = tuple((r.glob, r.categories) for r in rules if fnmatchcase(path, r.glob)) + owners[path] = matches + for _, cats in matches: + categories.update(cats) + if not matches and path.startswith(ENFORCED_PREFIXES): + unowned.append(path) + return Selection(owners=owners, categories=frozenset(categories), unowned=tuple(unowned)) + + +def changed_paths(base: str) -> list[str]: + out = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout + return [line for line in out.splitlines() if line.strip()] + + +def report(selection: Selection) -> str: + lines: list[str] = [] + for path, matches in selection.owners.items(): + if matches: + why = "; ".join(f"{glob} -> {','.join(cats)}" for glob, cats in matches) + lines.append(f" {path}: {why}") + elif path in selection.unowned: + lines.append(f" {path}: UNOWNED (no rule in ci_ownership.toml)") + else: + lines.append(f" {path}: no rule (outside openblade/ and tests/, not enforced)") + lines.append(f"selected categories: {', '.join(sorted(selection.categories)) or '(none)'}") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--base", help="git ref to diff against (base...HEAD)") + source.add_argument("--paths", nargs="+", help="explicit paths instead of a git diff") + parser.add_argument("--table", type=Path, default=DEFAULT_TABLE) + args = parser.parse_args(argv) + + paths = list(args.paths) if args.paths else changed_paths(args.base) + selection = select(paths, load_rules(args.table)) + print(f"ci-ownership: {len(paths)} changed path(s)") + print(report(selection)) + if selection.unowned: + print( + f"ci-ownership: FAIL - {len(selection.unowned)} path(s) under openblade/ or tests/ " + "have no owning category; add a rule to tools/ci_ownership.toml", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml new file mode 100644 index 0000000..024b683 --- /dev/null +++ b/tools/ci_ownership.toml @@ -0,0 +1,202 @@ +# Changed-path -> test-category ownership table for tools/ci_ownership.py. +# +# Every rule whose glob matches a path contributes its categories (union). +# Globs use fnmatch semantics: `*` also spans `/`, so "openblade/api/*" +# covers the whole subtree. There is deliberately NO catch-all +# "openblade/*" rule: a new subpackage must be given an owner here, or +# the ci-gate ownership check fails. +# +# Valid categories: unit, integration, safety, i3, compat, frontend, +# docs-only, ci-only. + +# --- tests ----------------------------------------------------------------- +[[rule]] +glob = "tests/unit/*" +categories = ["unit"] + +[[rule]] +glob = "tests/property/*" +categories = ["unit"] + +[[rule]] +glob = "tests/integration/*" +categories = ["integration"] + +[[rule]] +glob = "tests/fault/*" +categories = ["integration"] + +[[rule]] +glob = "tests/e2e/*" +categories = ["integration"] + +[[rule]] +glob = "tests/hardware/*" +categories = ["integration"] + +[[rule]] +glob = "tests/contract/*" +categories = ["integration"] + +[[rule]] +glob = "tests/fakes/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "tests/test_nas_protocol_render.py" +categories = ["unit"] + +[[rule]] +glob = "tests/safety/*" +categories = ["safety"] + +[[rule]] +glob = "tests/i3/*" +categories = ["i3"] + +[[rule]] +glob = "tests/compat/*" +categories = ["compat"] + +[[rule]] +glob = "tools/compat_capture*" +categories = ["compat"] + +[[rule]] +glob = "tests/fixtures/*" +categories = ["unit", "compat"] + +[[rule]] +glob = "tests/emulator_spec/*" +categories = ["i3"] + +# Shared test scaffolding at the top of tests/. +[[rule]] +glob = "tests/conftest.py" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "tests/__init__.py" +categories = ["unit"] + +[[rule]] +glob = "tests/assistant_support.py" +categories = ["unit", "safety"] + +[[rule]] +glob = "tests/gen_wiki_reference.py" +categories = ["docs-only"] + +# --- openblade ------------------------------------------------------------- +[[rule]] +glob = "openblade/__init__.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/api/*" +categories = ["unit", "integration", "i3"] + +[[rule]] +glob = "openblade/emulator_contract/*" +categories = ["unit", "i3", "compat"] + +[[rule]] +glob = "openblade/assistant/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/safety/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/domain/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/jobs/*" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "openblade/catalog/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/hardware/*" +categories = ["unit", "integration", "compat"] + +[[rule]] +glob = "openblade/simulator/*" +categories = ["unit", "integration", "i3"] + +[[rule]] +glob = "openblade/nas/*" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "openblade/fuse/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/sftp/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/cli/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/testing/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/web_flask/*" +categories = ["unit", "integration", "frontend"] + +[[rule]] +glob = "openblade/bootstrap.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/config.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/config_validation.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/deploy.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/dr.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/topology.py" +categories = ["unit", "integration"] + +# --- outside the enforced trees (informational) ------------------------------ +[[rule]] +glob = "docs/*" +categories = ["docs-only"] + +[[rule]] +glob = "*.md" +categories = ["docs-only"] + +[[rule]] +glob = ".github/*" +categories = ["ci-only"] + +[[rule]] +glob = "tools/ci_ownership*" +categories = ["ci-only"] + +[[rule]] +glob = "Makefile" +categories = ["ci-only"] + +[[rule]] +glob = ".yamllint" +categories = ["ci-only"] diff --git a/tools/compat_capture.py b/tools/compat_capture.py new file mode 100644 index 0000000..865cb0a --- /dev/null +++ b/tools/compat_capture.py @@ -0,0 +1,365 @@ +"""Compatibility capture + sanitize pipeline for the ``compatibility/`` corpus. + +Replays each case's ``request`` block against a live base URL (the OpenBlade +emulator or, on-site, a real Scalar i3) and writes a new case whose +``expected`` is filled from the observed response. Every written case passes +through :func:`sanitize` first so tokens, serials, hosts, timestamps and +passwords never reach the repository. + + capture --base-url URL --recipe --out + [--source self-captured|captured] [--profile scalar-i3-341g] + [--login user:pass] [--appliance-firmware VER --attest NAME] + capture --base-url URL --recipe --diff + +``source: captured`` is appliance-only and ENFORCED by the corpus harness, so +the tool refuses to write it without ``--appliance-firmware`` AND ``--attest``. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import httpx + +TOKEN = "" +SERIAL = "" +HOST = "" +TIMESTAMP = "" +REDACTED = "" +# A request password that was NOT the --login credential (a deliberate bad-password case). +REDACTED_INVALID = "" + +SOURCES = ("self-captured", "captured") +_COOKIE_ATTRS = frozenset({"path", "domain", "expires", "max-age", "samesite"}) + + +class CaptureRefusedError(Exception): + """The requested capture would mislabel provenance; nothing was written.""" + + +@dataclass(frozen=True) +class Rules: + # Key rules match on the lower-cased key with "_"/"-" removed, by substring. + token_keys: tuple[str, ...] = ("session", "token", "authorization", "apikey") + serial_keys: tuple[str, ...] = ("serial",) + password_keys: tuple[str, ...] = ("password", "passwd") + cookie_keys: tuple[str, ...] = ("cookie", "setcookie") + bearer: re.Pattern[str] = re.compile(r"(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+") + url_host: re.Pattern[str] = re.compile(r"(?i)\b(https?|wss?)://[^/\s\"'?#]+") + ipv4: re.Pattern[str] = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b") + timestamp: re.Pattern[str] = re.compile( + r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?" + r"|\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT" + ) + + +DEFAULT_RULES = Rules() + + +_XML_START_TAG = re.compile(r"<[A-Za-z_][\w.-]*\s[^<>]*>") +_XML_ATTR = re.compile(r"([A-Za-z_][\w.:-]*)\s*=\s*([\"'])(.*?)\2") +_XML_ELEMENT = re.compile(r"<([A-Za-z_][\w.-]*)((?:\s[^<>]*)?)>([^<]*)") + + +def _norm(key: str) -> str: + return key.lower().replace("_", "").replace("-", "") + + +def _key_hit(key: str, needles: Iterable[str]) -> bool: + k = _norm(key) + return any(n in k for n in needles) + + +def _scrub_cookie(value: str) -> str: + def repl(m: re.Match[str]) -> str: + name = m.group(1) + return m.group(0) if name.strip().lower() in _COOKIE_ATTRS else f"{name}={TOKEN}" + + return re.sub(r"([^=;,\s]+)=([^;,]*)", repl, value) + + +def _secret_mark(name: str, rules: Rules) -> str | None: + for needles, mark in ( + (rules.password_keys, REDACTED), + (rules.token_keys, TOKEN), + (rules.serial_keys, SERIAL), + ): + if _key_hit(name, needles): + return mark + return None + + +def _scrub_xml(value: str, rules: Rules) -> str: + # XML bodies (Accept: application/xml) carry the same secrets as JSON keys, + # both as element text and as attribute values (``sessionId="..."``). + def attr_repl(m: re.Match[str]) -> str: + mark = _secret_mark(m.group(1), rules) + return m.group(0) if mark is None else f"{m.group(1)}={m.group(2)}{mark}{m.group(2)}" + + def tag_repl(m: re.Match[str]) -> str: + return _XML_ATTR.sub(attr_repl, m.group(0)) + + def elem_repl(m: re.Match[str]) -> str: + tag, attrs, text = m.group(1), m.group(2), m.group(3) + if not text: # already a placeholder (or empty): keep idempotent + return m.group(0) + mark = _secret_mark(tag, rules) + return m.group(0) if mark is None else f"<{tag}{attrs}>{mark}" + + # Element text first: once an attribute holds a placeholder the start tag + # no longer matches _XML_ELEMENT (that is what keeps a re-run idempotent). + value = _XML_ELEMENT.sub(elem_repl, value) + return _XML_START_TAG.sub(tag_repl, value) + + +def _scrub_str(value: str, rules: Rules) -> str: + value = _scrub_xml(value, rules) + value = rules.bearer.sub(lambda m: f"{m.group(1)} {TOKEN}", value) + value = rules.url_host.sub(lambda m: f"{m.group(1)}://{HOST}", value) + value = rules.ipv4.sub(HOST, value) + return rules.timestamp.sub(TIMESTAMP, value) + + +def _mask_leaves(node: Any, mark: str, keep: tuple[str, ...] = ()) -> Any: + # A secret-class key masks EVERY scalar beneath it, however it is nested. + if node is None or isinstance(node, bool): + return node + if isinstance(node, dict): + return {k: _mask_leaves(v, mark, keep) for k, v in node.items()} + if isinstance(node, list): + return [_mask_leaves(item, mark, keep) for item in node] + return node if node in keep else mark + + +def _walk(node: Any, rules: Rules) -> Any: + if isinstance(node, dict): + out: dict[str, Any] = {} + for key, value in node.items(): + if value is None or isinstance(value, bool): + out[key] = value + elif _key_hit(key, rules.password_keys): + out[key] = _mask_leaves(value, REDACTED, (REDACTED, REDACTED_INVALID)) + elif _key_hit(key, rules.cookie_keys) and isinstance(value, str): + out[key] = _scrub_str(_scrub_cookie(value), rules) + elif _key_hit(key, rules.token_keys): + out[key] = _mask_leaves(value, TOKEN) + elif _key_hit(key, rules.serial_keys): + out[key] = _mask_leaves(value, SERIAL) + else: + out[key] = _walk(value, rules) + return out + if isinstance(node, list): + return [_walk(item, rules) for item in node] + if isinstance(node, str): + return _scrub_str(node, rules) + return node + + +def sanitize(case: dict[str, Any], rules: Rules = DEFAULT_RULES) -> dict[str, Any]: + """Pure: return a scrubbed deep copy of ``case``; the input is not mutated.""" + result: dict[str, Any] = _walk(copy.deepcopy(case), rules) + return result + + +def dumps_case(case: dict[str, Any]) -> str: + """Deterministic serialisation: same case -> byte-identical file.""" + return json.dumps(case, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def load_recipes(path: Path) -> list[dict[str, Any]]: + if not path.is_dir(): + return [json.loads(path.read_text())] + # A directory's own earlier outputs are not recipes: their passwords are redacted. + cases = [json.loads(p.read_text()) for p in sorted(path.rglob("*.json"))] + return [c for c in cases if c.get("source") not in SOURCES] + + +def _login(client: httpx.Client, login: str | None) -> None: + if login is None: + raise CaptureRefusedError("case needs auth but --login user:pass was not given") + user, _, password = login.partition(":") + resp = client.post("/aml/users/login", json={"name": user, "password": password}) + if resp.status_code != 200: + raise CaptureRefusedError(f"login as {user!r} failed with status {resp.status_code}") + + +def replay(client: httpx.Client, case: dict[str, Any], login: str | None) -> httpx.Response: + # Mirrors tests/compat/test_compatibility_corpus.py::_run so captures replay identically. + if case.get("auth"): + _login(client, login) + req = case["request"] + headers = dict(req.get("headers") or {}) + if "content_type" in req: + headers["Content-Type"] = req["content_type"] + kwargs: dict[str, Any] = {"headers": headers or None} + if "body" in req: + kwargs["content"] = req["body"] + else: + kwargs["json"] = req.get("json") + for _ in range(int(req.get("repeat", 0))): + client.request(req["method"], req["path"], **kwargs) + return client.request(req["method"], req["path"], **kwargs) + + +def _body(resp: httpx.Response) -> Any: + try: + return resp.json() + except ValueError: + return resp.text + + +def build_case( + recipe: dict[str, Any], + resp: httpx.Response, + *, + source: str, + profile: str | None = None, + appliance_firmware: str | None = None, + attest: str | None = None, + login: str | None = None, +) -> dict[str, Any]: + if source not in SOURCES: + raise CaptureRefusedError(f"source must be one of {SOURCES}, not {source!r}") + if source == "captured" and not (appliance_firmware and attest): + raise CaptureRefusedError( + "source=captured is appliance-only: --appliance-firmware and --attest are both required" + ) + raw = _body(resp) + clean = sanitize({"b": raw})["b"] + expected: dict[str, Any] = {"status": resp.status_code} + ctype = resp.headers.get("content-type") + if ctype: + expected["headers"] = {"content-type": ctype.split(";")[0].strip()} + if isinstance(raw, dict): + # Only values the sanitizer left untouched are enforceable. + expected["json_contains"] = {k: v for k, v in clean.items() if raw.get(k) == v} + observed_headers = {k.lower(): v for k, v in resp.headers.items() if k.lower() != "date"} + case: dict[str, Any] = { + key: recipe[key] + for key in ("id", "profile", "firmware", "auth", "manual_ref", "request") + if key in recipe + } + case.update( + source=source, + expected=expected, + observed={"headers": observed_headers, "body": clean}, + ) + good = login.partition(":")[2] if login else None + payload = case.get("request", {}).get("json") + if isinstance(payload, dict): + # Keep "right vs wrong password" replayable after redaction. + case["request"] = { + **case["request"], + "json": { + k: REDACTED_INVALID + if _key_hit(k, DEFAULT_RULES.password_keys) and isinstance(v, str) and v != good + else v + for k, v in payload.items() + }, + } + if profile: + case["profile"] = profile + if source == "captured": + case.update(firmware=appliance_firmware, attested_by=attest) + else: + case["captured_from"] = "openblade-emulator" + case["fidelity_notes"] = ( + "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression " + "guard of emulator behaviour, NOT evidence of appliance fidelity." + ) + return sanitize(case) + + +def drift(case: dict[str, Any], resp: httpx.Response) -> list[str]: + exp = case["expected"] + problems: list[str] = [] + if resp.status_code != exp["status"]: + problems.append(f"status {resp.status_code} != {exp['status']}") + for name, needle in exp.get("headers", {}).items(): + value = resp.headers.get(name) + if value is None or needle not in value: + problems.append(f"header {name}={value!r} lacks {needle!r}") + if "json_contains" in exp: + body = _body(resp) + if not isinstance(body, dict): + problems.append(f"body is not a JSON object: {str(body)[:80]!r}") + else: + for key, value in exp["json_contains"].items(): + if body.get(key, KeyError) != value: + problems.append(f"{key}: expected {value!r}, got {body.get(key)!r}") + return problems + + +ClientFactory = Callable[[], httpx.Client] + + +def run_diff(factory: ClientFactory, cases: list[dict[str, Any]], login: str | None) -> int: + total = 0 + for case in cases: + with factory() as client: + problems = drift(case, replay(client, case, login)) + total += len(problems) + for p in problems: + print(f"DRIFT {case['id']}: {p}") + print(f"[compat_capture] diff: {len(cases)} cases, {total} drift findings") + return 1 if total else 0 + + +def run_capture(factory: ClientFactory, cases: list[dict[str, Any]], out: Path, **opts: Any) -> int: + login = opts.pop("login", None) + built = [] + for recipe in cases: # build everything first: a refusal writes nothing + with factory() as client: + built.append(build_case(recipe, replay(client, recipe, login), login=login, **opts)) + out.mkdir(parents=True, exist_ok=True) + for case in built: + (out / f"{case['id']}.json").write_text(dumps_case(case)) + print(f"[compat_capture] wrote {len(built)} cases to {out}") + return 0 + + +def main(argv: list[str] | None = None, factory: ClientFactory | None = None) -> int: + ap = argparse.ArgumentParser(prog="compat_capture", description=__doc__.split("\n")[0]) + ap.add_argument("command", choices=["capture"]) + ap.add_argument("--base-url", required=True) + ap.add_argument("--recipe", required=True, type=Path) + ap.add_argument("--out", type=Path) + ap.add_argument("--diff", action="store_true") + ap.add_argument("--source", choices=SOURCES, default="self-captured") + ap.add_argument("--profile") + ap.add_argument("--login") + ap.add_argument("--appliance-firmware") + ap.add_argument("--attest") + args = ap.parse_args(argv) + make = factory or (lambda: httpx.Client(base_url=args.base_url, timeout=30.0)) + cases = load_recipes(args.recipe) + try: + if args.diff: + return run_diff(make, cases, args.login) + if args.out is None: + ap.error("--out is required unless --diff is given") + return run_capture( + make, + cases, + args.out, + login=args.login, + source=args.source, + profile=args.profile, + appliance_firmware=args.appliance_firmware, + attest=args.attest, + ) + except CaptureRefusedError as exc: + print(f"[compat_capture] REFUSED: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/hardware/__init__.py b/tools/hardware/__init__.py new file mode 100644 index 0000000..3dcb5a0 --- /dev/null +++ b/tools/hardware/__init__.py @@ -0,0 +1 @@ +"""Hardware-lane operator tools (allowlist gate, inventory snapshots).""" diff --git a/tools/hardware/allowlist.py b/tools/hardware/allowlist.py new file mode 100644 index 0000000..207a405 --- /dev/null +++ b/tools/hardware/allowlist.py @@ -0,0 +1,94 @@ +"""Sacrificial-barcode allowlist gate for the destructive hardware lane. + +Run BEFORE any hardware is touched:: + + python -m tools.hardware.allowlist --barcodes "TST001L8 TST002L8" + +The allowlist comes from env ``SACRIFICIAL_BARCODES``; the request comes from +``--barcodes`` or, when the flag is absent, env ``BARCODES``. Both accept +comma- and/or whitespace-separated lists (case-insensitive). Exit 0 when the +request is a non-empty subset of a non-empty allowlist and print exactly one +stdout line: the normalized request (upper-cased, de-duplicated, request order, +comma-joined, e.g. ``TST001L8,TST002L8``), suitable for +``OPENBLADE_SCRATCH_BARCODES``. Exit 2 otherwise (fail closed), naming every +offending barcode on stderr with nothing on stdout. All diagnostics go to stderr. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from collections.abc import Sequence + +EXIT_OK = 0 +EXIT_VIOLATION = 2 + +_SEPARATORS = re.compile(r"[\s,]+") + + +class AllowlistError(Exception): + """The requested barcodes are not safe to use destructively.""" + + +def parse_barcodes(raw: str | None) -> list[str]: + """Split a comma/whitespace list into upper-cased, de-duplicated barcodes.""" + seen: dict[str, None] = {} + for token in _SEPARATORS.split(raw or ""): + if token: + seen.setdefault(token.strip().upper(), None) + return list(seen) + + +def format_barcodes(barcodes: Sequence[str]) -> str: + """The normalized stdout form: comma-joined, no spaces.""" + return ",".join(barcodes) + + +def check_subset(requested: Sequence[str], allowlist: Sequence[str]) -> list[str]: + """Return ``requested`` if it is a non-empty subset of a non-empty allowlist. + + Raises :class:`AllowlistError` naming the offenders otherwise. + """ + if not allowlist: + raise AllowlistError( + "SACRIFICIAL_BARCODES is empty: no tape is approved for destructive use (fail closed)." + ) + if not requested: + raise AllowlistError("No barcodes requested (pass --barcodes or set BARCODES).") + allowed = set(allowlist) + offenders = [b for b in requested if b not in allowed] + if offenders: + raise AllowlistError( + "Barcodes outside the sacrificial allowlist: " + + ", ".join(offenders) + + ". Refusing before touching hardware." + ) + return list(requested) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m tools.hardware.allowlist") + parser.add_argument( + "--barcodes", + default=None, + help="Comma/space-separated barcodes to use (default: env BARCODES).", + ) + args = parser.parse_args(argv) + raw_request = args.barcodes if args.barcodes is not None else os.environ.get("BARCODES") + try: + approved = check_subset( + parse_barcodes(raw_request), + parse_barcodes(os.environ.get("SACRIFICIAL_BARCODES")), + ) + except AllowlistError as exc: + print(f"allowlist: REFUSED: {exc}", file=sys.stderr) + return EXIT_VIOLATION + print(f"allowlist: OK: {len(approved)} barcode(s) approved", file=sys.stderr) + print(format_barcodes(approved)) + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hardware/junit_gate.py b/tools/hardware/junit_gate.py new file mode 100644 index 0000000..1c0055b --- /dev/null +++ b/tools/hardware/junit_gate.py @@ -0,0 +1,83 @@ +"""Fail a rig lane that "passed" by skipping. + +Rig tests skip when their prerequisites are missing, and pytest exits 0 on an +all-skipped run. This gate reads the JUnit XML each rig pytest step wrote and +exits 1 if any test was skipped (outside an explicit expected-skip list) or if +nothing passed at all. + + python -m tools.hardware.junit_gate reports/*.xml [--expected-skip NODEID ...] +""" + +from __future__ import annotations + +import argparse +import sys +import xml.etree.ElementTree as ET # parses our own pytest output, not untrusted input +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class GateResult: + passed: int = 0 + failed: int = 0 + skipped: list[str] = field(default_factory=list) + + +def _node_id(case: ET.Element) -> str: + classname = case.get("classname", "") + name = case.get("name", "") + return f"{classname}::{name}" if classname else name + + +def collect(paths: list[Path]) -> GateResult: + result = GateResult() + for path in paths: + root = ET.parse(path).getroot() + for case in root.iter("testcase"): + if case.find("skipped") is not None: + result.skipped.append(_node_id(case)) + elif case.find("failure") is not None or case.find("error") is not None: + result.failed += 1 + else: + result.passed += 1 + return result + + +def evaluate(result: GateResult, expected_skips: set[str]) -> list[str]: + """Return the reasons the gate fails (empty list = pass).""" + problems: list[str] = [] + unexpected = sorted(s for s in result.skipped if s not in expected_skips) + if unexpected: + problems.append(f"{len(unexpected)} unexpected skip(s): " + ", ".join(unexpected)) + if result.passed == 0: + problems.append("0 tests passed") + if result.failed: + problems.append(f"{result.failed} test(s) failed or errored") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("reports", nargs="+", type=Path) + parser.add_argument( + "--expected-skip", + action="append", + default=[], + help="JUnit classname::name of a skip that is expected (repeatable)", + ) + args = parser.parse_args(argv) + missing = [str(p) for p in args.reports if not p.is_file()] + if missing: + print("::error::missing JUnit report(s): " + ", ".join(missing), file=sys.stderr) + return 1 + result = collect(args.reports) + problems = evaluate(result, set(args.expected_skip)) + print(f"passed={result.passed} failed={result.failed} skipped={len(result.skipped)}") + for problem in problems: + print(f"::error::rig lane gate: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hardware/snapshot.py b/tools/hardware/snapshot.py new file mode 100644 index 0000000..9f0b93e --- /dev/null +++ b/tools/hardware/snapshot.py @@ -0,0 +1,168 @@ +"""Before/after library-state snapshots for the destructive hardware lane. + + python -m tools.hardware.snapshot capture --out before.json + python -m tools.hardware.snapshot diff before.json after.json --out diff.json + +``capture`` reads inventory + drive state through ``InventoryService`` on the +library the configured backend builds (``OPENBLADE_BACKEND``; the simulator by +default, real only when ``OPENBLADE_REAL_HARDWARE_ENABLED=true`` too). It is +read-only, and refuses (exit 2, ``snapshot: REFUSED:`` on stderr, no file +written) when the configured backend is real but ``OPENBLADE_HARDWARE_DRY_RUN`` +is true: dry-run never touches the changer, so the capture would not describe +the library the lane is about to change. ``diff`` compares two captures and writes the diff file, then exits +0 when identical (library_id included), 1 when they differ, 2 on unreadable +input. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import enum +import json +import sys +from collections.abc import Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from openblade.config import BackendMode, OpenBladeConfig +from openblade.domain.backends import LibraryBackend + +EXIT_OK = 0 +EXIT_CHANGED = 1 +EXIT_BAD_INPUT = 2 +EXIT_REFUSED = 2 + + +class SnapshotError(Exception): + """A snapshot file is missing or malformed.""" + + +def _jsonable(value: Any) -> Any: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {f.name: _jsonable(getattr(value, f.name)) for f in dataclasses.fields(value)} + if isinstance(value, enum.Enum): + return _jsonable(value.value) + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + if isinstance(value, list | tuple): + return [_jsonable(v) for v in value] + if value is None or isinstance(value, bool | int | float | str): + return value + return str(value) + + +def capture_snapshot(library: LibraryBackend) -> dict[str, Any]: + """Return a JSON-able snapshot of ``library``'s inventory and drive state.""" + from openblade.jobs.inventory import InventoryService + + inventory = InventoryService(library).snapshot() + return { + "captured_at": datetime.now(UTC).isoformat(), + "library_id": inventory.library_id, + "slots": [_jsonable(s) for s in inventory.slots], + "drives": [_jsonable(d) for d in inventory.drives], + "changer_state": _jsonable(inventory.changer_state), + } + + +def _index(items: list[dict[str, Any]], key: str) -> dict[str, dict[str, Any]]: + return {str(item.get(key, i)): item for i, item in enumerate(items)} + + +def _diff_section( + before: list[dict[str, Any]], after: list[dict[str, Any]], key: str +) -> list[dict[str, Any]]: + b, a = _index(before, key), _index(after, key) + changes: list[dict[str, Any]] = [] + for ident in sorted(b.keys() | a.keys()): + if b.get(ident) != a.get(ident): + changes.append({key: ident, "before": b.get(ident), "after": a.get(ident)}) + return changes + + +def diff_snapshots(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + """Compare two captures element-by-element (slots by slot_id, drives by drive_id).""" + slots = _diff_section(before.get("slots", []), after.get("slots", []), "slot_id") + drives = _diff_section(before.get("drives", []), after.get("drives", []), "drive_id") + library_changed = before.get("library_id") != after.get("library_id") + changer = ( + [] + if before.get("changer_state") == after.get("changer_state") + else [{"before": before.get("changer_state"), "after": after.get("changer_state")}] + ) + return { + "library_id": {"before": before.get("library_id"), "after": after.get("library_id")}, + "slots": slots, + "drives": drives, + "changer_state": changer, + "changed": bool(library_changed or slots or drives or changer), + } + + +def _load(path: Path) -> dict[str, Any]: + try: + data = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise SnapshotError(f"cannot read snapshot {path}: {exc.__class__.__name__}") from exc + if not isinstance(data, dict): + raise SnapshotError(f"snapshot {path} is not a JSON object") + return data + + +def dry_run_refusal(config: OpenBladeConfig) -> str | None: + """Return why ``capture`` must refuse under ``config``, or None when it may run.""" + if config.backend is BackendMode.REAL and config.hardware_dry_run: + return ( + "OPENBLADE_HARDWARE_DRY_RUN=true with OPENBLADE_BACKEND=real: a dry-run " + "capture does not reflect the physical library." + ) + return None + + +def _build_library(config: OpenBladeConfig) -> LibraryBackend: + from openblade.bootstrap import create_context + + return create_context(config).library + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m tools.hardware.snapshot") + sub = parser.add_subparsers(dest="command", required=True) + cap = sub.add_parser("capture", help="Capture inventory + drive state to JSON.") + cap.add_argument("--out", required=True, type=Path) + dif = sub.add_parser("diff", help="Diff two captures.") + dif.add_argument("before", type=Path) + dif.add_argument("after", type=Path) + dif.add_argument("--out", required=True, type=Path) + args = parser.parse_args(argv) + + if args.command == "capture": + from openblade.config import load_config + + config = load_config() + reason = dry_run_refusal(config) + if reason is not None: + print(f"snapshot: REFUSED: {reason}", file=sys.stderr) + return EXIT_REFUSED + snap = capture_snapshot(_build_library(config)) + args.out.write_text(json.dumps(snap, indent=2, sort_keys=True)) + print(f"snapshot: captured {len(snap['slots'])} slots, {len(snap['drives'])} drives") + return EXIT_OK + try: + result = diff_snapshots(_load(args.before), _load(args.after)) + except SnapshotError as exc: + print(f"snapshot: {exc}", file=sys.stderr) + return EXIT_BAD_INPUT + args.out.write_text(json.dumps(result, indent=2, sort_keys=True)) + print( + f"snapshot diff: {len(result['slots'])} slot change(s), " + f"{len(result['drives'])} drive change(s), " + f"changer {'changed' if result['changer_state'] else 'unchanged'}" + ) + return EXIT_CHANGED if result["changed"] else EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/mutation_run.sh b/tools/mutation_run.sh new file mode 100755 index 0000000..4ab727b --- /dev/null +++ b/tools/mutation_run.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Mutation lane: run mutmut (config: [tool.mutmut] in pyproject.toml) over the +# safety-critical modules and ratchet surviving mutants against +# mutation/baseline.txt. More survivors than the baseline for any module => exit 1. +# +# Env: +# MUTATION_MODULES space-separated module paths to mutate +# (default: every module listed in mutation/baseline.txt) +# MUTATION_TIMEOUT wall-clock budget in seconds (default 3600); exceeding it exits 2 +# MUTATION_MAX_CHILDREN mutmut worker count (default: mutmut's own, = cpu count) +# PYTHON interpreter with mutmut installed (default: python) +set -euo pipefail + +cd "$(dirname "$0")/.." +PY="${PYTHON:-python}" +BASELINE="mutation/baseline.txt" +TIMEOUT="${MUTATION_TIMEOUT:-3600}" + +if [[ -n "${MUTATION_MODULES:-}" ]]; then + read -r -a modules <<<"$MUTATION_MODULES" +else + mapfile -t modules < <(grep -vE '^\s*(#|$)' "$BASELINE" | awk '{print $1}') +fi + +# mutmut names mutants ".__mutmut_"; select by module glob. +globs=() +for mod in "${modules[@]}"; do + [[ -f "$mod" ]] || { echo "mutation: no such module: $mod" >&2; exit 2; } + dotted="${mod%.py}" + globs+=("${dotted//\//.}.*") +done + +# Fresh run every time: a stale mutants/ cache would report old results. +rm -rf mutants + +run_args=(run) +[[ -n "${MUTATION_MAX_CHILDREN:-}" ]] && run_args+=(--max-children "$MUTATION_MAX_CHILDREN") +set +e +timeout "$TIMEOUT" "$PY" -m mutmut "${run_args[@]}" "${globs[@]}" +rc=$? +set -e +if [[ $rc -eq 124 ]]; then + echo "mutation: time budget of ${TIMEOUT}s exceeded - result is NOT a pass" >&2 + exit 2 +elif [[ $rc -ne 0 ]]; then + echo "mutation: mutmut run failed (exit $rc)" >&2 + exit "$rc" +fi + +results="$("$PY" -m mutmut results)" +fail=0 +for mod in "${modules[@]}"; do + dotted="${mod%.py}" + prefix="${dotted//\//.}." + survived=$(grep -F " ${prefix}" <<<"$results" | grep -cE ': (survived|no tests)$' || true) + allowed=$(awk -v m="$mod" '$1 == m {print $2}' "$BASELINE") + if [[ -z "$allowed" ]]; then + echo "mutation: $mod survived=$survived but has no baseline entry - add one to $BASELINE" >&2 + fail=1 + elif ((survived > allowed)); then + echo "mutation: $mod survived=$survived > baseline=$allowed (new survivors: kill them or justify)" >&2 + fail=1 + else + echo "mutation: $mod survived=$survived <= baseline=$allowed" + ((survived < allowed)) && echo "mutation: ratchet: lower $mod baseline to $survived" + fi +done +exit "$fail"