From c20661d76ceb904810d2e42db5c761779e91eacf Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:33:24 +0000 Subject: [PATCH 001/205] docs: spec for persistent drive leases with fencing and restart recovery (roadmap 7/8) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 docs/decisions/2026-10-09-persistent-drive-leases.md diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md new file mode 100644 index 0000000..a95b833 --- /dev/null +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -0,0 +1,163 @@ +# Persistent drive leases with fencing, and job recovery on restart + +Status: accepted 2026-10-09. Roadmap items 7 and 8 (Phase 2), first slice. + +## Where the four working copies stand (so nobody re-merges them) + +| checkout | branch | relation to `origin/master` | +|---|---|---| +| `/root/ob2-final` | `integrate/gap-closure` | tree-identical to master (`1863348`, PR #45) | +| `/root/ob-final` | `integrate/eight-items` | tree-identical to PR #44 squash, already on master | +| `/root/ob-int2` | `integrate/campaign-wiki-assistant` | fully contained in `ob-final` | +| `/root/OpenBlade` | `chore/py312-baseline-and-quantum-refs` | stale (37 behind); its 3 stray commits are already on master by content | + +Nothing is left to merge. All new work starts from `origin/master`. + +## Problem + +`DriveScheduler` is the only thing that stops two tape jobs from driving the +same physical drive, and it is constructed **per request** and held **in +memory**: + +- `openblade/api/routes_archive.py:196`, `routes_restore.py:159` and `:234`, + `openblade/cli/main.py:624`, `:698`, `:838` each do + `DriveScheduler(num_drives=len(context.library.inventory().drives))`. +- Two API requests (only the archive route has a process-wide lock), or the + CLI and the API against the same catalog, each get a scheduler that believes + every drive is free. +- After a crash or restart, nothing knows which drives a dead job held, and + catalog jobs stay `running` forever. + +The roadmap (items 7, 8) asks for DB-backed leases with a monotonic fencing +token and restart recovery. This note is the spec for that slice. + +## Decision + +### 1. Lease store in the catalog + +New table `drive_leases` (SQLAlchemy model in `openblade/catalog/models.py`, +created by `Base.metadata.create_all`; nothing to add to `_migrate_schema` +because it is a new table): + +| column | type | notes | +|---|---|---| +| `id` | str pk (uuid4) | lease id | +| `drive_id` | int, not null | scheduler lock key; `UNIQUE` while the lease is live (see below) | +| `job_id` | str, not null, indexed | owning catalog job | +| `barcode` | str(8), not null | cartridge the lease was taken for | +| `physical_drive_id` | int, nullable | mirrors `DriveHandle.physical_drive_id` | +| `fencing_token` | int, not null, unique | monotonic across the catalog | +| `acquired_at` | datetime utc | | +| `heartbeat_at` | datetime utc | updated by `heartbeat()` | +| `expires_at` | datetime utc | `heartbeat_at + ttl`; default ttl 15 min | +| `released_at` | datetime utc, nullable | null = live | + +Live-ness is `released_at IS NULL AND expires_at > now`. Enforce "one live +lease per drive" in code under a write transaction (SQLite `BEGIN IMMEDIATE` +via `connection.exec_driver_sql`, Postgres not a target here) — do **not** rely +on a partial unique index, SQLite support for those in SQLAlchemy is uneven. + +`fencing_token` = `COALESCE(MAX(fencing_token), 0) + 1` inside the same +transaction. It is never reused, even after release. + +Repository API (`CatalogRepository`, keep it small): + +```python +def acquire_drive_leases(self, *, job_id, barcodes: list[str], num_drives: int, ttl: timedelta) -> list[DriveLease] | None # None = not all free, caller waits +def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None +def release_leases(self, lease_ids: list[str]) -> None +def lease_is_live(self, lease_id: str, fencing_token: int) -> bool +def live_leases(self) -> list[DriveLease] +def release_leases_for_job(self, job_id: str) -> int +``` + +`DriveLease` is a frozen dataclass in `openblade/domain/models.py`. + +### 2. `DriveScheduler` becomes a façade over a lease store + +Keep the public shape every caller and ~all tests already use: +`acquire_drives(barcodes, timeout)`, `release_drives(handles)`, `status()`, +`available_count()`, `num_drives`. Add: + +- `DriveScheduler(num_drives, *, store: LeaseStore, job_id: str, ttl=...)`. +- `LeaseStore` is a `Protocol` with two implementations: + `InMemoryLeaseStore` (existing behaviour, used by unit tests that build a + scheduler directly) and `CatalogLeaseStore` (wraps the repository calls + above). This is the one new abstraction and it exists because tests and + production genuinely differ. +- `acquire_drives` keeps the all-or-nothing + timeout semantics; the wait loop + polls the store (0.25 s) instead of `Condition.wait` when the store is the + catalog one. +- `DriveHandle` gains `lease_id: str` and `fencing_token: int`. `drive_id` + stays immutable (existing test pins this). +- New `verify(handle)` → raises `StaleLeaseError` (new typed error in + `openblade/domain/errors.py`, subclass of the existing domain error base so + the jobs error sanitizer treats it as typed) when the lease is not live or + the token differs. +- New `heartbeat(handles)`. + +### 3. Fencing at the destructive edges + +In `openblade/jobs/sharded_archive.py`, `archive.py`, `restore.py`, +`tree_restore.py`: call `scheduler.verify(handle)` immediately before +`ltfs.mount(..., READ_WRITE)`, before each shard/file write batch, and before +unmount/unload. A stale lease aborts the job with `StaleLeaseError`; the error +path must still attempt the clean unmount/unload **and must not suppress its +errors** (existing rule). Call `heartbeat` after each file/shard write. + +### 4. One scheduler per process, bound to the catalog + +- `AppContext` (`openblade/bootstrap.py`) gets a `lease_store: LeaseStore` + built from the catalog repository. The six call sites above construct + `DriveScheduler(num_drives=..., store=context.lease_store, job_id=job.id)`. + The CLI paths that have no catalog job yet must create one first (they + already have `context.catalog`). +- Delete nothing else; `JobQueue` (in-memory, `bootstrap.py:607`) is not in + scope — note it in the PR as follow-up. + +### 5. Recovery on startup (item 8, recovery half only) + +New `openblade/jobs/recovery.py::recover_after_restart(catalog, library) -> RecoveryReport`, +called from bootstrap after `init_db` and before the API/CLI serves anything: + +1. Every catalog job in state `running` → `failed_recoverable` with error + `"interrupted by process restart; physical state unknown — reconcile before retry"`. +2. Every live lease whose job is not `running` → released (`released_at = now`). +3. For each lease released in step 2, compare expected (`barcode` in + `physical`) with `library.inventory()`; log one line per mismatch at WARNING + and include it in the report. **Never** move media here — report only. +4. Report is logged at INFO and returned; expose it read-only as + `GET /jobs/recovery` (native surface, bearer-gated like the other native + routes; 404 under `OPENBLADE_SCALAR_API_ONLY`). + +Resuming a job from staged shards is **deferred** (needs the STAGING / +VERIFYING instance states from item 4) — say so in the PR, do not stub it. + +## Tests (each must fail with the guard removed — mutation-check the first two) + +- `tests/unit/test_drive_leases.py`: two schedulers over the same catalog + store cannot both acquire drive 0; second waits then `DriveBusyError`. + Fencing tokens strictly increase; a released lease's token is never reissued. +- `tests/unit/test_lease_fencing.py`: expire/release a lease behind a running + sharded archive (monkeypatch the store) → `StaleLeaseError`, no + `mark_instance_archived`, unmount/unload still attempted. +- `tests/unit/test_recovery.py`: seed a `running` job + live lease, restart → + job `failed_recoverable`, lease released, mismatch reported when the + simulator's drive is empty. +- Existing `tests/unit/test_scheduler*.py` / `test_sharded_archive_atomicity.py` + stay green with `InMemoryLeaseStore`. +- `tests/integration/test_api.py` (or the nearest native route suite): two + overlapping archive requests against one catalog serialise on the lease, and + `GET /jobs/recovery` is bearer-gated and 404 in scalar-api-only mode. + +## Non-goals + +Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any +`/aml/*` or `/iblade/*` route, the emulator contract, Postgres. + +## Definition of done + +`make lint` clean, `.venv/bin/mypy` strict clean (CI blocks on both), +`tests/unit tests/integration tests/safety` green, frontend untouched, reviewer +subagent pass, one commit per logical change (models+repo, scheduler, fencing +at call sites, recovery+route, docs). From 6276db4fe94aca297f2985a62cd5d2458411bd0c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:36:34 +0000 Subject: [PATCH 002/205] feat(catalog): drive_leases table and repository lease API Adds DriveLeaseRecord (new table via create_all), the frozen DriveLease domain dataclass, and CatalogRepository acquire/heartbeat/release/ lease_is_live/live_leases/release_leases_for_job. One live lease per drive and the monotonic fencing token are enforced under SQLite BEGIN IMMEDIATE (verified against installed SQLAlchemy 2.1.4/pysqlite). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/models.py | 22 +++++ openblade/catalog/repository.py | 152 +++++++++++++++++++++++++++++++- openblade/domain/models.py | 16 ++++ 3 files changed, 188 insertions(+), 2 deletions(-) diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index cf51f58..93ac005 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -449,3 +449,25 @@ class NasRestoreJob(Base): created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) completed_at: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class DriveLeaseRecord(Base): + """Catalog-persisted drive lease (docs/decisions/2026-10-09-persistent-drive-leases.md). + + Live = ``released_at IS NULL AND expires_at > now``. "One live lease per + drive" is enforced in the repository under ``BEGIN IMMEDIATE``, not by a + partial unique index. Timestamps are naive UTC, like the rest of the catalog. + """ + + __tablename__ = "drive_leases" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + drive_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + job_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + barcode: Mapped[str] = mapped_column(String(8), nullable=False) + physical_drive_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + fencing_token: Mapped[int] = mapped_column(Integer, nullable=False, unique=True) + acquired_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + heartbeat_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + released_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 3126075..4f8152c 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -6,7 +6,7 @@ import threading from collections.abc import Mapping, Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime, timedelta from pathlib import PurePosixPath from typing import Any, Protocol, cast, runtime_checkable from uuid import uuid4 @@ -17,6 +17,7 @@ from openblade.catalog.models import ( Cartridge, CatalogRebuildRun, + DriveLeaseRecord, FileInstance, FileRecord, Job, @@ -40,7 +41,7 @@ VolumeGroup, ) from openblade.domain.errors import FileNotFoundError -from openblade.domain.models import FileInstanceState +from openblade.domain.models import DriveLease, FileInstanceState from openblade.domain.policies import SafetyToken from openblade.nas.types import ( CacheDriveConfig, @@ -1771,3 +1772,150 @@ def delete_safety_token(self, token_value: str) -> None: if row is not None: self.session.delete(row) self.session.commit() + + # ── Drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md) ── + + def _begin_immediate(self) -> None: + """Take SQLite's write lock up front so check-then-insert is atomic. + + Verified against installed SQLAlchemy 2.1.4 / pysqlite: with the default + isolation handling an explicit ``BEGIN IMMEDIATE`` on the session's + connection is honoured and ``session.commit()`` commits it. Any pending + session transaction is committed first (every repository write commits + anyway) because SQLite rejects a nested BEGIN. + """ + self.session.commit() + self.session.connection().exec_driver_sql("BEGIN IMMEDIATE") + + def acquire_drive_leases( + self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + ) -> list[DriveLease] | None: + """All-or-nothing: one live lease per barcode, or None if not enough free drives.""" + with self._lock: + self._begin_immediate() + try: + now = _lease_now() + busy = set( + self.session.scalars( + select(DriveLeaseRecord.drive_id).where( + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ) + ) + free = [d for d in range(num_drives) if d not in busy] + if len(free) < len(barcodes): + self.session.rollback() + return None + token = int( + self.session.scalar( + select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0)) + ) + or 0 + ) + rows: list[DriveLeaseRecord] = [] + for drive_id, barcode in zip(free, barcodes, strict=False): + token += 1 + row = DriveLeaseRecord( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + fencing_token=token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + self.session.add(row) + rows.append(row) + self.session.commit() + except BaseException: + self.session.rollback() + raise + return [_lease_from_row(row) for row in rows] + + def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None: + with self._lock: + now = _lease_now() + for row in self._live_lease_rows(lease_ids, now): + row.heartbeat_at = now + row.expires_at = now + ttl + self.session.commit() + + def release_leases(self, lease_ids: list[str]) -> None: + with self._lock: + now = _lease_now() + for row in self.session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.id.in_(lease_ids), DriveLeaseRecord.released_at.is_(None) + ) + ): + row.released_at = now + self.session.commit() + + def lease_is_live(self, lease_id: str, fencing_token: int) -> bool: + with self._lock: + row = self.session.get(DriveLeaseRecord, lease_id, populate_existing=True) + return ( + row is not None + and row.fencing_token == fencing_token + and row.released_at is None + and row.expires_at > _lease_now() + ) + + def live_leases(self) -> list[DriveLease]: + with self._lock: + now = _lease_now() + rows = self.session.scalars( + select(DriveLeaseRecord) + .where(DriveLeaseRecord.released_at.is_(None), DriveLeaseRecord.expires_at > now) + .order_by(DriveLeaseRecord.fencing_token) + .execution_options(populate_existing=True) + ) + return [_lease_from_row(row) for row in rows] + + def release_leases_for_job(self, job_id: str) -> int: + with self._lock: + now = _lease_now() + rows = list( + self.session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.job_id == job_id, DriveLeaseRecord.released_at.is_(None) + ) + ) + ) + for row in rows: + row.released_at = now + self.session.commit() + return len(rows) + + def _live_lease_rows(self, lease_ids: list[str], now: datetime) -> list[DriveLeaseRecord]: + return list( + self.session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.id.in_(lease_ids), + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ) + ) + + +def _lease_now() -> datetime: + """Naive UTC, matching how the catalog's DateTime columns store values.""" + return datetime.now(UTC).replace(tzinfo=None) + + +def _lease_from_row(row: DriveLeaseRecord) -> DriveLease: + return DriveLease( + id=row.id, + drive_id=row.drive_id, + job_id=row.job_id, + barcode=row.barcode, + physical_drive_id=row.physical_drive_id, + fencing_token=row.fencing_token, + acquired_at=row.acquired_at, + heartbeat_at=row.heartbeat_at, + expires_at=row.expires_at, + released_at=row.released_at, + ) diff --git a/openblade/domain/models.py b/openblade/domain/models.py index 9220239..d5d1dcf 100644 --- a/openblade/domain/models.py +++ b/openblade/domain/models.py @@ -174,3 +174,19 @@ class OperationResult: success: bool message: str details: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class DriveLease: + """A catalog-backed reservation of one scheduler drive slot for one job.""" + + id: str + drive_id: int + job_id: str + barcode: str + physical_drive_id: int | None + fencing_token: int + acquired_at: datetime + heartbeat_at: datetime + expires_at: datetime + released_at: datetime | None = None From 605978e92817f9bd02a9fb2f5ddb46afcd8e0446 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:36:34 +0000 Subject: [PATCH 003/205] feat(jobs): DriveScheduler over a LeaseStore, with fencing verify/heartbeat DriveScheduler keeps its public shape but delegates to a LeaseStore protocol: InMemoryLeaseStore (default; existing behaviour for unit tests) and CatalogLeaseStore (shared across processes via the catalog, polled at 0.25 s). DriveHandle gains lease_id/fencing_token; verify() raises the new typed StaleLeaseError, heartbeat() extends the ttl. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/errors.py | 4 + openblade/jobs/scheduler.py | 222 ++++++++++++++++++++++++++++---- tests/unit/test_drive_leases.py | 72 +++++++++++ 3 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_drive_leases.py diff --git a/openblade/domain/errors.py b/openblade/domain/errors.py index 65b4ed8..1d2f651 100644 --- a/openblade/domain/errors.py +++ b/openblade/domain/errors.py @@ -50,6 +50,10 @@ class DriveBusyError(OpenBladeError): """Drive is busy.""" +class StaleLeaseError(OpenBladeError): + """A drive lease expired, was released, or was superseded (fencing token mismatch).""" + + class ChangerBusyError(OpenBladeError): """Changer is busy with another operation.""" diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 1f13d89..c9e5016 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -4,9 +4,16 @@ import logging import threading -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Protocol +from uuid import uuid4 -from openblade.domain.errors import DriveBusyError +from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.models import DriveLease + +if TYPE_CHECKING: + from openblade.catalog.repository import CatalogRepository logger = logging.getLogger(__name__) @@ -22,6 +29,9 @@ class DriveHandle: # one (e.g. the tape was already loaded elsewhere). Physical tape ops target # `physical`; the scheduler lock still tracks `drive_id`. physical_drive_id: int | None = None + # Lease identity + fencing token (docs/decisions/2026-10-09-persistent-drive-leases.md). + lease_id: str = "" + fencing_token: int = 0 _released: bool = field(default=False, init=False, repr=False) @property @@ -30,20 +40,146 @@ def physical(self) -> int: return self.physical_drive_id if self.physical_drive_id is not None else self.drive_id +DEFAULT_LEASE_TTL = timedelta(minutes=15) + + +class LeaseStore(Protocol): + """Where drive leases live. In memory for unit tests, the catalog in production.""" + + poll_interval: float + + def acquire( + self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + ) -> list[DriveLease] | None: ... + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: ... + + def release(self, lease_ids: list[str]) -> None: ... + + def is_live(self, lease_id: str, fencing_token: int) -> bool: ... + + def live_leases(self) -> list[DriveLease]: ... + + +class InMemoryLeaseStore: + """Process-local lease store (the pre-catalog behaviour).""" + + poll_interval = 1.0 + + def __init__(self) -> None: + self._lock = threading.Lock() + self._leases: dict[str, DriveLease] = {} + self._last_token = 0 + + def acquire( + self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + ) -> list[DriveLease] | None: + with self._lock: + now = _now() + busy = {lease.drive_id for lease in self._leases.values() if _live(lease, now)} + free = [d for d in range(num_drives) if d not in busy] + if len(free) < len(barcodes): + return None + acquired: list[DriveLease] = [] + for drive_id, barcode in zip(free, barcodes, strict=False): + self._last_token += 1 + lease = DriveLease( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + physical_drive_id=None, + fencing_token=self._last_token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + self._leases[lease.id] = lease + acquired.append(lease) + return acquired + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: + with self._lock: + now = _now() + for lease_id in lease_ids: + lease = self._leases.get(lease_id) + if lease is not None and _live(lease, now): + self._leases[lease_id] = replace(lease, heartbeat_at=now, expires_at=now + ttl) + + def release(self, lease_ids: list[str]) -> None: + with self._lock: + now = _now() + for lease_id in lease_ids: + lease = self._leases.get(lease_id) + if lease is not None and lease.released_at is None: + self._leases[lease_id] = replace(lease, released_at=now) + + def is_live(self, lease_id: str, fencing_token: int) -> bool: + with self._lock: + lease = self._leases.get(lease_id) + return ( + lease is not None and lease.fencing_token == fencing_token and _live(lease, _now()) + ) + + def live_leases(self) -> list[DriveLease]: + with self._lock: + now = _now() + return [lease for lease in self._leases.values() if _live(lease, now)] + + +class CatalogLeaseStore: + """Lease store shared by every process using the same catalog.""" + + poll_interval = 0.25 + + def __init__(self, repository: CatalogRepository) -> None: + self._repo = repository + + def acquire( + self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + ) -> list[DriveLease] | None: + return self._repo.acquire_drive_leases( + job_id=job_id, barcodes=barcodes, num_drives=num_drives, ttl=ttl + ) + + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: + self._repo.heartbeat_leases(lease_ids, ttl) + + def release(self, lease_ids: list[str]) -> None: + self._repo.release_leases(lease_ids) + + def is_live(self, lease_id: str, fencing_token: int) -> bool: + return self._repo.lease_is_live(lease_id, fencing_token) + + def live_leases(self) -> list[DriveLease]: + return self._repo.live_leases() + + class DriveScheduler: """ Atomically allocates 1..N drives for parallel tape operations. Rules: - - A drive can be held by at most one job at a time. + - A drive can be held by at most one job at a time (across every scheduler + sharing the same lease store). - acquire_drives() waits (with timeout) until all requested drives are free. - - release_drives() marks drives as free and notifies waiting jobs. + - release_drives() releases the leases and wakes local waiters. + - verify() raises StaleLeaseError once a lease is no longer live (fencing). """ - def __init__(self, num_drives: int) -> None: + def __init__( + self, + num_drives: int, + *, + store: LeaseStore | None = None, + job_id: str = "local", + ttl: timedelta = DEFAULT_LEASE_TTL, + ) -> None: self._num_drives = num_drives + self._store: LeaseStore = store if store is not None else InMemoryLeaseStore() + self._job_id = job_id + self._ttl = ttl self._lock = threading.Condition(threading.Lock()) - self._held: dict[int, str | None] = {i: None for i in range(num_drives)} @property def num_drives(self) -> int: @@ -51,8 +187,8 @@ def num_drives(self) -> int: def available_count(self) -> int: """Number of currently free drives.""" - with self._lock: - return sum(1 for value in self._held.values() if value is None) + held = {lease.drive_id for lease in self._store.live_leases()} + return sum(1 for drive_id in range(self._num_drives) if drive_id not in held) def acquire_drives( self, @@ -77,39 +213,73 @@ def acquire_drives( with self._lock: deadline = _monotonic() + timeout while True: - free_drives = [drive_id for drive_id, value in self._held.items() if value is None] - if len(free_drives) >= len(barcodes): + leases = self._store.acquire( + job_id=self._job_id, + barcodes=barcodes, + num_drives=self._num_drives, + ttl=self._ttl, + ) + if leases is not None: break remaining = deadline - _monotonic() if remaining <= 0: raise DriveBusyError(f"Timed out waiting for {len(barcodes)} free drives") - self._lock.wait(timeout=min(remaining, 1.0)) - - handles: list[DriveHandle] = [] - free_iter = iter(free_drives) - for barcode in barcodes: - drive_id = next(free_iter) - self._held[drive_id] = barcode - handles.append(DriveHandle(drive_id=drive_id, barcode=barcode)) - logger.info("Allocated drive %d for barcode %s", drive_id, barcode) + self._lock.wait(timeout=min(remaining, self._store.poll_interval)) - return handles + handles: list[DriveHandle] = [] + for lease in leases: + handles.append( + DriveHandle( + drive_id=lease.drive_id, + barcode=lease.barcode, + lease_id=lease.id, + fencing_token=lease.fencing_token, + ) + ) + logger.info( + "Allocated drive %d for barcode %s (token %d)", + lease.drive_id, + lease.barcode, + lease.fencing_token, + ) + return handles def release_drives(self, handles: list[DriveHandle]) -> None: """Release drives and notify waiting jobs.""" + pending = [handle for handle in handles if not handle._released] + self._store.release([handle.lease_id for handle in pending]) with self._lock: - for handle in handles: - if handle._released: - continue - self._held[handle.drive_id] = None + for handle in pending: handle._released = True logger.info("Released drive %d (was %s)", handle.drive_id, handle.barcode) self._lock.notify_all() + def verify(self, handle: DriveHandle) -> None: + """Raise StaleLeaseError unless the handle's lease is still live with its token.""" + if handle._released or not self._store.is_live(handle.lease_id, handle.fencing_token): + raise StaleLeaseError( + f"Lease on drive {handle.drive_id} for {handle.barcode} is no longer live" + ) + + def heartbeat(self, handles: list[DriveHandle]) -> None: + """Extend the leases behind these handles by the scheduler's ttl.""" + self._store.heartbeat([handle.lease_id for handle in handles], self._ttl) + def status(self) -> dict[int, str | None]: """Return copy of drive allocation status.""" - with self._lock: - return dict(self._held) + status: dict[int, str | None] = {i: None for i in range(self._num_drives)} + for lease in self._store.live_leases(): + if lease.drive_id in status: + status[lease.drive_id] = lease.barcode + return status + + +def _now() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +def _live(lease: DriveLease, now: datetime) -> bool: + return lease.released_at is None and lease.expires_at > now def _monotonic() -> float: diff --git a/tests/unit/test_drive_leases.py b/tests/unit/test_drive_leases.py new file mode 100644 index 0000000..438fe3e --- /dev/null +++ b/tests/unit/test_drive_leases.py @@ -0,0 +1,72 @@ +"""Catalog-backed drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md).""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler + + +def _two_repos(tmp_path: Path) -> tuple[CatalogRepository, CatalogRepository]: + # A file DB with two independent sessions models two processes (CLI + API) + # sharing one catalog; an in-memory StaticPool DB would share one connection. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()), CatalogRepository(get_session()) + + +def test_two_schedulers_on_one_catalog_cannot_both_hold_drive_0(tmp_path: Path) -> None: + repo_a, repo_b = _two_repos(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="job-a") + sched_b = DriveScheduler(1, store=CatalogLeaseStore(repo_b), job_id="job-b") + + held = sched_a.acquire_drives(["AAA001L9"], timeout=1.0) + assert held[0].drive_id == 0 + + with pytest.raises(DriveBusyError): + sched_b.acquire_drives(["BBB001L9"], timeout=0.6) + assert sched_b.status() == {0: "AAA001L9"} + + sched_a.release_drives(held) + got = sched_b.acquire_drives(["BBB001L9"], timeout=1.0) + assert got[0].drive_id == 0 + assert got[0].fencing_token > held[0].fencing_token + + +def test_fencing_tokens_strictly_increase_and_are_never_reissued(tmp_path: Path) -> None: + repo, _ = _two_repos(tmp_path) + sched = DriveScheduler(2, store=CatalogLeaseStore(repo), job_id="job-1") + + seen: list[int] = [] + for _ in range(3): + handles = sched.acquire_drives(["AAA001L9", "AAA002L9"], timeout=1.0) + seen.extend(h.fencing_token for h in handles) + sched.release_drives(handles) + + assert seen == sorted(seen) + assert len(set(seen)) == len(seen) + + +def test_verify_raises_stale_lease_after_release_or_expiry(tmp_path: Path) -> None: + repo, other = _two_repos(tmp_path) + sched = DriveScheduler(2, store=CatalogLeaseStore(repo), job_id="job-1") + (handle,) = sched.acquire_drives(["AAA001L9"], timeout=1.0) + sched.verify(handle) # live + sched.heartbeat([handle]) + sched.verify(handle) + + assert other.release_leases_for_job("job-1") == 1 + with pytest.raises(StaleLeaseError): + sched.verify(handle) + + expired = DriveScheduler( + 2, store=CatalogLeaseStore(repo), job_id="job-2", ttl=timedelta(seconds=-1) + ) + (dead,) = expired.acquire_drives(["AAA002L9"], timeout=1.0) + with pytest.raises(StaleLeaseError): + expired.verify(dead) From 650279331e4b0af8ed0c6f9e3a1be5cf4cf96057 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:40:05 +0000 Subject: [PATCH 004/205] feat(jobs): fence sharded archive on drive leases (partial slice of step 3) verify() before each READ_WRITE mount (both stripe paths), before the shard write batch, and before unmount/unload+commit; heartbeat once per batch after the writes. StaleLeaseError aborts the job (re-raised past the batch's generic handler); the existing finally still unmounts and unloads. archive.py/restore.py/tree_restore.py, the six construction sites and AppContext wiring are NOT done yet. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/sharded_archive.py | 12 ++++++- .../test_sharded_archive_atomicity.py | 36 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 9ee07b8..70c1537 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -14,7 +14,7 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.capacity import has_room_for -from openblade.domain.errors import TapeFullError, safe_job_error +from openblade.domain.errors import StaleLeaseError, TapeFullError, safe_job_error from openblade.domain.models import MountHandle, MountMode from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import ( @@ -352,6 +352,7 @@ def _archive_stripe( for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) def _write_one( @@ -368,6 +369,8 @@ def _write_one( raise ValueError(f"Checksum mismatch: {source_file.name}") return source_file, barcode, tape_path, checksum, source_file.stat().st_size + for handle in handles: + scheduler.verify(handle) # fencing: before the shard write batch with concurrent.futures.ThreadPoolExecutor(max_workers=len(batch)) as pool: futures = [ pool.submit(_write_one, source_file, barcode, mounts[barcode]) @@ -413,6 +416,9 @@ def _write_one( # Every shard in the batch wrote and checksum-verified. Cleanly unmount and # unload BEFORE commit; a failed unmount raises and blocks the commit. + scheduler.heartbeat(handles) + for handle in handles: + scheduler.verify(handle) # fencing: before unmount/unload + commit _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) # COMMIT: only now is the batch durable. @@ -421,6 +427,9 @@ def _write_one( catalog.mark_instance_archived(shard_instance_id) files_archived += 1 bytes_archived += size_bytes + except StaleLeaseError: + # Fenced out: abort the whole job. `finally` still unmounts/unloads. + raise except Exception as exc: # noqa: BLE001 # Staged instances remain PENDING -> not exposed as archived; resumable. # This branch had no logging at all, so a batch that failed left no @@ -488,6 +497,7 @@ def _archive_block_stripe( for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) shard_tmp_files: list[Path] = [Path()] * len(plan.shards) diff --git a/tests/integration/test_sharded_archive_atomicity.py b/tests/integration/test_sharded_archive_atomicity.py index e2814a7..2aa57f1 100644 --- a/tests/integration/test_sharded_archive_atomicity.py +++ b/tests/integration/test_sharded_archive_atomicity.py @@ -9,13 +9,15 @@ from pathlib import Path +import pytest from sqlalchemy import func, select from openblade.catalog.db import get_session, init_db from openblade.catalog.models import FileInstance from openblade.catalog.repository import _ARCHIVED_INSTANCE_STATES, CatalogRepository +from openblade.domain.errors import StaleLeaseError from openblade.domain.policies import FormatConfirmation, SafetyToken -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore from openblade.jobs.shard import ShardMode from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive from openblade.simulator.library import MockLibraryBackend @@ -168,3 +170,35 @@ def test_unload_failure_blocks_commit(tmp_path: Path, monkeypatch) -> None: assert result.errors # stuck unload surfaced, not swallowed assert result.files_archived == 0 assert _archived_count(catalog) == 0 # tape stuck in drive -> batch must not be durable + + +def test_stale_lease_aborts_archive_commits_nothing_and_still_unmounts( + tmp_path: Path, monkeypatch +) -> None: + # Spec docs/decisions/2026-10-09-persistent-drive-leases.md §3: a lease + # released behind a running job fences it out before unmount + commit. + library, ltfs = _setup() + catalog = _catalog() + store = InMemoryLeaseStore() + scheduler = DriveScheduler(num_drives=2, store=store) + real_write, real_unmount = ltfs.write_file, ltfs.unmount + unmounted: list[object] = [] + + def write_then_lose_lease(*args, **kwargs): + result = real_write(*args, **kwargs) + store.release([lease.id for lease in store.live_leases()]) + return result + + def tracking_unmount(mount): + unmounted.append(mount) + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "write_file", write_then_lose_lease) + monkeypatch.setattr(ltfs, "unmount", tracking_unmount) + job = catalog.create_job("archive", {}) + + with pytest.raises(StaleLeaseError): + run_sharded_archive(_request(_source(tmp_path)), library, ltfs, catalog, scheduler, job.id) + + assert _archived_count(catalog) == 0 # no mark_instance_archived + assert unmounted # clean unmount still attempted on the error path From f36462314c1ef38938f68c094650c02a0d2ea031 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:48:12 +0000 Subject: [PATCH 005/205] feat(jobs): bind every scheduler to the catalog lease store; fence block-stripe writes - AppContext gains lease_store (CatalogLeaseStore over context.catalog). - The six DriveScheduler construction sites (archive/restore routes, CLI restore, restore-tree, archive-sharded) pass store= and job_id=; the CLI sites now build the scheduler after their catalog job exists. The tree restore route's worker thread uses a CatalogLeaseStore over its own session (the context session is not thread-safe). - sharded_archive block-stripe path: verify before the shard write batch and before unmount/unload, heartbeat after writes. - store stays optional on DriveScheduler: ~40 test constructions omit it. - Integration test: a lease held through a second catalog session makes POST /archive/sharded wait until it is released (mutation-checked). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_archive.py | 6 +- openblade/api/routes_restore.py | 20 ++++- openblade/bootstrap.py | 3 + openblade/cli/main.py | 18 +++- openblade/jobs/sharded_archive.py | 5 ++ .../test_drive_lease_serialisation.py | 85 +++++++++++++++++++ 6 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_drive_lease_serialisation.py diff --git a/openblade/api/routes_archive.py b/openblade/api/routes_archive.py index fef50b7..aa027be 100644 --- a/openblade/api/routes_archive.py +++ b/openblade/api/routes_archive.py @@ -193,7 +193,11 @@ async def enqueue_sharded_archive( "block_size_mb": request.block_size_mb, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) try: run_sharded_archive( ShardedArchiveRequest( diff --git a/openblade/api/routes_restore.py b/openblade/api/routes_restore.py index 47c1ec1..388b5c2 100644 --- a/openblade/api/routes_restore.py +++ b/openblade/api/routes_restore.py @@ -19,7 +19,7 @@ from openblade.domain.errors import OpenBladeError, safe_job_error from openblade.jobs.restore import RestoreRequest as RestoreJobRequest from openblade.jobs.restore import run_restore_job -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler from openblade.jobs.sharded_restore import ShardedRestoreRequest, run_sharded_restore from openblade.jobs.tree_restore import TreeRestoreRequest, TreeRestoreResult, run_tree_restore @@ -156,7 +156,11 @@ async def enqueue_restore( ) try: if use_sharded_restore: - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) run_sharded_restore( ShardedRestoreRequest(catalog_path=catalog_path, dest_path=Path(request.dest_path)), context.library, @@ -231,7 +235,7 @@ async def restore_tree( "dry_run": request.dry_run, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + num_drives = len(context.library.inventory().drives) tree_request = TreeRestoreRequest( catalog_prefix=request.catalog_prefix, dest_dir=Path(request.dest_dir), @@ -246,12 +250,20 @@ def _run() -> TreeRestoreResult: # target drive 0) and two callers fight over one drive. db_session = get_session() try: + worker_catalog = CatalogRepository(db_session) + # Lease store over the worker's own session: the context's store + # wraps the process-global Session, which is not thread-safe. + scheduler = DriveScheduler( + num_drives=num_drives, + store=CatalogLeaseStore(worker_catalog), + job_id=job.id, + ) with _ARCHIVE_REQUEST_LOCK: return run_tree_restore( tree_request, context.library, context.ltfs, - CatalogRepository(db_session), + worker_catalog, scheduler, job.id, ) diff --git a/openblade/bootstrap.py b/openblade/bootstrap.py index 82c487e..9209b96 100644 --- a/openblade/bootstrap.py +++ b/openblade/bootstrap.py @@ -26,6 +26,7 @@ from openblade.jobs.inventory import InventoryService, run_inventory_job from openblade.jobs.queue import JobQueue from openblade.jobs.restore import RestoreService +from openblade.jobs.scheduler import CatalogLeaseStore, LeaseStore from openblade.jobs.worker import Worker from openblade.nas.service import NasService from openblade.nas.types import ( @@ -403,6 +404,7 @@ class AppContext: library: LibraryBackend ltfs: LTFSBackend catalog: CatalogRepository + lease_store: LeaseStore queue: JobQueue worker: Worker inventory_service: InventoryService @@ -611,6 +613,7 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: library=library, ltfs=ltfs, catalog=catalog, + lease_store=CatalogLeaseStore(catalog), queue=queue, worker=worker, inventory_service=InventoryService(library), diff --git a/openblade/cli/main.py b/openblade/cli/main.py index 4173d28..ca9aa63 100644 --- a/openblade/cli/main.py +++ b/openblade/cli/main.py @@ -621,13 +621,17 @@ def restore_file( # Checked BEFORE the job row is created, so a typo does not leave an # orphaned `queued` restore job behind. raise typer.BadParameter(f"{catalog_path} is not in the catalog") - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) treat_as_dir = into_dir or dest.is_dir() dest_path = dest / PurePosixPath(catalog_path).name if treat_as_dir else dest dest_path.parent.mkdir(parents=True, exist_ok=True) job = context.catalog.create_job( "restore", {"catalog_path": catalog_path, "dest_path": str(dest_path)} ) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) sharded = bool(context.catalog.list_shard_records(record.id)) or (record.shard_count or 1) > 1 err_console.print( f"Restoring {catalog_path} -> {dest_path} ({'sharded' if sharded else 'single-instance'})" @@ -695,11 +699,15 @@ def restore_tree( to stdout, and any failure exits non-zero with the curated error. """ context = _get_context() - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) job = context.catalog.create_job( "restore", {"catalog_prefix": catalog_prefix, "dest_dir": str(dest), "bulk": True}, ) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) def _progress(tick: TreeRestoreProgress) -> None: marker = "[red]FAIL[/red]" if tick.failed else "ok" @@ -835,7 +843,11 @@ def archive_sharded( "block_size_mb": block_size_mb, }, ) - scheduler = DriveScheduler(num_drives=len(context.library.inventory().drives)) + scheduler = DriveScheduler( + num_drives=len(context.library.inventory().drives), + store=context.lease_store, + job_id=job.id, + ) err_console.print( f"Sharded archive {source} -> {volume_group} " f"mode={shard_mode.value} lanes={','.join(lane_barcodes)}" diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 70c1537..957c7a9 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -527,6 +527,8 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: raise ValueError(f"Shard {spec.shard_index} checksum mismatch") return spec.shard_index, checksum, shard_tmp.stat().st_size + for handle in handles: + scheduler.verify(handle) # fencing: before the shard write batch shard_checksums: list[str] = [""] * len(plan.shards) shard_sizes: list[int] = [0] * len(plan.shards) with concurrent.futures.ThreadPoolExecutor(max_workers=len(plan.shards)) as pool: @@ -577,6 +579,9 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: ) staged.extend((instance.id, shard_instance.id)) + scheduler.heartbeat(handles) + for handle in handles: + scheduler.verify(handle) # fencing: before unmount/unload + commit # Clean unmount/unload BEFORE commit; a failed unmount raises and blocks it. _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) diff --git a/tests/integration/test_drive_lease_serialisation.py b/tests/integration/test_drive_lease_serialisation.py new file mode 100644 index 0000000..c81bb97 --- /dev/null +++ b/tests/integration/test_drive_lease_serialisation.py @@ -0,0 +1,85 @@ +"""Two overlapping archive requests against one catalog serialise on the drive lease. + +The archive route's in-process lock cannot see a second process (the CLI, another +worker). The overlap here is a lease held through a *separate* catalog session on +the same SQLite file -- exactly what another process would hold -- while +``POST /archive/sharded`` runs. The request must wait for that lease, not drive +the same tape concurrently. +""" + +from __future__ import annotations + +import threading +from datetime import timedelta +from pathlib import Path + +from fastapi.testclient import TestClient +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from openblade.api.main import app +from openblade.catalog.db import get_session +from openblade.catalog.models import DriveLeaseRecord +from openblade.catalog.repository import CatalogRepository +from openblade.jobs.scheduler import CatalogLeaseStore +from tests.integration.test_sharded_archive_cli_api_parity import ( + LANES, + build_context, + seed_source, +) + +client = TestClient(app) + + +def test_overlapping_archive_waits_for_a_lease_held_by_another_session(tmp_path: Path) -> None: + context = build_context(tmp_path / "leases.db") + context.catalog.create_volume_group("shards") + source = tmp_path / "src" + seed_source(source) + + other_session = get_session() + try: + other = CatalogLeaseStore(CatalogRepository(other_session)) + held = other.acquire( + job_id="other-process", + barcodes=list(LANES), + num_drives=2, + ttl=timedelta(minutes=15), + ) + assert held is not None + + finished = threading.Event() + responses: list[int] = [] + + def _post() -> None: + response = client.post( + "/archive/sharded", + json={ + "source_path": str(source), + "volume_group": "shards", + "lane_barcodes": LANES, + "mode": "stripe", + }, + ) + responses.append(response.status_code) + finished.set() + + worker = threading.Thread(target=_post) + worker.start() + # While the other session holds both drives, the request must not finish. + assert not finished.wait(1.0), "archive ran while another job held the drives" + other.release([lease.id for lease in held]) + worker.join(timeout=60) + assert finished.is_set() + assert responses == [202] + + # The request ran under its own, strictly newer leases, now released. + assert other.live_leases() == [] + assert max(lease.fencing_token for lease in held) < _max_token(other_session) + finally: + other_session.close() + + +def _max_token(session: Session) -> int: + value = session.scalar(select(func.max(DriveLeaseRecord.fencing_token))) + return int(value or 0) From b4c32e7b26a540fe9e4500b56b840d56970c603c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:49:18 +0000 Subject: [PATCH 006/205] feat(jobs): recover_after_restart reconciles jobs and drive leases (report-only) Bootstrap hook and GET /jobs/recovery route not yet wired. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/recovery.py | 80 +++++++++++++++++++++++++++++++++++++ tests/unit/test_recovery.py | 35 ++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 openblade/jobs/recovery.py create mode 100644 tests/unit/test_recovery.py diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py new file mode 100644 index 0000000..92f0073 --- /dev/null +++ b/openblade/jobs/recovery.py @@ -0,0 +1,80 @@ +"""Startup recovery: reconcile catalog jobs and drive leases after a restart. + +Report-only for physical state: this module never moves media. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +from openblade.catalog.repository import CatalogRepository +from openblade.domain.backends import LibraryBackend + +logger = logging.getLogger(__name__) + +INTERRUPTED_ERROR = ( + "interrupted by process restart; physical state unknown — reconcile before retry" +) + + +@dataclass(frozen=True) +class DriveMismatch: + lease_id: str + job_id: str + drive_id: int + expected_barcode: str + observed_barcode: str | None + + +@dataclass(frozen=True) +class RecoveryReport: + interrupted_job_ids: list[str] = field(default_factory=list) + released_lease_ids: list[str] = field(default_factory=list) + mismatches: list[DriveMismatch] = field(default_factory=list) + + +def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) -> RecoveryReport: + """Fail interrupted jobs, release orphaned leases, and report drive mismatches.""" + interrupted: list[str] = [] + for job in catalog.list_jobs(state="running"): + catalog.update_job_state(job.id, "failed_recoverable", INTERRUPTED_ERROR) + interrupted.append(job.id) + + orphaned = [] + for lease in catalog.live_leases(): + owner = catalog.get_job(lease.job_id) + if owner is None or owner.state != "running": + orphaned.append(lease) + catalog.release_leases([lease.id for lease in orphaned]) + + observed: dict[int, str | None] = { + drive.drive_id: None if drive.barcode is None else str(drive.barcode) + for drive in library.inventory().drives + } + mismatches: list[DriveMismatch] = [] + for lease in orphaned: + physical = ( + lease.physical_drive_id if lease.physical_drive_id is not None else lease.drive_id + ) + seen = observed.get(physical) + if seen != lease.barcode: + mismatch = DriveMismatch(lease.id, lease.job_id, physical, lease.barcode, seen) + logger.warning( + "recovery: drive %s expected %s, observed %s (lease %s, job %s)", + physical, + lease.barcode, + seen, + lease.id, + lease.job_id, + ) + mismatches.append(mismatch) + + report = RecoveryReport(interrupted, [lease.id for lease in orphaned], mismatches) + logger.info( + "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es)", + len(report.interrupted_job_ids), + len(report.released_lease_ids), + len(report.mismatches), + ) + return report diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py new file mode 100644 index 0000000..d11f7e0 --- /dev/null +++ b/tests/unit/test_recovery.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.config import OpenBladeConfig +from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart +from openblade.simulator.library import MockLibraryBackend + + +def test_recover_after_restart_running_job_marked_failed_recoverable_and_lease_released( + tmp_path: Path, +) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + leases = catalog.acquire_drive_leases( + job_id=job.id, barcodes=["MCK00001"], num_drives=2, ttl=timedelta(minutes=15) + ) + assert leases is not None + empty_library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, empty_library) + + refreshed = catalog.get_job(job.id) + assert refreshed is not None + assert refreshed.state == "failed_recoverable" + assert refreshed.error == INTERRUPTED_ERROR + assert catalog.live_leases() == [] + assert report.released_lease_ids == [leases[0].id] + assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ + ("MCK00001", None) + ] From 4e67682fef6f4d5a38bfa37b0c84e1eaf9a6ec9f Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:52:08 +0000 Subject: [PATCH 007/205] feat(jobs): run restart recovery in create_context and expose it at GET /jobs/recovery Recovery runs right after the catalog opens, before anything can claim a drive. The report is read-only on the native surface: bearer-gated by the existing middleware (the auth sweep enumerates it) and 404 under OPENBLADE_SCALAR_API_ONLY. Wiki API reference regenerated for the new route. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/wiki/reference/api.md | 5 +++-- openblade/api/routes_jobs.py | 37 ++++++++++++++++++++++++++++++++++++ openblade/bootstrap.py | 6 ++++++ tests/unit/test_recovery.py | 26 +++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/docs/wiki/reference/api.md b/docs/wiki/reference/api.md index 115b313..9d24438 100644 --- a/docs/wiki/reference/api.md +++ b/docs/wiki/reference/api.md @@ -6,7 +6,7 @@ Introspected from the OpenAPI schema of `openblade.api.main:app`. -**1178 operations** total: 198 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. +**1179 operations** total: 199 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. The application serves two surfaces from one ASGI app. Setting `OPENBLADE_SCALAR_API_ONLY=true` puts it in emulator-only mode, where the @@ -83,11 +83,12 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/restore/` | Enqueue Restore | - | `openblade__api__routes_restore__RestoreRequest` | `EnqueuedJobResponse` | | `POST` | `/restore/tree` | Restore Tree | - | `TreeRestoreApiRequest` | `TreeRestoreResponse` | -### `jobs` (2 operations) +### `jobs` (3 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | | `GET` | `/jobs/` | List Jobs | `library_id`? (query) | - | `openblade__api__routes_jobs__JobResponse[]` | +| `GET` | `/jobs/recovery` | Get Recovery Report | - | - | `RecoveryReportResponse` | | `GET` | `/jobs/{job_id}` | Get Job | `job_id` (path) | - | `openblade__api__routes_jobs__JobResponse` | ### `catalog` (6 operations) diff --git a/openblade/api/routes_jobs.py b/openblade/api/routes_jobs.py index 4b0a217..9f1beed 100644 --- a/openblade/api/routes_jobs.py +++ b/openblade/api/routes_jobs.py @@ -45,6 +45,43 @@ async def list_jobs( return [job for job in jobs if job.library_id is None or job.library_id == library_id] +class DriveMismatchResponse(BaseModel): + lease_id: str + job_id: str + drive_id: int + expected_barcode: str + observed_barcode: str | None + + +class RecoveryReportResponse(BaseModel): + interrupted_job_ids: list[str] + released_lease_ids: list[str] + mismatches: list[DriveMismatchResponse] + + +# Declared before /{job_id} so the literal path wins route matching. +@router.get("/recovery", response_model=RecoveryReportResponse) +async def get_recovery_report( + context: AppContext = Depends(get_context), +) -> RecoveryReportResponse: + """What startup recovery did: interrupted jobs, released leases, drive mismatches.""" + report = context.recovery_report + return RecoveryReportResponse( + interrupted_job_ids=list(report.interrupted_job_ids), + released_lease_ids=list(report.released_lease_ids), + mismatches=[ + DriveMismatchResponse( + lease_id=m.lease_id, + job_id=m.job_id, + drive_id=m.drive_id, + expected_barcode=m.expected_barcode, + observed_barcode=m.observed_barcode, + ) + for m in report.mismatches + ], + ) + + @router.get("/{job_id}", response_model=JobResponse) async def get_job(job_id: str, context: AppContext = Depends(get_context)) -> JobResponse: job = context.catalog.get_job(job_id) diff --git a/openblade/bootstrap.py b/openblade/bootstrap.py index 9209b96..35c16d1 100644 --- a/openblade/bootstrap.py +++ b/openblade/bootstrap.py @@ -25,6 +25,7 @@ from openblade.jobs.format import FormatService from openblade.jobs.inventory import InventoryService, run_inventory_job from openblade.jobs.queue import JobQueue +from openblade.jobs.recovery import RecoveryReport, recover_after_restart from openblade.jobs.restore import RestoreService from openblade.jobs.scheduler import CatalogLeaseStore, LeaseStore from openblade.jobs.worker import Worker @@ -405,6 +406,7 @@ class AppContext: ltfs: LTFSBackend catalog: CatalogRepository lease_store: LeaseStore + recovery_report: RecoveryReport queue: JobQueue worker: Worker inventory_service: InventoryService @@ -595,6 +597,9 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: else: library, ltfs = _create_real_backends(active_config) catalog = CatalogRepository(get_session()) + # Before anything can claim a drive: fail jobs the previous process left + # running, release their leases, and report (never fix) drive mismatches. + recovery_report = recover_after_restart(catalog, library) _seed_nas_defaults(catalog) if active_config.backend == BackendMode.MOCK: _seed_library_defaults(catalog, active_config) @@ -614,6 +619,7 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: ltfs=ltfs, catalog=catalog, lease_store=CatalogLeaseStore(catalog), + recovery_report=recovery_report, queue=queue, worker=worker, inventory_service=InventoryService(library), diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py index d11f7e0..2694ae8 100644 --- a/tests/unit/test_recovery.py +++ b/tests/unit/test_recovery.py @@ -33,3 +33,29 @@ def test_recover_after_restart_running_job_marked_failed_recoverable_and_lease_r assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ ("MCK00001", None) ] + + +def test_restart_runs_recovery_and_exposes_it_read_only(tmp_path: Path) -> None: + # A job left `running` by a previous process is failed on the next start, + # and the report is visible on the native surface only. + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.bootstrap import reset_context + + db_url = f"sqlite:///{tmp_path / 'restart.db'}" + first = create_context(OpenBladeConfig(db_url=db_url)) + job = first.catalog.create_job("archive", {}) + first.catalog.update_job_state(job.id, "running") + + second = create_context(OpenBladeConfig(db_url=db_url)) + reset_context(second) + assert second.recovery_report.interrupted_job_ids == [job.id] + + response = TestClient(app).get("/jobs/recovery") + assert response.status_code == 200 + assert response.json()["interrupted_job_ids"] == [job.id] + + strict = create_context(OpenBladeConfig(db_url=db_url, scalar_api_only=True)) + reset_context(strict) + assert TestClient(app).get("/jobs/recovery").status_code == 404 From de4f999553f3eeda0d48e14adeeae3fd328e6527 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 06:56:18 +0000 Subject: [PATCH 008/205] docs: architecture section for drive leases/recovery; record implementation deviations in the spec Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/architecture.md | 3 +++ .../2026-10-09-persistent-drive-leases.md | 22 +++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 0353524..f452c15 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,5 +51,8 @@ The catalog uses SQLite tables for volume groups, barcode assignments, file reco ## Job system Jobs are stored in-process with explicit state transitions. The queue additionally tracks changer and drive ownership so concurrent operations cannot claim the same hardware resources. +## Drive leases and recovery +Drive exclusion for tape I/O is a catalog row, not process memory. `DriveScheduler` is a façade over a `LeaseStore`: `CatalogLeaseStore` in production (one per `AppContext`, shared by the API and the CLI through the same `drive_leases` table) and `InMemoryLeaseStore` for unit tests. Each lease carries a monotonic `fencing_token`; job modules call `scheduler.verify(handle)` before a read-write mount, a write batch, and unmount/unload, and `heartbeat` after writes, so a lease that expired or was released behind a running job aborts it with `StaleLeaseError` instead of writing to a drive it no longer owns. On startup `recover_after_restart` fails any job the previous process left `running`, releases its leases, and reports expected-vs-observed drive contents — it never moves media. The report is read-only at `GET /jobs/recovery`. Spec and rationale: `docs/decisions/2026-10-09-persistent-drive-leases.md`. + ## FUSE namespace The FUSE-oriented layer is intentionally thin: catalog entries define the namespace, and hydration delegates to restore workflows. This keeps the namespace authoritative and avoids bypassing safety or verification logic. diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index a95b833..244e376 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -6,7 +6,7 @@ Status: accepted 2026-10-09. Roadmap items 7 and 8 (Phase 2), first slice. | checkout | branch | relation to `origin/master` | |---|---|---| -| `/root/ob2-final` | `integrate/gap-closure` | tree-identical to master (`1863348`, PR #45) | +| `/root/ob2-final` | `feat/persistent-drive-leases` (was `integrate/gap-closure`) | branched from master (`1863348`, PR #45) for this work | | `/root/ob-final` | `integrate/eight-items` | tree-identical to PR #44 squash, already on master | | `/root/ob-int2` | `integrate/campaign-wiki-assistant` | fully contained in `ob-final` | | `/root/OpenBlade` | `chore/py312-baseline-and-quantum-refs` | stale (37 behind); its 3 stray commits are already on master by content | @@ -138,9 +138,9 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. - `tests/unit/test_drive_leases.py`: two schedulers over the same catalog store cannot both acquire drive 0; second waits then `DriveBusyError`. Fencing tokens strictly increase; a released lease's token is never reissued. -- `tests/unit/test_lease_fencing.py`: expire/release a lease behind a running - sharded archive (monkeypatch the store) → `StaleLeaseError`, no - `mark_instance_archived`, unmount/unload still attempted. +- Lease fencing (landed in `tests/integration/test_sharded_archive_atomicity.py` + to reuse its helpers): release a lease behind a running sharded archive → + `StaleLeaseError`, no `mark_instance_archived`, unmount/unload still attempted. - `tests/unit/test_recovery.py`: seed a `running` job + live lease, restart → job `failed_recoverable`, lease released, mismatch reported when the simulator's drive is empty. @@ -150,6 +150,20 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. overlapping archive requests against one catalog serialise on the lease, and `GET /jobs/recovery` is bearer-gated and 404 in scalar-api-only mode. +## Deviations recorded at implementation time + +- `DriveScheduler(store=...)` stays optional (defaults to `InMemoryLeaseStore`) + because ~40 existing tests construct it bare; every production call site + passes the catalog store. +- `archive.py` / `restore.py` (`run_archive_job` / `run_restore_job`) take no + scheduler and hold no lease today, so there is nothing to fence; giving them + leases is a follow-up. `tree_restore.py` only forwards the scheduler to the + sharded restore, which is fenced. +- The tree-restore route builds its scheduler in the worker thread over a + `CatalogLeaseStore(worker_catalog)` rather than `context.lease_store`: the + context's session is not thread-safe. Same table, same exclusion. +- Heartbeat runs once per write batch, not per shard. + ## Non-goals Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any From 096a2bf676d3a35caa56409b3ca2f1d06c92608f Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 07:16:07 +0000 Subject: [PATCH 009/205] fix(catalog): run lease operations on their own session, not the shared one A lease poll from a worker thread could commit or roll back another thread's pending rows on the shared Session. Lease ops now open a short-lived Session over the engine; on a file-backed catalog that is a separate connection, so BEGIN IMMEDIATE is a real cross-process lock. Adds expired_leases() and set_lease_physical_drive() for recovery. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 163 +++++++++++++++++--------------- 1 file changed, 87 insertions(+), 76 deletions(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 4f8152c..538d9ed 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -4,7 +4,8 @@ import json import threading -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import PurePosixPath @@ -1774,88 +1775,98 @@ def delete_safety_token(self, token_value: str) -> None: self.session.commit() # ── Drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md) ── - - def _begin_immediate(self) -> None: - """Take SQLite's write lock up front so check-then-insert is atomic. - - Verified against installed SQLAlchemy 2.1.4 / pysqlite: with the default - isolation handling an explicit ``BEGIN IMMEDIATE`` on the session's - connection is honoured and ``session.commit()`` commits it. Any pending - session transaction is committed first (every repository write commits - anyway) because SQLite rejects a nested BEGIN. - """ - self.session.commit() - self.session.connection().exec_driver_sql("BEGIN IMMEDIATE") + # + # Lease operations run on their OWN short-lived Session over the catalog's + # engine, never on the shared `self.session`: a lease poll from a worker + # thread must not be able to commit or roll back another thread's pending + # rows (review finding, 2026-10-09). On a file-backed catalog this is a + # separate connection, so `BEGIN IMMEDIATE` is a real cross-process lock. + + @contextmanager + def _lease_session(self) -> Iterator[Session]: + with Session(bind=self.session.get_bind(), expire_on_commit=False) as session: + yield session def acquire_drive_leases( self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta ) -> list[DriveLease] | None: """All-or-nothing: one live lease per barcode, or None if not enough free drives.""" - with self._lock: - self._begin_immediate() - try: - now = _lease_now() - busy = set( - self.session.scalars( - select(DriveLeaseRecord.drive_id).where( - DriveLeaseRecord.released_at.is_(None), - DriveLeaseRecord.expires_at > now, - ) + with self._lock, self._lease_session() as session: + # Verified against installed SQLAlchemy 2.1.4 / pysqlite: an explicit + # BEGIN IMMEDIATE on the session's connection is honoured and + # session.commit() commits it, so check-then-insert is atomic. + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + now = _lease_now() + busy = set( + session.scalars( + select(DriveLeaseRecord.drive_id).where( + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, ) ) - free = [d for d in range(num_drives) if d not in busy] - if len(free) < len(barcodes): - self.session.rollback() - return None - token = int( - self.session.scalar( - select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0)) - ) - or 0 + ) + free = [d for d in range(num_drives) if d not in busy] + if len(free) < len(barcodes): + session.rollback() + return None + token = int( + session.scalar(select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0))) + or 0 + ) + rows: list[DriveLeaseRecord] = [] + for drive_id, barcode in zip(free, barcodes, strict=False): + token += 1 + row = DriveLeaseRecord( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + fencing_token=token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, ) - rows: list[DriveLeaseRecord] = [] - for drive_id, barcode in zip(free, barcodes, strict=False): - token += 1 - row = DriveLeaseRecord( - id=str(uuid4()), - drive_id=drive_id, - job_id=job_id, - barcode=barcode, - fencing_token=token, - acquired_at=now, - heartbeat_at=now, - expires_at=now + ttl, - ) - self.session.add(row) - rows.append(row) - self.session.commit() - except BaseException: - self.session.rollback() - raise + session.add(row) + rows.append(row) + session.commit() return [_lease_from_row(row) for row in rows] def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None: - with self._lock: + with self._lock, self._lease_session() as session: now = _lease_now() - for row in self._live_lease_rows(lease_ids, now): + for row in session.scalars( + select(DriveLeaseRecord).where( + DriveLeaseRecord.id.in_(lease_ids), + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ): row.heartbeat_at = now row.expires_at = now + ttl - self.session.commit() + session.commit() + + def set_lease_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + """Record where the cartridge actually is, so recovery reconciles the right drive.""" + with self._lock, self._lease_session() as session: + row = session.get(DriveLeaseRecord, lease_id) + if row is not None: + row.physical_drive_id = physical_drive_id + session.commit() def release_leases(self, lease_ids: list[str]) -> None: - with self._lock: + with self._lock, self._lease_session() as session: now = _lease_now() - for row in self.session.scalars( + for row in session.scalars( select(DriveLeaseRecord).where( DriveLeaseRecord.id.in_(lease_ids), DriveLeaseRecord.released_at.is_(None) ) ): row.released_at = now - self.session.commit() + session.commit() def lease_is_live(self, lease_id: str, fencing_token: int) -> bool: - with self._lock: - row = self.session.get(DriveLeaseRecord, lease_id, populate_existing=True) + with self._lock, self._lease_session() as session: + row = session.get(DriveLeaseRecord, lease_id) return ( row is not None and row.fencing_token == fencing_token @@ -1864,21 +1875,32 @@ def lease_is_live(self, lease_id: str, fencing_token: int) -> bool: ) def live_leases(self) -> list[DriveLease]: - with self._lock: + with self._lock, self._lease_session() as session: now = _lease_now() - rows = self.session.scalars( + rows = session.scalars( select(DriveLeaseRecord) .where(DriveLeaseRecord.released_at.is_(None), DriveLeaseRecord.expires_at > now) .order_by(DriveLeaseRecord.fencing_token) - .execution_options(populate_existing=True) + ) + return [_lease_from_row(row) for row in rows] + + def expired_leases(self) -> list[DriveLease]: + """Unreleased leases whose owner stopped heartbeating: the only cross-process + evidence that a job is dead rather than running elsewhere.""" + with self._lock, self._lease_session() as session: + now = _lease_now() + rows = session.scalars( + select(DriveLeaseRecord) + .where(DriveLeaseRecord.released_at.is_(None), DriveLeaseRecord.expires_at <= now) + .order_by(DriveLeaseRecord.fencing_token) ) return [_lease_from_row(row) for row in rows] def release_leases_for_job(self, job_id: str) -> int: - with self._lock: + with self._lock, self._lease_session() as session: now = _lease_now() rows = list( - self.session.scalars( + session.scalars( select(DriveLeaseRecord).where( DriveLeaseRecord.job_id == job_id, DriveLeaseRecord.released_at.is_(None) ) @@ -1886,20 +1908,9 @@ def release_leases_for_job(self, job_id: str) -> int: ) for row in rows: row.released_at = now - self.session.commit() + session.commit() return len(rows) - def _live_lease_rows(self, lease_ids: list[str], now: datetime) -> list[DriveLeaseRecord]: - return list( - self.session.scalars( - select(DriveLeaseRecord).where( - DriveLeaseRecord.id.in_(lease_ids), - DriveLeaseRecord.released_at.is_(None), - DriveLeaseRecord.expires_at > now, - ) - ) - ) - def _lease_now() -> datetime: """Naive UTC, matching how the catalog's DateTime columns store values.""" From e2443f26253adba01b8b0f19577e20302b4a93e4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 07:16:07 +0000 Subject: [PATCH 010/205] =?UTF-8?q?fix(recovery):=20act=20only=20on=20expi?= =?UTF-8?q?red=20leases=20=E2=80=94=20never=20fail=20a=20job=20another=20p?= =?UTF-8?q?rocess=20may=20own?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog is shared by the API and the CLI, so 'state == running' is not evidence a job is dead. Recovery now fails only jobs whose lease stopped heartbeating, releases live leases of already-finished jobs as bookkeeping, and leaves everything else alone. Inventory read goes through InventoryService (SAFETY_003). Mutation-checked: the test fails if recovery regresses to failing every running job. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/recovery.py | 25 +++++++++++++----- tests/unit/test_recovery.py | 52 ++++++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index 92f0073..584b560 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -10,11 +10,13 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend +from openblade.jobs.inventory import InventoryService logger = logging.getLogger(__name__) INTERRUPTED_ERROR = ( - "interrupted by process restart; physical state unknown — reconcile before retry" + "interrupted: drive lease expired without a heartbeat; physical state unknown — " + "reconcile before retry" ) @@ -35,13 +37,22 @@ class RecoveryReport: def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) -> RecoveryReport: - """Fail interrupted jobs, release orphaned leases, and report drive mismatches.""" - interrupted: list[str] = [] - for job in catalog.list_jobs(state="running"): - catalog.update_job_state(job.id, "failed_recoverable", INTERRUPTED_ERROR) - interrupted.append(job.id) + """Fail jobs whose leases expired, release orphaned leases, report drive mismatches. + The catalog is shared by every process (API + CLI), so "state == running" is + NOT evidence that a job is dead — it may be running in another process. The + only cross-process evidence is a lease that stopped heartbeating. Jobs that + are running without a lease are left alone. + """ + interrupted: list[str] = [] orphaned = [] + for lease in catalog.expired_leases(): + orphaned.append(lease) + owner = catalog.get_job(lease.job_id) + if owner is not None and owner.state == "running" and owner.id not in interrupted: + catalog.update_job_state(owner.id, "failed_recoverable", INTERRUPTED_ERROR) + interrupted.append(owner.id) + # Bookkeeping: a live lease whose job already reached a terminal state. for lease in catalog.live_leases(): owner = catalog.get_job(lease.job_id) if owner is None or owner.state != "running": @@ -50,7 +61,7 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - observed: dict[int, str | None] = { drive.drive_id: None if drive.barcode is None else str(drive.barcode) - for drive in library.inventory().drives + for drive in InventoryService(library).snapshot().drives # SAFETY_003: via service } mismatches: list[DriveMismatch] = [] for lease in orphaned: diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py index 2694ae8..0c98097 100644 --- a/tests/unit/test_recovery.py +++ b/tests/unit/test_recovery.py @@ -4,35 +4,62 @@ from pathlib import Path from openblade.bootstrap import create_context +from openblade.catalog.repository import CatalogRepository from openblade.config import OpenBladeConfig from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart from openblade.simulator.library import MockLibraryBackend -def test_recover_after_restart_running_job_marked_failed_recoverable_and_lease_released( - tmp_path: Path, -) -> None: +def _lease(catalog: CatalogRepository, job_id: str, barcode: str, *, ttl: timedelta) -> str: + leases = catalog.acquire_drive_leases(job_id=job_id, barcodes=[barcode], num_drives=2, ttl=ttl) + assert leases is not None + return leases[0].id + + +def test_recovery_fails_only_jobs_whose_lease_expired(tmp_path: Path) -> None: + # Two `running` jobs share the catalog. One stopped heartbeating (dead + # process); the other holds a live lease — it may be running in ANOTHER + # process right now, and recovery must not touch it. context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) catalog = context.catalog - job = catalog.create_job("archive", {}) - catalog.update_job_state(job.id, "running") - leases = catalog.acquire_drive_leases( - job_id=job.id, barcodes=["MCK00001"], num_drives=2, ttl=timedelta(minutes=15) - ) - assert leases is not None + dead = catalog.create_job("archive", {}) + catalog.update_job_state(dead.id, "running") + dead_lease = _lease(catalog, dead.id, "MCK00001", ttl=timedelta(seconds=-1)) + alive = catalog.create_job("archive", {}) + catalog.update_job_state(alive.id, "running") + alive_lease = _lease(catalog, alive.id, "MCK00002", ttl=timedelta(minutes=15)) empty_library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) report = recover_after_restart(catalog, empty_library) - refreshed = catalog.get_job(job.id) + assert report.interrupted_job_ids == [dead.id] + assert report.released_lease_ids == [dead_lease] + refreshed = catalog.get_job(dead.id) assert refreshed is not None assert refreshed.state == "failed_recoverable" assert refreshed.error == INTERRUPTED_ERROR - assert catalog.live_leases() == [] - assert report.released_lease_ids == [leases[0].id] assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ ("MCK00001", None) ] + untouched = catalog.get_job(alive.id) + assert untouched is not None + assert untouched.state == "running" + assert [lease.id for lease in catalog.live_leases()] == [alive_lease] + + +def test_recovery_releases_live_lease_of_a_finished_job(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + done = catalog.create_job("archive", {}) + catalog.update_job_state(done.id, "completed") + lease_id = _lease(catalog, done.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.interrupted_job_ids == [] + assert report.released_lease_ids == [lease_id] + assert catalog.live_leases() == [] def test_restart_runs_recovery_and_exposes_it_read_only(tmp_path: Path) -> None: @@ -47,6 +74,7 @@ def test_restart_runs_recovery_and_exposes_it_read_only(tmp_path: Path) -> None: first = create_context(OpenBladeConfig(db_url=db_url)) job = first.catalog.create_job("archive", {}) first.catalog.update_job_state(job.id, "running") + _lease(first.catalog, job.id, "MCK00001", ttl=timedelta(seconds=-1)) second = create_context(OpenBladeConfig(db_url=db_url)) reset_context(second) From 5f2ba333318c3d98c53063395c9dd9ea638850d7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 07:16:07 +0000 Subject: [PATCH 011/205] fix(jobs): fenced-out jobs leave the drive alone; background heartbeat; persist physical drive - On StaleLeaseError the error path no longer unmounts/unloads: the drive may already hold the new lease-holder's mounted tape. Physical state is logged as unknown for reconciliation. Mutation-checked. - DriveScheduler keeps held leases alive from a daemon thread (ttl/3), so a single long LTO write cannot expire its own lease. Mutation-checked. - record_physical_drives() persists where a pre-loaded cartridge actually is, so the recovery report reconciles the right drive. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/scheduler.py | 52 ++++++++ openblade/jobs/sharded_archive.py | 111 +++++++++++------- openblade/jobs/sharded_restore.py | 2 + .../test_sharded_archive_atomicity.py | 10 +- tests/unit/test_drive_leases.py | 16 ++- 5 files changed, 146 insertions(+), 45 deletions(-) diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index c9e5016..4664703 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -54,6 +54,8 @@ def acquire( def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: ... + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: ... + def release(self, lease_ids: list[str]) -> None: ... def is_live(self, lease_id: str, fencing_token: int) -> bool: ... @@ -106,6 +108,12 @@ def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: if lease is not None and _live(lease, now): self._leases[lease_id] = replace(lease, heartbeat_at=now, expires_at=now + ttl) + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + with self._lock: + lease = self._leases.get(lease_id) + if lease is not None: + self._leases[lease_id] = replace(lease, physical_drive_id=physical_drive_id) + def release(self, lease_ids: list[str]) -> None: with self._lock: now = _now() @@ -145,6 +153,9 @@ def acquire( def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: self._repo.heartbeat_leases(lease_ids, ttl) + def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: + self._repo.set_lease_physical_drive(lease_id, physical_drive_id) + def release(self, lease_ids: list[str]) -> None: self._repo.release_leases(lease_ids) @@ -180,6 +191,12 @@ def __init__( self._job_id = job_id self._ttl = ttl self._lock = threading.Condition(threading.Lock()) + # Leases are kept alive by a daemon thread for as long as any handle is + # held, so a single long tape write (real LTO: minutes to hours per file) + # cannot outlive its own TTL and be fenced out after the data is down. + self._held: list[DriveHandle] = [] + self._stop_heartbeat = threading.Event() + self._heartbeat_thread: threading.Thread | None = None @property def num_drives(self) -> int: @@ -242,8 +259,33 @@ def acquire_drives( lease.barcode, lease.fencing_token, ) + with self._lock: + self._held.extend(handles) + self._ensure_heartbeat_thread() return handles + def _ensure_heartbeat_thread(self) -> None: + # Caller holds self._lock. + if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive(): + return + self._stop_heartbeat.clear() + self._heartbeat_thread = threading.Thread( + target=self._heartbeat_loop, name="drive-lease-heartbeat", daemon=True + ) + self._heartbeat_thread.start() + + def _heartbeat_loop(self) -> None: + interval = max(self._ttl.total_seconds() / 3.0, 0.05) + while not self._stop_heartbeat.wait(interval): + with self._lock: + handles = [handle for handle in self._held if not handle._released] + if not handles: + return + try: + self.heartbeat(handles) + except Exception: # noqa: BLE001 - a missed beat is logged; the next verify() fences + logger.warning("drive lease heartbeat failed", exc_info=True) + def release_drives(self, handles: list[DriveHandle]) -> None: """Release drives and notify waiting jobs.""" pending = [handle for handle in handles if not handle._released] @@ -252,6 +294,9 @@ def release_drives(self, handles: list[DriveHandle]) -> None: for handle in pending: handle._released = True logger.info("Released drive %d (was %s)", handle.drive_id, handle.barcode) + self._held = [handle for handle in self._held if not handle._released] + if not self._held: + self._stop_heartbeat.set() self._lock.notify_all() def verify(self, handle: DriveHandle) -> None: @@ -265,6 +310,13 @@ def heartbeat(self, handles: list[DriveHandle]) -> None: """Extend the leases behind these handles by the scheduler's ttl.""" self._store.heartbeat([handle.lease_id for handle in handles], self._ttl) + def record_physical_drives(self, handles: list[DriveHandle]) -> None: + """Persist where each cartridge actually ended up (when it differs from the + reserved drive) so restart recovery reconciles the right drive.""" + for handle in handles: + if handle.physical_drive_id is not None: + self._store.set_physical_drive(handle.lease_id, handle.physical_drive_id) + def status(self) -> dict[int, str | None]: """Return copy of drive allocation status.""" status: dict[int, str | None] = {i: None for i in range(self._num_drives)} diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 957c7a9..a951975 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -312,6 +312,49 @@ def _clean_unmount_and_unload( ) +def _log_fenced_out(handles: list[DriveHandle], job_id: str) -> None: + for handle in handles: + logger.error( + "job %s lost its lease on drive %d (%s); leaving the drive untouched — " + "physical state unknown, reconcile before reuse", + job_id, + handle.physical, + handle.barcode, + ) + + +def _best_effort_unmount_and_unload( + catalog: CatalogRepository, + library: LibraryBackend, + ltfs: LTFSBackend, + mounts: dict[str, MountHandle], + handles: list[DriveHandle], + loaded_slots: dict[int, int | None], + job_id: str, +) -> None: + """Error-path cleanup for a job that still owns its drives: unmount, then unload.""" + for mount in mounts.values(): + with suppress(Exception): + ltfs.unmount(mount) + for handle in handles: + slot_id = loaded_slots.get(handle.physical) + if slot_id is not None: + with suppress(Exception): + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=handle.barcode, + drive_id=handle.physical, + slot_id=slot_id, + requested_by="sharded-archive", + job_id=job_id, + ), + ) + + def _archive_stripe( files: list[Path], request: ShardedArchiveRequest, @@ -348,10 +391,13 @@ def _archive_stripe( # this batch. Nothing here is marked archived until the WHOLE batch has verified # and every tape has cleanly unmounted — the atomic commit boundary. staged: list[tuple[str, str, int]] = [] + fenced_out = False try: for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) @@ -428,7 +474,11 @@ def _write_one( files_archived += 1 bytes_archived += size_bytes except StaleLeaseError: - # Fenced out: abort the whole job. `finally` still unmounts/unloads. + # Fenced out: abort the whole job and leave the hardware ALONE. The + # drive may already be loaded + mounted by whoever holds the lease now, + # so unmount/unload here could eject their tape mid-write. Physical + # state is unknown -> reconcile (recovery report / inventory). + fenced_out = True raise except Exception as exc: # noqa: BLE001 # Staged instances remain PENDING -> not exposed as archived; resumable. @@ -443,26 +493,12 @@ def _write_one( ) errors.append(safe_job_error(exc)) finally: - for mount in mounts.values(): - with suppress(Exception): - ltfs.unmount(mount) - for handle in handles: - slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=handle.barcode, - drive_id=handle.physical, - slot_id=slot_id, - requested_by="sharded-archive", - job_id=job_id, - ), - ) + if fenced_out: + _log_fenced_out(handles, job_id) + else: + _best_effort_unmount_and_unload( + catalog, library, ltfs, mounts, handles, loaded_slots, job_id + ) scheduler.release_drives(handles) return files_archived, bytes_archived @@ -492,11 +528,14 @@ def _archive_block_stripe( loaded_slots: dict[int, int | None] = {} shard_dir = scratch_dir / plan.shard_group_id shard_dir.mkdir(parents=True, exist_ok=True) + fenced_out = False try: for handle in handles: drive_id, slot_id = _load_barcode(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: scheduler.verify(handle) # fencing: never mount RW on a stale lease mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_WRITE) @@ -588,27 +627,17 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: # COMMIT: block-striped file is durable only when all shards are down. for instance_id in staged: catalog.mark_instance_archived(instance_id) + except StaleLeaseError: + # Fenced out: abort and leave the hardware alone (see _archive_stripe). + fenced_out = True + raise finally: - for mount in mounts.values(): - with suppress(Exception): - ltfs.unmount(mount) - for handle in handles: - slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=handle.barcode, - drive_id=handle.physical, - slot_id=slot_id, - requested_by="sharded-archive", - job_id=job_id, - ), - ) + if fenced_out: + _log_fenced_out(handles, job_id) + else: + _best_effort_unmount_and_unload( + catalog, library, ltfs, mounts, handles, loaded_slots, job_id + ) scheduler.release_drives(handles) shutil.rmtree(shard_dir, ignore_errors=True) diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index 9cf4786..abd375b 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -232,6 +232,8 @@ def _restore_sharded( for handle in handles: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) loaded_slots[drive_id] = slot_id + scheduler.record_physical_drives(handles) + for handle in handles: mounts[handle.barcode] = ltfs.mount(handle.barcode, MountMode.READ_ONLY) def _read_shard(index: int, instance: Any) -> Path: diff --git a/tests/integration/test_sharded_archive_atomicity.py b/tests/integration/test_sharded_archive_atomicity.py index 2aa57f1..db07b01 100644 --- a/tests/integration/test_sharded_archive_atomicity.py +++ b/tests/integration/test_sharded_archive_atomicity.py @@ -172,11 +172,13 @@ def test_unload_failure_blocks_commit(tmp_path: Path, monkeypatch) -> None: assert _archived_count(catalog) == 0 # tape stuck in drive -> batch must not be durable -def test_stale_lease_aborts_archive_commits_nothing_and_still_unmounts( +def test_stale_lease_aborts_archive_commits_nothing_and_leaves_the_drive_alone( tmp_path: Path, monkeypatch ) -> None: # Spec docs/decisions/2026-10-09-persistent-drive-leases.md §3: a lease - # released behind a running job fences it out before unmount + commit. + # released behind a running job fences it out before unmount + commit, and + # the job must NOT touch the drive afterwards — whoever holds the lease now + # may already have their own tape loaded and mounted there. library, ltfs = _setup() catalog = _catalog() store = InMemoryLeaseStore() @@ -201,4 +203,6 @@ def tracking_unmount(mount): run_sharded_archive(_request(_source(tmp_path)), library, ltfs, catalog, scheduler, job.id) assert _archived_count(catalog) == 0 # no mark_instance_archived - assert unmounted # clean unmount still attempted on the error path + assert unmounted == [] # fenced out: no unmount + loaded = {str(drive.barcode) for drive in library.inventory().drives if drive.barcode} + assert loaded == set(BARCODES) # ...and no unload: the drives are left as they were diff --git a/tests/unit/test_drive_leases.py b/tests/unit/test_drive_leases.py index 438fe3e..4f12511 100644 --- a/tests/unit/test_drive_leases.py +++ b/tests/unit/test_drive_leases.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from datetime import timedelta from pathlib import Path @@ -10,7 +11,7 @@ from openblade.catalog.db import get_session, init_db from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import DriveBusyError, StaleLeaseError -from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler, InMemoryLeaseStore def _two_repos(tmp_path: Path) -> tuple[CatalogRepository, CatalogRepository]: @@ -70,3 +71,16 @@ def test_verify_raises_stale_lease_after_release_or_expiry(tmp_path: Path) -> No (dead,) = expired.acquire_drives(["AAA002L9"], timeout=1.0) with pytest.raises(StaleLeaseError): expired.verify(dead) + + +def test_held_leases_are_kept_alive_without_explicit_heartbeats() -> None: + # A single long tape write must not outlive its own lease: the scheduler + # heartbeats in the background for as long as a handle is held. + store = InMemoryLeaseStore() + scheduler = DriveScheduler(num_drives=1, store=store, ttl=timedelta(seconds=0.3)) + handles = scheduler.acquire_drives(["MCK00001"]) + time.sleep(0.8) # > 2 TTLs, with no heartbeat() call from the "job" + scheduler.verify(handles[0]) # still live + scheduler.release_drives(handles) + time.sleep(0.4) + assert store.live_leases() == [] # released, and the thread did not revive it From 47dc55088070dcd319216898830b7c76118ed636 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 07:18:06 +0000 Subject: [PATCH 012/205] docs(spec): record the review-driven recovery scoping and fenced-out cleanup rules Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 244e376..6e7f238 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -98,6 +98,12 @@ Keep the public shape every caller and ~all tests already use: ### 3. Fencing at the destructive edges +A fenced-out job (`StaleLeaseError`) must **not** unmount or unload: the drive +may already hold the new lease-holder's mounted tape. It logs the drive as +"physical state unknown" and leaves it for reconciliation. Held leases are +kept alive by a daemon heartbeat thread in the scheduler (every `ttl/3`), so a +single long write cannot expire its own lease. + In `openblade/jobs/sharded_archive.py`, `archive.py`, `restore.py`, `tree_restore.py`: call `scheduler.verify(handle)` immediately before `ltfs.mount(..., READ_WRITE)`, before each shard/file write batch, and before @@ -120,8 +126,12 @@ errors** (existing rule). Call `heartbeat` after each file/shard write. New `openblade/jobs/recovery.py::recover_after_restart(catalog, library) -> RecoveryReport`, called from bootstrap after `init_db` and before the API/CLI serves anything: -1. Every catalog job in state `running` → `failed_recoverable` with error - `"interrupted by process restart; physical state unknown — reconcile before retry"`. +1. **Only jobs whose lease has EXPIRED** (no heartbeat within the TTL) → + `failed_recoverable`. The catalog is shared by the API and the CLI, so a + `running` row is not evidence that a job is dead — it may be running in + another process. A running job with no lease at all is left alone. + (Adversarial review 2026-10-09 caught the original "fail every running + job" wording as a cross-process regression; it never shipped.) 2. Every live lease whose job is not `running` → released (`released_at = now`). 3. For each lease released in step 2, compare expected (`barcode` in `physical`) with `library.inventory()`; log one line per mismatch at WARNING @@ -140,7 +150,8 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. Fencing tokens strictly increase; a released lease's token is never reissued. - Lease fencing (landed in `tests/integration/test_sharded_archive_atomicity.py` to reuse its helpers): release a lease behind a running sharded archive → - `StaleLeaseError`, no `mark_instance_archived`, unmount/unload still attempted. + `StaleLeaseError`, no `mark_instance_archived`, and the drive is left + untouched (no unmount, no unload). - `tests/unit/test_recovery.py`: seed a `running` job + live lease, restart → job `failed_recoverable`, lease released, mismatch reported when the simulator's drive is empty. @@ -162,7 +173,8 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. - The tree-restore route builds its scheduler in the worker thread over a `CatalogLeaseStore(worker_catalog)` rather than `context.lease_store`: the context's session is not thread-safe. Same table, same exclusion. -- Heartbeat runs once per write batch, not per shard. +- Explicit heartbeats run once per write batch; the scheduler's background + heartbeat thread covers the gaps. ## Non-goals From da84f5b96710e0039e2528c703f89e9edc9ba70c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:57:08 +0000 Subject: [PATCH 013/205] feat(domain): add STAGING and VERIFYING file-instance states Sharded-archive lifecycle states. Stored as plain strings in file_instances.state (no CHECK/enum type), so no migration is needed; every catalog 'archived/restorable' filter is an explicit allowlist of {archived, verified}, so the new states are excluded by construction. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openblade/domain/models.py b/openblade/domain/models.py index d5d1dcf..7a4042b 100644 --- a/openblade/domain/models.py +++ b/openblade/domain/models.py @@ -65,6 +65,11 @@ class MountMode(str, Enum): class FileInstanceState(str, Enum): PENDING = "pending" + # Sharded-archive lifecycle: written but not yet verified (STAGING), read + # back and being verified (VERIFYING). Neither is restorable or "archived"; + # only the single commit at the end of a job moves them to ARCHIVED. + STAGING = "staging" + VERIFYING = "verifying" ARCHIVED = "archived" VERIFIED = "verified" FAILED = "failed" From 7ff9ba9dd6363ab6cc6525e518868e8ba12607db Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:57:11 +0000 Subject: [PATCH 014/205] ci(operability): pin actions to full commit SHAs Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/operability.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/operability.yml b/.github/workflows/operability.yml index 0b52fd3..7eda9e4 100644 --- a/.github/workflows/operability.yml +++ b/.github/workflows/operability.yml @@ -27,8 +27,8 @@ jobs: name: operability-gate runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" From 94322784cbdd7030a523155dfe72bb150a128c1c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:57:22 +0000 Subject: [PATCH 015/205] feat(domain): model placement, striping and protection as separate axes Striping is a performance choice; only redundant copies make a policy protected. Erasure coding is modelled but rejected by require_plannable(). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/protection.py | 201 +++++++++++++++++++++++++++++ tests/unit/test_protection_axes.py | 101 +++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 openblade/domain/protection.py create mode 100644 tests/unit/test_protection_axes.py diff --git a/openblade/domain/protection.py b/openblade/domain/protection.py new file mode 100644 index 0000000..b36d1a0 --- /dev/null +++ b/openblade/domain/protection.py @@ -0,0 +1,201 @@ +"""Protection axes for archived data (roadmap item 10). + +Placement, striping and protection are independent axes. Spreading data across +tapes (striping/sharding) is a *performance* choice; it never makes data more +durable on its own. Only redundant copies make a policy "protected". + +Pure value objects: no IO, no library access. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any + + +class UnsupportedProtectionError(ValueError): + """A protection scheme that is modelled but cannot be planned (e.g. erasure coding).""" + + +class InvalidProtectionPolicyError(ValueError): + """A protection policy whose axes contradict each other.""" + + +class Placement(StrEnum): + SINGLE_TAPE = "single_tape" + STRIPED = "striped" + + +class StripingMode(StrEnum): + NONE = "none" + STRIPE = "stripe" + BLOCK_STRIPE = "block_stripe" + + +class ProtectionKind(StrEnum): + NONE = "none" + REPLICATION = "replication" + ERASURE_CODE = "erasure_code" + + +class FailureDomain(StrEnum): + DRIVE = "drive" + TAPE = "tape" + LIBRARY = "library" + + +class DurabilityClass(StrEnum): + UNPROTECTED = "unprotected" + PERFORMANCE_ONLY = "performance-only" + PROTECTED = "protected" + + +@dataclass(frozen=True) +class Striping: + mode: StripingMode = StripingMode.NONE + block_size: int | None = None + + def __post_init__(self) -> None: + if self.mode is StripingMode.BLOCK_STRIPE: + if self.block_size is None or self.block_size <= 0: + raise InvalidProtectionPolicyError("block striping requires a positive block_size") + elif self.block_size is not None: + raise InvalidProtectionPolicyError("block_size is only valid for block striping") + + +@dataclass(frozen=True) +class Protection: + kind: ProtectionKind = ProtectionKind.NONE + copies: int = 1 + k: int = 0 + m: int = 0 + + def __post_init__(self) -> None: + if self.copies < 1: + raise InvalidProtectionPolicyError("copies must be >= 1") + if self.kind is ProtectionKind.ERASURE_CODE and (self.k < 1 or self.m < 1): + raise InvalidProtectionPolicyError("erasure coding requires k >= 1 and m >= 1") + + @classmethod + def none(cls) -> Protection: + return cls() + + @classmethod + def replication(cls, copies: int) -> Protection: + return cls(kind=ProtectionKind.REPLICATION, copies=copies) + + @classmethod + def erasure_code(cls, k: int, m: int) -> Protection: + return cls(kind=ProtectionKind.ERASURE_CODE, k=k, m=m) + + @property + def supported(self) -> bool: + """Erasure coding is modelled only; the planner cannot produce it.""" + return self.kind is not ProtectionKind.ERASURE_CODE + + @property + def redundant(self) -> bool: + if self.kind is ProtectionKind.REPLICATION: + return self.copies >= 2 + return self.kind is ProtectionKind.ERASURE_CODE + + +@dataclass(frozen=True) +class RestoreQuorum: + tapes_required: int + tapes_total: int + + def __post_init__(self) -> None: + if self.tapes_required < 1 or self.tapes_total < 1: + raise InvalidProtectionPolicyError("restore quorum counts must be >= 1") + if self.tapes_required > self.tapes_total: + raise InvalidProtectionPolicyError("restore quorum cannot exceed total tapes") + + +@dataclass(frozen=True) +class ProtectionPolicy: + placement: Placement = Placement.SINGLE_TAPE + striping: Striping = field(default_factory=Striping) + protection: Protection = field(default_factory=Protection) + failure_domain: FailureDomain = FailureDomain.TAPE + quorum: RestoreQuorum | None = None + + def __post_init__(self) -> None: + striped = self.striping.mode is not StripingMode.NONE + if striped != (self.placement is Placement.STRIPED): + raise InvalidProtectionPolicyError("striping mode must match STRIPED placement") + + @property + def is_protected(self) -> bool: + return self.protection.redundant + + @property + def durability_class(self) -> DurabilityClass: + if self.is_protected: + return DurabilityClass.PROTECTED + if self.striping.mode is not StripingMode.NONE: + return DurabilityClass.PERFORMANCE_ONLY + return DurabilityClass.UNPROTECTED + + def require_plannable(self) -> None: + """Raise if the planner cannot realise this policy.""" + if not self.protection.supported: + raise UnsupportedProtectionError( + f"protection '{self.protection.kind.value}' is modelled but not supported" + ) + + def to_dict(self) -> dict[str, Any]: + return { + "placement": self.placement.value, + "striping": {"mode": self.striping.mode.value, "block_size": self.striping.block_size}, + "protection": { + "kind": self.protection.kind.value, + "copies": self.protection.copies, + "k": self.protection.k, + "m": self.protection.m, + }, + "failure_domain": self.failure_domain.value, + "quorum": None + if self.quorum is None + else { + "tapes_required": self.quorum.tapes_required, + "tapes_total": self.quorum.tapes_total, + }, + "durability_class": self.durability_class.value, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> ProtectionPolicy: + striping = data.get("striping") or {} + protection = data.get("protection") or {} + quorum = data.get("quorum") + return cls( + placement=Placement(data.get("placement", Placement.SINGLE_TAPE.value)), + striping=Striping( + mode=StripingMode(striping.get("mode", StripingMode.NONE.value)), + block_size=striping.get("block_size"), + ), + protection=Protection( + kind=ProtectionKind(protection.get("kind", ProtectionKind.NONE.value)), + copies=int(protection.get("copies", 1)), + k=int(protection.get("k", 0)), + m=int(protection.get("m", 0)), + ), + failure_domain=FailureDomain(data.get("failure_domain", FailureDomain.TAPE.value)), + quorum=None + if quorum is None + else RestoreQuorum(int(quorum["tapes_required"]), int(quorum["tapes_total"])), + ) + + +def protection_from_pool(pool: Any) -> ProtectionPolicy: + """Map a legacy ``NasPool.replication_factor`` onto the protection axis. + + ``replication_factor`` stays the source of truth for existing pools; this only + expresses it as ``Protection.REPLICATION(copies=n)``. Placement is single-tape: + the legacy field says nothing about striping. + """ + copies = max(1, int(getattr(pool, "replication_factor", 1) or 1)) + protection = Protection.replication(copies) if copies >= 2 else Protection.none() + return ProtectionPolicy(protection=protection) diff --git a/tests/unit/test_protection_axes.py b/tests/unit/test_protection_axes.py new file mode 100644 index 0000000..025ca12 --- /dev/null +++ b/tests/unit/test_protection_axes.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from openblade.domain.protection import ( + DurabilityClass, + FailureDomain, + InvalidProtectionPolicyError, + Placement, + Protection, + ProtectionPolicy, + RestoreQuorum, + Striping, + StripingMode, + UnsupportedProtectionError, + protection_from_pool, +) + +_STRIPINGS = [ + (Placement.SINGLE_TAPE, Striping()), + (Placement.STRIPED, Striping(StripingMode.STRIPE)), + (Placement.STRIPED, Striping(StripingMode.BLOCK_STRIPE, block_size=1 << 20)), +] +_PROTECTIONS = [ + (Protection.none(), False), + (Protection.replication(1), False), + (Protection.replication(2), True), + (Protection.replication(3), True), + (Protection.erasure_code(4, 2), True), +] + + +@pytest.mark.parametrize("domain", list(FailureDomain)) +@pytest.mark.parametrize(("placement", "striping"), _STRIPINGS) +@pytest.mark.parametrize(("protection", "expected"), _PROTECTIONS) +def test_is_protected_for_every_combination( + domain: FailureDomain, + placement: Placement, + striping: Striping, + protection: Protection, + expected: bool, +) -> None: + policy = ProtectionPolicy( + placement=placement, striping=striping, protection=protection, failure_domain=domain + ) + assert policy.is_protected is expected + assert (policy.durability_class is DurabilityClass.PROTECTED) is expected + + +@pytest.mark.parametrize("mode", [StripingMode.STRIPE, StripingMode.BLOCK_STRIPE]) +@pytest.mark.parametrize("copies", [1]) +def test_striping_never_implies_protection(mode: StripingMode, copies: int) -> None: + block = 4096 if mode is StripingMode.BLOCK_STRIPE else None + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(mode, block_size=block), + protection=Protection.replication(copies), + ) + assert policy.is_protected is False + assert policy.durability_class is DurabilityClass.PERFORMANCE_ONLY + + +def test_quorum_cannot_exceed_total() -> None: + with pytest.raises(InvalidProtectionPolicyError): + RestoreQuorum(tapes_required=3, tapes_total=2) + assert RestoreQuorum(2, 2).tapes_required == 2 + + +def test_striping_must_match_placement() -> None: + with pytest.raises(InvalidProtectionPolicyError): + ProtectionPolicy(placement=Placement.SINGLE_TAPE, striping=Striping(StripingMode.STRIPE)) + with pytest.raises(InvalidProtectionPolicyError): + Striping(StripingMode.BLOCK_STRIPE) + + +def test_erasure_code_is_modelled_but_not_plannable() -> None: + policy = ProtectionPolicy(protection=Protection.erasure_code(4, 2)) + assert policy.protection.supported is False + with pytest.raises(UnsupportedProtectionError): + policy.require_plannable() + ProtectionPolicy(protection=Protection.replication(2)).require_plannable() + + +@pytest.mark.parametrize(("factor", "copies", "protected"), [(1, 1, False), (2, 2, True)]) +def test_protection_from_pool(factor: int, copies: int, protected: bool) -> None: + policy = protection_from_pool(SimpleNamespace(replication_factor=factor)) + assert policy.protection.copies == copies + assert policy.is_protected is protected + + +def test_policy_dict_round_trip() -> None: + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(StripingMode.BLOCK_STRIPE, block_size=8192), + protection=Protection.replication(2), + failure_domain=FailureDomain.LIBRARY, + quorum=RestoreQuorum(2, 4), + ) + assert ProtectionPolicy.from_dict(policy.to_dict()) == policy From daff4dac20d78a5457335caca3d417f552af4e39 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:58:17 +0000 Subject: [PATCH 016/205] ci: add nightly/manual NAS protocol rig workflow Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/nas-protocols.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/nas-protocols.yml diff --git a/.github/workflows/nas-protocols.yml b/.github/workflows/nas-protocols.yml new file mode 100644 index 0000000..bd9d23e --- /dev/null +++ b/.github/workflows/nas-protocols.yml @@ -0,0 +1,44 @@ +name: NAS protocols + +# Nightly / manual only. Deliberately NOT a pull_request or push check: the rig +# builds Samba + NFS-Ganesha containers and needs a privileged client container. +on: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" + +permissions: + contents: read + +jobs: + protocols: + name: smb-nfs-protocol-rig + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Start protocol rig + run: make protocols-up + - name: Run protocol tests + run: make test-protocols + - name: Dump rig logs + if: failure() + run: | + mkdir -p protocol-logs + docker compose -f deploy/nas-protocols/docker-compose.yml ps > protocol-logs/ps.txt 2>&1 || true + docker compose -f deploy/nas-protocols/docker-compose.yml logs --no-color > protocol-logs/compose.log 2>&1 || true + - name: Upload rig logs + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nas-protocols-logs + path: protocol-logs/ + if-no-files-found: warn + - name: Tear down protocol rig + if: always() + run: make protocols-down From de01abe362893239c5b29344045a5e9d9ef86673 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:58:29 +0000 Subject: [PATCH 017/205] docs: describe NAS protocol test rig Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/test-plan.md | 3 +++ docs/wiki/guides/fuse-and-nas.md | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/docs/test-plan.md b/docs/test-plan.md index 549dc48..75d00b8 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -20,3 +20,6 @@ Verify injected mount, write, and capacity faults surface as typed errors. ## Safety regressions Prove the default config blocks real hardware, formatting requires confirmation, and unload-while-mounted is rejected. + +## Protocol tests +Run the SMB and NFS protocol rig with `make protocols-up && make test-protocols && make protocols-down` (tests are marked `protocols`, live in `tests/e2e/protocols`, and skip with a capability-naming reason when the rig is unavailable). They are not part of the default suite; `.github/workflows/nas-protocols.yml` runs them nightly and on manual dispatch, and CI records the results. The rig uses a hydrator shim to simulate tape recall, so it does not cover the real FUSE hydrator. See `docs/wiki/guides/fuse-and-nas.md`. diff --git a/docs/wiki/guides/fuse-and-nas.md b/docs/wiki/guides/fuse-and-nas.md index 02bbad8..d8df05b 100644 --- a/docs/wiki/guides/fuse-and-nas.md +++ b/docs/wiki/guides/fuse-and-nas.md @@ -205,6 +205,45 @@ policy-resolution helper for ingest and has nothing to do with mounting. --- +## Protocol test rig + +`deploy/nas-protocols/docker-compose.yml` brings up a throwaway SMB and NFS +stack so the protocol exports can be exercised by real clients. Services: + +- `samba` — serves shares from an `smb.conf` rendered by `openblade/nas/samba.py`. +- `nfs-ganesha` — serves exports from a config rendered by `openblade/nas/nfs.py`. +- `client` — a privileged container that mounts the NFS export internally + (mounting needs privileges the CI runner's test process does not have). + +Both config files are rendered from `NasShare` records, so the rig tests the +renderers' output rather than hand-written configs. + +Run it: + +```bash +make protocols-up && make test-protocols && make protocols-down +``` + +`protocols-up` builds, starts and waits for healthy; `test-protocols` runs +`python -m pytest -m protocols tests/e2e/protocols -q`; `protocols-down` tears +down with `-v`. Host ports come from `OB_SMB_PORT` and `OB_NFS_PORT`. Tests +carry the `protocols` marker and skip, with a reason naming the missing +capability, when the rig is not available. + +**What the rig simulates.** The rig contains a hydrator shim, not the real +hydrator. The shim models a tape recall: a stub file becomes real content after +a delay, or never does for an offline cartridge. It exists to check how SMB and +NFS clients behave while a read blocks or times out. The real hydrator is the +FUSE path described above, and it is not exercised by the rig, so rig results +say nothing about actual recall from tape. + +**CI.** `.github/workflows/nas-protocols.yml` runs the rig nightly and on manual +dispatch. It is not triggered by pull requests or pushes and is not a required +check. Per-scenario results are recorded by CI runs; on failure the workflow +uploads the compose logs as an artifact. + +--- + ## Doc corrections - `docs/fuse.md` calls this "a lightweight namespace abstraction over the From 58d0f047ab20e76e5a53b8348fe299c9020267a1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 14:59:47 +0000 Subject: [PATCH 018/205] docs: rewrite tech-debt with verified state; document CI lanes Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- README.md | 32 ++++++++--- docs/tech-debt.md | 137 +++++++++++++++++++++++++++++----------------- docs/test-plan.md | 23 ++++++++ 3 files changed, 134 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 5755345..849d62a 100644 --- a/README.md +++ b/README.md @@ -19,16 +19,34 @@ OpenBlade is a simulator-first DIY tape archive controller inspired by iBlade-st 5. **Continuous verification**: enforce compatibility and regression evidence in CI/CD before changes land on `master`. ## Layered CI/CD (targeted) -OpenBlade CI/CD is intentionally split by layer so each change runs only relevant checks: +OpenBlade CI/CD is split by layer so that each change runs only the checks that matter for it. A single required check, `ci-gate`, then aggregates the results. -| Layer | Primary workflow/jobs | Trigger scope | +**PR lanes (`CI` / `ci.yml`).** These lanes run on pull requests, and `detect-changes` path-filters them: + +| Layer | Jobs | Trigger scope | | --- | --- | --- | -| API + backend domain | `CI`: `backend-lint`, `backend-typecheck`, `backend-tests`, `api-aml-integration` | `openblade/**/*.py`, AML integration tests, backend config | -| Simulator/emulator parity | `CI`: `i3-smoke`; `i3-emulator-compliance`; `emulator-change-gates` | simulator, AML routes, emulator contract/tools, i3 tests, compose/runtime wiring | -| Frontend/UI | `CI`: `frontend-build-test` (React SPA), `web-flask-smoke` (Flask NAS UI) | `frontend/**`; `openblade/web_flask/**`, `Dockerfile.web`, `docker-compose.yml` | -| CI/CD policy layer | `CI`: `cicd-workflow-validate` | `.github/workflows/**` | +| API + backend domain | `backend-lint` (ruff check + format), `backend-typecheck` (`mypy openblade`, strict), `backend-tests`, `api-aml-integration` | `openblade/**/*.py`, AML integration tests, backend config | +| Simulator/emulator parity | `i3-smoke`; `i3-emulator-compliance`; `emulator-change-gates` | simulator, AML routes, emulator contract/tools, i3 tests, compose/runtime wiring | +| Frontend/UI | `frontend-build-test` (React SPA), `web-flask-smoke` (Flask NAS UI) | `frontend/**`; `openblade/web_flask/**`, `Dockerfile.web`, `docker-compose.yml` | +| CI/CD policy | `cicd-workflow-validate`; `workflow-lint` | `.github/workflows/**` | + +- **Blocking.** Lint, format and `backend-typecheck` all block merge. None of these jobs uses `continue-on-error`, and `ci-gate` fails on any required job whose result is not `success` or `skipped`. +- **Ownership check.** `tools/ci_ownership.py --base origin/master` takes the changed paths from `git diff --name-only origin/master...HEAD`. It maps them through the glob table in `tools/ci_ownership.toml` to these categories: `unit`, `integration`, `safety`, `i3`, `compat`, `frontend`, `docs-only` and `ci-only`. It prints its reasoning, and it fails if a changed `openblade/**` or `tests/**` path has no owning category. +- **Emulator boot.** The reusable workflow `.github/workflows/_emulator-boot-test.yml` (`workflow_call`; inputs include `timing_profile`, `pytest_args` and `python_version`) boots the emulator, waits for it and runs pytest. `emulator-change-gates.yml` and `i3-emulator-compliance.yml` call it instead of copying those steps. + +**Workflow hygiene (`workflow-lint.yml`).** This workflow runs `actionlint` (with shellcheck on inline `run:` blocks), `zizmor` and `yamllint` (`.yamllint`). Every action is pinned to a full commit SHA with a `# vX.Y.Z` comment. Workflows declare `permissions: contents: read` at the top level and grant write scopes per job. + +**Nightly (`nightly.yml`).** Runs on a cron schedule and through `workflow_dispatch`. It is not a required PR check, but each of its lanes fails red: + +| Lane | Selection | +| --- | --- | +| `slow` | `-m slow` | +| `stress` | `tests/fault` + `-m stress` | +| `fuzz` | `tests/property` (`-m fuzz`) | +| `rebuild` | `tests/unit/test_catalog_rebuild*` (`-m rebuild`) | +| `mutation` | `tools/mutation_run.sh` (mutmut) over the safety-critical modules; ratchet against `mutation/baseline.txt`, so new surviving mutants fail the lane. Run it locally with `make mutation`. | -This keeps checks up to date and targeted while preserving full coverage on workflow dispatch and on emulator-specific workflows. +**i3 timing profiles.** `I3_TIMING_PROFILE` selects a profile for both the emulator and pytest. The names `instant`, `realistic` and `hardware` still work. The new profiles are `normal`, `slow-robotics`, `busy-library`, `intermittent-drive`, `session-expiry`, `rebooting` and `degraded-media`. Profile delays go through a `Clock` protocol (`RealClock`/`VirtualClock` in `tests/i3/timing.py`), so profile tests run deterministically under `VirtualClock` without real sleeps. ## Quick start ```bash diff --git a/docs/tech-debt.md b/docs/tech-debt.md index b7c5272..23ffd29 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -1,57 +1,92 @@ # Tracked technical debt -Known, quantified debt that is deliberately not blocking CI, with the safe way to -pay it down. Advisory CI jobs (`continue-on-error: true`) still run and report; -they just don't fail the build or block trusted-PR auto-merge. +Current, verified state only. Every number below was produced by the command +shown next to it, run from the repo root with the pinned dev toolchain +(`ruff==0.15.22`, `mypy==2.3.0` from the `dev` extra) on 2026-10-09. If you +change a number, re-run the command and paste the new result; do not estimate. -## Lint / format backlog — PAID DOWN (2026-09-12) +## Lint and format: clean and blocking -**This section is history.** The backlog described here is cleared; see +| Check | Command | Result | +| --- | --- | --- | +| Lint | `ruff check .` | `All checks passed!` (0 findings) | +| Format | `ruff format --check .` | `365 files already formatted` | + +Both run in the `backend-lint` job of `.github/workflows/ci.yml`. That job has no +`continue-on-error`, and `ci-gate` both lists it in `needs:` and fails on any +result other than `success`/`skipped`, so a lint or format failure blocks merge. + +The older note that `openblade/assistant/tools.py` was the one file failing +`ruff format --check` is no longer true: +`ruff format --check openblade/assistant/tools.py` reports `1 file already formatted`. + +History of how the backlog was paid down: [`decisions/2026-09-11-pin-ruff-toolchain.md`](decisions/2026-09-11-pin-ruff-toolchain.md). -What it used to say: ~180 `ruff check` findings + ~160 files needing -`ruff format`, with `A002` (~68 findings) called out as un-fixable because -FastAPI path/query params named `id`/`type`/`filter` are the wire contract. - -What actually happened: `A002` was added to the `ignore` list as that entry -recommended, which is why the count fell from ~180 to **13**. Those 13 were then -hand-fixed (12 real fixes + 1 justified `# noqa: SIM222`), and the 176-file -format backlog was swept in a single format-only commit verified AST-identical. -`ruff` is now pinned **exactly** (`ruff==0.15.22` in the `dev` extra), so -`make lint`, the editor hook, and CI enforce the same rules on the same version -instead of whatever each happened to resolve. - -Two items remain, both deliberate and both scoped: - -- `openblade/assistant/tools.py` is the one file still failing - `ruff format --check` — the package has a live owner and was left - byte-identical. It needs one `ruff format` pass from that owner. -- 27 `# noqa` markers reference rules that are **not selected** (`BLE001` x23, - `S603`/`S404`/`S310` x4), so they are inert documentation. Enabling those - families was measured, not guessed: `BLE` = 44 findings, `S` = 3,748 (3,655 of - them `S101`, bare `assert` in tests, which would need a per-file ignore). - Do not delete those markers as dead code — they record intent for that work. - -Rule for new debt of this kind: pin the tool version at the same time you pin -the rule set. An unpinned linter makes "is the tree clean?" a question with a -different answer per machine. - -## Type-check backlog (advisory: `backend-typecheck`) - -- ~300 `mypy openblade` errors across 51 files, concentrated in the god-file - modules (`aml_state.py`, `routes_aml_*.py`). -- Pay down **per module**, not in one sweep — annotate one file, keep it green, - and (eventually) move it to a mypy per-file strict allowlist so it can't - regress. Brute-forcing all 300 at once is high-regression-risk and unreviewable. - -## Why the type-check backlog is still advisory - -Hand-fixing 300 type errors across wire-contract code in one sweep is a large, -unreviewable, high-regression change. `backend-typecheck` therefore stays -`continue-on-error: true`. The required checks (`backend-tests`, `i3-smoke`) and -the `operability-gate` remain hard gates. - -`backend-lint`'s `continue-on-error` no longer has a justification and should be -removed — the exact workflow diff is in -[`decisions/2026-09-11-pin-ruff-toolchain.md`](decisions/2026-09-11-pin-ruff-toolchain.md); -it needs an owner of `.github/workflows/**` to apply it. +## Type check: strict, zero errors, blocking + +| Check | Command (as in `ci.yml`) | Result | +| --- | --- | --- | +| Backend | `mypy openblade` (`backend-typecheck`) | `Success: no issues found in 170 source files` | +| Flask UI | `mypy openblade/web_flask` (`web-flask-smoke`) | `Success: no issues found in 4 source files` | + +`[tool.mypy]` sets `strict = true`. `backend-typecheck` has no +`continue-on-error`. On this branch it is a required dependency of `ci-gate`: +it is listed under `needs:`, and the gate's aggregate step checks its result. +The old "advisory, ~300 errors" entry is obsolete. + +## Remaining `# noqa` markers + +Inventory command (counts rule codes, which is the same as counting lines here, +because no marker lists more than one rule): + +```bash +grep -rhoE "# noqa: ?[A-Z0-9, ]+" --include=*.py openblade tests tools \ + | sed -E 's/# noqa: ?//' | tr ',' '\n' | tr -d ' ' | grep -v '^$' \ + | sort | uniq -c | sort -rn +``` + +| Rule | Count | +| --- | --- | +| `BLE001` (blind `except Exception`) | 65 | +| `B009` (`getattr` with a constant) | 7 | +| `SIM222` | 1 | +| `F401` | 1 | +| `A001` | 1 | +| **Total** | **75** (openblade 55, tests 20, tools 0) | + +There are no bare `# noqa` markers without a rule code (0 matches for +`# noqa` not followed by `:`). `BLE` is now in the selected rule set +(`[tool.ruff.lint] select`), so the `BLE001` markers are active suppressions, +not inert documentation as an earlier version of this file said. Each marker is +supposed to carry a written reason (project rule). Paying them down means +narrowing each `except Exception` to the typed errors the call can actually +raise. + +## `datetime.utcnow` (deprecated since Python 3.12) + +```bash +grep -roE "datetime\.utcnow\b" --include=*.py openblade tests tools | wc -l # 54 +grep -rlE "datetime\.utcnow\b" --include=*.py openblade tests tools | wc -l # 16 files +``` + +54 references in 16 files, all under `openblade/` (0 in `tests/` and `tools/`). +They include both calls and bare references such as SQLAlchemy column defaults +(`default=datetime.utcnow` in `openblade/catalog/models.py`). The largest +groups are `openblade/catalog/models.py`, `openblade/catalog/repository.py` +and `openblade/nas/protocol_gateway.py`. These return naive datetimes. Migrate +them module by module to `datetime.now(timezone.utc)`, and check every column +or comparison that mixes naive and aware values before you switch it. +`openblade/api/aml_state.py` already uses its own aware `_utcnow()` helper and +is not counted here. + +## In-memory job queue + +`openblade/jobs/queue.py:23` (`class JobQueue`) keeps jobs, drive owners and the +changer owner in process memory (`dict`s guarded by a `threading.RLock`). It is +created once in `openblade/bootstrap.py:614`. Nothing in `queue.py` persists +anything, so a process restart loses queued and running job records and the +ownership map. Any durable recovery has to come from elsewhere (for example, +drive leases and `openblade/jobs/recovery.py`), not from the queue. The queue +also assumes a single process, so running more than one worker process would +give each one its own independent queue. diff --git a/docs/test-plan.md b/docs/test-plan.md index 549dc48..32bab23 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -20,3 +20,26 @@ Verify injected mount, write, and capacity faults surface as typed errors. ## Safety regressions Prove the default config blocks real hardware, formatting requires confirmation, and unload-while-mounted is rejected. + +## CI lanes +Lane definitions live in the README section "Layered CI/CD". This section lists which tests each lane runs. + +- **PR (`ci.yml`, required through `ci-gate`).** + - `backend-lint`: `ruff check .` and `ruff format --check .`. + - `backend-typecheck`: `mypy openblade` under strict mode. It blocks merge. + - `backend-tests`: unit and integration tests. + - `api-aml-integration`. + - `i3-smoke`. + - `frontend-build-test`. + - `web-flask-smoke`. + - The ownership check `tools/ci_ownership.py --base origin/master`. It maps every changed path, through `tools/ci_ownership.toml`, to `unit`, `integration`, `safety`, `i3`, `compat`, `frontend`, `docs-only` or `ci-only`. It fails on any changed `openblade/**` or `tests/**` path that has no owner. If you add a new test directory or package, add its glob to the TOML table. +- **Emulator.** `emulator-change-gates.yml` and `i3-emulator-compliance.yml` both run `tests/i3` through the reusable `_emulator-boot-test.yml`, with an `I3_TIMING_PROFILE` input. + - The timing profiles are `instant`, `realistic`, `hardware`, `normal`, `slow-robotics`, `busy-library`, `intermittent-drive`, `session-expiry`, `rebooting` and `degraded-media`. + - `tests/i3/test_timing_profiles.py` checks each profile's effect under `VirtualClock`, with no real sleeping. +- **Nightly (`nightly.yml`, not a PR check).** Run the same selections locally with pytest. + - `slow`: `-m slow`. + - `stress`: `tests/fault` plus `-m stress`. + - `fuzz`: `tests/property`. + - `rebuild`: `tests/unit/test_catalog_rebuild*`. + - `mutation`: `make mutation` / `tools/mutation_run.sh`. It runs mutmut against the safety-critical modules using `tests/safety` plus the scheduler, drive-lease, recovery and scalar-coordinate unit tests. Surviving mutants are compared with `mutation/baseline.txt`, and any new survivor fails the lane. + - The pytest markers `stress`, `fuzz`, `rebuild` and `mutation` are registered in `pyproject.toml`. From ef6bc31d64482a6592dd1b5d56e67aa77e85bce4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:00:15 +0000 Subject: [PATCH 019/205] feat(nas): render full smb.conf and Ganesha config from NasShare records Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/export_render.py | 49 ++++++++++++++++++++++++++++++++++ openblade/nas/nfs.py | 42 +++++++++++++++++++++++++++++ openblade/nas/samba.py | 35 +++++++++++++++++++++++- 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 openblade/nas/export_render.py diff --git a/openblade/nas/export_render.py b/openblade/nas/export_render.py new file mode 100644 index 0000000..e908c38 --- /dev/null +++ b/openblade/nas/export_render.py @@ -0,0 +1,49 @@ +"""Map catalog ``NasShare`` records onto Samba/NFS export definitions. + +``NasShare.config_json`` keys honoured: ``read_only`` (bool, default True), +``case_sensitive`` (bool, default False), ``client`` (NFS client spec, default +``*``), ``export_id`` (int) and ``pseudo`` (NFSv4 pseudo path). +""" + +import json +from pathlib import Path +from typing import Any + +from openblade.catalog.models import NasShare +from openblade.nas.nfs import NfsExport +from openblade.nas.samba import SambaShare + + +class InvalidShareConfigError(ValueError): + """``NasShare.config_json`` is not a JSON object.""" + + +def _config(share: NasShare) -> dict[str, Any]: + try: + raw = json.loads(share.config_json or "{}") + except json.JSONDecodeError as exc: + raise InvalidShareConfigError(f"share {share.name!r}: config_json is not JSON") from exc + if not isinstance(raw, dict): + raise InvalidShareConfigError(f"share {share.name!r}: config_json must be an object") + return raw + + +def samba_share_from(share: NasShare) -> SambaShare: + cfg = _config(share) + return SambaShare( + name=share.name, + path=Path(share.path), + read_only=bool(cfg.get("read_only", True)), + case_sensitive=bool(cfg.get("case_sensitive", False)), + ) + + +def nfs_export_from(share: NasShare, export_id: int = 1) -> NfsExport: + cfg = _config(share) + return NfsExport( + path=Path(share.path), + client=str(cfg.get("client", "*")), + read_only=bool(cfg.get("read_only", True)), + export_id=int(cfg.get("export_id", export_id)), + pseudo=cfg.get("pseudo"), + ) diff --git a/openblade/nas/nfs.py b/openblade/nas/nfs.py index 55188d2..27f0116 100644 --- a/openblade/nas/nfs.py +++ b/openblade/nas/nfs.py @@ -1,5 +1,6 @@ """NFS export configuration rendering.""" +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -9,7 +10,48 @@ class NfsExport: path: Path client: str = "*" read_only: bool = True + export_id: int = 1 + pseudo: str | None = None def render(self) -> str: mode = "ro" if self.read_only else "rw" return f"{self.path} {self.client}({mode},sync,no_subtree_check)" + + def render_ganesha(self) -> str: + """Render this export as an NFS-Ganesha ``EXPORT`` block.""" + access = "RO" if self.read_only else "RW" + pseudo = self.pseudo or str(self.path) + return ( + "EXPORT {\n" + f" Export_Id = {self.export_id};\n" + f' Path = "{self.path}";\n' + f' Pseudo = "{pseudo}";\n' + " Protocols = 4;\n" + " Transports = TCP;\n" + f" Access_Type = {access};\n" + " Squash = No_Root_Squash;\n" + " SecType = sys;\n" + f' CLIENT {{ Clients = "{self.client}"; Access_Type = {access}; }}\n' + " FSAL { Name = VFS; }\n" + "}\n" + ) + + +def render_exports(exports: Iterable[NfsExport]) -> str: + """Render a kernel-nfsd ``/etc/exports`` file.""" + return "".join(export.render() + "\n" for export in exports) + + +def render_ganesha_conf(exports: Iterable[NfsExport]) -> str: + """Render a complete NFS-Ganesha config (NFSv4 only, VFS FSAL).""" + header = ( + "NFS_CORE_PARAM {\n" + " Protocols = 4;\n" + " NFS_Port = 2049;\n" + "}\n" + "NFSV4 {\n" + " Grace_Period = 5;\n" + " Lease_Lifetime = 5;\n" + "}\n" + ) + return header + "".join(export.render_ganesha() for export in exports) diff --git a/openblade/nas/samba.py b/openblade/nas/samba.py index feb8930..ca42f61 100644 --- a/openblade/nas/samba.py +++ b/openblade/nas/samba.py @@ -1,5 +1,6 @@ """Samba share configuration rendering.""" +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -9,7 +10,39 @@ class SambaShare: name: str path: Path read_only: bool = True + case_sensitive: bool = False + guest_ok: bool = True def render(self) -> str: writable = "no" if self.read_only else "yes" - return f"[{self.name}]\n path = {self.path}\n writeable = {writable}\n" + lines = [ + f"[{self.name}]", + f" path = {self.path}", + f" writeable = {writable}", + f" case sensitive = {'yes' if self.case_sensitive else 'no'}", + f" guest ok = {'yes' if self.guest_ok else 'no'}", + " browseable = yes", + ] + return "\n".join(lines) + "\n" + + +def render_smb_conf(shares: Iterable[SambaShare], *, force_user: str = "root") -> str: + """Render a complete ``smb.conf`` for ``shares``. + + The global section serves guest SMB2/3 only (no SMB1) with POSIX locking on + so byte-range locks are visible across protocols. + """ + header = ( + "[global]\n" + " workgroup = OPENBLADE\n" + " server role = standalone server\n" + " server min protocol = SMB2_10\n" + " map to guest = Bad User\n" + f" guest account = {force_user}\n" + " load printers = no\n" + " disable spoolss = yes\n" + " posix locking = yes\n" + " unix extensions = no\n" + " log file = /var/log/samba/%m.log\n" + ) + return header + "".join("\n" + share.render() for share in shares) From 870e83684c47975bc1fd5b5feaabe5375f9990e0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:00:20 +0000 Subject: [PATCH 020/205] feat(deploy): read-only appliance postcheck and digest rollback postcheck --read-only-appliance-checks wraps every probe in ReadOnlyProbe so no non-GET request reaches the appliance, then GETs /inventory/ and /jobs/recovery. rollback --to refuses tags, redeploys application code only via OPENBLADE_IMAGE, and prints the tape-state notice before and after. Both guards mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/deploy.py | 96 +++++++++++++++++++ scripts/deploy.py | 97 ++++++++++++++++--- tests/unit/test_deploy_release.py | 151 ++++++++++++++++++++++++++++++ 3 files changed, 333 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_deploy_release.py diff --git a/openblade/deploy.py b/openblade/deploy.py index 14cdce7..3632dbb 100644 --- a/openblade/deploy.py +++ b/openblade/deploy.py @@ -11,6 +11,7 @@ from __future__ import annotations +import re from collections.abc import Callable from dataclasses import asdict, dataclass, field from enum import Enum @@ -20,6 +21,7 @@ class Stage(str, Enum): PRECHECK = "precheck" DEPLOY = "deploy" POSTCHECK = "postcheck" + ROLLBACK = "rollback" @dataclass @@ -74,3 +76,97 @@ def run_deploy_pipeline( report.results.append(post) report.promoted = post.ok return report + + +# --- Read-only appliance postcheck ------------------------------------------- + +# A probe takes (method, path) and returns an HTTP status code (same contract as +# openblade.topology). These are the appliance checks run after a deploy; every +# one is a GET so a postcheck can never move a cartridge or start a job. +READ_ONLY_APPLIANCE_PROBES: tuple[tuple[str, str], ...] = ( + ("GET", "/inventory/"), + ("GET", "/jobs/recovery"), +) +_READ_ONLY_METHODS = frozenset({"GET", "HEAD"}) + + +class ReadOnlyProbe: + """Wraps a probe so that only GET/HEAD requests are ever sent. + + A non-read-only request is never forwarded: it is recorded in ``skipped`` and + answered with 405 (which topology treats as "present, not verified"), so the + caller can report exactly which checks were not exercised. + """ + + def __init__(self, probe: Callable[[str, str], int]) -> None: + self._probe = probe + self.skipped: list[str] = [] + + def __call__(self, method: str, path: str) -> int: + if method.upper() not in _READ_ONLY_METHODS: + self.skipped.append(f"{method.upper()} {path}") + return 405 + return self._probe(method.upper(), path) + + +def run_read_only_appliance_checks(probe: Callable[[str, str], int]) -> StageResult: + """GET each appliance endpoint; any non-2xx status is a blocking finding.""" + guarded = probe if isinstance(probe, ReadOnlyProbe) else ReadOnlyProbe(probe) + failures = [ + f"{method} {path} -> {status}" + for method, path in READ_ONLY_APPLIANCE_PROBES + if not 200 <= (status := guarded(method, path)) < 300 + ] + ok = not failures + return StageResult( + Stage.POSTCHECK, + ok, + "read-only appliance checks OK" if ok else f"appliance checks failed: {failures}", + findings=failures, + ) + + +# --- Rollback ----------------------------------------------------------------- + +ROLLBACK_NOTICE = ( + "!!! ROLLBACK REDEPLOYS APPLICATION CODE ONLY !!!\n" + "Physical tape state is NEVER rolled back: cartridges moved, written or loaded\n" + "by the rolled-back version stay where they are. Review reconciliation at\n" + "GET /jobs/recovery before resuming operations." +) + +_DIGEST_RE = re.compile(r"^(?:(?P[a-z0-9][a-z0-9._/:-]*)@)?(?Psha256:[0-9a-f]{64})$") + + +class InvalidDigestError(ValueError): + """The rollback target is not an immutable ``sha256:`` digest reference.""" + + +def parse_image_digest(ref: str, default_image: str) -> str: + """Return ``image@sha256:...`` for a bare digest or full digest reference. + + Tags are refused: a rollback must name the exact bytes that ran before. + """ + m = _DIGEST_RE.match(ref.strip()) + if m is None: + raise InvalidDigestError( + "rollback target must be sha256:<64 hex> or @sha256:<64 hex>" + ) + return f"{m.group('image') or default_image}@{m.group('digest')}" + + +def run_rollback( + *, + image_ref: str, + redeploy: Callable[[str], StageResult], + notify: Callable[[str], None], +) -> StageResult: + """Redeploy ``image_ref`` (application code only), announcing the tape caveat. + + The notice is emitted before the redeploy and again after it, whatever the + outcome, so it cannot be missed in a long log. + """ + notify(ROLLBACK_NOTICE) + result = redeploy(image_ref) + notify(ROLLBACK_NOTICE) + return StageResult(Stage.ROLLBACK, result.ok, result.detail, findings=list(result.findings)) diff --git a/scripts/deploy.py b/scripts/deploy.py index e63d355..4996b01 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -20,9 +20,19 @@ import sys import urllib.error import urllib.request +from dataclasses import asdict from openblade.config_validation import is_deployable, validate_config -from openblade.deploy import Stage, StageResult, run_deploy_pipeline +from openblade.deploy import ( + InvalidDigestError, + ReadOnlyProbe, + Stage, + StageResult, + parse_image_digest, + run_deploy_pipeline, + run_read_only_appliance_checks, + run_rollback, +) from openblade.topology import is_healthy_topology, verify_topology @@ -34,12 +44,12 @@ def _precheck() -> StageResult: return StageResult(Stage.PRECHECK, ok, detail, findings=blocking) -def _deploy(cmd: list[str] | None) -> StageResult: +def _deploy(cmd: list[str] | None, env: dict[str, str] | None = None) -> StageResult: if not cmd: return StageResult(Stage.DEPLOY, True, "skipped (no --deploy-cmd)") # `cmd` is already an argv list (shlex.split of --deploy-cmd) and no shell=True: # an operator string can therefore never become shell syntax. - completed = subprocess.run(cmd, check=False) + completed = subprocess.run(cmd, check=False, env=env) ok = completed.returncode == 0 return StageResult(Stage.DEPLOY, ok, f"`{' '.join(cmd)}` exited {completed.returncode}") @@ -66,8 +76,8 @@ def _postcheck_in_process() -> StageResult: ) -def _postcheck_live(base_url: str) -> StageResult: - def probe(method: str, path: str) -> int: +def _postcheck_live(base_url: str, *, read_only_appliance_checks: bool = False) -> StageResult: + def raw_probe(method: str, path: str) -> int: req = urllib.request.Request(base_url.rstrip("/") + path, method=method) try: # The URL is operator-supplied (--postcheck-url), so the scheme is not @@ -84,15 +94,19 @@ def probe(method: str, path: str) -> int: # satisfied by _AllWired and only endpoint reachability is meaningful. Pass no # emulator URLs so the fleet check surfaces its honest "unverified" warning # rather than falsely reporting the remote fleet as configured. + # In read-only appliance mode every request goes through ReadOnlyProbe, so a + # non-GET topology probe (e.g. the login POST) is never sent; it is reported. + probe = ReadOnlyProbe(raw_probe) if read_only_appliance_checks else raw_probe findings = verify_topology(probe=probe, context=_AllWired(), emulator_urls=[]) ok = is_healthy_topology(findings) blocking = [f.code for f in findings if f.severity == "blocking"] - return StageResult( - Stage.POSTCHECK, - ok, - "live topology OK" if ok else f"blocking: {blocking}", - findings=blocking, - ) + detail = "live topology OK" if ok else f"blocking: {blocking}" + if isinstance(probe, ReadOnlyProbe): + appliance = run_read_only_appliance_checks(probe) + ok = ok and appliance.ok + blocking += appliance.findings + detail += f"; {appliance.detail}; not sent (non-GET): {probe.skipped}" + return StageResult(Stage.POSTCHECK, ok, detail, findings=blocking) class _AllWired: @@ -102,7 +116,68 @@ def __getattr__(self, _name: str) -> object: return self +def _print_report(results: list[StageResult], as_json: bool) -> None: + if as_json: + print(json.dumps([asdict(r) | {"stage": r.stage.value} for r in results], indent=2)) + return + for r in results: + print(f" {'✓' if r.ok else '✗'} {r.stage.value}: {r.detail}") + + +def postcheck_main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(prog="deploy.py postcheck") + ap.add_argument("--base-url", required=True, help="the deployed appliance to verify") + ap.add_argument( + "--read-only-appliance-checks", + action="store_true", + help="also GET /inventory and /jobs/recovery; never sends a non-GET request", + ) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + result = _postcheck_live( + args.base_url, read_only_appliance_checks=args.read_only_appliance_checks + ) + _print_report([result], args.json) + return 0 if result.ok else 1 + + +def rollback_main(argv: list[str]) -> int: + # Deliberately no option that touches tape state: rollback redeploys the + # application image only (asserted by tests/unit/test_deploy_release.py). + ap = argparse.ArgumentParser(prog="deploy.py rollback") + ap.add_argument( + "--to", required=True, help="previous image digest: sha256: or @sha256:" + ) + ap.add_argument( + "--image", default="ghcr.io/amantux/openblade", help="image name for a bare digest" + ) + ap.add_argument( + "--deploy-cmd", + required=True, + help="redeploy command; receives OPENBLADE_IMAGE=@ in its environment", + ) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + try: + image_ref = parse_image_digest(args.to, args.image) + except InvalidDigestError as exc: + print(f"rollback refused: {exc}", file=sys.stderr) + return 2 + cmd = shlex.split(args.deploy_cmd) + result = run_rollback( + image_ref=image_ref, + redeploy=lambda ref: _deploy(cmd, env=os.environ | {"OPENBLADE_IMAGE": ref}), + notify=lambda msg: print(msg, file=sys.stderr), + ) + _print_report([result], args.json) + return 0 if result.ok else 1 + + def main(argv: list[str]) -> int: + if argv and argv[0] == "postcheck": + return postcheck_main(argv[1:]) + if argv and argv[0] == "rollback": + return rollback_main(argv[1:]) ap = argparse.ArgumentParser() ap.add_argument("--deploy-cmd", help="deploy command, e.g. 'docker compose up -d'") ap.add_argument("--skip-deploy", action="store_true", help="run pre/post checks only") diff --git a/tests/unit/test_deploy_release.py b/tests/unit/test_deploy_release.py new file mode 100644 index 0000000..61737bc --- /dev/null +++ b/tests/unit/test_deploy_release.py @@ -0,0 +1,151 @@ +"""Release-pipeline deploy stages: read-only appliance postcheck and rollback. + +Safety invariants under test: +- the read-only postcheck never sends a non-GET request to the appliance; +- rollback redeploys application code only, has no option that could touch tape + state, refuses a mutable tag, and always prints the tape-state notice. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from openblade.deploy import ( + READ_ONLY_APPLIANCE_PROBES, + ROLLBACK_NOTICE, + InvalidDigestError, + ReadOnlyProbe, + Stage, + StageResult, + parse_image_digest, + run_read_only_appliance_checks, + run_rollback, +) + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) +import deploy as deploy_cli # noqa: E402 - scripts/ is not a package; path set just above + +DIGEST = "sha256:" + "a" * 64 + + +def _recording_probe(status: int = 200) -> tuple[list[tuple[str, str]], ReadOnlyProbe]: + sent: list[tuple[str, str]] = [] + + def probe(method: str, path: str) -> int: + sent.append((method, path)) + return status + + return sent, ReadOnlyProbe(probe) + + +def test_appliance_checks_are_all_gets() -> None: + assert {m for m, _ in READ_ONLY_APPLIANCE_PROBES} == {"GET"} + assert {p for _, p in READ_ONLY_APPLIANCE_PROBES} == {"/inventory/", "/jobs/recovery"} + + +def test_appliance_checks_pass_on_2xx() -> None: + sent, probe = _recording_probe(200) + result = run_read_only_appliance_checks(probe) + assert result.ok and result.stage is Stage.POSTCHECK + assert sent == list(READ_ONLY_APPLIANCE_PROBES) + + +@pytest.mark.parametrize("status", [404, 500, 503, 307]) +def test_appliance_checks_fail_on_non_2xx(status: int) -> None: + _, probe = _recording_probe(status) + result = run_read_only_appliance_checks(probe) + assert not result.ok + assert len(result.findings) == 2 + + +@pytest.mark.parametrize("method", ["POST", "PUT", "PATCH", "DELETE", "post"]) +def test_read_only_probe_never_forwards_mutating_requests(method: str) -> None: + sent, probe = _recording_probe() + assert probe(method, "/aml/users/login") == 405 + assert sent == [] + assert probe.skipped == [f"{method.upper()} /aml/users/login"] + + +def test_live_postcheck_read_only_mode_sends_only_gets(monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[str] = [] + + class _Resp: + status = 200 + + def __enter__(self) -> _Resp: + return self + + def __exit__(self, *_: object) -> None: + return None + + def fake_urlopen(req: object, timeout: float) -> _Resp: + sent.append(req.get_method()) # type: ignore[attr-defined] + return _Resp() + + monkeypatch.setattr(deploy_cli.urllib.request, "urlopen", fake_urlopen) + result = deploy_cli._postcheck_live("http://appliance", read_only_appliance_checks=True) + assert sent and set(sent) == {"GET"} + assert "not sent (non-GET)" in result.detail + + +@pytest.mark.parametrize( + ("ref", "expected"), + [ + (DIGEST, f"ghcr.io/amantux/openblade@{DIGEST}"), + (f"ghcr.io/other/img@{DIGEST}", f"ghcr.io/other/img@{DIGEST}"), + ], +) +def test_parse_image_digest_accepts_digests(ref: str, expected: str) -> None: + assert parse_image_digest(ref, "ghcr.io/amantux/openblade") == expected + + +@pytest.mark.parametrize("ref", ["latest", "ghcr.io/amantux/openblade:0.5.0", "sha256:abc", ""]) +def test_parse_image_digest_refuses_tags(ref: str) -> None: + with pytest.raises(InvalidDigestError): + parse_image_digest(ref, "ghcr.io/amantux/openblade") + + +def test_rollback_prints_notice_and_redeploys_image_only() -> None: + notices: list[str] = [] + redeployed: list[str] = [] + + def redeploy(ref: str) -> StageResult: + redeployed.append(ref) + return StageResult(Stage.DEPLOY, False, "exited 1") + + result = run_rollback(image_ref=f"img@{DIGEST}", redeploy=redeploy, notify=notices.append) + assert redeployed == [f"img@{DIGEST}"] + assert result.stage is Stage.ROLLBACK and not result.ok + assert notices == [ROLLBACK_NOTICE, ROLLBACK_NOTICE] + assert "tape state is NEVER rolled back" in ROLLBACK_NOTICE + assert "/jobs/recovery" in ROLLBACK_NOTICE + + +def test_rollback_cli_has_no_tape_state_option(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + deploy_cli.rollback_main(["--help"]) + help_text = capsys.readouterr().out.lower() + for forbidden in ("tape", "cartridge", "inventory", "unload", "move", "slot", "drive"): + assert f"--{forbidden}" not in help_text + assert set(help_text.split()) & {"--to", "--image", "--deploy-cmd", "--json"} == { + "--to", + "--image", + "--deploy-cmd", + "--json", + } + + +def test_rollback_cli_refuses_tag(capsys: pytest.CaptureFixture[str]) -> None: + rc = deploy_cli.main(["rollback", "--to", "latest", "--deploy-cmd", "true"]) + assert rc == 2 + assert "rollback refused" in capsys.readouterr().err + + +def test_rollback_cli_passes_digest_and_prints_notice(capsys: pytest.CaptureFixture[str]) -> None: + cmd = f"{sys.executable} -c \"import os,sys; sys.exit(0 if os.environ['OPENBLADE_IMAGE'].endswith('{DIGEST}') else 3)\"" + rc = deploy_cli.main(["rollback", "--to", DIGEST, "--deploy-cmd", cmd]) + assert rc == 0 + assert capsys.readouterr().err.count("Physical tape state is NEVER rolled back") == 2 From 64bb29eb4fa0895901525a6dfaa6152dc9653be5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:00:27 +0000 Subject: [PATCH 021/205] feat(fuse): byte-budget LRU cache and batching Hydrator over the restore service Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 99 +++++++++++++- openblade/fuse/hydration.py | 266 +++++++++++++++++++++++++++++++++++- 2 files changed, 360 insertions(+), 5 deletions(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 379c481..95e8585 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -3,13 +3,99 @@ from __future__ import annotations import hashlib +import os +import threading +from collections import OrderedDict from pathlib import Path +class CacheEntryInUseError(RuntimeError): + """Raised when eviction is asked to drop a file that still has open handles.""" + + +class CacheChecksumError(ValueError): + """Raised when staged bytes do not match the catalog checksum.""" + + class HydrationCache: - def __init__(self, cache_dir: str) -> None: + """Checksum-addressed staging cache. + + ``max_bytes`` turns on a byte budget with LRU eviction. Entries with open + handles (``acquire``/``release`` refcount) are never evicted; the budget may + be exceeded rather than drop a file a reader is using. + """ + + def __init__(self, cache_dir: str, max_bytes: int | None = None) -> None: self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) + self.max_bytes = max_bytes + self._lock = threading.RLock() + self._lru: OrderedDict[str, int] = OrderedDict() + self._refs: dict[str, int] = {} + + # -- refcounting / LRU --------------------------------------------------- + def acquire(self, checksum: str) -> None: + with self._lock: + self._refs[checksum] = self._refs.get(checksum, 0) + 1 + if checksum in self._lru: + self._lru.move_to_end(checksum) + + def release(self, checksum: str) -> None: + with self._lock: + count = self._refs.get(checksum, 0) - 1 + if count <= 0: + self._refs.pop(checksum, None) + else: + self._refs[checksum] = count + self._enforce_budget() + + def open_count(self, checksum: str) -> int: + with self._lock: + return self._refs.get(checksum, 0) + + def used_bytes(self) -> int: + with self._lock: + return sum(self._lru.values()) + + def _track(self, checksum: str, size: int) -> None: + with self._lock: + self._lru[checksum] = size + self._lru.move_to_end(checksum) + self._enforce_budget() + + def _enforce_budget(self) -> None: + if self.max_bytes is None: + return + for checksum in list(self._lru): + if sum(self._lru.values()) <= self.max_bytes: + return + if self._refs.get(checksum, 0) > 0: + continue # never evict an open file + self.evict(checksum) + + def store_verified(self, checksum: str, staged: Path) -> Path: + """Move ``staged`` into the cache only if its sha256 matches. + + Bytes are written to a temp name beside the final key and renamed only + after the checksum verifies, so a failed restore never leaves a partial + file under the checksum name. + """ + final = self.cache_key(checksum) + final.parent.mkdir(parents=True, exist_ok=True) + tmp = final.with_name(f".{checksum}.{os.getpid()}.{threading.get_ident()}.part") + digest = hashlib.sha256() + try: + with staged.open("rb") as src, tmp.open("wb") as dst: + for chunk in iter(lambda: src.read(1 << 20), b""): + digest.update(chunk) + dst.write(chunk) + if digest.hexdigest() != checksum: + raise CacheChecksumError(f"staged bytes for {checksum} failed verification") + os.replace(tmp, final) + finally: + tmp.unlink(missing_ok=True) + self._track(checksum, final.stat().st_size) + return final def cache_key(self, checksum: str) -> Path: return self.cache_dir / checksum[:2] / checksum @@ -22,6 +108,7 @@ def store(self, checksum: str, data: bytes) -> Path: path.parent.mkdir(parents=True, exist_ok=True) with path.open("wb") as handle: handle.write(data) + self._track(checksum, len(data)) return path def retrieve(self, checksum: str) -> bytes: @@ -37,6 +124,10 @@ def retrieve(self, checksum: str) -> bytes: return data def evict(self, checksum: str) -> None: - path = self.cache_key(checksum) - if path.exists(): - path.unlink() + with self._lock: + if self._refs.get(checksum, 0) > 0: + raise CacheEntryInUseError(f"{checksum} has open handles") + self._lru.pop(checksum, None) + path = self.cache_key(checksum) + if path.exists(): + path.unlink() diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 17e80ad..09e5dfa 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -2,7 +2,16 @@ from __future__ import annotations -from dataclasses import dataclass +import json +import os +import threading +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, Protocol + +from openblade.fuse.cache import CacheChecksumError, HydrationCache @dataclass @@ -23,3 +32,258 @@ def pending(self) -> list[HydrationRequest]: def clear(self) -> None: self._queue.clear() + + +# -- hydration data plane ---------------------------------------------------- +# +# Layering: the Hydrator owns tickets, batching and the cache hand-off; a +# RestoreEngine owns *how* bytes come back from tape. The production engine goes +# through RestoreService (the job machinery). Nothing here touches a tape +# backend, formats media, or moves cartridges directly. + +DEFAULT_HYDRATE_TIMEOUT_S = 300.0 +DEFAULT_BATCH_WINDOW_MS = 250 + + +def hydrate_timeout_from_env() -> float: + raw = os.environ.get("OPENBLADE_FUSE_HYDRATE_TIMEOUT", "").strip() + return float(raw) if raw else DEFAULT_HYDRATE_TIMEOUT_S + + +def batch_window_from_env() -> float: + raw = os.environ.get("OPENBLADE_FUSE_BATCH_WINDOW_MS", "").strip() + return (int(raw) if raw else DEFAULT_BATCH_WINDOW_MS) / 1000.0 + + +class HydrationState(str, Enum): + OFFLINE = "offline" + HYDRATING = "hydrating" + ONLINE = "online" + + +class HydrationFailedError(RuntimeError): + """A restore for hydration failed; the message is curated, never raw.""" + + +class HydrationTimeoutError(TimeoutError): + """The file did not come online within the wait budget.""" + + +@dataclass +class HydrationTicket: + catalog_path: str + checksum: str + tape_key: str + job_id: str | None = None + error: str | None = None + done: threading.Event = field(default_factory=threading.Event) + + +class RestoreEngine(Protocol): + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + """Restore every path (all on one tape) and return staged file paths.""" + + def find_active_job(self, catalog_path: str) -> str | None: + """Return the id of a pending/running restore job for the path, if any.""" + + def resume(self, job_id: str) -> Path: + """Wait for an existing job to finish and return its staged file.""" + + +class JobRestoreEngine: + """Production engine: one restore job per file via ``RestoreService``. + + Files in one batch share a tape and are restored back to back, so the + library loads that cartridge once (scheduler/lease policy is the jobs + layer's concern, not ours). + """ + + def __init__(self, restore_service: Any, catalog: Any, staging_root: Path) -> None: + self.restore_service = restore_service + self.catalog = catalog + self.staging_root = staging_root + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + del tape_key + staged: dict[str, Path] = {} + for path in catalog_paths: + record = self.catalog.get_file_record(path) + if record is None: + raise HydrationFailedError(f"{path} is not in the catalog") + dest = self.staging_root / record.id + dest.parent.mkdir(parents=True, exist_ok=True) + job = self.restore_service.enqueue(path, dest) + if job.state != "completed": + raise HydrationFailedError(f"restore job {job.id} for {path} ended {job.state}") + staged[path] = dest + return staged + + def find_active_job(self, catalog_path: str) -> str | None: + for job in self.catalog.list_jobs(): + if job.job_type != "restore" or job.state not in {"pending", "running"}: + continue + if json.loads(job.metadata_json or "{}").get("catalog_path") == catalog_path: + return str(job.id) + return None + + def resume(self, job_id: str) -> Path: + while True: + job = self.catalog.get_job(job_id) + if job is None: + raise HydrationFailedError(f"restore job {job_id} disappeared") + if job.state == "completed": + return Path(json.loads(job.metadata_json)["dest_path"]) + if job.state not in {"pending", "running"}: + raise HydrationFailedError(f"restore job {job_id} ended {job.state}") + time.sleep(0.5) + + +class Hydrator: + def __init__( + self, + catalog: Any, + cache: HydrationCache, + engine: RestoreEngine, + *, + batch_window_s: float | None = None, + ) -> None: + self.catalog = catalog + self.cache = cache + self.engine = engine + self.batch_window_s = batch_window_from_env() if batch_window_s is None else batch_window_s + self._lock = threading.Lock() + self._tickets: dict[str, HydrationTicket] = {} + self._pending: dict[str, list[HydrationTicket]] = {} + self._timers: dict[str, threading.Timer] = {} + self._workers: list[threading.Thread] = [] + self._errors: dict[str, str] = {} + self._closed = False + + def _tape_key(self, catalog_path: str, record: Any) -> str: + try: + _, instance = self.catalog.get_latest_instance_for_path(catalog_path) + return str(instance.barcode) + except FileNotFoundError: + return f"vg:{record.volume_group_id}" + + def status(self, catalog_path: str) -> HydrationState: + record = self.catalog.get_file_record(catalog_path) + if record is not None and self.cache.is_cached(record.checksum_sha256): + return HydrationState.ONLINE + with self._lock: + ticket = self._tickets.get(catalog_path) + if ticket is not None and not ticket.done.is_set(): + return HydrationState.HYDRATING + return HydrationState.OFFLINE + + def last_error(self, catalog_path: str) -> str | None: + with self._lock: + return self._errors.get(catalog_path) + + def request(self, catalog_path: str) -> HydrationTicket: + record = self.catalog.get_file_record(catalog_path) + if record is None: + raise FileNotFoundError(catalog_path) + with self._lock: + if self._closed: + raise HydrationFailedError("hydrator is shut down") + existing = self._tickets.get(catalog_path) + if existing is not None and not existing.done.is_set(): + return existing + ticket = HydrationTicket(catalog_path, record.checksum_sha256, "") + if self.cache.is_cached(record.checksum_sha256): + ticket.done.set() + return ticket + self._tickets[catalog_path] = ticket + # Restart-safety: a job already in flight for this path is re-attached, + # not duplicated. + job_id = self.engine.find_active_job(catalog_path) + if job_id is not None: + ticket.job_id = job_id + self._spawn(self._resume, ticket) + return ticket + ticket.tape_key = self._tape_key(catalog_path, record) + with self._lock: + self._pending.setdefault(ticket.tape_key, []).append(ticket) + if ticket.tape_key not in self._timers: + timer = threading.Timer(self.batch_window_s, self._flush, args=(ticket.tape_key,)) + timer.daemon = True + self._timers[ticket.tape_key] = timer + timer.start() + return ticket + + def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: + if not ticket.done.wait(timeout): + raise HydrationTimeoutError(f"{ticket.catalog_path} still hydrating") + if ticket.error is not None: + raise HydrationFailedError(ticket.error) + return self.cache.retrieve(ticket.checksum) + + def shutdown(self, timeout: float = 30.0) -> None: + """Cancel batches not yet started and wait for in-flight restores.""" + with self._lock: + self._closed = True + timers, self._timers = self._timers, {} + pending, self._pending = self._pending, {} + workers = list(self._workers) + for timer in timers.values(): + timer.cancel() + for tickets in pending.values(): + for ticket in tickets: + self._fail(ticket, "hydration cancelled by unmount") + for worker in workers: + worker.join(timeout) + + # -- internals ----------------------------------------------------------- + def _spawn(self, target: Any, *args: Any) -> None: + worker = threading.Thread(target=target, args=args, daemon=True) + with self._lock: + self._workers = [w for w in self._workers if w.is_alive()] + self._workers.append(worker) + worker.start() + + def _flush(self, tape_key: str) -> None: + with self._lock: + self._timers.pop(tape_key, None) + batch = self._pending.pop(tape_key, []) + if self._closed or not batch: + return + self._workers.append(threading.current_thread()) + try: + staged = self.engine.restore_batch(tape_key, [t.catalog_path for t in batch]) + except (HydrationFailedError, OSError) as exc: + reason = str(exc) if isinstance(exc, HydrationFailedError) else "restore I/O error" + for ticket in batch: + self._fail(ticket, reason) + return + for ticket in batch: + self._commit(ticket, staged.get(ticket.catalog_path)) + + def _resume(self, ticket: HydrationTicket) -> None: + assert ticket.job_id is not None + try: + self._commit(ticket, self.engine.resume(ticket.job_id)) + except (HydrationFailedError, OSError, KeyError): + self._fail(ticket, f"restore job {ticket.job_id} did not complete") + + def _commit(self, ticket: HydrationTicket, staged: Path | None) -> None: + if staged is None: + self._fail(ticket, "restore produced no file") + return + try: + self.cache.store_verified(ticket.checksum, staged) + except CacheChecksumError: + self._fail(ticket, "restored bytes failed checksum verification") + return + except OSError: + self._fail(ticket, "restore I/O error") + return + with self._lock: + self._errors.pop(ticket.catalog_path, None) + ticket.done.set() + + def _fail(self, ticket: HydrationTicket, reason: str) -> None: + ticket.error = reason + with self._lock: + self._errors[ticket.catalog_path] = reason + ticket.done.set() From 331adbf92b8461993d9db5aa9f1bcfe33d6ccae0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:05 +0000 Subject: [PATCH 022/205] feat(fuse): state xattr, open-handle refcount, blocking/EAGAIN hydration, unmount shutdown Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/mount.py | 67 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/openblade/fuse/mount.py b/openblade/fuse/mount.py index 4bf5e75..61e339b 100644 --- a/openblade/fuse/mount.py +++ b/openblade/fuse/mount.py @@ -35,6 +35,15 @@ from typing import Any from openblade.fuse.filesystem import CatalogFilesystem, VirtualDirEntry +from openblade.fuse.hydration import ( + HydrationFailedError, + HydrationState, + HydrationTimeoutError, + Hydrator, + hydrate_timeout_from_env, +) + +STATE_XATTR = "user.openblade.state" logger = logging.getLogger(__name__) @@ -100,11 +109,20 @@ def __init__( filesystem: CatalogFilesystem, *, hydrator: Callable[[str], bytes] | None = None, + data_plane: Hydrator | None = None, + blocking: bool = True, + hydrate_timeout: float | None = None, uid: int | None = None, gid: int | None = None, ) -> None: self.filesystem = filesystem self.hydrator = hydrator + self.data_plane = data_plane + self.blocking = blocking + self.hydrate_timeout = ( + hydrate_timeout_from_env() if hydrate_timeout is None else hydrate_timeout + ) + self._handle_checksums: dict[int, str] = {} self.uid = os.getuid() if uid is None else uid self.gid = os.getgid() if gid is None else gid self._lock = threading.Lock() @@ -146,6 +164,9 @@ def init(self, path: str) -> None: def destroy(self, path: str) -> None: del path + if self.data_plane is not None: + # Unmount: cancel batches not yet started, wait for in-flight restores. + self.data_plane.shutdown() with self._lock: self._open_files.clear() @@ -201,12 +222,22 @@ def releasedir(self, path: str, fh: int) -> int: return 0 def getxattr(self, path: str, name: str, position: int = 0) -> bytes: - del path, name, position - raise OSError(errno.ENODATA, os.strerror(errno.ENODATA)) + del position + entry = self._entry(path) + if entry.is_dir or name != STATE_XATTR: + raise OSError(errno.ENODATA, os.strerror(errno.ENODATA)) + return self._state(str(entry.path)).value.encode() def listxattr(self, path: str) -> list[str]: - del path - return [] + entry = self._entry(path) + return [] if entry.is_dir else [STATE_XATTR] + + def _state(self, catalog_path: str) -> HydrationState: + if self.data_plane is not None: + return self.data_plane.status(catalog_path) + if self.filesystem.is_hydrated(catalog_path): + return HydrationState.ONLINE + return HydrationState.OFFLINE # -- reads -------------------------------------------------------------- def open(self, path: str, flags: int) -> int: @@ -220,6 +251,11 @@ def open(self, path: str, flags: int) -> int: handle = self._next_handle self._next_handle += 1 self._open_files[handle] = data + record = self.filesystem.catalog.get_file_record(str(entry.path)) + if record is not None: + self.filesystem.cache.acquire(record.checksum_sha256) + with self._lock: + self._handle_checksums[handle] = record.checksum_sha256 return handle def read(self, path: str, size: int, offset: int, fh: int) -> bytes: @@ -243,6 +279,9 @@ def release(self, path: str, fh: int) -> int: del path with self._lock: self._open_files.pop(fh, None) + checksum = self._handle_checksums.pop(fh, None) + if checksum is not None: + self.filesystem.cache.release(checksum) return 0 def _materialise(self, catalog_path: str) -> bytes: @@ -261,6 +300,8 @@ def _materialise(self, catalog_path: str) -> bytes: except (OSError, ValueError) as exc: logger.warning("fuse: cache read failed for %s: %s", catalog_path, exc) raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + if self.data_plane is not None: + return self._hydrate(catalog_path) if self.hydrator is None: logger.warning( "fuse: %s is not in the staging cache; returning EIO. " @@ -276,6 +317,18 @@ def _materialise(self, catalog_path: str) -> bytes: logger.warning("fuse: hydration failed for %s: %s", catalog_path, exc) raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + def _hydrate(self, catalog_path: str) -> bytes: + assert self.data_plane is not None + ticket = self.data_plane.request(catalog_path) + timeout = self.hydrate_timeout if self.blocking else 0.0 + try: + return self.data_plane.wait(ticket, timeout) + except HydrationTimeoutError: + raise OSError(errno.EAGAIN, os.strerror(errno.EAGAIN), catalog_path) from None + except (HydrationFailedError, OSError, ValueError): + logger.warning("fuse: hydration failed for %s", catalog_path) + raise OSError(errno.EIO, os.strerror(errno.EIO), catalog_path) from None + def _catalog_path(path: str) -> str: """Map a FUSE path onto a catalog path, rejecting traversal.""" @@ -290,6 +343,8 @@ def mount_catalog( mountpoint: str, *, hydrator: Callable[[str], bytes] | None = None, + data_plane: Hydrator | None = None, + blocking: bool = True, allow_other: bool = False, foreground: bool = True, ) -> None: @@ -301,7 +356,9 @@ def mount_catalog( fuse = load_fuse() if not os.path.isdir(mountpoint): raise NotADirectoryError(f"Mount point is not a directory: {mountpoint}") - operations = CatalogFuseOperations(filesystem, hydrator=hydrator) + operations = CatalogFuseOperations( + filesystem, hydrator=hydrator, data_plane=data_plane, blocking=blocking + ) fuse.FUSE( operations, mountpoint, From 0babc3a4d57e7584f83b5be6f613df3a97edb500 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:14 +0000 Subject: [PATCH 023/205] ci: pin actions to SHAs and scope permissions per job Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/auto-merge-trusted.yml | 13 +++++++++---- .github/workflows/copilot-setup-steps.yml | 7 ++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/auto-merge-trusted.yml b/.github/workflows/auto-merge-trusted.yml index 8318a4c..62b82ba 100644 --- a/.github/workflows/auto-merge-trusted.yml +++ b/.github/workflows/auto-merge-trusted.yml @@ -1,13 +1,15 @@ name: Auto-merge trusted PRs -on: +# dangerous-triggers is ignored on `on:` below: this workflow runs only the GitHub API via +# github-script on the trusted-author allowlist; never checks out or executes +# PR code, which is what makes workflow_run dangerous. +on: # zizmor: ignore[dangerous-triggers] -- see comment above workflow_run: workflows: ["CI"] types: [completed] permissions: - contents: write - pull-requests: write + contents: read jobs: merge-trusted-pr: @@ -15,9 +17,12 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - name: Merge trusted PRs targeting master - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const owner = context.repo.owner; diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 8f71130..807f1f0 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -9,6 +9,9 @@ on: paths: - .github/workflows/copilot-setup-steps.yml +permissions: + contents: read + jobs: copilot-setup-steps: runs-on: ubuntu-latest @@ -18,4 +21,6 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + # zizmor: ignore[artipacked] -- the Copilot coding agent works in this + # checkout and may need its git credentials; nothing here uploads artifacts. + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 From 0ff26ed3a081da270054385f7e12c8c3158e69c8 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:15 +0000 Subject: [PATCH 024/205] ci: consolidate emulator boot/test into reusable workflow (SHA-pinned) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/_emulator-boot-test.yml | 112 ++++++++++++++++ .github/workflows/emulator-change-gates.yml | 65 +++------- .github/workflows/i3-emulator-compliance.yml | 130 +++++-------------- 3 files changed, 160 insertions(+), 147 deletions(-) create mode 100644 .github/workflows/_emulator-boot-test.yml diff --git a/.github/workflows/_emulator-boot-test.yml b/.github/workflows/_emulator-boot-test.yml new file mode 100644 index 0000000..5dce2fe --- /dev/null +++ b/.github/workflows/_emulator-boot-test.yml @@ -0,0 +1,112 @@ +# Reusable: boot the mock-backend emulator API on 127.0.0.1:8000, wait for +# /health, then run pytest against it. Callers: emulator-change-gates.yml, +# i3-emulator-compliance.yml. Check names become " / boot-test". +name: _emulator-boot-test + +on: + workflow_call: + inputs: + timing_profile: + description: I3_TIMING_PROFILE for both the server and pytest. + type: string + default: instant + pytest_args: + description: >- + Space-separated pytest arguments (paths, -m, -q ...). Word-split by + the shell on purpose; must not contain quotes. + type: string + required: true + followup_pytest_args: + description: Optional second pytest run against the same booted API. + type: string + default: "" + python_version: + type: string + default: "3.12" + emulator_profile: + description: EMULATOR_PROFILE for the booted server and pytest ("" = default shape). + type: string + default: "" + validate_contract: + description: Run tools/emulator_spec/validate_cross_repo_contract.py first. + type: boolean + default: true + db_name: + description: Basename of the throwaway sqlite DB under /tmp. + type: string + default: openblade-emulator + artifact_name: + description: Upload test-results/ + the API log under this name ("" = no upload). + type: string + default: "" + +permissions: + contents: read + +jobs: + boot-test: + name: boot-test + runs-on: ubuntu-latest + env: + I3_TEST_MODE: emulator + I3_TIMING_PROFILE: ${{ inputs.timing_profile }} + I3_AML_URL: http://127.0.0.1:8000 + PYTEST_ARGS: ${{ inputs.pytest_args }} + FOLLOWUP_PYTEST_ARGS: ${{ inputs.followup_pytest_args }} + DB_PATH: /tmp/${{ inputs.db_name }}.db + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + # Job-wide EMULATOR_PROFILE only when one was given: an empty value must + # mean "unset" (default shape) for every later step, not "profile named ''". + - name: Export EMULATOR_PROFILE + env: + PROFILE: ${{ inputs.emulator_profile }} + run: | + if [ -n "$PROFILE" ]; then echo "EMULATOR_PROFILE=$PROFILE" >> "$GITHUB_ENV"; fi + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ inputs.python_version }} + cache: "pip" + - run: pip install -e '.[dev]' + - if: ${{ inputs.validate_contract }} + run: python3 tools/emulator_spec/validate_cross_repo_contract.py + - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py + - name: Boot emulator API + run: | + rm -f "$DB_PATH" + OPENBLADE_BACKEND=mock OPENBLADE_DB_URL="sqlite:///$DB_PATH" \ + nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & + - name: Wait for API health + run: | + python - <<'PY' + import time + import urllib.request + for _ in range(60): + try: + with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: + if response.status == 200: + raise SystemExit(0) + except Exception: + time.sleep(1) + raise SystemExit("API did not become healthy in time") + PY + - run: mkdir -p test-results + - name: Run pytest against the booted emulator + run: | + # shellcheck disable=SC2086 # PYTEST_ARGS is a word list by contract + python -m pytest $PYTEST_ARGS + - name: Follow-up pytest run + if: ${{ inputs.followup_pytest_args != '' }} + run: | + # shellcheck disable=SC2086 # word list by contract + python -m pytest $FOLLOWUP_PYTEST_ARGS + - name: Upload artifacts + if: ${{ always() && inputs.artifact_name != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact_name }} + path: | + test-results/ + /tmp/openblade-api.log diff --git a/.github/workflows/emulator-change-gates.yml b/.github/workflows/emulator-change-gates.yml index 7727d0b..62c6007 100644 --- a/.github/workflows/emulator-change-gates.yml +++ b/.github/workflows/emulator-change-gates.yml @@ -52,51 +52,20 @@ permissions: jobs: emulator-contract: name: emulator-contract - runs-on: ubuntu-latest - env: - I3_TIMING_PROFILE: instant - I3_TEST_MODE: emulator - I3_AML_URL: http://127.0.0.1:8000 - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - - run: python3 tools/emulator_spec/validate_cross_repo_contract.py - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - # Boot BEFORE the suites: this ordering was wrong since the workflow was - # written, masked for months because the contract-validator step above - # failed first — the suites below dial 127.0.0.1:8000. - - run: | - rm -f /tmp/openblade-emulator-gates.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-emulator-gates.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: > - python -m pytest - tests/i3/test_14_emulator_profile.py - tests/i3/test_18_config_profiles.py - tests/i3/test_15_manual_matrix_contract.py - tests/i3/test_16_manual_matrix_generated_suites.py - tests/i3/test_17_matrix_endpoint_coverage.py - tests/integration/test_aml_emulator_latency*.py - tests/integration/test_aml_scope_lockdown.py - tests/integration/test_aml_library_spec_parity.py - tests/unit/test_emulator_client_config.py - -q --tb=short - - run: python -m pytest tests/i3/ -m i3 -q --tb=short + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + db_name: openblade-emulator-gates + pytest_args: >- + tests/i3/test_14_emulator_profile.py + tests/i3/test_18_config_profiles.py + tests/i3/test_15_manual_matrix_contract.py + tests/i3/test_16_manual_matrix_generated_suites.py + tests/i3/test_17_matrix_endpoint_coverage.py + tests/integration/test_aml_emulator_latency*.py + tests/integration/test_aml_scope_lockdown.py + tests/integration/test_aml_library_spec_parity.py + tests/unit/test_emulator_client_config.py + -q --tb=short + followup_pytest_args: tests/i3/ -m i3 -q --tb=short diff --git a/.github/workflows/i3-emulator-compliance.yml b/.github/workflows/i3-emulator-compliance.yml index 1625bf1..b62db2f 100644 --- a/.github/workflows/i3-emulator-compliance.yml +++ b/.github/workflows/i3-emulator-compliance.yml @@ -52,7 +52,6 @@ permissions: jobs: i3-emulator-suite: name: i3-emulator-${{ matrix.suite }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -63,56 +62,21 @@ jobs: tests: "tests/i3/test_06_operations.py tests/i3/test_07_ltfs.py tests/i3/test_08_archive_cycle.py tests/i3/test_09_restore_cycle.py" - suite: system tests: "tests/i3/test_10_fault_scenarios.py tests/i3/test_11_diagnostics.py tests/i3/test_12_multi_library.py tests/i3/test_13_ui_scenarios.py" - env: - I3_TEST_MODE: emulator - I3_TIMING_PROFILE: instant - I3_AML_URL: http://127.0.0.1:8000 - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - - run: python3 tools/emulator_spec/validate_cross_repo_contract.py - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - - run: | - rm -f /tmp/openblade-i3-compliance.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-i3-compliance.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: mkdir -p test-results - - run: pytest ${{ matrix.tests }} -q --tb=short --junitxml=test-results/i3-${{ matrix.suite }}.xml - - name: Upload i3 suite artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: i3-emulator-suite-${{ matrix.suite }} - path: | - test-results/i3-${{ matrix.suite }}.xml - /tmp/openblade-api.log + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + db_name: openblade-i3-compliance + pytest_args: ${{ matrix.tests }} -q --tb=short --junitxml=test-results/i3-${{ matrix.suite }}.xml + artifact_name: i3-emulator-suite-${{ matrix.suite }} i3-config-profiles: # Boot the emulator once per supported i3 configuration and assert it behaves # compliantly for that shape — the emulated i3 must behave like a real i3 across # the range of valid configurations (drive count, LTO generation, slot/module - # scaling, partition count). EMULATOR_PROFILE is set at job level so both the - # booted server and the pytest process observe the same shape. + # scaling, partition count). EMULATOR_PROFILE reaches both the booted server + # and the pytest process via the reusable workflow's emulator_profile input. name: i3-config-${{ matrix.profile }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -122,64 +86,32 @@ jobs: - scalar-i3-100-3 - scalar-i3-50-6 - scalar-i3-50-3-lto9 - env: - I3_TEST_MODE: emulator - I3_TIMING_PROFILE: instant - I3_AML_URL: http://127.0.0.1:8000 - EMULATOR_PROFILE: ${{ matrix.profile }} - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "pip" - - run: pip install -e '.[dev]' - - run: python3 tools/emulator_spec/generate_matrix_endpoint_catalog.py - - run: | - rm -f /tmp/openblade-i3-profile.db - OPENBLADE_BACKEND=mock OPENBLADE_DB_URL=sqlite:////tmp/openblade-i3-profile.db \ - nohup uvicorn openblade.api.main:app --host 127.0.0.1 --port 8000 >/tmp/openblade-api.log 2>&1 & - - name: Wait for API health - run: | - python - <<'PY' - import time - import urllib.request - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=2) as response: - if response.status == 200: - raise SystemExit(0) - except Exception: - time.sleep(1) - raise SystemExit("API did not become healthy in time") - PY - - run: mkdir -p test-results - - name: Run the i3 suite against the ${{ matrix.profile }} emulator - run: > - pytest - tests/i3/test_01_auth.py - tests/i3/test_02_inventory.py - tests/i3/test_03_changer.py - tests/i3/test_04_drives.py - tests/i3/test_05_media.py - tests/i3/test_14_emulator_profile.py - tests/i3/test_18_config_profiles.py - -q --tb=short --junitxml=test-results/i3-config-${{ matrix.profile }}.xml - - name: Upload i3 config-profile artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: i3-config-profile-${{ matrix.profile }} - path: | - test-results/i3-config-${{ matrix.profile }}.xml - /tmp/openblade-api.log + # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. + uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] + with: + timing_profile: instant + emulator_profile: ${{ matrix.profile }} + validate_contract: false + db_name: openblade-i3-profile + pytest_args: >- + tests/i3/test_01_auth.py + tests/i3/test_02_inventory.py + tests/i3/test_03_changer.py + tests/i3/test_04_drives.py + tests/i3/test_05_media.py + tests/i3/test_14_emulator_profile.py + tests/i3/test_18_config_profiles.py + -q --tb=short --junitxml=test-results/i3-config-${{ matrix.profile }}.xml + artifact_name: i3-config-profile-${{ matrix.profile }} aml-compatibility-suite: name: aml-compatibility runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -198,7 +130,7 @@ jobs: --junitxml=test-results/aml-compatibility.xml - name: Upload AML compatibility artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: aml-compatibility-suite path: test-results/aml-compatibility.xml From 045d3c10f56e210cf50fa6cf8ddd619d24ccd9ac Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:15 +0000 Subject: [PATCH 025/205] ci: add workflow-lint (actionlint, zizmor, shellcheck, yamllint) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/workflow-lint.yml | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/workflow-lint.yml diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml new file mode 100644 index 0000000..5903cb5 --- /dev/null +++ b/.github/workflows/workflow-lint.yml @@ -0,0 +1,50 @@ +# Static checks for the workflows themselves: actionlint (incl. shellcheck on +# every inline run: block), zizmor (security audit) and yamllint. +name: workflow-lint + +on: + pull_request: + paths: + - ".github/workflows/**" + - ".yamllint" + push: + branches: [master] + paths: + - ".github/workflows/**" + - ".yamllint" + workflow_dispatch: + +permissions: + contents: read + +env: + ACTIONLINT_VERSION: "1.7.12" + # sha256 of actionlint_1.7.12_linux_amd64.tar.gz from the release checksums.txt + ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + +jobs: + workflow-lint: + name: workflow-lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install zizmor and yamllint (pinned) + run: python -m pip install zizmor==1.30.1 yamllint==1.38.0 + - name: Install actionlint (pinned, checksum-verified) + run: | + tarball="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -sSfL -o "$RUNNER_TEMP/$tarball" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/$tarball" + echo "$ACTIONLINT_SHA256 $RUNNER_TEMP/$tarball" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/$tarball" -C "$RUNNER_TEMP" actionlint + - name: actionlint (+ shellcheck on run blocks; shellcheck is preinstalled on ubuntu runners) + run: '"$RUNNER_TEMP/actionlint" -color' + - name: zizmor + run: zizmor --no-progress --min-severity low .github/workflows/ + - name: yamllint + run: yamllint -c .yamllint .github/workflows/ From e5b444ebd22a9f8c90aaa74261a9f20c52898cab Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:35 +0000 Subject: [PATCH 026/205] feat(nas): protocol gateway on_open/on_read_range/on_close/on_evict over the Hydrator Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/protocol_gateway.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 515bb4d..9bafc48 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -19,6 +19,9 @@ from enum import Enum from typing import Any +from openblade.fuse.cache import CacheEntryInUseError +from openblade.fuse.hydration import HydrationFailedError, HydrationState, Hydrator + class GatewayStatus(str, Enum): RUNNING = "running" @@ -367,6 +370,52 @@ def _get_session(self, session_id: str) -> GatewaySession | None: return session return None + # -- hydration entry points (single path for the SMB/NFS rig) ------------ + def attach_hydrator(self, hydrator: Hydrator | None) -> None: + self._hydrator = hydrator + + def _require_hydrator(self) -> Hydrator: + hydrator: Hydrator | None = getattr(self, "_hydrator", None) + if hydrator is None: + raise HydrationFailedError("no hydrator attached to the protocol gateway") + return hydrator + + def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> HydrationState: + """Pin the file and start hydration; optionally wait ``timeout`` seconds.""" + hydrator = self._require_hydrator() + ticket = hydrator.request(catalog_path) + if timeout: + hydrator.wait(ticket, timeout) + state = hydrator.status(catalog_path) + if state is HydrationState.ONLINE: + hydrator.cache.acquire(ticket.checksum) + return state + + def on_read_range( + self, catalog_path: str, offset: int, length: int, *, timeout: float | None = None + ) -> bytes: + hydrator = self._require_hydrator() + data = hydrator.wait(hydrator.request(catalog_path), timeout) + return data[offset : offset + length] + + def on_close(self, catalog_path: str) -> None: + hydrator = self._require_hydrator() + record = hydrator.catalog.get_file_record(catalog_path) + if record is not None: + hydrator.cache.release(record.checksum_sha256) + + def on_evict(self, catalog_path: str) -> bool: + """Drop the cached copy; False when the file still has open handles.""" + hydrator = self._require_hydrator() + record = hydrator.catalog.get_file_record(catalog_path) + if record is None: + return False + try: + hydrator.cache.evict(record.checksum_sha256) + except CacheEntryInUseError: + return False + return True + _gateway = ProtocolGateway() From 62deb3f29fad61232e3c4a7ff6625cb1e47fe880 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:47 +0000 Subject: [PATCH 027/205] feat(nas): manifest v2 with commit marker written last and generation chain finalize_tape_generation() is the single place that orders metadata writes: manifest -> catalog-shard -> tape.json -> commit-.json. Manifest v2 adds generation_id, sibling_tapes, placement and protection; v1 still parses, unknown schemas raise UnsupportedManifestSchemaError. TapeJson gains current_generation/previous_generation (its existing 'generation' is LTO media). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/archive_lifecycle.py | 53 +++++++++++ openblade/nas/catalog_shard.py | 3 + openblade/nas/ltfs_manifest.py | 65 ++++++++++++- openblade/nas/manifest_validator.py | 17 +++- tests/unit/test_ltfs_manifest.py | 132 ++++++++++++++++++++++++++ tests/unit/test_manifest_validator.py | 2 +- 6 files changed, 263 insertions(+), 9 deletions(-) diff --git a/openblade/nas/archive_lifecycle.py b/openblade/nas/archive_lifecycle.py index c6b3ae5..aebf1c1 100644 --- a/openblade/nas/archive_lifecycle.py +++ b/openblade/nas/archive_lifecycle.py @@ -15,6 +15,7 @@ CatalogShardWriter, ) from openblade.nas.ltfs_manifest import ( + CommitMarker, ManifestFileEntry, ManifestJson, TapeJson, @@ -492,6 +493,58 @@ def _resolve_tape_path(self, barcode: str, file_record_id: str) -> str: return "" +def sibling_tapes_from_shard(shard: CatalogShard) -> list[str]: + """Other barcodes needed to restore every file of this generation (from shard records).""" + barcodes: list[str | None] = [] + for dataset in shard.datasets: + barcodes.extend(dataset.tape_set) + barcodes.extend(dataset.shard_set) + return sorted(set(_ordered_unique(barcodes)) - {shard.barcode}) + + +def finalize_tape_generation( + writer: TapeMetadataWriter, + shard_writer: CatalogShardWriter, + barcode: str, + *, + generation_id: str, + manifest: ManifestJson, + shard: CatalogShard, + tape_json: TapeJson, +) -> CommitMarker: + """Write a generation's metadata in the one durable order, commit marker LAST. + + Precondition: all file data for this generation is already on tape. Order: + manifest.json (+.sha256) -> catalog-shard.json (+.sha256) -> tape.json -> commit marker. + A crash before the marker leaves the generation uncommitted; rebuild ignores it. + """ + manifest = manifest.model_copy( + update={"generation_id": generation_id, "sibling_tapes": sibling_tapes_from_shard(shard)} + ) + manifest_sha = writer.write_manifest(barcode, manifest) + writer.write_manifest_checksum(barcode, manifest_sha) + shard_sha = shard_writer.write_shard(barcode, shard) + previous = tape_json.current_generation + writer.write_tape_json( + barcode, + tape_json.model_copy( + update={ + "current_generation": generation_id, + "previous_generation": previous if previous != generation_id else None, + "last_openblade_write_at": _utcnow_iso(), + } + ), + ) + marker = CommitMarker( + generation_id=generation_id, + manifest_sha256=manifest_sha, + catalog_shard_sha256=shard_sha, + written_at=_utcnow_iso(), + ) + writer.write_commit_marker(barcode, marker) + return marker + + def _ordered_unique(values: list[str | None]) -> list[str]: ordered: list[str] = [] seen: set[str] = set() diff --git a/openblade/nas/catalog_shard.py b/openblade/nas/catalog_shard.py index dd6730f..9ee602f 100644 --- a/openblade/nas/catalog_shard.py +++ b/openblade/nas/catalog_shard.py @@ -2,6 +2,7 @@ import json from datetime import datetime +from typing import Any from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator @@ -38,6 +39,8 @@ class CatalogShardDatasetEntry(BaseModel): total_bytes: int = 0 tape_set: list[str] = Field(default_factory=list) shard_set: list[str] = Field(default_factory=list) + # Serialized openblade.domain.protection.ProtectionPolicy; None on pre-v2 shards. + protection: dict[str, Any] | None = None class CatalogShard(BaseModel): diff --git a/openblade/nas/ltfs_manifest.py b/openblade/nas/ltfs_manifest.py index dd51f20..9e362ce 100644 --- a/openblade/nas/ltfs_manifest.py +++ b/openblade/nas/ltfs_manifest.py @@ -8,6 +8,16 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +MANIFEST_SCHEMA_V1 = "openblade.manifest.v1" +MANIFEST_SCHEMA_V2 = "openblade.manifest.v2" +MANIFEST_SCHEMA = MANIFEST_SCHEMA_V2 +SUPPORTED_MANIFEST_SCHEMAS = frozenset({MANIFEST_SCHEMA_V1, MANIFEST_SCHEMA_V2}) +COMMIT_MARKER_SCHEMA = "openblade.commit.v1" + + +class UnsupportedManifestSchemaError(ValueError): + """manifest.json declares a schema this build does not understand (e.g. a newer one).""" + class TapeJson(BaseModel): schema_: str = Field(default="openblade.tape.v1", alias="schema") @@ -23,6 +33,9 @@ class TapeJson(BaseModel): state: str = "active" generation: str = "LTO-8" notes: str = "" + # `generation` above is the LTO media generation; the write-generation chain is below. + current_generation: str | None = None + previous_generation: str | None = None model_config = ConfigDict(populate_by_name=True) @@ -41,7 +54,7 @@ class ManifestFileEntry(BaseModel): class ManifestJson(BaseModel): - schema_: str = Field(default="openblade.manifest.v1", alias="schema") + schema_: str = Field(default=MANIFEST_SCHEMA, alias="schema") barcode: str openblade_tape_id: str volume_group: str = "" @@ -52,6 +65,11 @@ class ManifestJson(BaseModel): file_count: int = 0 total_logical_bytes: int = 0 files: list[ManifestFileEntry] = Field(default_factory=list) + # v2 fields; a v1 manifest parses with these defaults. + generation_id: str = "" + sibling_tapes: list[str] = Field(default_factory=list) + placement: str = "" + protection: dict[str, Any] | None = None model_config = ConfigDict(populate_by_name=True) @@ -88,6 +106,30 @@ class ShardSetManifest(BaseModel): model_config = ConfigDict(populate_by_name=True) +class CommitMarker(BaseModel): + """/.openblade/commit-.json — written LAST; its absence means uncommitted.""" + + schema_: str = Field(default=COMMIT_MARKER_SCHEMA, alias="schema") + generation_id: str + manifest_sha256: str + catalog_shard_sha256: str + written_at: str + + model_config = ConfigDict(populate_by_name=True) + + +def parse_manifest(payload: dict[str, Any]) -> ManifestJson: + """Validate a manifest payload, rejecting schemas newer than this build understands. + + Raises UnsupportedManifestSchemaError for an unknown schema and pydantic's + ValidationError for a malformed supported one. + """ + schema = str(payload.get("schema", MANIFEST_SCHEMA_V1)) + if schema not in SUPPORTED_MANIFEST_SCHEMAS: + raise UnsupportedManifestSchemaError(f"unsupported manifest schema: {schema}") + return ManifestJson.model_validate(payload) + + class ChecksumEntry(BaseModel): checksum: str path: str @@ -166,6 +208,23 @@ def write_shard_set_manifest(self, barcode: str, shard_set: ShardSetManifest) -> shard_set.model_dump(by_alias=True), ) + def write_commit_marker(self, barcode: str, marker: CommitMarker) -> None: + """Write /.openblade/commit-.json. Callers must write it last.""" + self.ensure_openblade_dirs(barcode) + name = f"commit-{self._safe_name(marker.generation_id)}.json" + self._write_json(barcode, self._metadata_path(name), marker.model_dump(by_alias=True)) + + def read_commit_marker(self, barcode: str, generation_id: str) -> CommitMarker | None: + """Read a generation's commit marker; None when missing or corrupt.""" + name = f"commit-{self._safe_name(generation_id)}.json" + try: + payload = self._read_json(barcode, self._metadata_path(name)) + if payload is None: + return None + return CommitMarker.model_validate(payload) + except (json.JSONDecodeError, ValidationError): + return None + def ensure_openblade_dirs(self, barcode: str) -> None: """Ensure all reserved /.openblade/ subdirectories exist in the backend.""" for directory in self.RESERVED_DIRS: @@ -193,8 +252,8 @@ def read_manifest(self, barcode: str) -> ManifestJson | None: payload = self._read_json(barcode, self._metadata_path("manifest.json")) if payload is None: return None - return ManifestJson.model_validate(payload) - except (json.JSONDecodeError, ValidationError): + return parse_manifest(payload) + except (json.JSONDecodeError, ValidationError, UnsupportedManifestSchemaError): return None def initialize_tape( diff --git a/openblade/nas/manifest_validator.py b/openblade/nas/manifest_validator.py index 91b5be5..6facf24 100644 --- a/openblade/nas/manifest_validator.py +++ b/openblade/nas/manifest_validator.py @@ -6,7 +6,13 @@ from pydantic import BaseModel, Field, ValidationError from openblade.nas.catalog_shard import CatalogShard, CatalogShardWriter -from openblade.nas.ltfs_manifest import ManifestJson, TapeMetadataWriter +from openblade.nas.ltfs_manifest import ( + MANIFEST_SCHEMA, + ManifestJson, + TapeMetadataWriter, + UnsupportedManifestSchemaError, + parse_manifest, +) class ManifestValidationResult(BaseModel): @@ -60,7 +66,7 @@ class ManifestValidator: MANIFEST_CHECKSUM_PATH = "/.openblade/manifest.sha256" CATALOG_SHARD_PATH = "/.openblade/catalog-shard.json" CATALOG_SHARD_CHECKSUM_PATH = "/.openblade/catalog-shard.sha256" - MANIFEST_SCHEMA = "openblade.manifest.v1" + MANIFEST_SCHEMA = MANIFEST_SCHEMA CATALOG_SHARD_SCHEMA = "openblade.catalog_shard.v1" def __init__( @@ -83,15 +89,16 @@ def validate_manifest(self, barcode: str) -> ManifestValidationResult: result.schema_version = str(payload.get("schema", "")) try: - manifest = ManifestJson.model_validate(payload) + manifest = parse_manifest(payload) + except UnsupportedManifestSchemaError: + result.errors.append(f"manifest schema mismatch: {result.schema_version}") + return result except ValidationError: result.errors.append("manifest.json failed schema validation") return result result.file_count = manifest.file_count result.total_bytes = manifest.total_logical_bytes - if result.schema_version != self.MANIFEST_SCHEMA: - result.errors.append(f"manifest schema mismatch: {result.schema_version}") raw_files = payload.get("files", []) if not isinstance(raw_files, list): diff --git a/tests/unit/test_ltfs_manifest.py b/tests/unit/test_ltfs_manifest.py index aaac9a4..57016d8 100644 --- a/tests/unit/test_ltfs_manifest.py +++ b/tests/unit/test_ltfs_manifest.py @@ -2,6 +2,17 @@ import json +import pytest + +from openblade.domain.protection import ( + Placement, + Protection, + ProtectionPolicy, + Striping, + StripingMode, +) +from openblade.nas.archive_lifecycle import finalize_tape_generation +from openblade.nas.catalog_shard import CatalogShard, CatalogShardDatasetEntry, CatalogShardWriter from openblade.nas.ltfs_manifest import ( ChecksumEntry, ManifestFileEntry, @@ -10,6 +21,8 @@ TapeJson, TapeMetadataWriter, TapeSetManifest, + UnsupportedManifestSchemaError, + parse_manifest, ) from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend @@ -281,3 +294,122 @@ def test_metadata_path_rejects_traversal_attempt() -> None: with pytest.raises(ValueError): writer._metadata_path("../etc/passwd") + + +class RecordingMetadataBackend(FakeMetadataBackend): + def __init__(self) -> None: + super().__init__() + self.write_order: list[str] = [] + + def write_bytes(self, path: str, content: bytes) -> None: + self.write_order.append(path.rsplit("/", 1)[-1]) + super().write_bytes(path, content) + + +def _generation_inputs(barcode: str = "OB0001L8") -> tuple[ManifestJson, CatalogShard, TapeJson]: + manifest = ManifestJson(barcode=barcode, openblade_tape_id="tape-1") + shard = CatalogShard( + barcode=barcode, + openblade_tape_id="tape-1", + datasets=[ + CatalogShardDatasetEntry( + dataset_id="ds-1", shard_set=[barcode, "OB0002L8"], tape_set=["OB0003L8"] + ) + ], + ) + tape = TapeJson( + openblade_tape_id="tape-1", + barcode=barcode, + created_at="2026-01-01T00:00:00Z", + last_openblade_write_at="2026-01-01T00:00:00Z", + current_generation="gen-1", + ) + return manifest, shard, tape + + +def test_finalize_tape_generation_writes_commit_marker_last() -> None: + backend = RecordingMetadataBackend() + writer = TapeMetadataWriter(backend) + manifest, shard, tape = _generation_inputs() + + finalize_tape_generation( + writer, + CatalogShardWriter(writer), + "OB0001L8", + generation_id="gen-2", + manifest=manifest, + shard=shard, + tape_json=tape, + ) + + order = [name for name in backend.write_order if not name.startswith(".")] + assert order == [ + "manifest.json", + "manifest.sha256", + "catalog-shard.json", + "catalog-shard.sha256", + "tape.json", + "commit-gen-2.json", + ] + + +def test_finalize_tape_generation_chains_generations_and_records_siblings() -> None: + _, writer = _writer() + manifest, shard, tape = _generation_inputs() + + marker = finalize_tape_generation( + writer, + CatalogShardWriter(writer), + "OB0001L8", + generation_id="gen-2", + manifest=manifest, + shard=shard, + tape_json=tape, + ) + + stored_tape = writer.read_tape_json("OB0001L8") + stored_manifest = writer.read_manifest("OB0001L8") + assert stored_tape is not None and stored_manifest is not None + assert (stored_tape.current_generation, stored_tape.previous_generation) == ("gen-2", "gen-1") + assert stored_manifest.sibling_tapes == ["OB0002L8", "OB0003L8"] + assert writer.read_commit_marker("OB0001L8", "gen-2") == marker + assert marker.manifest_sha256 == writer.compute_json_checksum( + stored_manifest.model_dump(by_alias=True) + ) + + +def test_v1_manifest_parses_with_v2_defaults() -> None: + manifest = parse_manifest( + {"schema": "openblade.manifest.v1", "barcode": "OB0001L8", "openblade_tape_id": "t"} + ) + assert manifest.sibling_tapes == [] + assert manifest.protection is None + assert manifest.generation_id == "" + + +def test_newer_manifest_schema_is_rejected_with_typed_error() -> None: + with pytest.raises(UnsupportedManifestSchemaError): + parse_manifest( + {"schema": "openblade.manifest.v3", "barcode": "OB0001L8", "openblade_tape_id": "t"} + ) + + +def test_manifest_round_trip_preserves_protection_policy() -> None: + policy = ProtectionPolicy( + placement=Placement.STRIPED, + striping=Striping(StripingMode.STRIPE), + protection=Protection.replication(2), + ) + _, writer = _writer() + manifest = ManifestJson( + barcode="OB0001L8", + openblade_tape_id="t", + placement=policy.placement.value, + protection=policy.to_dict(), + ) + + writer.write_manifest("OB0001L8", manifest) + stored = writer.read_manifest("OB0001L8") + + assert stored is not None and stored.protection is not None + assert ProtectionPolicy.from_dict(stored.protection) == policy diff --git a/tests/unit/test_manifest_validator.py b/tests/unit/test_manifest_validator.py index 1b8bca2..3ed93f6 100644 --- a/tests/unit/test_manifest_validator.py +++ b/tests/unit/test_manifest_validator.py @@ -158,7 +158,7 @@ def test_validate_manifest_sets_schema_version_from_content() -> None: result = validator.validate_manifest(manifest.barcode) - assert result.schema_version == "openblade.manifest.v1" + assert result.schema_version == "openblade.manifest.v2" def test_validate_manifest_sets_file_count_from_manifest() -> None: From 79f4e6e25138077916193c67b430011fa11d5d37 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:01:49 +0000 Subject: [PATCH 028/205] ci(release): build-once digest pipeline with SBOM, scan, signing and staged deploy build -> scan (pip-audit + trivy HIGH+, SBOMs) -> sign (keyless cosign + SLSA provenance, id-token only here) -> verify -> release + staging (suites, read-only postcheck, rollback) -> production behind environment approval. All actions pinned to SHAs; digest-pinned compose examples for both envs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/release.yml | 269 +++++++++++++++++++++++---- .trivyignore | 3 + deploy/production/docker-compose.yml | 18 ++ deploy/staging/docker-compose.yml | 18 ++ 4 files changed, 270 insertions(+), 38 deletions(-) create mode 100644 .trivyignore create mode 100644 deploy/production/docker-compose.yml create mode 100644 deploy/staging/docker-compose.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da266b7..6e4bdcf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,15 @@ name: Release -# Tag-triggered release: builds the controller image, pushes it to GHCR, and -# cuts a GitHub Release from the tag. Runs on `v*` tag pushes only -- it never +# Tag-triggered release: build once -> SBOM + vuln scan -> sign + provenance -> +# verify -> staging (+ suites) -> approval -> production, all by image DIGEST. +# Stage-by-stage rationale and the rollback rule: docs/runbooks/release.md. Runs on `v*` tag pushes only -- it never # runs on a branch push, so it cannot race the version-bump automation in # other workflows. # # NOTE: this workflow has never been exercised by an actual tag push (no `v*` # tag exists on this repo yet). The first real run is what proves the GHCR # login/push and release-notes generation actually work end to end; until -# then, treat it as reviewed-but-unverified. See docs/wiki/guides/releasing.md +# then, treat it as reviewed-but-unverified. See docs/runbooks/release.md # for what to check if that first run fails, and keep changes to this file # small and easy to bisect for exactly that reason. @@ -29,64 +30,256 @@ permissions: contents: read jobs: - release: - name: build-and-release + # 1. Build ONCE. Every later job consumes needs.build.outputs.digest, never a tag. + build: runs-on: ubuntu-latest permissions: - contents: write # to create the GitHub Release from the tag - packages: write # to push images to ghcr.io + contents: read + packages: write # push the image to ghcr.io + outputs: + digest: ${{ steps.push.outputs.digest }} + version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v5 - + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" - name: Derive version from the tag id: version run: | tag="${GITHUB_REF#refs/tags/}" - version="${tag#v}" echo "tag=${tag}" >> "$GITHUB_OUTPUT" - echo "version=${version}" >> "$GITHUB_OUTPUT" - + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" - name: Cross-check tag against pyproject [project] version run: | pyproject_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" if [ "$pyproject_version" != "${{ steps.version.outputs.version }}" ]; then - echo "::error::tag v${{ steps.version.outputs.version }} does not match pyproject version ${pyproject_version} -- bump pyproject.toml and re-tag" + echo "::error::tag does not match pyproject version ${pyproject_version} -- bump pyproject.toml and re-tag" exit 1 fi - - - name: Log in to GHCR - uses: docker/login-action@v3 + - name: Build wheel + run: python -m pip install build && python -m build --wheel --outdir dist/ + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel + path: dist/*.whl + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Controller image: the root Dockerfile (FastAPI/Typer app). This is the - # only image this workflow builds -- see the Dockerfile.web note below. - - name: Build and push controller image - uses: docker/build-push-action@v6 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build and push controller image (once) + id: push + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . file: Dockerfile push: true - tags: | - ghcr.io/amantux/openblade:${{ steps.version.outputs.version }} - ghcr.io/amantux/openblade:latest + tags: ghcr.io/amantux/openblade:${{ steps.version.outputs.version }} + + # 2 + 3. SBOMs (image spdx-json, wheel cyclonedx) and vuln scan; HIGH+ fails. + scan: + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel + path: dist/ + - name: Resolve wheel dependencies + run: | + python -m venv /tmp/rel + /tmp/rel/bin/pip install dist/*.whl + /tmp/rel/bin/pip freeze --exclude openblade > resolved-requirements.txt + python -m pip install pip-audit + - name: Wheel SBOM (CycloneDX) + run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip -f cyclonedx-json -o wheel-sbom.cdx.json || true + - name: pip-audit (fails on any known vulnerability) + run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip --strict + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Image SBOM (SPDX) + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 + with: + image: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} + format: spdx-json + output-file: image-sbom.spdx.json + upload-artifact: false + - name: Trivy image scan by digest (CRITICAL/HIGH fail the release) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} + severity: CRITICAL,HIGH + exit-code: "1" + trivyignores: .trivyignore + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sbom + path: | + image-sbom.spdx.json + wheel-sbom.cdx.json + + # 4. Keyless sign + SLSA provenance. id-token: write exists ONLY on this job. + sign: + needs: [build, scan] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # push signature + attestation to ghcr.io + id-token: write # OIDC for keyless signing + attestations: write # SLSA provenance + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: cosign sign by digest + run: cosign sign --yes "${IMAGE}@${{ needs.build.outputs.digest }}" + - name: SLSA build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ needs.build.outputs.digest }} + push-to-registry: true - # `Dockerfile.web` is not tracked on master as of this workflow's - # authoring (only `frontend/Dockerfile` exists, consumed by - # docker-compose's `web` service, not published standalone to GHCR). - # If/when a top-level `Dockerfile.web` is added for a standalone web - # image, add a matching build-push step here tagged - # `ghcr.io/amantux/openblade-web:{version,latest}` -- do not invent one - # now. + verify: + needs: [build, sign] + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + steps: + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: cosign verify (expected release identity) + run: >- + cosign verify + --certificate-oidc-issuer https://token.actions.githubusercontent.com + --certificate-identity-regexp '^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$' + "${IMAGE}@${{ needs.build.outputs.digest }}" - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + release: + needs: [build, verify] + runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub Release + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - tag_name: ${{ steps.version.outputs.tag }} - name: ${{ steps.version.outputs.tag }} + path: assets/ + merge-multiple: true + - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 + with: + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} generate_release_notes: true + body: "Image: `ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }}`" + files: assets/* + + # 5 + 6. Staging: deploy the digest, read-only postcheck, suites, rollback on failure. + deploy-staging: + needs: [build, verify] + runs-on: ubuntu-latest + environment: staging + permissions: + contents: read + packages: read + env: + OPENBLADE_IMAGE: ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }} + OPENBLADE_ENV: staging + I3_TIMING_PROFILE: instant + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel + path: dist/ + - name: Install the released wheel (+ test extras) + run: pip install "$(ls dist/*.whl)[dev]" + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: precheck -> deploy -> postcheck + run: >- + python scripts/deploy.py + --deploy-cmd "docker compose -f deploy/staging/docker-compose.yml up -d --wait" + --base-url http://localhost:8000 + - name: Read-only appliance postcheck + id: postcheck + run: python scripts/deploy.py postcheck --base-url http://localhost:8000 --read-only-appliance-checks + - name: Suites (emulator smoke, NAS journey, rebuild, restore) + run: >- + pytest -q --tb=short + tests/i3/test_01_auth.py tests/i3/test_02_inventory.py tests/i3/test_03_changer.py + tests/i3/test_12_multi_library.py tests/i3/test_14_emulator_profile.py + tests/e2e/test_nas_journey.py tests/unit/test_catalog_rebuild.py + tests/unit/test_catalog_rebuild_worker.py tests/e2e/test_mock_archive_restore.py + - name: Rollback on postcheck failure (application code only) + if: failure() && steps.postcheck.outcome == 'failure' && vars.STAGING_LAST_GOOD_DIGEST != '' + run: >- + python scripts/deploy.py rollback --to "${{ vars.STAGING_LAST_GOOD_DIGEST }}" + --deploy-cmd "docker compose -f deploy/staging/docker-compose.yml up -d --wait" + + # Required reviewers on the `production` environment are the approval gate + # (repo Settings -> Environments); this job waits there before any step runs. + deploy-production: + needs: [build, deploy-staging] + runs-on: [self-hosted, openblade-production] + environment: production + permissions: + contents: read + packages: read + env: + IMAGE: ghcr.io/amantux/openblade + OPENBLADE_IMAGE: ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }} + OPENBLADE_ENV: production + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: cosign verify (expected release identity) + run: >- + cosign verify + --certificate-oidc-issuer https://token.actions.githubusercontent.com + --certificate-identity-regexp '^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$' + "${IMAGE}@${{ needs.build.outputs.digest }}" + - name: precheck -> deploy -> postcheck (same digest as staging) + run: >- + python3 scripts/deploy.py + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" + --base-url http://localhost:8000 + - name: Read-only appliance postcheck + id: postcheck + run: python3 scripts/deploy.py postcheck --base-url http://localhost:8000 --read-only-appliance-checks + - name: Rollback on postcheck failure (application code only) + if: failure() && steps.postcheck.outcome == 'failure' && vars.PRODUCTION_LAST_GOOD_DIGEST != '' + run: >- + python3 scripts/deploy.py rollback --to "${{ vars.PRODUCTION_LAST_GOOD_DIGEST }}" + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000..1478d1b --- /dev/null +++ b/.trivyignore @@ -0,0 +1,3 @@ +# Trivy ignore list for the release image scan (.github/workflows/release.yml). +# Every entry MUST carry a written justification on the line above it, an owner, +# and a review-by date. See docs/runbooks/release.md "Vulnerability exceptions". diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml new file mode 100644 index 0000000..e1bc85f --- /dev/null +++ b/deploy/production/docker-compose.yml @@ -0,0 +1,18 @@ +# OpenBlade production stack -- ALWAYS pinned by digest, never by tag. +# The release workflow (and scripts/deploy.py rollback) set OPENBLADE_IMAGE to +# ghcr.io/amantux/openblade@sha256:<64 hex> +# Compose refuses to start if it is unset, so a mutable tag can't sneak in. +services: + openblade: + image: ${OPENBLADE_IMAGE:?set OPENBLADE_IMAGE to ghcr.io/amantux/openblade@sha256:} + ports: + - "8000:8000" + environment: + OPENBLADE_ENV: production + OPENBLADE_DB_URL: sqlite:////data/openblade.db + volumes: + - openblade-production-data:/data + restart: unless-stopped + +volumes: + openblade-production-data: diff --git a/deploy/staging/docker-compose.yml b/deploy/staging/docker-compose.yml new file mode 100644 index 0000000..28e15bf --- /dev/null +++ b/deploy/staging/docker-compose.yml @@ -0,0 +1,18 @@ +# OpenBlade staging stack -- ALWAYS pinned by digest, never by tag. +# The release workflow (and scripts/deploy.py rollback) set OPENBLADE_IMAGE to +# ghcr.io/amantux/openblade@sha256:<64 hex> +# Compose refuses to start if it is unset, so a mutable tag can't sneak in. +services: + openblade: + image: ${OPENBLADE_IMAGE:?set OPENBLADE_IMAGE to ghcr.io/amantux/openblade@sha256:} + ports: + - "8000:8000" + environment: + OPENBLADE_ENV: staging + OPENBLADE_DB_URL: sqlite:////data/openblade.db + volumes: + - openblade-staging-data:/data + restart: unless-stopped + +volumes: + openblade-staging-data: From 82d6688065a15c819b3e5bf1b049499d9a09363d Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:02:19 +0000 Subject: [PATCH 029/205] docs: release runbook and digest-pinned deployment examples Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/deployment.md | 24 ++++++++++++---- docs/runbooks/release.md | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 docs/runbooks/release.md diff --git a/docs/deployment.md b/docs/deployment.md index 0efab13..3b5307a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -65,10 +65,21 @@ Exit 0 only when the deploy is **promoted** (all three pass). OPENBLADE_ENV=production \ OPENBLADE_ADMIN_PASSWORD=... OPENBLADE_SERVICE_PASSWORD=... \ OPENBLADE_SERVICE_TOKEN=... OPENBLADE_DB_URL=sqlite:////data/openblade.db \ +OPENBLADE_IMAGE=ghcr.io/amantux/openblade@sha256:<64-hex digest from the release> \ python3 scripts/deploy.py \ - --deploy-cmd "docker compose up -d" \ + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" \ --base-url https://openblade.internal +# Then the read-only appliance postcheck (GET /inventory/ + GET /jobs/recovery on +# top of the live topology; no non-GET request is ever sent): +python3 scripts/deploy.py postcheck --base-url https://openblade.internal \ + --read-only-appliance-checks + +# Roll back APPLICATION CODE ONLY to the previous digest (tags are refused). +# Physical tape state is never rolled back -- review GET /jobs/recovery after. +python3 scripts/deploy.py rollback --to sha256: \ + --deploy-cmd "docker compose -f deploy/production/docker-compose.yml up -d --wait" + # Re-run pre/post checks only (no deploy), e.g. as a post-deploy smoke: python3 scripts/deploy.py --skip-deploy @@ -88,7 +99,10 @@ deploy fails the build. No production data or credentials are used. - `--base-url` postcheck probes endpoint reachability only (it cannot introspect the remote process's in-process wiring); the in-process postcheck additionally asserts the AppContext is fully wired. -- Rollback orchestration and blue/green promotion are out of scope here; pair this - gate with your container platform's rollout. On a failed postcheck, do not - promote — roll back to the last verified image and restore the catalog per - docs/disaster-recovery.md if needed. +- Images are always deployed by digest (`deploy/staging/` and + `deploy/production/` compose files refuse to start without `OPENBLADE_IMAGE`). + The tag-triggered release pipeline (build once, scan, sign, staging, approval, + production, rollback) is described in docs/runbooks/release.md; it has **not + yet been exercised on a real tag**. +- `rollback` redeploys application code only. It never touches tape state; if + the catalog needs restoring, follow docs/disaster-recovery.md. diff --git a/docs/runbooks/release.md b/docs/runbooks/release.md new file mode 100644 index 0000000..50537a5 --- /dev/null +++ b/docs/runbooks/release.md @@ -0,0 +1,60 @@ +# Release runbook + +> **Status: this pipeline has not yet been exercised on a real tag.** No `v*` tag +> has been pushed since `.github/workflows/release.yml` was written. Treat the +> first run as the test: watch every job, and expect to fix something. + +Trigger: pushing a `v*` tag whose version equals `[project].version` in +`pyproject.toml` (the build job refuses a mismatch). + +## Stages (all keyed on the image DIGEST, never a tag) + +| # | Job | What it does | Fails the release when | +|---|-----|--------------|------------------------| +| 1 | `build` | Builds the wheel (`python -m build`) and the container **once**; pushes `ghcr.io/amantux/openblade:` and outputs `digest`. Wheel uploaded as an artifact. | tag/pyproject mismatch, build error | +| 2 | `scan` | Image SBOM (anchore, SPDX JSON) + wheel SBOM (pip-audit CycloneDX); `pip-audit --strict` on the wheel's resolved deps; Trivy on `image@digest` with `severity: CRITICAL,HIGH`, `exit-code: 1`. | any known vuln in deps; any HIGH/CRITICAL in the image not in `.trivyignore` | +| 3 | `sign` | `cosign sign` (keyless, GitHub OIDC) of `image@digest`; SLSA provenance via `actions/attest-build-provenance`, pushed to the registry. | signing/attestation error | +| 4 | `verify` | `cosign verify` with issuer `token.actions.githubusercontent.com` and identity regex `^https://github\.com/Amantux/openblade/\.github/workflows/release\.yml@refs/tags/v.+$`. | signature missing or signed by any other identity | +| 5 | `release` | GitHub Release with the wheel, both SBOMs, and the digest in the body. | — | +| 6 | `deploy-staging` | Environment `staging`. `scripts/deploy.py` precheck → `docker compose -f deploy/staging/docker-compose.yml up -d --wait` → postcheck, then `postcheck --read-only-appliance-checks`, then the suites (i3 CI subset with `I3_TIMING_PROFILE=instant`, `tests/e2e/test_nas_journey.py`, `tests/unit/test_catalog_rebuild*.py`, `tests/e2e/test_mock_archive_restore.py`). | any check or suite fails | +| 7 | `deploy-production` | Environment `production` (**approval gate**). Re-runs `cosign verify`, deploys the **same digest**, read-only postcheck. | any check fails | + +The suites do not accept a base URL, so they run in-process on the runner from +the tagged checkout with the released wheel installed — not against the staging +container. The staging container is covered by the two postchecks. + +## Approval gate + +Configure in repo **Settings → Environments → production → Required reviewers** +(at least one person who did not push the tag). That setting cannot live in the +repo; without it `deploy-production` runs unattended. The job also needs a +self-hosted runner labelled `openblade-production` on the production host. + +## Rollback rule + +On a failed read-only postcheck, the job runs +`scripts/deploy.py rollback --to `. +Update that environment variable to the new digest after a release is accepted. +If it is unset, no automatic rollback happens and an operator must run it. + +- Rollback redeploys **application code only** (`OPENBLADE_IMAGE=@`). + It refuses tags and has no option that touches tape state. +- **Physical tape state is never rolled back.** Cartridges moved, loaded or + written by the bad version stay where they are. After any rollback, review + `GET /jobs/recovery` and reconcile before resuming operations; restore the + catalog per `docs/disaster-recovery.md` if needed. + +## Credentials + +No long-lived deploy secrets. GHCR uses the job's `GITHUB_TOKEN`; signing uses +GitHub OIDC (`id-token: write` exists only on the `sign` job). Deploy targets +must authenticate by OIDC too (or run on a self-hosted runner on the target +host, as production does). Do not add registry or cloud keys as repo secrets. +Top-level `permissions: contents: read`; write scopes are granted per job. +Every action is pinned to a commit SHA with a `# vX.Y.Z` comment. + +## Vulnerability exceptions (`.trivyignore`) + +An entry is allowed only with, on the line above it: why the finding does not +apply (or why no fix exists), who owns it, and a review-by date. No blanket +ignores. Remove the entry once a fixed base image or dependency ships. From bb6aeff094b5c78db752f8a934eb07a6e4d78cd1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:02:56 +0000 Subject: [PATCH 030/205] ci(hardware): add rig-guarded read-only and destructive i3 lanes; rehearse allowlist+snapshot on mhvtl Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/hardware-destructive.yml | 156 ++++++++++++++++++ .github/workflows/hardware-library-smoke.yml | 155 ++++++++--------- .../workflows/hardware-nightly-readonly.yml | 97 +++++++++++ .github/workflows/mhvtl-weekly.yml | 69 ++++++-- 4 files changed, 378 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/hardware-destructive.yml create mode 100644 .github/workflows/hardware-nightly-readonly.yml diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml new file mode 100644 index 0000000..e428c25 --- /dev/null +++ b/.github/workflows/hardware-destructive.yml @@ -0,0 +1,156 @@ +name: Hardware destructive + +# Destructive lane against the physical Scalar i3 (docs/runbooks/hardware-lanes.md). +# Rehearsed first on mhvtl (mhvtl-weekly.yml runs the same allowlist + snapshot steps). +# +# Guard rails, in order: +# 1. environment `hardware-destructive` — required reviewers approve every run +# (configured in repo settings; a scheduled run waits for approval too); +# 2. rig guard — skipped (neutral, never green) without HARDWARE_RIG_AVAILABLE; +# 3. allowlist — exits non-zero before ANY hardware step if a barcode is +# outside vars.SACRIFICIAL_BARCODES, or the list is empty; +# 4. before/after inventory + drive snapshot, diffed and uploaded; +# 5. /jobs/recovery must be clean at the end. +# LTFS format only ever goes through the two-phase dry-run + safety-token flow. + +on: + schedule: + # 05:43 UTC every Saturday. Odd minute on purpose. + - cron: "43 5 * * 6" + workflow_dispatch: + inputs: + barcodes: + description: "Comma-separated cartridges to use; must be a subset of vars.SACRIFICIAL_BARCODES (default: all of them)" + required: false + default: "" + type: string + +permissions: + contents: read + +concurrency: + # One rig, one lock: never two destructive runs (or a read-only run) at once, + # and never cancel one mid-sequence — that can strand a cartridge in a drive. + group: hardware-rig-i3 + cancel-in-progress: false + +jobs: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' + runs-on: ubuntu-latest + steps: + - name: Record that the lane did not run + run: | + { + echo "## Hardware destructive: NOT RUN (neutral)" + echo + echo "HARDWARE_RIG_AVAILABLE is not 'true', so the destructive sequence was skipped." + echo "This is not a pass. See docs/runbooks/hardware-lanes.md." + } >> "$GITHUB_STEP_SUMMARY" + + destructive: + name: hardware-destructive + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + environment: hardware-destructive + timeout-minutes: 240 + env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES }} + # Never interpolated into a script: read from the environment only. + BARCODES: ${{ inputs.barcodes || vars.SACRIFICIAL_BARCODES }} + I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} + I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: pip install -e '.[dev]' + + - name: Allowlist — refuse any cartridge outside the sacrificial set + id: allowlist + run: | + set -euo pipefail + python -m tools.hardware.allowlist --barcodes "$BARCODES" + echo "OPENBLADE_SCRATCH_BARCODES=$BARCODES" >> "$GITHUB_ENV" + + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + python -m tools.hardware.snapshot capture --out snapshots/before.json + + - name: Load / unload + run: python -m pytest tests/hardware/test_changer_operations.py -m real_hardware -v --tb=short -rs + + # test_mkltfs_formats_tape is deselected: it calls mkltfs directly, which + # bypasses the two-phase dry-run + token flow this lane requires. The + # remaining LTFS tests need a cartridge already formatted via that flow. + - name: LTFS write / read / checksum + run: | + python -m pytest tests/hardware/test_ltfs_operations.py -m real_hardware -v --tb=short -rs \ + --deselect tests/hardware/test_ltfs_operations.py::test_mkltfs_formats_tape + + - name: Archive + restore + run: python -m pytest tests/hardware/test_archive_restore.py -m real_hardware -v --tb=short -rs + + - name: Sharded write + restore + run: python -m pytest tests/hardware/test_sharded_operations.py -m real_hardware -v --tb=short -rs + + - name: Safe failure injection + env: + OPENBLADE_FAULT_TESTS: enabled + run: python -m pytest tests/hardware/test_fault_recovery.py -m real_hardware -v --tb=short -rs + + # Reboot persistence is a MANUAL step (the runner cannot power-cycle the + # appliance): see docs/runbooks/hardware-lanes.md "Reboot persistence". + + - name: Recovery report must be clean + # Only once the allowlist passed: before that, nothing was touched. + if: always() && steps.allowlist.outcome == 'success' + run: | + set -euo pipefail + # Starting the app runs startup recovery against the same catalog the + # suite used; /jobs/recovery then reports what it had to repair. + python - <<'PY' + import json + import sys + + from fastapi.testclient import TestClient + + from openblade.api.main import app + + with TestClient(app) as client: + response = client.get("/jobs/recovery") + response.raise_for_status() + report = response.json() + print(json.dumps(report, indent=2)) + dirty = {k: v for k, v in report.items() if v} + if dirty: + print("::error::Recovery report is not clean: " + ", ".join(sorted(dirty))) + sys.exit(1) + PY + + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: hardware-destructive-${{ github.run_id }} + path: snapshots/ + if-no-files-found: warn diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index 29e4a63..87a8eec 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -1,123 +1,102 @@ name: Quantum hardware smoke +# Dispatchable subset of hardware-nightly-readonly.yml: the same read-only +# suite (tests/hardware/readonly/), on demand, on the same rig. +# +# Why kept rather than deleted: the workflow name and the job id +# `register-and-validate` are kept so any branch-protection rule or status +# check that references them keeps resolving. The old ubuntu-latest job could +# not reach a physical library anyway; the real check needs the [self-hosted, i3] +# runner (docs/runbooks/hardware-lanes.md). Motion/control-plane inputs were +# removed on purpose: motion belongs to hardware-destructive.yml, behind the +# environment approval and the cartridge allowlist. + on: workflow_dispatch: inputs: - target_aml_url: - description: "AML API base URL for the physical Quantum library" - required: true - type: string - target_model: - description: "Hardware model under test" - required: true - default: "Scalar i3" - type: choice - options: - - Scalar i3 - - Scalar i6 - - Other - target_name: - description: "Display name when registering with OpenBlade" - required: true - default: "Quantum Library Under Test" - type: string - target_serial: - description: "Serial number (optional, used for idempotent registration)" + test_filter: + description: "Optional pytest -k expression to narrow the read-only suite" required: false default: "" type: string - openblade_api_url: - description: "Optional OpenBlade API base URL for library registration" - required: false - default: "" - type: string - include_motion_tests: - description: "Include changer/move command checks in command matrix" - required: true - default: false - type: boolean - include_control_plane_checks: - description: "Validate /api/libraries command compatibility (requires OpenBlade target URL)" - required: true - default: false - type: boolean - timing_profile: - description: "Timing profile for real hardware smoke tests" - required: true - default: hardware - type: choice - options: - - instant - - realistic - - hardware permissions: contents: read +concurrency: + group: hardware-rig-i3 + cancel-in-progress: false + jobs: - register-and-validate: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' runs-on: ubuntu-latest + steps: + - name: Record that the smoke did not run + run: | + { + echo "## Quantum hardware smoke: NOT RUN (neutral)" + echo + echo "HARDWARE_RIG_AVAILABLE is not 'true', so register-and-validate was skipped." + echo "This is not a pass. See docs/runbooks/hardware-lanes.md." + } >> "$GITHUB_STEP_SUMMARY" + + register-and-validate: + name: register-and-validate + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + timeout-minutes: 30 env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} - OPENBLADE_ADMIN_USER: ${{ secrets.OPENBLADE_ADMIN_USER }} - OPENBLADE_ADMIN_PASSWORD: ${{ secrets.OPENBLADE_ADMIN_PASSWORD }} steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - cache: "pip" + - run: pip install -e '.[dev]' - - name: Validate required secrets + - name: Snapshot inventory + drive state (before) run: | set -euo pipefail - if [ -z "${I3_AML_USER}" ] || [ -z "${I3_AML_PASSWORD}" ]; then - echo "::error::Set QUANTUM_AML_USER and QUANTUM_AML_PASSWORD repository secrets." - exit 1 - fi + mkdir -p snapshots + python -m tools.hardware.snapshot capture --out snapshots/before.json - - name: Register target library in OpenBlade - if: ${{ inputs.openblade_api_url != '' }} + - name: Run the read-only suite env: - TARGET_URL: ${{ inputs.target_aml_url }} - TARGET_NAME: ${{ inputs.target_name }} - TARGET_MODEL: ${{ inputs.target_model }} - TARGET_SERIAL: ${{ inputs.target_serial }} - OPENBLADE_URL: ${{ inputs.openblade_api_url }} + TEST_FILTER: ${{ inputs.test_filter }} run: | set -euo pipefail - if [ -z "${OPENBLADE_ADMIN_USER}" ] || [ -z "${OPENBLADE_ADMIN_PASSWORD}" ]; then - echo "::error::Set OPENBLADE_ADMIN_USER and OPENBLADE_ADMIN_PASSWORD secrets to register libraries." - exit 1 + args=(tests/hardware/readonly/ -m real_hardware -v --tb=short -rs --junitxml=readonly-results.xml) + if [ -n "$TEST_FILTER" ]; then + args+=(-k "$TEST_FILTER") fi - python3 tests/i3/register_target_library.py \ - --openblade-url "${OPENBLADE_URL}" \ - --target-aml-url "${TARGET_URL}" \ - --target-name "${TARGET_NAME}" \ - --target-model "${TARGET_MODEL}" \ - --target-serial "${TARGET_SERIAL}" \ - --username "${OPENBLADE_ADMIN_USER}" \ - --password "${OPENBLADE_ADMIN_PASSWORD}" + python -m pytest "${args[@]}" - - name: Run real hardware command matrix - env: - I3_TEST_MODE: real - I3_REAL_HARDWARE_ENABLED: "true" - I3_AML_URL: ${{ inputs.target_aml_url }} - I3_TIMING_PROFILE: ${{ inputs.timing_profile }} - I3_COMMAND_MATRIX_INCLUDE_MOTION: ${{ inputs.include_motion_tests }} - I3_COMMAND_MATRIX_INCLUDE_CONTROL_PLANE: ${{ inputs.include_control_plane_checks }} - I3_COMMAND_MATRIX_REPORT: /tmp/i3-command-matrix.json + - name: Snapshot inventory + drive state (after) and diff + if: always() run: | set -euo pipefail - python3 -m pytest tests/i3/test_00_command_matrix.py -q --tb=short - cat /tmp/i3-command-matrix.json + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json - - name: Upload hardware command matrix report + - name: Upload results, differential report and snapshots if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: i3-command-matrix - path: /tmp/i3-command-matrix.json + name: hardware-smoke-${{ github.run_id }} + path: | + readonly-results.xml + differential-report.json + snapshots/ + if-no-files-found: warn diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml new file mode 100644 index 0000000..3c9c3d1 --- /dev/null +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -0,0 +1,97 @@ +name: Hardware nightly read-only + +# Read-only lane against the physical Scalar i3 (docs/runbooks/hardware-lanes.md). +# Every test in tests/hardware/readonly/ is read-only by construction (an AST +# safety test in that directory rejects load/unload/move/format/mount calls). +# The before/after snapshot diff is the runtime proof that nothing moved. +# +# Without a rig (HARDWARE_RIG_AVAILABLE != 'true') the hardware job is SKIPPED, +# and the no-rig job says so in the run summary. A skipped lane is neutral, +# never green: it proves nothing about the appliance. + +on: + schedule: + # 02:17 UTC nightly. Odd minute: top-of-hour schedules queue behind everyone. + - cron: "17 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Shares the rig lock with the destructive lane and the smoke subset: a + # read-only pass must never observe a destructive run mid-sequence. + group: hardware-rig-i3 + cancel-in-progress: false + +jobs: + no-rig-notice: + name: no-rig-notice + if: vars.HARDWARE_RIG_AVAILABLE != 'true' + runs-on: ubuntu-latest + steps: + - name: Record that the lane did not run + run: | + { + echo "## Hardware nightly read-only: NOT RUN (neutral)" + echo + echo "No i3 rig is registered for this repository (repo variable HARDWARE_RIG_AVAILABLE is not 'true')," + echo "so the read-only hardware job was skipped. This is not a pass: nothing was verified against the appliance." + echo "See docs/runbooks/hardware-lanes.md to register the self-hosted runner." + } >> "$GITHUB_STEP_SUMMARY" + + readonly: + name: hardware-readonly + if: vars.HARDWARE_RIG_AVAILABLE == 'true' + runs-on: [self-hosted, i3] + timeout-minutes: 60 + env: + OPENBLADE_BACKEND: real + OPENBLADE_REAL_HARDWARE_ENABLED: "true" + OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json + I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} + I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: pip install -e '.[dev]' + + # No allowlist step: this lane names no cartridges and performs no + # motion. The snapshot diff below is what would catch a violation. + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + python -m tools.hardware.snapshot capture --out snapshots/before.json + + - name: Run the read-only suite + run: | + set -euo pipefail + python -m pytest tests/hardware/readonly/ -m real_hardware -v --tb=short -rs \ + --junitxml=readonly-results.xml + + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + python -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload results, differential report and snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: hardware-readonly-${{ github.run_id }} + path: | + readonly-results.xml + differential-report.json + snapshots/ + if-no-files-found: warn diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 571e088..9bb38f7 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -98,9 +98,9 @@ jobs: # ever trips, look at the per-step durations before raising it. timeout-minutes: 45 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -160,7 +160,7 @@ jobs: - name: Restore cached LTFS build id: ltfs-cache if: ${{ inputs.skip_ltfs != true }} - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /tmp/ltfs-stage.tar key: ${{ steps.ltfs-key.outputs.key }} @@ -276,7 +276,7 @@ jobs: lsscsi -g || true ls -l /dev/sg* /dev/nst* /dev/st* /dev/sch* 2>&1 || true lsmod | grep mhvtl || true - ps ax | grep -E 'vtltape|vtllibrary' | grep -v grep || true + pgrep -af 'vtltape|vtllibrary' || true systemctl status 'vtltape@*' 'vtllibrary@*' --no-pager 2>&1 | head -60 || true sudo dmesg | tail -80 || true cat /etc/mhvtl/device.conf || true @@ -306,6 +306,29 @@ jobs: printf '%s\n' "$block" \ | sed -n 's/^export \([A-Z_][A-Z0-9_]*\)=\(.*\)$/\1=\2/p' >> "$GITHUB_ENV" + # Rehearsal of the destructive lane's guard rails (hardware-destructive.yml): + # the same allowlist gate and before/after snapshot, so a broken guard is + # found here on a disposable VM before it matters on the real i3. + # Without the repo variable, the sacrificial set defaults to the scratch + # cartridges mhvtl loads for formatting (scripts/mhvtl/env.sh). + - name: Allowlist — refuse any cartridge outside the sacrificial set + env: + SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES || 'OB0007L8,OB0008L8' }} + run: | + set -euo pipefail + python -m tools.hardware.allowlist --barcodes "$OPENBLADE_SCRATCH_BARCODES" + + - name: Snapshot inventory + drive state (before) + run: | + set -euo pipefail + mkdir -p snapshots + sudo env \ + OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ + OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ + OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ + "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/before.json + - name: Run the hardware suite against the rig timeout-minutes: 20 env: @@ -325,16 +348,16 @@ jobs: env | grep '^OPENBLADE_' | sort if [ "${{ steps.ltfs.outputs.available }}" = "true" ]; then - suites="tests/hardware/" + suites=(tests/hardware/) else # Documented cut: everything below needs mkltfs to have formatted # the scratch cartridges. Without it those suites fail on mount # rather than skip, which would make the canary permanently red for # a reason that is not a product defect. - suites="tests/hardware/test_device_discovery.py \ - tests/hardware/test_changer_operations.py \ - tests/hardware/test_drive_health.py \ - tests/hardware/test_catalog_integrity.py" + suites=(tests/hardware/test_device_discovery.py + tests/hardware/test_changer_operations.py + tests/hardware/test_drive_health.py + tests/hardware/test_catalog_integrity.py) fi # `sudo env VAR=...` rather than `sudo -E`: -E depends on the host's @@ -349,18 +372,42 @@ jobs: OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ OPENBLADE_SCRATCH_BARCODES="$OPENBLADE_SCRATCH_BARCODES" \ OPENBLADE_FAULT_TESTS="${OPENBLADE_FAULT_TESTS:-}" \ - "$(command -v python)" -m pytest $suites \ + "$(command -v python)" -m pytest "${suites[@]}" \ -m real_hardware -v --tb=short -rs \ --junitxml=/tmp/mhvtl-hardware-results.xml - name: Upload hardware results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: mhvtl-hardware-results path: /tmp/mhvtl-hardware-results.xml if-no-files-found: warn + - name: Snapshot inventory + drive state (after) and diff + if: always() + run: | + set -euo pipefail + if [ ! -f snapshots/before.json ]; then + echo "::warning::No before-snapshot was captured; nothing to diff." + exit 0 + fi + sudo env \ + OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ + OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ + OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ + "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/after.json + python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + + - name: Upload snapshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: mhvtl-snapshots + path: snapshots/ + if-no-files-found: warn + - name: Tear the rig down if: always() run: sudo scripts/mhvtl/teardown.sh || true From 8e23d57a3275aa37ad560f185f69e9c5a9228fd2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:02:56 +0000 Subject: [PATCH 031/205] docs(hardware): runbook for self-hosted i3 lanes; gate write paths on the drive-identity bridge Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/hardware-setup.md | 15 +++++ docs/runbooks/hardware-lanes.md | 106 ++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 docs/runbooks/hardware-lanes.md diff --git a/docs/hardware-setup.md b/docs/hardware-setup.md index 989c8ed..edd58a5 100644 --- a/docs/hardware-setup.md +++ b/docs/hardware-setup.md @@ -8,6 +8,21 @@ 6. Keep the application in mock mode until read-only inventory and LTFS validation succeed end-to-end. 7. Document barcode conventions and slot maps before allowing write workflows. +## Write paths stay disabled until the drive-identity bridge has passed on the rig + +Do not enable any write path (format, archive, the destructive CI lane) on a +real library until `tests/hardware/readonly/test_drive_identity_bridge.py` has +passed **on that rig**. It walks the whole identity chain once, end to end: +AML element coordinate → drive serial → `sg_inq` on the host → the +`/dev/tape/by-id` node → an LTFS mount **dry run** (the mount is not performed; +the test asserts the resolved device path exists and the serial matches, then +stops). Until it passes, a write could land on a different physical drive than +the one the library reports — the failure mode the serial correlation above +exists to prevent. + +The bridge test runs in the nightly read-only lane. How the lanes are wired, +gated and approved: [runbooks/hardware-lanes.md](runbooks/hardware-lanes.md). + ## Always use the no-rewind device node (`/dev/nstN`, never `/dev/stN`) Linux exposes every tape drive twice: `/dev/stN` **rewinds the tape on every diff --git a/docs/runbooks/hardware-lanes.md b/docs/runbooks/hardware-lanes.md new file mode 100644 index 0000000..f4bcb38 --- /dev/null +++ b/docs/runbooks/hardware-lanes.md @@ -0,0 +1,106 @@ +# Runbook: hardware CI lanes + +Four workflows exercise real (or virtual) tape hardware. None of them runs on a +GitHub-hosted runner against a physical library; the physical lanes need the +self-hosted runner described below. + +| Workflow | Trigger | Runs on | Touches cartridges? | +|---|---|---|---| +| `hardware-nightly-readonly.yml` | nightly cron + dispatch | `[self-hosted, i3]` | No — read-only by construction | +| `hardware-library-smoke.yml` | dispatch only | `[self-hosted, i3]` | No — dispatchable subset of the nightly lane | +| `hardware-destructive.yml` | weekly cron + dispatch | `[self-hosted, i3]`, environment `hardware-destructive` | Yes — sacrificial cartridges only | +| `mhvtl-weekly.yml` | weekly cron + dispatch | `ubuntu-latest` (mhvtl VM) | Virtual only — rehearsal of the destructive guard rails | + +All three physical lanes share the concurrency group `hardware-rig-i3` with +`cancel-in-progress: false`: only one lane holds the rig at a time, and a +running destructive sequence is never cancelled halfway (that can strand a +cartridge in a drive). + +`hardware-library-smoke.yml` was kept rather than deleted so its workflow name +and job id (`register-and-validate`) keep resolving for any branch-protection +rule that references them. + +## 1. Register the self-hosted runner + +1. On the host attached to the i3 (SAS/FC to the changer and drives), create an + unprivileged runner user that is in the groups owning `/dev/sg*` and + `/dev/nst*` (usually `tape` and `disk`). Do not run the runner as root. +2. Repo → **Settings → Actions → Runners → New self-hosted runner**, follow the + download/`config.sh` steps, and give it the labels **`self-hosted,i3`** + (`./config.sh --labels i3`; `self-hosted` is implicit). Install it as a + service (`sudo ./svc.sh install `). +3. In the runner's `.env` file, set the rig's device map so the suites can + address it: `OPENBLADE_CHANGER_DEVICE`, `OPENBLADE_DRIVE_DEVICES`, and the + drive map described in [../hardware-setup.md](../hardware-setup.md). +4. Install host tools the suites call: `sg3-utils`, `lsscsi`, `mtx`, LTFS. +5. Add repo secrets `QUANTUM_AML_USER` / `QUANTUM_AML_PASSWORD` (AML login). + +## 2. Repository variables + +Repo → **Settings → Secrets and variables → Actions → Variables**: + +- **`HARDWARE_RIG_AVAILABLE`** = `true` — only once the runner above is online. + Every physical lane's hardware job has `if: vars.HARDWARE_RIG_AVAILABLE == 'true'`. +- **`SACRIFICIAL_BARCODES`** — comma-separated list of cartridges that may be + loaded, formatted and overwritten (e.g. `OB0007L8,OB0008L8`). Nothing else may + ever be touched by the destructive lane. + +## 3. Environment approval (settings-only — cannot be committed) + +Repo → **Settings → Environments → New environment** `hardware-destructive`: + +- **Required reviewers**: at least one person who can physically check the + library. Every run waits for approval, including the weekly scheduled one. +- Optionally restrict **Deployment branches** to `main`. + +Without the environment, GitHub creates it on first use with no protection — +so create it, with reviewers, before setting `HARDWARE_RIG_AVAILABLE`. + +## 4. The cartridge allowlist + +Before any hardware step, the destructive lane (and its mhvtl rehearsal) runs: + +```bash +python -m tools.hardware.allowlist --barcodes "$BARCODES" +``` + +`BARCODES` is the dispatch input `barcodes` (or all of `SACRIFICIAL_BARCODES` +on a scheduled run). The step exits non-zero — and the job stops — if the list +is empty or names any barcode outside `SACRIFICIAL_BARCODES`. The input is +passed through `env:`, never interpolated into the script. mhvtl falls back to +the scratch cartridges its rig loads (`OB0007L8,OB0008L8`) when the variable is +unset. + +Each lane also captures inventory + drive state before and after +(`tools.hardware.snapshot capture`), diffs them, and uploads `snapshots/` as an +artifact even on failure. On the read-only lanes the diff must be empty. + +## 5. The destructive sequence + +Load/unload → LTFS write/read/checksum → archive + restore → sharded write + +restore → safe fault injection → `/jobs/recovery` must report no interrupted +jobs, released leases or drive mismatches. + +LTFS format goes only through the two-phase flow (dry-run plan → explicit +barcode + one-time safety token). `test_mkltfs_formats_tape` is deselected in +this lane because it calls `mkltfs` directly. + +### Reboot persistence (manual) + +The runner cannot power-cycle the appliance. Once per quarter, after a green +destructive run: reboot the i3 from its web UI, wait for it to come Ready, +dispatch `hardware-library-smoke.yml`, and confirm the snapshot diff against +the pre-reboot `after.json` shows the same cartridges in the same slots. + +## 6. What "no rig" looks like — and the no-green-by-skipping rule + +Without `HARDWARE_RIG_AVAILABLE=true`, the hardware job shows as **Skipped** +and a small `no-rig-notice` job writes "NOT RUN (neutral)" to the run summary. + +**No lane may be reported or treated as green because it skipped.** A skipped +hardware job proves nothing about the appliance. Do not add it as a required +status check that a skip satisfies, do not quote a skipped run as evidence, and +do not "fix" a red lane by unsetting the variable. + +`actionlint` needs to know the custom label: add `.github/actionlint.yaml` with +`self-hosted-runner: { labels: [i3] }` if you lint these files locally. From bf072c676d4530de6b638cd92fe01f60ff7db9a3 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:03:06 +0000 Subject: [PATCH 032/205] feat(nas): cartridge lifecycle state machine with typed invalid transitions Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/state_machine.py | 95 ++++++++++++++++++++++++++++++ tests/unit/test_nas_tape_states.py | 66 +++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 openblade/nas/state_machine.py create mode 100644 tests/unit/test_nas_tape_states.py diff --git a/openblade/nas/state_machine.py b/openblade/nas/state_machine.py new file mode 100644 index 0000000..6178185 --- /dev/null +++ b/openblade/nas/state_machine.py @@ -0,0 +1,95 @@ +"""Tape/cartridge lifecycle state machine for the tape-native NAS. + +This models the *cartridge* lifecycle. Per-file states live in +``openblade.nas.types.NasFileState`` and are deliberately not duplicated here: +a file is ``OFFLINE_ON_TAPE``/``MISSING_TAPE``/``EXPORTED`` as a consequence of +the cartridge state it sits on (see :func:`file_state_for_tape`). +""" + +from __future__ import annotations + +from enum import Enum + +from openblade.nas.types import NasFileState + + +class TapeState(str, Enum): + ONLINE = "online" + OFFLINE = "offline" + DEGRADED = "degraded" + HYDRATING = "hydrating" + EXPORTING = "exporting" + EXPORTED = "exported" + SEQUESTERED = "sequestered" + REPAIR_REQUIRED = "repair_required" + + +class TapeEvent(str, Enum): + GO_OFFLINE = "go_offline" + BRING_ONLINE = "bring_online" + ERROR_DETECTED = "error_detected" + HYDRATE_START = "hydrate_start" + HYDRATE_DONE = "hydrate_done" + EXPORT_START = "export_start" + EXPORT_DONE = "export_done" + EXPORT_CANCEL = "export_cancel" + IMPORT = "import" + SEQUESTER = "sequester" + OPERATOR_REPAIR = "operator_repair" + OPERATOR_RELEASE = "operator_release" + REPAIR_DONE = "repair_done" + + +class InvalidTransitionError(ValueError): + """Raised when an event is not allowed from the current tape state.""" + + def __init__(self, state: TapeState, event: TapeEvent) -> None: + super().__init__(f"transition {event.value!r} not allowed from {state.value!r}") + self.state = state + self.event = event + + +_S = TapeState +_E = TapeEvent + +# Explicit, exhaustive transition table. Anything absent is forbidden. +TRANSITIONS: dict[tuple[TapeState, TapeEvent], TapeState] = { + (_S.ONLINE, _E.GO_OFFLINE): _S.OFFLINE, + (_S.ONLINE, _E.ERROR_DETECTED): _S.DEGRADED, + (_S.ONLINE, _E.HYDRATE_START): _S.HYDRATING, + (_S.ONLINE, _E.EXPORT_START): _S.EXPORTING, + (_S.ONLINE, _E.SEQUESTER): _S.SEQUESTERED, + (_S.OFFLINE, _E.BRING_ONLINE): _S.ONLINE, + (_S.OFFLINE, _E.SEQUESTER): _S.SEQUESTERED, + (_S.DEGRADED, _E.OPERATOR_REPAIR): _S.REPAIR_REQUIRED, + (_S.DEGRADED, _E.SEQUESTER): _S.SEQUESTERED, + (_S.DEGRADED, _E.GO_OFFLINE): _S.OFFLINE, + (_S.HYDRATING, _E.HYDRATE_DONE): _S.ONLINE, + (_S.HYDRATING, _E.ERROR_DETECTED): _S.DEGRADED, + (_S.EXPORTING, _E.EXPORT_DONE): _S.EXPORTED, + (_S.EXPORTING, _E.EXPORT_CANCEL): _S.ONLINE, + (_S.EXPORTED, _E.IMPORT): _S.OFFLINE, + # Sequestered media only leaves quarantine through an explicit operator action. + (_S.SEQUESTERED, _E.OPERATOR_REPAIR): _S.REPAIR_REQUIRED, + (_S.SEQUESTERED, _E.OPERATOR_RELEASE): _S.ONLINE, + (_S.REPAIR_REQUIRED, _E.REPAIR_DONE): _S.ONLINE, + (_S.REPAIR_REQUIRED, _E.SEQUESTER): _S.SEQUESTERED, +} + + +def transition(state: TapeState, event: TapeEvent) -> TapeState: + try: + return TRANSITIONS[(state, event)] + except KeyError: + raise InvalidTransitionError(state, event) from None + + +def file_state_for_tape(state: TapeState) -> NasFileState: + """Planning view of a file whose only copy is on a tape in ``state``.""" + if state is TapeState.EXPORTED: + return NasFileState.EXPORTED + if state is TapeState.HYDRATING: + return NasFileState.HYDRATING + if state in (TapeState.SEQUESTERED, TapeState.REPAIR_REQUIRED): + return NasFileState.MISSING_TAPE + return NasFileState.OFFLINE_ON_TAPE diff --git a/tests/unit/test_nas_tape_states.py b/tests/unit/test_nas_tape_states.py new file mode 100644 index 0000000..73a24a5 --- /dev/null +++ b/tests/unit/test_nas_tape_states.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import itertools + +import pytest + +from openblade.nas.state_machine import ( + TRANSITIONS, + InvalidTransitionError, + TapeEvent, + TapeState, + file_state_for_tape, + transition, +) +from openblade.nas.types import NasFileState + +S = TapeState +E = TapeEvent + + +@pytest.mark.parametrize(("pair", "target"), list(TRANSITIONS.items())) +def test_every_allowed_transition(pair: tuple[TapeState, TapeEvent], target: TapeState) -> None: + assert transition(*pair) is target + + +_FORBIDDEN = [p for p in itertools.product(S, E) if p not in TRANSITIONS] + + +@pytest.mark.parametrize(("state", "event"), _FORBIDDEN) +def test_every_forbidden_transition_raises_typed(state: TapeState, event: TapeEvent) -> None: + with pytest.raises(InvalidTransitionError) as exc: + transition(state, event) + assert exc.value.state is state and exc.value.event is event + + +def test_exported_cannot_come_online_without_import() -> None: + with pytest.raises(InvalidTransitionError): + transition(S.EXPORTED, E.BRING_ONLINE) + assert transition(transition(S.EXPORTED, E.IMPORT), E.BRING_ONLINE) is S.ONLINE + + +def test_exporting_cannot_jump_online_except_by_cancel() -> None: + for event in E: + if event in (E.EXPORT_DONE, E.EXPORT_CANCEL): + continue + with pytest.raises(InvalidTransitionError): + transition(S.EXPORTING, event) + + +def test_sequestered_leaves_only_via_operator() -> None: + exits = {e for (s, e) in TRANSITIONS if s is S.SEQUESTERED} + assert exits == {E.OPERATOR_REPAIR, E.OPERATOR_RELEASE} + assert {TRANSITIONS[(S.SEQUESTERED, e)] for e in exits} == {S.REPAIR_REQUIRED, S.ONLINE} + + +def test_degraded_never_returns_online_without_repair() -> None: + assert (S.DEGRADED, E.BRING_ONLINE) not in TRANSITIONS + assert (S.DEGRADED, E.REPAIR_DONE) not in TRANSITIONS + + +def test_file_state_projection_reuses_nas_file_state() -> None: + assert file_state_for_tape(S.EXPORTED) is NasFileState.EXPORTED + assert file_state_for_tape(S.SEQUESTERED) is NasFileState.MISSING_TAPE + assert file_state_for_tape(S.REPAIR_REQUIRED) is NasFileState.MISSING_TAPE + assert file_state_for_tape(S.HYDRATING) is NasFileState.HYDRATING + assert file_state_for_tape(S.OFFLINE) is NasFileState.OFFLINE_ON_TAPE From 99e2c6ec8dba197e7f67307a58103c9e1141fc89 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:03:06 +0000 Subject: [PATCH 033/205] feat(nas): LTO compat matrix, foreign-media guard and reservations in archive planner Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/media.py | 93 ++++++++++++++++++++++++++++ openblade/nas/planner.py | 20 +++++- openblade/nas/types.py | 6 ++ tests/unit/test_nas_media_compat.py | 95 +++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 openblade/nas/media.py create mode 100644 tests/unit/test_nas_media_compat.py diff --git a/openblade/nas/media.py b/openblade/nas/media.py new file mode 100644 index 0000000..ecdc41f --- /dev/null +++ b/openblade/nas/media.py @@ -0,0 +1,93 @@ +"""LTO media generations, drive compatibility, and foreign-media detection.""" + +from __future__ import annotations + +from enum import IntEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class MediaGeneration(IntEnum): + UNKNOWN = 0 + LTO5 = 5 + LTO6 = 6 + LTO7 = 7 + LTO8 = 8 + LTO9 = 9 + + +# LTO Program (lto.org) compatibility rules: +# - LTO-5, LTO-6, LTO-7 drives follow the classic rule: read back two generations +# (N-2) and write back one generation (N-1). +# - LTO-8 broke the N-2 read rule: an LTO-8 drive reads and writes LTO-7 and +# LTO-8 media only (no LTO-6 read). +# - LTO-9 reads and writes LTO-8 and LTO-9 media only. +# (LTO-7 Type M / M8 media is not modelled; it is reported as LTO7.) +_READ: dict[MediaGeneration, frozenset[MediaGeneration]] = { + # LTO-5 also reads LTO-3/LTO-4, which are outside the modelled range. + MediaGeneration.LTO5: frozenset({MediaGeneration.LTO5}), + MediaGeneration.LTO6: frozenset({MediaGeneration.LTO5, MediaGeneration.LTO6}), + MediaGeneration.LTO7: frozenset( + {MediaGeneration.LTO5, MediaGeneration.LTO6, MediaGeneration.LTO7} + ), + MediaGeneration.LTO8: frozenset({MediaGeneration.LTO7, MediaGeneration.LTO8}), + MediaGeneration.LTO9: frozenset({MediaGeneration.LTO8, MediaGeneration.LTO9}), +} +_WRITE: dict[MediaGeneration, frozenset[MediaGeneration]] = { + MediaGeneration.LTO5: frozenset({MediaGeneration.LTO5}), + MediaGeneration.LTO6: frozenset({MediaGeneration.LTO5, MediaGeneration.LTO6}), + MediaGeneration.LTO7: frozenset({MediaGeneration.LTO6, MediaGeneration.LTO7}), + MediaGeneration.LTO8: frozenset({MediaGeneration.LTO7, MediaGeneration.LTO8}), + MediaGeneration.LTO9: frozenset({MediaGeneration.LTO8, MediaGeneration.LTO9}), +} + + +def can_read(drive_gen: MediaGeneration, media_gen: MediaGeneration) -> bool: + return media_gen in _READ.get(drive_gen, frozenset()) + + +def can_write(drive_gen: MediaGeneration, media_gen: MediaGeneration) -> bool: + return media_gen in _WRITE.get(drive_gen, frozenset()) + + +class IncompatibleMediaError(ValueError): + """Drive generation cannot perform the requested operation on the media.""" + + def __init__(self, drive_gen: MediaGeneration, media_gen: MediaGeneration, op: str) -> None: + super().__init__(f"{drive_gen.name} drive cannot {op} {media_gen.name} media") + self.drive_gen = drive_gen + self.media_gen = media_gen + self.op = op + + +class ForeignMediaError(ValueError): + """Media not formatted by OpenBlade; must be explicitly adopted first.""" + + def __init__(self, barcode: str) -> None: + super().__init__(f"cartridge {barcode} is foreign media; adopt it before use") + self.barcode = barcode + + +class Cartridge(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + generation: MediaGeneration = MediaGeneration.UNKNOWN + has_tape_json: bool = False + adopted: bool = False + + +def is_foreign(cartridge: Cartridge) -> bool: + """True for media OpenBlade did not format (no tape.json) and that was not adopted. + + Foreign media is read-only for planning and is never auto-formatted. + """ + return not cartridge.has_tape_json and not cartridge.adopted + + +def require_writable(cartridge: Cartridge, drive_gen: MediaGeneration) -> None: + """Planner guard for a tape write: refuses foreign or incompatible media.""" + if is_foreign(cartridge): + raise ForeignMediaError(cartridge.barcode) + if not can_write(drive_gen, cartridge.generation): + raise IncompatibleMediaError(drive_gen, cartridge.generation, "write") diff --git a/openblade/nas/planner.py b/openblade/nas/planner.py index 2a4b1ee..876c816 100644 --- a/openblade/nas/planner.py +++ b/openblade/nas/planner.py @@ -5,6 +5,7 @@ import posixpath from datetime import datetime, timezone +from openblade.nas.media import ForeignMediaError, is_foreign, require_writable from openblade.nas.types import ( ArchivePlan, ArchivePlanRequest, @@ -42,6 +43,7 @@ def plan(self, request: ArchivePlanRequest) -> ArchivePlan: created_at=datetime.now(timezone.utc).isoformat(), ) + self._check_media(request) self._apply_common_checks(request, plan) if not plan.is_safe_to_enqueue: return plan @@ -362,7 +364,23 @@ def _single_tape_capacity(self, request: ArchivePlanRequest) -> int: return self.DEFAULT_TAPE_CAPACITY def _tape_capacity(self, request: ArchivePlanRequest, barcode: str) -> int: - return request.tape_capacities.get(barcode, self.DEFAULT_TAPE_CAPACITY) + capacity = request.tape_capacities.get(barcode, self.DEFAULT_TAPE_CAPACITY) + return max(0, capacity - request.reserved_bytes.get(barcode, 0)) + + def _check_media(self, request: ArchivePlanRequest) -> None: + """Refuse foreign (un-adopted) or drive-incompatible target cartridges. + + Raises ForeignMediaError / IncompatibleMediaError; foreign media is never + auto-formatted. + """ + for barcode in request.available_tapes: + cartridge = request.cartridges.get(barcode) + if cartridge is None: + continue + if is_foreign(cartridge): + raise ForeignMediaError(barcode) + if request.drive_generation is not None: + require_writable(cartridge, request.drive_generation) def _file_size(self, request: ArchivePlanRequest, path: str) -> int: return request.file_sizes.get(path, 0) diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 97d5225..94a5792 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator +from openblade.nas.media import Cartridge, MediaGeneration + class EffectivePolicySource(str, Enum): SYSTEM_DEFAULT = "system_default" @@ -448,6 +450,10 @@ class ArchivePlanRequest(BaseModel): shard_size_bytes: int | None = Field(default=None, ge=1) shard_strategy: ShardStrategy | None = None max_parallelism: int = Field(default=1, ge=1, le=16) + # Tape-native planning inputs (all optional; empty = previous behaviour). + cartridges: dict[str, Cartridge] = Field(default_factory=dict) + drive_generation: MediaGeneration | None = None + reserved_bytes: dict[str, int] = Field(default_factory=dict) class CacheDriveConfig(BaseModel): diff --git a/tests/unit/test_nas_media_compat.py b/tests/unit/test_nas_media_compat.py new file mode 100644 index 0000000..51dabb8 --- /dev/null +++ b/tests/unit/test_nas_media_compat.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import pytest + +from openblade.nas.media import ( + Cartridge, + ForeignMediaError, + IncompatibleMediaError, + MediaGeneration, + can_read, + can_write, + is_foreign, + require_writable, +) +from openblade.nas.planner import ArchivePlanner +from openblade.nas.types import ArchivePlanRequest, PolicyType + +G = MediaGeneration + + +@pytest.mark.parametrize( + ("drive", "readable", "writable"), + [ + (G.LTO6, {G.LTO5, G.LTO6}, {G.LTO5, G.LTO6}), + (G.LTO7, {G.LTO5, G.LTO6, G.LTO7}, {G.LTO6, G.LTO7}), + # LTO-8 and LTO-9 break the N-2 read rule. + (G.LTO8, {G.LTO7, G.LTO8}, {G.LTO7, G.LTO8}), + (G.LTO9, {G.LTO8, G.LTO9}, {G.LTO8, G.LTO9}), + ], +) +def test_compat_matrix(drive: G, readable: set[G], writable: set[G]) -> None: + assert {m for m in G if can_read(drive, m)} == readable + assert {m for m in G if can_write(drive, m)} == writable + + +def test_lto8_cannot_read_lto6_and_lto9_cannot_read_lto7() -> None: + assert not can_read(G.LTO8, G.LTO6) + assert not can_read(G.LTO9, G.LTO7) + + +def test_unknown_media_never_compatible() -> None: + assert not any(can_read(d, G.UNKNOWN) or can_write(d, G.UNKNOWN) for d in G) + + +def test_foreign_detection() -> None: + assert is_foreign(Cartridge(barcode="F00001L8")) + assert not is_foreign(Cartridge(barcode="F00001L8", adopted=True)) + assert not is_foreign(Cartridge(barcode="A00001L8", has_tape_json=True)) + + +def test_require_writable_typed_errors() -> None: + with pytest.raises(ForeignMediaError): + require_writable(Cartridge(barcode="F1", generation=G.LTO8), G.LTO8) + with pytest.raises(IncompatibleMediaError): + require_writable(Cartridge(barcode="A1", generation=G.LTO6, has_tape_json=True), G.LTO8) + + +def _request(cart: Cartridge, drive: G | None = None) -> ArchivePlanRequest: + return ArchivePlanRequest( + files=["/a"], + file_sizes={"/a": 10}, + available_tapes=[cart.barcode], + cartridges={cart.barcode: cart}, + drive_generation=drive, + ) + + +def test_archive_planner_refuses_foreign_media_without_adopt() -> None: + foreign = Cartridge(barcode="F00001L8", generation=G.LTO8) + with pytest.raises(ForeignMediaError): + ArchivePlanner().plan(_request(foreign)) + adopted = foreign.model_copy(update={"adopted": True}) + ArchivePlanner().plan(_request(adopted)) + + +def test_archive_planner_refuses_unwritable_pairing() -> None: + old = Cartridge(barcode="A00001L6", generation=G.LTO6, has_tape_json=True) + with pytest.raises(IncompatibleMediaError): + ArchivePlanner().plan(_request(old, G.LTO8)) + ArchivePlanner().plan(_request(old, G.LTO7)) + + +def test_archive_plan_reserved_bytes_block_plan_that_would_otherwise_fit() -> None: + def req(reserved: int) -> ArchivePlanRequest: + return ArchivePlanRequest( + policy_type=PolicyType.CRITICAL_SEQUENTIAL, + files=["/a"], + file_sizes={"/a": 80}, + available_tapes=["T1"], + tape_capacities={"T1": 100}, + reserved_bytes={"T1": reserved}, + ) + + assert ArchivePlanner().plan(req(0)).is_safe_to_enqueue + assert not ArchivePlanner().plan(req(30)).is_safe_to_enqueue From 5d5e98a5697c25e26f40862ef0ef26d289587223 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:03:07 +0000 Subject: [PATCH 034/205] feat(nas): scratch, fragmentation, reservation, replication, span, export and drive planning models Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/capacity.py | 305 +++++++++++++++++++++++++++ tests/unit/test_nas_domain_models.py | 189 +++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 openblade/nas/capacity.py create mode 100644 tests/unit/test_nas_domain_models.py diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py new file mode 100644 index 0000000..4121990 --- /dev/null +++ b/openblade/nas/capacity.py @@ -0,0 +1,305 @@ +"""Tape-native NAS planning models: scratch thresholds, fragmentation, +capacity reservations, VG replication, file spanning, export sets/vaults, +drive planning, and small-file aggregation. + +Pure models and functions; no I/O. Planners compose these. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from datetime import datetime, timedelta +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from openblade.nas.types import NasFileState + +# --- 2. scratch thresholds ------------------------------------------------- + + +class ScratchHealth(str, Enum): + OK = "ok" + WARN = "warn" + CRITICAL = "critical" + + +class ScratchThresholds(BaseModel): + model_config = ConfigDict(frozen=True) + + scratch_min: int = Field(ge=0) + scratch_warn: int = Field(ge=0) + + @model_validator(mode="after") + def _warn_above_min(self) -> ScratchThresholds: + if self.scratch_warn < self.scratch_min: + raise ValueError("scratch_warn must be >= scratch_min") + return self + + +def evaluate_scratch(scratch_count: int, thresholds: ScratchThresholds) -> ScratchHealth: + if scratch_count < thresholds.scratch_min: + return ScratchHealth.CRITICAL + if scratch_count < thresholds.scratch_warn: + return ScratchHealth.WARN + return ScratchHealth.OK + + +# --- 6. fragmentation ------------------------------------------------------ + +DEFAULT_CONSOLIDATION_RATIO = 0.5 + + +class TapeFragmentation(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + live_bytes: int = Field(ge=0) + written_bytes: int = Field(ge=0) + + @model_validator(mode="after") + def _live_le_written(self) -> TapeFragmentation: + if self.live_bytes > self.written_bytes: + raise ValueError("live_bytes cannot exceed written_bytes") + return self + + @property + def live_ratio(self) -> float: + return 1.0 if self.written_bytes == 0 else self.live_bytes / self.written_bytes + + def needs_consolidation(self, threshold: float = DEFAULT_CONSOLIDATION_RATIO) -> bool: + return self.written_bytes > 0 and self.live_ratio < threshold + + +# --- 7. capacity reservations --------------------------------------------- + + +class ReservationError(ValueError): + """A reservation cannot be satisfied from the pool's free capacity.""" + + +class CapacityReservation(BaseModel): + model_config = ConfigDict(frozen=True) + + pool_id: str = Field(min_length=1) + bytes: int = Field(gt=0) + owner: str = Field(min_length=1) + expires_at: datetime + + def active(self, now: datetime) -> bool: + return now < self.expires_at + + +class ReservationLedger: + """In-memory ledger of capacity reservations per pool.""" + + def __init__(self) -> None: + self._items: list[CapacityReservation] = [] + + def reserved(self, pool_id: str, now: datetime) -> int: + return sum(r.bytes for r in self._items if r.pool_id == pool_id and r.active(now)) + + def available(self, pool_id: str, free_bytes: int, now: datetime) -> int: + return max(0, free_bytes - self.reserved(pool_id, now)) + + def reserve( + self, + pool_id: str, + nbytes: int, + owner: str, + ttl: timedelta, + *, + free_bytes: int, + now: datetime, + ) -> CapacityReservation: + if nbytes > self.available(pool_id, free_bytes, now): + raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") + res = CapacityReservation(pool_id=pool_id, bytes=nbytes, owner=owner, expires_at=now + ttl) + self._items.append(res) + return res + + def release(self, pool_id: str, owner: str) -> int: + before = len(self._items) + self._items = [r for r in self._items if not (r.pool_id == pool_id and r.owner == owner)] + return before - len(self._items) + + +# --- 5. VG replication ----------------------------------------------------- + + +class ReplicationState(str, Enum): + IN_SYNC = "in_sync" + LAGGING = "lagging" + BROKEN = "broken" + + +class ExportPrepError(ValueError): + """Export preparation refused (e.g. replicas not in sync).""" + + +class VolumeGroupReplication(BaseModel): + model_config = ConfigDict(frozen=True) + + vg_id: str = Field(min_length=1) + barcodes: tuple[str, ...] = () + replicas_required: int = Field(ge=1) + replicas_present: int = Field(ge=0) + + @property + def state(self) -> ReplicationState: + if self.replicas_present >= self.replicas_required: + return ReplicationState.IN_SYNC + if self.replicas_present == 0: + return ReplicationState.BROKEN + return ReplicationState.LAGGING + + def replica_write_completed(self) -> VolumeGroupReplication: + return self.model_copy(update={"replicas_present": self.replicas_present + 1}) + + +def merge_volume_groups( + a: VolumeGroupReplication, b: VolumeGroupReplication, new_id: str +) -> VolumeGroupReplication: + """Merged VG needs the stricter requirement and has only the replicas both share.""" + return VolumeGroupReplication( + vg_id=new_id, + barcodes=a.barcodes + tuple(x for x in b.barcodes if x not in a.barcodes), + replicas_required=max(a.replicas_required, b.replicas_required), + replicas_present=min(a.replicas_present, b.replicas_present), + ) + + +def require_export_ready(vg: VolumeGroupReplication) -> None: + if vg.state is not ReplicationState.IN_SYNC: + raise ExportPrepError(f"volume group {vg.vg_id} is {vg.state.value}; cannot export-prep") + + +# --- 8. file spanning ------------------------------------------------------ + + +class SpanSegment(BaseModel): + model_config = ConfigDict(frozen=True) + + barcode: str = Field(min_length=1) + offset: int = Field(ge=0) + length: int = Field(gt=0) + + +class FileSpan(BaseModel): + model_config = ConfigDict(frozen=True) + + path: str = Field(min_length=1) + segments: tuple[SpanSegment, ...] = Field(min_length=1) + + @property + def barcodes(self) -> list[str]: + return list(dict.fromkeys(s.barcode for s in self.segments)) + + +def span_state(span: FileSpan, available: set[str]) -> NasFileState: + if all(b in available for b in span.barcodes): + return NasFileState.OFFLINE_ON_TAPE + return NasFileState.MISSING_TAPE + + +def required_tapes(spans: Iterable[FileSpan]) -> list[str]: + return list(dict.fromkeys(b for s in spans for b in s.barcodes)) + + +# --- 9. export sets + vault ------------------------------------------------ + + +class ExportSetState(str, Enum): + PREPARING = "preparing" + READY = "ready" + EXPORTED = "exported" + IMPORTED = "imported" + + +class ExportSet(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(min_length=1) + barcodes: tuple[str, ...] = Field(min_length=1) + state: ExportSetState = ExportSetState.PREPARING + + +class Vault(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(min_length=1) + barcodes: frozenset[str] = frozenset() + location: str = Field(min_length=1) + + +def mark_export_ready(export_set: ExportSet, groups: Sequence[VolumeGroupReplication]) -> ExportSet: + """PREPARING → READY only if every member tape's VG replication is IN_SYNC.""" + if export_set.state is not ExportSetState.PREPARING: + raise ExportPrepError(f"export set {export_set.id} is {export_set.state.value}") + for barcode in export_set.barcodes: + owners = [g for g in groups if barcode in g.barcodes] + if not owners: + raise ExportPrepError(f"tape {barcode} has no known replication state") + for g in owners: + require_export_ready(g) + return export_set.model_copy(update={"state": ExportSetState.READY}) + + +def planning_available(library_barcodes: Iterable[str], vaults: Iterable[Vault]) -> set[str]: + """Barcodes usable for planning: anything in a vault is OFFLINE.""" + vaulted = {b for v in vaults for b in v.barcodes} + return {b for b in library_barcodes if b not in vaulted} + + +# --- 10. required drives / estimated swaps -------------------------------- + + +class DrivePlan(BaseModel): + model_config = ConfigDict(frozen=True) + + tapes: tuple[str, ...] + required_drives: int + estimated_swaps: int + + +def plan_drives(spans: Iterable[FileSpan], available_drives: int) -> DrivePlan: + if available_drives < 1: + raise ValueError("available_drives must be >= 1") + tapes = required_tapes(spans) + used = min(len(tapes), available_drives) + return DrivePlan( + tapes=tuple(tapes), + required_drives=used, + estimated_swaps=max(0, len(tapes) - available_drives), + ) + + +# --- 11. small-file aggregation ------------------------------------------- + + +class AggregationPolicy(BaseModel): + model_config = ConfigDict(frozen=True) + + aggregate_below_bytes: int = Field(ge=0) + max_container_bytes: int = Field(gt=0) + + +def group_for_write( + files: Sequence[tuple[str, int]], policy: AggregationPolicy +) -> list[list[tuple[str, int]]]: + """Group files into write units: small files share containers, large stand alone.""" + units: list[list[tuple[str, int]]] = [] + current: list[tuple[str, int]] = [] + current_size = 0 + for path, size in files: + if size >= policy.aggregate_below_bytes: + units.append([(path, size)]) + continue + if current and current_size + size > policy.max_container_bytes: + units.append(current) + current, current_size = [], 0 + current.append((path, size)) + current_size += size + if current: + units.append(current) + return units diff --git a/tests/unit/test_nas_domain_models.py b/tests/unit/test_nas_domain_models.py new file mode 100644 index 0000000..c4a1a61 --- /dev/null +++ b/tests/unit/test_nas_domain_models.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from pydantic import ValidationError + +from openblade.nas.capacity import ( + AggregationPolicy, + ExportPrepError, + ExportSet, + ExportSetState, + FileSpan, + ReplicationState, + ReservationError, + ReservationLedger, + ScratchHealth, + ScratchThresholds, + SpanSegment, + TapeFragmentation, + Vault, + VolumeGroupReplication, + evaluate_scratch, + group_for_write, + mark_export_ready, + merge_volume_groups, + plan_drives, + planning_available, + require_export_ready, + required_tapes, + span_state, +) +from openblade.nas.types import NasFileState + +NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +# --- scratch thresholds +def test_scratch_thresholds_evaluate_ok_warn_critical() -> None: + t = ScratchThresholds(scratch_min=2, scratch_warn=5) + assert evaluate_scratch(1, t) is ScratchHealth.CRITICAL + assert evaluate_scratch(2, t) is ScratchHealth.WARN + assert evaluate_scratch(5, t) is ScratchHealth.OK + + +def test_scratch_thresholds_warn_below_min_rejected() -> None: + with pytest.raises(ValidationError): + ScratchThresholds(scratch_min=5, scratch_warn=2) + + +# --- fragmentation +def test_fragmentation_below_threshold_needs_consolidation() -> None: + assert TapeFragmentation(barcode="T1", live_bytes=40, written_bytes=100).needs_consolidation() + assert not TapeFragmentation( + barcode="T1", live_bytes=60, written_bytes=100 + ).needs_consolidation() + assert not TapeFragmentation(barcode="T1", live_bytes=0, written_bytes=0).needs_consolidation() + + +def test_fragmentation_live_above_written_rejected() -> None: + with pytest.raises(ValidationError): + TapeFragmentation(barcode="T1", live_bytes=2, written_bytes=1) + + +# --- reservations +def test_reservation_counts_against_free_capacity() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 60, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + assert ledger.available("p1", 100, NOW) == 40 + assert ledger.available("p2", 100, NOW) == 100 + + +def test_reservation_over_reserve_fails_typed() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 60, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + with pytest.raises(ReservationError): + ledger.reserve("p1", 41, "job-b", timedelta(minutes=5), free_bytes=100, now=NOW) + + +def test_reservation_expired_is_ignored() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 100, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + later = NOW + timedelta(minutes=6) + assert ledger.available("p1", 100, later) == 100 + ledger.reserve("p1", 100, "job-b", timedelta(minutes=5), free_bytes=100, now=later) + + +def test_reservation_release_frees_capacity() -> None: + ledger = ReservationLedger() + ledger.reserve("p1", 100, "job-a", timedelta(minutes=5), free_bytes=100, now=NOW) + assert ledger.release("p1", "job-a") == 1 + assert ledger.available("p1", 100, NOW) == 100 + + +# --- VG replication +def _vg(vg_id: str, req: int, present: int, *barcodes: str) -> VolumeGroupReplication: + return VolumeGroupReplication( + vg_id=vg_id, barcodes=barcodes, replicas_required=req, replicas_present=present + ) + + +def test_replication_state_assignment() -> None: + assert _vg("a", 2, 2).state is ReplicationState.IN_SYNC + assert _vg("a", 2, 1).state is ReplicationState.LAGGING + assert _vg("a", 2, 0).state is ReplicationState.BROKEN + + +def test_replication_merge_re_evaluates_state() -> None: + merged = merge_volume_groups(_vg("a", 1, 1, "T1"), _vg("b", 2, 2, "T2"), "ab") + assert merged.replicas_required == 2 + assert merged.state is ReplicationState.LAGGING + assert merged.barcodes == ("T1", "T2") + + +def test_replication_lagging_vg_cannot_export_prep() -> None: + with pytest.raises(ExportPrepError): + require_export_ready(_vg("a", 2, 1)) + + +def test_replication_broken_repairs_only_after_replica_write() -> None: + vg = _vg("a", 1, 0) + assert vg.state is ReplicationState.BROKEN + assert vg.replica_write_completed().state is ReplicationState.IN_SYNC + + +# --- spanning +def _span(path: str, *barcodes: str) -> FileSpan: + segs = tuple(SpanSegment(barcode=b, offset=0, length=10) for b in barcodes) + return FileSpan(path=path, segments=segs) + + +def test_span_required_tapes_deduplicated_in_order() -> None: + assert required_tapes([_span("/a", "T1", "T2"), _span("/b", "T2", "T3")]) == ["T1", "T2", "T3"] + + +def test_span_on_missing_tape_is_missing_tape() -> None: + span = _span("/a", "T1", "T2") + assert span_state(span, {"T1", "T2"}) is NasFileState.OFFLINE_ON_TAPE + assert span_state(span, {"T1"}) is NasFileState.MISSING_TAPE + + +def test_span_requires_segments() -> None: + with pytest.raises(ValidationError): + FileSpan(path="/a", segments=()) + + +# --- export sets + vault +def test_export_set_ready_requires_all_members_in_sync() -> None: + es = ExportSet(id="e1", barcodes=("T1", "T2")) + groups = [_vg("a", 1, 1, "T1"), _vg("b", 2, 1, "T2")] + with pytest.raises(ExportPrepError): + mark_export_ready(es, groups) + groups[1] = groups[1].replica_write_completed() + assert mark_export_ready(es, groups).state is ExportSetState.READY + + +def test_export_set_unknown_member_refused() -> None: + with pytest.raises(ExportPrepError): + mark_export_ready(ExportSet(id="e1", barcodes=("T9",)), [_vg("a", 1, 1, "T1")]) + + +def test_vaulted_barcode_is_offline_for_planning() -> None: + vault = Vault(id="v1", barcodes=frozenset({"T2"}), location="offsite") + avail = planning_available(["T1", "T2"], [vault]) + assert avail == {"T1"} + assert span_state(_span("/a", "T2"), avail) is NasFileState.MISSING_TAPE + + +# --- drive planning +def test_plan_drives_one_drive_three_tapes() -> None: + plan = plan_drives([_span("/a", "T1", "T2"), _span("/b", "T3")], available_drives=1) + assert plan.required_drives == 1 + assert plan.estimated_swaps == 2 + + +def test_plan_drives_enough_drives_no_swaps() -> None: + plan = plan_drives([_span("/a", "T1", "T2", "T3")], available_drives=4) + assert (plan.required_drives, plan.estimated_swaps) == (3, 0) + + +# --- aggregation +def test_aggregation_groups_small_files_into_containers() -> None: + policy = AggregationPolicy(aggregate_below_bytes=10, max_container_bytes=12) + files = [("/s1", 5), ("/big", 50), ("/s2", 5), ("/s3", 5)] + assert group_for_write(files, policy) == [ + [("/big", 50)], + [("/s1", 5), ("/s2", 5)], + [("/s3", 5)], + ] From 2e4eb316203528f56ae3738a668789d199e937a5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:03:07 +0000 Subject: [PATCH 035/205] docs(nas): document tape-native planning models Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/architecture.md | 9 +++++++++ docs/wiki/guides/fuse-and-nas.md | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/architecture.md b/docs/architecture.md index f452c15..bd55902 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,3 +56,12 @@ Drive exclusion for tape I/O is a catalog row, not process memory. `DriveSchedul ## FUSE namespace The FUSE-oriented layer is intentionally thin: catalog entries define the namespace, and hydration delegates to restore workflows. This keeps the namespace authoritative and avoids bypassing safety or verification logic. + +## NAS tape-native planning models + +Cartridge lifecycle (`openblade/nas/state_machine.py`), LTO media/drive +compatibility and foreign-media detection (`openblade/nas/media.py`), and pure +capacity/replication/export/spanning planners (`openblade/nas/capacity.py`) +sit beside the existing archive and restore planners. They hold no I/O and no +persistence yet; see the "Tape-native domain models" section of +`docs/wiki/guides/fuse-and-nas.md` for the model list and tests. diff --git a/docs/wiki/guides/fuse-and-nas.md b/docs/wiki/guides/fuse-and-nas.md index 02bbad8..1a0cbfc 100644 --- a/docs/wiki/guides/fuse-and-nas.md +++ b/docs/wiki/guides/fuse-and-nas.md @@ -229,6 +229,26 @@ policy-resolution helper for ingest and has nothing to do with mounting. --- +## Tape-native domain models + +Pure planning models (no I/O). Each is validated and covered by unit tests. + +| Model / function | Module | Tests | +|---|---|---| +| `TapeState` / `TapeEvent` / `transition()` -> `InvalidTransitionError`; `file_state_for_tape()` maps onto the existing `NasFileState` | `openblade/nas/state_machine.py` | `tests/unit/test_nas_tape_states.py` (every allowed edge, every forbidden edge) | +| `MediaGeneration`, `can_read` / `can_write` (LTO-8/9 break the N-2 read rule), `Cartridge`, `is_foreign`, `require_writable` | `openblade/nas/media.py` | `tests/unit/test_nas_media_compat.py` | +| `ArchivePlanRequest.cartridges` / `drive_generation` / `reserved_bytes`: the archive planner raises `ForeignMediaError` for un-adopted media without `tape.json` (never auto-formatted) and `IncompatibleMediaError` for an unwritable drive/media pairing; reserved bytes reduce per-tape capacity | `openblade/nas/planner.py` | `tests/unit/test_nas_media_compat.py` | +| `ScratchThresholds` + `evaluate_scratch` (OK/WARN/CRITICAL) | `openblade/nas/capacity.py` | `tests/unit/test_nas_domain_models.py` | +| `TapeFragmentation.needs_consolidation` (`live_bytes / written_bytes`) | same | same | +| `ReservationLedger.reserve/release` (TTL; expired ignored; over-reserve -> `ReservationError`) | same | same | +| `VolumeGroupReplication` (IN_SYNC/LAGGING/BROKEN), `merge_volume_groups`, `require_export_ready` | same | same | +| `FileSpan` / `SpanSegment`, `required_tapes`, `span_state` (missing tape -> `MISSING_TAPE`) | same | same | +| `ExportSet` (PREPARING/READY/EXPORTED/IMPORTED), `mark_export_ready`, `Vault`, `planning_available` (vaulted = offline) | same | same | +| `plan_drives` -> `required_drives`, `estimated_swaps` | same | same | +| `AggregationPolicy.aggregate_below_bytes` + `group_for_write` (grouping only, no container format) | same | same | + +No read-only HTTP view is exposed yet: there is no per-pool health hook to attach to. + ## Related - [The catalog](the-catalog.md) — the two independent catalogs From 58827b31a31463f65bc6e85f8f5d7166ce007395 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:04:10 +0000 Subject: [PATCH 036/205] fix(fuse): never budget-evict the entry just stored for a waiting reader Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 95e8585..12d2652 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -66,7 +66,8 @@ def _track(self, checksum: str, size: int) -> None: def _enforce_budget(self) -> None: if self.max_bytes is None: return - for checksum in list(self._lru): + # The newest entry is exempt: it was just stored for a reader about to open it. + for checksum in list(self._lru)[:-1]: if sum(self._lru.values()) <= self.max_bytes: return if self._refs.get(checksum, 0) > 0: From d465a5e17488a5ce51715b07b7c8c745a9d49e55 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:04:10 +0000 Subject: [PATCH 037/205] test(fuse): data-plane integration tests (hydrate, tickets, timeout, eviction, failure, batching, restart, unmount) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_fuse_data_plane.py | 170 ++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tests/integration/test_fuse_data_plane.py diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py new file mode 100644 index 0000000..09d44c2 --- /dev/null +++ b/tests/integration/test_fuse_data_plane.py @@ -0,0 +1,170 @@ +"""FUSE data plane: hydration, cache, batching, safety (no kernel FUSE needed).""" + +from __future__ import annotations + +import errno +import hashlib +import os +import threading +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.fuse.cache import CacheEntryInUseError, HydrationCache +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import HydrationFailedError, Hydrator, JobRestoreEngine +from openblade.fuse.mount import STATE_XATTR, CatalogFuseOperations + +FILES = { + "/a.bin": (b"alpha" * 100, "TAPE01"), + "/b.bin": (b"bravo" * 50, "TAPE01"), + "/c.bin": (b"charlie", "TAPE02"), +} + + +class FakeEngine: + """Stands in for the restore job path; records one call per tape batch.""" + + def __init__(self, staging: Path, *, corrupt: bool = False) -> None: + self.staging = staging + self.corrupt = corrupt + self.batches: list[tuple[str, list[str]]] = [] + self.gate = threading.Event() + self.gate.set() + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + self.gate.wait(10) + self.batches.append((tape_key, sorted(catalog_paths))) + out = {} + for path in catalog_paths: + dest = self.staging / path.strip("/") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(b"half" if self.corrupt else FILES[path][0]) + out[path] = dest + return out + + def find_active_job(self, catalog_path: str) -> str | None: + return None + + def resume(self, job_id: str) -> Path: + raise HydrationFailedError(job_id) + + +@pytest.fixture +def catalog(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + for path, (data, barcode) in FILES.items(): + rec = repo.create_file_record(path, len(data), hashlib.sha256(data).hexdigest(), group.id) + inst = repo.create_file_instance(rec.id, barcode, path) + repo.mark_instance_archived(inst.id) + return repo + + +def _plane(catalog: CatalogRepository, tmp_path: Path, **kw: object): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "staging", corrupt=bool(kw.pop("corrupt", False))) + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.05) + ops = CatalogFuseOperations(fs, data_plane=hyd, hydrate_timeout=5.0, **kw) # type: ignore[arg-type] + return ops, hyd, engine + + +def test_offline_read_hydrates_and_bytes_match(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, _, _ = _plane(catalog, tmp_path) + assert ops.getattr("/a.bin")["st_size"] == len(FILES["/a.bin"][0]) # real size while offline + assert ops.getxattr("/a.bin", STATE_XATTR) == b"offline" + assert ops.listxattr("/a.bin") == [STATE_XATTR] + fh = ops.open("/a.bin", os.O_RDONLY) + assert ops.read("/a.bin", 10_000, 0, fh) == FILES["/a.bin"][0] + assert ops.getxattr("/a.bin", STATE_XATTR) == b"online" + + +def test_concurrent_requests_share_one_ticket(catalog, tmp_path): # type: ignore[no-untyped-def] + _, hyd, engine = _plane(catalog, tmp_path) + engine.gate.clear() + t1, t2 = hyd.request("/a.bin"), hyd.request("/a.bin") + assert t1 is t2 + assert hyd.status("/a.bin").value == "hydrating" + engine.gate.set() + hyd.wait(t1, 5) + assert len(engine.batches) == 1 + + +def test_nonblocking_open_returns_eagain_then_succeeds(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path, blocking=False) + engine.gate.clear() + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EAGAIN + engine.gate.set() + hyd.wait(hyd.request("/a.bin"), 5) + assert ops.open("/a.bin", os.O_RDONLY) > 0 + + +def test_blocking_open_times_out_with_eagain(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path) + ops.hydrate_timeout = 0.2 + engine.gate.clear() + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EAGAIN + engine.gate.set() + hyd.shutdown() + + +def test_eviction_never_drops_an_open_file(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, _ = _plane(catalog, tmp_path) + fh = ops.open("/a.bin", os.O_RDONLY) + checksum = catalog.get_file_record("/a.bin").checksum_sha256 # type: ignore[union-attr] + with pytest.raises(CacheEntryInUseError): + hyd.cache.evict(checksum) + hyd.cache.max_bytes = 1 # budget pressure must skip the open file too + ops.open("/c.bin", os.O_RDONLY) + assert hyd.cache.is_cached(checksum) + ops.release("/a.bin", fh) + assert not hyd.cache.is_cached(checksum) # released -> evictable under budget + + +def test_failed_restore_leaves_file_offline_without_partial(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, _ = _plane(catalog, tmp_path, corrupt=True) + with pytest.raises(OSError) as exc: + ops.open("/a.bin", os.O_RDONLY) + assert exc.value.errno == errno.EIO + assert ops.getxattr("/a.bin", STATE_XATTR) == b"offline" + assert hyd.last_error("/a.bin") == "restored bytes failed checksum verification" + leftovers = [p for p in (tmp_path / "cache").rglob("*") if p.is_file()] + assert leftovers == [] # no half file under the checksum name or a temp name + + +def test_same_tape_batching(catalog, tmp_path): # type: ignore[no-untyped-def] + _, hyd, engine = _plane(catalog, tmp_path) + tickets = [hyd.request(p) for p in ("/a.bin", "/b.bin", "/c.bin")] + for t in tickets: + hyd.wait(t, 5) + assert sorted(engine.batches) == [("TAPE01", ["/a.bin", "/b.bin"]), ("TAPE02", ["/c.bin"])] + + +def test_restart_rederives_ticket_from_job_state(catalog, tmp_path): # type: ignore[no-untyped-def] + dest = tmp_path / "staged" / "a" + job = catalog.create_job("restore", {"catalog_path": "/a.bin", "dest_path": str(dest)}) + cache = HydrationCache(str(tmp_path / "cache")) + hyd = Hydrator(catalog, cache, JobRestoreEngine(None, catalog, tmp_path), batch_window_s=0.05) + ticket = hyd.request("/a.bin") + assert ticket.job_id == job.id # re-attached, not a second restore + dest.parent.mkdir(parents=True) + dest.write_bytes(FILES["/a.bin"][0]) + job.state = "completed" + catalog.session.commit() + assert hyd.wait(ticket, 5) == FILES["/a.bin"][0] + + +def test_unmount_cancels_pending_batches(catalog, tmp_path): # type: ignore[no-untyped-def] + ops, hyd, engine = _plane(catalog, tmp_path) + hyd.batch_window_s = 30 + ticket = hyd.request("/a.bin") + ops.destroy("/") + assert ticket.done.is_set() and ticket.error == "hydration cancelled by unmount" + assert engine.batches == [] From 61905841594931c6c4bbb5bf015bfa74a4da206a Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:04:11 +0000 Subject: [PATCH 038/205] build: install fuse3/libfuse3 so the FUSE mount works in the shipped image Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6cd0525..b0dc4ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ mtx \ gosu \ + fuse3 \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml README.md ./ From a06aeb02fe8d59d5c0e9ef1744c239387b577e95 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:04:26 +0000 Subject: [PATCH 039/205] test(i3): Clock abstraction and fault-shaped timing profiles Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/i3/test_timing_profiles.py | 248 ++++++++++++++++++++++++++ tests/i3/timing.py | 293 ++++++++++++++++++++++++++++++- 2 files changed, 533 insertions(+), 8 deletions(-) create mode 100644 tests/i3/test_timing_profiles.py diff --git a/tests/i3/test_timing_profiles.py b/tests/i3/test_timing_profiles.py new file mode 100644 index 0000000..a434ea0 --- /dev/null +++ b/tests/i3/test_timing_profiles.py @@ -0,0 +1,248 @@ +"""Unit tests for tests/i3/timing.py profiles. + +Runs entirely under ``VirtualClock`` -- no emulator, no real sleeping. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Iterator + +import pytest + +from tests.i3.timing import ( + PROFILE_NAMES, + ProfileName, + ProfileRuntime, + RealClock, + RetryableOpError, + SessionExpiredError, + VirtualClock, + emulator_env, + get_clock, + get_profile_name, + use_clock, + wait_for_op, +) + +_MAX_WALL_S = 1.0 + + +@pytest.fixture +def vclock(monkeypatch: pytest.MonkeyPatch) -> Iterator[VirtualClock]: + monkeypatch.delenv("I3_TIMING_PROFILE", raising=False) + monkeypatch.delenv("I3_TEST_MODE", raising=False) + clock = VirtualClock() + started = time.monotonic() + with use_clock(clock): + yield clock + assert time.monotonic() - started < _MAX_WALL_S, "profile test slept for real" + + +def _runtime(name: ProfileName, clock: VirtualClock) -> ProfileRuntime: + return ProfileRuntime(name, clock=clock) + + +# --- Clock ------------------------------------------------------------------ + + +def test_virtual_clock_sleep_advances_and_records() -> None: + clock = VirtualClock(start=10.0) + + clock.sleep(2.5) + clock.advance(1.0) + + assert clock.monotonic() == 13.5 + assert clock.sleeps == [2.5] + + +def test_virtual_clock_negative_duration_raises() -> None: + clock = VirtualClock() + + with pytest.raises(ValueError): + clock.sleep(-1.0) + with pytest.raises(ValueError): + clock.advance(-1.0) + + +def test_use_clock_restores_previous_clock() -> None: + before = get_clock() + + with use_clock(VirtualClock()) as installed: + assert get_clock() is installed + + assert get_clock() is before + assert isinstance(before, RealClock) + + +# --- Profile selection -------------------------------------------------------- + + +@pytest.mark.parametrize("name", PROFILE_NAMES) +def test_profile_name_from_env_resolves(name: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", f" {name.upper()} ") + + assert get_profile_name() == name + + +def test_unknown_profile_falls_back_to_mode_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", "bogus") + monkeypatch.setenv("I3_TEST_MODE", "real") + + assert get_profile_name() == "hardware" + + +# --- Delays through the clock ------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "expected"), [("instant", []), ("realistic", [3.0]), ("hardware", [35.0])] +) +def test_wait_for_op_legacy_profiles_sleep_via_clock( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch, name: str, expected: list[float] +) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", name) + + wait_for_op("tape_load") + + assert vclock.sleeps == expected + + +def test_wait_for_op_hardware_rewind_does_not_really_sleep( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("I3_TIMING_PROFILE", "hardware") + + wait_for_op("rewind", multiplier=2.0) + + assert vclock.monotonic() == 180.0 + + +def test_slow_robotics_move_is_three_times_normal(vclock: VirtualClock) -> None: + normal, slow = _runtime("normal", vclock), _runtime("slow-robotics", vclock) + + normal.perform("move") + slow.perform("move") + + assert vclock.sleeps == [1.5, 4.5] + + +def test_busy_library_adds_queue_wait_to_mechanical_ops_only(vclock: VirtualClock) -> None: + busy = _runtime("busy-library", vclock) + + busy.perform("tape_load") + busy.perform("auth") + + assert vclock.sleeps == [5.0, 0.1] + + +# --- Faults ----------------------------------------------------------------- + + +def _load_outcomes(runtime: ProfileRuntime, count: int) -> list[int | None]: + outcomes: list[int | None] = [] + for _ in range(count): + try: + runtime.perform("tape_load") + outcomes.append(None) + except RetryableOpError as exc: + outcomes.append(exc.status) + return outcomes + + +def test_intermittent_drive_every_third_load_fails_retryably(vclock: VirtualClock) -> None: + runtime = _runtime("intermittent-drive", vclock) + + outcomes = _load_outcomes(runtime, 6) + + assert outcomes == [None, None, 409, None, None, 409] + + +def test_normal_profile_loads_never_fail(vclock: VirtualClock) -> None: + assert _load_outcomes(_runtime("normal", vclock), 6) == [None] * 6 + + +def test_session_expiry_rejects_session_after_ttl(vclock: VirtualClock) -> None: + runtime = _runtime("session-expiry", vclock) + issued = runtime.issue_session() + + vclock.advance(4.9) + runtime.check_session(issued) + vclock.advance(0.1) + + with pytest.raises(SessionExpiredError): + runtime.check_session(issued) + + +def test_normal_profile_session_survives_an_hour(vclock: VirtualClock) -> None: + runtime = _runtime("normal", vclock) + issued = runtime.issue_session() + + vclock.advance(3600.0) + + runtime.check_session(issued) + + +def test_rebooting_returns_503_until_window_closes(vclock: VirtualClock) -> None: + runtime = _runtime("rebooting", vclock) + statuses: list[int] = [] + + while True: + try: + runtime.perform("inventory") + break + except RetryableOpError as exc: + statuses.append(exc.status) + vclock.sleep(5.0) + + assert statuses == [503, 503, 503, 503] + assert vclock.monotonic() == 22.0 + + +def test_degraded_media_verify_slow_with_checksum_retry(vclock: VirtualClock) -> None: + runtime = _runtime("degraded-media", vclock) + + for _ in range(4): + runtime.perform("verify") + runtime.perform("read") + + assert runtime.checksum_retries == 1 + assert vclock.sleeps == [9.0, 9.0, 9.0, 9.0, 9.0, 6.0] + + +def test_normal_profile_verify_has_no_checksum_retry(vclock: VirtualClock) -> None: + runtime = _runtime("normal", vclock) + + for _ in range(8): + runtime.perform("verify") + + assert runtime.checksum_retries == 0 + + +# --- Emulator env wiring ------------------------------------------------------ + + +@pytest.mark.parametrize("name", ["instant", "realistic", "hardware"]) +def test_emulator_env_legacy_profiles_unchanged(name: ProfileName) -> None: + assert emulator_env(name) == { + "I3_TIMING_PROFILE": name, + "OPENBLADE_EMULATOR_LATENCY_PROFILE": name, + } + + +def test_emulator_env_latency_override_accepted_by_emulator_hook( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from openblade.api.aml_state import _env_emulator_latency_profile_ms + + env = emulator_env("slow-robotics") + monkeypatch.setenv( + "OPENBLADE_EMULATOR_LATENCY_PROFILE_MS", env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"] + ) + + parsed = _env_emulator_latency_profile_ms() + + assert env["OPENBLADE_EMULATOR_LATENCY_PROFILE"] == "realistic" + assert parsed == json.loads(env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"]) + assert parsed is not None and parsed["move"]["realistic"] == 4500 diff --git a/tests/i3/timing.py b/tests/i3/timing.py index 05de605..deb7e5d 100644 --- a/tests/i3/timing.py +++ b/tests/i3/timing.py @@ -1,20 +1,44 @@ """ Timing profiles for Quantum i3 test suite. -Three profiles: +Legacy profiles (names kept stable for CI and existing callers): instant — zero delays, for fast CI runs realistic — short delays that feel like real hardware (default in emulator mode) hardware — real Quantum i3 mechanical tolerances (used with real i3 target) +Scenario profiles (realistic baseline plus one stressor each): + normal — alias of realistic delays, no faults + slow-robotics — robot/drive mechanics 3x slower + busy-library — every mechanical op waits behind a contended robot queue + intermittent-drive — every Nth tape load fails with a RetryableOpError + session-expiry — auth sessions expire after a short TTL + rebooting — a window of HTTP 503 at the start of the run + degraded-media — read/verify 3x slower + periodic checksum retry + +All sleeping goes through a ``Clock`` (``RealClock`` by default). Unit tests +install a ``VirtualClock`` with ``use_clock()`` so profile behaviour can be +asserted deterministically without real sleeping. + +Delays are applied client-side by ``wait_for_op``. Faults (load failures, +session expiry, 503 window, checksum retries) are applied client-side by a +``ProfileRuntime`` because the emulator has no env-driven fault hook for them; +``emulator_env()`` exports the env vars the emulator DOES honour +(``I3_TIMING_PROFILE``, ``OPENBLADE_EMULATOR_LATENCY_PROFILE``, +``OPENBLADE_EMULATOR_LATENCY_PROFILE_MS``) so its own latency matches. + Usage: from tests.i3.timing import get_profile, wait_for_op, assert_within_tolerance """ from __future__ import annotations +import json import os import time -from typing import Literal +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Literal, Protocol, get_args OpType = Literal[ "tape_load", @@ -26,9 +50,96 @@ "unmount", "inventory", "auth", + "read", + "verify", +] + +LegacyProfileName = Literal["instant", "realistic", "hardware"] +ProfileName = Literal[ + "instant", + "realistic", + "hardware", + "normal", + "slow-robotics", + "busy-library", + "intermittent-drive", + "session-expiry", + "rebooting", + "degraded-media", ] +PROFILE_NAMES: tuple[ProfileName, ...] = get_args(ProfileName) + + +# --------------------------------------------------------------------------- +# Clock +# --------------------------------------------------------------------------- + + +class Clock(Protocol): + """Time source used by every profile sleep.""" + + def monotonic(self) -> float: ... + + def sleep(self, seconds: float) -> None: ... + + +class RealClock: + """Wall-clock implementation backed by :mod:`time`.""" + + def monotonic(self) -> float: + return time.monotonic() + + def sleep(self, seconds: float) -> None: + if seconds > 0: + time.sleep(seconds) + + +class VirtualClock: + """Deterministic clock: ``sleep`` advances virtual time instantly.""" + + def __init__(self, start: float = 0.0) -> None: + self._now = start + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self._now + + def sleep(self, seconds: float) -> None: + if seconds < 0: + raise ValueError("cannot sleep a negative duration") + self.sleeps.append(seconds) + self._now += seconds + + def advance(self, seconds: float) -> None: + """Move virtual time forward without recording a sleep.""" + if seconds < 0: + raise ValueError("cannot advance a negative duration") + self._now += seconds + + +_clock: Clock = RealClock() + + +def get_clock() -> Clock: + return _clock + + +def set_clock(clock: Clock) -> Clock: + """Install ``clock`` globally and return the previous one.""" + global _clock + previous = _clock + _clock = clock + return previous + + +@contextmanager +def use_clock(clock: Clock) -> Iterator[Clock]: + previous = set_clock(clock) + try: + yield clock + finally: + set_clock(previous) -ProfileName = Literal["instant", "realistic", "hardware"] TIMING_PROFILES: dict[ProfileName, dict[OpType, float]] = { "instant": { @@ -41,6 +152,8 @@ "unmount": 0.0, "inventory": 0.0, "auth": 0.0, + "read": 0.0, + "verify": 0.0, }, "realistic": { "tape_load": 3.0, @@ -52,6 +165,8 @@ "unmount": 1.5, "inventory": 2.0, "auth": 0.1, + "read": 2.0, + "verify": 3.0, }, "hardware": { "tape_load": 35.0, @@ -63,9 +178,168 @@ "unmount": 10.0, "inventory": 45.0, "auth": 0.5, + "read": 60.0, + "verify": 120.0, }, } +_MECHANICAL_OPS: tuple[OpType, ...] = ( + "tape_load", + "tape_unload", + "move", + "mount", + "unmount", + "inventory", +) +_REALISTIC = TIMING_PROFILES["realistic"] +_BUSY_QUEUE_WAIT_S = 2.0 + + +def _scaled(ops: tuple[OpType, ...], factor: float) -> dict[OpType, float]: + return {op: (v * factor if op in ops else v) for op, v in _REALISTIC.items()} + + +TIMING_PROFILES.update( + { + "normal": dict(_REALISTIC), + "slow-robotics": _scaled(_MECHANICAL_OPS, 3.0), + "busy-library": { + op: (v + _BUSY_QUEUE_WAIT_S if op in _MECHANICAL_OPS else v) + for op, v in _REALISTIC.items() + }, + "intermittent-drive": dict(_REALISTIC), + "session-expiry": dict(_REALISTIC), + "rebooting": dict(_REALISTIC), + "degraded-media": _scaled(("read", "verify"), 3.0), + } +) + + +@dataclass(frozen=True) +class ProfileFaults: + """Client-side fault knobs for a scenario profile (0/None = off).""" + + base: LegacyProfileName = "realistic" + load_fail_every: int = 0 + auth_ttl_s: float | None = None + reboot_window_s: float = 0.0 + checksum_retry_every: int = 0 + + +PROFILE_FAULTS: dict[ProfileName, ProfileFaults] = { + "instant": ProfileFaults(base="instant"), + "realistic": ProfileFaults(base="realistic"), + "hardware": ProfileFaults(base="hardware"), + "normal": ProfileFaults(), + "slow-robotics": ProfileFaults(), + "busy-library": ProfileFaults(), + "intermittent-drive": ProfileFaults(load_fail_every=3), + "session-expiry": ProfileFaults(auth_ttl_s=5.0), + "rebooting": ProfileFaults(reboot_window_s=20.0), + "degraded-media": ProfileFaults(checksum_retry_every=4), +} + + +class RetryableOpError(Exception): + """A transient failure the caller is expected to retry.""" + + def __init__(self, op_type: OpType, status: int, message: str) -> None: + super().__init__(message) + self.op_type = op_type + self.status = status + + +class SessionExpiredError(Exception): + """The session issued by ``ProfileRuntime.issue_session`` has expired (HTTP 401).""" + + +@dataclass +class ProfileRuntime: + """Stateful, per-test application of a profile's delays and faults.""" + + name: ProfileName + clock: Clock = field(default_factory=get_clock) + loads: int = 0 + verifies: int = 0 + checksum_retries: int = 0 + started_at: float = field(init=False) + + def __post_init__(self) -> None: + self.started_at = self.clock.monotonic() + + @property + def faults(self) -> ProfileFaults: + return PROFILE_FAULTS[self.name] + + def _sleep_for(self, op_type: OpType, multiplier: float = 1.0) -> None: + self.clock.sleep(TIMING_PROFILES[self.name].get(op_type, 0.0) * multiplier) + + def perform(self, op_type: OpType, multiplier: float = 1.0) -> None: + """Apply delay + faults for one operation; raise on an injected fault.""" + window = self.faults.reboot_window_s + if window and self.clock.monotonic() - self.started_at < window: + raise RetryableOpError(op_type, 503, "library rebooting (503)") + self._sleep_for(op_type, multiplier) + if op_type == "tape_load": + self.loads += 1 + every = self.faults.load_fail_every + if every and self.loads % every == 0: + raise RetryableOpError(op_type, 409, f"drive load {self.loads} failed (retryable)") + if op_type == "verify": + self.verifies += 1 + every = self.faults.checksum_retry_every + if every and self.verifies % every == 0: + self.checksum_retries += 1 + self._sleep_for("verify", multiplier) + + def issue_session(self) -> float: + """Return the clock time at which a freshly issued session was created.""" + self._sleep_for("auth") + return self.clock.monotonic() + + def check_session(self, issued_at: float) -> None: + ttl = self.faults.auth_ttl_s + if ttl is not None and self.clock.monotonic() - issued_at >= ttl: + raise SessionExpiredError(f"session expired after {ttl:.1f}s") + + +_EMULATOR_LATENCY_OPS: tuple[OpType, ...] = ( + "auth", + "inventory", + "mount", + "unmount", + "move", + "format", +) + + +def emulator_env(name: ProfileName) -> dict[str, str]: + """Env vars to start the emulator with so its latency matches ``name``. + + Uses only hooks the emulator already reads: ``I3_TIMING_PROFILE`` + (simulator load delay; legacy names only), ``OPENBLADE_EMULATOR_LATENCY_PROFILE`` + and ``OPENBLADE_EMULATOR_LATENCY_PROFILE_MS`` (``openblade/api/aml_state.py``). + Faults have no env hook and stay client-side in ``ProfileRuntime``. + """ + base = PROFILE_FAULTS[name].base + env: dict[str, str] = {"I3_TIMING_PROFILE": base, "OPENBLADE_EMULATOR_LATENCY_PROFILE": base} + if name not in get_args(LegacyProfileName): + delays = TIMING_PROFILES[name] + hardware = TIMING_PROFILES["hardware"] + env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"] = json.dumps( + { + op: { + "instant": 0, + "realistic": round(delays[op] * 1000), + "hardware": round(hardware[op] * 1000), + } + for op in _EMULATOR_LATENCY_OPS + }, + sort_keys=True, + ) + return env + + # Tolerance multipliers for hardware mode (real i3 timing can vary) HARDWARE_TOLERANCE: dict[OpType, float] = { "tape_load": 0.4, # ±40% — mechanical variation @@ -83,8 +357,9 @@ def get_profile_name() -> ProfileName: """Read profile from env. Defaults to 'instant' for emulator, 'hardware' for real.""" env = os.environ.get("I3_TIMING_PROFILE", "").strip().lower() - if env in ("instant", "realistic", "hardware"): - return env # type: ignore[return-value] + for name in PROFILE_NAMES: + if env == name: + return name mode = os.environ.get("I3_TEST_MODE", "emulator").strip().lower() return "hardware" if mode == "real" else "instant" @@ -97,12 +372,14 @@ def get_profile() -> dict[OpType, float]: def wait_for_op(op_type: OpType, multiplier: float = 1.0) -> None: """Sleep for the appropriate time for this operation. - In instant mode this is a no-op. In realistic/hardware modes the sleep - simulates the mechanical delay an operator would observe on a real i3. + In instant mode this is a no-op. Otherwise the sleep simulates the + mechanical delay an operator would observe on a real i3. The sleep goes + through the active ``Clock`` (see ``use_clock``). Faults are NOT applied + here -- use ``ProfileRuntime.perform`` for fault-bearing profiles. """ delay = get_profile().get(op_type, 0.0) * multiplier if delay > 0: - time.sleep(delay) + get_clock().sleep(delay) def assert_within_tolerance( From f22c4a8515c05ea5ee90197f6a329d1dde19ffa0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:04:35 +0000 Subject: [PATCH 040/205] docs(fuse): describe the five data-plane layers honestly; correct the wiki gap list Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/fuse.md | 15 +++++++++++++-- docs/wiki/guides/fuse-and-nas.md | 8 ++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/fuse.md b/docs/fuse.md index d2a53a0..f7f68d8 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -1,3 +1,14 @@ -# FUSE namespace +# FUSE data plane -OpenBlade's FUSE-facing layer is currently a lightweight namespace abstraction over the catalog. It is designed so future kernel FUSE work can reuse the same hydration and metadata contracts without bypassing archive safety rules. +Five layers, top to bottom, and what is real today: + +| Layer | Status | +|---|---| +| SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`). | +| Namespace/metadata | **Real.** `CatalogFuseOperations` (`openblade/fuse/mount.py`): offline files report their real size; xattr `user.openblade.state` = `offline`/`hydrating`/`online`. | +| Hydration/cache | **Real.** `Hydrator` (`openblade/fuse/hydration.py`): one shared ticket per path, `OPENBLADE_FUSE_HYDRATE_TIMEOUT` (default 300 s) blocking wait, or `EAGAIN` with `blocking=False`; failure = `EIO`, file stays `offline`, reason in `last_error()`. `HydrationCache` (`cache.py`): optional byte budget with LRU eviction that never evicts a file with open handles; restores land under a temp name and are renamed only after the sha256 matches. | +| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue` (one restore job per file); never a tape backend directly. A restarted hydrator re-attaches to a pending/running restore job for the path. | +| Placement/robotics | **Partial.** Requests are batched per tape barcode for `OPENBLADE_FUSE_BATCH_WINDOW_MS` (default 250 ms) and restored back to back; whether that is one physical load is the jobs scheduler's decision. | + +Unmount (`destroy`) cancels batches not yet started and waits for in-flight restores. +The `openblade fuse` CLI still uses the older synchronous `--hydrate` callable; the Hydrator is not wired into `create_context` yet. diff --git a/docs/wiki/guides/fuse-and-nas.md b/docs/wiki/guides/fuse-and-nas.md index 02bbad8..c562d6a 100644 --- a/docs/wiki/guides/fuse-and-nas.md +++ b/docs/wiki/guides/fuse-and-nas.md @@ -73,15 +73,15 @@ Other limits, stated rather than hidden: |---|---| | `OPENBLADE_FUSE_MOUNT_POINT` | The config field `fuse_mount_point` exists, is **read by nothing**, and has no environment override. `openblade fuse mount` takes the mount point as an argument. | | A FUSE mount of the **NAS** namespace | **No.** Only the catalog namespace is mountable. | -| An SMB (Samba) server | **No.** `openblade/nas/samba.py` is a 15-line class whose only method renders a 3-line `smb.conf` stanza. It is referenced by nothing — not by the app, not by tests. | -| An NFS server | **No.** Same shape: `nfs.py` renders one `/etc/exports` line and is referenced by nothing. | +| An SMB (Samba) server | **Not in the image.** The protocol rig lives in `deploy/nas-protocols`; see it for status. | +| An NFS server | **Not in the image.** See `deploy/nas-protocols`. | | An SFTP listener | **Effectively no.** See below. | -| `libfuse` in the shipped image | **No.** The extra and `libfuse2` are a host-side install; the container does not ship them. | +| `libfuse` in the shipped image | `fuse3` is now apt-installed in the Dockerfile. The `fuse` Python extra (fusepy, libfuse2 API) and `/dev/fuse` + `SYS_ADMIN` at runtime are still the operator's job — unverified in a container. | The `nas_shares` table and the `/nas/shares` CRUD endpoints store share definitions as **metadata only**. Nothing renders or applies them. -The code itself says so — `openblade/nas/fuse_hook.py`: +Hydration for the **catalog** mount is now real (`openblade/fuse/hydration.py`, see `docs/fuse.md`). The NAS-pool hook below is still the old stub — `openblade/nas/fuse_hook.py`: > Stub for optional FUSE virtual filesystem integration. In v1, this is a no-op > stub that records access attempts and returns appropriate offline/hydrating From 4c75df346dc15410099e38b38cbee168bec0e250 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:09 +0000 Subject: [PATCH 041/205] test: register contract marker and make test-contract target Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- Makefile | 7 ++++++- pyproject.toml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7a15dd5..3c09b18 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash .ONESHELL: -.PHONY: up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware +.PHONY: up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration test-contract seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware # Start all services up: @@ -65,6 +65,11 @@ test-unit: test-integration: python3 -m pytest tests/integration/ -q --tb=short +# Run the backend contract suite against every backend pairing +# (the real+real pairing skips unless OPENBLADE_REAL_HARDWARE_ENABLED=true) +test-contract: + python3 -m pytest tests/contract/ -m contract -q --tb=short + # Seed default library instances (library-1, library-2, library-3) seed-libraries: python3 -c $$'from openblade.catalog.db import get_session, init_db\nfrom openblade.catalog.repository import CatalogRepository\nfrom openblade.config import load_config\n\ndb_url = load_config().db_url\ninit_db(db_url)\nrepo = CatalogRepository(get_session())\nfor name, url in [("library-1", "http://localhost:8010"), ("library-2", "http://localhost:8011"), ("library-3", "http://localhost:8012")]:\n existing = repo.get_library_instance_by_name(name)\n if existing:\n print(f"Skipped {name} (already exists)")\n continue\n repo.create_library_instance(name=name, emulator_url=url, model="Scalar i3")\n print(f"Created {name}")' diff --git a/pyproject.toml b/pyproject.toml index 5cf6af1..96dc560 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,4 +134,5 @@ markers = [ "slow: long-running simulator stress tests", "i3: Quantum i3 emulator and protocol tests", "real_i3: tests requiring a physical Quantum Scalar i3", + "contract: behavioural contract suite run against every backend pairing", ] From 39a27f80c85016f99118bea4f2b5adc6580a1e77 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:09 +0000 Subject: [PATCH 042/205] test: backend contract suite across sim, in-process emulator and real pairings Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/contract/__init__.py | 0 tests/contract/conftest.py | 147 ++++++++++++++++++ .../contract/test_archive_restore_contract.py | 99 ++++++++++++ tests/contract/test_library_contract.py | 76 +++++++++ tests/contract/test_ltfs_contract.py | 75 +++++++++ 5 files changed, 397 insertions(+) create mode 100644 tests/contract/__init__.py create mode 100644 tests/contract/conftest.py create mode 100644 tests/contract/test_archive_restore_contract.py create mode 100644 tests/contract/test_library_contract.py create mode 100644 tests/contract/test_ltfs_contract.py diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/contract/conftest.py b/tests/contract/conftest.py new file mode 100644 index 0000000..c234ddc --- /dev/null +++ b/tests/contract/conftest.py @@ -0,0 +1,147 @@ +"""Backend pairings for the behavioural contract suite. + +Every contract test receives a ``BackendPair`` and talks to it ONLY through the +``LibraryBackend`` / ``LTFSBackend`` Protocols. A new backend must pass this suite +before it may be selected via ``OPENBLADE_BACKEND`` (docs/test-plan.md). +""" + +from __future__ import annotations + +import inspect +import os +from collections.abc import Callable, Generator, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.backends import LibraryBackend, LTFSBackend +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.inventory import InventoryService +from openblade.jobs.scheduler import DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend + +CAPACITY = 50 * 1024 * 1024 +SIM_BARCODES = ["CTR001L8", "CTR002L8", "CTR003L8"] + + +@dataclass +class BackendPair: + name: str + library: LibraryBackend + ltfs: LTFSBackend + num_drives: int + barcodes: list[str] + + def format_all(self) -> None: + for barcode in self.barcodes: + self.ltfs.format( + barcode, + FormatConfirmation( + expected_barcode=barcode, + safety_token=SafetyToken.generate("format", barcode), + ), + ) + + def unload_everything(self) -> None: + """Return every loaded cartridge to its home slot (emulator state is shared).""" + snapshot = InventoryService(self.library).snapshot() + empty_slots = [slot.slot_id for slot in snapshot.slots if not slot.occupied] + for drive in snapshot.drives: + if drive.barcode is not None and empty_slots: + self.library.unload(drive.drive_id, empty_slots.pop(0)) + + +def _sim_pair() -> Iterator[BackendPair]: + library = MockLibraryBackend(num_slots=20, num_drives=3) + for slot_id, barcode in enumerate(SIM_BARCODES, start=1): + library.add_cartridge(slot_id, barcode) + ltfs = MockLTFSBackend(library, capacity_bytes=CAPACITY) + yield BackendPair("sim+sim", library, ltfs, 3, list(SIM_BARCODES)) + + +def _emulator_pair(tmp_path: Path) -> Iterator[BackendPair]: + os.environ.setdefault("OPENBLADE_BACKEND", "mock") + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.hardware.scalar_http.library_backend import ScalarHttpLibraryBackend + from openblade.hardware.scalar_http.session import ScalarHttpSession + + with TestClient(app) as client: + library = ScalarHttpLibraryBackend( + ScalarHttpSession(client, username="admin", password="password") + ) + snapshot = InventoryService(library).snapshot() + barcodes = [slot.barcode.value for slot in snapshot.slots if slot.barcode][:3] + # The simulator LTFS is typed against MockLibraryBackend but only consumes + # the LibraryBackend surface; the cast is the pairing under test. + ltfs = MockLTFSBackend(cast(MockLibraryBackend, library), capacity_bytes=CAPACITY) + pair = BackendPair("emulator+sim-ltfs", library, ltfs, len(snapshot.drives), barcodes) + pair.unload_everything() + try: + yield pair + finally: + pair.unload_everything() + + +def _real_pair() -> Iterator[BackendPair]: + from openblade.bootstrap import get_library, get_ltfs + + library = get_library() + snapshot = InventoryService(library).snapshot() + barcodes = [slot.barcode.value for slot in snapshot.slots if slot.barcode][:1] + yield BackendPair("real+real", library, get_ltfs(), len(snapshot.drives), barcodes) + + +_REAL_ENABLED = os.environ.get("OPENBLADE_REAL_HARDWARE_ENABLED", "false").lower() == "true" + +PAIRINGS = [ + pytest.param("sim+sim", id="sim+sim"), + pytest.param("emulator+sim-ltfs", id="emulator+sim-ltfs"), + pytest.param( + "real+real", + id="real+real", + marks=[ + pytest.mark.real_hardware, + pytest.mark.skipif( + not _REAL_ENABLED, reason="OPENBLADE_REAL_HARDWARE_ENABLED is not true" + ), + ], + ), +] + + +@pytest.fixture(params=PAIRINGS) +def backend_pair(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[BackendPair]: + name = request.param + if name == "sim+sim": + yield from _sim_pair() + elif name == "emulator+sim-ltfs": + yield from _emulator_pair(tmp_path) + else: + yield from _real_pair() + + +@pytest.fixture +def catalog(backend_pair: BackendPair, tmp_path: Path) -> CatalogRepository: + # Depends on backend_pair so the emulator app's own init_db runs first. + init_db(f"sqlite:///{tmp_path / 'contract-catalog.db'}") + return CatalogRepository(get_session()) + + +def call_with_scheduler( + fn: Callable[..., Any], *args: Any, scheduler: DriveScheduler, **kwargs: Any +) -> Any: + """Call a job runner whether or not its signature takes ``scheduler``. + + ``run_archive_job`` / ``run_restore_job`` are gaining a ``scheduler=`` kwarg in a + parallel workstream; this keeps the contract test valid across both shapes. + """ + if "scheduler" in inspect.signature(fn).parameters: + kwargs["scheduler"] = scheduler + return fn(*args, **kwargs) diff --git a/tests/contract/test_archive_restore_contract.py b/tests/contract/test_archive_restore_contract.py new file mode 100644 index 0000000..f324175 --- /dev/null +++ b/tests/contract/test_archive_restore_contract.py @@ -0,0 +1,99 @@ +"""Archive → restore byte equality through the job layer, for every backend pairing.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openblade.catalog.repository import CatalogRepository +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_job +from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.shard import ShardMode +from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive +from openblade.jobs.sharded_restore import ShardedRestoreRequest, run_sharded_restore +from tests.contract.conftest import BackendPair, call_with_scheduler + +pytestmark = pytest.mark.contract + + +def _source_files(root: Path, count: int = 4) -> list[Path]: + root.mkdir() + files = [] + for index in range(count): + path = root / f"file_{index}.bin" + path.write_bytes(bytes((index + value) % 256 for value in range(700 + index))) + files.append(path) + return files + + +def test_sharded_archive_then_restore_is_byte_identical( + backend_pair: BackendPair, catalog: CatalogRepository, tmp_path: Path +) -> None: + backend_pair.format_all() + lanes = backend_pair.barcodes[: min(len(backend_pair.barcodes), backend_pair.num_drives)] + scheduler = DriveScheduler(num_drives=backend_pair.num_drives) + files = _source_files(tmp_path / "src") + request = ShardedArchiveRequest( + source_path=tmp_path / "src", + volume_group_name="contract", + lane_barcodes=lanes, + mode=ShardMode.STRIPE, + ) + job = catalog.create_job("archive", {}) + + result = run_sharded_archive( + request, backend_pair.library, backend_pair.ltfs, catalog, scheduler, job.id + ) + + assert result.errors == [] + for path in files: + dest = tmp_path / f"restored_{path.name}" + restore_job = catalog.create_job("restore", {}) + restored = run_sharded_restore( + ShardedRestoreRequest(catalog_path=str(path), dest_path=dest), + backend_pair.library, + backend_pair.ltfs, + catalog, + scheduler, + restore_job.id, + ) + assert restored.error is None + assert restored.checksum_verified + assert dest.read_bytes() == path.read_bytes() + + +def test_archive_job_then_restore_job_is_byte_identical( + backend_pair: BackendPair, catalog: CatalogRepository, tmp_path: Path +) -> None: + backend_pair.format_all() + scheduler = DriveScheduler(num_drives=backend_pair.num_drives) + files = _source_files(tmp_path / "src", count=2) + job = catalog.create_job("archive", {}) + + result = call_with_scheduler( + run_archive_job, + ArchiveRequest(source_path=tmp_path / "src", volume_group_name="contract"), + backend_pair.library, + backend_pair.ltfs, + catalog, + job_id=job.id, + scheduler=scheduler, + ) + + assert result.errors == [] + for path in files: + dest = tmp_path / f"restored_{path.name}" + restore_job = catalog.create_job("restore", {}) + restored = call_with_scheduler( + run_restore_job, + RestoreRequest(catalog_path=f"/contract/{path.name}", dest_path=dest), + backend_pair.library, + backend_pair.ltfs, + catalog, + job_id=restore_job.id, + scheduler=scheduler, + ) + assert restored.error is None + assert dest.read_bytes() == path.read_bytes() diff --git a/tests/contract/test_library_contract.py b/tests/contract/test_library_contract.py new file mode 100644 index 0000000..da99261 --- /dev/null +++ b/tests/contract/test_library_contract.py @@ -0,0 +1,76 @@ +"""LibraryBackend behavioural contract, run against every backend pairing.""" + +from __future__ import annotations + +import pytest + +from openblade.domain.backends import LibraryBackend, LTFSBackend +from openblade.domain.errors import OpenBladeError +from openblade.jobs.inventory import InventoryService +from tests.contract.conftest import BackendPair + +pytestmark = pytest.mark.contract + + +def _home_slot(library: LibraryBackend, barcode: str) -> int: + slot = library.find_slot_by_barcode(barcode) + assert slot is not None, f"{barcode} not found in any slot" + return slot + + +def test_backends_satisfy_runtime_protocols(backend_pair: BackendPair) -> None: + assert isinstance(backend_pair.library, LibraryBackend) + assert isinstance(backend_pair.ltfs, LTFSBackend) + + +def test_inventory_shape_is_consistent(backend_pair: BackendPair) -> None: + snapshot = InventoryService(backend_pair.library).snapshot() + + slot_ids = [slot.slot_id for slot in snapshot.slots] + drive_ids = [drive.drive_id for drive in snapshot.drives] + assert slot_ids and drive_ids + assert len(set(slot_ids)) == len(slot_ids) + assert len(set(drive_ids)) == len(drive_ids) + for slot in snapshot.slots: + assert slot.occupied == (slot.barcode is not None) + slot_barcodes = {slot.barcode.value for slot in snapshot.slots if slot.barcode} + assert set(backend_pair.barcodes) <= slot_barcodes + + +def test_load_unload_round_trip_keeps_barcode_lookups_consistent( + backend_pair: BackendPair, +) -> None: + library = backend_pair.library + barcode = backend_pair.barcodes[0] + slot = _home_slot(library, barcode) + drive_id = InventoryService(library).snapshot().drives[0].drive_id + + assert library.load(slot, drive_id).success + assert library.find_drive_by_barcode(barcode) == drive_id + assert library.find_slot_by_barcode(barcode) is None + loaded = library.get_drive(drive_id).barcode + assert loaded is not None and loaded.value == barcode + + assert library.unload(drive_id, slot).success + assert library.find_drive_by_barcode(barcode) is None + assert library.find_slot_by_barcode(barcode) == slot + assert library.get_drive(drive_id).barcode is None + + +def test_double_load_into_occupied_drive_is_an_error(backend_pair: BackendPair) -> None: + library = backend_pair.library + first, second = backend_pair.barcodes[0], backend_pair.barcodes[1] + first_slot, second_slot = _home_slot(library, first), _home_slot(library, second) + drive_id = InventoryService(library).snapshot().drives[0].drive_id + assert library.load(first_slot, drive_id).success + + try: + result = library.load(second_slot, drive_id) + except OpenBladeError: + pass # a typed domain error is an acceptable refusal + else: + assert not result.success, "loading into an occupied drive must fail" + + assert library.find_drive_by_barcode(first) == drive_id + assert library.find_slot_by_barcode(second) == second_slot + assert library.unload(drive_id, first_slot).success diff --git a/tests/contract/test_ltfs_contract.py b/tests/contract/test_ltfs_contract.py new file mode 100644 index 0000000..6a23d3d --- /dev/null +++ b/tests/contract/test_ltfs_contract.py @@ -0,0 +1,75 @@ +"""LTFSBackend behavioural contract (mount/write/read/checksum + unload safety gate).""" + +from __future__ import annotations + +import hashlib +from pathlib import Path, PurePosixPath + +import pytest + +from openblade.domain.errors import TapeMountedError +from openblade.domain.models import MountMode +from openblade.jobs.inventory import InventoryService +from tests.contract.conftest import BackendPair + +pytestmark = pytest.mark.contract + + +def _load_first(pair: BackendPair) -> tuple[str, int, int]: + barcode = pair.barcodes[0] + slot = pair.library.find_slot_by_barcode(barcode) + assert slot is not None + drive_id = InventoryService(pair.library).snapshot().drives[0].drive_id + assert pair.library.load(slot, drive_id).success + return barcode, slot, drive_id + + +def test_mount_rw_write_then_ro_read_round_trips_checksum( + backend_pair: BackendPair, tmp_path: Path +) -> None: + backend_pair.format_all() + barcode, slot, drive_id = _load_first(backend_pair) + payload = bytes(range(256)) * 64 + source = tmp_path / "in.bin" + source.write_bytes(payload) + dest = PurePosixPath("/contract/in.bin") + + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_WRITE) + backend_pair.ltfs.write_file(handle, source, dest) + stat = backend_pair.ltfs.stat(handle, dest) + assert stat.size_bytes == len(payload) + assert stat.checksum_sha256 == hashlib.sha256(payload).hexdigest() + assert backend_pair.ltfs.unmount(handle).success + + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_ONLY) + restored = tmp_path / "out.bin" + assert backend_pair.ltfs.read_file(handle, dest, restored).success + assert backend_pair.ltfs.unmount(handle).success + + assert hashlib.sha256(restored.read_bytes()).digest() == hashlib.sha256(payload).digest() + assert backend_pair.library.unload(drive_id, slot).success + + +def test_unload_while_mounted_is_refused( + backend_pair: BackendPair, request: pytest.FixtureRequest +) -> None: + if backend_pair.name == "emulator+sim-ltfs": + request.applymarker( + pytest.mark.xfail( + strict=True, + reason=( + "openblade/hardware/scalar_http/library_backend.py:260 unload() has no " + "LTFS mount-state gate; the AML emulator moves a mounted cartridge" + ), + ) + ) + backend_pair.format_all() + barcode, slot, drive_id = _load_first(backend_pair) + handle = backend_pair.ltfs.mount(barcode, MountMode.READ_WRITE) + + with pytest.raises(TapeMountedError): + backend_pair.library.unload(drive_id, slot) + + assert backend_pair.library.find_drive_by_barcode(barcode) == drive_id + assert backend_pair.ltfs.unmount(handle).success + assert backend_pair.library.unload(drive_id, slot).success From ecf205200c7737b700e74306a9a252aedea9a961 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:09 +0000 Subject: [PATCH 043/205] docs: document the backend contract suite and new-backend rule Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- README.md | 3 +++ docs/test-plan.md | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/README.md b/README.md index 5755345..b75a6dd 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ OpenBlade is a simulator-first DIY tape archive controller inspired by iBlade-st 5. **Continuous verification**: enforce compatibility and regression evidence in CI/CD before changes land on `master`. ## Layered CI/CD (targeted) + + +Backend contract suite: `make test-contract` runs `tests/contract/` against every backend pairing (simulator, in-process AML emulator, and real hardware when explicitly enabled). A new backend must pass it before it may be selected via `OPENBLADE_BACKEND`; see docs/test-plan.md "Backend contract suite". OpenBlade CI/CD is intentionally split by layer so each change runs only relevant checks: | Layer | Primary workflow/jobs | Trigger scope | diff --git a/docs/test-plan.md b/docs/test-plan.md index 549dc48..aaa80a9 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -20,3 +20,24 @@ Verify injected mount, write, and capacity faults surface as typed errors. ## Safety regressions Prove the default config blocks real hardware, formatting requires confirmation, and unload-while-mounted is rejected. + +## Backend contract suite + +`tests/contract/` (`make test-contract`, marker `contract`) runs one behavioural +contract against every library/LTFS pairing, using only the `LibraryBackend` / +`LTFSBackend` Protocols: inventory shape, load/unload round-trip, double-load +refusal, barcode lookup consistency, RW/RO mount + write + read + checksum, the +unload-while-mounted safety gate, and archive → restore byte equality (sharded and +single-job) through `InventoryService`. + +| Pairing | Library | LTFS | Runs | +|---|---|---|---| +| `sim+sim` | `MockLibraryBackend` | `MockLTFSBackend` | always | +| `emulator+sim-ltfs` | `ScalarHttpLibraryBackend` over in-process ASGI (`openblade.api.main:app`) | `MockLTFSBackend` | always | +| `real+real` | `get_library()` | `get_ltfs()` | only with `OPENBLADE_REAL_HARDWARE_ENABLED=true` (marker `real_hardware`) | + +Known contract violations are `xfail(strict=True)` with the defect's file:line, so a +fix flips them to XPASS and fails the run until the marker is removed. + +**Rule: a new backend must pass this suite before it may be selected via +`OPENBLADE_BACKEND`.** Add it as a named pairing in `tests/contract/conftest.py`. From 1dc34c0d54b55ca936b76fece1a115c771480aa7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:25 +0000 Subject: [PATCH 044/205] docs: protection axis table and commit-marker recovery rules Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/disaster-recovery.md | 15 +++++++++++++++ docs/sharding.md | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md index 0af4a9f..318dbd0 100644 --- a/docs/disaster-recovery.md +++ b/docs/disaster-recovery.md @@ -55,3 +55,18 @@ against a freshly-seeded DB — this is what CI runs, with no production data. - A restore drill against a **real production backup** artifact. CI only exercises the mechanism on seeded data; never restore production data in a test path. - Scheduled backup rotation + off-host replication. + +## Commit markers and uncommitted generations + +Each tape write generation ends with `/.openblade/commit-.json` +(schema, generation id, sha256 of `manifest.json` and `catalog-shard.json`, +written_at). `finalize_tape_generation()` is the only writer and enforces the order: +data -> `manifest.json` (+`.sha256`) -> `catalog-shard.json` (+`.sha256`) -> +`tape.json` (`current_generation`/`previous_generation` chain) -> commit marker. + +On rebuild, a v2 manifest whose marker is **missing** or whose hashes **do not +match** is an uncommitted generation (an interrupted write): its files are not +restored as archived and the plan warns `uncommitted generation : `. +Pre-v2 manifests carry no generation id and are treated as committed. Each +manifest lists `sibling_tapes`; a sibling not included in the rebuild request is +reported as `missing sibling tape ` — restore needs every sibling. diff --git a/docs/sharding.md b/docs/sharding.md index bbfb1f3..cf8f745 100644 --- a/docs/sharding.md +++ b/docs/sharding.md @@ -129,3 +129,23 @@ POST /archive/sharded - Restore always uses `READ_ONLY` mounts, even on shard tapes - DriveScheduler never grants the same drive to two jobs - All drives are released (even on error) via try/finally + +## Protection is a separate axis + +Sharding/striping spreads data across tapes for throughput. It does **not** make +data durable: losing any one tape of a STRIPE set loses data. Durability comes only +from redundant copies. `openblade/domain/protection.py` models the axes +independently (`Placement`, `Striping`, `Protection`, `FailureDomain`, `RestoreQuorum`) +and `ProtectionPolicy.durability_class` reports the result: + +| Striping | Protection | `is_protected` | `durability_class` | +|---|---|---|---| +| NONE | NONE / REPLICATION(1) | false | `unprotected` | +| STRIPE / BLOCK_STRIPE | NONE / REPLICATION(1) | false | `performance-only` | +| any | REPLICATION(copies >= 2) | true | `protected` | +| any | ERASURE_CODE(k, m) | true (modelled only) | planner raises `UnsupportedProtectionError` | + +Legacy `NasPool.replication_factor` maps via `protection_from_pool()`. The policy is +recorded in `manifest.json` (`protection`, `placement`) and per dataset in +`catalog-shard.json`, so a rebuild can recover it. Pinned by +`test_striping_never_implies_protection`. From f790ba5c914ec132124345a17645d946b6d48f34 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:35 +0000 Subject: [PATCH 045/205] feat(nas): rebuild skips uncommitted generations and reports missing siblings Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/catalog_rebuild.py | 37 +++++++++++++++++- tests/unit/test_catalog_rebuild.py | 60 ++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index 9795f89..85741c0 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -12,7 +12,7 @@ CatalogShard, CatalogShardWriter, ) -from openblade.nas.ltfs_manifest import TapeMetadataWriter +from openblade.nas.ltfs_manifest import ManifestJson, TapeMetadataWriter from openblade.nas.manifest_validator import ManifestValidator from openblade.nas.path_mapping import PathMappingService from openblade.nas.types import ( @@ -116,6 +116,18 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: warnings.append(f"{barcode}: catalog-shard.json missing") continue + uncommitted = self._uncommitted_reason(barcode, manifest) + if uncommitted is not None: + # Never restore an uncommitted generation's files as archived. + warnings.append( + f"{barcode}: uncommitted generation {manifest.generation_id}: {uncommitted}" + ) + continue + requested = set(request.barcodes) + for sibling in manifest.sibling_tapes: + if sibling not in requested: + warnings.append(f"{barcode}: missing sibling tape {sibling}") + barcodes_to_scan.append(barcode) estimated_files += len(shard.files) estimated_datasets += len(shard.datasets) @@ -159,6 +171,29 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: safe_to_enqueue=safe_to_enqueue, ) + def _uncommitted_reason(self, barcode: str, manifest: ManifestJson) -> str | None: + """Why this tape's generation is uncommitted, or None when its commit marker checks out. + + Pre-v2 manifests carry no generation_id and predate commit markers; they are + accepted as committed so existing tapes stay rebuildable. + """ + if not manifest.generation_id: + return None + marker = self.metadata_writer.read_commit_marker(barcode, manifest.generation_id) + if marker is None: + return "commit marker missing" + manifest_sha = self.metadata_writer.compute_json_checksum( + manifest.model_dump(by_alias=True) + ) + shard_sha = self.metadata_writer._read_text( + barcode, self.shard_writer.SHARD_CHECKSUM_METADATA_PATH + ) + if marker.manifest_sha256 != manifest_sha: + return "commit marker manifest hash mismatch" + if shard_sha is None or marker.catalog_shard_sha256 != shard_sha.strip(): + return "commit marker catalog-shard hash mismatch" + return None + def execute_rebuild_run(self, run_id: str) -> CatalogRebuildRunRecord: """ Execute a planned rebuild run and persist progress in the catalog database. diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index e2d0f87..9b1c262 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -406,3 +406,63 @@ def test_api_get_run_returns_record(rebuild_env: dict[str, object]) -> None: assert response.status_code == 200 assert response.json()["id"] == run_id assert response.json()["status"] == RebuildRunStatus.PLANNED.value + + +def _stamp_generation( + rebuild_env: dict[str, object], barcode: str, *, marker: str, siblings: list[str] | None = None +) -> None: + from openblade.nas.ltfs_manifest import CommitMarker + + planner = rebuild_env["planner"] + writer = planner.metadata_writer + manifest = writer.read_manifest(barcode).model_copy( + update={"generation_id": "gen-7", "sibling_tapes": siblings or []} + ) + manifest_sha = writer.write_manifest(barcode, manifest) + writer.write_manifest_checksum(barcode, manifest_sha) + shard_sha = writer._read_text(barcode, planner.shard_writer.SHARD_CHECKSUM_METADATA_PATH) + if marker == "missing": + return + writer.write_commit_marker( + barcode, + CommitMarker( + generation_id="gen-7", + manifest_sha256=manifest_sha if marker == "ok" else "0" * 64, + catalog_shard_sha256=shard_sha.strip(), + written_at="2026-10-09T00:00:00Z", + ), + ) + + +@pytest.mark.parametrize( + ("marker", "reason"), + [("missing", "commit marker missing"), ("bad", "commit marker manifest hash mismatch")], +) +def test_plan_rebuild_uncommitted_generation_not_scanned( + rebuild_env: dict[str, object], marker: str, reason: str +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker=marker) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.barcodes_to_scan == [] + assert f"{barcode}: uncommitted generation gen-7: {reason}" in result.warnings + + +def test_plan_rebuild_committed_generation_reports_missing_sibling( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker="ok", siblings=["ZZ9999L8"]) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.barcodes_to_scan == [barcode] + assert f"{barcode}: missing sibling tape ZZ9999L8" in result.warnings From 7feed229537d99323a00bcfc2cb7925782d901d2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:49 +0000 Subject: [PATCH 046/205] feat(hardware): resolve_tape_by_id maps a drive serial to its /dev/tape/by-id link (read-only) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/hardware/correlation.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/openblade/hardware/correlation.py b/openblade/hardware/correlation.py index f5da8c3..f76b560 100644 --- a/openblade/hardware/correlation.py +++ b/openblade/hardware/correlation.py @@ -61,6 +61,7 @@ from __future__ import annotations import logging +import os import subprocess from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import dataclass @@ -445,3 +446,33 @@ def _refuse_on_mismatch( def _format_serials(live: Mapping[str, str]) -> str: return ", ".join(f"{device}={serial or ''}" for device, serial in sorted(live.items())) + + +TAPE_BY_ID_DIR = "/dev/tape/by-id" +"""udev's stable per-serial tape symlink directory.""" + + +def resolve_tape_by_id(serial: str, by_id_dir: str = TAPE_BY_ID_DIR) -> str | None: + """Return the ``/dev/tape/by-id`` entry whose name carries ``serial``, or ``None``. + + Read-only: lists one directory and never opens a device. Prefers the + non-rewinding ``-nst`` link when udev publishes both. The match is on the + normalized serial as a whole ``-``/``_``-delimited token, so serial ``123`` + cannot match ``...-1234-nst``. + """ + wanted = _normalize_serial(serial) + if not wanted: + return None + try: + names = sorted(os.listdir(by_id_dir)) + except OSError: + return None + matches = [ + name + for name in names + if wanted in {_normalize_serial(t) for t in name.replace("_", "-").split("-")} + ] + if not matches: + return None + preferred = [m for m in matches if m.endswith("-nst")] or matches + return os.path.join(by_id_dir, preferred[0]) From 5492fcc24a5a00846050c95dee510d7a64abfe02 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:49 +0000 Subject: [PATCH 047/205] feat(tools): sacrificial-barcode allowlist gate and inventory/drive snapshot+diff for hardware lanes Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/hardware/test_allowlist.py | 58 +++++++++++++ tests/hardware/test_snapshot.py | 60 +++++++++++++ tools/hardware/__init__.py | 1 + tools/hardware/allowlist.py | 85 ++++++++++++++++++ tools/hardware/snapshot.py | 145 +++++++++++++++++++++++++++++++ 5 files changed, 349 insertions(+) create mode 100644 tests/hardware/test_allowlist.py create mode 100644 tests/hardware/test_snapshot.py create mode 100644 tools/hardware/__init__.py create mode 100644 tools/hardware/allowlist.py create mode 100644 tools/hardware/snapshot.py diff --git a/tests/hardware/test_allowlist.py b/tests/hardware/test_allowlist.py new file mode 100644 index 0000000..5a659e8 --- /dev/null +++ b/tests/hardware/test_allowlist.py @@ -0,0 +1,58 @@ +"""Unit tests for tools/hardware/allowlist.py — run without the rig.""" + +from __future__ import annotations + +import pytest + +from tools.hardware.allowlist import ( + EXIT_OK, + EXIT_VIOLATION, + AllowlistError, + check_subset, + main, + parse_barcodes, +) + + +def test_parse_barcodes_splits_commas_and_whitespace() -> None: + assert parse_barcodes(" a00001l8, A00002L8\nA00003L8 a00001L8 ") == [ + "A00001L8", + "A00002L8", + "A00003L8", + ] + + +def test_check_subset_accepts_subset() -> None: + assert check_subset(["A1"], ["A1", "A2"]) == ["A1"] + + +def test_check_subset_outside_allowlist_names_offenders() -> None: + with pytest.raises(AllowlistError, match="PROD9, PROD8"): + check_subset(["A1", "PROD9", "PROD8"], ["A1", "A2"]) + + +def test_check_subset_empty_allowlist_fails_closed() -> None: + with pytest.raises(AllowlistError, match="fail closed"): + check_subset(["A1"], []) + + +def test_check_subset_empty_request_refused() -> None: + with pytest.raises(AllowlistError, match="No barcodes"): + check_subset([], ["A1"]) + + +def test_main_offender_exits_2_with_message(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1,A2") + assert main(["--barcodes", "A1 PROD9"]) == EXIT_VIOLATION + assert "PROD9" in capsys.readouterr().err + + +def test_main_falls_back_to_env_barcodes(monkeypatch) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1 A2") + monkeypatch.setenv("BARCODES", "a2") + assert main([]) == EXIT_OK + + +def test_main_unset_allowlist_exits_2(monkeypatch) -> None: + monkeypatch.delenv("SACRIFICIAL_BARCODES", raising=False) + assert main(["--barcodes", "A1"]) == EXIT_VIOLATION diff --git a/tests/hardware/test_snapshot.py b/tests/hardware/test_snapshot.py new file mode 100644 index 0000000..5a38096 --- /dev/null +++ b/tests/hardware/test_snapshot.py @@ -0,0 +1,60 @@ +"""Unit tests for tools/hardware/snapshot.py — simulator backend, no rig.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.config import OpenBladeConfig +from tools.hardware.snapshot import ( + EXIT_BAD_INPUT, + EXIT_CHANGED, + capture_snapshot, + diff_snapshots, + main, +) + + +def _snap(slots, drives, changer="idle"): + return {"library_id": "L", "slots": slots, "drives": drives, "changer_state": changer} + + +def test_capture_against_simulator_is_json_serializable(tmp_path: Path) -> None: + library = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 's.db'}")).library + snap = capture_snapshot(library) + assert snap["slots"] and "slot_id" in snap["slots"][0] + assert json.loads(json.dumps(snap)) == snap + + +def test_diff_identical_snapshots_reports_unchanged() -> None: + s = _snap([{"slot_id": 1, "barcode": "A"}], [{"drive_id": 0, "barcode": None}]) + assert diff_snapshots(s, s)["changed"] is False + + +def test_diff_reports_moved_cartridge_per_element() -> None: + before = _snap([{"slot_id": 1, "barcode": "A"}], [{"drive_id": 0, "barcode": None}]) + after = _snap([{"slot_id": 1, "barcode": None}], [{"drive_id": 0, "barcode": "A"}]) + result = diff_snapshots(before, after) + assert result["changed"] is True + assert result["slots"][0]["slot_id"] == "1" and result["slots"][0]["after"]["barcode"] is None + assert result["drives"][0]["after"]["barcode"] == "A" + + +def test_diff_cli_writes_file_and_rejects_bad_input(tmp_path: Path) -> None: + good = tmp_path / "a.json" + good.write_text(json.dumps(_snap([], []))) + out = tmp_path / "d.json" + assert main(["diff", str(good), str(good), "--out", str(out)]) == 0 + assert json.loads(out.read_text())["changed"] is False + bad = tmp_path / "bad.json" + bad.write_text("not json") + assert main(["diff", str(good), str(bad), "--out", str(out)]) == EXIT_BAD_INPUT + + +def test_diff_cli_changed_snapshots_exit_1_after_writing(tmp_path: Path) -> None: + a, b, out = tmp_path / "a.json", tmp_path / "b.json", tmp_path / "d.json" + a.write_text(json.dumps(_snap([{"slot_id": 1, "barcode": "A"}], []))) + b.write_text(json.dumps(_snap([{"slot_id": 1, "barcode": None}], []))) + assert main(["diff", str(a), str(b), "--out", str(out)]) == EXIT_CHANGED + assert json.loads(out.read_text())["changed"] is True diff --git a/tools/hardware/__init__.py b/tools/hardware/__init__.py new file mode 100644 index 0000000..3dcb5a0 --- /dev/null +++ b/tools/hardware/__init__.py @@ -0,0 +1 @@ +"""Hardware-lane operator tools (allowlist gate, inventory snapshots).""" diff --git a/tools/hardware/allowlist.py b/tools/hardware/allowlist.py new file mode 100644 index 0000000..1d0d151 --- /dev/null +++ b/tools/hardware/allowlist.py @@ -0,0 +1,85 @@ +"""Sacrificial-barcode allowlist gate for the destructive hardware lane. + +Run BEFORE any hardware is touched:: + + python -m tools.hardware.allowlist --barcodes "TST001L8 TST002L8" + +The allowlist comes from env ``SACRIFICIAL_BARCODES``; the request comes from +``--barcodes`` or, when the flag is absent, env ``BARCODES``. Both accept +comma- and/or whitespace-separated lists. Exit 0 when the request is a +non-empty subset of a non-empty allowlist; exit 2 otherwise (fail closed), +naming every offending barcode. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from collections.abc import Sequence + +EXIT_OK = 0 +EXIT_VIOLATION = 2 + +_SEPARATORS = re.compile(r"[\s,]+") + + +class AllowlistError(Exception): + """The requested barcodes are not safe to use destructively.""" + + +def parse_barcodes(raw: str | None) -> list[str]: + """Split a comma/whitespace list into upper-cased, de-duplicated barcodes.""" + seen: dict[str, None] = {} + for token in _SEPARATORS.split(raw or ""): + if token: + seen.setdefault(token.strip().upper(), None) + return list(seen) + + +def check_subset(requested: Sequence[str], allowlist: Sequence[str]) -> list[str]: + """Return ``requested`` if it is a non-empty subset of a non-empty allowlist. + + Raises :class:`AllowlistError` naming the offenders otherwise. + """ + if not allowlist: + raise AllowlistError( + "SACRIFICIAL_BARCODES is empty: no tape is approved for destructive use (fail closed)." + ) + if not requested: + raise AllowlistError("No barcodes requested (pass --barcodes or set BARCODES).") + allowed = set(allowlist) + offenders = [b for b in requested if b not in allowed] + if offenders: + raise AllowlistError( + "Barcodes outside the sacrificial allowlist: " + + ", ".join(offenders) + + ". Refusing before touching hardware." + ) + return list(requested) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m tools.hardware.allowlist") + parser.add_argument( + "--barcodes", + default=None, + help="Comma/space-separated barcodes to use (default: env BARCODES).", + ) + args = parser.parse_args(argv) + raw_request = args.barcodes if args.barcodes is not None else os.environ.get("BARCODES") + try: + approved = check_subset( + parse_barcodes(raw_request), + parse_barcodes(os.environ.get("SACRIFICIAL_BARCODES")), + ) + except AllowlistError as exc: + print(f"allowlist: REFUSED: {exc}", file=sys.stderr) + return EXIT_VIOLATION + print("allowlist: OK: " + " ".join(approved)) + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hardware/snapshot.py b/tools/hardware/snapshot.py new file mode 100644 index 0000000..874988c --- /dev/null +++ b/tools/hardware/snapshot.py @@ -0,0 +1,145 @@ +"""Before/after library-state snapshots for the destructive hardware lane. + + python -m tools.hardware.snapshot capture --out before.json + python -m tools.hardware.snapshot diff before.json after.json --out diff.json + +``capture`` reads inventory + drive state through ``InventoryService`` on the +library the configured backend builds (``OPENBLADE_BACKEND``; the simulator by +default, real only when ``OPENBLADE_REAL_HARDWARE_ENABLED=true`` too). It is +read-only. ``diff`` compares two captures and writes the diff file, then exits +0 when identical, 1 when they differ, 2 on unreadable input. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import enum +import json +import sys +from collections.abc import Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from openblade.domain.backends import LibraryBackend + +EXIT_OK = 0 +EXIT_CHANGED = 1 +EXIT_BAD_INPUT = 2 + + +class SnapshotError(Exception): + """A snapshot file is missing or malformed.""" + + +def _jsonable(value: Any) -> Any: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {f.name: _jsonable(getattr(value, f.name)) for f in dataclasses.fields(value)} + if isinstance(value, enum.Enum): + return _jsonable(value.value) + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + if isinstance(value, list | tuple): + return [_jsonable(v) for v in value] + if value is None or isinstance(value, bool | int | float | str): + return value + return str(value) + + +def capture_snapshot(library: LibraryBackend) -> dict[str, Any]: + """Return a JSON-able snapshot of ``library``'s inventory and drive state.""" + from openblade.jobs.inventory import InventoryService + + inventory = InventoryService(library).snapshot() + return { + "captured_at": datetime.now(UTC).isoformat(), + "library_id": inventory.library_id, + "slots": [_jsonable(s) for s in inventory.slots], + "drives": [_jsonable(d) for d in inventory.drives], + "changer_state": _jsonable(inventory.changer_state), + } + + +def _index(items: list[dict[str, Any]], key: str) -> dict[str, dict[str, Any]]: + return {str(item.get(key, i)): item for i, item in enumerate(items)} + + +def _diff_section( + before: list[dict[str, Any]], after: list[dict[str, Any]], key: str +) -> list[dict[str, Any]]: + b, a = _index(before, key), _index(after, key) + changes: list[dict[str, Any]] = [] + for ident in sorted(b.keys() | a.keys()): + if b.get(ident) != a.get(ident): + changes.append({key: ident, "before": b.get(ident), "after": a.get(ident)}) + return changes + + +def diff_snapshots(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + """Compare two captures element-by-element (slots by slot_id, drives by drive_id).""" + slots = _diff_section(before.get("slots", []), after.get("slots", []), "slot_id") + drives = _diff_section(before.get("drives", []), after.get("drives", []), "drive_id") + changer = ( + [] + if before.get("changer_state") == after.get("changer_state") + else [{"before": before.get("changer_state"), "after": after.get("changer_state")}] + ) + return { + "library_id": {"before": before.get("library_id"), "after": after.get("library_id")}, + "slots": slots, + "drives": drives, + "changer_state": changer, + "changed": bool(slots or drives or changer), + } + + +def _load(path: Path) -> dict[str, Any]: + try: + data = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise SnapshotError(f"cannot read snapshot {path}: {exc.__class__.__name__}") from exc + if not isinstance(data, dict): + raise SnapshotError(f"snapshot {path} is not a JSON object") + return data + + +def _build_library() -> LibraryBackend: + from openblade.bootstrap import create_context + from openblade.config import load_config + + return create_context(load_config()).library + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m tools.hardware.snapshot") + sub = parser.add_subparsers(dest="command", required=True) + cap = sub.add_parser("capture", help="Capture inventory + drive state to JSON.") + cap.add_argument("--out", required=True, type=Path) + dif = sub.add_parser("diff", help="Diff two captures.") + dif.add_argument("before", type=Path) + dif.add_argument("after", type=Path) + dif.add_argument("--out", required=True, type=Path) + args = parser.parse_args(argv) + + if args.command == "capture": + snap = capture_snapshot(_build_library()) + args.out.write_text(json.dumps(snap, indent=2, sort_keys=True)) + print(f"snapshot: captured {len(snap['slots'])} slots, {len(snap['drives'])} drives") + return EXIT_OK + try: + result = diff_snapshots(_load(args.before), _load(args.after)) + except SnapshotError as exc: + print(f"snapshot: {exc}", file=sys.stderr) + return EXIT_BAD_INPUT + args.out.write_text(json.dumps(result, indent=2, sort_keys=True)) + print( + f"snapshot diff: {len(result['slots'])} slot change(s), " + f"{len(result['drives'])} drive change(s), " + f"changer {'changed' if result['changer_state'] else 'unchanged'}" + ) + return EXIT_CHANGED if result["changed"] else EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) From 7f8389f5c3ac5f02a57107ebe9a3239a143d6669 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:49 +0000 Subject: [PATCH 048/205] test(hardware): read-only rig suite, drive-identity bridge, and AST guard forbidding motion in readonly/ Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/hardware/readonly/__init__.py | 1 + tests/hardware/readonly/conftest.py | 117 +++++++++++++++++ .../readonly/test_drive_identity_bridge.py | 57 ++++++++ .../readonly/test_readonly_ast_safety.py | 109 +++++++++++++++ tests/hardware/readonly/test_readonly_rig.py | 124 ++++++++++++++++++ 5 files changed, 408 insertions(+) create mode 100644 tests/hardware/readonly/__init__.py create mode 100644 tests/hardware/readonly/conftest.py create mode 100644 tests/hardware/readonly/test_drive_identity_bridge.py create mode 100644 tests/hardware/readonly/test_readonly_ast_safety.py create mode 100644 tests/hardware/readonly/test_readonly_rig.py diff --git a/tests/hardware/readonly/__init__.py b/tests/hardware/readonly/__init__.py new file mode 100644 index 0000000..6af84bf --- /dev/null +++ b/tests/hardware/readonly/__init__.py @@ -0,0 +1 @@ +"""Read-only real-hardware tests (see test_readonly_ast_safety.py).""" diff --git a/tests/hardware/readonly/conftest.py b/tests/hardware/readonly/conftest.py new file mode 100644 index 0000000..ec06463 --- /dev/null +++ b/tests/hardware/readonly/conftest.py @@ -0,0 +1,117 @@ +"""Fixtures for the read-only real-hardware lane. + +Gating mirrors tests/hardware/conftest.py: every test here requests +``real_hardware_guard`` (skips unless OPENBLADE_BACKEND=real and +OPENBLADE_REAL_HARDWARE_ENABLED=true). The appliance's AML web services are +reached at OPENBLADE_APPLIANCE_URL with OPENBLADE_APPLIANCE_USER / +OPENBLADE_APPLIANCE_PASSWORD; tests needing them skip with that reason when unset. +""" + +from __future__ import annotations + +import json +import os +import ssl +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any + +import pytest + +from openblade.config import load_config +from openblade.domain.policies import RealHardwareGuard +from openblade.hardware.safety import require_real_hardware + +CORPUS_DIR = Path(__file__).resolve().parents[3] / "compatibility" + + +def read_corpus() -> list[dict[str, Any]]: + return [json.loads(p.read_text()) for p in sorted(CORPUS_DIR.rglob("*.json"))] + + +@dataclass +class ApplianceResponse: + status: int + body: Any + + +@dataclass +class ApplianceClient: + """Minimal cookie-session HTTP client for the real appliance's AML API.""" + + base_url: str + user: str + password: str + insecure: bool = False + _opener: urllib.request.OpenerDirector = field(init=False) + + def __post_init__(self) -> None: + context = ssl.create_default_context() + if self.insecure: + # Opt-in via OPENBLADE_APPLIANCE_INSECURE=true for a rig whose + # appliance only has its factory self-signed certificate. + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + self._opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(CookieJar()), + urllib.request.HTTPSHandler(context=context), + ) + + def request(self, method: str, path: str, json_body: Any = None) -> ApplianceResponse: + data = None if json_body is None else json.dumps(json_body).encode() + req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config, scheme checked in fixture + self.base_url.rstrip("/") + path, + data=data, + method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with self._opener.open(req, timeout=30) as resp: + status, raw = resp.status, resp.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + try: + body: Any = json.loads(raw) if raw else None + except ValueError: + body = None + return ApplianceResponse(status=status, body=body) + + def login(self) -> ApplianceResponse: + return self.request( + "POST", "/aml/users/login", {"name": self.user, "password": self.password} + ) + + def logout(self) -> ApplianceResponse: + return self.request("POST", "/aml/auth/logout") + + +@pytest.fixture +def appliance(real_hardware_guard) -> ApplianceClient: + url = os.environ.get("OPENBLADE_APPLIANCE_URL", "") + user = os.environ.get("OPENBLADE_APPLIANCE_USER", "") + password = os.environ.get("OPENBLADE_APPLIANCE_PASSWORD", "") + if not (url and user and password): + pytest.skip( + "Appliance AML API not configured: set OPENBLADE_APPLIANCE_URL, " + "OPENBLADE_APPLIANCE_USER and OPENBLADE_APPLIANCE_PASSWORD" + ) + if not url.startswith(("https://", "http://")): + pytest.fail(f"OPENBLADE_APPLIANCE_URL must be http(s)://, got scheme of {url[:8]!r}") + insecure = os.environ.get("OPENBLADE_APPLIANCE_INSECURE") == "true" + return ApplianceClient(base_url=url, user=user, password=password, insecure=insecure) + + +@pytest.fixture +def authed_appliance(appliance: ApplianceClient): + resp = appliance.login() + assert resp.status == 200, f"appliance login failed: HTTP {resp.status}" + yield appliance + appliance.logout() + + +@pytest.fixture +def hardware_guard(real_hardware_guard) -> RealHardwareGuard: + return require_real_hardware(load_config()) diff --git a/tests/hardware/readonly/test_drive_identity_bridge.py b/tests/hardware/readonly/test_drive_identity_bridge.py new file mode 100644 index 0000000..f37cba0 --- /dev/null +++ b/tests/hardware/readonly/test_drive_identity_bridge.py @@ -0,0 +1,57 @@ +"""AML coordinate -> serial -> sg_inq -> /dev/tape/by-id -> LTFS mount DRY. + +Resolves the device LTFS *would* mount for each drive and stops: asserts the +by-id path exists and its sg_inq serial matches the library's, then performs +no mount. +""" + +from __future__ import annotations + +import os +from typing import Any + +import pytest + +from openblade.domain.scalar_coordinate import ScalarCoordinate +from openblade.hardware.correlation import resolve_tape_by_id +from openblade.hardware.runner import SafeRunner +from openblade.hardware.sg import sg_inq + +from .conftest import ApplianceClient + +pytestmark = [pytest.mark.real_hardware, pytest.mark.usefixtures("real_hardware_guard")] + +_DRIVE_ELEMENT_TYPE = 4 # SMC Data Transfer Element + + +def _drive_elements(node: Any) -> list[dict[str, Any]]: + if isinstance(node, dict): + coord = node.get("coordinate") + if isinstance(coord, dict) and node.get("serialNumber"): + return [node] + return [e for v in node.values() for e in _drive_elements(v)] + if isinstance(node, list): + return [e for v in node for e in _drive_elements(v)] + return [] + + +def test_drive_coordinate_resolves_to_matching_by_id_device( + authed_appliance: ApplianceClient, hardware_guard +) -> None: + resp = authed_appliance.request("GET", "/aml/physicalLibrary/elements") + assert resp.status == 200 + drives = [ + e + for e in _drive_elements(resp.body) + if ScalarCoordinate.from_dict(e["coordinate"]).element_type == _DRIVE_ELEMENT_TYPE + ] + if not drives: + pytest.skip("Appliance reported no drive elements with coordinate + serialNumber") + for element in drives: + serial = str(element["serialNumber"]) + by_id = resolve_tape_by_id(serial) + assert by_id is not None, f"no /dev/tape/by-id entry for drive serial {serial}" + assert os.path.exists(by_id) + inquiry = sg_inq(os.path.realpath(by_id), SafeRunner(dry_run=False), hardware_guard) + assert inquiry.serial.strip().casefold() == serial.strip().casefold() + # DRY: the LTFS mount target is resolved; no mount is performed. diff --git a/tests/hardware/readonly/test_readonly_ast_safety.py b/tests/hardware/readonly/test_readonly_ast_safety.py new file mode 100644 index 0000000..f197871 --- /dev/null +++ b/tests/hardware/readonly/test_readonly_ast_safety.py @@ -0,0 +1,109 @@ +"""Plain-CI guard: tests/hardware/readonly/ must be read-only by construction. + +Fails if any module there imports or calls a load/unload/move/format/mount +API. Resolution helpers ("resolve_*", "*dry_run*", "plan_*") stay allowed. +Not rig-gated. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +READONLY_DIR = Path(__file__).resolve().parent + +FORBIDDEN_EXACT = frozenset( + { + "load", "unload", "move", "move_medium", "exchange", "transfer", "eject", + "format", "mkltfs", "mount", "unmount", "umount", "ltfs", "erase", + "run_inventory_job", "inventory", + } +) # fmt: skip +FORBIDDEN_PREFIXES = ("load_", "unload_", "move_", "format_", "mount_", "erase_", "mkltfs") +FORBIDDEN_SUBSTRINGS = ("_load", "_unload", "_move", "_format", "_mount", "_erase") +ALLOWED_MARKERS = ("dry_run", "resolve_", "plan_") +# Config loading reads a file, not a cartridge; exempt by exact name only. +ALLOWED_EXACT = frozenset({"load_config"}) + + +def is_forbidden_name(name: str) -> bool: + lowered = name.lower() + if lowered in ALLOWED_EXACT or any(marker in lowered for marker in ALLOWED_MARKERS): + return False + return ( + lowered in FORBIDDEN_EXACT + or lowered.startswith(FORBIDDEN_PREFIXES) + or any(s in lowered for s in FORBIDDEN_SUBSTRINGS) + ) + + +def find_forbidden(source: str) -> list[str]: + """Return ``line:name`` for every forbidden import or call in ``source``.""" + hits: list[str] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom | ast.Import): + for alias in node.names: + if is_forbidden_name(alias.name.rsplit(".", 1)[-1]): + hits.append(f"{node.lineno}:{alias.name}") + elif isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + name = func.id + elif isinstance(func, ast.Attribute): + if isinstance(func.value, ast.Constant) and isinstance(func.value.value, str): + continue # "...".format(...) is string formatting, not a tape API + name = func.attr + else: + continue + if is_forbidden_name(name): + hits.append(f"{node.lineno}:{name}") + return hits + + +def _readonly_modules() -> list[Path]: + return sorted(p for p in READONLY_DIR.glob("*.py") if p.name != Path(__file__).name) + + +def test_readonly_dir_has_modules_to_scan() -> None: + assert len(_readonly_modules()) >= 2 + + +@pytest.mark.parametrize("path", _readonly_modules(), ids=lambda p: p.name) +def test_readonly_module_calls_no_mutating_api(path: Path) -> None: + assert find_forbidden(path.read_text()) == [] + + +@pytest.mark.parametrize( + "snippet", + [ + "library.load(1, 'A00001L8')", + "backend.unload(0)", + "changer.move_medium(src, dst)", + "ltfs.format_tape('A', '/dev/nst0')", + "ltfs.format('A', confirmation)", + "mount_ltfs('/dev/nst0', '/mnt')", + "from openblade.hardware.ltfs import ltfs_mount", + "from openblade.jobs.inventory import run_inventory_job", + "library.inventory()", + "svc.load_tape(1)", + ], +) +def test_scanner_flags_forbidden_snippet(snippet: str) -> None: + assert find_forbidden(snippet) + + +@pytest.mark.parametrize( + "snippet", + [ + "resolve_tape_by_id('X')", + "LTFSCommandBackend.format_dry_run_plan('A', '/dev/nst0')", + "'{}'.format(1)", + "InventoryService(lib).snapshot()", + "sg_inq(dev, runner, guard)", + "from openblade.config import load_config", + ], +) +def test_scanner_allows_read_only_snippet(snippet: str) -> None: + assert find_forbidden(snippet) == [] diff --git a/tests/hardware/readonly/test_readonly_rig.py b/tests/hardware/readonly/test_readonly_rig.py new file mode 100644 index 0000000..847f7ba --- /dev/null +++ b/tests/hardware/readonly/test_readonly_rig.py @@ -0,0 +1,124 @@ +"""Read-only checks against the real i3 rig (nightly lane). + +Nothing here loads, moves, formats or mounts anything — enforced by +test_readonly_ast_safety.py. Skipped without the rig (see conftest gating). +""" + +from __future__ import annotations + +import json +import os +import tempfile +import warnings +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +from openblade.domain.scalar_coordinate import ScalarCoordinate +from openblade.hardware.runner import SafeRunner +from openblade.hardware.sg import sg_inq +from openblade.hardware.tapealert import read_tape_alerts +from openblade.jobs.inventory import InventoryService + +from .conftest import ApplianceClient, read_corpus + +pytestmark = [pytest.mark.real_hardware, pytest.mark.usefixtures("real_hardware_guard")] + + +def test_appliance_login_then_logout_succeeds(appliance: ApplianceClient) -> None: + assert appliance.login().status == 200 + assert appliance.logout().status in {200, 204} + + +def test_changer_firmware_and_identity_are_discoverable(changer_device, hardware_guard) -> None: + inquiry = sg_inq(changer_device, SafeRunner(dry_run=False), hardware_guard) + assert inquiry.vendor and inquiry.product and inquiry.revision + + +def test_inventory_snapshot_reports_slots_and_drives(real_library_backend) -> None: + inventory = InventoryService(real_library_backend).snapshot() + assert inventory.slots and inventory.drives + + +def test_element_coordinates_round_trip(authed_appliance: ApplianceClient) -> None: + resp = authed_appliance.request("GET", "/aml/physicalLibrary/elements") + assert resp.status == 200 + flat = _find_coordinates(resp.body) + assert flat, "appliance returned no element coordinates" + for raw in flat: + assert ScalarCoordinate.from_dict(raw).to_dict() == {k: raw[k] for k in _COORD_KEYS} + + +def test_partitions_are_visible(authed_appliance: ApplianceClient) -> None: + resp = authed_appliance.request("GET", "/aml/partitions") + assert resp.status == 200 and resp.body + + +def test_tapealert_health_reads_on_every_drive(drive_devices, hardware_guard) -> None: + for device in drive_devices: + report = read_tape_alerts(device, SafeRunner(dry_run=False), hardware_guard) + assert report.device + + +_COORD_KEYS = ("frame", "rack", "section", "column", "row", "type") + + +def _find_coordinates(node: Any) -> list[dict[str, Any]]: + if isinstance(node, dict): + if all(k in node for k in _COORD_KEYS): + return [node] + return [c for v in node.values() for c in _find_coordinates(v)] + if isinstance(node, list): + return [c for v in node for c in _find_coordinates(v)] + return [] + + +def _report_path() -> Path: + default = Path(tempfile.gettempdir()) / "differential-report.json" + return Path(os.environ.get("OPENBLADE_DIFFERENTIAL_REPORT", str(default))) + + +def _is_read_only(case: dict[str, Any]) -> bool: + req = case["request"] + return req["method"] == "GET" or req["path"] in {"/aml/users/login", "/aml/auth/logout"} + + +def test_emulator_matches_appliance_on_compatibility_corpus( + appliance: ApplianceClient, tmp_path: Path +) -> None: + from openblade.api.main import app + from openblade.bootstrap import create_context, reset_context + from openblade.config import OpenBladeConfig + + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'diff.db'}"))) + emulator = TestClient(app) + divergences: list[dict[str, Any]] = [] + for case in filter(_is_read_only, read_corpus()): + req = case["request"] + if case.get("auth") == "admin": + emulator.post("/aml/users/login", json={"name": "admin", "password": "password"}) + appliance.login() + # Credentials in corpus cases are emulator defaults; the appliance gets the rig's. + body = req.get("json") + if req["path"] == "/aml/users/login" and body and body.get("password") == "password": + body = {"name": appliance.user, "password": appliance.password} + emu = emulator.request(req["method"], req["path"], json=req.get("json")) + real = appliance.request(req["method"], req["path"], body) + if emu.status_code != real.status: + divergences.append( + { + "id": case["id"], + "source": case["source"], + "path": req["path"], + "emulator_status": emu.status_code, + "appliance_status": real.status, + } + ) + _report_path().write_text(json.dumps({"divergences": divergences}, indent=2)) + captured = [d for d in divergences if d["source"] == "captured"] + for d in divergences: + if d["source"] != "captured": + warnings.warn(f"inferred case diverges from appliance: {d}", stacklevel=1) + assert not captured, f"captured-case divergence (see {_report_path()}): {captured}" From 2105632fa90cd9c65508172484ecb891e1eb2df4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:05:50 +0000 Subject: [PATCH 049/205] ci(hardware): pass appliance config to rig jobs and fail preflight instead of skipping green Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/hardware-destructive.yml | 19 +++++++++++++++++++ .github/workflows/hardware-library-smoke.yml | 19 +++++++++++++++++++ .../workflows/hardware-nightly-readonly.yml | 19 +++++++++++++++++++ docs/runbooks/hardware-lanes.md | 8 ++++++++ 4 files changed, 65 insertions(+) diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml index e428c25..0b96a81 100644 --- a/.github/workflows/hardware-destructive.yml +++ b/.github/workflows/hardware-destructive.yml @@ -58,6 +58,10 @@ jobs: env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES }} # Never interpolated into a script: read from the environment only. BARCODES: ${{ inputs.barcodes || vars.SACRIFICIAL_BARCODES }} @@ -72,6 +76,21 @@ jobs: - run: pip install -e '.[dev]' + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present + run: | + set -euo pipefail + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" + exit 1 + fi + - name: Allowlist — refuse any cartridge outside the sacrificial set id: allowlist run: | diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index 87a8eec..f66b637 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -50,6 +50,10 @@ jobs: env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} @@ -62,6 +66,21 @@ jobs: - run: pip install -e '.[dev]' + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present + run: | + set -euo pipefail + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" + exit 1 + fi + - name: Snapshot inventory + drive state (before) run: | set -euo pipefail diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml index 3c9c3d1..7142d39 100644 --- a/.github/workflows/hardware-nightly-readonly.yml +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -48,6 +48,10 @@ jobs: env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} + OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} + OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} + OPENBLADE_APPLIANCE_INSECURE: ${{ vars.OPENBLADE_APPLIANCE_INSECURE }} OPENBLADE_DIFFERENTIAL_REPORT: differential-report.json I3_AML_USER: ${{ secrets.QUANTUM_AML_USER }} I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} @@ -60,6 +64,21 @@ jobs: - run: pip install -e '.[dev]' + # A rig lane must never go green by skipping: the rig tests skip when + # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE + # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. + - name: Preflight — rig configuration present + run: | + set -euo pipefail + missing=() + for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + echo "::error::rig lane misconfigured, missing: ${missing[*]}" + exit 1 + fi + # No allowlist step: this lane names no cartridges and performs no # motion. The snapshot diff below is what would catch a violation. - name: Snapshot inventory + drive state (before) diff --git a/docs/runbooks/hardware-lanes.md b/docs/runbooks/hardware-lanes.md index f4bcb38..6b59cc8 100644 --- a/docs/runbooks/hardware-lanes.md +++ b/docs/runbooks/hardware-lanes.md @@ -44,6 +44,14 @@ Repo → **Settings → Secrets and variables → Actions → Variables**: - **`SACRIFICIAL_BARCODES`** — comma-separated list of cartridges that may be loaded, formatted and overwritten (e.g. `OB0007L8,OB0008L8`). Nothing else may ever be touched by the destructive lane. +- **`OPENBLADE_APPLIANCE_URL`**, **`OPENBLADE_APPLIANCE_USER`** (variables), + **`OPENBLADE_APPLIANCE_PASSWORD`** (secret — put it on the `hardware-destructive` + environment too), optional **`OPENBLADE_APPLIANCE_INSECURE`** = `true` for a + self-signed appliance cert. `OPENBLADE_CHANGER_DEVICE` and + `OPENBLADE_DRIVE_DEVICES` are set in the runner host's environment (`.env` of + the runner service). Every rig job has a **Preflight** step that fails if any + of these is empty — otherwise the rig tests would skip and the lane would go + green without testing anything. ## 3. Environment approval (settings-only — cannot be committed) From 46d5a74c50923c74a35a0c06945b2221e6272b2c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:08:03 +0000 Subject: [PATCH 050/205] fix(scalar_http): refuse unload while the host-side LTFS state is mounted or dirty The Web Services backend had no mount-state gate, so a moveMedium could eject a mounted cartridge. Found by the new backend contract suite (strict xfail now removed); inventory also reports the tracked mount state. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/hardware/scalar_http/library_backend.py | 10 ++++++++-- tests/contract/test_ltfs_contract.py | 14 +------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/openblade/hardware/scalar_http/library_backend.py b/openblade/hardware/scalar_http/library_backend.py index a1565ff..055cbf1 100644 --- a/openblade/hardware/scalar_http/library_backend.py +++ b/openblade/hardware/scalar_http/library_backend.py @@ -69,7 +69,7 @@ import httpx -from openblade.domain.errors import DriveCorrelationError, OpenBladeError +from openblade.domain.errors import DriveCorrelationError, OpenBladeError, TapeMountedError from openblade.domain.models import ( Barcode, CartridgeState, @@ -184,7 +184,7 @@ def inventory(self) -> LibraryInventory: drive_id=int(address), barcode=_barcode_or_none(element.get("barcode")), drive_state=_drive_state(element.get("state")), - mount_state=MountState.UNMOUNTED, + mount_state=self._mount_states.get(int(address), MountState.UNMOUNTED), ) ) return LibraryInventory( @@ -258,6 +258,12 @@ def load(self, source_slot: int, drive_id: int) -> OperationResult: ) def unload(self, drive_id: int, target_slot: int) -> OperationResult: + # Safety gate: the Web Services API has no notion of LTFS mount state, so + # the host-side record is the only thing standing between a mounted (or + # dirty) volume and a moveMedium that ejects it. + mount_state = self._mount_states.get(drive_id, MountState.UNMOUNTED) + if mount_state is not MountState.UNMOUNTED: + raise TapeMountedError(f"Drive {drive_id} cannot be unloaded while {mount_state.value}") # Real i3 unload uses moveClass=8 (bit field) with the drive source; the # target slot is sent as a hint (Web Services manual). See # docs/reference/i3-contract-notes.md. diff --git a/tests/contract/test_ltfs_contract.py b/tests/contract/test_ltfs_contract.py index 6a23d3d..c95211c 100644 --- a/tests/contract/test_ltfs_contract.py +++ b/tests/contract/test_ltfs_contract.py @@ -50,19 +50,7 @@ def test_mount_rw_write_then_ro_read_round_trips_checksum( assert backend_pair.library.unload(drive_id, slot).success -def test_unload_while_mounted_is_refused( - backend_pair: BackendPair, request: pytest.FixtureRequest -) -> None: - if backend_pair.name == "emulator+sim-ltfs": - request.applymarker( - pytest.mark.xfail( - strict=True, - reason=( - "openblade/hardware/scalar_http/library_backend.py:260 unload() has no " - "LTFS mount-state gate; the AML emulator moves a mounted cartridge" - ), - ) - ) +def test_unload_while_mounted_is_refused(backend_pair: BackendPair) -> None: backend_pair.format_all() barcode, slot, drive_id = _load_first(backend_pair) handle = backend_pair.ltfs.mount(barcode, MountMode.READ_WRITE) From 8d3cded7d4b46c5b6127be3eace27681ab866ab4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:11:00 +0000 Subject: [PATCH 051/205] feat(catalog): job journal and staged-instance lifecycle methods - job_journal table (JobJournalEntry; created by create_all like drive_leases) with CatalogRepository.journal() / job_journal(). - create_staged_instance(): STAGING/VERIFYING instance + 'shard_staged' journal row in one transaction; the journal row is the job linkage (file_instances has no job column, and adding one needs a db.py migration). - list_staged_instances(job_id): still-uncommitted instances of a job, with barcode, tape_path and the shard record's shard_index. - mark_instances_verifying / mark_instances_archived: all-or-nothing bulk transitions; archived requires every instance to be VERIFYING. - delete_file_record_if_unarchived keeps records with staged instances so failure cleanup cannot orphan shards needed for resume. Tests mutation-checked: dropping the delete guard or the all-or-nothing validation fails the suite. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/models.py | 22 ++++ openblade/catalog/repository.py | 166 ++++++++++++++++++++++++++++++ tests/unit/test_staged_catalog.py | 132 ++++++++++++++++++++++++ 3 files changed, 320 insertions(+) create mode 100644 tests/unit/test_staged_catalog.py diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index 93ac005..3884fbe 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -471,3 +471,25 @@ class DriveLeaseRecord(Base): heartbeat_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) released_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + + +class JobJournalEntry(Base): + """Append-only per-job event journal (lease acquired, shard staged, commit, failure...). + + Created by ``Base.metadata.create_all`` like ``drive_leases`` -- a new table + needs no ``_migrate_schema`` step. ``detail_json`` is a JSON object; a + ``shard_staged`` row carries ``instance_id`` and is how staged instances are + linked back to their job (``file_instances`` has no job column). + """ + + __tablename__ = "job_journal" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + job_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow) + event: Mapped[str] = mapped_column(String(64), nullable=False) + detail_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + @property + def detail(self) -> dict[str, Any]: + return cast("dict[str, Any]", json.loads(self.detail_json or "{}")) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 538d9ed..c383830 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -22,6 +22,7 @@ FileInstance, FileRecord, Job, + JobJournalEntry, LibraryInstance, ManifestVersion, NasCacheDrive, @@ -126,6 +127,24 @@ class CatalogBrowseEntry: shard_count: int +@dataclass(frozen=True) +class StagedInstance: + """A sharded-archive instance not yet committed (STAGING or VERIFYING).""" + + instance_id: str + file_record_id: str + barcode: str + tape_path: str + shard_index: int | None + state: str + + +_STAGED_INSTANCE_STATES = frozenset( + {FileInstanceState.STAGING.value, FileInstanceState.VERIFYING.value} +) +_SHARD_STAGED_EVENT = "shard_staged" + + class CatalogRepository: def __init__(self, session: Session) -> None: self.session = session @@ -662,6 +681,151 @@ def mark_instance_failed(self, instance_id: str, error: str) -> None: instance.state = FileInstanceState.FAILED.value self.session.commit() + # -- Sharded-archive staging + job journal --------------------------------- + # Lifecycle: create_staged_instance (STAGING) -> mark_instances_verifying -> + # mark_instances_archived, the last being the job's ONE commit. Each bulk + # call validates every id before mutating anything, so it is all-or-nothing. + + def journal( + self, job_id: str, event: str, detail: Mapping[str, object] | None = None + ) -> JobJournalEntry: + """Append one event to ``job_id``'s journal and commit.""" + entry = JobJournalEntry( + job_id=job_id, + at=datetime.utcnow(), + event=event, + detail_json=json.dumps(dict(detail or {}), sort_keys=True, default=str), + ) + self.session.add(entry) + self.session.commit() + return entry + + def job_journal(self, job_id: str) -> list[JobJournalEntry]: + """Every journal entry for ``job_id``, oldest first (ties broken by id).""" + stmt = ( + select(JobJournalEntry) + .where(JobJournalEntry.job_id == job_id) + .order_by(JobJournalEntry.at, JobJournalEntry.id) + ) + return list(self.session.scalars(stmt)) + + def create_staged_instance( + self, + job_id: str, + file_record_id: str, + barcode: str, + tape_path: str, + *, + state: FileInstanceState = FileInstanceState.STAGING, + ) -> FileInstance: + """Create an uncommitted shard instance and its ``shard_staged`` journal row. + + Both rows land in one transaction; the journal row is the job linkage + that ``list_staged_instances`` reads. + """ + if state.value not in _STAGED_INSTANCE_STATES: + raise ValueError(f"staged instance state must be staging/verifying, not {state.value}") + record = self.session.get(FileRecord, file_record_id) + if record is None: + raise FileNotFoundError(f"File record {file_record_id} not found") + instance = FileInstance( + file_record_id=file_record_id, + barcode=barcode, + tape_path=str(PurePosixPath(tape_path)), + state=state.value, + ) + self.session.add(instance) + self.session.flush() + self.session.add( + JobJournalEntry( + job_id=job_id, + at=datetime.utcnow(), + event=_SHARD_STAGED_EVENT, + detail_json=json.dumps( + { + "instance_id": instance.id, + "barcode": barcode, + "tape_path": instance.tape_path, + "shard_index": record.shard_index, + }, + sort_keys=True, + ), + ) + ) + self.session.commit() + self.session.refresh(instance) + return instance + + def list_staged_instances(self, job_id: str) -> list[StagedInstance]: + """Instances staged by ``job_id`` that are still STAGING or VERIFYING.""" + instance_ids = [ + str(entry.detail.get("instance_id")) + for entry in self.job_journal(job_id) + if entry.event == _SHARD_STAGED_EVENT + ] + if not instance_ids: + return [] + stmt = ( + select(FileInstance, FileRecord.shard_index) + .join(FileRecord, FileInstance.file_record_id == FileRecord.id) + .where( + FileInstance.id.in_(instance_ids), + FileInstance.state.in_(_STAGED_INSTANCE_STATES), + ) + ) + staged = [ + StagedInstance( + instance_id=instance.id, + file_record_id=instance.file_record_id, + barcode=instance.barcode, + tape_path=instance.tape_path, + shard_index=shard_index, + state=instance.state, + ) + for instance, shard_index in self.session.execute(stmt) + ] + staged.sort(key=lambda s: (s.shard_index is None, s.shard_index or 0, s.barcode)) + return staged + + def _load_instances_in( + self, instance_ids: Sequence[str], allowed_states: frozenset[str] + ) -> list[FileInstance]: + instances: list[FileInstance] = [] + for instance_id in dict.fromkeys(instance_ids): + instance = self.session.get(FileInstance, instance_id) + if instance is None: + raise FileNotFoundError(f"File instance {instance_id} not found") + if instance.state not in allowed_states: + raise ValueError( + f"File instance {instance_id} is {instance.state}, " + f"expected one of {sorted(allowed_states)}" + ) + instances.append(instance) + return instances + + def mark_instances_verifying(self, instance_ids: Sequence[str]) -> None: + """Move STAGING (or already VERIFYING) instances to VERIFYING in one commit.""" + instances = self._load_instances_in(instance_ids, _STAGED_INSTANCE_STATES) + for instance in instances: + instance.state = FileInstanceState.VERIFYING.value + self.session.commit() + + def mark_instances_archived( + self, instance_ids: Sequence[str], *, checksum_verified: bool = True + ) -> None: + """The job's single commit: every VERIFYING instance becomes ARCHIVED together. + + Raises (changing nothing) if any id is missing or not VERIFYING. + """ + verifying = frozenset({FileInstanceState.VERIFYING.value}) + instances = self._load_instances_in(instance_ids, verifying) + now = datetime.utcnow() + for instance in instances: + instance.state = FileInstanceState.ARCHIVED.value + instance.archived_at = now + instance.checksum_verified = checksum_verified + self.session.commit() + def create_job(self, job_type: str, metadata: dict[str, object]) -> Job: job = Job(job_type=job_type, state="pending", metadata_json=json.dumps(metadata)) self.session.add(job) @@ -756,6 +920,8 @@ def delete_file_record_if_unarchived(self, path: str) -> None: return if any( instance.state in {FileInstanceState.ARCHIVED.value, FileInstanceState.VERIFIED.value} + # Staged shards must stay discoverable for resume / reconcile. + or instance.state in _STAGED_INSTANCE_STATES for instance in record.instances ): return diff --git a/tests/unit/test_staged_catalog.py b/tests/unit/test_staged_catalog.py new file mode 100644 index 0000000..be44f58 --- /dev/null +++ b/tests/unit/test_staged_catalog.py @@ -0,0 +1,132 @@ +"""Staged (STAGING/VERIFYING) instances and the job journal in the catalog. + +Follow-up to docs/decisions/2026-10-09-persistent-drive-leases.md: a sharded +archive stages its instances and exposes them as ARCHIVED only in one commit. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.models import FileInstance +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import FileNotFoundError +from openblade.domain.models import FileInstanceState + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()) + + +def _stage_two_shards(repo: CatalogRepository, job_id: str = "job-1") -> list[FileInstance]: + vg = repo.create_volume_group("vg-staged") + instances = [] + for index, barcode in enumerate(["AAA001L9", "BBB001L9"]): + record = repo.create_file_record( + f"/data/big.bin.shard{index}", 10, "0" * 64, vg.id, shard_index=index + ) + instances.append( + repo.create_staged_instance(job_id, record.id, barcode, f"/big.bin.shard{index}") + ) + return instances + + +def _states(repo: CatalogRepository, instances: list[FileInstance]) -> list[str]: + repo.session.expire_all() + return [repo.session.get(FileInstance, i.id).state for i in instances] # type: ignore[union-attr] + + +def test_journal_round_trip_is_ordered_and_scoped_to_the_job(repo: CatalogRepository) -> None: + repo.journal("job-1", "lease_acquired", {"drive_id": 0, "barcode": "AAA001L9"}) + repo.journal("job-2", "lease_acquired") + repo.journal("job-1", "failure", {"error": "LaneWriteError"}) + + entries = repo.job_journal("job-1") + + assert [e.event for e in entries] == ["lease_acquired", "failure"] + assert entries[0].detail == {"drive_id": 0, "barcode": "AAA001L9"} + assert entries[1].detail == {"error": "LaneWriteError"} + assert repo.job_journal("job-2")[0].detail == {} + assert repo.job_journal("nope") == [] + + +def test_create_staged_instance_journals_and_lists_by_job(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + other_job_instance = repo.create_staged_instance( + "job-2", instances[0].file_record_id, "CCC001L9", "/other" + ) + + staged = repo.list_staged_instances("job-1") + + assert [(s.barcode, s.tape_path, s.shard_index, s.state) for s in staged] == [ + ("AAA001L9", "/big.bin.shard0", 0, "staging"), + ("BBB001L9", "/big.bin.shard1", 1, "staging"), + ] + assert [s.instance_id for s in repo.list_staged_instances("job-2")] == [other_job_instance.id] + assert [e.event for e in repo.job_journal("job-1")] == ["shard_staged", "shard_staged"] + assert repo.list_staged_instances("no-such-job") == [] + + +def test_create_staged_instance_refuses_a_committed_state(repo: CatalogRepository) -> None: + vg = repo.create_volume_group("vg-x") + record = repo.create_file_record("/x", 1, "0" * 64, vg.id) + with pytest.raises(ValueError): + repo.create_staged_instance( + "job-1", record.id, "AAA001L9", "/x", state=FileInstanceState.ARCHIVED + ) + assert repo.list_instances_for_barcode("AAA001L9") == [] + + +def test_bulk_commit_archives_every_instance_together(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + ids = [i.id for i in instances] + + repo.mark_instances_verifying(ids) + assert _states(repo, instances) == ["verifying", "verifying"] + assert [s.state for s in repo.list_staged_instances("job-1")] == ["verifying"] * 2 + + repo.mark_instances_archived(ids) + + assert _states(repo, instances) == ["archived", "archived"] + assert all(repo.session.get(FileInstance, i).archived_at is not None for i in ids) # type: ignore[union-attr] + assert repo.list_staged_instances("job-1") == [] + + +def test_bulk_commit_is_all_or_nothing(repo: CatalogRepository) -> None: + instances = _stage_two_shards(repo) + repo.mark_instances_verifying([instances[0].id]) + + # One shard never reached VERIFYING: nothing may become ARCHIVED. + with pytest.raises(ValueError): + repo.mark_instances_archived([i.id for i in instances]) + assert _states(repo, instances) == ["verifying", "staging"] + + with pytest.raises(FileNotFoundError): + repo.mark_instances_archived([instances[0].id, "missing-id"]) + assert _states(repo, instances) == ["verifying", "staging"] + + +def test_staged_instances_are_never_listed_as_archived_or_restorable( + repo: CatalogRepository, +) -> None: + instances = _stage_two_shards(repo) + repo.mark_instances_verifying([instances[1].id]) # one STAGING, one VERIFYING + + assert repo.list_catalog_tape_barcodes() == [] + assert repo.list_ltfs_entries() == [] + for index in (0, 1): + with pytest.raises(FileNotFoundError): + repo.get_latest_instance_for_path(f"/data/big.bin.shard{index}") + + # Failure cleanup must not drop the records staged shards hang off. + repo.delete_file_record_if_unarchived("/data/big.bin.shard0") + assert repo.get_file_record("/data/big.bin.shard0") is not None + + repo.mark_instances_verifying([i.id for i in instances]) + repo.mark_instances_archived([i.id for i in instances]) + assert repo.list_catalog_tape_barcodes() == ["AAA001L9", "BBB001L9"] From 3389085b24a0c374c59e4d5a6cccb537bb5e4f33 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:11:23 +0000 Subject: [PATCH 052/205] test(integration): rendezvous all three lease holders before asserting disjoint drive ownership The started barrier only aligned threads before acquire_drives; a descheduled thread could receive drive 0 after another job released it ([0,0,1]). An all_holding barrier inside the lease's try/finally makes the assertion mean concurrent ownership. Scheduler unchanged (not a race). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_three_drive_concurrency.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_three_drive_concurrency.py b/tests/integration/test_three_drive_concurrency.py index ed3c7e4..320fe9f 100644 --- a/tests/integration/test_three_drive_concurrency.py +++ b/tests/integration/test_three_drive_concurrency.py @@ -62,6 +62,11 @@ def test_three_jobs_run_on_three_drives_with_disjoint_ownership( jobs = [queue.create_job(JobType.ARCHIVE, {"barcode": barcode}) for barcode in BARCODES] started = threading.Barrier(len(jobs)) + # Rendezvous reached by each job only while it HOLDS its drive lease. Without + # it, a fast job can finish and release drive 0 before a descheduled job even + # acquires, and the scheduler then (correctly) hands drive 0 out again — so + # "three jobs, three drives" is only asserted when all three overlap. + all_holding = threading.Barrier(len(jobs)) ownership: list[tuple[str, int]] = [] errors: list[BaseException] = [] ownership_lock = threading.Lock() @@ -75,6 +80,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: with ownership_lock: ownership.append((job_id, handle.drive_id)) try: + all_holding.wait(timeout=10) # One robot, three drives: the changer serializes, the I/O does not. _load_with_retry(library, slot_id, handle.drive_id) mount = ltfs.mount(barcode, MountMode.READ_WRITE) @@ -97,6 +103,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: for thread in threads: thread.join(timeout=15) + assert not any(thread.is_alive() for thread in threads) assert errors == [] assert len(ownership) == 3 # Every job got its own drive, and all three drives were used. From 24dbb3281dbe2c60b303aeb1ad41151c7a76cf6d Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:11:29 +0000 Subject: [PATCH 053/205] fix(nas): Ganesha config disables NLM/RQUOTA for the NFSv4-only export Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/nfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openblade/nas/nfs.py b/openblade/nas/nfs.py index 27f0116..8611ba6 100644 --- a/openblade/nas/nfs.py +++ b/openblade/nas/nfs.py @@ -48,6 +48,8 @@ def render_ganesha_conf(exports: Iterable[NfsExport]) -> str: "NFS_CORE_PARAM {\n" " Protocols = 4;\n" " NFS_Port = 2049;\n" + " Enable_NLM = false;\n" + " Enable_RQUOTA = false;\n" "}\n" "NFSV4 {\n" " Grace_Period = 5;\n" From 3278209cf383988031f494331d813fde2d107ad7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:11:29 +0000 Subject: [PATCH 054/205] feat(deploy): containerised Samba + NFS-Ganesha protocol rig with hydrator shim Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/.gitignore | 1 + deploy/nas-protocols/Dockerfile | 12 +++++ deploy/nas-protocols/docker-compose.yml | 35 ++++++++++++++ deploy/nas-protocols/hydrator_shim.py | 61 +++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 deploy/nas-protocols/.gitignore create mode 100644 deploy/nas-protocols/Dockerfile create mode 100644 deploy/nas-protocols/docker-compose.yml create mode 100644 deploy/nas-protocols/hydrator_shim.py diff --git a/deploy/nas-protocols/.gitignore b/deploy/nas-protocols/.gitignore new file mode 100644 index 0000000..dc3b76b --- /dev/null +++ b/deploy/nas-protocols/.gitignore @@ -0,0 +1 @@ +state/ diff --git a/deploy/nas-protocols/Dockerfile b/deploy/nas-protocols/Dockerfile new file mode 100644 index 0000000..67dcee6 --- /dev/null +++ b/deploy/nas-protocols/Dockerfile @@ -0,0 +1,12 @@ +# One pinned base for all three rig roles; the role is chosen by build arg. +# debian:bookworm-slim pinned by digest (docker inspect RepoDigests). +FROM debian@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 +ARG ROLE +RUN apt-get update \ + && if [ "$ROLE" = samba ]; then pkgs="samba smbclient"; \ + elif [ "$ROLE" = nfs ]; then pkgs="nfs-ganesha nfs-ganesha-vfs rpcbind dbus"; \ + else pkgs="smbclient cifs-utils nfs-common python3 python3-pip attr procps"; fi \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $pkgs \ + && apt-get clean \ + && if [ "$ROLE" = client ]; then pip3 install --break-system-packages --no-cache-dir smbprotocol==1.15.0; fi +COPY hydrator_shim.py /usr/local/bin/hydrator_shim.py diff --git a/deploy/nas-protocols/docker-compose.yml b/deploy/nas-protocols/docker-compose.yml new file mode 100644 index 0000000..94ac072 --- /dev/null +++ b/deploy/nas-protocols/docker-compose.yml @@ -0,0 +1,35 @@ +# OpenBlade NAS protocol test rig (roadmap item 19). `make protocols-up`. +# The harness (tests/e2e/protocols/conftest.py) renders state/smb.conf and +# state/ganesha.conf through openblade.nas.samba / openblade.nas.nfs and +# populates state/export/ before bringing the rig up. +name: ob-nas-protocols +services: + samba: + build: { context: ., args: { ROLE: samba } } + command: ["smbd", "--foreground", "--no-process-group", "--debug-stdout", "-s", "/rig/smb.conf"] + volumes: ["./state:/rig"] + ports: ["${OB_SMB_PORT:-14450}:445"] + healthcheck: + test: ["CMD", "smbclient", "-N", "-L", "localhost", "-m", "SMB3"] + interval: 2s + retries: 30 + nfs-ganesha: + build: { context: ., args: { ROLE: nfs } } + # Ganesha's VFS FSAL needs open_by_handle_at (CAP_DAC_READ_SEARCH) etc. + privileged: true + command: ["sh", "-c", "rpcbind && exec ganesha.nfsd -F -L STDOUT -p /tmp/ganesha.pid -f /rig/ganesha.conf"] + volumes: ["./state:/rig"] + ports: ["${OB_NFS_PORT:-20490}:2049"] + healthcheck: + test: ["CMD-SHELL", "grep -qi ':0801 ' /proc/net/tcp /proc/net/tcp6"] + interval: 2s + retries: 30 + client: + build: { context: ., args: { ROLE: client } } + # Kernel cifs/nfs4 mounts need CAP_SYS_ADMIN: privileged client container. + privileged: true + command: ["python3", "/usr/local/bin/hydrator_shim.py", "/rig/export/rw"] + volumes: ["./state:/rig"] + depends_on: + samba: { condition: service_healthy } + nfs-ganesha: { condition: service_healthy } diff --git a/deploy/nas-protocols/hydrator_shim.py b/deploy/nas-protocols/hydrator_shim.py new file mode 100644 index 0000000..6754aac --- /dev/null +++ b/deploy/nas-protocols/hydrator_shim.py @@ -0,0 +1,61 @@ +"""Hydrator shim for the protocol rig (NOT the real hydrator). + +The real hydrator is the FUSE path (openblade/fuse): a read of an offline file +blocks inside the filesystem until tape recall completes. This rig exports a +plain directory, so an offline file is a stub whose content starts with +STUB_MAGIC followed by a JSON body {"state": "offline_on_tape", "mode": ...}. +A client asks for recall by creating `/.ob-control/.req`; then, +per the stub's mode: + "delay:" -> after seconds atomically replace the stub with the bytes + in `/.ob-tape/` (simulated tape recall), + "never" -> leave it offline (hydration timeout), + "tape_error" -> write `.ob-control/.err` = OFFLINE_TAPE_UNAVAILABLE. +Each recall is logged once to `.ob-control/hydration.log`; requests for a path +already in flight are coalesced (one hydration event per path). +""" + +import json +import os +import sys +import threading +import time +from pathlib import Path + +STUB_MAGIC = b"OPENBLADE-OFFLINE-STUB\n" + + +def _hydrate(root: Path, rel: str, mode: str, inflight: set[str]) -> None: + target = root / rel + if mode == "tape_error": + (root / ".ob-control" / f"{rel}.err").write_text("OFFLINE_TAPE_UNAVAILABLE") + elif mode.startswith("delay:"): + time.sleep(float(mode.split(":", 1)[1])) + tmp = target.with_name(f".{target.name}.hydrating") + tmp.write_bytes((root / ".ob-tape" / rel).read_bytes()) + os.replace(tmp, target) + with (root / ".ob-control" / "hydration.log").open("a") as log: + log.write(f"hydrated {rel}\n") + inflight.discard(rel) + + +def main(root: Path) -> None: + control = root / ".ob-control" + inflight: set[str] = set() + while True: + control.mkdir(parents=True, exist_ok=True) + for req in sorted(control.rglob("*.req")): + rel = str(req.relative_to(control))[: -len(".req")] + req.unlink(missing_ok=True) + data = (root / rel).read_bytes() if (root / rel).is_file() else b"" + if rel in inflight or not data.startswith(STUB_MAGIC): + continue + mode = json.loads(data[len(STUB_MAGIC) :])["mode"] + inflight.add(rel) + with (control / "hydration.log").open("a") as log: + log.write(f"recall {rel} mode={mode}\n") + threading.Thread(target=_hydrate, args=(root, rel, mode, inflight), daemon=True).start() + time.sleep(0.1) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) From 286e1d887ce02501bf03c25d13fec82970f989b5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:11:29 +0000 Subject: [PATCH 055/205] test(e2e): SMB/NFS protocol scenarios, protocols marker, make targets Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- Makefile | 15 +++- pyproject.toml | 1 + tests/e2e/protocols/__init__.py | 0 tests/e2e/protocols/conftest.py | 80 +++++++++++++++++ tests/e2e/protocols/rig.py | 114 ++++++++++++++++++++++++ tests/e2e/protocols/scenarios.py | 148 +++++++++++++++++++++++++++++++ tests/e2e/protocols/test_nfs.py | 28 ++++++ tests/e2e/protocols/test_smb.py | 54 +++++++++++ 8 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/protocols/__init__.py create mode 100644 tests/e2e/protocols/conftest.py create mode 100644 tests/e2e/protocols/rig.py create mode 100644 tests/e2e/protocols/scenarios.py create mode 100644 tests/e2e/protocols/test_nfs.py create mode 100644 tests/e2e/protocols/test_smb.py diff --git a/Makefile b/Makefile index 7a15dd5..4143954 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash .ONESHELL: -.PHONY: up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware +.PHONY: up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware protocols-up protocols-down test-protocols # Start all services up: @@ -106,3 +106,16 @@ all: lint test build test-hardware: OPENBLADE_BACKEND=real OPENBLADE_REAL_HARDWARE_ENABLED=true python3 -m pytest tests/hardware/ -v + +# NAS protocol rig (SMB/NFS, containerised). Host ports: OB_SMB_PORT / OB_NFS_PORT. +PROTOCOLS_COMPOSE = docker compose -f deploy/nas-protocols/docker-compose.yml + +protocols-up: + python -m tests.e2e.protocols.rig + OB_SMB_PORT=$${OB_SMB_PORT:-14450} OB_NFS_PORT=$${OB_NFS_PORT:-20490} $(PROTOCOLS_COMPOSE) up -d --build --wait + +protocols-down: + $(PROTOCOLS_COMPOSE) down -v --remove-orphans + +test-protocols: + python -m pytest -m protocols tests/e2e/protocols -q diff --git a/pyproject.toml b/pyproject.toml index 5cf6af1..e3cd90c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,4 +134,5 @@ markers = [ "slow: long-running simulator stress tests", "i3: Quantum i3 emulator and protocol tests", "real_i3: tests requiring a physical Quantum Scalar i3", + "protocols: containerised SMB/NFS protocol rig tests (make test-protocols)", ] diff --git a/tests/e2e/protocols/__init__.py b/tests/e2e/protocols/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/protocols/conftest.py b/tests/e2e/protocols/conftest.py new file mode 100644 index 0000000..3feaf78 --- /dev/null +++ b/tests/e2e/protocols/conftest.py @@ -0,0 +1,80 @@ +"""Fixtures for the containerised SMB/NFS protocol rig (marker: protocols). + +Path taken for NFS (and SMB file I/O): KERNEL mounts (cifs vers=3.0, nfs4) +inside the privileged `client` container; checks run there via +`docker compose exec`. Without docker every test here SKIPS. +""" + +import shutil +import subprocess +from collections.abc import Iterator + +import pytest + +from tests.e2e.protocols import rig + +MOUNTS = { + "smb": "mount -t cifs //samba/{s} /mnt/smb/{s} -o guest,vers=3.0,cache=none,actimeo=0", + "nfs": "mount -t nfs4 -o port=2049,noac,lookupcache=none,timeo=50 nfs-ganesha:/{s} /mnt/nfs/{s}", +} + + +class Rig: + def __init__(self, env: dict[str, str]) -> None: + self.env = env + + def compose(self, *args: str, timeout: float = 120) -> subprocess.CompletedProcess[str]: + return subprocess.run( # argv list, shell=False + [*rig.COMPOSE, *args], env=self.env, capture_output=True, text=True, timeout=timeout + ) + + def sh(self, script: str, timeout: float = 60) -> subprocess.CompletedProcess[str]: + """Run a shell snippet inside the client container.""" + return self.compose("exec", "-T", "client", "sh", "-c", script, timeout=timeout) + + def py(self, code: str, timeout: float = 60) -> subprocess.CompletedProcess[str]: + return self.compose("exec", "-T", "client", "python3", "-c", code, timeout=timeout) + + def mount(self, proto: str) -> None: + for share in ("ro", "rw"): + cmd = MOUNTS[proto].format(s=share) + res = self.sh( + f"mkdir -p /mnt/{proto}/{share}; mountpoint -q /mnt/{proto}/{share} || {cmd}" + ) + if res.returncode != 0: + pytest.skip( + f"kernel {proto} mount refused in client container: {res.stderr.strip()[:200]}" + ) + + +def _docker_ok() -> bool: + if shutil.which("docker") is None: + return False + try: + info = subprocess.run(["docker", "info"], capture_output=True, timeout=20) # argv list + except (OSError, subprocess.TimeoutExpired): + return False + return info.returncode == 0 + + +@pytest.fixture(scope="session") +def protocol_rig() -> Iterator[Rig]: + if not _docker_ok(): + pytest.skip("docker daemon not available: protocol rig cannot start") + env = rig.compose_env() + handle = Rig(env) + running = handle.compose("ps", "--status", "running", "-q", "client") + if not running.stdout.strip(): + rig.prepare() + up = handle.compose("up", "-d", "--build", "--wait", timeout=600) + if up.returncode != 0: + pytest.skip( + f"protocol rig failed to start (docker compose up): {up.stderr.strip()[-300:]}" + ) + yield handle + + +@pytest.fixture +def mounted(protocol_rig: Rig, proto: str) -> Rig: + protocol_rig.mount(proto) + return protocol_rig diff --git a/tests/e2e/protocols/rig.py b/tests/e2e/protocols/rig.py new file mode 100644 index 0000000..2d6af15 --- /dev/null +++ b/tests/e2e/protocols/rig.py @@ -0,0 +1,114 @@ +"""Protocol rig harness: render configs from NasShare records, stage the export. + +`python -m tests.e2e.protocols.rig` prepares deploy/nas-protocols/state/ (used +by `make protocols-up`). Offline files are stubs handled by the rig's hydrator +shim (deploy/nas-protocols/hydrator_shim.py); the real hydrator is FUSE. +""" + +import json +import os +import socket +import subprocess +from pathlib import Path + +from openblade.catalog.models import NasShare +from openblade.nas.export_render import nfs_export_from, samba_share_from +from openblade.nas.nfs import render_ganesha_conf +from openblade.nas.samba import render_smb_conf + +ROOT = Path(__file__).resolve().parents[3] +RIG = ROOT / "deploy" / "nas-protocols" +COMPOSE = ["docker", "compose", "-f", str(RIG / "docker-compose.yml")] +STATE = RIG / "state" +STUB_MAGIC = b"OPENBLADE-OFFLINE-STUB\n" +PAYLOAD = bytes(range(256)) * 4096 # 1 MiB, position-identifiable +DEEP_PARTS = ["d" * 200] * 19 # ~3.8 KiB relative path, under PATH_MAX 4096 +LONG_NAME = "n" * 251 + ".bin" # 255 bytes +UNICODE_NAME = "réсумé-日本語-🎞.txt" +# (scenario, stub mode) per protocol; each test gets its own stub. +OFFLINE = { + "blocking": "delay:2", + "timeout": "never", + "tape_error": "tape_error", + "concurrent": "delay:2", + "evict": "delay:0", + "restart": "delay:4", +} + +SHARES = [ + NasShare( + path="/rig/export/ro", + name="ro", + share_type="smb+nfs", + config_json=json.dumps({"pseudo": "/ro"}), + ), + NasShare( + path="/rig/export/rw", + name="rw", + share_type="smb+nfs", + config_json=json.dumps({"read_only": False, "case_sensitive": True, "pseudo": "/rw"}), + ), +] + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def compose_env() -> dict[str, str]: + env = dict(os.environ) + env.setdefault("OB_SMB_PORT", str(free_port())) + env.setdefault("OB_NFS_PORT", str(free_port())) + return env + + +def prepare() -> None: + """Render configs through openblade.nas and stage a fresh export tree.""" + export = STATE / "export" + if export.exists(): # container-created files are root-owned: clean in-container + subprocess.run( # argv list, shell=False + [ + "docker", + "run", + "--rm", + "-v", + f"{STATE}:/s", + "debian:bookworm-slim", + "rm", + "-rf", + "/s/export", + ], + check=True, + ) + STATE.mkdir(parents=True, exist_ok=True) + (STATE / "smb.conf").write_text(render_smb_conf(samba_share_from(s) for s in SHARES)) + exports = [nfs_export_from(s, export_id=i + 1) for i, s in enumerate(SHARES)] + (STATE / "ganesha.conf").write_text(render_ganesha_conf(exports)) + ro, rw = export / "ro", export / "rw" + deep = ro.joinpath(*DEEP_PARTS) + deep.mkdir(parents=True) + (deep / "leaf.txt").write_text("deep") + (rw / "offline").mkdir(parents=True) + (rw / ".ob-tape" / "offline").mkdir(parents=True) + (ro / "readme.txt").write_text("hello openblade\n") + (ro / "data.bin").write_bytes(PAYLOAD) + (ro / UNICODE_NAME).write_text("unicode") + (ro / LONG_NAME).write_text("long") + (ro / "mode640.txt").write_text("acl") + (ro / "mode640.txt").chmod(0o640) + for proto in ("smb", "nfs"): + for scenario, mode in OFFLINE.items(): + rel = f"offline/{proto}-{scenario}.bin" + (rw / rel).write_bytes( + STUB_MAGIC + json.dumps({"state": "offline_on_tape", "mode": mode}).encode() + ) + (rw / ".ob-tape" / rel).write_bytes(PAYLOAD) + os.chmod(export, 0o777) + for path in [rw, *rw.rglob("*")]: + path.chmod(0o777 if path.is_dir() else 0o666) + + +if __name__ == "__main__": + prepare() diff --git a/tests/e2e/protocols/scenarios.py b/tests/e2e/protocols/scenarios.py new file mode 100644 index 0000000..178a244 --- /dev/null +++ b/tests/e2e/protocols/scenarios.py @@ -0,0 +1,148 @@ +"""Scenarios shared by test_smb.py and test_nfs.py (each binds the `proto` fixture). + +All file I/O runs inside the client container against the kernel mount at +/mnt//{ro,rw}. Hydration uses the rig shim (not FUSE): a client requests +recall by creating .ob-control/.req through the share, then polls. +""" + +import time + +import pytest + +from tests.e2e.protocols import rig +from tests.e2e.protocols.conftest import Rig + +pytestmark = pytest.mark.protocols + +POLL = """ +import sys, time +p, deadline = sys.argv[1], time.time() + float(sys.argv[2]) +open(p.replace('/offline/', '/.ob-control/offline/') + '.req', 'w').close() +while time.time() < deadline: + data = open(p, 'rb').read() + if not data.startswith(b'OPENBLADE-OFFLINE-STUB'): + print('HYDRATED', len(data)); sys.exit(0) + time.sleep(0.2) +print('HYDRATION_TIMEOUT'); sys.exit(3) +""" + + +def _poll(r: Rig, proto: str, scenario: str, timeout: float) -> tuple[int, str]: + path = f"/mnt/{proto}/rw/offline/{proto}-{scenario}.bin" + res = r.compose( + "exec", "-T", "client", "python3", "-c", POLL, path, str(timeout), timeout=timeout + 30 + ) + return res.returncode, res.stdout + + +def _hydration_log(r: Rig) -> str: + return (rig.STATE / "export" / "rw" / ".ob-control" / "hydration.log").read_text() + + +def test_listing_shows_staged_files(mounted: Rig, proto: str) -> None: + res = mounted.sh(f"ls /mnt/{proto}/ro") + assert {"readme.txt", "data.bin", rig.UNICODE_NAME} <= set(res.stdout.split("\n")) + + +def test_open_read_seek_partial_range(mounted: Rig, proto: str) -> None: + code = ( + f"f=open('/mnt/{proto}/ro/data.bin','rb'); f.seek(1000); print(f.read(4).hex()); f.close()" + ) + assert mounted.py(code).stdout.strip() == rig.PAYLOAD[1000:1004].hex() + + +def test_rename_delete_on_rw_share_succeeds(mounted: Rig, proto: str) -> None: + d = f"/mnt/{proto}/rw" + res = mounted.sh( + f"echo x > {d}/a.txt && mv {d}/a.txt {d}/b.txt && rm {d}/b.txt && ! test -e {d}/b.txt" + ) + assert res.returncode == 0, res.stderr + + +def test_write_on_ro_share_is_refused(mounted: Rig, proto: str) -> None: + res = mounted.sh(f"mv /mnt/{proto}/ro/readme.txt /mnt/{proto}/ro/x.txt") + assert res.returncode != 0 + assert mounted.sh(f"test -e /mnt/{proto}/ro/readme.txt").returncode == 0 + + +def test_unicode_long_name_and_deep_path_readable(mounted: Rig, proto: str) -> None: + deep = "/".join(rig.DEEP_PARTS) + res = mounted.sh( + f"cat '/mnt/{proto}/ro/{rig.UNICODE_NAME}' /mnt/{proto}/ro/{rig.LONG_NAME} /mnt/{proto}/ro/{deep}/leaf.txt" + ) + assert res.stdout == "unicodelongdeep", res.stderr + + +def test_sparse_write_past_end_keeps_apparent_size(mounted: Rig, proto: str) -> None: + name = f"sparse-{proto}.bin" + mounted.py(f"f=open('/mnt/{proto}/rw/{name}','wb'); f.seek(64<<20); f.write(b'z'); f.close()") + st = (rig.STATE / "export" / "rw" / name).stat() + assert st.st_size == (64 << 20) + 1 + assert st.st_blocks * 512 < st.st_size // 2, f"server allocated {st.st_blocks * 512} bytes" + + +def test_interrupted_reader_then_reread_ok(mounted: Rig, proto: str) -> None: + reader = f"while true; do cat /mnt/{proto}/ro/data.bin > /dev/null; done" + mounted.sh( + f"nohup sh -c '{reader}' >/dev/null 2>&1 & sleep 1; pkill -9 -f 'data.bin > /dev/null' ; true" + ) + code = f"import hashlib;print(hashlib.sha256(open('/mnt/{proto}/ro/data.bin','rb').read()).hexdigest())" + import hashlib + + assert mounted.py(code).stdout.strip() == hashlib.sha256(rig.PAYLOAD).hexdigest() + + +def test_hydration_blocks_until_shim_recalls(mounted: Rig, proto: str) -> None: + start = time.monotonic() + rc, out = _poll(mounted, proto, "blocking", 20) + assert rc == 0 and f"HYDRATED {len(rig.PAYLOAD)}" in out + assert time.monotonic() - start >= 2 # stub mode delay:2 + + +def test_hydration_never_recalled_times_out(mounted: Rig, proto: str) -> None: + rc, out = _poll(mounted, proto, "timeout", 3) + assert (rc, out.strip()) == (3, "HYDRATION_TIMEOUT") + + +def test_offline_tape_error_is_reported(mounted: Rig, proto: str) -> None: + rc, _ = _poll(mounted, proto, "tape_error", 3) + err = rig.STATE / "export" / "rw" / ".ob-control" / "offline" / f"{proto}-tape_error.bin.err" + assert rc == 3 and err.read_text() == "OFFLINE_TAPE_UNAVAILABLE" + + +def test_concurrent_opens_trigger_one_hydration(mounted: Rig, proto: str) -> None: + path = f"/mnt/{proto}/rw/offline/{proto}-concurrent.bin" + script = f'for i in 1 2 3 4 5 6 7 8; do python3 -c "$P" {path} 20 & done; wait' + res = mounted.compose("exec", "-T", "-e", f"P={POLL}", "client", "sh", "-c", script, timeout=60) + assert res.stdout.count("HYDRATED") == 8 + assert _hydration_log(mounted).count(f"recall offline/{proto}-concurrent.bin") == 1 + + +def test_eviction_while_open_keeps_old_bytes_or_estale(mounted: Rig, proto: str) -> None: + """Shim-style atomic replace (new inode) while a handle is open.""" + rel = f"offline/{proto}-evict.bin" + assert _poll(mounted, proto, "evict", 20)[0] == 0 + code = f""" +import os, time, errno +f = open('/mnt/{proto}/rw/{rel}', 'rb'); first = f.read(4) +srv = '/rig/export/rw/{rel}' # server-side eviction, as the real evictor would +open(srv + '.tmp', 'wb').write(b'OPENBLADE-OFFLINE-STUB\\n{{}}'); os.replace(srv + '.tmp', srv) +time.sleep(1) +try: + f.seek(0); print('OLD' if f.read(4) == first else 'NEW') +except OSError as e: + print(errno.errorcode[e.errno]) +""" + proc = mounted.compose("exec", "-T", "client", "python3", "-c", code, timeout=30) + assert proc.stdout.strip() in {"OLD", "ESTALE"}, proc.stdout + proc.stderr + + +def test_server_restart_during_hydration_completes(mounted: Rig, proto: str) -> None: + service = "samba" if proto == "smb" else "nfs-ganesha" + path = f"/mnt/{proto}/rw/offline/{proto}-restart.bin" + mounted.compose( + "exec", "-T", "client", "touch", path.replace("/offline/", "/.ob-control/offline/") + ".req" + ) + assert mounted.compose("restart", service, timeout=90).returncode == 0 + rc, out = _poll(mounted, proto, "restart", 60) + assert rc == 0 and "HYDRATED" in out diff --git a/tests/e2e/protocols/test_nfs.py b/tests/e2e/protocols/test_nfs.py new file mode 100644 index 0000000..a271561 --- /dev/null +++ b/tests/e2e/protocols/test_nfs.py @@ -0,0 +1,28 @@ +"""NFSv4 (NFS-Ganesha, kernel nfs4 client) protocol scenarios.""" + +import pytest + +from tests.e2e.protocols.conftest import Rig +from tests.e2e.protocols.scenarios import * # noqa: F403 -- re-collect the shared scenarios bound to proto=nfs + +pytestmark = pytest.mark.protocols + + +@pytest.fixture +def proto() -> str: + return "nfs" + + +def test_nfs_is_case_sensitive(mounted: Rig) -> None: + assert mounted.sh("cat /mnt/nfs/ro/README.TXT").returncode != 0 + + +def test_nfs_posix_lock_granted_by_server(mounted: Rig) -> None: + code = "import fcntl;f=open('/mnt/nfs/rw/lock.bin','wb');fcntl.lockf(f,fcntl.LOCK_EX|fcntl.LOCK_NB,10);print('LOCKED')" + assert mounted.py(code).stdout.strip() == "LOCKED" + + +def test_nfs_acl_mapping() -> None: + pytest.skip( + "NFS ACL mapping needs nfs4_getfacl (nfs4-acl-tools) and Ganesha NFSv4 ACL support; not in rig" + ) diff --git a/tests/e2e/protocols/test_smb.py b/tests/e2e/protocols/test_smb.py new file mode 100644 index 0000000..2a6cd75 --- /dev/null +++ b/tests/e2e/protocols/test_smb.py @@ -0,0 +1,54 @@ +"""SMB (Samba, kernel cifs vers=3.0) protocol scenarios.""" + +import pytest + +from tests.e2e.protocols.conftest import Rig +from tests.e2e.protocols.scenarios import * # noqa: F403 -- re-collect the shared scenarios bound to proto=smb + +pytestmark = pytest.mark.protocols + + +@pytest.fixture +def proto() -> str: + return "smb" + + +def test_smb_case_insensitive_default_share(mounted: Rig) -> None: + assert mounted.sh("cat /mnt/smb/ro/README.TXT").stdout == "hello openblade\n" + + +def test_smb_case_sensitive_share_keeps_distinct_names(mounted: Rig) -> None: + res = mounted.sh( + "cd /mnt/smb/rw && echo a > Case.txt && echo b > case.txt && cat Case.txt case.txt" + ) + assert res.stdout == "a\nb\n", res.stderr + + +def test_smb_acl_maps_posix_mode(mounted: Rig) -> None: + res = mounted.sh("smbcacls -N //samba/ro mode640.txt") + assert res.returncode == 0, res.stderr + assert "ACL:Everyone:ALLOWED/0x0/0x0" not in res.stdout or "Unix Group" in res.stdout + assert "Unix User\\root:ALLOWED/0x0/RW" in res.stdout, res.stdout + + +def test_smb_byte_range_lock_conflicts_across_sessions(mounted: Rig) -> None: + code = """ +import smbclient, smbclient._io as io +from smbprotocol.exceptions import SMBOSError +from smbprotocol.open import SMB2LockElement, LockFlags +smbclient.ClientConfig(username='guest', password='', auth_protocol='ntlm') +p = r'\\\\samba\\rw\\lock.bin' +with smbclient.open_file(p, 'wb') as f: f.write(b'0' * 100) +a = smbclient.open_file(p, 'rb', share_access='rw'); b = smbclient.open_file(p, 'rb', share_access='rw') +el = SMB2LockElement(); el['offset'] = 0; el['length'] = 10 +el['flags'] = LockFlags.SMB2_LOCKFLAG_EXCLUSIVE_LOCK | LockFlags.SMB2_LOCKFLAG_FAIL_IMMEDIATELY +a.fd.lock([el]) +try: + b.fd.lock([el]); print('GRANTED') +except Exception as e: + print(type(e).__name__) +""" + res = mounted.py(code) + if "Error" in res.stderr and not res.stdout: + pytest.skip(f"smbprotocol lock driver unavailable: {res.stderr.strip()[-200:]}") + assert res.stdout.strip() not in {"", "GRANTED"}, res.stdout + res.stderr From 42bfac45b10f471edea3f2682f28421fbb7e9811 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:13:02 +0000 Subject: [PATCH 056/205] feat(aml-auth): session cap, strict 412, XML login negotiation, Retry-After Inferred behaviours (Rev D documents none of them); default behaviour unchanged except 415 for login without a supported Content-Type. XXE/DTD rejected; guards mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/aml_state.py | 25 ++ openblade/api/routes_aml_auth.py | 100 ++++++- tests/integration/test_aml_auth_fidelity.py | 302 ++++++++++++++++++++ 3 files changed, 420 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_aml_auth_fidelity.py diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 6c1b2dd..48b7ed6 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -2462,8 +2462,33 @@ def purge_expired_sessions() -> None: _STATE.sessions.pop(token, None) +_DEFAULT_MAX_SESSIONS_PER_USER = 5 + + +def max_sessions_per_user() -> int: + """Concurrent-session cap per user (inferred; the Rev D manual documents no limit). + + Read from ``OPENBLADE_AML_MAX_SESSIONS`` at call time; unparseable or < 1 falls + back to the default. + """ + raw = os.environ.get("OPENBLADE_AML_MAX_SESSIONS", "") + try: + value = int(raw) + except ValueError: + return _DEFAULT_MAX_SESSIONS_PER_USER + return value if value >= 1 else _DEFAULT_MAX_SESSIONS_PER_USER + + def create_session(user: AmlUser) -> SessionRecord: purge_expired_sessions() + # Evict the user's oldest sessions so the new login always succeeds: refusing + # would let abandoned sessions lock an operator (incl. admin) out until expiry. + own = sorted( + (record for record in _STATE.sessions.values() if record.user_name == user.name), + key=lambda record: record.created_at, + ) + for stale in own[: max(0, len(own) - max_sessions_per_user() + 1)]: + _STATE.sessions.pop(stale.token, None) now = _utcnow() record = SessionRecord( token=str(uuid4()), diff --git a/openblade/api/routes_aml_auth.py b/openblade/api/routes_aml_auth.py index b2188b5..1bd3c8b 100644 --- a/openblade/api/routes_aml_auth.py +++ b/openblade/api/routes_aml_auth.py @@ -4,10 +4,12 @@ import collections import os +import re import string import time from collections.abc import Callable from typing import Any, TypeVar +from xml.etree import ElementTree import pyotp from fastapi import ( @@ -58,6 +60,7 @@ def _check_rate_limit(remote_ip: str) -> None: raise HTTPException( status_code=429, detail=f"Too many login attempts. Try again in {_LOGIN_WINDOW_SECONDS // 60} minutes.", + headers={"Retry-After": str(max(1, int(deque[0] - window_start) + 1) if deque else 1)}, ) deque.append(now) @@ -345,18 +348,86 @@ async def require_auth(request: Request, context: AppContext = Depends(get_conte token = auth.split(" ", 1)[1] user = aml_state.get_session_user(token) if user is None: - raise HTTPException(status_code=401, detail="Authentication required") + headers = None + if session_id: + # Expired/unknown cookie: tell the browser to drop it. + headers = {"Set-Cookie": 'sessionID=""; Max-Age=0; Path=/; HttpOnly; SameSite=lax'} + raise HTTPException(status_code=401, detail="Authentication required", headers=headers) return user +_XML_MEDIA_TYPES = frozenset({"application/xml", "text/xml"}) +_FORM_MEDIA_TYPES = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +_ACCEPTABLE_MEDIA_TYPES = frozenset({"*/*", "application/*", "text/*", "application/json"}) +_XML_FORBIDDEN_MARKUP = re.compile(rb" None: + super().__init__(summary) + self.status_code = status_code + self.summary = summary + + +def _media_type(header_value: str) -> str: + return header_value.split(";", 1)[0].strip().lower() + + +def _wants_xml(request: Request) -> bool: + """True when Accept names an XML type and not JSON (inferred; manual is JSON-only).""" + accepted = {_media_type(part) for part in request.headers.get("accept", "").split(",")} + return bool(accepted & _XML_MEDIA_TYPES) and "application/json" not in accepted + + +def _check_acceptable(request: Request) -> None: + raw = request.headers.get("accept", "").strip() + if not raw: + return + accepted = {_media_type(part) for part in raw.split(",")} + if not accepted & (_ACCEPTABLE_MEDIA_TYPES | _XML_MEDIA_TYPES): + raise _LoginNegotiationError(406, "Unsupported Accept media type") + + +def _parse_login_xml(body: bytes) -> dict[str, Any]: + # defusedxml is not installed; refuse any DTD/entity declaration outright so + # neither external entities (XXE) nor entity expansion can reach the parser. + if _XML_FORBIDDEN_MARKUP.search(body): + raise _LoginNegotiationError(400, "XML DTD/entity declarations are not allowed") + try: + root = ElementTree.fromstring(body) + except ElementTree.ParseError as exc: + raise _LoginNegotiationError(400, "Malformed XML login payload") from exc + return {child.tag: (child.text or "") for child in root} + + +def _negotiated(request: Request, status_code: int, body: dict[str, Any]) -> Response: + if not _wants_xml(request): + return JSONResponse(status_code=status_code, content=body) + root = ElementTree.Element("WSResultCode") + for key, value in body.items(): + if value is not None: + ElementTree.SubElement(root, key).text = str(value) + return Response( + content=ElementTree.tostring(root, encoding="unicode"), + status_code=status_code, + media_type="application/xml", + ) + + async def _parse_login_request(request: Request) -> LoginRequest: - content_type = request.headers.get("content-type", "") + media_type = _media_type(request.headers.get("content-type", "")) payload: dict[str, Any] - if "json" in content_type: + if media_type == "application/json": payload = await request.json() - else: + elif media_type in _XML_MEDIA_TYPES: + payload = _parse_login_xml(await request.body()) + elif media_type in _FORM_MEDIA_TYPES: form = await request.form() payload = dict(form) + else: + raise _LoginNegotiationError(415, "Unsupported or missing Content-Type") # Backwards compatible: accept 'username' as an alias for 'name' if isinstance(payload, dict) and "username" in payload and "name" not in payload: @@ -505,7 +576,11 @@ async def test_ldap( @no_auth async def login(request: Request, context: AppContext = Depends(get_context)) -> Response: _ensure_state(context) - payload = await _parse_login_request(request) + try: + _check_acceptable(request) + payload = await _parse_login_request(request) + except _LoginNegotiationError as exc: + return _ws_error(exc.status_code, exc.summary) user_name = _validate_user_name(payload.name) remote_address = _remote_address(request) or "unknown" @@ -533,7 +608,14 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> body = _ws_result("Login successful").model_dump() # Provide session token in response body for API clients/tests that expect it body["token"] = session_record.token - response = JSONResponse(content=body) + # Inferred: Rev D signals a default password only via the Warning header. With + # OPENBLADE_AML_STRICT_PRECONDITIONS=true the same login answers 412 instead of + # 200; the session is still issued so the client can change the password. + strict = os.environ.get("OPENBLADE_AML_STRICT_PRECONDITIONS", "false").lower() == "true" + precondition_failed = strict and user.require_password_change + if precondition_failed: + body.update(code=412, description="Error", summary="Default password must be changed") + response = _negotiated(request, 412 if precondition_failed else 200, body) _is_production = os.environ.get("OPENBLADE_ENV", "development").lower() == "production" response.set_cookie( "sessionID", @@ -562,8 +644,12 @@ async def logout( context: AppContext = Depends(get_context), ) -> Response: _ensure_state(context) + try: + _check_acceptable(request) + except _LoginNegotiationError as exc: + return _ws_error(exc.status_code, exc.summary) aml_state.clear_session(request.cookies.get("sessionID", "")) - response = JSONResponse(content=_ws_result(f"Logged out {current_user.name}").model_dump()) + response = _negotiated(request, 200, _ws_result(f"Logged out {current_user.name}").model_dump()) response.delete_cookie("sessionID") return response diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py new file mode 100644 index 0000000..2bb6966 --- /dev/null +++ b/tests/integration/test_aml_auth_fidelity.py @@ -0,0 +1,302 @@ +"""AML auth fidelity (roadmap item 18): sessions, preconditions, negotiation, envelopes. + +Every behaviour here is *inferred* unless noted: the Rev D manual documents JSON +login/logout, the ``Warning: Default Password Supplied`` header and the +``sessionID`` cookie, but no session cap, 412, XML, 415/406 or 429 semantics. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from openblade.api import aml_state, routes_aml_auth +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig + +CREDS = {"name": "admin", "password": "password"} +LEAK = "postgres://u:SECRET@db/leak" + + +@pytest.fixture() +def client(tmp_path: Path) -> Iterator[TestClient]: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'fid.db'}")) + reset_context(context) + routes_aml_auth._login_attempts.clear() + yield TestClient(app, raise_server_exceptions=False) + routes_aml_auth._login_attempts.clear() + + +def _login(client: TestClient) -> str: + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200, response.text + return str(response.json()["token"]) + + +# --- 1. max sessions per user (oldest evicted) --------------------------------- + + +def test_session_cap_evicts_oldest(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENBLADE_AML_MAX_SESSIONS", "3") + tokens = [_login(client) for _ in range(4)] + client.cookies.clear() + first = client.get("/aml/users", headers={"Authorization": f"Bearer {tokens[0]}"}) + assert first.status_code == 401 + for token in tokens[1:]: + ok = client.get("/aml/users", headers={"Authorization": f"Bearer {token}"}) + assert ok.status_code == 200 + + +def test_session_cap_default_is_five(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENBLADE_AML_MAX_SESSIONS", raising=False) + assert aml_state.max_sessions_per_user() == 5 + monkeypatch.setenv("OPENBLADE_AML_MAX_SESSIONS", "bogus") + assert aml_state.max_sessions_per_user() == 5 + + +# --- 2. 412 behind OPENBLADE_AML_STRICT_PRECONDITIONS -------------------------- + + +def test_default_password_lenient_is_200_with_warning( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENBLADE_AML_STRICT_PRECONDITIONS", raising=False) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + assert response.headers["Warning"] == "Default Password Supplied" + + +def test_default_password_strict_is_412_and_session_usable( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 412 + body = response.json() + assert body["code"] == 412 and body["summary"] == "Default password must be changed" + assert response.headers["Warning"] == "Default Password Supplied" + assert client.get("/aml/users").status_code == 200 + + +# --- 3. content negotiation ---------------------------------------------------- + +XML_LOGIN = b"adminpassword" + + +def test_xml_login_and_logout_round_trip(client: TestClient) -> None: + xml = {"Content-Type": "application/xml", "Accept": "application/xml"} + response = client.post("/aml/users/login", content=XML_LOGIN, headers=xml) + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/xml") + assert "Login successful" in response.text + out = client.delete("/aml/users/login", headers={"Accept": "text/xml"}) + assert out.status_code == 200 + assert out.text.startswith("") + + +def test_text_xml_request_gets_json_by_default(client: TestClient) -> None: + response = client.post( + "/aml/users/login", content=XML_LOGIN, headers={"Content-Type": "text/xml"} + ) + assert response.status_code == 200 + assert response.json()["summary"] == "Login successful" + + +@pytest.mark.parametrize( + "payload", + [ + b']>' + b"admin&x;", + b']>' + b"admin&b;", + ], + ids=["xxe", "entity-expansion"], +) +def test_xml_dtd_and_entities_rejected(client: TestClient, payload: bytes) -> None: + response = client.post( + "/aml/users/login", content=payload, headers={"Content-Type": "application/xml"} + ) + assert response.status_code == 400 + assert response.json()["summary"] == "XML DTD/entity declarations are not allowed" + assert "root:" not in response.text + + +def test_malformed_xml_is_400(client: TestClient) -> None: + response = client.post( + "/aml/users/login", content=b"", headers={"Content-Type": "text/xml"} + ) + assert response.status_code == 400 + assert response.json()["summary"] == "Malformed XML login payload" + + +@pytest.mark.parametrize("content_type", [None, "text/plain", "application/yaml", ";;"]) +def test_unsupported_content_type_is_415(client: TestClient, content_type: str | None) -> None: + headers = {"Content-Type": content_type} if content_type else {} + response = client.post("/aml/users/login", content=b"name=admin", headers=headers) + assert response.status_code == 415 + body = response.json() + assert body["code"] == 415 + assert body["summary"] == "Unsupported or missing Content-Type" + + +def test_unsupported_accept_is_406(client: TestClient) -> None: + response = client.post("/aml/users/login", json=CREDS, headers={"Accept": "image/png"}) + assert response.status_code == 406 + assert response.json()["code"] == 406 + token = _login(client) + out = client.delete( + "/aml/users/login", + headers={"Accept": "image/png", "Authorization": f"Bearer {token}"}, + ) + assert out.status_code == 406 + + +# --- 5. expiry + rate limit + constant-time compare ---------------------------- + + +def _expire_all_sessions() -> None: + for record in aml_state._STATE.sessions.values(): + record.expires_at = record.expires_at - timedelta(days=2) + + +_HEADERS_DROPPED = ( + "openblade/api/main.py handle_http_exception drops HTTPException.headers on /aml " + "paths; cross-cutting fix requested (pass headers=exc.headers)" +) + + +def test_expired_cookie_is_401(client: TestClient) -> None: + _login(client) + _expire_all_sessions() + assert client.get("/aml/users").status_code == 401 + + +@pytest.mark.xfail(strict=True, reason=_HEADERS_DROPPED) +def test_expired_cookie_is_cleared(client: TestClient) -> None: + _login(client) + _expire_all_sessions() + response = client.get("/aml/users") + set_cookie = response.headers.get("set-cookie", "") + assert set_cookie.startswith("sessionID=") and "Max-Age=0" in set_cookie + + +def test_expired_bearer_is_401(client: TestClient) -> None: + token = _login(client) + client.cookies.clear() + _expire_all_sessions() + response = client.get("/aml/users", headers={"Authorization": f"Bearer {token}"}) + assert response.status_code == 401 + assert "set-cookie" not in response.headers + + +def test_eleventh_login_attempt_is_429_with_retry_after( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 10) + bad = {"name": "admin", "password": "wrong-password"} + for _ in range(10): + assert client.post("/aml/users/login", json=bad).status_code == 401 + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 429 + + +@pytest.mark.xfail(strict=True, reason=_HEADERS_DROPPED) +def test_rate_limited_login_carries_retry_after( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 0) + response = client.post("/aml/users/login", json=CREDS) + assert int(response.headers["Retry-After"]) >= 1 + + +def test_password_compare_is_constant_time() -> None: + import inspect + + assert "compare_digest" in inspect.getsource(aml_state.verify_password) + assert "compare_digest" in inspect.getsource(aml_state.authenticate_ldap_user) + + +# --- 4. error-code envelope matrix --------------------------------------------- + + +def _trigger(status: int, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> object: + if status == 401: + return client.get("/aml/users") + if status == 403: + monkeypatch.setattr(aml_state, "get_login_mode", lambda: 2) + return client.post("/aml/users/login", json=CREDS) + if status == 404: + _login(client) + return client.get("/aml/users/no-such-user") + if status == 412: + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + return client.post("/aml/users/login", json=CREDS) + if status == 429: + monkeypatch.setattr(routes_aml_auth, "_LOGIN_MAX_ATTEMPTS", 0) + return client.post("/aml/users/login", json=CREDS) + if status == 500: + + def boom(*_a: object, **_k: object) -> None: + raise RuntimeError(LEAK) + + monkeypatch.setattr(aml_state, "verify_credentials", boom) + return client.post("/aml/users/login", json=CREDS) + assert status == 503 + monkeypatch.setattr(aml_state, "get_service_access", lambda: {"enabled": False}) + real = aml_state.verify_credentials + + def as_service(name: str, password: str) -> object: + user = real(name, password) + assert user is not None + return SimpleNamespace( + name=user.name, role=2, require_password_change=user.require_password_change + ) + + monkeypatch.setattr(aml_state, "verify_credentials", as_service) + return client.post("/aml/users/login", json=CREDS) + + +_NO_500_ENVELOPE = ( + "openblade/api/main.py has no catch-all Exception handler for /aml paths, so an " + "unhandled error is Starlette's plain-text 500; cross-cutting fix requested" +) + + +def test_unhandled_error_is_500_without_leak( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + response = _trigger(500, client, monkeypatch) + assert response.status_code == 500 # type: ignore[attr-defined] + text = response.text # type: ignore[attr-defined] + assert LEAK not in text and "Traceback" not in text and "RuntimeError" not in text + + +@pytest.mark.parametrize( + "status", + [ + 401, + 403, + 404, + 412, + 429, + pytest.param(500, marks=pytest.mark.xfail(strict=True, reason=_NO_500_ENVELOPE)), + 503, + ], +) +def test_error_envelope_shape( + status: int, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + response = _trigger(status, client, monkeypatch) + assert response.status_code == status, response.text # type: ignore[attr-defined] + text = response.text # type: ignore[attr-defined] + body = response.json() # type: ignore[attr-defined] + assert isinstance(body, dict) + assert "Traceback" not in text and LEAK not in text and "RuntimeError" not in text + message = body.get("summary", body.get("detail", body.get("message"))) + assert isinstance(message, str) and message From 8bc5fd78469dda88a7feaaa5d96a23f388d7840a Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:13:03 +0000 Subject: [PATCH 057/205] docs(compat): inferred auth cases, iblade-rev-a scaffold, auth contract notes Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- compatibility/README.md | 6 ++++++ compatibility/iblade-rev-a/README.md | 11 ++++++++++ .../iblade/auth-login-success.json | 10 ++++++++++ .../iblade-rev-a/iblade/auth-logout.json | 11 ++++++++++ .../iblade/auth-unauthenticated-401.json | 10 ++++++++++ .../aml/auth-login-malformed-xml.json | 10 ++++++++++ .../aml/auth-login-missing-content-type.json | 10 ++++++++++ .../aml/auth-login-unsupported-accept.json | 10 ++++++++++ .../aml/auth-login-xml-response.json | 10 ++++++++++ docs/reference/i3-contract-notes.md | 20 +++++++++++++++++++ 10 files changed, 108 insertions(+) create mode 100644 compatibility/iblade-rev-a/README.md create mode 100644 compatibility/iblade-rev-a/iblade/auth-login-success.json create mode 100644 compatibility/iblade-rev-a/iblade/auth-logout.json create mode 100644 compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-xml-response.json diff --git a/compatibility/README.md b/compatibility/README.md index c3d797a..20056dd 100644 --- a/compatibility/README.md +++ b/compatibility/README.md @@ -59,3 +59,9 @@ exact body match (use for `captured` cases where the full body is authoritative) 3. Run the harness. If the emulator diverges, either the emulator is wrong (fix it) or the capture reveals a real contract we hadn't modeled (fix the emulator to match). The emulator is required to satisfy the corpus — not the reverse. + +## iblade-rev-a + +`iblade-rev-a/` is a scaffold profile for the Rev A (2017) iBlade surface. All its +cases are `inferred`; see `iblade-rev-a/README.md`. The harness discovers it +automatically (it globs `compatibility/**/*.json`). diff --git a/compatibility/iblade-rev-a/README.md b/compatibility/iblade-rev-a/README.md new file mode 100644 index 0000000..4ed55d9 --- /dev/null +++ b/compatibility/iblade-rev-a/README.md @@ -0,0 +1,11 @@ +# iblade-rev-a — Scalar-LTFS / Windows iBlade (Rev A, Sep 2017) + +Scope: the iBlade management surface (`/iblade/*`) as described by the local +iBlade reference manual, Rev A (2017). This profile is a **scaffold**. + +- Every case here is `inferred`: none has been checked against an appliance or + quoted from the manual. They lock current emulator behaviour only. +- The login, logout and 401 cases currently exercise the shared AML auth routes + (`/aml/users/login`, `/aml/auth/logout`) because no iBlade-specific auth path + is modelled yet. Replace them with `/iblade/*` paths once confirmed. +- Add `captured` cases next to these files, following `../README.md`. diff --git a/compatibility/iblade-rev-a/iblade/auth-login-success.json b/compatibility/iblade-rev-a/iblade/auth-login-success.json new file mode 100644 index 0000000..c79f1c9 --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-login-success.json @@ -0,0 +1,10 @@ +{ + "id": "iblade-login-success", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "fidelity_notes": "Scaffold: assumes the iBlade surface shares the AML login contract at /aml/users/login. Unverified.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" } }, + "expected": { "status": 200, "json_contains": { "code": 0, "description": "OK" } } +} diff --git a/compatibility/iblade-rev-a/iblade/auth-logout.json b/compatibility/iblade-rev-a/iblade/auth-logout.json new file mode 100644 index 0000000..d71e75e --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-logout.json @@ -0,0 +1,11 @@ +{ + "id": "iblade-logout", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "auth": "admin", + "fidelity_notes": "Scaffold: logout via the shared AML auth route. Unverified.", + "request": { "method": "POST", "path": "/aml/auth/logout" }, + "expected": { "status": 200 } +} diff --git a/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json b/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json new file mode 100644 index 0000000..5f55130 --- /dev/null +++ b/compatibility/iblade-rev-a/iblade/auth-unauthenticated-401.json @@ -0,0 +1,10 @@ +{ + "id": "iblade-unauthenticated-401", + "profile": "iblade-rev-a", + "source": "inferred", + "firmware": "RevA", + "manual_ref": "iBlade Rev A (Sep 2017), scaffold", + "fidelity_notes": "Scaffold: unauthenticated request is rejected with 401. Unverified.", + "request": { "method": "GET", "path": "/aml/physicalLibrary/elements" }, + "expected": { "status": 401 } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json b/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json new file mode 100644 index 0000000..9fee9b4 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-malformed-xml.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-malformed-xml", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Body sent with Content-Type application/xml but is not valid XML (the harness serialises json=), so it exercises the malformed-XML 400. A true XXE (DOCTYPE/ENTITY) case needs raw-body support in the harness; covered in integration tests instead.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Content-Type": "application/xml" } }, + "expected": { "status": 400, "json_contains": { "code": 400, "summary": "Malformed XML login payload" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json b/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json new file mode 100644 index 0000000..e3a4c13 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-missing-content-type.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-missing-content-type", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Missing Content-Type on login returns 415. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login" }, + "expected": { "status": 415, "json_contains": { "code": 415, "description": "Error", "summary": "Unsupported or missing Content-Type" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json b/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json new file mode 100644 index 0000000..6b9b355 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-unsupported-accept.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-unsupported-accept", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Unsupported Accept on login returns 406. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Accept": "image/png" } }, + "expected": { "status": 406, "json_contains": { "code": 406, "summary": "Unsupported Accept media type" } } +} diff --git a/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json b/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json new file mode 100644 index 0000000..16a8a06 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-xml-response.json @@ -0,0 +1,10 @@ +{ + "id": "aml-auth-login-xml-response", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "XML response (root WSResultCode) when Accept names an XML type and not application/json. The harness only parses JSON bodies and cannot send a raw XML request body, so this checks status only; the XML request path and XXE rejection are covered in tests/integration/test_aml_auth_fidelity.py.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" }, "headers": { "Accept": "application/xml" } }, + "expected": { "status": 200 } +} diff --git a/docs/reference/i3-contract-notes.md b/docs/reference/i3-contract-notes.md index 3e9d00e..9d00ffd 100644 --- a/docs/reference/i3-contract-notes.md +++ b/docs/reference/i3-contract-notes.md @@ -56,3 +56,23 @@ The Web Services guide is a *control-plane* API spec; drive/robotics performance figures live in hardware datasheets, not here. The emulator's latency profiles therefore remain OpenBlade-chosen approximations, not doc-certified numbers — do not present them as vendor-specified. + +## AML authentication behaviours + +Only the items marked "manual" are documented in Rev D; everything else is +inferred (OpenBlade-chosen) and unverified against an appliance. + +| Behaviour | Source | +|---|---| +| JSON login/logout; `sessionID` cookie | manual §2.5 (users/login) | +| `Warning: Default Password Supplied` header on login with a default password (200 kept by default) | manual §2.5 (users/login) | +| 412 `{"code":412,"description":"Error","summary":"Default password must be changed"}` plus token, cookie and Warning header when `OPENBLADE_AML_STRICT_PRECONDITIONS=true` (default false) | inferred | +| Per-user session cap, `OPENBLADE_AML_MAX_SESSIONS` (default 5); a login beyond the cap evicts the user's oldest session, whose token then gets 401 | inferred | +| Login also accepts form and `application/xml` / `text/xml` bodies (`....`) | inferred | +| XML response (root ``) when Accept names an XML type and not `application/json`; login and `DELETE /aml/users/login` only | inferred | +| ` Date: Fri, 9 Oct 2026 15:16:28 +0000 Subject: [PATCH 058/205] test(hardware): format LTFS test tapes via FormatService dry-run + token, never raw mkltfs Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/hardware/test_ltfs_operations.py | 71 +++++++++++++++++--------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/tests/hardware/test_ltfs_operations.py b/tests/hardware/test_ltfs_operations.py index e644561..df06c5d 100644 --- a/tests/hardware/test_ltfs_operations.py +++ b/tests/hardware/test_ltfs_operations.py @@ -42,23 +42,37 @@ def _unload_barcode(changer_device: str, runner, slot_id: int): assert result.returncode == 0, result.stderr -def _format_tape(runner, drive_device: str, barcode: str): - # --tape-serial takes exactly 6 alphanumeric characters. Real LTO barcodes - # are 6-8 (typically 8: six characters plus a two-character media-type - # suffix such as "L8"), so passing a barcode there fails every time with - # "LTFS15029E Tape serial must be 6 characters." The barcode belongs in - # --volume-name, which is unbounded - and that is what - # openblade.hardware.ltfs.LTFSCommandBackend.format_tape uses. - result = runner.run( - [ - "mkltfs", - f"--device={resolve_sg_device(drive_device)}", - f"--volume-name={barcode}", - "--force", - ], - timeout=LTFS_TIMEOUT, - ) - assert result.returncode == 0, result.stderr +def _format_tape(app_context, drive_device: str, barcode: str): + """Format ``barcode`` through the product's two-phase flow - never mkltfs directly. + + Phase 1 is ``FormatService.dry_run``: it checks the barcode against live + inventory, persists a one-time SafetyToken and returns the plan. Phase 2 is + ``FormatService.confirm`` with exactly that token, which re-validates it + against the stored row before LTFSCommandBackend.format_tape runs mkltfs. + The cartridge is already in drive 0 (``_load_barcode``), and the orchestrator + formats a loaded cartridge in place without moving it. + """ + service = app_context.format_service + # The tests mount ``drive_device``; make sure the service formats the same + # drive, otherwise every later mount would be testing an unformatted tape. + library = app_context.library + drive_id = library.find_drive_by_barcode(barcode) + assert drive_id is not None, f"{barcode} is not loaded in a drive the app can see" + drive_device_for = getattr(library, "drive_device", None) + if drive_device_for is not None: + assert resolve_sg_device(drive_device_for(drive_id)) == resolve_sg_device(drive_device), ( + f"App would format {barcode} in drive {drive_id}, not {drive_device}" + ) + + plan, token = service.dry_run(barcode) + assert plan.operation == "format" + assert plan.is_destructive + assert plan.affected_barcodes == [barcode] + assert token.operation == "format" + assert token.target_barcode == barcode + + result = service.confirm(barcode, token.token) + assert result.success, result.message return result @@ -98,12 +112,13 @@ def test_mkltfs_formats_tape( drive_devices, runner, scratch_barcode, + real_app_context, ): """Requires: a scratch tape that is safe to format and load into drive 0.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - result = _format_tape(runner, drive_devices[0], scratch_barcode) - assert result.returncode == 0 + result = _format_tape(real_app_context, drive_devices[0], scratch_barcode) + assert result.success finally: _unload_barcode(changer_device, runner, slot_id) @@ -114,12 +129,13 @@ def test_ltfs_mount_formatted( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) result = _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) assert result.returncode == 0 assert tmp_mount_dir.exists() @@ -137,6 +153,7 @@ def test_ltfs_write_small_file( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -144,7 +161,7 @@ def test_ltfs_write_small_file( slot_id = _load_barcode(changer_device, runner, scratch_barcode) payload = tmp_path / "small.bin" try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) _write_payload(payload) target = tmp_mount_dir / payload.name @@ -165,6 +182,7 @@ def test_ltfs_read_small_file( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -174,7 +192,7 @@ def test_ltfs_read_small_file( restored = tmp_path / "restored.bin" try: expected_sha = _write_payload(source) - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) target = tmp_mount_dir / source.name target.write_bytes(source.read_bytes()) @@ -195,12 +213,13 @@ def test_ltfs_unmount_clean( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) result = _unmount_ltfs(runner, tmp_mount_dir) assert result.returncode == 0 @@ -214,6 +233,7 @@ def test_ltfs_remount_persistence( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, tmp_path, ): @@ -221,7 +241,7 @@ def test_ltfs_remount_persistence( slot_id = _load_barcode(changer_device, runner, scratch_barcode) payload = tmp_path / "persist.bin" try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) expected_sha = _write_payload(payload) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) target = tmp_mount_dir / payload.name @@ -245,12 +265,13 @@ def test_ltfs_capacity_reporting( drive_devices, runner, scratch_barcode, + real_app_context, tmp_mount_dir, ): """Requires: a scratch tape that is safe to format and mount.""" slot_id = _load_barcode(changer_device, runner, scratch_barcode) try: - _format_tape(runner, drive_devices[0], scratch_barcode) + _format_tape(real_app_context, drive_devices[0], scratch_barcode) _mount_ltfs(runner, drive_devices[0], tmp_mount_dir) result = runner.run(["df", "-h", str(tmp_mount_dir)], timeout=30) assert result.returncode == 0, result.stderr From cd361fb75e703271a508b383df8886c059e1089d Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:16:28 +0000 Subject: [PATCH 059/205] test(hardware): enforce read-only at runtime (HTTP + argv allowlists) and harden the AST motion guard Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/hardware/readonly/conftest.py | 74 +++++++++ .../readonly/test_drive_identity_bridge.py | 8 +- .../readonly/test_readonly_ast_safety.py | 120 +++++++++++--- tests/hardware/readonly/test_readonly_rig.py | 19 ++- .../readonly/test_readonly_runtime_guards.py | 156 ++++++++++++++++++ 5 files changed, 348 insertions(+), 29 deletions(-) create mode 100644 tests/hardware/readonly/test_readonly_runtime_guards.py diff --git a/tests/hardware/readonly/conftest.py b/tests/hardware/readonly/conftest.py index ec06463..22c71d1 100644 --- a/tests/hardware/readonly/conftest.py +++ b/tests/hardware/readonly/conftest.py @@ -11,6 +11,7 @@ import json import os +import re import ssl import urllib.error import urllib.request @@ -23,6 +24,7 @@ from openblade.config import load_config from openblade.domain.policies import RealHardwareGuard +from openblade.hardware.runner import CommandResult, SafeRunner from openblade.hardware.safety import require_real_hardware CORPUS_DIR = Path(__file__).resolve().parents[3] / "compatibility" @@ -32,6 +34,72 @@ def read_corpus() -> list[dict[str, Any]]: return [json.loads(p.read_text()) for p in sorted(CORPUS_DIR.rglob("*.json"))] +class ReadOnlyViolation(RuntimeError): + """A read-only-lane test attempted an HTTP request or command that could mutate the rig.""" + + +# The only non-GET requests the read-only lane may send: the session handshake. +AUTH_REQUESTS = frozenset({("POST", "/aml/users/login"), ("POST", "/aml/auth/logout")}) + + +def check_appliance_request(method: str, path: str) -> None: + """Raise ReadOnlyViolation unless ``method path`` is a GET or the login/logout call. + + The method is matched exactly (HTTP methods are case-sensitive), so ``"get"`` + is refused rather than normalised. + """ + verb = method + if not path.startswith("/"): + raise ReadOnlyViolation(f"appliance path must be absolute, got {path!r}") + if verb == "GET" or (verb, path) in AUTH_REQUESTS: + return + raise ReadOnlyViolation(f"read-only lane refuses {verb} {path}") + + +_DEVICE = re.compile(r"/dev/[A-Za-z0-9_./:+-]+") +_LOG_PAGE = re.compile(r"(0x)?[0-9A-Fa-f]{1,2}") + + +def _is_device(arg: str) -> bool: + return _DEVICE.fullmatch(arg) is not None and ".." not in arg + + +def _is_path(arg: str) -> bool: + return arg.startswith("/") and ".." not in arg + + +def is_allowed_argv(argv: list[str]) -> bool: + """Exact argv shapes the read-only lane may execute; anything else is refused. + + Positions are fixed so a flag can't be smuggled in (``sg_logs -R`` resets log + pages; ``mtx -f dev load`` moves a cartridge). + """ + match argv: + case ["sg_inq", dev]: + return _is_device(dev) + case ["sg_logs", "-p", page, dev]: + return _LOG_PAGE.fullmatch(page) is not None and _is_device(dev) + case ["mtx", "-f", dev, "status"]: + return _is_device(dev) + case ["ls", *paths] if paths: + return all(_is_path(p) for p in paths) + return False + + +class ReadOnlyRunner(SafeRunner): + """SafeRunner that only executes the read-only argv shapes in ``is_allowed_argv``.""" + + def run( + self, + args: list[str], + timeout: int | None = None, + redact_args: list[int] | None = None, + ) -> CommandResult: + if isinstance(args, str) or not is_allowed_argv(list(args)): + raise ReadOnlyViolation(f"read-only lane refuses command {args!r}") + return super().run(args, timeout=timeout, redact_args=redact_args) + + @dataclass class ApplianceResponse: status: int @@ -61,6 +129,7 @@ def __post_init__(self) -> None: ) def request(self, method: str, path: str, json_body: Any = None) -> ApplianceResponse: + check_appliance_request(method, path) data = None if json_body is None else json.dumps(json_body).encode() req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config, scheme checked in fixture self.base_url.rstrip("/") + path, @@ -112,6 +181,11 @@ def authed_appliance(appliance: ApplianceClient): appliance.logout() +@pytest.fixture +def readonly_runner() -> ReadOnlyRunner: + return ReadOnlyRunner(dry_run=False) + + @pytest.fixture def hardware_guard(real_hardware_guard) -> RealHardwareGuard: return require_real_hardware(load_config()) diff --git a/tests/hardware/readonly/test_drive_identity_bridge.py b/tests/hardware/readonly/test_drive_identity_bridge.py index f37cba0..6ffb9da 100644 --- a/tests/hardware/readonly/test_drive_identity_bridge.py +++ b/tests/hardware/readonly/test_drive_identity_bridge.py @@ -14,8 +14,8 @@ from openblade.domain.scalar_coordinate import ScalarCoordinate from openblade.hardware.correlation import resolve_tape_by_id -from openblade.hardware.runner import SafeRunner from openblade.hardware.sg import sg_inq +from tests.hardware.readonly.conftest import ReadOnlyRunner from .conftest import ApplianceClient @@ -45,13 +45,15 @@ def test_drive_coordinate_resolves_to_matching_by_id_device( for e in _drive_elements(resp.body) if ScalarCoordinate.from_dict(e["coordinate"]).element_type == _DRIVE_ELEMENT_TYPE ] + # On the rig (real_hardware_guard already skipped us off-rig) a library with + # no resolvable drive is a failure, not a pass-by-skip: the bridge is unproven. if not drives: - pytest.skip("Appliance reported no drive elements with coordinate + serialNumber") + pytest.fail("Appliance reported no drive elements with coordinate + serialNumber") for element in drives: serial = str(element["serialNumber"]) by_id = resolve_tape_by_id(serial) assert by_id is not None, f"no /dev/tape/by-id entry for drive serial {serial}" assert os.path.exists(by_id) - inquiry = sg_inq(os.path.realpath(by_id), SafeRunner(dry_run=False), hardware_guard) + inquiry = sg_inq(os.path.realpath(by_id), ReadOnlyRunner(dry_run=False), hardware_guard) assert inquiry.serial.strip().casefold() == serial.strip().casefold() # DRY: the LTFS mount target is resolved; no mount is performed. diff --git a/tests/hardware/readonly/test_readonly_ast_safety.py b/tests/hardware/readonly/test_readonly_ast_safety.py index f197871..8a35367 100644 --- a/tests/hardware/readonly/test_readonly_ast_safety.py +++ b/tests/hardware/readonly/test_readonly_ast_safety.py @@ -1,8 +1,12 @@ """Plain-CI guard: tests/hardware/readonly/ must be read-only by construction. -Fails if any module there imports or calls a load/unload/move/format/mount -API. Resolution helpers ("resolve_*", "*dry_run*", "plan_*") stay allowed. -Not rig-gated. +Fails if any module there (recursively) imports, calls or merely references a +load/unload/move/format/mount API, fetches one via ``getattr(x, "")``, +puts a mutating verb into an argv list literal, or sends a non-GET request +other than the login/logout handshake. Resolution helpers stay allowed, matched +by prefix/suffix only ("resolve_*", "plan_*", "dry_run_*", "*_dry_run_plan"). +Not rig-gated. The runtime guards in conftest.py (ReadOnlyViolation) back this +up for anything a static scan can't see. """ from __future__ import annotations @@ -21,16 +25,30 @@ "run_inventory_job", "inventory", } ) # fmt: skip -FORBIDDEN_PREFIXES = ("load_", "unload_", "move_", "format_", "mount_", "erase_", "mkltfs") +FORBIDDEN_PREFIXES = ("load_", "unload_", "move_", "format_", "mount_", "erase_", "mkltfs_") FORBIDDEN_SUBSTRINGS = ("_load", "_unload", "_move", "_format", "_mount", "_erase") -ALLOWED_MARKERS = ("dry_run", "resolve_", "plan_") +ALLOWED_PREFIXES = ("resolve_", "plan_", "dry_run_") +ALLOWED_SUFFIXES = ("_dry_run_plan",) # Config loading reads a file, not a cartridge; exempt by exact name only. ALLOWED_EXACT = frozenset({"load_config"}) +# Whole argv items that make a command mutate media (mtx load, mkltfs, ltfs, ...). +FORBIDDEN_ARGV_WORDS = frozenset( + { + "load", "unload", "mkltfs", "ltfs", "mount", "umount", "unmount", "transfer", + "move", "format", "eject", "erase", "exchange", + } +) # fmt: skip +WRITE_VERBS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) +AUTH_PATHS = frozenset({"/aml/users/login", "/aml/auth/logout"}) def is_forbidden_name(name: str) -> bool: lowered = name.lower() - if lowered in ALLOWED_EXACT or any(marker in lowered for marker in ALLOWED_MARKERS): + if ( + lowered in ALLOWED_EXACT + or lowered.startswith(ALLOWED_PREFIXES) + or lowered.endswith(ALLOWED_SUFFIXES) + ): return False return ( lowered in FORBIDDEN_EXACT @@ -39,35 +57,78 @@ def is_forbidden_name(name: str) -> bool: ) +def _str_const(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _write_request_hit(node: ast.Call) -> str | None: + """Name a non-GET HTTP call unless it is a literal login/logout path.""" + func = node.func + if not isinstance(func, ast.Attribute): + return None + args = list(node.args) + if func.attr.upper() in WRITE_VERBS: + verb = func.attr.upper() + elif func.attr == "request" and args and _str_const(args[0]) is not None: + verb = str(_str_const(args[0])).upper() + args = args[1:] + if verb not in WRITE_VERBS: + return None + else: + return None + path = _str_const(args[0]) if args else None + if verb == "POST" and path in AUTH_PATHS: + return None + return f"{verb} {path if path is not None else ''}" + + def find_forbidden(source: str) -> list[str]: - """Return ``line:name`` for every forbidden import or call in ``source``.""" + """Return ``line:what`` for every mutating reference in ``source``.""" hits: list[str] = [] for node in ast.walk(ast.parse(source)): + line = getattr(node, "lineno", 0) if isinstance(node, ast.ImportFrom | ast.Import): for alias in node.names: if is_forbidden_name(alias.name.rsplit(".", 1)[-1]): - hits.append(f"{node.lineno}:{alias.name}") + hits.append(f"{line}:{alias.name}") + elif isinstance(node, ast.Attribute): + # Any reference, not just a call: `op = lib.load; op(1)` is caught here. + # Bare names are checked only as call targets (below): a forbidden bare + # name can only arrive via an import (flagged above) or a local binding + # of a flagged value, and flagging every Name trips on locals such as + # `inventory = InventoryService(...).snapshot()`. + if _str_const(node.value) is not None: + continue # "...".format(...) is string formatting, not a tape API + if is_forbidden_name(node.attr): + hits.append(f"{line}:{node.attr}") + elif isinstance(node, ast.List | ast.Tuple): + for elt in node.elts: + word = _str_const(elt) + if word is not None and word.lower() in FORBIDDEN_ARGV_WORDS: + hits.append(f"{line}:argv {word!r}") elif isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Name): - name = func.id - elif isinstance(func, ast.Attribute): - if isinstance(func.value, ast.Constant) and isinstance(func.value.value, str): - continue # "...".format(...) is string formatting, not a tape API - name = func.attr - else: - continue - if is_forbidden_name(name): - hits.append(f"{node.lineno}:{name}") + if isinstance(node.func, ast.Name) and is_forbidden_name(node.func.id): + hits.append(f"{line}:{node.func.id}") + if isinstance(node.func, ast.Name) and node.func.id in {"getattr", "setattr"}: + attr = _str_const(node.args[1]) if len(node.args) > 1 else None + if attr is not None and is_forbidden_name(attr): + hits.append(f"{line}:getattr {attr!r}") + write = _write_request_hit(node) + if write is not None: + hits.append(f"{line}:{write}") return hits def _readonly_modules() -> list[Path]: - return sorted(p for p in READONLY_DIR.glob("*.py") if p.name != Path(__file__).name) + # The scanner scans its own source too: its snippets are whole-statement + # strings, never argv-shaped single words, so they don't misfire. + return sorted(READONLY_DIR.rglob("*.py")) def test_readonly_dir_has_modules_to_scan() -> None: - assert len(_readonly_modules()) >= 2 + assert len(_readonly_modules()) >= 3 @pytest.mark.parametrize("path", _readonly_modules(), ids=lambda p: p.name) @@ -88,6 +149,18 @@ def test_readonly_module_calls_no_mutating_api(path: Path) -> None: "from openblade.jobs.inventory import run_inventory_job", "library.inventory()", "svc.load_tape(1)", + # B3 bypasses: each must be flagged. + "client.post('/aml/media/move', json={'src': 1})", + "requests.put(url + '/format')", + "runner.run(['mtx', '-f', d, 'load', '1', '0'])", + "subprocess.run(['mkltfs', '--force'])", + "getattr(lib, 'load')(1)", + "op = lib.load\nop(1)", + "lib.load_plan_now(1)", + "backend.unload_dry_run(0)", + "client.request('DELETE', '/aml/users/login')", + "client.delete('/aml/partitions/1')", + "appliance.request('POST', '/aml/physicalLibrary/elements/move', {})", ], ) def test_scanner_flags_forbidden_snippet(snippet: str) -> None: @@ -103,6 +176,11 @@ def test_scanner_flags_forbidden_snippet(snippet: str) -> None: "InventoryService(lib).snapshot()", "sg_inq(dev, runner, guard)", "from openblade.config import load_config", + "runner.run(['mtx', '-f', d, 'status'])", + "client.get('/aml/partitions')", + "client.request('GET', '/aml/physicalLibrary/elements')", + "client.request('POST', '/aml/users/login', body)", + "emulator.post('/aml/auth/logout')", ], ) def test_scanner_allows_read_only_snippet(snippet: str) -> None: diff --git a/tests/hardware/readonly/test_readonly_rig.py b/tests/hardware/readonly/test_readonly_rig.py index 847f7ba..8346db9 100644 --- a/tests/hardware/readonly/test_readonly_rig.py +++ b/tests/hardware/readonly/test_readonly_rig.py @@ -17,12 +17,11 @@ from fastapi.testclient import TestClient from openblade.domain.scalar_coordinate import ScalarCoordinate -from openblade.hardware.runner import SafeRunner from openblade.hardware.sg import sg_inq from openblade.hardware.tapealert import read_tape_alerts from openblade.jobs.inventory import InventoryService -from .conftest import ApplianceClient, read_corpus +from .conftest import CORPUS_DIR, ApplianceClient, ReadOnlyRunner, read_corpus pytestmark = [pytest.mark.real_hardware, pytest.mark.usefixtures("real_hardware_guard")] @@ -33,7 +32,7 @@ def test_appliance_login_then_logout_succeeds(appliance: ApplianceClient) -> Non def test_changer_firmware_and_identity_are_discoverable(changer_device, hardware_guard) -> None: - inquiry = sg_inq(changer_device, SafeRunner(dry_run=False), hardware_guard) + inquiry = sg_inq(changer_device, ReadOnlyRunner(dry_run=False), hardware_guard) assert inquiry.vendor and inquiry.product and inquiry.revision @@ -58,7 +57,7 @@ def test_partitions_are_visible(authed_appliance: ApplianceClient) -> None: def test_tapealert_health_reads_on_every_drive(drive_devices, hardware_guard) -> None: for device in drive_devices: - report = read_tape_alerts(device, SafeRunner(dry_run=False), hardware_guard) + report = read_tape_alerts(device, ReadOnlyRunner(dry_run=False), hardware_guard) assert report.device @@ -80,6 +79,11 @@ def _report_path() -> Path: return Path(os.environ.get("OPENBLADE_DIFFERENTIAL_REPORT", str(default))) +# The differential must compare SOMETHING: an empty/mis-filtered corpus would +# otherwise produce zero divergences and pass vacuously. +MIN_DIFFERENTIAL_CASES = 1 + + def _is_read_only(case: dict[str, Any]) -> bool: req = case["request"] return req["method"] == "GET" or req["path"] in {"/aml/users/login", "/aml/auth/logout"} @@ -95,7 +99,12 @@ def test_emulator_matches_appliance_on_compatibility_corpus( reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'diff.db'}"))) emulator = TestClient(app) divergences: list[dict[str, Any]] = [] - for case in filter(_is_read_only, read_corpus()): + cases = [c for c in read_corpus() if _is_read_only(c)] + assert len(cases) >= MIN_DIFFERENTIAL_CASES, ( + f"differential ran {len(cases)} read-only corpus case(s) from {CORPUS_DIR}; " + f"need >= {MIN_DIFFERENTIAL_CASES} (refusing to pass vacuously)" + ) + for case in cases: req = case["request"] if case.get("auth") == "admin": emulator.post("/aml/users/login", json={"name": "admin", "password": "password"}) diff --git a/tests/hardware/readonly/test_readonly_runtime_guards.py b/tests/hardware/readonly/test_readonly_runtime_guards.py new file mode 100644 index 0000000..3257527 --- /dev/null +++ b/tests/hardware/readonly/test_readonly_runtime_guards.py @@ -0,0 +1,156 @@ +"""Plain-CI unit tests for the read-only lane's runtime guards (no rig needed). + +ApplianceClient.request and ReadOnlyRunner.run must refuse anything that could +mutate the rig with ReadOnlyViolation *before* touching the network or a +subprocess. The transport and subprocess.run are replaced so nothing executes. +""" + +from __future__ import annotations + +import io +import json +import subprocess +from typing import Any + +import pytest + +from .conftest import ApplianceClient, ReadOnlyRunner, ReadOnlyViolation + + +class _FakeResponse(io.BytesIO): + status = 200 + + +@pytest.fixture +def sent() -> list[tuple[str, str]]: + return [] + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, sent: list[tuple[str, str]]) -> ApplianceClient: + appliance = ApplianceClient(base_url="https://rig.invalid", user="u", password="p") + + def fake_open(req: Any, timeout: int = 0) -> _FakeResponse: + sent.append((req.get_method(), req.full_url)) + return _FakeResponse(json.dumps({"ok": True}).encode()) + + monkeypatch.setattr(appliance._opener, "open", fake_open) + return appliance + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("POST", "/aml/media/move"), + ("PUT", "/aml/partitions/1"), + ("PATCH", "/aml/partitions/1"), + ("DELETE", "/aml/partitions/1"), + ("DELETE", "/aml/users/login"), + ("POST", "/aml/users/login/../media/move"), + ("post", "/aml/physicalLibrary/elements/move"), + ("GET", "aml/partitions"), + ("get", "/aml/partitions"), + ], +) +def test_appliance_request_mutating_call_raises_before_sending( + client: ApplianceClient, sent: list[tuple[str, str]], method: str, path: str +) -> None: + with pytest.raises(ReadOnlyViolation): + client.request(method, path) + + assert sent == [] + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("GET", "/aml/partitions"), + ("POST", "/aml/users/login"), + ("POST", "/aml/auth/logout"), + ], +) +def test_appliance_request_read_or_auth_call_is_sent( + client: ApplianceClient, sent: list[tuple[str, str]], method: str, path: str +) -> None: + resp = client.request(method, path) + + assert resp.status == 200 + assert sent == [(method, "https://rig.invalid" + path)] + + +def test_appliance_login_and_logout_pass_the_guard( + client: ApplianceClient, sent: list[tuple[str, str]] +) -> None: + client.login() + client.logout() + + assert [m for m, _ in sent] == ["POST", "POST"] + + +@pytest.fixture +def executed(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_run(args: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + # SafeRunner calls subprocess.run via the module, so this intercepts it. + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + +# Commands are space-joined strings (split at call time) rather than list +# literals so test_readonly_ast_safety's argv scan of this file stays clean. +@pytest.mark.parametrize( + "command", + [ + "mtx -f /dev/sg0 load 1 0", + "mtx -f /dev/sg0 unload 1 0", + "mtx -f /dev/sg0 transfer 1 2", + "mtx -f /dev/sg0 status load", + "mkltfs --force -d /dev/nst0", + "ltfs /mnt/tape -o devname=/dev/nst0", + "mount /dev/nst0 /mnt", + "sg_logs -R /dev/sg1", + "sg_logs -p 0x2e -R /dev/sg1", + "sg_inq --page=0x83 /dev/sg1", + "sg_inq /etc/passwd", + "mt -f /dev/nst0 erase", + "ls -R /dev", + "ls", + ], +) +def test_readonly_runner_forbidden_argv_raises_before_executing( + executed: list[list[str]], command: str +) -> None: + with pytest.raises(ReadOnlyViolation): + ReadOnlyRunner(dry_run=False).run(command.split()) + + assert executed == [] + + +def test_readonly_runner_string_command_raises(executed: list[list[str]]) -> None: + with pytest.raises(ReadOnlyViolation): + ReadOnlyRunner(dry_run=False).run("mtx -f /dev/sg0 status") # type: ignore[arg-type] + + assert executed == [] + + +# Commands are space-joined strings (split at call time) rather than list +# literals so test_readonly_ast_safety's argv scan of this file stays clean. +@pytest.mark.parametrize( + "command", + [ + "mtx -f /dev/sg0 status", + "sg_inq /dev/sg1", + "sg_inq /dev/tape/by-id/scsi-3500e09e0bb562001-nst", + "sg_logs -p 0x2e /dev/sg1", + "ls /dev/tape/by-id", + ], +) +def test_readonly_runner_allowed_argv_executes(executed: list[list[str]], command: str) -> None: + result = ReadOnlyRunner(dry_run=False).run(command.split()) + + assert result.returncode == 0 + assert executed == [command.split()] From 5351b3ec4e6d6b8404daa41d8ab460b08db34a6c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:16:28 +0000 Subject: [PATCH 060/205] feat(tools): junit skip gate, normalized allowlist output, snapshot refuses dry-run backend Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/hardware/conftest.py | 7 +- tests/hardware/test_allowlist.py | 32 +++++++++ tests/hardware/test_differential_corpus.py | 16 +++++ tests/hardware/test_junit_gate.py | 49 +++++++++++++ tests/hardware/test_snapshot.py | 34 ++++++++- tools/hardware/allowlist.py | 17 +++-- tools/hardware/junit_gate.py | 83 ++++++++++++++++++++++ tools/hardware/snapshot.py | 37 ++++++++-- 8 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 tests/hardware/test_differential_corpus.py create mode 100644 tests/hardware/test_junit_gate.py create mode 100644 tools/hardware/junit_gate.py diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 05df03c..a023897 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -57,13 +57,18 @@ def drive_devices(): return [d.strip() for d in devs.split(",") if d.strip()] +def split_barcode_env(raw: str) -> list[str]: + """Split a comma-separated barcode env value (the form tools.hardware.allowlist prints).""" + return [b.strip() for b in raw.split(",") if b.strip()] + + @pytest.fixture def scratch_barcodes(): _skip_if_no_real_hardware() barcodes_str = os.environ.get("OPENBLADE_SCRATCH_BARCODES", "") if not barcodes_str: pytest.skip("No scratch barcodes configured (OPENBLADE_SCRATCH_BARCODES)") - return [b.strip() for b in barcodes_str.split(",") if b.strip()] + return split_barcode_env(barcodes_str) @pytest.fixture diff --git a/tests/hardware/test_allowlist.py b/tests/hardware/test_allowlist.py index 5a659e8..4d40274 100644 --- a/tests/hardware/test_allowlist.py +++ b/tests/hardware/test_allowlist.py @@ -9,6 +9,7 @@ EXIT_VIOLATION, AllowlistError, check_subset, + format_barcodes, main, parse_barcodes, ) @@ -56,3 +57,34 @@ def test_main_falls_back_to_env_barcodes(monkeypatch) -> None: def test_main_unset_allowlist_exits_2(monkeypatch) -> None: monkeypatch.delenv("SACRIFICIAL_BARCODES", raising=False) assert main(["--barcodes", "A1"]) == EXIT_VIOLATION + + +def test_main_success_prints_only_normalized_list_on_stdout(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1 A2 A3") + assert main(["--barcodes", " a2, A1\na2 "]) == EXIT_OK + out, err = capsys.readouterr() + assert out == "A2,A1\n" + assert "OK" in err + + +def test_main_refusal_prints_nothing_on_stdout(monkeypatch, capsys) -> None: + monkeypatch.setenv("SACRIFICIAL_BARCODES", "A1") + assert main(["--barcodes", "PROD9"]) == EXIT_VIOLATION + assert capsys.readouterr().out == "" + + +def test_allowlist_output_round_trips_through_conftest_parser(monkeypatch, capsys) -> None: + from tests.hardware.conftest import split_barcode_env + + monkeypatch.setenv("SACRIFICIAL_BARCODES", "TST001L8,TST002L8,TST003L8") + assert main(["--barcodes", "tst003l8 TST001L8,tst003L8"]) == EXIT_OK + printed = capsys.readouterr().out.strip() + assert split_barcode_env(printed) == ["TST003L8", "TST001L8"] + assert format_barcodes(split_barcode_env(printed)) == printed + + +def test_conftest_parser_still_accepts_spaced_comma_lists() -> None: + from tests.hardware.conftest import split_barcode_env + + assert split_barcode_env(" A1 , A2,,") == ["A1", "A2"] + assert split_barcode_env("") == [] diff --git a/tests/hardware/test_differential_corpus.py b/tests/hardware/test_differential_corpus.py new file mode 100644 index 0000000..ebe1aea --- /dev/null +++ b/tests/hardware/test_differential_corpus.py @@ -0,0 +1,16 @@ +"""Off-rig guard for the read-only differential: its corpus filter must select cases. + +The on-rig differential asserts a minimum case count too, but only on the rig; +this catches an emptied or mis-filtered corpus in ordinary CI first. +""" + +from __future__ import annotations + +from tests.hardware.readonly.conftest import read_corpus +from tests.hardware.readonly.test_readonly_rig import MIN_DIFFERENTIAL_CASES, _is_read_only + + +def test_differential_corpus_has_minimum_read_only_cases() -> None: + cases = [c for c in read_corpus() if _is_read_only(c)] + assert MIN_DIFFERENTIAL_CASES > 0 + assert len(cases) >= MIN_DIFFERENTIAL_CASES diff --git a/tests/hardware/test_junit_gate.py b/tests/hardware/test_junit_gate.py new file mode 100644 index 0000000..d55d511 --- /dev/null +++ b/tests/hardware/test_junit_gate.py @@ -0,0 +1,49 @@ +"""Unit tests for the skip-as-pass gate (no rig needed).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools.hardware import junit_gate + + +def _report(tmp_path: Path, body: str, name: str = "r.xml") -> Path: + path = tmp_path / name + path.write_text(f'{body}') + return path + + +PASS = '' +SKIP = '' +FAIL = '' + + +def test_all_passed_is_green(tmp_path: Path) -> None: + assert junit_gate.main([str(_report(tmp_path, PASS))]) == 0 + + +@pytest.mark.parametrize("body", [SKIP, PASS + SKIP, "", FAIL]) +def test_skip_or_nothing_passed_is_red(tmp_path: Path, body: str) -> None: + assert junit_gate.main([str(_report(tmp_path, body))]) == 1 + + +def test_expected_skip_is_allowed(tmp_path: Path) -> None: + path = _report(tmp_path, PASS + SKIP) + assert junit_gate.main([str(path), "--expected-skip", "t.m::test_skip"]) == 0 + + +def test_expected_skip_does_not_mask_zero_passed(tmp_path: Path) -> None: + path = _report(tmp_path, SKIP) + assert junit_gate.main([str(path), "--expected-skip", "t.m::test_skip"]) == 1 + + +def test_missing_report_is_red(tmp_path: Path) -> None: + assert junit_gate.main([str(tmp_path / "absent.xml")]) == 1 + + +def test_multiple_reports_aggregate(tmp_path: Path) -> None: + a = _report(tmp_path, PASS, "a.xml") + b = _report(tmp_path, SKIP, "b.xml") + assert junit_gate.main([str(a), str(b)]) == 1 diff --git a/tests/hardware/test_snapshot.py b/tests/hardware/test_snapshot.py index 5a38096..1de119e 100644 --- a/tests/hardware/test_snapshot.py +++ b/tests/hardware/test_snapshot.py @@ -6,12 +6,14 @@ from pathlib import Path from openblade.bootstrap import create_context -from openblade.config import OpenBladeConfig +from openblade.config import BackendMode, OpenBladeConfig from tools.hardware.snapshot import ( EXIT_BAD_INPUT, EXIT_CHANGED, + EXIT_REFUSED, capture_snapshot, diff_snapshots, + dry_run_refusal, main, ) @@ -58,3 +60,33 @@ def test_diff_cli_changed_snapshots_exit_1_after_writing(tmp_path: Path) -> None b.write_text(json.dumps(_snap([{"slot_id": 1, "barcode": None}], []))) assert main(["diff", str(a), str(b), "--out", str(out)]) == EXIT_CHANGED assert json.loads(out.read_text())["changed"] is True + + +def test_diff_detects_library_id_change_alone() -> None: + before = _snap([{"slot_id": 1, "barcode": "A"}], []) + after = dict(before, library_id="OTHER") + result = diff_snapshots(before, after) + assert result["changed"] is True + assert result["library_id"] == {"before": "L", "after": "OTHER"} + + +def test_dry_run_refusal_only_for_real_backend_in_dry_run() -> None: + assert dry_run_refusal(OpenBladeConfig(backend=BackendMode.REAL, hardware_dry_run=True)) + assert dry_run_refusal(OpenBladeConfig(backend=BackendMode.REAL)) is None + assert dry_run_refusal(OpenBladeConfig(hardware_dry_run=True)) is None + + +def test_capture_cli_refuses_real_dry_run_with_exit_2(monkeypatch, tmp_path: Path, capsys) -> None: + import tools.hardware.snapshot as snapshot + + def _must_not_build(_config): + raise AssertionError("capture touched the backend despite dry-run") + + monkeypatch.setattr(snapshot, "_build_library", _must_not_build) + monkeypatch.setenv("OPENBLADE_BACKEND", "real") + monkeypatch.setenv("OPENBLADE_HARDWARE_DRY_RUN", "true") + out = tmp_path / "before.json" + assert main(["capture", "--out", str(out)]) == EXIT_REFUSED == 2 + assert not out.exists() + captured = capsys.readouterr() + assert "snapshot: REFUSED:" in captured.err and captured.out == "" diff --git a/tools/hardware/allowlist.py b/tools/hardware/allowlist.py index 1d0d151..207a405 100644 --- a/tools/hardware/allowlist.py +++ b/tools/hardware/allowlist.py @@ -6,9 +6,12 @@ The allowlist comes from env ``SACRIFICIAL_BARCODES``; the request comes from ``--barcodes`` or, when the flag is absent, env ``BARCODES``. Both accept -comma- and/or whitespace-separated lists. Exit 0 when the request is a -non-empty subset of a non-empty allowlist; exit 2 otherwise (fail closed), -naming every offending barcode. +comma- and/or whitespace-separated lists (case-insensitive). Exit 0 when the +request is a non-empty subset of a non-empty allowlist and print exactly one +stdout line: the normalized request (upper-cased, de-duplicated, request order, +comma-joined, e.g. ``TST001L8,TST002L8``), suitable for +``OPENBLADE_SCRATCH_BARCODES``. Exit 2 otherwise (fail closed), naming every +offending barcode on stderr with nothing on stdout. All diagnostics go to stderr. """ from __future__ import annotations @@ -38,6 +41,11 @@ def parse_barcodes(raw: str | None) -> list[str]: return list(seen) +def format_barcodes(barcodes: Sequence[str]) -> str: + """The normalized stdout form: comma-joined, no spaces.""" + return ",".join(barcodes) + + def check_subset(requested: Sequence[str], allowlist: Sequence[str]) -> list[str]: """Return ``requested`` if it is a non-empty subset of a non-empty allowlist. @@ -77,7 +85,8 @@ def main(argv: Sequence[str] | None = None) -> int: except AllowlistError as exc: print(f"allowlist: REFUSED: {exc}", file=sys.stderr) return EXIT_VIOLATION - print("allowlist: OK: " + " ".join(approved)) + print(f"allowlist: OK: {len(approved)} barcode(s) approved", file=sys.stderr) + print(format_barcodes(approved)) return EXIT_OK diff --git a/tools/hardware/junit_gate.py b/tools/hardware/junit_gate.py new file mode 100644 index 0000000..1c0055b --- /dev/null +++ b/tools/hardware/junit_gate.py @@ -0,0 +1,83 @@ +"""Fail a rig lane that "passed" by skipping. + +Rig tests skip when their prerequisites are missing, and pytest exits 0 on an +all-skipped run. This gate reads the JUnit XML each rig pytest step wrote and +exits 1 if any test was skipped (outside an explicit expected-skip list) or if +nothing passed at all. + + python -m tools.hardware.junit_gate reports/*.xml [--expected-skip NODEID ...] +""" + +from __future__ import annotations + +import argparse +import sys +import xml.etree.ElementTree as ET # parses our own pytest output, not untrusted input +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class GateResult: + passed: int = 0 + failed: int = 0 + skipped: list[str] = field(default_factory=list) + + +def _node_id(case: ET.Element) -> str: + classname = case.get("classname", "") + name = case.get("name", "") + return f"{classname}::{name}" if classname else name + + +def collect(paths: list[Path]) -> GateResult: + result = GateResult() + for path in paths: + root = ET.parse(path).getroot() + for case in root.iter("testcase"): + if case.find("skipped") is not None: + result.skipped.append(_node_id(case)) + elif case.find("failure") is not None or case.find("error") is not None: + result.failed += 1 + else: + result.passed += 1 + return result + + +def evaluate(result: GateResult, expected_skips: set[str]) -> list[str]: + """Return the reasons the gate fails (empty list = pass).""" + problems: list[str] = [] + unexpected = sorted(s for s in result.skipped if s not in expected_skips) + if unexpected: + problems.append(f"{len(unexpected)} unexpected skip(s): " + ", ".join(unexpected)) + if result.passed == 0: + problems.append("0 tests passed") + if result.failed: + problems.append(f"{result.failed} test(s) failed or errored") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("reports", nargs="+", type=Path) + parser.add_argument( + "--expected-skip", + action="append", + default=[], + help="JUnit classname::name of a skip that is expected (repeatable)", + ) + args = parser.parse_args(argv) + missing = [str(p) for p in args.reports if not p.is_file()] + if missing: + print("::error::missing JUnit report(s): " + ", ".join(missing), file=sys.stderr) + return 1 + result = collect(args.reports) + problems = evaluate(result, set(args.expected_skip)) + print(f"passed={result.passed} failed={result.failed} skipped={len(result.skipped)}") + for problem in problems: + print(f"::error::rig lane gate: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hardware/snapshot.py b/tools/hardware/snapshot.py index 874988c..9f0b93e 100644 --- a/tools/hardware/snapshot.py +++ b/tools/hardware/snapshot.py @@ -6,8 +6,12 @@ ``capture`` reads inventory + drive state through ``InventoryService`` on the library the configured backend builds (``OPENBLADE_BACKEND``; the simulator by default, real only when ``OPENBLADE_REAL_HARDWARE_ENABLED=true`` too). It is -read-only. ``diff`` compares two captures and writes the diff file, then exits -0 when identical, 1 when they differ, 2 on unreadable input. +read-only, and refuses (exit 2, ``snapshot: REFUSED:`` on stderr, no file +written) when the configured backend is real but ``OPENBLADE_HARDWARE_DRY_RUN`` +is true: dry-run never touches the changer, so the capture would not describe +the library the lane is about to change. ``diff`` compares two captures and writes the diff file, then exits +0 when identical (library_id included), 1 when they differ, 2 on unreadable +input. """ from __future__ import annotations @@ -22,11 +26,13 @@ from pathlib import Path from typing import Any +from openblade.config import BackendMode, OpenBladeConfig from openblade.domain.backends import LibraryBackend EXIT_OK = 0 EXIT_CHANGED = 1 EXIT_BAD_INPUT = 2 +EXIT_REFUSED = 2 class SnapshotError(Exception): @@ -80,6 +86,7 @@ def diff_snapshots(before: dict[str, Any], after: dict[str, Any]) -> dict[str, A """Compare two captures element-by-element (slots by slot_id, drives by drive_id).""" slots = _diff_section(before.get("slots", []), after.get("slots", []), "slot_id") drives = _diff_section(before.get("drives", []), after.get("drives", []), "drive_id") + library_changed = before.get("library_id") != after.get("library_id") changer = ( [] if before.get("changer_state") == after.get("changer_state") @@ -90,7 +97,7 @@ def diff_snapshots(before: dict[str, Any], after: dict[str, Any]) -> dict[str, A "slots": slots, "drives": drives, "changer_state": changer, - "changed": bool(slots or drives or changer), + "changed": bool(library_changed or slots or drives or changer), } @@ -104,11 +111,20 @@ def _load(path: Path) -> dict[str, Any]: return data -def _build_library() -> LibraryBackend: +def dry_run_refusal(config: OpenBladeConfig) -> str | None: + """Return why ``capture`` must refuse under ``config``, or None when it may run.""" + if config.backend is BackendMode.REAL and config.hardware_dry_run: + return ( + "OPENBLADE_HARDWARE_DRY_RUN=true with OPENBLADE_BACKEND=real: a dry-run " + "capture does not reflect the physical library." + ) + return None + + +def _build_library(config: OpenBladeConfig) -> LibraryBackend: from openblade.bootstrap import create_context - from openblade.config import load_config - return create_context(load_config()).library + return create_context(config).library def main(argv: Sequence[str] | None = None) -> int: @@ -123,7 +139,14 @@ def main(argv: Sequence[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "capture": - snap = capture_snapshot(_build_library()) + from openblade.config import load_config + + config = load_config() + reason = dry_run_refusal(config) + if reason is not None: + print(f"snapshot: REFUSED: {reason}", file=sys.stderr) + return EXIT_REFUSED + snap = capture_snapshot(_build_library(config)) args.out.write_text(json.dumps(snap, indent=2, sort_keys=True)) print(f"snapshot: captured {len(snap['slots'])} slots, {len(snap['drives'])} drives") return EXIT_OK From 06b740d08b1978c2f3e16e7f349c3fb9c1506764 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:16:29 +0000 Subject: [PATCH 061/205] ci(hardware): gate rig lanes on junit (no skip-as-pass), job-level rig lock, pin dry-run off Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/hardware-destructive.yml | 52 +++++++++++-------- .github/workflows/hardware-library-smoke.yml | 16 ++++-- .../workflows/hardware-nightly-readonly.yml | 27 ++++++---- .github/workflows/mhvtl-weekly.yml | 4 +- docs/runbooks/hardware-lanes.md | 17 +++++- 5 files changed, 79 insertions(+), 37 deletions(-) diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml index 0b96a81..a1d2151 100644 --- a/.github/workflows/hardware-destructive.yml +++ b/.github/workflows/hardware-destructive.yml @@ -6,7 +6,9 @@ name: Hardware destructive # Guard rails, in order: # 1. environment `hardware-destructive` — required reviewers approve every run # (configured in repo settings; a scheduled run waits for approval too); -# 2. rig guard — skipped (neutral, never green) without HARDWARE_RIG_AVAILABLE; +# 2. rig guard — without HARDWARE_RIG_AVAILABLE only a NOT RUN summary job +# runs, so the run concludes *success*; this lane must never be a required +# check or a badge (docs/runbooks/hardware-lanes.md); # 3. allowlist — exits non-zero before ANY hardware step if a barcode is # outside vars.SACRIFICIAL_BARCODES, or the list is empty; # 4. before/after inventory + drive snapshot, diffed and uploaded; @@ -28,12 +30,6 @@ on: permissions: contents: read -concurrency: - # One rig, one lock: never two destructive runs (or a read-only run) at once, - # and never cancel one mid-sequence — that can strand a cartridge in a drive. - group: hardware-rig-i3 - cancel-in-progress: false - jobs: no-rig-notice: name: no-rig-notice @@ -54,10 +50,19 @@ jobs: if: vars.HARDWARE_RIG_AVAILABLE == 'true' runs-on: [self-hosted, i3] environment: hardware-destructive + # Job-level (after the environment gate): a run waiting for approval must + # not hold the rig slot. One rig, one lock — never two rig jobs at once, and + # never cancel one mid-sequence (that can strand a cartridge in a drive). + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false timeout-minutes: 240 env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would turn every step into a no-op + # that still reports success. + OPENBLADE_HARDWARE_DRY_RUN: "false" OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} @@ -95,36 +100,39 @@ jobs: id: allowlist run: | set -euo pipefail - python -m tools.hardware.allowlist --barcodes "$BARCODES" - echo "OPENBLADE_SCRATCH_BARCODES=$BARCODES" >> "$GITHUB_ENV" + # Export the tool's normalized list, never the raw input. + normalized="$(python -m tools.hardware.allowlist --barcodes "$BARCODES")" + echo "OPENBLADE_SCRATCH_BARCODES=$normalized" >> "$GITHUB_ENV" - name: Snapshot inventory + drive state (before) run: | set -euo pipefail - mkdir -p snapshots + mkdir -p snapshots reports python -m tools.hardware.snapshot capture --out snapshots/before.json - name: Load / unload - run: python -m pytest tests/hardware/test_changer_operations.py -m real_hardware -v --tb=short -rs + run: python -m pytest tests/hardware/test_changer_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_changer_operations.xml - # test_mkltfs_formats_tape is deselected: it calls mkltfs directly, which - # bypasses the two-phase dry-run + token flow this lane requires. The - # remaining LTFS tests need a cartridge already formatted via that flow. + # Every LTFS test formats via FormatService.dry_run -> confirm(token), the + # two-phase dry-run + safety-token flow; none invokes mkltfs directly. - name: LTFS write / read / checksum - run: | - python -m pytest tests/hardware/test_ltfs_operations.py -m real_hardware -v --tb=short -rs \ - --deselect tests/hardware/test_ltfs_operations.py::test_mkltfs_formats_tape + run: python -m pytest tests/hardware/test_ltfs_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_ltfs_operations.xml - name: Archive + restore - run: python -m pytest tests/hardware/test_archive_restore.py -m real_hardware -v --tb=short -rs + run: python -m pytest tests/hardware/test_archive_restore.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_archive_restore.xml - name: Sharded write + restore - run: python -m pytest tests/hardware/test_sharded_operations.py -m real_hardware -v --tb=short -rs + run: python -m pytest tests/hardware/test_sharded_operations.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_sharded_operations.xml - name: Safe failure injection env: OPENBLADE_FAULT_TESTS: enabled - run: python -m pytest tests/hardware/test_fault_recovery.py -m real_hardware -v --tb=short -rs + run: python -m pytest tests/hardware/test_fault_recovery.py -m real_hardware -v --tb=short -rs --junitxml=reports/test_fault_recovery.xml + + # Runs only if every test step succeeded (no `if: always()`): a red test + # step already fails the job; this catches the green-by-skipping case. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate reports/*.xml # Reboot persistence is a MANUAL step (the runner cannot power-cycle the # appliance): see docs/runbooks/hardware-lanes.md "Reboot persistence". @@ -171,5 +179,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: hardware-destructive-${{ github.run_id }} - path: snapshots/ + path: | + snapshots/ + reports/ if-no-files-found: warn diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index f66b637..d2ba97d 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -23,10 +23,6 @@ on: permissions: contents: read -concurrency: - group: hardware-rig-i3 - cancel-in-progress: false - jobs: no-rig-notice: name: no-rig-notice @@ -46,10 +42,16 @@ jobs: name: register-and-validate if: vars.HARDWARE_RIG_AVAILABLE == 'true' runs-on: [self-hosted, i3] + # Job-level, so only a job that actually holds the rig takes the lock. + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false timeout-minutes: 30 env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would make the lane a no-op. + OPENBLADE_HARDWARE_DRY_RUN: "false" OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} @@ -98,6 +100,12 @@ jobs: fi python -m pytest "${args[@]}" + # Runs only after the suite succeeded (no `if: always()`): catches a + # green-by-skipping run. A -k filter deselects (absent from the XML), + # it does not skip, so it cannot trip this gate. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate readonly-results.xml + - name: Snapshot inventory + drive state (after) and diff if: always() run: | diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml index 7142d39..53bebfc 100644 --- a/.github/workflows/hardware-nightly-readonly.yml +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -5,9 +5,10 @@ name: Hardware nightly read-only # safety test in that directory rejects load/unload/move/format/mount calls). # The before/after snapshot diff is the runtime proof that nothing moved. # -# Without a rig (HARDWARE_RIG_AVAILABLE != 'true') the hardware job is SKIPPED, -# and the no-rig job says so in the run summary. A skipped lane is neutral, -# never green: it proves nothing about the appliance. +# Without a rig (HARDWARE_RIG_AVAILABLE != 'true') the hardware job is SKIPPED +# and the no-rig job writes a NOT RUN summary. The run then concludes +# *success* even though nothing was verified, so this lane must never be a +# required check or a badge (docs/runbooks/hardware-lanes.md). on: schedule: @@ -18,12 +19,6 @@ on: permissions: contents: read -concurrency: - # Shares the rig lock with the destructive lane and the smoke subset: a - # read-only pass must never observe a destructive run mid-sequence. - group: hardware-rig-i3 - cancel-in-progress: false - jobs: no-rig-notice: name: no-rig-notice @@ -44,10 +39,18 @@ jobs: name: hardware-readonly if: vars.HARDWARE_RIG_AVAILABLE == 'true' runs-on: [self-hosted, i3] + # Job-level, so only a job that actually holds the rig takes the lock. + # Shared with the destructive lane and the smoke subset: a read-only pass + # must never observe a destructive run mid-sequence. + concurrency: + group: hardware-rig-i3 + cancel-in-progress: false timeout-minutes: 60 env: OPENBLADE_BACKEND: real OPENBLADE_REAL_HARDWARE_ENABLED: "true" + # Pinned: a runner-host dry-run setting would make the lane a no-op. + OPENBLADE_HARDWARE_DRY_RUN: "false" OPENBLADE_APPLIANCE_URL: ${{ vars.OPENBLADE_APPLIANCE_URL }} OPENBLADE_APPLIANCE_USER: ${{ vars.OPENBLADE_APPLIANCE_USER }} OPENBLADE_APPLIANCE_PASSWORD: ${{ secrets.OPENBLADE_APPLIANCE_PASSWORD }} @@ -93,6 +96,12 @@ jobs: python -m pytest tests/hardware/readonly/ -m real_hardware -v --tb=short -rs \ --junitxml=readonly-results.xml + # Runs only after the suite succeeded (no `if: always()`): catches a + # green-by-skipping run. Add `--expected-skip classname::name` only with + # a comment saying why that skip is legitimate on the rig. + - name: Gate — fail on any skip or zero passes + run: python -m tools.hardware.junit_gate readonly-results.xml + - name: Snapshot inventory + drive state (after) and diff if: always() run: | diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 9bb38f7..92f62d7 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -316,7 +316,9 @@ jobs: SACRIFICIAL_BARCODES: ${{ vars.SACRIFICIAL_BARCODES || 'OB0007L8,OB0008L8' }} run: | set -euo pipefail - python -m tools.hardware.allowlist --barcodes "$OPENBLADE_SCRATCH_BARCODES" + # Same contract as the i3 lane: export the tool's normalized list. + normalized="$(python -m tools.hardware.allowlist --barcodes "$OPENBLADE_SCRATCH_BARCODES")" + echo "OPENBLADE_SCRATCH_BARCODES=$normalized" >> "$GITHUB_ENV" - name: Snapshot inventory + drive state (before) run: | diff --git a/docs/runbooks/hardware-lanes.md b/docs/runbooks/hardware-lanes.md index 6b59cc8..4adf692 100644 --- a/docs/runbooks/hardware-lanes.md +++ b/docs/runbooks/hardware-lanes.md @@ -14,7 +14,11 @@ self-hosted runner described below. All three physical lanes share the concurrency group `hardware-rig-i3` with `cancel-in-progress: false`: only one lane holds the rig at a time, and a running destructive sequence is never cancelled halfway (that can strand a -cartridge in a drive). +cartridge in a drive). The group is declared on the **rig job**, not the +workflow, so a destructive run waiting on environment approval does not hold +the rig slot, and a no-rig run never takes it. Every rig job also pins +`OPENBLADE_HARDWARE_DRY_RUN=false`, and `tools.hardware.snapshot capture` +refuses (exit 2) if the backend is in hardware dry-run mode. `hardware-library-smoke.yml` was kept rather than deleted so its workflow name and job id (`register-and-validate`) keep resolving for any branch-protection @@ -51,7 +55,10 @@ Repo → **Settings → Secrets and variables → Actions → Variables**: `OPENBLADE_DRIVE_DEVICES` are set in the runner host's environment (`.env` of the runner service). Every rig job has a **Preflight** step that fails if any of these is empty — otherwise the rig tests would skip and the lane would go - green without testing anything. + green without testing anything. As a second layer, every rig pytest step + writes JUnit XML and a final **Gate** step (`python -m tools.hardware.junit_gate`) + fails the job if any test skipped or nothing passed. An expected skip must be + passed explicitly (`--expected-skip classname::name`) with a comment saying why. ## 3. Environment approval (settings-only — cannot be committed) @@ -104,6 +111,12 @@ the pre-reboot `after.json` shows the same cartridges in the same slots. Without `HARDWARE_RIG_AVAILABLE=true`, the hardware job shows as **Skipped** and a small `no-rig-notice` job writes "NOT RUN (neutral)" to the run summary. +Because `no-rig-notice` succeeds, **the workflow run concludes `success`** — +GitHub shows it with a green check even though nothing touched the appliance. +That is why these lanes must **never be required status checks or README +badges**: a no-rig run would satisfy the check and colour the badge green. Read +the run summary (`NOT RUN`) or the rig job's own result, never the run +conclusion. **No lane may be reported or treated as green because it skipped.** A skipped hardware job proves nothing about the appliance. Do not add it as a required From 51e19e3b5b3edbcfb0313ad4b50c7f1c5efa1c75 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:19:49 +0000 Subject: [PATCH 062/205] fix(nas): render Ganesha Clients unquoted so EXPORTs load Ganesha rejected Clients = "*" (Expected a client) and dropped every EXPORT, so nfs4 mounts of /ro and /rw failed with ENOENT. Adds non-docker unit tests for the SMB/NFS renderers; the regression test was mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/nfs.py | 4 ++- tests/test_nas_protocol_render.py | 52 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/test_nas_protocol_render.py diff --git a/openblade/nas/nfs.py b/openblade/nas/nfs.py index 8611ba6..60ff0f6 100644 --- a/openblade/nas/nfs.py +++ b/openblade/nas/nfs.py @@ -31,7 +31,9 @@ def render_ganesha(self) -> str: f" Access_Type = {access};\n" " Squash = No_Root_Squash;\n" " SecType = sys;\n" - f' CLIENT {{ Clients = "{self.client}"; Access_Type = {access}; }}\n' + # Ganesha parses Clients as a bare client-list token; a quoted + # "*" is rejected ("Expected a client") and drops the whole EXPORT. + f" CLIENT {{ Clients = {self.client}; Access_Type = {access}; }}\n" " FSAL { Name = VFS; }\n" "}\n" ) diff --git a/tests/test_nas_protocol_render.py b/tests/test_nas_protocol_render.py new file mode 100644 index 0000000..7251ef4 --- /dev/null +++ b/tests/test_nas_protocol_render.py @@ -0,0 +1,52 @@ +"""Unit tests for the SMB/NFS config renderers (no docker; not marked protocols).""" + +from pathlib import Path + +from openblade.nas.nfs import NfsExport, render_exports, render_ganesha_conf +from openblade.nas.samba import SambaShare, render_smb_conf + + +def test_ganesha_client_list_is_unquoted() -> None: + # Regression: Ganesha rejects `Clients = "*"` ("Expected a client") and then + # silently drops the whole EXPORT, so NFS mounts fail with ENOENT. + block = NfsExport(Path("/rig/export/ro"), pseudo="/ro").render_ganesha() + + assert "Clients = *;" in block + assert '"*"' not in block + + +def test_ganesha_conf_renders_one_export_per_share_with_access_type() -> None: + conf = render_ganesha_conf( + [ + NfsExport(Path("/e/ro"), export_id=1, pseudo="/ro"), + NfsExport(Path("/e/rw"), export_id=2, pseudo="/rw", read_only=False), + ] + ) + + assert conf.count("EXPORT {") == 2 + assert 'Pseudo = "/ro";' in conf and "Access_Type = RO;" in conf + assert 'Pseudo = "/rw";' in conf and "Access_Type = RW;" in conf + assert "Protocols = 4;" in conf + + +def test_ganesha_pseudo_defaults_to_path() -> None: + assert 'Pseudo = "/data/x";' in NfsExport(Path("/data/x")).render_ganesha() + + +def test_kernel_exports_line_reflects_read_only() -> None: + out = render_exports([NfsExport(Path("/a")), NfsExport(Path("/b"), read_only=False)]) + + assert out == "/a *(ro,sync,no_subtree_check)\n/b *(rw,sync,no_subtree_check)\n" + + +def test_smb_conf_maps_share_flags() -> None: + conf = render_smb_conf( + [ + SambaShare("ro", Path("/e/ro")), + SambaShare("rw", Path("/e/rw"), read_only=False, case_sensitive=True), + ] + ) + + ro, rw = conf.split("[ro]", 1)[1].split("[rw]", 1) + assert "writeable = no" in ro and "case sensitive = no" in ro + assert "writeable = yes" in rw and "case sensitive = yes" in rw From 68a5d4091b89361d7ba4b6bc98fd6a748569ad64 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:19:49 +0000 Subject: [PATCH 063/205] fix(e2e): stage guest-writable .ob-control/offline for recall requests Clients could not create .req files (ENOENT: the shim only creates a root-owned .ob-control), so no hydration scenario ever reached the shim. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/rig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/protocols/rig.py b/tests/e2e/protocols/rig.py index 2d6af15..5a31309 100644 --- a/tests/e2e/protocols/rig.py +++ b/tests/e2e/protocols/rig.py @@ -92,6 +92,9 @@ def prepare() -> None: (deep / "leaf.txt").write_text("deep") (rw / "offline").mkdir(parents=True) (rw / ".ob-tape" / "offline").mkdir(parents=True) + # Clients drop recall requests here; must exist and be guest-writable before + # the shim starts (it only creates .ob-control itself, root-owned 0755). + (rw / ".ob-control" / "offline").mkdir(parents=True) (ro / "readme.txt").write_text("hello openblade\n") (ro / "data.bin").write_bytes(PAYLOAD) (ro / UNICODE_NAME).write_text("unicode") From 135a92ee7cecb1c7a44425be1204ea7958dc94c7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:19:49 +0000 Subject: [PATCH 064/205] test(e2e): assert smbcacls owner/group/everyone ACEs for mode 0640 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/test_smb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/e2e/protocols/test_smb.py b/tests/e2e/protocols/test_smb.py index 2a6cd75..bd5f1ce 100644 --- a/tests/e2e/protocols/test_smb.py +++ b/tests/e2e/protocols/test_smb.py @@ -27,8 +27,14 @@ def test_smb_case_sensitive_share_keeps_distinct_names(mounted: Rig) -> None: def test_smb_acl_maps_posix_mode(mounted: Rig) -> None: res = mounted.sh("smbcacls -N //samba/ro mode640.txt") assert res.returncode == 0, res.stderr - assert "ACL:Everyone:ALLOWED/0x0/0x0" not in res.stdout or "Unix Group" in res.stdout - assert "Unix User\\root:ALLOWED/0x0/RW" in res.stdout, res.stdout + # smbcacls prints "ACL::ALLOWED//". Mode 0640 must map + # to owner=RW, group=R, Everyone=no rights. The owner's name is the server's + # netbios name (container id), so match on the OWNER: line rather than a literal. + lines = res.stdout.splitlines() + owner = next(line.split(":", 1)[1] for line in lines if line.startswith("OWNER:")) + assert f"ACL:{owner}:ALLOWED/0x0/RW" in lines, res.stdout + assert "ACL:Unix Group\\root:ALLOWED/0x0/R" in lines, res.stdout + assert "ACL:Everyone:ALLOWED/0x0/" in lines, res.stdout def test_smb_byte_range_lock_conflicts_across_sessions(mounted: Rig) -> None: From 1cce6d3c1523cfeaa7b6903782cf15e7dbabe5a5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:19:49 +0000 Subject: [PATCH 065/205] docs(deploy): note the catalog has no on-disk OFFLINE_ON_TAPE form Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/hydrator_shim.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/nas-protocols/hydrator_shim.py b/deploy/nas-protocols/hydrator_shim.py index 6754aac..90c9fbf 100644 --- a/deploy/nas-protocols/hydrator_shim.py +++ b/deploy/nas-protocols/hydrator_shim.py @@ -12,6 +12,15 @@ "tape_error" -> write `.ob-control/.err` = OFFLINE_TAPE_UNAVAILABLE. Each recall is logged once to `.ob-control/hydration.log`; requests for a path already in flight are coalesced (one hydration event per path). + +The stub format is rig-only. The catalog has NO on-disk form for +NasFileState.OFFLINE_ON_TAPE (openblade/nas/types.py): no stub, sidecar or +xattr. An offline file exists only as a catalog NasFileRecord. /virtual reports +it with the catalog size and status, and a read raises CartridgeOfflineError. +The FUSE mount shows it read-only (0444) at catalog size. A read returns EIO, or +with --hydrate blocks until restore. So this stub only stands in for "offline" +when there is no FUSE layer. Its content, unlike FUSE, does not keep the +catalog size. """ import json From 6e9d9581de3924cd3a90438e103406629b15699b Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:22:08 +0000 Subject: [PATCH 066/205] sharded_archive: stage instances, one bulk commit, journal physical_state_unknown Shard instances are created STAGING before their first write, moved to VERIFYING once every lane read back its shard, and archived by ONE mark_instances_archived after clean unmount + inventory reconcile. Cleanup no longer suppresses exceptions: each failure is logged and journaled as physical_state_unknown and the other drives are still cleaned. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/sharded_archive.py | 312 +++++++++++++++++++----------- 1 file changed, 197 insertions(+), 115 deletions(-) diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index a951975..25ec6d6 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -7,7 +7,6 @@ import os import shutil import uuid -from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -16,6 +15,7 @@ from openblade.domain.capacity import has_room_for from openblade.domain.errors import StaleLeaseError, TapeFullError, safe_job_error from openblade.domain.models import MountHandle, MountMode +from openblade.jobs.inventory import InventoryService from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import ( DEFAULT_BLOCK_SIZE, @@ -214,7 +214,7 @@ def run_sharded_archive( files_archived += 1 except Exception as exc: # noqa: BLE001 logger.exception("Shard archive failed for %s", source_file) - errors.append(safe_job_error(exc)) + errors.append(_job_error(exc)) else: files_archived, bytes_archived = _archive_stripe( files, @@ -323,6 +323,36 @@ def _log_fenced_out(handles: list[DriveHandle], job_id: str) -> None: ) +PHYSICAL_STATE_UNKNOWN = "physical state unknown — reconcile before reuse" + + +def _job_error(exc: Exception) -> str: + """jobs.error text: the typed physical-state message, else safe_job_error.""" + if isinstance(exc, TapePhysicalStateError): + return PHYSICAL_STATE_UNKNOWN + return safe_job_error(exc) + + +def record_physical_state_unknown( + catalog: CatalogRepository, job_id: str, op: str, barcode: str, drive: int | None +) -> None: + """Log + journal a cleanup op that raised; the drive must be reconciled before reuse.""" + logger.exception( + "job %s: %s of %s (drive %s) failed during cleanup; %s", + job_id, + op, + barcode, + drive, + PHYSICAL_STATE_UNKNOWN, + ) + try: + catalog.journal( + job_id, "physical_state_unknown", {"op": op, "barcode": barcode, "drive": drive} + ) + except Exception: # noqa: BLE001 - journal is evidence, must not stop cleanup of other drives + logger.exception("job %s: could not journal physical_state_unknown", job_id) + + def _best_effort_unmount_and_unload( catalog: CatalogRepository, library: LibraryBackend, @@ -331,28 +361,75 @@ def _best_effort_unmount_and_unload( handles: list[DriveHandle], loaded_slots: dict[int, int | None], job_id: str, + errors: list[str] | None, ) -> None: - """Error-path cleanup for a job that still owns its drives: unmount, then unload.""" - for mount in mounts.values(): - with suppress(Exception): + """Error-path cleanup for a job that still owns its drives: unmount, then unload. + + Every op is attempted even if an earlier one raised. A failure is logged and + journaled as ``physical_state_unknown``; it becomes the job error only when + ``errors`` is empty (the original lane error wins). ``errors=None`` means an + exception is already propagating and will become the job error. + """ + drive_by_barcode = {handle.barcode: handle.physical for handle in handles} + failed = False + for barcode, mount in mounts.items(): + try: ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues + failed = True + record_physical_state_unknown( + catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode) + ) for handle in handles: slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: - with suppress(Exception): - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=handle.barcode, - drive_id=handle.physical, - slot_id=slot_id, - requested_by="sharded-archive", - job_id=job_id, - ), - ) + if slot_id is None: + continue + try: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=handle.barcode, + drive_id=handle.physical, + slot_id=slot_id, + requested_by="sharded-archive", + job_id=job_id, + ), + ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues + failed = True + record_physical_state_unknown( + catalog, job_id, "unload", handle.barcode, handle.physical + ) + if failed and errors is not None and not errors: + errors.append(PHYSICAL_STATE_UNKNOWN) + + +def _mark_verifying(catalog: CatalogRepository, job_id: str, staged_ids: list[str]) -> None: + catalog.journal(job_id, "verify_started", {"instances": len(staged_ids)}) + catalog.mark_instances_verifying(staged_ids) + catalog.journal(job_id, "verify_finished", {"instances": len(staged_ids)}) + + +def _commit_staged( + catalog: CatalogRepository, + library: LibraryBackend, + job_id: str, + handles: list[DriveHandle], + staged_ids: list[str], +) -> None: + """The atomic commit: inventory reconciled, then ONE mark_instances_archived.""" + inventory = InventoryService(library).snapshot() # SAFETY_003: via service + seen = {str(slot.barcode) for slot in inventory.slots if slot.barcode is not None} + seen |= {str(drive.barcode) for drive in inventory.drives if drive.barcode is not None} + missing = sorted({handle.barcode for handle in handles} - seen) + if missing: + logger.error("job %s: barcodes %s not in any slot or drive after unload", job_id, missing) + raise TapePhysicalStateError(f"inventory does not show {', '.join(missing)}") + catalog.mark_instances_archived(staged_ids) + catalog.journal(job_id, "committed", {"instances": len(staged_ids)}) def _archive_stripe( @@ -385,12 +462,14 @@ def _archive_stripe( for batch in batches: batch_barcodes = list(dict.fromkeys(barcode for _, barcode in batch)) handles = scheduler.acquire_drives(batch_barcodes) + catalog.journal(job_id, "lease_acquired", {"barcodes": batch_barcodes}) mounts: dict[str, MountHandle] = {} loaded_slots: dict[int, int | None] = {} - # (main_instance_id, shard_instance_id, size_bytes) for shards written+verified - # this batch. Nothing here is marked archived until the WHOLE batch has verified - # and every tape has cleanly unmounted — the atomic commit boundary. - staged: list[tuple[str, str, int]] = [] + # Instances are created STAGING before their first write and stay invisible + # until the WHOLE batch has verified, every tape cleanly unmounted and the + # inventory reconciled — then ONE mark_instances_archived (atomic commit). + staged_ids: list[str] = [] + batch_bytes = 0 fenced_out = False try: for handle in handles: @@ -405,60 +484,67 @@ def _write_one( source_file: Path, barcode: str, mount: MountHandle, - ) -> tuple[Path, str, str, str, int]: - tape_path = _stripe_tape_path(source_file, request.source_path) + tape_path: str, + checksum: str, + ) -> None: _require_lane_room(ltfs, barcode, source_file.stat().st_size) - checksum = compute_checksum(source_file) ltfs.write_file(mount, source_file, PurePosixPath(tape_path)) stat = ltfs.stat(mount, PurePosixPath(tape_path)) if stat.checksum_sha256 != checksum: raise ValueError(f"Checksum mismatch: {source_file.name}") - return source_file, barcode, tape_path, checksum, source_file.stat().st_size + + # Stage every instance (main thread: the catalog session is not + # thread-safe) BEFORE the first write, so a crash mid-write leaves + # STAGING rows that recovery can find, never ARCHIVED ones. + planned: list[tuple[Path, str, str, str]] = [] + for source_file, barcode in batch: + tape_path = _stripe_tape_path(source_file, request.source_path) + checksum = compute_checksum(source_file) + size_bytes = source_file.stat().st_size + file_record = catalog.create_file_record( + path=str(source_file), + size_bytes=size_bytes, + checksum=checksum, + vg_id=vg_id, + shard_count=1, + shard_index=None, + block_size=None, + shard_profile=_archive_profile(request.mode), + parent_id=None, + ) + instance = catalog.create_staged_instance( + job_id, file_record.id, barcode, tape_path + ) + shard_record = catalog.create_file_record( + path=_shard_record_path(source_file, 0), + size_bytes=size_bytes, + checksum=checksum, + vg_id=vg_id, + shard_count=1, + shard_index=0, + block_size=None, + shard_profile=_archive_profile(request.mode), + parent_id=file_record.id, + ) + shard_instance = catalog.create_staged_instance( + job_id, shard_record.id, barcode, tape_path + ) + staged_ids.extend((instance.id, shard_instance.id)) + batch_bytes += size_bytes + planned.append((source_file, barcode, tape_path, checksum)) + shard_group_ids.append(str(uuid.uuid4())) for handle in handles: scheduler.verify(handle) # fencing: before the shard write batch with concurrent.futures.ThreadPoolExecutor(max_workers=len(batch)) as pool: futures = [ - pool.submit(_write_one, source_file, barcode, mounts[barcode]) - for source_file, barcode in batch + pool.submit(_write_one, source_file, barcode, mounts[barcode], path, csum) + for source_file, barcode, path, csum in planned ] for future in concurrent.futures.as_completed(futures): - source_file, barcode, tape_path, checksum, size_bytes = future.result() - file_record = catalog.create_file_record( - path=str(source_file), - size_bytes=size_bytes, - checksum=checksum, - vg_id=vg_id, - shard_count=1, - shard_index=None, - block_size=None, - shard_profile=_archive_profile(request.mode), - parent_id=None, - ) - instance = catalog.create_file_instance( - file_record_id=file_record.id, - barcode=barcode, - tape_path=tape_path, - ) - shard_record = catalog.create_file_record( - path=_shard_record_path(source_file, 0), - size_bytes=size_bytes, - checksum=checksum, - vg_id=vg_id, - shard_count=1, - shard_index=0, - block_size=None, - shard_profile=_archive_profile(request.mode), - parent_id=file_record.id, - ) - shard_instance = catalog.create_file_instance( - file_record_id=shard_record.id, - barcode=barcode, - tape_path=tape_path, - ) - # Created PENDING; not archived yet. - staged.append((instance.id, shard_instance.id, size_bytes)) - shard_group_ids.append(str(uuid.uuid4())) + future.result() + # Every lane wrote and read back (stat checksum) its shard. + _mark_verifying(catalog, job_id, staged_ids) # Every shard in the batch wrote and checksum-verified. Cleanly unmount and # unload BEFORE commit; a failed unmount raises and blocks the commit. @@ -466,19 +552,16 @@ def _write_one( for handle in handles: scheduler.verify(handle) # fencing: before unmount/unload + commit _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) - - # COMMIT: only now is the batch durable. - for instance_id, shard_instance_id, size_bytes in staged: - catalog.mark_instance_archived(instance_id) - catalog.mark_instance_archived(shard_instance_id) - files_archived += 1 - bytes_archived += size_bytes + _commit_staged(catalog, library, job_id, handles, staged_ids) + files_archived += len(planned) + bytes_archived += batch_bytes except StaleLeaseError: # Fenced out: abort the whole job and leave the hardware ALONE. The # drive may already be loaded + mounted by whoever holds the lease now, # so unmount/unload here could eject their tape mid-write. Physical # state is unknown -> reconcile (recovery report / inventory). fenced_out = True + catalog.journal(job_id, "fenced_out", {"barcodes": batch_barcodes}) raise except Exception as exc: # noqa: BLE001 # Staged instances remain PENDING -> not exposed as archived; resumable. @@ -491,13 +574,14 @@ def _write_one( exc, exc_info=True, ) - errors.append(safe_job_error(exc)) + catalog.journal(job_id, "failed", {"error": type(exc).__name__}) + errors.append(_job_error(exc)) finally: if fenced_out: _log_fenced_out(handles, job_id) else: _best_effort_unmount_and_unload( - catalog, library, ltfs, mounts, handles, loaded_slots, job_id + catalog, library, ltfs, mounts, handles, loaded_slots, job_id, errors ) scheduler.release_drives(handles) @@ -524,6 +608,7 @@ def _archive_block_stripe( ) shard_group_ids.append(plan.shard_group_id) handles = scheduler.acquire_drives(request.lane_barcodes) + catalog.journal(job_id, "lease_acquired", {"barcodes": list(request.lane_barcodes)}) mounts: dict[str, MountHandle] = {} loaded_slots: dict[int, int | None] = {} shard_dir = scratch_dir / plan.shard_group_id @@ -556,30 +641,7 @@ def _archive_block_stripe( shard_index = future_map[future] shard_tmp_files[shard_index] = future.result() - def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: - _require_lane_room(ltfs, spec.barcode, shard_tmp.stat().st_size) - checksum = compute_checksum(shard_tmp) - mount = mounts[spec.barcode] - ltfs.write_file(mount, shard_tmp, PurePosixPath(spec.tape_path)) - stat = ltfs.stat(mount, PurePosixPath(spec.tape_path)) - if stat.checksum_sha256 != checksum: - raise ValueError(f"Shard {spec.shard_index} checksum mismatch") - return spec.shard_index, checksum, shard_tmp.stat().st_size - - for handle in handles: - scheduler.verify(handle) # fencing: before the shard write batch - shard_checksums: list[str] = [""] * len(plan.shards) - shard_sizes: list[int] = [0] * len(plan.shards) - with concurrent.futures.ThreadPoolExecutor(max_workers=len(plan.shards)) as pool: - futures = [ - pool.submit(_write_shard, spec, shard_tmp_files[spec.shard_index]) - for spec in plan.shards - ] - for shard_future in concurrent.futures.as_completed(futures): - shard_index, checksum, shard_size = shard_future.result() - shard_checksums[shard_index] = checksum - shard_sizes[shard_index] = shard_size - + shard_checksums = [compute_checksum(path) for path in shard_tmp_files] file_record = catalog.create_file_record( path=str(source_file), size_bytes=plan.file_size, @@ -591,18 +653,17 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: shard_profile=_archive_profile(request.mode), parent_id=None, ) - # All shards verified above. Create records/instances PENDING; do not mark - # archived until every tape has cleanly unmounted (atomic commit boundary). + # Stage records/instances STAGING before the first shard write (main thread: + # the catalog session is not thread-safe). Nothing is ARCHIVED until every + # lane verified, every tape cleanly unmounted and inventory reconciled. staged: list[str] = [] for spec in plan.shards: - instance = catalog.create_file_instance( - file_record_id=file_record.id, - barcode=spec.barcode, - tape_path=spec.tape_path, + instance = catalog.create_staged_instance( + job_id, file_record.id, spec.barcode, spec.tape_path ) shard_record = catalog.create_file_record( path=_shard_record_path(source_file, spec.shard_index), - size_bytes=shard_sizes[spec.shard_index], + size_bytes=shard_tmp_files[spec.shard_index].stat().st_size, checksum=shard_checksums[spec.shard_index], vg_id=vg_id, shard_count=len(plan.shards), @@ -611,32 +672,53 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: shard_profile=_archive_profile(request.mode), parent_id=file_record.id, ) - shard_instance = catalog.create_file_instance( - file_record_id=shard_record.id, - barcode=spec.barcode, - tape_path=spec.tape_path, + shard_instance = catalog.create_staged_instance( + job_id, shard_record.id, spec.barcode, spec.tape_path ) staged.extend((instance.id, shard_instance.id)) + def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: + _require_lane_room(ltfs, spec.barcode, shard_tmp.stat().st_size) + checksum = shard_checksums[spec.shard_index] + mount = mounts[spec.barcode] + ltfs.write_file(mount, shard_tmp, PurePosixPath(spec.tape_path)) + stat = ltfs.stat(mount, PurePosixPath(spec.tape_path)) + if stat.checksum_sha256 != checksum: + raise ValueError(f"Shard {spec.shard_index} checksum mismatch") + return spec.shard_index, checksum, shard_tmp.stat().st_size + + for handle in handles: + scheduler.verify(handle) # fencing: before the shard write batch + with concurrent.futures.ThreadPoolExecutor(max_workers=len(plan.shards)) as pool: + futures = [ + pool.submit(_write_shard, spec, shard_tmp_files[spec.shard_index]) + for spec in plan.shards + ] + for shard_future in concurrent.futures.as_completed(futures): + shard_future.result() + # Every lane wrote and read back (stat checksum) its shard. + _mark_verifying(catalog, job_id, staged) + scheduler.heartbeat(handles) for handle in handles: scheduler.verify(handle) # fencing: before unmount/unload + commit # Clean unmount/unload BEFORE commit; a failed unmount raises and blocks it. _clean_unmount_and_unload(catalog, library, ltfs, mounts, handles, loaded_slots, job_id) - - # COMMIT: block-striped file is durable only when all shards are down. - for instance_id in staged: - catalog.mark_instance_archived(instance_id) + _commit_staged(catalog, library, job_id, handles, staged) except StaleLeaseError: # Fenced out: abort and leave the hardware alone (see _archive_stripe). fenced_out = True + catalog.journal(job_id, "fenced_out", {"barcodes": list(request.lane_barcodes)}) + raise + except Exception as exc: + catalog.journal(job_id, "failed", {"error": type(exc).__name__}) raise finally: if fenced_out: _log_fenced_out(handles, job_id) else: _best_effort_unmount_and_unload( - catalog, library, ltfs, mounts, handles, loaded_slots, job_id + catalog, library, ltfs, mounts, handles, loaded_slots, job_id, None ) scheduler.release_drives(handles) shutil.rmtree(shard_dir, ignore_errors=True) From a128e022ea68e1314d06e576f68cd5fb2624c6bd Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:23:39 +0000 Subject: [PATCH 067/205] feat(jobs): take non-sharded archive/restore drives through DriveScheduler leases Acquire via scheduler.acquire_drives, verify before mount and before unmount/unload, release in finally, journal lease_acquired/released. StaleLeaseError leaves hardware untouched (sharded fenced_out pattern). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/archive.py | 114 +++++++++++++++++++++++++++++--------- openblade/jobs/restore.py | 113 +++++++++++++++++++++++++++++-------- 2 files changed, 177 insertions(+), 50 deletions(-) diff --git a/openblade/jobs/archive.py b/openblade/jobs/archive.py index ecc424a..c29d9aa 100644 --- a/openblade/jobs/archive.py +++ b/openblade/jobs/archive.py @@ -12,9 +12,16 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.capacity import has_room_for -from openblade.domain.errors import ChecksumMismatchError, NoScratchMediaError, safe_job_error +from openblade.domain.errors import ( + ChecksumMismatchError, + NoScratchMediaError, + StaleLeaseError, + safe_job_error, +) from openblade.domain.models import JobType, MountMode +from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue +from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import TapeOperationFailedError, execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -158,37 +165,40 @@ def _load_if_needed( catalog: CatalogRepository, library: LibraryBackend, ltfs: LTFSBackend, - barcode: str, + handle: DriveHandle, job_id: str, ) -> tuple[int, int | None]: + barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: + # Already in a drive: record it as the PHYSICAL drive; the lease key + # (handle.drive_id) stays what the scheduler reserved. + if drive_id != handle.drive_id: + handle.physical_drive_id = drive_id return drive_id, None slot_id = library.find_slot_by_barcode(barcode) if slot_id is None: raise NoScratchMediaError(f"Barcode {barcode} not found in inventory") - for drive in library.inventory().drives: - if drive.barcode is not None: - continue - try: - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.LOAD, - barcode=barcode, - drive_id=drive.drive_id, - slot_id=slot_id, - requested_by="archive-job", - job_id=job_id, - ), - raise_on_failed=True, - ) - return drive.drive_id, slot_id - except TapeOperationFailedError: - continue - raise NoScratchMediaError("No available drives for archive load") + try: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.LOAD, + barcode=barcode, + drive_id=handle.drive_id, + slot_id=slot_id, + requested_by="archive-job", + job_id=job_id, + ), + raise_on_failed=True, + ) + except TapeOperationFailedError as exc: + raise NoScratchMediaError( + f"Leased drive {handle.drive_id} could not load {barcode}" + ) from exc + return handle.drive_id, slot_id def run_archive_job( @@ -197,8 +207,16 @@ def run_archive_job( ltfs: LTFSBackend, catalog: CatalogRepository, job_id: str, + *, + scheduler: DriveScheduler | None = None, ) -> ArchiveResult: - """Archive source files onto LTFS media and record them in the catalog.""" + """Archive source files onto LTFS media and record them in the catalog. + + Each tape's drive is taken through ``scheduler`` (a lease shared with every + other job on the same store). ``None`` falls back to a job-local in-memory + scheduler, which serialises nothing across jobs -- callers that can reach the + app's lease store must pass one. + """ catalog.update_job_state(job_id, "running") files = _iter_source_files(request.source_path) volume_group = catalog.get_volume_group(request.volume_group_name) @@ -215,6 +233,12 @@ def run_archive_job( catalog.update_job_state(job_id, "completed") return result + if scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + leases = scheduler + current_lease: DriveHandle | None = None current_barcode: str | None = None current_handle = None current_slot_id: int | None = None @@ -233,10 +257,13 @@ def finalize_current_mount() -> None: pending_instance_ids if current_handle is None or current_barcode is None or current_drive_id is None: return + assert current_lease is not None + leases.verify(current_lease) # fencing: before unmount ltfs.unmount(current_handle) for instance_id in pending_instance_ids: catalog.mark_instance_archived(instance_id, checksum_verified=True) pending_instance_ids = [] + leases.verify(current_lease) # fencing: before unload if current_slot_id is not None: execute_tape_request( catalog, @@ -263,6 +290,18 @@ def finalize_current_mount() -> None: current_handle = None current_slot_id = None current_drive_id = None + release_current_lease() + + def release_current_lease() -> None: + nonlocal current_lease + if current_lease is None: + return + lease = current_lease + current_lease = None + leases.release_drives([lease]) + catalog.journal( + job_id, "lease_released", {"drive_id": lease.drive_id, "barcode": lease.barcode} + ) try: for file_path in files: @@ -278,14 +317,22 @@ def finalize_current_mount() -> None: selected_barcode = _choose_tape(catalog, library, ltfs, volume_group.id, size_bytes) if selected_barcode != current_barcode: finalize_current_mount() + current_lease = leases.acquire_drives([selected_barcode])[0] + catalog.journal( + job_id, + "lease_acquired", + {"drive_id": current_lease.drive_id, "barcode": current_lease.barcode}, + ) current_drive_id, current_slot_id = _load_if_needed( catalog, library, ltfs, - selected_barcode, + current_lease, job_id, ) + leases.record_physical_drives([current_lease]) _mark_aml_drive_busy(selected_barcode, current_drive_id) + leases.verify(current_lease) # fencing: never mount RW on a stale lease current_handle = ltfs.mount(selected_barcode, MountMode.READ_WRITE) current_barcode = selected_barcode tapes_used.append(selected_barcode) @@ -310,6 +357,19 @@ def finalize_current_mount() -> None: bytes_archived += size_bytes files_archived += 1 finalize_current_mount() + except StaleLeaseError as exc: + # Fenced out: leave the hardware ALONE -- whoever holds the lease now may + # have their own tape in that drive. Pending instances stay unarchived. + lost = current_lease + logger.error( + "job %s lost its lease on drive %s (%s); leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + lost.physical if lost is not None else "?", + current_barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise except Exception as exc: if current_handle is not None: try: @@ -340,6 +400,8 @@ def finalize_current_mount() -> None: logger.exception("failed to reset archive AML drive state for job %s", job_id) catalog.update_job_state(job_id, "failed", safe_job_error(exc)) raise + finally: + release_current_lease() result = ArchiveResult( job_id=job_id, diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index 12a3ff6..dee505f 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -10,13 +11,22 @@ from openblade.catalog.models import Job from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend -from openblade.domain.errors import CartridgeOfflineError, ChecksumMismatchError +from openblade.domain.errors import ( + CartridgeOfflineError, + ChecksumMismatchError, + StaleLeaseError, + safe_job_error, +) from openblade.domain.models import JobType, MountMode +from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue +from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType +logger = logging.getLogger(__name__) + @dataclass class RestoreRequest: @@ -89,16 +99,21 @@ def _load_if_needed( catalog: CatalogRepository, library: LibraryBackend, ltfs: LTFSBackend, - barcode: str, + handle: DriveHandle, job_id: str, ) -> tuple[int, int | None]: + barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: + # Already in a drive: record it as the PHYSICAL drive; the lease key + # (handle.drive_id) stays what the scheduler reserved. + if drive_id != handle.drive_id: + handle.physical_drive_id = drive_id return drive_id, None slot_id = library.find_slot_by_barcode(barcode) if slot_id is None: raise CartridgeOfflineError(f"Cartridge {barcode} is offline") - drive_id = 0 + drive_id = handle.drive_id execute_tape_request( catalog, library, @@ -122,8 +137,16 @@ def run_restore_job( ltfs: LTFSBackend, catalog: CatalogRepository, job_id: str, + *, + scheduler: DriveScheduler | None = None, ) -> RestoreResult: - """Restore a cataloged file from tape to a local path.""" + """Restore a cataloged file from tape to a local path. + + The drive is taken through ``scheduler`` (a lease shared with every other job + on the same store). ``None`` falls back to a job-local in-memory scheduler, + which serialises nothing across jobs -- callers that can reach the app's + lease store must pass one. + """ catalog.update_job_state(job_id, "running") record, instance = catalog.get_latest_instance_for_path(request.catalog_path) cartridge = catalog.get_cartridge(instance.barcode) @@ -135,35 +158,77 @@ def run_restore_job( return RestoreResult( job_id=job_id, source_barcode=instance.barcode, checksum_verified=False ) - drive_id, slot_id = _load_if_needed(catalog, library, ltfs, instance.barcode, job_id) - _mark_aml_drive_busy(instance.barcode, drive_id) + if scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + handles = scheduler.acquire_drives([instance.barcode]) + handle = handles[0] + catalog.journal( + job_id, "lease_acquired", {"drive_id": handle.drive_id, "barcode": handle.barcode} + ) final_dest = ( request.dest_path / PurePosixPath(request.catalog_path).name if request.dest_path.exists() and request.dest_path.is_dir() else request.dest_path ) + drive_id: int | None = None + slot_id: int | None = None + fenced_out = False try: - handle = ltfs.mount(instance.barcode, MountMode.READ_ONLY) try: - ltfs.read_file(handle, PurePosixPath(instance.tape_path), final_dest) + drive_id, slot_id = _load_if_needed(catalog, library, ltfs, handle, job_id) + scheduler.record_physical_drives(handles) + _mark_aml_drive_busy(instance.barcode, drive_id) + scheduler.verify(handle) # fencing: never mount on a stale lease + mount_handle = ltfs.mount(instance.barcode, MountMode.READ_ONLY) + try: + ltfs.read_file(mount_handle, PurePosixPath(instance.tape_path), final_dest) + finally: + scheduler.verify(handle) # fencing: before unmount + ltfs.unmount(mount_handle) + except StaleLeaseError: + fenced_out = True + raise finally: - ltfs.unmount(handle) + if not fenced_out and drive_id is not None: + try: + scheduler.verify(handle) # fencing: before unload + except StaleLeaseError: + fenced_out = True + raise + if slot_id is not None: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=instance.barcode, + drive_id=drive_id, + slot_id=slot_id, + requested_by="restore-job", + job_id=job_id, + ), + ) + _mark_aml_drive_idle(instance.barcode, drive_id, slot_id) + except StaleLeaseError as exc: + # Fenced out: leave the hardware ALONE -- whoever holds the lease now may + # have their own tape in that drive. Physical state is unknown -> reconcile. + logger.error( + "job %s lost its lease on drive %d (%s); leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + handle.physical, + handle.barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise finally: - if slot_id is not None: - execute_tape_request( - catalog, - library, - ltfs, - TapeOpRequest( - op_type=TapeOpType.UNLOAD, - barcode=instance.barcode, - drive_id=drive_id, - slot_id=slot_id, - requested_by="restore-job", - job_id=job_id, - ), - ) - _mark_aml_drive_idle(instance.barcode, drive_id, slot_id) + scheduler.release_drives(handles) + catalog.journal( + job_id, "lease_released", {"drive_id": handle.drive_id, "barcode": handle.barcode} + ) actual_checksum = sha256sum(final_dest) if actual_checksum != record.checksum_sha256: quarantine = final_dest.with_name(f"{final_dest.name}.quarantine") From 579628e39972e20e6cfcf601b105aea860f9e272 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:23:40 +0000 Subject: [PATCH 068/205] feat(api,cli): pass the shared lease-store scheduler to run_archive_job/run_restore_job Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_archive.py | 6 ++++++ openblade/api/routes_restore.py | 6 ++++++ openblade/cli/main.py | 1 + 3 files changed, 13 insertions(+) diff --git a/openblade/api/routes_archive.py b/openblade/api/routes_archive.py index aa027be..852dd8e 100644 --- a/openblade/api/routes_archive.py +++ b/openblade/api/routes_archive.py @@ -15,6 +15,7 @@ from openblade.domain.errors import FileNotFoundError from openblade.jobs.archive import ArchiveRequest as ArchiveJobRequest from openblade.jobs.archive import run_archive_job +from openblade.jobs.inventory import InventoryService from openblade.jobs.scheduler import DriveScheduler from openblade.jobs.shard import ShardMode from openblade.jobs.sharded_archive import ShardedArchiveRequest, run_sharded_archive @@ -140,6 +141,11 @@ async def enqueue_archive( context.ltfs, context.catalog, job.id, + scheduler=DriveScheduler( + num_drives=len(InventoryService(context.library).snapshot().drives), + store=context.lease_store, + job_id=job.id, + ), ) except Exception as exc: _cleanup_failed_archive(context, source_path, request.volume_group) diff --git a/openblade/api/routes_restore.py b/openblade/api/routes_restore.py index 388b5c2..b30d7b4 100644 --- a/openblade/api/routes_restore.py +++ b/openblade/api/routes_restore.py @@ -17,6 +17,7 @@ from openblade.catalog.db import get_session from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import OpenBladeError, safe_job_error +from openblade.jobs.inventory import InventoryService from openblade.jobs.restore import RestoreRequest as RestoreJobRequest from openblade.jobs.restore import run_restore_job from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler @@ -176,6 +177,11 @@ async def enqueue_restore( context.ltfs, context.catalog, job.id, + scheduler=DriveScheduler( + num_drives=len(InventoryService(context.library).snapshot().drives), + store=context.lease_store, + job_id=job.id, + ), ) except Exception as exc: context.catalog.update_job_state(job.id, "failed", str(exc)) diff --git a/openblade/cli/main.py b/openblade/cli/main.py index ca9aa63..42af56a 100644 --- a/openblade/cli/main.py +++ b/openblade/cli/main.py @@ -663,6 +663,7 @@ def restore_file( context.ltfs, context.catalog, job.id, + scheduler=scheduler, ) payload = { "jobId": job.id, From ddaed9dec1166787fc09ae0b06522a18b242964f Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:23:49 +0000 Subject: [PATCH 069/205] fix(api): AML error envelope keeps exception headers and covers unhandled 500s Retry-After and the session-clearing Set-Cookie were dropped by the HTTPException handler; unhandled exceptions on /aml paths now return the WSResultCode envelope with a curated message instead of plain text. Also registers the self-hosted runner labels for actionlint. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/actionlint.yaml | 4 ++++ openblade/api/main.py | 14 +++++++++++++- tests/integration/test_aml_auth_fidelity.py | 4 +--- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..ed3c8ce --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - i3 + - openblade-production diff --git a/openblade/api/main.py b/openblade/api/main.py index f4bc959..870e5e5 100644 --- a/openblade/api/main.py +++ b/openblade/api/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import os import time from typing import Any @@ -370,10 +371,21 @@ async def handle_http_exception(request: Request, exc: HTTPException) -> Respons if not _is_aml_compatible_path(request.url.path): return await http_exception_handler(request, exc) return JSONResponse( - status_code=exc.status_code, content=_aml_error_payload(exc.status_code, exc.detail) + status_code=exc.status_code, + content=_aml_error_payload(exc.status_code, exc.detail), + headers=exc.headers, ) +@app.exception_handler(Exception) +async def handle_unexpected_exception(request: Request, exc: Exception) -> Response: + """Last-resort handler: AML paths get the error envelope, never the exception text.""" + if not _is_aml_compatible_path(request.url.path): + raise exc + logging.getLogger(__name__).exception("Unhandled error on %s", request.url.path) + return JSONResponse(status_code=500, content=_aml_error_payload(500, "Internal server error")) + + @app.exception_handler(RequestValidationError) async def handle_request_validation_error( request: Request, exc: RequestValidationError diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py index 2bb6966..3ef0a3a 100644 --- a/tests/integration/test_aml_auth_fidelity.py +++ b/tests/integration/test_aml_auth_fidelity.py @@ -177,7 +177,6 @@ def test_expired_cookie_is_401(client: TestClient) -> None: assert client.get("/aml/users").status_code == 401 -@pytest.mark.xfail(strict=True, reason=_HEADERS_DROPPED) def test_expired_cookie_is_cleared(client: TestClient) -> None: _login(client) _expire_all_sessions() @@ -206,7 +205,6 @@ def test_eleventh_login_attempt_is_429_with_retry_after( assert response.status_code == 429 -@pytest.mark.xfail(strict=True, reason=_HEADERS_DROPPED) def test_rate_limited_login_carries_retry_after( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -285,7 +283,7 @@ def test_unhandled_error_is_500_without_leak( 404, 412, 429, - pytest.param(500, marks=pytest.mark.xfail(strict=True, reason=_NO_500_ENVELOPE)), + 500, 503, ], ) From a96533c8f6a4f42a25b9ab795d3d196f5c31c9b0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:24:38 +0000 Subject: [PATCH 070/205] sharded_restore: journal physical_state_unknown instead of suppressing cleanup errors Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/sharded_restore.py | 38 +++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index abd375b..263de9d 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -6,7 +6,6 @@ import logging import shutil import uuid -from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -17,6 +16,7 @@ from openblade.domain.models import MountMode from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import DEFAULT_BLOCK_SIZE, compute_checksum, reassemble_block_stripe +from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, record_physical_state_unknown from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -135,6 +135,16 @@ def run_sharded_restore( shutil.rmtree(scratch_dir, ignore_errors=True) +def _cleanup_failed( + catalog: CatalogRepository, job_id: str, op: str, barcode: str, drive: int | None +) -> None: + """Journal a failed cleanup op; it is the job error only if none exists yet.""" + record_physical_state_unknown(catalog, job_id, op, barcode, drive) + job = catalog.get_job(job_id) + if job is not None and not job.error: + catalog.update_job_state(job_id, "failed", error=PHYSICAL_STATE_UNKNOWN) + + def _restore_single( instance: Any, file_record: Any, @@ -157,6 +167,7 @@ def _restore_single( drive_id = handle.drive_id slot_id: int | None = None + cleanup_failed = False try: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) mount = ltfs.mount(instance.barcode, MountMode.READ_ONLY) @@ -178,7 +189,7 @@ def _restore_single( ltfs.unmount(mount) finally: if slot_id is not None: - with suppress(Exception): + try: execute_tape_request( catalog, library, @@ -192,9 +203,13 @@ def _restore_single( job_id=job_id, ), ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + _cleanup_failed(catalog, job_id, "unload", instance.barcode, drive_id) scheduler.release_drives(handles) - catalog.update_job_state(job_id, "completed") + if not cleanup_failed: + catalog.update_job_state(job_id, "completed") return ShardedRestoreResult( job_id=job_id, source_barcodes=[instance.barcode], @@ -228,6 +243,7 @@ def _restore_sharded( mounts: dict[str, Any] = {} loaded_slots: dict[int, int | None] = {} + cleanup_failed = False try: for handle in handles: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) @@ -264,13 +280,17 @@ def _read_shard(index: int, instance: Any) -> Path: catalog.update_job_state(job_id, "failed", error=error) raise ChecksumMismatchError(error) finally: - for mount in mounts.values(): - with suppress(Exception): + drive_by_barcode = {handle.barcode: handle.physical for handle in handles} + for barcode, mount in mounts.items(): + try: ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + _cleanup_failed(catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode)) for handle in handles: slot_id = loaded_slots.get(handle.physical) if slot_id is not None: - with suppress(Exception): + try: execute_tape_request( catalog, library, @@ -284,9 +304,13 @@ def _read_shard(index: int, instance: Any) -> Path: job_id=job_id, ), ) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + _cleanup_failed(catalog, job_id, "unload", handle.barcode, handle.physical) scheduler.release_drives(handles) - catalog.update_job_state(job_id, "completed") + if not cleanup_failed: + catalog.update_job_state(job_id, "completed") return ShardedRestoreResult( job_id=job_id, source_barcodes=barcodes, From 02cc807a6e7a7067b4968113ff4e66f39cefc523 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:24:43 +0000 Subject: [PATCH 071/205] test(jobs): lease contention and stale-lease fencing for non-sharded archive/restore Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/unit/test_nonsharded_leases.py | 146 +++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/unit/test_nonsharded_leases.py diff --git a/tests/unit/test_nonsharded_leases.py b/tests/unit/test_nonsharded_leases.py new file mode 100644 index 0000000..5768a05 --- /dev/null +++ b/tests/unit/test_nonsharded_leases.py @@ -0,0 +1,146 @@ +"""Non-sharded run_archive_job / run_restore_job take drives through DriveScheduler leases.""" + +from __future__ import annotations + +import functools +from pathlib import Path, PurePosixPath +from typing import Any + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_job +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend +from openblade.simulator.scenarios import one_drive_twenty_slots_five_cartridges + + +def _stack( + tmp_path: Path, +) -> tuple[CatalogRepository, CatalogRepository, MockLibraryBackend, MockLTFSBackend]: + # Two sessions on one file DB model two processes sharing one lease store. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + library, ltfs = one_drive_twenty_slots_five_cartridges() + barcode = str(library.inventory().slots[0].barcode) + library.load(1, 0) + ltfs.format(barcode, FormatConfirmation(barcode, SafetyToken.generate("format", barcode))) + library.unload(0, 1) + group = repo_a.create_volume_group("photos") + repo_a.add_barcode_to_volume_group(group.id, barcode) + return repo_a, repo_b, library, ltfs + + +def _archive( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + source: Path, + scheduler: DriveScheduler, +) -> None: + job = repo.create_job("archive", {"source_path": str(source), "volume_group": "photos"}) + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name="photos"), + library, + ltfs, + repo, + job.id, + scheduler=scheduler, + ) + + +def test_archive_holding_drive_0_blocks_restore_until_released(tmp_path: Path) -> None: + repo_a, repo_b, library, ltfs = _stack(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="archive") + first = tmp_path / "a.txt" + first.write_text("first") + _archive(repo_a, library, ltfs, first, sched_a) + + sched_b = DriveScheduler(1, store=CatalogLeaseStore(repo_b), job_id="restore") + sched_b.acquire_drives = functools.partial(sched_b.acquire_drives, timeout=0.2) # type: ignore[method-assign] + restore_job = repo_b.create_job("restore", {}) + blocked: list[BaseException] = [] + real_write = ltfs.write_file + + def write_then_contend(handle: Any, src: Path, dest: PurePosixPath) -> Any: + result = real_write(handle, src, dest) + with pytest.raises(DriveBusyError) as caught: + run_restore_job( + RestoreRequest("/photos/a.txt", tmp_path / "blocked.txt"), + library, + ltfs, + repo_b, + restore_job.id, + scheduler=sched_b, + ) + blocked.append(caught.value) + return result + + ltfs.write_file = write_then_contend # type: ignore[method-assign] + second = tmp_path / "b.txt" + second.write_text("second") + _archive(repo_a, library, ltfs, second, sched_a) + ltfs.write_file = real_write # type: ignore[method-assign] + assert len(blocked) == 1 + assert not (tmp_path / "blocked.txt").exists() + + # Released after the archive finished: the same restore now gets drive 0. + result = run_restore_job( + RestoreRequest("/photos/a.txt", tmp_path / "restored.txt"), + library, + ltfs, + repo_b, + repo_b.create_job("restore", {}).id, + scheduler=sched_b, + ) + assert result.checksum_verified + assert (tmp_path / "restored.txt").read_text() == "first" + events = [entry.event for entry in repo_a.job_journal(repo_a.list_jobs()[1].id)] + assert events.count("lease_acquired") == 1 + assert events.count("lease_released") == 1 + + +def test_stale_lease_aborts_archive_without_unmount_or_unload(tmp_path: Path) -> None: + repo_a, repo_b, library, ltfs = _stack(tmp_path) + sched_a = DriveScheduler(1, store=CatalogLeaseStore(repo_a), job_id="archive") + other_store = CatalogLeaseStore(repo_b) + calls: list[str] = [] + real_write, real_unmount, real_unload = ltfs.write_file, ltfs.unmount, library.unload + + def write_then_lose_lease(handle: Any, src: Path, dest: PurePosixPath) -> Any: + result = real_write(handle, src, dest) + # Another process reclaims the lease (e.g. TTL expiry) mid-job. + other_store.release([lease.id for lease in other_store.live_leases()]) + return result + + def record_unmount(handle: Any) -> None: + calls.append("unmount") + real_unmount(handle) + + def record_unload(*args: Any, **kwargs: Any) -> Any: + calls.append("unload") + return real_unload(*args, **kwargs) + + ltfs.write_file = write_then_lose_lease # type: ignore[method-assign] + ltfs.unmount = record_unmount # type: ignore[method-assign] + library.unload = record_unload # type: ignore[method-assign] + source = tmp_path / "a.txt" + source.write_text("payload") + job = repo_a.create_job("archive", {"source_path": str(source), "volume_group": "photos"}) + with pytest.raises(StaleLeaseError): + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name="photos"), + library, + ltfs, + repo_a, + job.id, + scheduler=sched_a, + ) + assert calls == [] + refreshed = repo_a.get_job(job.id) + assert refreshed is not None and refreshed.state == "failed" From c94cd6c22a75814c2e940ccfae6915aff63f4264 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:25:06 +0000 Subject: [PATCH 072/205] test: staged sharded archive never archives on lane failure; cleanup journals physical_state_unknown Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../test_staged_sharded_archive.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/integration/test_staged_sharded_archive.py diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py new file mode 100644 index 0000000..c33b748 --- /dev/null +++ b/tests/integration/test_staged_sharded_archive.py @@ -0,0 +1,91 @@ +"""Staged (STAGING -> VERIFYING -> one bulk ARCHIVED) sharded archive + cleanup journaling.""" + +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Any + +from sqlalchemy import select + +from openblade.catalog.models import FileInstance +from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, run_sharded_archive +from tests.integration.test_sharded_archive_atomicity import ( + BARCODES, + _catalog, + _request, + _setup, + _source, +) + + +def _states(catalog: Any) -> set[str]: + return {str(i.state) for i in catalog.session.scalars(select(FileInstance)).all()} + + +def _events(catalog: Any, job_id: str) -> list[str]: + return [entry.event for entry in catalog.job_journal(job_id)] + + +def test_second_lane_write_failure_stages_but_never_archives(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + real_write, real_unmount = ltfs.write_file, ltfs.unmount + calls = {"write": 0} + unmounted: list[Any] = [] + lock = threading.Lock() + + def write(*args: Any, **kwargs: Any) -> Any: + with lock: + calls["write"] += 1 + nth = calls["write"] + if nth == 2: + raise OSError("lane 2 write failed") + return real_write(*args, **kwargs) + + def unmount(mount: Any) -> Any: + unmounted.append(mount) + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "write_file", write) + monkeypatch.setattr(ltfs, "unmount", unmount) + job = catalog.create_job("archive", {}) + result = run_sharded_archive( + _request(_source(tmp_path)), library, ltfs, catalog, DriveScheduler(num_drives=2), job.id + ) + + assert result.errors and result.files_archived == 0 + states = _states(catalog) + assert states and states <= {"staging", "verifying"}, states # nothing ARCHIVED + events = _events(catalog, job.id) + assert "failed" in events and "committed" not in events + assert len(unmounted) == 2 # both lanes unmounted on the error path + assert all(d.barcode is None for d in library.inventory().drives) # and unloaded + + +def test_unmount_raise_in_cleanup_is_journaled_and_other_drive_unloaded( + tmp_path: Path, monkeypatch +) -> None: + library, ltfs = _setup() + catalog = _catalog() + real_unmount = ltfs.unmount + + def unmount(mount: Any) -> Any: + if str(mount.barcode) == BARCODES[0]: + raise RuntimeError("stuck") + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "unmount", unmount) + job = catalog.create_job("archive", {}) + run_sharded_archive( + _request(_source(tmp_path)), library, ltfs, catalog, DriveScheduler(num_drives=2), job.id + ) + + stored = catalog.get_job(job.id) + assert stored is not None and PHYSICAL_STATE_UNKNOWN in str(stored.error) + rows = [e for e in catalog.job_journal(job.id) if e.event == "physical_state_unknown"] + assert rows and rows[0].detail["barcode"] == BARCODES[0] + in_drives = {str(d.barcode) for d in library.inventory().drives if d.barcode is not None} + assert BARCODES[1] not in in_drives # the OTHER drive was still unloaded + assert "archived" not in _states(catalog) From a4245bc1b74e806d1a39bdd9eff069964a36b163 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:26:18 +0000 Subject: [PATCH 073/205] fix(nas): gateway on_open pins every open so on_close cannot drop another handle's pin on_open pinned only files already online while on_close always released, so a client that opened mid-hydration could close and unpin a file another client still held open, making it evictable. Regression test fails without the fix. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/protocol_gateway.py | 21 ++++++++++++++------- tests/integration/test_fuse_data_plane.py | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 9bafc48..041b585 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -381,15 +381,22 @@ def _require_hydrator(self) -> Hydrator: return hydrator def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> HydrationState: - """Pin the file and start hydration; optionally wait ``timeout`` seconds.""" + """Pin the file and start hydration; optionally wait ``timeout`` seconds. + + Every successful ``on_open`` takes exactly one pin, whatever the state, + so it pairs with exactly one ``on_close`` release. Pinning only online + files let a close of a mid-hydration open drop another handle's pin. + """ hydrator = self._require_hydrator() ticket = hydrator.request(catalog_path) - if timeout: - hydrator.wait(ticket, timeout) - state = hydrator.status(catalog_path) - if state is HydrationState.ONLINE: - hydrator.cache.acquire(ticket.checksum) - return state + hydrator.cache.acquire(ticket.checksum) + try: + if timeout: + hydrator.wait(ticket, timeout) + return hydrator.status(catalog_path) + except BaseException: + hydrator.cache.release(ticket.checksum) + raise def on_read_range( self, catalog_path: str, offset: int, length: int, *, timeout: float | None = None diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 09d44c2..1336039 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -168,3 +168,23 @@ def test_unmount_cancels_pending_batches(catalog, tmp_path): # type: ignore[no- ops.destroy("/") assert ticket.done.is_set() and ticket.error == "hydration cancelled by unmount" assert engine.batches == [] + + +def test_gateway_close_of_unpinned_open_does_not_unpin_other_handle(catalog, tmp_path): # type: ignore[no-untyped-def] + from openblade.nas.protocol_gateway import ProtocolGateway + + _, hyd, engine = _plane(catalog, tmp_path) + gw = ProtocolGateway() + gw.attach_hydrator(hyd) + checksum = catalog.get_file_record("/a.bin").checksum_sha256 # type: ignore[union-attr] + engine.gate.clear() + assert gw.on_open("/a.bin").value == "hydrating" # client B opens mid-hydration + engine.gate.set() + hyd.wait(hyd.request("/a.bin"), 5) + assert gw.on_open("/a.bin").value == "online" # client A opens the online file + + gw.on_close("/a.bin") # client B closes; A still holds the file open + hyd.cache.max_bytes = 1 + gw.on_open("/c.bin", timeout=5) # budget pressure + + assert hyd.cache.is_cached(checksum) From 0e3d1451ed0716dcb863bea14674cf313b6a146a Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:27:10 +0000 Subject: [PATCH 074/205] fix(jobs): never attempt an unload after a failed unmount; curate route error text; lease tree-restore children A failed unmount is journaled as physical_state_unknown and the drive is left for reconciliation instead of an unload that the backend gate must reject. Archive/restore routes stop recording str(exc) on bare exceptions, and tree_restore passes its lease-backed scheduler to run_restore_job. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_archive.py | 10 +++++----- openblade/api/routes_restore.py | 4 ++-- openblade/jobs/sharded_archive.py | 13 +++++++++++++ openblade/jobs/sharded_restore.py | 17 ++++++++++++++--- openblade/jobs/tree_restore.py | 1 + 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/openblade/api/routes_archive.py b/openblade/api/routes_archive.py index 852dd8e..c3019b7 100644 --- a/openblade/api/routes_archive.py +++ b/openblade/api/routes_archive.py @@ -12,7 +12,7 @@ from openblade.api import aml_state from openblade.bootstrap import AppContext, get_context -from openblade.domain.errors import FileNotFoundError +from openblade.domain.errors import FileNotFoundError, safe_job_error from openblade.jobs.archive import ArchiveRequest as ArchiveJobRequest from openblade.jobs.archive import run_archive_job from openblade.jobs.inventory import InventoryService @@ -149,14 +149,14 @@ async def enqueue_archive( ) except Exception as exc: _cleanup_failed_archive(context, source_path, request.volume_group) - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", source_path=request.source_path, volume_group=request.volume_group, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) @@ -220,14 +220,14 @@ async def enqueue_sharded_archive( job.id, ) except Exception as exc: - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", source_path=request.source_path, volume_group=request.volume_group, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) diff --git a/openblade/api/routes_restore.py b/openblade/api/routes_restore.py index b30d7b4..d932a1d 100644 --- a/openblade/api/routes_restore.py +++ b/openblade/api/routes_restore.py @@ -184,14 +184,14 @@ async def enqueue_restore( ), ) except Exception as exc: - context.catalog.update_job_state(job.id, "failed", str(exc)) + context.catalog.update_job_state(job.id, "failed", safe_job_error(exc)) _bridge_to_aml( context, job_id=job.id, status="failed", catalog_path=catalog_path, dest_path=request.dest_path, - error=str(exc), + error=safe_job_error(exc), ) raise refreshed = context.catalog.get_job(job.id) diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 25ec6d6..79c5605 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -274,16 +274,23 @@ def _clean_unmount_and_unload( caller does NOT mark the batch archived. """ failures: list[str] = [] + still_mounted: set[str] = set() for barcode, mount in list(mounts.items()): try: ltfs.unmount(mount) mounts.pop(barcode, None) except Exception as exc: # noqa: BLE001 - aggregated and re-raised below failures.append(f"unmount {barcode}: {exc}") + still_mounted.add(barcode) for handle in handles: slot_id = loaded_slots.get(handle.physical) if slot_id is None: continue + if handle.barcode in still_mounted: + # Never unload while LTFS is mounted or dirty: a failed unmount leaves + # the drive in an unknown state, which is reported, not forced. + failures.append(f"unload {handle.barcode}: skipped, unmount failed") + continue try: # raise_on_failed=True is essential: execute_tape_request otherwise # SWALLOWS an unload failure (returns a FAILED record) and the batch would @@ -372,11 +379,13 @@ def _best_effort_unmount_and_unload( """ drive_by_barcode = {handle.barcode: handle.physical for handle in handles} failed = False + still_mounted: set[str] = set() for barcode, mount in mounts.items(): try: ltfs.unmount(mount) except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues failed = True + still_mounted.add(barcode) record_physical_state_unknown( catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode) ) @@ -384,6 +393,10 @@ def _best_effort_unmount_and_unload( slot_id = loaded_slots.get(handle.physical) if slot_id is None: continue + if handle.barcode in still_mounted: + # Never unload while LTFS is mounted or dirty; the unmount failure above + # is already journaled as physical_state_unknown for this drive. + continue try: execute_tape_request( catalog, diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index 263de9d..009e907 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -168,6 +168,7 @@ def _restore_single( slot_id: int | None = None cleanup_failed = False + still_mounted = False try: drive_id, slot_id = _ensure_loaded(catalog, library, ltfs, handle, job_id) mount = ltfs.mount(instance.barcode, MountMode.READ_ONLY) @@ -186,9 +187,16 @@ def _restore_single( raise ChecksumMismatchError(error) shutil.copy2(tmp_dest, request.dest_path) finally: - ltfs.unmount(mount) + try: + ltfs.unmount(mount) + except Exception: # noqa: BLE001 - recorded as physical_state_unknown + cleanup_failed = True + still_mounted = True + _cleanup_failed(catalog, job_id, "unmount", instance.barcode, drive_id) finally: - if slot_id is not None: + # Never unload while LTFS is mounted or dirty: a failed unmount is + # journaled above and the drive is left for reconciliation. + if slot_id is not None and not still_mounted: try: execute_tape_request( catalog, @@ -281,15 +289,18 @@ def _read_shard(index: int, instance: Any) -> Path: raise ChecksumMismatchError(error) finally: drive_by_barcode = {handle.barcode: handle.physical for handle in handles} + still_mounted_barcodes: set[str] = set() for barcode, mount in mounts.items(): try: ltfs.unmount(mount) except Exception: # noqa: BLE001 - recorded as physical_state_unknown cleanup_failed = True + still_mounted_barcodes.add(barcode) _cleanup_failed(catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode)) for handle in handles: slot_id = loaded_slots.get(handle.physical) - if slot_id is not None: + # Never unload while LTFS is mounted or dirty (failed unmount above). + if slot_id is not None and handle.barcode not in still_mounted_barcodes: try: execute_tape_request( catalog, diff --git a/openblade/jobs/tree_restore.py b/openblade/jobs/tree_restore.py index b8baf1d..e4d16fa 100644 --- a/openblade/jobs/tree_restore.py +++ b/openblade/jobs/tree_restore.py @@ -304,6 +304,7 @@ def run_tree_restore( ltfs, catalog, child.id, + scheduler=scheduler, ) verified = plain_result.checksum_verified barcodes = [plain_result.source_barcode] From 1845198a5cddea195a5a8fc4d804f128e962eab9 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:27:16 +0000 Subject: [PATCH 075/205] feat(rebuild): structured uncommitted_generations and missing_siblings in rebuild plan Replace per-tape free-text warnings with typed lists on RebuildPlanResult; keep one summary warning line per category for operators. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/catalog_rebuild.py | 38 +++++++++++++++++++++++++++--- openblade/nas/types.py | 18 ++++++++++++++ tests/unit/test_catalog_rebuild.py | 15 ++++++++++-- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index 85741c0..e726ca5 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -20,12 +20,14 @@ DatasetStatus, IngestMode, ManifestVersionRecord, + MissingSibling, NasDataset, NasFileRecord, NasFileState, RebuildPlanRequest, RebuildPlanResult, RebuildRunStatus, + UncommittedGeneration, ) logger = logging.getLogger(__name__) @@ -82,6 +84,8 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: barcodes_missing_shard: list[str] = [] barcodes_invalid: list[str] = [] warnings: list[str] = [] + uncommitted_generations: list[UncommittedGeneration] = [] + missing_siblings: list[MissingSibling] = [] estimated_files = 0 estimated_datasets = 0 estimated_path_mappings = 0 @@ -119,20 +123,46 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: uncommitted = self._uncommitted_reason(barcode, manifest) if uncommitted is not None: # Never restore an uncommitted generation's files as archived. - warnings.append( - f"{barcode}: uncommitted generation {manifest.generation_id}: {uncommitted}" + # _uncommitted_reason only returns a reason when generation_id is set. + uncommitted_generations.append( + UncommittedGeneration( + barcode=barcode, + generation_id=manifest.generation_id or "", + reason=uncommitted, + ) ) continue requested = set(request.barcodes) for sibling in manifest.sibling_tapes: if sibling not in requested: - warnings.append(f"{barcode}: missing sibling tape {sibling}") + missing_siblings.append( + MissingSibling( + barcode=sibling, + required_by_barcode=barcode, + generation_id=manifest.generation_id or None, + ) + ) barcodes_to_scan.append(barcode) estimated_files += len(shard.files) estimated_datasets += len(shard.datasets) estimated_path_mappings += len(self.shard_writer.shard_to_path_mappings(shard)) + # One operator-facing line per category; the structured lists carry the detail. + if uncommitted_generations: + warnings.append( + f"{len(uncommitted_generations)} uncommitted generation(s) skipped: " + + ", ".join( + f"{u.barcode} ({u.generation_id}: {u.reason})" for u in uncommitted_generations + ) + ) + if missing_siblings: + warnings.append( + f"{len(missing_siblings)} sibling tape(s) missing from request: " + + ", ".join( + f"{m.barcode} (required by {m.required_by_barcode})" for m in missing_siblings + ) + ) safe_to_enqueue = len(barcodes_invalid) == 0 and len(barcodes_to_scan) > 0 run_id = "" if not request.dry_run: @@ -167,6 +197,8 @@ def plan_rebuild(self, request: RebuildPlanRequest) -> RebuildPlanResult: estimated_files=estimated_files, estimated_datasets=estimated_datasets, estimated_path_mappings=estimated_path_mappings, + uncommitted_generations=uncommitted_generations, + missing_siblings=missing_siblings, warnings=warnings, safe_to_enqueue=safe_to_enqueue, ) diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 94a5792..607cb00 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -689,6 +689,22 @@ class RebuildPlanRequest(BaseModel): dry_run: bool = False +class UncommittedGeneration(BaseModel): + """A tape whose generation has no valid commit marker; it is never rebuilt.""" + + barcode: str + generation_id: str + reason: str + + +class MissingSibling(BaseModel): + """A sibling tape a planned tape's generation needs but the request omitted.""" + + barcode: str + required_by_barcode: str + generation_id: str | None = None + + class RebuildPlanResult(BaseModel): run_id: str dry_run: bool @@ -699,6 +715,8 @@ class RebuildPlanResult(BaseModel): estimated_files: int = 0 estimated_datasets: int = 0 estimated_path_mappings: int = 0 + uncommitted_generations: list[UncommittedGeneration] = Field(default_factory=list) + missing_siblings: list[MissingSibling] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list) safe_to_enqueue: bool diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index 9b1c262..6265d9e 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -450,7 +450,13 @@ def test_plan_rebuild_uncommitted_generation_not_scanned( ) assert result.barcodes_to_scan == [] - assert f"{barcode}: uncommitted generation gen-7: {reason}" in result.warnings + assert [u.model_dump() for u in result.uncommitted_generations] == [ + {"barcode": barcode, "generation_id": "gen-7", "reason": reason} + ] + assert result.missing_siblings == [] + assert len(result.warnings) == 1 + assert barcode in result.warnings[0] + assert reason in result.warnings[0] def test_plan_rebuild_committed_generation_reports_missing_sibling( @@ -465,4 +471,9 @@ def test_plan_rebuild_committed_generation_reports_missing_sibling( ) assert result.barcodes_to_scan == [barcode] - assert f"{barcode}: missing sibling tape ZZ9999L8" in result.warnings + assert [m.model_dump() for m in result.missing_siblings] == [ + {"barcode": "ZZ9999L8", "required_by_barcode": barcode, "generation_id": "gen-7"} + ] + assert result.uncommitted_generations == [] + assert len(result.warnings) == 1 + assert "ZZ9999L8" in result.warnings[0] From e61a0298649c76b31cd535d2b5436136c54e239d Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:27:45 +0000 Subject: [PATCH 076/205] test(jobs): a drive whose unmount failed is never asked to unload Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_staged_sharded_archive.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py index c33b748..610cf5d 100644 --- a/tests/integration/test_staged_sharded_archive.py +++ b/tests/integration/test_staged_sharded_archive.py @@ -77,6 +77,14 @@ def unmount(mount: Any) -> Any: return real_unmount(mount) monkeypatch.setattr(ltfs, "unmount", unmount) + real_unload = library.unload + unload_requests: list[int] = [] + + def unload(drive_id: int, target_slot: int) -> Any: + unload_requests.append(drive_id) + return real_unload(drive_id, target_slot) + + monkeypatch.setattr(library, "unload", unload) job = catalog.create_job("archive", {}) run_sharded_archive( _request(_source(tmp_path)), library, ltfs, catalog, DriveScheduler(num_drives=2), job.id @@ -86,6 +94,10 @@ def unmount(mount: Any) -> Any: assert stored is not None and PHYSICAL_STATE_UNKNOWN in str(stored.error) rows = [e for e in catalog.job_journal(job.id) if e.event == "physical_state_unknown"] assert rows and rows[0].detail["barcode"] == BARCODES[0] - in_drives = {str(d.barcode) for d in library.inventory().drives if d.barcode is not None} + drives = {str(d.barcode): d.drive_id for d in library.inventory().drives if d.barcode} + in_drives = set(drives) assert BARCODES[1] not in in_drives # the OTHER drive was still unloaded + # Never unload while LTFS is mounted: the stuck drive must not even be asked. + assert BARCODES[0] in in_drives + assert drives[BARCODES[0]] not in unload_requests assert "archived" not in _states(catalog) From 4b249334f56a8248d354ad31285e2cbd5d9bcff2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:28:04 +0000 Subject: [PATCH 077/205] feat(nas): NasPool.protection explicit policy; protection_from_pool prefers it Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/protection.py | 6 +++++- openblade/nas/types.py | 3 +++ tests/unit/test_protection_axes.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/openblade/domain/protection.py b/openblade/domain/protection.py index b36d1a0..e68e091 100644 --- a/openblade/domain/protection.py +++ b/openblade/domain/protection.py @@ -194,8 +194,12 @@ def protection_from_pool(pool: Any) -> ProtectionPolicy: ``replication_factor`` stays the source of truth for existing pools; this only expresses it as ``Protection.REPLICATION(copies=n)``. Placement is single-tape: - the legacy field says nothing about striping. + the legacy field says nothing about striping. An explicit ``pool.protection`` + policy, when set, wins over the legacy field. """ + explicit = getattr(pool, "protection", None) + if isinstance(explicit, ProtectionPolicy): + return explicit copies = max(1, int(getattr(pool, "replication_factor", 1) or 1)) protection = Protection.replication(copies) if copies >= 2 else Protection.none() return ProtectionPolicy(protection=protection) diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 607cb00..28ea24e 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator +from openblade.domain.protection import ProtectionPolicy from openblade.nas.media import Cartridge, MediaGeneration @@ -530,6 +531,8 @@ class NasPool(BaseModel): virtual_mount_enabled: bool = True hydration_behavior: HydrationBehavior = HydrationBehavior.QUEUE replication_factor: int = Field(default=1, ge=1, le=4) + # Explicit protection/placement policy; None means derive from replication_factor. + protection: ProtectionPolicy | None = None backup_order_mode: Literal["sequential", "parallel"] = "sequential" cache_target_id: str | None = None restore_target_path: str = "/openblade/restore" diff --git a/tests/unit/test_protection_axes.py b/tests/unit/test_protection_axes.py index 025ca12..9b915b3 100644 --- a/tests/unit/test_protection_axes.py +++ b/tests/unit/test_protection_axes.py @@ -90,6 +90,16 @@ def test_protection_from_pool(factor: int, copies: int, protected: bool) -> None assert policy.is_protected is protected +def test_protection_from_pool_prefers_explicit_policy() -> None: + from openblade.nas.types import NasPool + + explicit = ProtectionPolicy(protection=Protection.replication(3)) + pool = NasPool(name="p", replication_factor=1, protection=explicit) + assert protection_from_pool(pool) == explicit + # Unset field falls back to the legacy replication_factor. + assert protection_from_pool(NasPool(name="q", replication_factor=2)).protection.copies == 2 + + def test_policy_dict_round_trip() -> None: policy = ProtectionPolicy( placement=Placement.STRIPED, From 5750f090718791d5ba3100869ada60aa355d1ebc Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:28:23 +0000 Subject: [PATCH 078/205] fix(fuse): hydration workers complete tickets on any engine error Typed OpenBladeError keeps its curated message; any other exception is logged and recorded as a generic message, so the Timer thread never dies with the ticket stuck HYDRATING. request() unregisters its ticket if the job lookup or tape-key resolution raises. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 37 ++++++++++++++++++----- tests/integration/test_fuse_data_plane.py | 37 +++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 09e5dfa..1ea9c5e 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import os import threading import time @@ -11,8 +12,13 @@ from pathlib import Path from typing import Any, Protocol +from openblade.domain.errors import OpenBladeError from openblade.fuse.cache import CacheChecksumError, HydrationCache +logger = logging.getLogger(__name__) + +_UNEXPECTED = "restore failed unexpectedly" + @dataclass class HydrationRequest: @@ -197,12 +203,20 @@ def request(self, catalog_path: str) -> HydrationTicket: self._tickets[catalog_path] = ticket # Restart-safety: a job already in flight for this path is re-attached, # not duplicated. - job_id = self.engine.find_active_job(catalog_path) + try: + job_id = self.engine.find_active_job(catalog_path) + if job_id is None: + ticket.tape_key = self._tape_key(catalog_path, record) + except BaseException: + # Never leave a registered ticket that no worker will complete. + with self._lock: + if self._tickets.get(catalog_path) is ticket: + del self._tickets[catalog_path] + raise if job_id is not None: ticket.job_id = job_id self._spawn(self._resume, ticket) return ticket - ticket.tape_key = self._tape_key(catalog_path, record) with self._lock: self._pending.setdefault(ticket.tape_key, []).append(ticket) if ticket.tape_key not in self._timers: @@ -251,20 +265,29 @@ def _flush(self, tape_key: str) -> None: self._workers.append(threading.current_thread()) try: staged = self.engine.restore_batch(tape_key, [t.catalog_path for t in batch]) - except (HydrationFailedError, OSError) as exc: - reason = str(exc) if isinstance(exc, HydrationFailedError) else "restore I/O error" + except (HydrationFailedError, OpenBladeError) as exc: + reason = str(exc) # typed errors carry a curated message + except OSError: + reason = "restore I/O error" + except Exception: + logger.exception("fuse: unexpected restore failure for tape %s", tape_key) + reason = _UNEXPECTED + else: for ticket in batch: - self._fail(ticket, reason) + self._commit(ticket, staged.get(ticket.catalog_path)) return for ticket in batch: - self._commit(ticket, staged.get(ticket.catalog_path)) + self._fail(ticket, reason) def _resume(self, ticket: HydrationTicket) -> None: assert ticket.job_id is not None try: self._commit(ticket, self.engine.resume(ticket.job_id)) - except (HydrationFailedError, OSError, KeyError): + except (HydrationFailedError, OpenBladeError, OSError, KeyError): self._fail(ticket, f"restore job {ticket.job_id} did not complete") + except Exception: + logger.exception("fuse: unexpected failure resuming job %s", ticket.job_id) + self._fail(ticket, _UNEXPECTED) def _commit(self, ticket: HydrationTicket, staged: Path | None) -> None: if staged is None: diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 1336039..cf2ae65 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -12,6 +12,7 @@ from openblade.catalog.db import get_session, init_db from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import CartridgeOfflineError from openblade.fuse.cache import CacheEntryInUseError, HydrationCache from openblade.fuse.filesystem import CatalogFilesystem from openblade.fuse.hydration import HydrationFailedError, Hydrator, JobRestoreEngine @@ -188,3 +189,39 @@ def test_gateway_close_of_unpinned_open_does_not_unpin_other_handle(catalog, tmp gw.on_open("/c.bin", timeout=5) # budget pressure assert hyd.cache.is_cached(checksum) + + +class _RaisingEngine(FakeEngine): + def __init__(self, staging: Path, exc: Exception) -> None: + super().__init__(staging) + self.exc = exc + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + raise self.exc + + +@pytest.mark.parametrize( + ("exc", "expected"), + [ + (CartridgeOfflineError("cartridge TAPE01 is offline"), "cartridge TAPE01 is offline"), + (RuntimeError("dsn=postgres://u:secret@db"), "restore failed unexpectedly"), + ], +) +def test_untyped_engine_error_fails_ticket_and_file_goes_offline(catalog, tmp_path, exc, expected): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + hyd = Hydrator(catalog, fs.cache, _RaisingEngine(tmp_path / "s", exc), batch_window_s=0.01) + ticket = hyd.request("/a.bin") + assert ticket.done.wait(3), "worker died without completing the ticket" + assert ticket.error == expected + assert "secret" not in (hyd.last_error("/a.bin") or "") + assert hyd.status("/a.bin").value == "offline" + + +def test_request_unregisters_ticket_when_job_lookup_raises(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s") + engine.find_active_job = lambda p: (_ for _ in ()).throw(RuntimeError("db down")) # type: ignore[method-assign] + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + with pytest.raises(RuntimeError): + hyd.request("/a.bin") + assert hyd.status("/a.bin").value == "offline" From 4d3782db224d7d9d9cc05660d3fd059d1f459abd Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:30:38 +0000 Subject: [PATCH 079/205] fix(fuse): resume() re-reads the job each poll and stops at a deadline The engine refreshes the job row (Session.refresh) so a job completed by another session is seen, and raises HydrationTimeoutError after resume_timeout_s (defaults to the hydrate timeout); the hydrator fails the ticket with a curated 'timed out' message instead of hanging. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 30 ++++++++++++++++++++-- tests/integration/test_fuse_data_plane.py | 31 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 1ea9c5e..8790376 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any, Protocol +from sqlalchemy.exc import InvalidRequestError + from openblade.domain.errors import OpenBladeError from openblade.fuse.cache import CacheChecksumError, HydrationCache @@ -104,7 +106,19 @@ class JobRestoreEngine: layer's concern, not ours). """ - def __init__(self, restore_service: Any, catalog: Any, staging_root: Path) -> None: + def __init__( + self, + restore_service: Any, + catalog: Any, + staging_root: Path, + *, + poll_interval_s: float = 0.5, + resume_timeout_s: float | None = None, + ) -> None: + self.poll_interval_s = poll_interval_s + self.resume_timeout_s = ( + hydrate_timeout_from_env() if resume_timeout_s is None else resume_timeout_s + ) self.restore_service = restore_service self.catalog = catalog self.staging_root = staging_root @@ -133,15 +147,25 @@ def find_active_job(self, catalog_path: str) -> str | None: return None def resume(self, job_id: str) -> Path: + deadline = time.monotonic() + self.resume_timeout_s while True: job = self.catalog.get_job(job_id) + if job is not None: + # The job is advanced by another session; re-read the row instead + # of trusting this session's identity-map copy. + try: + self.catalog.session.refresh(job) + except InvalidRequestError: + job = None if job is None: raise HydrationFailedError(f"restore job {job_id} disappeared") if job.state == "completed": return Path(json.loads(job.metadata_json)["dest_path"]) if job.state not in {"pending", "running"}: raise HydrationFailedError(f"restore job {job_id} ended {job.state}") - time.sleep(0.5) + if time.monotonic() >= deadline: + raise HydrationTimeoutError(f"restore job {job_id} did not finish in time") + time.sleep(self.poll_interval_s) class Hydrator: @@ -283,6 +307,8 @@ def _resume(self, ticket: HydrationTicket) -> None: assert ticket.job_id is not None try: self._commit(ticket, self.engine.resume(ticket.job_id)) + except HydrationTimeoutError: + self._fail(ticket, f"restore job {ticket.job_id} timed out") except (HydrationFailedError, OpenBladeError, OSError, KeyError): self._fail(ticket, f"restore job {ticket.job_id} did not complete") except Exception: diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index cf2ae65..3d866a6 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -4,6 +4,7 @@ import errno import hashlib +import json import os import threading from pathlib import Path @@ -15,7 +16,12 @@ from openblade.domain.errors import CartridgeOfflineError from openblade.fuse.cache import CacheEntryInUseError, HydrationCache from openblade.fuse.filesystem import CatalogFilesystem -from openblade.fuse.hydration import HydrationFailedError, Hydrator, JobRestoreEngine +from openblade.fuse.hydration import ( + HydrationFailedError, + HydrationTimeoutError, + Hydrator, + JobRestoreEngine, +) from openblade.fuse.mount import STATE_XATTR, CatalogFuseOperations FILES = { @@ -225,3 +231,26 @@ def test_request_unregisters_ticket_when_job_lookup_raises(catalog, tmp_path): with pytest.raises(RuntimeError): hyd.request("/a.bin") assert hyd.status("/a.bin").value == "offline" + + +def test_resume_sees_job_completed_by_another_session(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"catalog_path": "/a.bin"}) + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=3.0 + ) + assert catalog.get_job(job.id).state == "pending" # load into the identity map + other = CatalogRepository(get_session()) + row = other.get_job(job.id) + row.metadata_json = json.dumps({"catalog_path": "/a.bin", "dest_path": str(tmp_path / "x")}) + row.state = "completed" + other.session.commit() + assert engine.resume(str(job.id)) == tmp_path / "x" + + +def test_resume_gives_up_at_deadline_with_typed_error(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"catalog_path": "/a.bin"}) + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=0.2 + ) + with pytest.raises(HydrationTimeoutError): + engine.resume(str(job.id)) From fc08942b97d27d9b5d305d7d19562cfd51be33bb Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:30:43 +0000 Subject: [PATCH 080/205] feat(rebuild): recover dataset protection policy into nas_datasets.protection_json Rebuild writes each shard dataset entry's ProtectionPolicy back onto the dataset row (new nullable TEXT column + additive _migrate_schema guard). Malformed policies are logged and skipped; pre-v2 shards leave it unset. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/db.py | 4 ++ openblade/catalog/models.py | 2 + openblade/nas/catalog_rebuild.py | 27 +++++++++++++ tests/unit/test_catalog_rebuild.py | 62 ++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/openblade/catalog/db.py b/openblade/catalog/db.py index 5976278..b01f305 100644 --- a/openblade/catalog/db.py +++ b/openblade/catalog/db.py @@ -113,6 +113,10 @@ def _migrate_schema(engine: Engine) -> None: library_columns = { column["name"] for column in inspector.get_columns("library_instances") } + if "nas_datasets" in inspector.get_table_names(): + dataset_columns = {column["name"] for column in inspector.get_columns("nas_datasets")} + if "protection_json" not in dataset_columns: + connection.execute(text("ALTER TABLE nas_datasets ADD COLUMN protection_json TEXT")) if library_columns and "role" not in library_columns: with suppress(Exception): connection.execute( diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index 3884fbe..be779f7 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -327,6 +327,8 @@ class NasDataset(Base): status: Mapped[str] = mapped_column(String, default="pending") copies_completed: Mapped[int] = mapped_column(Integer, default=0) manifest_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # Serialized ProtectionPolicy.to_dict(); written by catalog rebuild, None if unknown. + protection_json: Mapped[str | None] = mapped_column(Text, nullable=True) created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index e726ca5..3343e79 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -2,12 +2,15 @@ from __future__ import annotations +import json import logging from datetime import datetime from typing import cast from uuid import uuid4 +from openblade.catalog.models import NasDataset as NasDatasetRow from openblade.catalog.repository import CatalogRepository +from openblade.domain.protection import ProtectionPolicy from openblade.nas.catalog_shard import ( CatalogShard, CatalogShardWriter, @@ -343,10 +346,34 @@ def _recover_datasets(self, barcode: str, shard: CatalogShard) -> int: updated_at=_utcnow_iso(), ) self.repo.upsert_nas_dataset(dataset.model_dump(mode="json")) + self._recover_dataset_protection(barcode, entry.dataset_id, entry.protection) if is_new: count += 1 return count + def _recover_dataset_protection( + self, barcode: str, dataset_id: str, protection: dict[str, object] | None + ) -> None: + """Write the shard's protection/placement policy back onto the dataset row. + + Pre-v2 shards carry no policy; the existing row value is kept. A malformed + policy is logged and skipped so one bad shard entry cannot abort the rebuild. + """ + if protection is None: + return + try: + policy = ProtectionPolicy.from_dict(dict(protection)) + except (AttributeError, KeyError, TypeError, ValueError): + logger.warning( + "Skipping malformed protection policy for dataset %s on %s", dataset_id, barcode + ) + return + row = self.repo.session.get(NasDatasetRow, dataset_id) + if row is None: + return + row.protection_json = json.dumps(policy.to_dict(), sort_keys=True) + self.repo.session.commit() + def _recover_files(self, barcode: str, shard: CatalogShard) -> int: """Upsert file records from shard; count only files not already in the catalog.""" count = 0 diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index 6265d9e..c2d6a9c 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -82,6 +82,7 @@ def _seed_tape( include_shard: bool = True, files_per_tape: int = 2, create_versions: int = 0, + protection: dict[str, object] | None = None, ) -> dict[str, object]: metadata_writer: TapeMetadataWriter = rebuild_env["metadata_writer"] shard_writer: CatalogShardWriter = rebuild_env["shard_writer"] @@ -165,6 +166,7 @@ def _seed_tape( total_bytes=sum(item.size for item in shard_files), tape_set=[barcode], shard_set=[], + protection=protection, ) ], files=shard_files, @@ -477,3 +479,63 @@ def test_plan_rebuild_committed_generation_reports_missing_sibling( assert result.uncommitted_generations == [] assert len(result.warnings) == 1 assert "ZZ9999L8" in result.warnings[0] + + +def _dataset_protection_json(rebuild_env: dict[str, object], dataset_id: str) -> str | None: + from openblade.catalog.models import NasDataset as NasDatasetRow + + repo = rebuild_env["planner"].repo + repo.session.expire_all() + row = repo.session.get(NasDatasetRow, dataset_id) + assert row is not None + return row.protection_json + + +def test_execute_rebuild_recovers_dataset_protection_policy( + rebuild_env: dict[str, object], +) -> None: + import json + + from openblade.domain.protection import Protection, ProtectionPolicy + + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + policy = ProtectionPolicy(protection=Protection.replication(3)) + _seed_tape(rebuild_env, barcode, protection=policy.to_dict()) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + stored = _dataset_protection_json(rebuild_env, f"dataset-{barcode}") + assert stored is not None + assert ProtectionPolicy.from_dict(json.loads(stored)) == policy + + +def test_execute_rebuild_pre_v2_shard_leaves_protection_unset( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + assert _dataset_protection_json(rebuild_env, f"dataset-{barcode}") is None + + +def test_execute_rebuild_skips_malformed_protection_policy( + rebuild_env: dict[str, object], +) -> None: + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode, protection={"placement": "bogus"}) + plan = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=False) + ) + + result = rebuild_env["planner"].execute_rebuild_run(plan.run_id) + + assert result.datasets_recovered == 1 + assert _dataset_protection_json(rebuild_env, f"dataset-{barcode}") is None From 2df1d67ef55631a1327d4722bd9f1a7925ae2fd1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:32:42 +0000 Subject: [PATCH 081/205] feat(recovery): report staged instances and stale pending jobs RecoveryReport gains staged_instances (uncommitted STAGING/VERIFYING instances per failed_recoverable job) and stale_pending_job_ids (pending jobs older than one lease TTL with no live lease; report only). Each job recovery fails gets a `recovered` journal entry. GET /jobs/recovery exposes both fields. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_jobs.py | 27 ++++++++++++++++- openblade/jobs/recovery.py | 48 +++++++++++++++++++++++++++--- tests/unit/test_recovery.py | 57 ++++++++++++++++++++++++++++++++++-- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/openblade/api/routes_jobs.py b/openblade/api/routes_jobs.py index 9f1beed..15d14f9 100644 --- a/openblade/api/routes_jobs.py +++ b/openblade/api/routes_jobs.py @@ -53,10 +53,20 @@ class DriveMismatchResponse(BaseModel): observed_barcode: str | None +class StagedInstanceResponse(BaseModel): + instance_id: str + barcode: str + tape_path: str + shard_index: int | None + state: str + + class RecoveryReportResponse(BaseModel): interrupted_job_ids: list[str] released_lease_ids: list[str] mismatches: list[DriveMismatchResponse] + staged_instances: dict[str, list[StagedInstanceResponse]] + stale_pending_job_ids: list[str] # Declared before /{job_id} so the literal path wins route matching. @@ -64,7 +74,8 @@ class RecoveryReportResponse(BaseModel): async def get_recovery_report( context: AppContext = Depends(get_context), ) -> RecoveryReportResponse: - """What startup recovery did: interrupted jobs, released leases, drive mismatches.""" + """What startup recovery did: interrupted jobs, released leases, drive mismatches, + uncommitted staged instances per recoverable job, and stale pending jobs.""" report = context.recovery_report return RecoveryReportResponse( interrupted_job_ids=list(report.interrupted_job_ids), @@ -79,6 +90,20 @@ async def get_recovery_report( ) for m in report.mismatches ], + staged_instances={ + job_id: [ + StagedInstanceResponse( + instance_id=i.instance_id, + barcode=i.barcode, + tape_path=i.tape_path, + shard_index=i.shard_index, + state=i.state, + ) + for i in instances + ] + for job_id, instances in report.staged_instances.items() + }, + stale_pending_job_ids=list(report.stale_pending_job_ids), ) diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index 584b560..f7e17a5 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -7,10 +7,12 @@ import logging from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta -from openblade.catalog.repository import CatalogRepository +from openblade.catalog.repository import CatalogRepository, StagedInstance from openblade.domain.backends import LibraryBackend from openblade.jobs.inventory import InventoryService +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL logger = logging.getLogger(__name__) @@ -34,6 +36,18 @@ class RecoveryReport: interrupted_job_ids: list[str] = field(default_factory=list) released_lease_ids: list[str] = field(default_factory=list) mismatches: list[DriveMismatch] = field(default_factory=list) + # Uncommitted (STAGING/VERIFYING) instances per `failed_recoverable` job: what an + # operator must reconcile on tape before retrying or discarding the job. + staged_instances: dict[str, list[StagedInstance]] = field(default_factory=dict) + # `pending` jobs older than one lease TTL with no live lease. Report only. + stale_pending_job_ids: list[str] = field(default_factory=list) + + +def _is_older_than(created_at: datetime, ttl: timedelta, now: datetime) -> bool: + # Job.created_at is written as naive UTC; tolerate an aware value too. + if created_at.tzinfo is not None: + created_at = created_at.astimezone(UTC).replace(tzinfo=None) + return created_at < now - ttl def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) -> RecoveryReport: @@ -52,8 +66,13 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - if owner is not None and owner.state == "running" and owner.id not in interrupted: catalog.update_job_state(owner.id, "failed_recoverable", INTERRUPTED_ERROR) interrupted.append(owner.id) + catalog.journal( + owner.id, "recovered", {"lease_id": lease.id, "error": INTERRUPTED_ERROR} + ) # Bookkeeping: a live lease whose job already reached a terminal state. - for lease in catalog.live_leases(): + live = catalog.live_leases() + leased_job_ids = {lease.job_id for lease in live} + for lease in live: owner = catalog.get_job(lease.job_id) if owner is None or owner.state != "running": orphaned.append(lease) @@ -81,11 +100,32 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - ) mismatches.append(mismatch) - report = RecoveryReport(interrupted, [lease.id for lease in orphaned], mismatches) + staged = { + job.id: catalog.list_staged_instances(job.id) + for job in catalog.list_jobs(state="failed_recoverable") + } + # Evidence is the lease set observed before this pass released anything. + now = datetime.now(UTC).replace(tzinfo=None) + stale_pending = [ + job.id + for job in catalog.list_jobs(state="pending") + if job.id not in leased_job_ids and _is_older_than(job.created_at, DEFAULT_LEASE_TTL, now) + ] + + report = RecoveryReport( + interrupted_job_ids=interrupted, + released_lease_ids=[lease.id for lease in orphaned], + mismatches=mismatches, + staged_instances=staged, + stale_pending_job_ids=stale_pending, + ) logger.info( - "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es)", + "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es), " + "%d staged instance(s) awaiting reconcile, %d stale pending job(s)", len(report.interrupted_job_ids), len(report.released_lease_ids), len(report.mismatches), + sum(len(items) for items in staged.values()), + len(stale_pending), ) return report diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py index 0c98097..42ba1e7 100644 --- a/tests/unit/test_recovery.py +++ b/tests/unit/test_recovery.py @@ -1,12 +1,13 @@ from __future__ import annotations -from datetime import timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path from openblade.bootstrap import create_context from openblade.catalog.repository import CatalogRepository from openblade.config import OpenBladeConfig from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL from openblade.simulator.library import MockLibraryBackend @@ -82,8 +83,60 @@ def test_restart_runs_recovery_and_exposes_it_read_only(tmp_path: Path) -> None: response = TestClient(app).get("/jobs/recovery") assert response.status_code == 200 - assert response.json()["interrupted_job_ids"] == [job.id] + body = response.json() + assert body["interrupted_job_ids"] == [job.id] + assert body["staged_instances"] == {job.id: []} + assert body["stale_pending_job_ids"] == [] strict = create_context(OpenBladeConfig(db_url=db_url, scalar_api_only=True)) reset_context(strict) assert TestClient(app).get("/jobs/recovery").status_code == 404 + + +def test_recovery_lists_staged_instances_of_an_interrupted_job(tmp_path: Path) -> None: + # A sharded archive died after staging shards but before the commit marker: + # the operator needs to know exactly which tape paths are uncommitted. + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + _lease(catalog, job.id, "AAA001L9", ttl=timedelta(seconds=-1)) + vg = catalog.create_volume_group("vg-staged") + record = catalog.create_file_record("/data/big.bin.shard0", 10, "0" * 64, vg.id, shard_index=0) + staged = catalog.create_staged_instance(job.id, record.id, "AAA001L9", "/big.bin.shard0") + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.interrupted_job_ids == [job.id] + [listed] = report.staged_instances[job.id] + assert (listed.instance_id, listed.barcode, listed.tape_path, listed.shard_index) == ( + staged.id, + "AAA001L9", + "/big.bin.shard0", + 0, + ) + assert listed.state == "staging" + assert "recovered" in [entry.event for entry in catalog.job_journal(job.id)] + + +def test_recovery_reports_stale_pending_jobs_without_touching_them(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + stale = catalog.create_job("archive", {}) + fresh = catalog.create_job("archive", {}) + old_but_leased = catalog.create_job("archive", {}) + long_ago = datetime.now(UTC).replace(tzinfo=None) - DEFAULT_LEASE_TTL - timedelta(minutes=1) + for job in (stale, old_but_leased): + job.created_at = long_ago + catalog.session.commit() + _lease(catalog, old_but_leased.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.stale_pending_job_ids == [stale.id] + for job_id in (stale.id, fresh.id): + refreshed = catalog.get_job(job_id) + assert refreshed is not None + assert refreshed.state == "pending" From 4c213b9cadec680ecca1de7063adfd916a1bb487 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:33:19 +0000 Subject: [PATCH 082/205] test(compat): raw body, content_type, repeat and expected.headers in corpus harness Add inferred cases: XXE login body -> 400, rate-limited login -> 429 with Retry-After, expired session cookie -> 401 with a clearing Set-Cookie. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- compatibility/README.md | 11 ++++++ .../aml/auth-expired-cookie.json | 4 +++ .../aml/auth-login-rate-limited.json | 4 +++ .../aml/auth-login-xxe-body.json | 5 +++ tests/compat/test_compatibility_corpus.py | 34 ++++++++++++++++--- 5 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 compatibility/scalar-i3-341g/aml/auth-expired-cookie.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json diff --git a/compatibility/README.md b/compatibility/README.md index 20056dd..2c048af 100644 --- a/compatibility/README.md +++ b/compatibility/README.md @@ -65,3 +65,14 @@ exact body match (use for `captured` cases where the full body is authoritative) `iblade-rev-a/` is a scaffold profile for the Rev A (2017) iBlade surface. All its cases are `inferred`; see `iblade-rev-a/README.md`. The harness discovers it automatically (it globs `compatibility/**/*.json`). + +## Request/expectation keys beyond `json` + +- `request.body` — raw request body string, sent verbatim (use for XML or malformed + payloads). When present, `request.json` is ignored. +- `request.content_type` — sets the `Content-Type` header (overrides `request.headers`). + Omit it entirely to test a missing Content-Type. +- `request.repeat` — send the request this many extra times before the asserted one + (e.g. to trip the login rate limiter). The harness clears the limiter around each case. +- `expected.headers` — `{name: substring}`; each header must be present and contain the + substring (`""` asserts presence only). diff --git a/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json b/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json new file mode 100644 index 0000000..9fac14e --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-expired-cookie.json @@ -0,0 +1,4 @@ +{ "id": "aml-auth-expired-cookie", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "An unknown/expired sessionID cookie returns 401 and a Set-Cookie that clears it. Inferred; unverified against an appliance.", + "request": { "method": "GET", "path": "/aml/physicalLibrary/elements", "headers": { "Cookie": "sessionID=expired-session" } }, + "expected": { "status": 401, "headers": { "Set-Cookie": "Max-Age=0" } } } diff --git a/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json b/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json new file mode 100644 index 0000000..faf8077 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-rate-limited.json @@ -0,0 +1,4 @@ +{ "id": "aml-auth-login-rate-limited", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "Repeated failed logins from one client return 429 with Retry-After. Default limit 10/300s. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "json": {"name": "admin", "password": "wrong"}, "repeat": 10 }, + "expected": { "status": 429, "headers": { "Retry-After": "" } } } diff --git a/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json b/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json new file mode 100644 index 0000000..60ffd41 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-xxe-body.json @@ -0,0 +1,5 @@ +{ "id": "aml-auth-login-xxe-body", "profile": "scalar-i3-341g", "source": "inferred", "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", + "fidelity_notes": "XML login body with a DOCTYPE/external entity is rejected with 400. Inferred; unverified against an appliance.", + "request": { "method": "POST", "path": "/aml/users/login", "content_type": "application/xml", + "body": "]>&x;p" }, + "expected": { "status": 400 } } diff --git a/tests/compat/test_compatibility_corpus.py b/tests/compat/test_compatibility_corpus.py index a1be904..fb536e3 100644 --- a/tests/compat/test_compatibility_corpus.py +++ b/tests/compat/test_compatibility_corpus.py @@ -14,11 +14,13 @@ from __future__ import annotations import json +from collections.abc import Iterator from pathlib import Path import pytest from fastapi.testclient import TestClient +from openblade.api import routes_aml_auth from openblade.api.main import app from openblade.bootstrap import create_context, reset_context from openblade.config import OpenBladeConfig @@ -36,9 +38,12 @@ def _load_cases() -> list[dict]: @pytest.fixture() -def client(tmp_path: Path) -> TestClient: +def client(tmp_path: Path) -> Iterator[TestClient]: reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'compat.db'}"))) - return TestClient(app) + # The login limiter is module-global; isolate cases that use request.repeat. + routes_aml_auth._login_attempts.clear() + yield TestClient(app) + routes_aml_auth._login_attempts.clear() def _authenticate(client: TestClient) -> None: @@ -53,9 +58,27 @@ def _run(client: TestClient, case: dict): if case.get("auth") == "admin": _authenticate(client) req = case["request"] - return client.request( - req["method"], req["path"], json=req.get("json"), headers=req.get("headers") - ) + headers = dict(req.get("headers") or {}) + if "content_type" in req: + headers["Content-Type"] = req["content_type"] + kwargs: dict[str, object] = {"headers": headers or None} + if "body" in req: + kwargs["content"] = req["body"] + else: + kwargs["json"] = req.get("json") + # request.repeat: send the request this many extra times first (e.g. to trip a limiter). + for _ in range(int(req.get("repeat", 0))): + client.request(req["method"], req["path"], **kwargs) + return client.request(req["method"], req["path"], **kwargs) + + +def _header_problems(case: dict, resp) -> list[str]: + problems = [] + for name, needle in case["expected"].get("headers", {}).items(): + value = resp.headers.get(name) + if value is None or needle not in value: + problems.append(f"header {name}={value!r} lacks {needle!r}") + return problems def _subset_matches(expected: dict, actual: dict) -> list[str]: @@ -82,6 +105,7 @@ def test_inferred_case_behaviour_is_stable(case: dict, client: TestClient) -> No assert resp.status_code == case["expected"]["status"], ( f"{case['id']}: status {resp.status_code} != {case['expected']['status']}" ) + assert not _header_problems(case, resp), f"{case['id']}: {_header_problems(case, resp)}" if "json_contains" in case["expected"]: problems = _subset_matches(case["expected"]["json_contains"], resp.json()) assert not problems, f"{case['id']}: {problems}" From f91518ed23dc4d814bcb355dc6e605db59bafd6a Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:33:33 +0000 Subject: [PATCH 083/205] docs: record phase-2 recovery report fields and staged-commit follow-up Architecture "Drive leases and recovery" now states the expired-lease rule accurately and lists staged_instances / stale_pending_job_ids. The decision record gains a "Follow-up landed 2026-10-09" section (states, journal, non-sharded leases, recovery report fields). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/architecture.md | 2 +- .../2026-10-09-persistent-drive-leases.md | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index bd55902..35c450c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ The catalog uses SQLite tables for volume groups, barcode assignments, file reco Jobs are stored in-process with explicit state transitions. The queue additionally tracks changer and drive ownership so concurrent operations cannot claim the same hardware resources. ## Drive leases and recovery -Drive exclusion for tape I/O is a catalog row, not process memory. `DriveScheduler` is a façade over a `LeaseStore`: `CatalogLeaseStore` in production (one per `AppContext`, shared by the API and the CLI through the same `drive_leases` table) and `InMemoryLeaseStore` for unit tests. Each lease carries a monotonic `fencing_token`; job modules call `scheduler.verify(handle)` before a read-write mount, a write batch, and unmount/unload, and `heartbeat` after writes, so a lease that expired or was released behind a running job aborts it with `StaleLeaseError` instead of writing to a drive it no longer owns. On startup `recover_after_restart` fails any job the previous process left `running`, releases its leases, and reports expected-vs-observed drive contents — it never moves media. The report is read-only at `GET /jobs/recovery`. Spec and rationale: `docs/decisions/2026-10-09-persistent-drive-leases.md`. +Drive exclusion for tape I/O is a catalog row, not process memory. `DriveScheduler` is a façade over a `LeaseStore`: `CatalogLeaseStore` in production (one per `AppContext`, shared by the API and the CLI through the same `drive_leases` table) and `InMemoryLeaseStore` for unit tests. Each lease carries a monotonic `fencing_token`; job modules call `scheduler.verify(handle)` before a read-write mount, a write batch, and unmount/unload, and `heartbeat` after writes, so a lease that expired or was released behind a running job aborts it with `StaleLeaseError` instead of writing to a drive it no longer owns. On startup `recover_after_restart` fails (`failed_recoverable`, journalled as `recovered`) every `running` job whose lease expired without a heartbeat — a `running` job with a live lease may belong to another process and is left alone — releases orphaned leases, and reports expected-vs-observed drive contents. It also reports, without changing anything, the uncommitted `STAGING`/`VERIFYING` instances of every `failed_recoverable` job (`staged_instances`: barcode, tape path, shard index, state — what to reconcile on tape before retry) and `pending` jobs older than one `DEFAULT_LEASE_TTL` with no live lease (`stale_pending_job_ids`). It never moves media. The report is read-only at `GET /jobs/recovery`. Spec and rationale: `docs/decisions/2026-10-09-persistent-drive-leases.md`. ## FUSE namespace The FUSE-oriented layer is intentionally thin: catalog entries define the namespace, and hydration delegates to restore workflows. This keeps the namespace authoritative and avoids bypassing safety or verification logic. diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 6e7f238..046d271 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -176,6 +176,37 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. - Explicit heartbeats run once per write batch; the scheduler's background heartbeat thread covers the gaps. +## Follow-up landed 2026-10-09 (phase 2: staged commits, journal, recovery report) + +Item 4's states and the job journal — listed under Non-goals above for phase 1 — +landed as a foundation plus three parallel workstreams: + +- **States.** `FileInstanceState.STAGING` / `VERIFYING` mark a sharded-archive + instance that is written but not committed. Instances are created STAGING + before their first write (`create_staged_instance`), moved to VERIFYING after + every lane wrote and read back its shard, and moved to ARCHIVED in ONE + all-or-nothing `mark_instances_archived` call only after the manifest/commit + marker (`finalize_tape_generation`, marker written last), clean unmount and + inventory reconcile. Staged instances are never listed as archived or + restorable. +- **Journal.** `job_journal` rows (`CatalogRepository.journal` / + `job_journal`) record lease_acquired/released, shard_staged, verify_started, + verify_finished, committed, failed, fenced_out, physical_state_unknown (a + cleanup step that raised; the original lane error is kept), and `recovered` + (written by startup recovery for each job it fails). +- **Non-sharded leases.** `run_archive_job` / `run_restore_job` take a + `DriveScheduler` and acquire, verify and release their drive through it like + the sharded path, so non-sharded and sharded jobs exclude each other across + processes; a stale lease aborts without touching hardware. +- **Recovery report fields.** `RecoveryReport` / `GET /jobs/recovery` gain + `staged_instances` — `{job_id: [{instance_id, barcode, tape_path, + shard_index, state}]}` for every `failed_recoverable` job, i.e. what to + reconcile on tape before retry — and `stale_pending_job_ids`: `pending` jobs + created more than one `DEFAULT_LEASE_TTL` ago with no live lease (judged from + the leases observed before recovery released any). Both are report-only: + no state change, no media movement. Resuming from staged shards remains + deferred. + ## Non-goals Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any From e4a3fceb65fe21b8dfd18302ba10c9d3303a9f31 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:35:11 +0000 Subject: [PATCH 084/205] test(e2e): rewrite asserts in protocol scenarios so failures show values Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/protocols/conftest.py b/tests/e2e/protocols/conftest.py index 3feaf78..6f2592e 100644 --- a/tests/e2e/protocols/conftest.py +++ b/tests/e2e/protocols/conftest.py @@ -11,7 +11,10 @@ import pytest -from tests.e2e.protocols import rig +# Before any test module imports scenarios, so its bare asserts show values. +pytest.register_assert_rewrite("tests.e2e.protocols.scenarios") + +from tests.e2e.protocols import rig # noqa: E402 -- must follow register_assert_rewrite MOUNTS = { "smb": "mount -t cifs //samba/{s} /mnt/smb/{s} -o guest,vers=3.0,cache=none,actimeo=0", From 966d39904c9321a6d73f646e1cbeae202ef87670 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:35:11 +0000 Subject: [PATCH 085/205] fix(nas): disable oplocks/SMB2 leases so SMB clients see hydrated files The hydrator swaps an offline stub for recalled bytes outside smbd, so smbd cannot break a client's handle lease. The kernel cifs client (deferred close, closetimeo=1) kept reusing its cached handle on the stub's old inode while it reopened faster than 1s, and never observed the recall. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/samba.py | 8 ++++++++ tests/test_nas_protocol_render.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/openblade/nas/samba.py b/openblade/nas/samba.py index ca42f61..b3e7d3e 100644 --- a/openblade/nas/samba.py +++ b/openblade/nas/samba.py @@ -31,6 +31,11 @@ def render_smb_conf(shares: Iterable[SambaShare], *, force_user: str = "root") - The global section serves guest SMB2/3 only (no SMB1) with POSIX locking on so byte-range locks are visible across protocols. + + Oplocks and SMB2 leases are off: the hydrator replaces offline stubs with + recalled bytes behind smbd's back, and smbd cannot break a client lease + for a change it did not make. A client holding a handle lease keeps its + (deferred-close) handle on the stub's old inode and never sees the recall. """ header = ( "[global]\n" @@ -42,6 +47,9 @@ def render_smb_conf(shares: Iterable[SambaShare], *, force_user: str = "root") - " load printers = no\n" " disable spoolss = yes\n" " posix locking = yes\n" + " oplocks = no\n" + " level2 oplocks = no\n" + " smb2 leases = no\n" " unix extensions = no\n" " log file = /var/log/samba/%m.log\n" ) diff --git a/tests/test_nas_protocol_render.py b/tests/test_nas_protocol_render.py index 7251ef4..e33de63 100644 --- a/tests/test_nas_protocol_render.py +++ b/tests/test_nas_protocol_render.py @@ -50,3 +50,11 @@ def test_smb_conf_maps_share_flags() -> None: ro, rw = conf.split("[ro]", 1)[1].split("[rw]", 1) assert "writeable = no" in ro and "case sensitive = no" in ro assert "writeable = yes" in rw and "case sensitive = yes" in rw + + +def test_smb_conf_disables_client_caching_of_hydrating_files() -> None: + conf = render_smb_conf([SambaShare("rw", Path("/e/rw"), read_only=False)]) + globals_ = conf.split("[rw]", 1)[0] + + for option in ("oplocks = no", "level2 oplocks = no", "smb2 leases = no"): + assert option in globals_ From bd9dffe24cb00f0eb2793fbe70a775884e2907af Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:35:12 +0000 Subject: [PATCH 086/205] fix(e2e): tolerate .req vanishing mid-create under concurrent SMB recalls When the shim unlinks one client's request while another client's SMB create of the same name is in flight, the create fails ENOENT; the recall is already queued, so the poller just keeps polling. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/scenarios.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/protocols/scenarios.py b/tests/e2e/protocols/scenarios.py index 178a244..f64b091 100644 --- a/tests/e2e/protocols/scenarios.py +++ b/tests/e2e/protocols/scenarios.py @@ -17,7 +17,10 @@ POLL = """ import sys, time p, deadline = sys.argv[1], time.time() + float(sys.argv[2]) -open(p.replace('/offline/', '/.ob-control/offline/') + '.req', 'w').close() +try: + open(p.replace('/offline/', '/.ob-control/offline/') + '.req', 'w').close() +except FileNotFoundError: + pass # SMB: shim unlinked a concurrent client's .req mid-create; recall already queued while time.time() < deadline: data = open(p, 'rb').read() if not data.startswith(b'OPENBLADE-OFFLINE-STUB'): From 9a1f977145543777efd930336d1452a72a77fddd Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:35:14 +0000 Subject: [PATCH 087/205] fix(fuse): waiters pin the cache entry so a batch cannot evict it unread Hydrator.wait() acquires a cache pin before the restore commits and releases it on every exit, so storing a later file of the same batch can no longer evict an earlier one before its waiter reads it (was FileNotFoundError -> EIO). mount.open pins before materialising and releases if materialising fails. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 17 +++++++++---- openblade/fuse/mount.py | 13 +++++++--- tests/integration/test_fuse_data_plane.py | 31 +++++++++++++++++++++++ 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 8790376..b235651 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -251,11 +251,18 @@ def request(self, catalog_path: str) -> HydrationTicket: return ticket def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: - if not ticket.done.wait(timeout): - raise HydrationTimeoutError(f"{ticket.catalog_path} still hydrating") - if ticket.error is not None: - raise HydrationFailedError(ticket.error) - return self.cache.retrieve(ticket.checksum) + # Each waiter pins the entry before the restore commits, so the budget + # cannot evict it (e.g. while a later file of the same batch is stored) + # before this waiter has read it. Released on every exit path. + self.cache.acquire(ticket.checksum) + try: + if not ticket.done.wait(timeout): + raise HydrationTimeoutError(f"{ticket.catalog_path} still hydrating") + if ticket.error is not None: + raise HydrationFailedError(ticket.error) + return self.cache.retrieve(ticket.checksum) + finally: + self.cache.release(ticket.checksum) def shutdown(self, timeout: float = 30.0) -> None: """Cancel batches not yet started and wait for in-flight restores.""" diff --git a/openblade/fuse/mount.py b/openblade/fuse/mount.py index 61e339b..8f106ac 100644 --- a/openblade/fuse/mount.py +++ b/openblade/fuse/mount.py @@ -246,14 +246,21 @@ def open(self, path: str, flags: int) -> int: entry = self._entry(path) if entry.is_dir: raise OSError(errno.EISDIR, os.strerror(errno.EISDIR), path) - data = self._materialise(str(entry.path)) + # Pin before materialising so the entry cannot be evicted in between. + record = self.filesystem.catalog.get_file_record(str(entry.path)) + if record is not None: + self.filesystem.cache.acquire(record.checksum_sha256) + try: + data = self._materialise(str(entry.path)) + except BaseException: + if record is not None: + self.filesystem.cache.release(record.checksum_sha256) + raise with self._lock: handle = self._next_handle self._next_handle += 1 self._open_files[handle] = data - record = self.filesystem.catalog.get_file_record(str(entry.path)) if record is not None: - self.filesystem.cache.acquire(record.checksum_sha256) with self._lock: self._handle_checksums[handle] = record.checksum_sha256 return handle diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 3d866a6..4db042b 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -254,3 +254,34 @@ def test_resume_gives_up_at_deadline_with_typed_error(catalog, tmp_path): # typ ) with pytest.raises(HydrationTimeoutError): engine.resume(str(job.id)) + + +def test_batch_member_is_not_evicted_before_its_waiter_reads_it(catalog, tmp_path): # type: ignore[no-untyped-def] + class SlowReader(HydrationCache): + def retrieve(self, checksum: str) -> bytes: + threading.Event().wait(0.3) # let the rest of the batch commit first + return super().retrieve(checksum) + + cache = SlowReader(str(tmp_path / "cache"), max_bytes=600) # a (500) + b (250) > 600 + engine = FakeEngine(tmp_path / "s") + engine.gate.clear() + hyd = Hydrator(catalog, cache, engine, batch_window_s=0.05) + tickets = [hyd.request("/a.bin"), hyd.request("/b.bin")] + results: dict[str, object] = {} + + def waiter(t): # type: ignore[no-untyped-def] + try: + results[t.catalog_path] = hyd.wait(t, 5) + except Exception as exc: # noqa: BLE001 - the test records whatever the waiter saw + results[t.catalog_path] = exc + + threads = [threading.Thread(target=waiter, args=(t,)) for t in tickets] + for th in threads: + th.start() + threading.Event().wait(0.2) + engine.gate.set() + for th in threads: + th.join(10) + assert results == {"/a.bin": FILES["/a.bin"][0], "/b.bin": FILES["/b.bin"][0]} + assert cache.open_count(tickets[0].checksum) == 0 # waiter pins are released + assert cache.open_count(tickets[1].checksum) == 0 From 67a7440066c5b1de48ac3c5f8b0d3c934d512399 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:36:04 +0000 Subject: [PATCH 088/205] docs: scope phase-2 follow-up to what this branch carries Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/decisions/2026-10-09-persistent-drive-leases.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 046d271..8077f5e 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -179,7 +179,11 @@ VERIFYING instance states from item 4) — say so in the PR, do not stub it. ## Follow-up landed 2026-10-09 (phase 2: staged commits, journal, recovery report) Item 4's states and the job journal — listed under Non-goals above for phase 1 — -landed as a foundation plus three parallel workstreams: +landed as a foundation plus three parallel workstreams. The foundation (states, +`journal`/`job_journal`, staged-instance repository methods) and the recovery report +are on this branch; the sharded-archive wiring of those states and journal events and +the non-sharded leases come from the sibling `fleet/jobs-sharded` and +`fleet/jobs-leases` branches and are only true once those merge: - **States.** `FileInstanceState.STAGING` / `VERIFYING` mark a sharded-archive instance that is written but not committed. Instances are created STAGING @@ -202,8 +206,9 @@ landed as a foundation plus three parallel workstreams: `staged_instances` — `{job_id: [{instance_id, barcode, tape_path, shard_index, state}]}` for every `failed_recoverable` job, i.e. what to reconcile on tape before retry — and `stale_pending_job_ids`: `pending` jobs - created more than one `DEFAULT_LEASE_TTL` ago with no live lease (judged from - the leases observed before recovery released any). Both are report-only: + created more than one `DEFAULT_LEASE_TTL` ago. Jobs mark themselves `running` + before they acquire a lease, so in practice this is an age signal: a job + legitimately queued longer than the TTL is reported too. Both are report-only: no state change, no media movement. Resuming from staged shards remains deferred. From 8d677c0498e05f4b57159435ba5e12656dd9be35 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:36:35 +0000 Subject: [PATCH 089/205] docs(nas): correct protocol rig mounts/ports and record SMB caching semantics Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/wiki/guides/fuse-and-nas.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/wiki/guides/fuse-and-nas.md b/docs/wiki/guides/fuse-and-nas.md index d8df05b..7b8d7d9 100644 --- a/docs/wiki/guides/fuse-and-nas.md +++ b/docs/wiki/guides/fuse-and-nas.md @@ -212,8 +212,9 @@ stack so the protocol exports can be exercised by real clients. Services: - `samba` — serves shares from an `smb.conf` rendered by `openblade/nas/samba.py`. - `nfs-ganesha` — serves exports from a config rendered by `openblade/nas/nfs.py`. -- `client` — a privileged container that mounts the NFS export internally - (mounting needs privileges the CI runner's test process does not have). +- `client` — a privileged container that kernel-mounts both protocols + internally (SMB via `cifs vers=3.0,cache=none,actimeo=0`, NFS via `nfs4`); + mounting needs privileges the CI runner's test process does not have. Both config files are rendered from `NasShare` records, so the rig tests the renderers' output rather than hand-written configs. @@ -226,7 +227,9 @@ make protocols-up && make test-protocols && make protocols-down `protocols-up` builds, starts and waits for healthy; `test-protocols` runs `python -m pytest -m protocols tests/e2e/protocols -q`; `protocols-down` tears -down with `-v`. Host ports come from `OB_SMB_PORT` and `OB_NFS_PORT`. Tests +down with `-v`. Host ports come from `OB_SMB_PORT` and `OB_NFS_PORT` +(Makefile defaults 14450 and 20490); the pytest fixture instead picks free +ports when those variables are unset. Tests carry the `protocols` marker and skip, with a reason naming the missing capability, when the rig is not available. @@ -235,7 +238,19 @@ hydrator. The shim models a tape recall: a stub file becomes real content after a delay, or never does for an offline cartridge. It exists to check how SMB and NFS clients behave while a read blocks or times out. The real hydrator is the FUSE path described above, and it is not exercised by the rig, so rig results -say nothing about actual recall from tape. +say nothing about actual recall from tape. The offline stub file is a rig-only +format: the catalog has no on-disk OFFLINE_ON_TAPE representation. + +**SMB caching semantics.** The shim (like an evictor) swaps files outside +smbd, so smbd cannot break client leases. The rendered `smb.conf` therefore +sets `oplocks = no`, `level2 oplocks = no` and `smb2 leases = no`; with leases +on, the Linux cifs client's deferred close (`closetimeo=1`) kept a cached +handle on the stub's old inode while a poller reopened it more often than once +a second, and the recall was never seen. Observed with leases off: a handle +opened before an eviction keeps reading the old bytes, and while that handle +stays open a reopen by path also returns the old bytes; after it closes, the +next open sees the new file. A Samba restart mid-recall is survived (the soft +mount reconnects and the poll completes). **CI.** `.github/workflows/nas-protocols.yml` runs the rig nightly and on manual dispatch. It is not triggered by pull requests or pushes and is not a required From c81e74e5e9e486289f5f87249626fdfd9c6aad71 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:36:58 +0000 Subject: [PATCH 090/205] fix(fuse): shutdown never strands a racing ticket; one overall wait deadline A request() that passed the _closed check before shutdown() now fails its ticket with the 'cancelled by unmount' message instead of queueing a batch that _flush silently discards; _flush also fails any batch it drops. shutdown() joins workers against one overall deadline, not timeout per worker. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 37 ++++++++++++++++------- tests/integration/test_fuse_data_plane.py | 31 +++++++++++++++++++ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index b235651..889c461 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) _UNEXPECTED = "restore failed unexpectedly" +_UNMOUNTING = "hydration cancelled by unmount" @dataclass @@ -242,12 +243,19 @@ def request(self, catalog_path: str) -> HydrationTicket: self._spawn(self._resume, ticket) return ticket with self._lock: - self._pending.setdefault(ticket.tape_key, []).append(ticket) - if ticket.tape_key not in self._timers: - timer = threading.Timer(self.batch_window_s, self._flush, args=(ticket.tape_key,)) - timer.daemon = True - self._timers[ticket.tape_key] = timer - timer.start() + closed = self._closed + if not closed: + self._pending.setdefault(ticket.tape_key, []).append(ticket) + if ticket.tape_key not in self._timers: + timer = threading.Timer( + self.batch_window_s, self._flush, args=(ticket.tape_key,) + ) + timer.daemon = True + self._timers[ticket.tape_key] = timer + timer.start() + if closed: + # shutdown() ran after the _closed check above; never strand the ticket. + self._fail(ticket, _UNMOUNTING) return ticket def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: @@ -275,9 +283,10 @@ def shutdown(self, timeout: float = 30.0) -> None: timer.cancel() for tickets in pending.values(): for ticket in tickets: - self._fail(ticket, "hydration cancelled by unmount") + self._fail(ticket, _UNMOUNTING) + deadline = time.monotonic() + timeout for worker in workers: - worker.join(timeout) + worker.join(max(0.0, deadline - time.monotonic())) # -- internals ----------------------------------------------------------- def _spawn(self, target: Any, *args: Any) -> None: @@ -291,9 +300,15 @@ def _flush(self, tape_key: str) -> None: with self._lock: self._timers.pop(tape_key, None) batch = self._pending.pop(tape_key, []) - if self._closed or not batch: - return - self._workers.append(threading.current_thread()) + closed = self._closed + if not closed and batch: + self._workers.append(threading.current_thread()) + if closed: + for ticket in batch: + self._fail(ticket, _UNMOUNTING) + return + if not batch: + return try: staged = self.engine.restore_batch(tape_key, [t.catalog_path for t in batch]) except (HydrationFailedError, OpenBladeError) as exc: diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 4db042b..7738dc5 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -7,6 +7,7 @@ import json import os import threading +import time from pathlib import Path import pytest @@ -285,3 +286,33 @@ def waiter(t): # type: ignore[no-untyped-def] assert results == {"/a.bin": FILES["/a.bin"][0], "/b.bin": FILES["/b.bin"][0]} assert cache.open_count(tickets[0].checksum) == 0 # waiter pins are released assert cache.open_count(tickets[1].checksum) == 0 + + +def test_request_racing_shutdown_fails_ticket_instead_of_stranding_it(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + hyd = Hydrator(catalog, fs.cache, FakeEngine(tmp_path / "s"), batch_window_s=0.01) + real_tape_key = hyd._tape_key + + def shutdown_mid_request(path, record): # type: ignore[no-untyped-def] + hyd.shutdown(timeout=1) # lands after request() passed its _closed check + return real_tape_key(path, record) + + hyd._tape_key = shutdown_mid_request # type: ignore[method-assign] + ticket = hyd.request("/a.bin") + assert ticket.done.wait(2), "ticket stranded by shutdown" + assert ticket.error == "hydration cancelled by unmount" + + +def test_shutdown_waits_on_one_overall_deadline(catalog, tmp_path): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s") + engine.gate.clear() + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + hyd.request("/a.bin") # TAPE01 + hyd.request("/c.bin") # TAPE02 -> a second blocked worker + threading.Event().wait(0.2) + started = time.monotonic() + hyd.shutdown(timeout=0.4) + elapsed = time.monotonic() - started + engine.gate.set() + assert elapsed < 0.6, f"shutdown took {elapsed:.2f}s for a 0.4s budget" From d9894e5a9ebea72bd72c9bba0dd18ab533d3ec2f Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:38:16 +0000 Subject: [PATCH 091/205] fix(recovery): keep live leases whose owner is still pending A pending job's live lease may belong to a process that is about to run it; only expired leases are reclaimed from non-terminal owners. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/recovery.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index f7e17a5..7ee016f 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -16,6 +16,8 @@ logger = logging.getLogger(__name__) +_NON_TERMINAL_JOB_STATES = frozenset({"pending", "running"}) + INTERRUPTED_ERROR = ( "interrupted: drive lease expired without a heartbeat; physical state unknown — " "reconcile before retry" @@ -74,7 +76,9 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - leased_job_ids = {lease.job_id for lease in live} for lease in live: owner = catalog.get_job(lease.job_id) - if owner is None or owner.state != "running": + # A pending or running owner may belong to another live process; its + # lease is only reclaimed once it expires (handled above). + if owner is None or owner.state not in _NON_TERMINAL_JOB_STATES: orphaned.append(lease) catalog.release_leases([lease.id for lease in orphaned]) From cc35a6e574fbe47a985b3a5424d9385469553ce5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:38:19 +0000 Subject: [PATCH 092/205] fix(fuse): delete the staging copy after a verified store or checksum failure Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 10 ++++++++++ tests/integration/test_fuse_data_plane.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 889c461..b4f862d 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -344,15 +344,25 @@ def _commit(self, ticket: HydrationTicket, staged: Path | None) -> None: try: self.cache.store_verified(ticket.checksum, staged) except CacheChecksumError: + self._discard_staged(staged) self._fail(ticket, "restored bytes failed checksum verification") return except OSError: self._fail(ticket, "restore I/O error") return + self._discard_staged(staged) with self._lock: self._errors.pop(ticket.catalog_path, None) ticket.done.set() + @staticmethod + def _discard_staged(staged: Path) -> None: + """Remove the staging copy once the cache holds (or rejected) its bytes.""" + try: + staged.unlink(missing_ok=True) + except OSError: + logger.warning("fuse: could not remove staging file %s", staged) + def _fail(self, ticket: HydrationTicket, reason: str) -> None: ticket.error = reason with self._lock: diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 7738dc5..9f719e1 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -316,3 +316,14 @@ def test_shutdown_waits_on_one_overall_deadline(catalog, tmp_path): # type: ign elapsed = time.monotonic() - started engine.gate.set() assert elapsed < 0.6, f"shutdown took {elapsed:.2f}s for a 0.4s budget" + + +@pytest.mark.parametrize("corrupt", [False, True]) +def test_staging_copy_is_removed_after_verified_store(catalog, tmp_path, corrupt): # type: ignore[no-untyped-def] + fs = CatalogFilesystem(catalog, cache_dir=str(tmp_path / "cache")) + engine = FakeEngine(tmp_path / "s", corrupt=corrupt) + hyd = Hydrator(catalog, fs.cache, engine, batch_window_s=0.01) + ticket = hyd.request("/c.bin") + assert ticket.done.wait(3) + assert (ticket.error is not None) is corrupt + assert not (tmp_path / "s" / "c.bin").exists(), "staging copy leaked" From b18c7b641761e5a043694f2c84e82f9f3eb6b054 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:40:47 +0000 Subject: [PATCH 093/205] ci: add changed-path ownership check (tools/ci_ownership.py) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/unit/test_ci_ownership.py | 89 +++++++++++++++ tools/ci_ownership.py | 121 +++++++++++++++++++++ tools/ci_ownership.toml | 186 ++++++++++++++++++++++++++++++++ 3 files changed, 396 insertions(+) create mode 100644 tests/unit/test_ci_ownership.py create mode 100644 tools/ci_ownership.py create mode 100644 tools/ci_ownership.toml diff --git a/tests/unit/test_ci_ownership.py b/tests/unit/test_ci_ownership.py new file mode 100644 index 0000000..85cf402 --- /dev/null +++ b/tests/unit/test_ci_ownership.py @@ -0,0 +1,89 @@ +"""Tests for tools/ci_ownership.py (changed-path -> CI category mapper).""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "ci_ownership", ROOT / "tools" / "ci_ownership.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_ownership"] = module + spec.loader.exec_module(module) + return module + + +ci = _load() +RULES = ci.load_rules(ROOT / "tools" / "ci_ownership.toml") + + +def test_scheduler_change_selects_safety_and_unit() -> None: + sel = ci.select(["openblade/jobs/scheduler.py"], RULES) + assert {"unit", "safety"} <= sel.categories + assert sel.unowned == () + + +def test_docs_only_change_selects_only_docs() -> None: + sel = ci.select(["docs/test-plan.md", "README.md"], RULES) + assert sel.categories == frozenset({"docs-only"}) + assert sel.unowned == () + + +def test_unowned_openblade_path_is_reported() -> None: + sel = ci.select(["openblade/brand_new_pkg/mod.py", "tests/unit/test_x.py"], RULES) + assert sel.unowned == ("openblade/brand_new_pkg/mod.py",) + + +def test_unowned_tests_path_is_reported() -> None: + sel = ci.select(["tests/new_lane/test_y.py"], RULES) + assert sel.unowned == ("tests/new_lane/test_y.py",) + + +def test_unmatched_path_outside_enforced_trees_is_not_a_failure() -> None: + sel = ci.select(["scripts/whatever.sh"], RULES) + assert sel.unowned == () + assert sel.categories == frozenset() + + +def test_main_exits_1_on_unowned_path(capsys: pytest.CaptureFixture[str]) -> None: + assert ci.main(["--paths", "openblade/brand_new_pkg/mod.py"]) == 1 + captured = capsys.readouterr() + assert "UNOWNED" in captured.out + assert "FAIL" in captured.err + + +def test_main_exits_0_and_prints_reasoning(capsys: pytest.CaptureFixture[str]) -> None: + assert ci.main(["--paths", "openblade/domain/policies.py"]) == 0 + out = capsys.readouterr().out + assert "openblade/domain/* -> unit,safety" in out + assert "selected categories:" in out + + +def test_invalid_category_in_table_is_rejected(tmp_path: Path) -> None: + table = tmp_path / "t.toml" + table.write_text('[[rule]]\nglob = "x/*"\ncategories = ["bogus"]\n', encoding="utf-8") + with pytest.raises(ValueError, match="bogus"): + ci.load_rules(table) + + +def test_every_tracked_file_is_owned() -> None: + files = subprocess.run( + ["git", "ls-files", "openblade", "tests"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.split() + assert files + assert ci.select(files, RULES).unowned == () diff --git a/tools/ci_ownership.py b/tools/ci_ownership.py new file mode 100644 index 0000000..1745dbd --- /dev/null +++ b/tools/ci_ownership.py @@ -0,0 +1,121 @@ +"""Map changed paths to the CI test categories that own them. + +Usage (CI calls it this way, before any dependency is installed, so this +module is stdlib-only):: + + python tools/ci_ownership.py --base origin/master + python tools/ci_ownership.py --paths openblade/jobs/scheduler.py docs/x.md + +Prints the per-path selection reasoning and the selected category set. Exits +1 when any changed ``openblade/**`` or ``tests/**`` path has no owning +category in ``tools/ci_ownership.toml``. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tomllib +from collections.abc import Sequence +from dataclasses import dataclass +from fnmatch import fnmatchcase +from pathlib import Path + +CATEGORIES = frozenset( + {"unit", "integration", "safety", "i3", "compat", "frontend", "docs-only", "ci-only"} +) +ENFORCED_PREFIXES = ("openblade/", "tests/") +DEFAULT_TABLE = Path(__file__).with_name("ci_ownership.toml") + + +@dataclass(frozen=True) +class Rule: + glob: str + categories: tuple[str, ...] + + +@dataclass(frozen=True) +class Selection: + owners: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] + """path -> ((matching glob, its categories), ...)""" + categories: frozenset[str] + unowned: tuple[str, ...] + """Enforced (openblade/ or tests/) paths that no rule matched.""" + + +def load_rules(table: Path) -> list[Rule]: + data = tomllib.loads(table.read_text(encoding="utf-8")) + rules: list[Rule] = [] + for raw in data.get("rule", []): + cats = tuple(raw["categories"]) + bad = set(cats) - CATEGORIES + if bad or not cats: + raise ValueError(f"rule {raw['glob']!r}: invalid categories {sorted(bad) or '[]'}") + rules.append(Rule(glob=raw["glob"], categories=cats)) + return rules + + +def select(paths: Sequence[str], rules: Sequence[Rule]) -> Selection: + """Pure mapping: changed paths -> owning categories + unowned enforced paths.""" + owners: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = {} + categories: set[str] = set() + unowned: list[str] = [] + for path in paths: + matches = tuple((r.glob, r.categories) for r in rules if fnmatchcase(path, r.glob)) + owners[path] = matches + for _, cats in matches: + categories.update(cats) + if not matches and path.startswith(ENFORCED_PREFIXES): + unowned.append(path) + return Selection(owners=owners, categories=frozenset(categories), unowned=tuple(unowned)) + + +def changed_paths(base: str) -> list[str]: + out = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout + return [line for line in out.splitlines() if line.strip()] + + +def report(selection: Selection) -> str: + lines: list[str] = [] + for path, matches in selection.owners.items(): + if matches: + why = "; ".join(f"{glob} -> {','.join(cats)}" for glob, cats in matches) + lines.append(f" {path}: {why}") + elif path in selection.unowned: + lines.append(f" {path}: UNOWNED (no rule in ci_ownership.toml)") + else: + lines.append(f" {path}: no rule (outside openblade/ and tests/, not enforced)") + lines.append(f"selected categories: {', '.join(sorted(selection.categories)) or '(none)'}") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--base", help="git ref to diff against (base...HEAD)") + source.add_argument("--paths", nargs="+", help="explicit paths instead of a git diff") + parser.add_argument("--table", type=Path, default=DEFAULT_TABLE) + args = parser.parse_args(argv) + + paths = list(args.paths) if args.paths else changed_paths(args.base) + selection = select(paths, load_rules(args.table)) + print(f"ci-ownership: {len(paths)} changed path(s)") + print(report(selection)) + if selection.unowned: + print( + f"ci-ownership: FAIL - {len(selection.unowned)} path(s) under openblade/ or tests/ " + "have no owning category; add a rule to tools/ci_ownership.toml", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml new file mode 100644 index 0000000..81b973a --- /dev/null +++ b/tools/ci_ownership.toml @@ -0,0 +1,186 @@ +# Changed-path -> test-category ownership table for tools/ci_ownership.py. +# +# Every rule whose glob matches a path contributes its categories (union). +# Globs use fnmatch semantics: `*` also spans `/`, so "openblade/api/*" +# covers the whole subtree. There is deliberately NO catch-all +# "openblade/*" rule: a new subpackage must be given an owner here, or +# the ci-gate ownership check fails. +# +# Valid categories: unit, integration, safety, i3, compat, frontend, +# docs-only, ci-only. + +# --- tests ----------------------------------------------------------------- +[[rule]] +glob = "tests/unit/*" +categories = ["unit"] + +[[rule]] +glob = "tests/property/*" +categories = ["unit"] + +[[rule]] +glob = "tests/integration/*" +categories = ["integration"] + +[[rule]] +glob = "tests/fault/*" +categories = ["integration"] + +[[rule]] +glob = "tests/e2e/*" +categories = ["integration"] + +[[rule]] +glob = "tests/hardware/*" +categories = ["integration"] + +[[rule]] +glob = "tests/safety/*" +categories = ["safety"] + +[[rule]] +glob = "tests/i3/*" +categories = ["i3"] + +[[rule]] +glob = "tests/compat/*" +categories = ["compat"] + +[[rule]] +glob = "tests/fixtures/*" +categories = ["unit", "compat"] + +[[rule]] +glob = "tests/emulator_spec/*" +categories = ["i3"] + +# Shared test scaffolding at the top of tests/. +[[rule]] +glob = "tests/conftest.py" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "tests/__init__.py" +categories = ["unit"] + +[[rule]] +glob = "tests/assistant_support.py" +categories = ["unit", "safety"] + +[[rule]] +glob = "tests/gen_wiki_reference.py" +categories = ["docs-only"] + +# --- openblade ------------------------------------------------------------- +[[rule]] +glob = "openblade/__init__.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/api/*" +categories = ["unit", "integration", "i3"] + +[[rule]] +glob = "openblade/emulator_contract/*" +categories = ["unit", "i3", "compat"] + +[[rule]] +glob = "openblade/assistant/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/safety/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/domain/*" +categories = ["unit", "safety"] + +[[rule]] +glob = "openblade/jobs/*" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "openblade/catalog/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/hardware/*" +categories = ["unit", "integration", "compat"] + +[[rule]] +glob = "openblade/simulator/*" +categories = ["unit", "integration", "i3"] + +[[rule]] +glob = "openblade/nas/*" +categories = ["unit", "integration", "safety"] + +[[rule]] +glob = "openblade/fuse/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/sftp/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/cli/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/testing/*" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/web_flask/*" +categories = ["unit", "integration", "frontend"] + +[[rule]] +glob = "openblade/bootstrap.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/config.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/config_validation.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/deploy.py" +categories = ["unit"] + +[[rule]] +glob = "openblade/dr.py" +categories = ["unit", "integration"] + +[[rule]] +glob = "openblade/topology.py" +categories = ["unit", "integration"] + +# --- outside the enforced trees (informational) ------------------------------ +[[rule]] +glob = "docs/*" +categories = ["docs-only"] + +[[rule]] +glob = "*.md" +categories = ["docs-only"] + +[[rule]] +glob = ".github/*" +categories = ["ci-only"] + +[[rule]] +glob = "tools/ci_ownership*" +categories = ["ci-only"] + +[[rule]] +glob = "Makefile" +categories = ["ci-only"] + +[[rule]] +glob = ".yamllint" +categories = ["ci-only"] From 22d6f41aa22374152ba97d000f7515d8a3f59b99 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:40:47 +0000 Subject: [PATCH 094/205] ci: SHA-pin ci.yml, add ownership job to ci-gate, drop stale advisory comment Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 100067d..fd986d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,10 @@ jobs: cicd: ${{ steps.filter.outputs.cicd }} wiki: ${{ steps.filter.outputs.wiki }} steps: - - uses: actions/checkout@v5 - - uses: dorny/paths-filter@v3 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3.0.4 id: filter with: filters: | @@ -99,8 +101,10 @@ jobs: # goes red, the fix is `ruff check --fix` / `ruff format` with the PINNED # version, never a version bump in the same change. steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -122,8 +126,10 @@ jobs: # `# type: ignore` comments left in openblade/ (down from 18) carries its # justification at the site. steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -136,8 +142,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.backend == 'true' || needs.changes.outputs.api == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -162,8 +170,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.api == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -185,8 +195,10 @@ jobs: env: I3_TIMING_PROFILE: instant steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -216,8 +228,10 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: "npm" @@ -234,17 +248,16 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.webflask == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" - run: pip install -e '.[dev]' - # BLOCKING mypy, scoped to this package alone. `mypy openblade` in - # backend-typecheck has to stay advisory because of ~300 pre-existing errors - # in the god-file modules; openblade/web_flask is clean under strict mode - # today, and this job is what keeps it that way instead of letting it join - # the backlog. + # BLOCKING mypy, scoped to this package alone, so a web_flask-only change + # is gated here too (backend-typecheck is path-filtered to backend paths). - run: mypy openblade/web_flask # Redundant with backend-tests (tests/unit covers this file) but kept so the # job is a complete gate on its own for a web_flask-only change. Seconds. @@ -260,8 +273,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.wiki == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" @@ -278,8 +293,10 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.cicd == 'true' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - run: python -m pip install --upgrade pip pyyaml yamllint @@ -297,6 +314,26 @@ jobs: - name: yamllint workflows (real YAML errors — see .yamllint) run: yamllint -c .yamllint .github/workflows/ + # Changed-path ownership: every changed openblade/** or tests/** path must map + # to a test category in tools/ci_ownership.toml, so a new module can't land + # outside every CI lane unnoticed. Always runs (not path-filtered). + ownership: + name: ownership + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Check changed paths have an owning test category + env: + # PRs diff against their target branch; pushes/dispatch against master. + BASE_REF: ${{ github.base_ref || 'master' }} + run: python tools/ci_ownership.py --base "origin/$BASE_REF" + # Single always-running aggregation gate. Make THIS the only required # branch-protection check (plus operability-gate). It understands a valid # `skipped` result from path-filtered jobs and fails on any real failure or @@ -305,6 +342,7 @@ jobs: ci-gate: name: ci-gate needs: + - ownership - backend-lint - backend-typecheck - backend-tests @@ -326,6 +364,7 @@ jobs: *) echo "FAIL $1=$2"; fail=1 ;; esac } + check ownership "${{ needs.ownership.result }}" check backend-tests "${{ needs.backend-tests.result }}" check api-aml-integration "${{ needs.api-aml-integration.result }}" check i3-smoke "${{ needs.i3-smoke.result }}" @@ -360,8 +399,10 @@ jobs: env: I3_TIMING_PROFILE: instant steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" From 953b18e36ca38358fc8bef861010cd9b1b3efe8f Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:40:57 +0000 Subject: [PATCH 095/205] test: add stress/fuzz/rebuild/mutation markers, enable --strict-markers Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- pyproject.toml | 6 +++++- tests/fault/test_fault_injection.py | 2 ++ tests/property/test_robot_transitions.py | 3 +++ tests/unit/test_catalog_rebuild.py | 2 ++ tests/unit/test_catalog_rebuild_worker.py | 2 ++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5cf6af1..7f39af7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ plugins = [] warn_unused_configs = true [tool.pytest.ini_options] -addopts = "-ra" +addopts = "-ra --strict-markers" asyncio_default_fixture_loop_scope = "function" testpaths = ["tests"] markers = [ @@ -134,4 +134,8 @@ markers = [ "slow: long-running simulator stress tests", "i3: Quantum i3 emulator and protocol tests", "real_i3: tests requiring a physical Quantum Scalar i3", + "stress: fault-injection / stress tests (nightly stress lane)", + "fuzz: property-based / fuzz tests (nightly fuzz lane)", + "rebuild: catalog rebuild tests (nightly rebuild lane)", + "mutation: tests used as the kill-set for the mutation lane", ] diff --git a/tests/fault/test_fault_injection.py b/tests/fault/test_fault_injection.py index af02852..a00a9d4 100644 --- a/tests/fault/test_fault_injection.py +++ b/tests/fault/test_fault_injection.py @@ -22,6 +22,8 @@ from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend +pytestmark = pytest.mark.stress + def _confirmation(barcode: str): from openblade.domain.policies import FormatConfirmation, SafetyToken diff --git a/tests/property/test_robot_transitions.py b/tests/property/test_robot_transitions.py index 62fe713..d04a1ab 100644 --- a/tests/property/test_robot_transitions.py +++ b/tests/property/test_robot_transitions.py @@ -2,6 +2,7 @@ from pathlib import Path, PurePosixPath +import pytest from hypothesis import given, settings from hypothesis import strategies as st @@ -10,6 +11,8 @@ from openblade.domain.policies import FormatConfirmation, SafetyToken from openblade.simulator.scenarios import two_drive_library +pytestmark = pytest.mark.fuzz + @given(st.lists(st.integers(min_value=1, max_value=20), min_size=0, max_size=50)) @settings(max_examples=50) diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index e2d0f87..e4c1654 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -32,6 +32,8 @@ RebuildRunStatus, ) +pytestmark = pytest.mark.rebuild + client = TestClient(app) diff --git a/tests/unit/test_catalog_rebuild_worker.py b/tests/unit/test_catalog_rebuild_worker.py index 032789a..7dbd213 100644 --- a/tests/unit/test_catalog_rebuild_worker.py +++ b/tests/unit/test_catalog_rebuild_worker.py @@ -39,6 +39,8 @@ RebuildRunStatus, ) +pytestmark = pytest.mark.rebuild + client = TestClient(app) From d8afb6e7602a300c36173a0910e7143cb7c4a219 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:40:58 +0000 Subject: [PATCH 096/205] ci: add mutmut mutation lane with survivor ratchet baseline Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 5 ++++ pyproject.toml | 24 +++++++++++++++ tools/mutation_run.sh | 68 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 mutation/baseline.txt create mode 100755 tools/mutation_run.sh diff --git a/mutation/baseline.txt b/mutation/baseline.txt new file mode 100644 index 0000000..9ae0bc1 --- /dev/null +++ b/mutation/baseline.txt @@ -0,0 +1,5 @@ +# Mutation ratchet: . +# Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-09. Only ever lower them. +# Modules not yet baselined (policies, safety/*, recovery, manifest_validator) need a first run. +openblade/domain/scalar_coordinate.py 4 +openblade/jobs/scheduler.py 87 diff --git a/pyproject.toml b/pyproject.toml index 7f39af7..db940b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,3 +139,27 @@ markers = [ "rebuild: catalog rebuild tests (nightly rebuild lane)", "mutation: tests used as the kill-set for the mutation lane", ] + +[tool.mutmut] +# mutmut 3.x (verified against installed 3.8.0 configuration.py). Scope: the +# safety-critical modules; tools/mutation_run.sh selects a subset per run. +source_paths = ["openblade"] +only_mutate = [ + "openblade/domain/policies.py", + "openblade/safety/*", + "openblade/jobs/scheduler.py", + "openblade/jobs/recovery.py", + "openblade/domain/scalar_coordinate.py", + "openblade/nas/manifest_validator.py", +] +pytest_add_cli_args = ["-p", "no:cacheprovider", "-q"] +# tests/safety/test_assistant_read_only.py is excluded: it does not exercise the +# mutated modules and stalled >60s locally (2026-10-09), which blocks mutmut stats. +pytest_add_cli_args_test_selection = [ + "tests/safety/test_import_guard.py", + "tests/safety/test_safety_regression.py", + "tests/unit/test_scheduler.py", + "tests/unit/test_drive_leases.py", + "tests/unit/test_recovery.py", + "tests/unit/test_scalar_coordinate.py", +] diff --git a/tools/mutation_run.sh b/tools/mutation_run.sh new file mode 100755 index 0000000..4ab727b --- /dev/null +++ b/tools/mutation_run.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Mutation lane: run mutmut (config: [tool.mutmut] in pyproject.toml) over the +# safety-critical modules and ratchet surviving mutants against +# mutation/baseline.txt. More survivors than the baseline for any module => exit 1. +# +# Env: +# MUTATION_MODULES space-separated module paths to mutate +# (default: every module listed in mutation/baseline.txt) +# MUTATION_TIMEOUT wall-clock budget in seconds (default 3600); exceeding it exits 2 +# MUTATION_MAX_CHILDREN mutmut worker count (default: mutmut's own, = cpu count) +# PYTHON interpreter with mutmut installed (default: python) +set -euo pipefail + +cd "$(dirname "$0")/.." +PY="${PYTHON:-python}" +BASELINE="mutation/baseline.txt" +TIMEOUT="${MUTATION_TIMEOUT:-3600}" + +if [[ -n "${MUTATION_MODULES:-}" ]]; then + read -r -a modules <<<"$MUTATION_MODULES" +else + mapfile -t modules < <(grep -vE '^\s*(#|$)' "$BASELINE" | awk '{print $1}') +fi + +# mutmut names mutants ".__mutmut_"; select by module glob. +globs=() +for mod in "${modules[@]}"; do + [[ -f "$mod" ]] || { echo "mutation: no such module: $mod" >&2; exit 2; } + dotted="${mod%.py}" + globs+=("${dotted//\//.}.*") +done + +# Fresh run every time: a stale mutants/ cache would report old results. +rm -rf mutants + +run_args=(run) +[[ -n "${MUTATION_MAX_CHILDREN:-}" ]] && run_args+=(--max-children "$MUTATION_MAX_CHILDREN") +set +e +timeout "$TIMEOUT" "$PY" -m mutmut "${run_args[@]}" "${globs[@]}" +rc=$? +set -e +if [[ $rc -eq 124 ]]; then + echo "mutation: time budget of ${TIMEOUT}s exceeded - result is NOT a pass" >&2 + exit 2 +elif [[ $rc -ne 0 ]]; then + echo "mutation: mutmut run failed (exit $rc)" >&2 + exit "$rc" +fi + +results="$("$PY" -m mutmut results)" +fail=0 +for mod in "${modules[@]}"; do + dotted="${mod%.py}" + prefix="${dotted//\//.}." + survived=$(grep -F " ${prefix}" <<<"$results" | grep -cE ': (survived|no tests)$' || true) + allowed=$(awk -v m="$mod" '$1 == m {print $2}' "$BASELINE") + if [[ -z "$allowed" ]]; then + echo "mutation: $mod survived=$survived but has no baseline entry - add one to $BASELINE" >&2 + fail=1 + elif ((survived > allowed)); then + echo "mutation: $mod survived=$survived > baseline=$allowed (new survivors: kill them or justify)" >&2 + fail=1 + else + echo "mutation: $mod survived=$survived <= baseline=$allowed" + ((survived < allowed)) && echo "mutation: ratchet: lower $mod baseline to $survived" + fi +done +exit "$fail" From 823d8e15effcde4b023ac15eeccc389b53bf0ce6 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:40:58 +0000 Subject: [PATCH 097/205] ci: add nightly slow/stress/fuzz/rebuild/mutation lanes Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/nightly.yml | 111 ++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..7ab23d6 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,111 @@ +# Nightly lanes that are too slow or too noisy for the PR path. Not a required +# PR check. Every lane is blocking within this workflow (no continue-on-error): +# a red nightly is a real regression to triage the next morning. +name: nightly + +on: + schedule: + - cron: "41 3 * * *" # odd minute: top-of-hour schedules queue behind everyone + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: false + +jobs: + slow: + # Tests excluded from full-suite by `not slow`. + name: nightly-slow + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Slow-marked tests + run: pytest -m "slow and not real_hardware and not real_i3" -q --tb=short + + stress: + # Fault-injection suite plus any stress-marked test elsewhere. + name: nightly-stress + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Fault-injection suite + run: pytest tests/fault -q --tb=short + - name: Stress-marked tests outside tests/fault + run: | + # Exit 5 = no tests collected: legitimate while nothing outside + # tests/fault carries the marker. Any other non-zero exit fails. + rc=0 + pytest tests --ignore=tests/fault -m "stress and not real_hardware and not real_i3" -q --tb=short || rc=$? + if [ "$rc" -ne 0 ] && [ "$rc" -ne 5 ]; then exit "$rc"; fi + + fuzz: + # Property-based (hypothesis) suite. + name: nightly-fuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Property tests + run: pytest tests/property -q --tb=short + + rebuild: + # Catalog rebuild from tape (incl. the worker). + name: nightly-rebuild + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Catalog rebuild tests + run: pytest tests/unit/test_catalog_rebuild.py tests/unit/test_catalog_rebuild_worker.py -q --tb=short + + mutation: + # mutmut over the safety-critical modules; fails on new survivors vs mutation/baseline.txt. + name: nightly-mutation + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: "pip" + - run: pip install -e '.[dev]' + - name: Install mutmut + run: pip install mutmut==3.8.0 + - name: Mutation run + run: bash tools/mutation_run.sh From 0744515c8734e196f235770a97b4bb683e0bb5b5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:42:50 +0000 Subject: [PATCH 098/205] ci: persist-credentials=false on every checkout; zizmor/actionlint/yamllint clean across all workflows Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/hardware-destructive.yml | 2 + .github/workflows/hardware-library-smoke.yml | 2 + .../workflows/hardware-nightly-readonly.yml | 2 + .github/workflows/mhvtl-weekly.yml | 2 + .github/workflows/nas-protocols.yml | 2 + .github/workflows/operability.yml | 6 +- .github/workflows/release.yml | 60 +++++++++++-------- 7 files changed, 48 insertions(+), 28 deletions(-) diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml index a1d2151..6a1902f 100644 --- a/.github/workflows/hardware-destructive.yml +++ b/.github/workflows/hardware-destructive.yml @@ -74,6 +74,8 @@ jobs: I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index d2ba97d..5301d21 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -61,6 +61,8 @@ jobs: I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml index 53bebfc..413020b 100644 --- a/.github/workflows/hardware-nightly-readonly.yml +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -60,6 +60,8 @@ jobs: I3_AML_PASSWORD: ${{ secrets.QUANTUM_AML_PASSWORD }} steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 92f62d7..5be2a08 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -99,6 +99,8 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/nas-protocols.yml b/.github/workflows/nas-protocols.yml index bd9d23e..246b80d 100644 --- a/.github/workflows/nas-protocols.yml +++ b/.github/workflows/nas-protocols.yml @@ -17,6 +17,8 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" diff --git a/.github/workflows/operability.yml b/.github/workflows/operability.yml index 7eda9e4..f883731 100644 --- a/.github/workflows/operability.yml +++ b/.github/workflows/operability.yml @@ -27,8 +27,10 @@ jobs: name: operability-gate runs-on: ubuntu-latest steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e4bdcf..981470d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,8 +40,10 @@ jobs: digest: ${{ steps.push.outputs.digest }} version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Derive version from the tag @@ -59,19 +61,19 @@ jobs: fi - name: Build wheel run: python -m pip install build && python -m build --wheel --outdir dist/ - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel path: dist/*.whl - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build and push controller image (once) id: push - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . file: Dockerfile @@ -88,11 +90,13 @@ jobs: env: IMAGE: ghcr.io/amantux/openblade steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: wheel path: dist/ @@ -106,26 +110,26 @@ jobs: run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip -f cyclonedx-json -o wheel-sbom.cdx.json || true - name: pip-audit (fails on any known vulnerability) run: pip-audit -r resolved-requirements.txt --no-deps --disable-pip --strict - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Image SBOM (SPDX) - uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: image: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} format: spdx-json output-file: image-sbom.spdx.json upload-artifact: false - name: Trivy image scan by digest (CRITICAL/HIGH fail the release) - uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ${{ env.IMAGE }}@${{ needs.build.outputs.digest }} severity: CRITICAL,HIGH exit-code: "1" trivyignores: .trivyignore - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: sbom path: | @@ -144,8 +148,8 @@ jobs: env: IMAGE: ghcr.io/amantux/openblade steps: - - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -153,7 +157,7 @@ jobs: - name: cosign sign by digest run: cosign sign --yes "${IMAGE}@${{ needs.build.outputs.digest }}" - name: SLSA build provenance - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-name: ${{ env.IMAGE }} subject-digest: ${{ needs.build.outputs.digest }} @@ -168,8 +172,8 @@ jobs: env: IMAGE: ghcr.io/amantux/openblade steps: - - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -187,11 +191,11 @@ jobs: permissions: contents: write # create the GitHub Release steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: assets/ merge-multiple: true - - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 + - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} @@ -212,17 +216,19 @@ jobs: OPENBLADE_ENV: staging I3_TIMING_PROFILE: instant steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: wheel path: dist/ - name: Install the released wheel (+ test extras) run: pip install "$(ls dist/*.whl)[dev]" - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -262,8 +268,10 @@ jobs: OPENBLADE_IMAGE: ghcr.io/amantux/openblade@${{ needs.build.outputs.digest }} OPENBLADE_ENV: production steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: cosign verify (expected release identity) run: >- cosign verify From 5fb1d29e3e8eaf938d03b1c462000c9f05501848 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:51:58 +0000 Subject: [PATCH 099/205] =?UTF-8?q?fix(verify):=20skip=20STAGING/VERIFYING?= =?UTF-8?q?=20instances=20=E2=80=94=20recovery=20owns=20them,=20not=20the?= =?UTF-8?q?=20verify=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/verify.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/openblade/jobs/verify.py b/openblade/jobs/verify.py index 84488dc..ec5ee9c 100644 --- a/openblade/jobs/verify.py +++ b/openblade/jobs/verify.py @@ -8,10 +8,14 @@ from openblade.catalog.models import FileRecord from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend -from openblade.domain.models import MountMode +from openblade.domain.models import FileInstanceState, MountMode from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType +_UNCOMMITTED_STATES = frozenset( + {FileInstanceState.STAGING.value, FileInstanceState.VERIFYING.value} +) + def sha256sum(path: Path) -> str: digest = hashlib.sha256() @@ -63,6 +67,10 @@ def run_verify_job( handle = ltfs.mount(barcode, MountMode.READ_ONLY) try: for instance in catalog.list_instances_for_barcode(barcode): + if instance.state in _UNCOMMITTED_STATES: + # A staged shard from an interrupted archive is not a + # durability claim yet; recovery owns it, not verify. + continue record = catalog.session.get(FileRecord, instance.file_record_id) if record is None: continue From c25258e9e7f002d2fe0485d9a67b96a2331349e0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:56:02 +0000 Subject: [PATCH 100/205] fuse: hydrator worker threads use a private catalog Session JobRestoreEngine.restore_batch/resume run on Hydrator worker threads but shared the caller's SQLAlchemy Session (not thread-safe; made test_restart_rederives_ticket_from_job_state flaky). Each call now opens its own CatalogRepository (catalog_factory; default a Session on the same bind) and can build its RestoreService over it (restore_service_factory). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 51 ++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index b4f862d..5a270b1 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -7,12 +7,15 @@ import os import threading import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import Any, Protocol from sqlalchemy.exc import InvalidRequestError +from sqlalchemy.orm import Session from openblade.domain.errors import OpenBladeError from openblade.fuse.cache import CacheChecksumError, HydrationCache @@ -115,6 +118,8 @@ def __init__( *, poll_interval_s: float = 0.5, resume_timeout_s: float | None = None, + catalog_factory: Callable[[], Any] | None = None, + restore_service_factory: Callable[[Any], Any] | None = None, ) -> None: self.poll_interval_s = poll_interval_s self.resume_timeout_s = ( @@ -123,17 +128,51 @@ def __init__( self.restore_service = restore_service self.catalog = catalog self.staging_root = staging_root + self.catalog_factory = catalog_factory + # Builds a RestoreService over the worker's own catalog so the restore + # job's writes never go through the caller's (other-thread) Session. + self.restore_service_factory = restore_service_factory + + @contextmanager + def _worker_catalog(self) -> Iterator[Any]: + """A catalog over a Session private to the calling (worker) thread. + + ``restore_batch``/``resume`` run on Hydrator worker threads; SQLAlchemy + Sessions are not thread-safe, so the caller's session is never used here. + """ + if self.catalog_factory is not None: + repo = self.catalog_factory() + else: + from openblade.catalog.repository import CatalogRepository # local: avoid import cycle + + bind = self.catalog.session.get_bind() + repo = CatalogRepository(Session(bind=bind, expire_on_commit=False)) + try: + yield repo + finally: + repo.session.close() def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + with self._worker_catalog() as catalog: + return self._restore_batch(catalog, tape_key, catalog_paths) + + def _restore_batch( + self, catalog: Any, tape_key: str, catalog_paths: list[str] + ) -> dict[str, Path]: del tape_key + service = ( + self.restore_service + if self.restore_service_factory is None + else self.restore_service_factory(catalog) + ) staged: dict[str, Path] = {} for path in catalog_paths: - record = self.catalog.get_file_record(path) + record = catalog.get_file_record(path) if record is None: raise HydrationFailedError(f"{path} is not in the catalog") dest = self.staging_root / record.id dest.parent.mkdir(parents=True, exist_ok=True) - job = self.restore_service.enqueue(path, dest) + job = service.enqueue(path, dest) if job.state != "completed": raise HydrationFailedError(f"restore job {job.id} for {path} ended {job.state}") staged[path] = dest @@ -148,14 +187,18 @@ def find_active_job(self, catalog_path: str) -> str | None: return None def resume(self, job_id: str) -> Path: + with self._worker_catalog() as catalog: + return self._resume(catalog, job_id) + + def _resume(self, catalog: Any, job_id: str) -> Path: deadline = time.monotonic() + self.resume_timeout_s while True: - job = self.catalog.get_job(job_id) + job = catalog.get_job(job_id) if job is not None: # The job is advanced by another session; re-read the row instead # of trusting this session's identity-map copy. try: - self.catalog.session.refresh(job) + catalog.session.refresh(job) except InvalidRequestError: job = None if job is None: From 94bee9b0e18c50f068b67c21b32a6a3d72044793 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:56:02 +0000 Subject: [PATCH 101/205] fuse: `fuse mount --hydrate` builds the batched Hydrator data plane The CLI passes data_plane=Hydrator(JobRestoreEngine(...)) to mount_catalog instead of the per-read restore closure. Timeout and batch window come from new settings fields fuse_hydrate_timeout_s / fuse_batch_window_ms (same env vars and defaults as before). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/cli/fuse_and_health.py | 53 ++++++++++----- openblade/config.py | 8 +++ tests/unit/test_fuse_cli_wiring.py | 103 +++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_fuse_cli_wiring.py diff --git a/openblade/cli/fuse_and_health.py b/openblade/cli/fuse_and_health.py index 463d05f..2457f8f 100644 --- a/openblade/cli/fuse_and_health.py +++ b/openblade/cli/fuse_and_health.py @@ -7,22 +7,26 @@ """ import logging -from collections.abc import Callable from pathlib import Path +from typing import Any import typer from rich.console import Console from rich.table import Table +from sqlalchemy.orm import Session +from openblade.catalog.repository import CatalogRepository from openblade.config import load_config from openblade.domain.errors import RealHardwareDisabledError from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import Hydrator, JobRestoreEngine from openblade.fuse.mount import FuseUnavailableError, mount_catalog from openblade.hardware.discovery import discover_library from openblade.hardware.runner import SafeRunner from openblade.hardware.safety import require_real_hardware from openblade.hardware.sg import sg_inq from openblade.hardware.tapealert import TapeAlertSeverity, read_tape_alerts +from openblade.jobs.restore import RestoreService console = Console(highlight=False) @@ -36,6 +40,37 @@ } +def build_data_plane(context: Any, filesystem: CatalogFilesystem) -> Hydrator: + """Production FUSE data plane: batched restores into the verified cache. + + Worker threads build their own catalog + RestoreService over a private + Session (``JobRestoreEngine``); the CLI's session stays on this thread. + """ + config = context.config + bind = context.catalog.session.get_bind() + + def _catalog() -> CatalogRepository: + return CatalogRepository(Session(bind=bind, expire_on_commit=False)) + + def _service(catalog: CatalogRepository) -> RestoreService: + return RestoreService(context.library, context.ltfs, catalog, context.queue) + + engine = JobRestoreEngine( + None, + context.catalog, + Path(config.restore_dir) / "fuse-hydrate", + resume_timeout_s=config.fuse_hydrate_timeout_s, + catalog_factory=_catalog, + restore_service_factory=_service, + ) + return Hydrator( + context.catalog, + filesystem.cache, + engine, + batch_window_s=config.fuse_batch_window_ms / 1000.0, + ) + + @fuse_app.command("mount") def fuse_mount( mountpoint: str = typer.Argument(..., help="Existing empty directory to mount on"), @@ -67,19 +102,7 @@ def fuse_mount( context = _get_context() filesystem = CatalogFilesystem(context.catalog, cache_dir=context.config.cache_dir) - def _hydrate_through_restore(catalog_path: str) -> bytes: - """Restore through the existing restore service, then cache the bytes.""" - record = context.catalog.get_file_record(catalog_path) - if record is None: - raise FileNotFoundError(catalog_path) - staging = Path(context.config.restore_dir) / "fuse-hydrate" / record.id - staging.parent.mkdir(parents=True, exist_ok=True) - context.restore_service.enqueue(catalog_path, staging) - data = staging.read_bytes() - filesystem.cache.store(record.checksum_sha256, data) - return data - - hydrator: Callable[[str], bytes] | None = _hydrate_through_restore if hydrate else None + data_plane = build_data_plane(context, filesystem) if hydrate else None console.print( f"[green]Mounting[/green] catalog at {mountpoint} " @@ -89,7 +112,7 @@ def _hydrate_through_restore(catalog_path: str) -> bytes: mount_catalog( filesystem, mountpoint, - hydrator=hydrator, + data_plane=data_plane, allow_other=allow_other, ) except (FuseUnavailableError, NotADirectoryError) as exc: diff --git a/openblade/config.py b/openblade/config.py index 67f0a39..27286ae 100644 --- a/openblade/config.py +++ b/openblade/config.py @@ -39,6 +39,8 @@ class OpenBladeConfig: staging_dir: str = str(_DEFAULT_HOME / "staging") restore_dir: str = str(_DEFAULT_HOME / "restore") fuse_mount_point: str = str(_DEFAULT_HOME / "mount") + fuse_hydrate_timeout_s: float = 300.0 + fuse_batch_window_ms: int = 250 ltfs_mount_root: str = str(_DEFAULT_HOME / "ltfs") job_timeout_seconds: int = 3600 changer_timeout_seconds: int = 60 @@ -184,6 +186,12 @@ def load_config() -> OpenBladeConfig: staging_dir=os.environ.get("OPENBLADE_STAGING_DIR", str(_DEFAULT_HOME / "staging")), restore_dir=os.environ.get("OPENBLADE_RESTORE_DIR", str(_DEFAULT_HOME / "restore")), ltfs_mount_root=os.environ.get("OPENBLADE_LTFS_MOUNT_ROOT", str(_DEFAULT_HOME / "ltfs")), + fuse_hydrate_timeout_s=float( + os.environ.get("OPENBLADE_FUSE_HYDRATE_TIMEOUT", "").strip() or 300.0 + ), + fuse_batch_window_ms=int( + os.environ.get("OPENBLADE_FUSE_BATCH_WINDOW_MS", "").strip() or 250 + ), hardware_dry_run=os.environ.get("OPENBLADE_HARDWARE_DRY_RUN", "false").lower() == "true", changer_device=os.environ.get("OPENBLADE_CHANGER_DEVICE") or None, drive_devices=drive_devices, diff --git a/tests/unit/test_fuse_cli_wiring.py b/tests/unit/test_fuse_cli_wiring.py new file mode 100644 index 0000000..8457b87 --- /dev/null +++ b/tests/unit/test_fuse_cli_wiring.py @@ -0,0 +1,103 @@ +"""`openblade fuse mount --hydrate` builds the production data plane.""" + +from __future__ import annotations + +import threading +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from typer.testing import CliRunner + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.cli import fuse_and_health +from openblade.config import load_config +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import Hydrator + + +def _context(tmp_path: Path, **config_overrides: Any) -> SimpleNamespace: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + config = replace( + load_config(), + cache_dir=str(tmp_path / "cache"), + restore_dir=str(tmp_path / "restore"), + **config_overrides, + ) + return SimpleNamespace( + config=config, + catalog=CatalogRepository(get_session()), + library=object(), + ltfs=object(), + queue=object(), + ) + + +def test_build_data_plane_reads_timeouts_from_settings(tmp_path: Path) -> None: + ctx = _context(tmp_path, fuse_hydrate_timeout_s=7.5, fuse_batch_window_ms=40) + + plane = fuse_and_health.build_data_plane( + ctx, CatalogFilesystem(ctx.catalog, ctx.config.cache_dir) + ) + + assert isinstance(plane, Hydrator) + assert plane.batch_window_s == pytest.approx(0.04) + assert plane.engine.resume_timeout_s == 7.5 # type: ignore[attr-defined] + + +def test_worker_restore_never_uses_the_callers_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + group = ctx.catalog.create_volume_group("g") + ctx.catalog.create_file_record("/a.bin", 3, "0" * 64, group.id) + plane = fuse_and_health.build_data_plane( + ctx, CatalogFilesystem(ctx.catalog, ctx.config.cache_dir) + ) + seen: list[Any] = [] + + class StubService: + def __init__(self, library: Any, ltfs: Any, catalog: Any, queue: Any) -> None: + seen.append(catalog) + + def enqueue(self, path: str, dest: Path) -> Any: + return SimpleNamespace(id="j1", state="completed") + + monkeypatch.setattr(fuse_and_health, "RestoreService", StubService) + worker = threading.Thread(target=plane.engine.restore_batch, args=("T1", ["/a.bin"])) + worker.start() + worker.join(5) + + assert len(seen) == 1 + assert seen[0].session is not ctx.catalog.session + + +def test_mount_with_hydrate_passes_data_plane( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + calls: dict[str, Any] = {} + monkeypatch.setattr("openblade.cli.main._get_context", lambda: ctx) + monkeypatch.setattr(fuse_and_health, "mount_catalog", lambda fs, mp, **kw: calls.update(kw)) + + result = CliRunner().invoke(fuse_and_health.fuse_app, [str(tmp_path), "--hydrate"]) + + assert result.exit_code == 0, result.output + assert isinstance(calls["data_plane"], Hydrator) + + +def test_mount_without_hydrate_has_no_data_plane( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ctx = _context(tmp_path) + calls: dict[str, Any] = {} + monkeypatch.setattr("openblade.cli.main._get_context", lambda: ctx) + monkeypatch.setattr(fuse_and_health, "mount_catalog", lambda fs, mp, **kw: calls.update(kw)) + + result = CliRunner().invoke(fuse_and_health.fuse_app, [str(tmp_path)]) + + assert result.exit_code == 0, result.output + assert calls["data_plane"] is None From aea84212e7656eff41e1a81e10c05a8d75b648e8 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 15:56:20 +0000 Subject: [PATCH 102/205] docs(fuse): CLI data plane and per-thread sessions are now real Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/fuse.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fuse.md b/docs/fuse.md index f7f68d8..bfb8250 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -7,8 +7,8 @@ Five layers, top to bottom, and what is real today: | SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`). | | Namespace/metadata | **Real.** `CatalogFuseOperations` (`openblade/fuse/mount.py`): offline files report their real size; xattr `user.openblade.state` = `offline`/`hydrating`/`online`. | | Hydration/cache | **Real.** `Hydrator` (`openblade/fuse/hydration.py`): one shared ticket per path, `OPENBLADE_FUSE_HYDRATE_TIMEOUT` (default 300 s) blocking wait, or `EAGAIN` with `blocking=False`; failure = `EIO`, file stays `offline`, reason in `last_error()`. `HydrationCache` (`cache.py`): optional byte budget with LRU eviction that never evicts a file with open handles; restores land under a temp name and are renamed only after the sha256 matches. | -| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue` (one restore job per file); never a tape backend directly. A restarted hydrator re-attaches to a pending/running restore job for the path. | +| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue` (one restore job per file); never a tape backend directly. A restarted hydrator re-attaches to a pending/running restore job for the path. Every `restore_batch`/`resume` call runs on a Hydrator worker thread over its **own** catalog Session (`catalog_factory`), never the caller's. Files of one tape batch are still restored one job per file (not yet one load per tape). | | Placement/robotics | **Partial.** Requests are batched per tape barcode for `OPENBLADE_FUSE_BATCH_WINDOW_MS` (default 250 ms) and restored back to back; whether that is one physical load is the jobs scheduler's decision. | Unmount (`destroy`) cancels batches not yet started and waits for in-flight restores. -The `openblade fuse` CLI still uses the older synchronous `--hydrate` callable; the Hydrator is not wired into `create_context` yet. +`openblade fuse mount --hydrate` builds this data plane (`build_data_plane` in `openblade/cli/fuse_and_health.py`) and passes it as `data_plane=`. Timeout and batch window come from the settings fields `fuse_hydrate_timeout_s` / `fuse_batch_window_ms` (env `OPENBLADE_FUSE_HYDRATE_TIMEOUT` / `OPENBLADE_FUSE_BATCH_WINDOW_MS`). The cache byte budget (`max_bytes`) is not yet configurable from the CLI, so the mount's cache is unbounded. From 6a57f3a303fc7d836da8f51527fe83d1c6ff5d31 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:01:50 +0000 Subject: [PATCH 103/205] test: own the contract suite and protocol render test; bridge test expects curated job errors Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_archive_aml_bridge.py | 8 ++++---- tools/ci_ownership.toml | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_archive_aml_bridge.py b/tests/integration/test_archive_aml_bridge.py index 73ed689..8b57773 100644 --- a/tests/integration/test_archive_aml_bridge.py +++ b/tests/integration/test_archive_aml_bridge.py @@ -168,7 +168,7 @@ def _boom(*args, **kwargs): monkeypatch.setattr(context.ltfs, "write_file", original_write_file) jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "archive bridge boom") + aml_job = next(job for job in jobs if job["result"] == "Job failed (RuntimeError); see server logs for detail") assert aml_job["type"] == "archive" assert aml_job["status"] == "failed" @@ -179,7 +179,7 @@ def _boom(*args, **kwargs): if event["component"] == "archive" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "archive bridge boom" + assert failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" drive = ( client.get("/aml/drive/DRV-001").json().get("drive") @@ -225,7 +225,7 @@ def _boom(*args, **kwargs): assert response.status_code == 500 jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "restore bridge boom") + aml_job = next(job for job in jobs if job["result"] == "Job failed (RuntimeError); see server logs for detail") assert aml_job["type"] == "restore" assert aml_job["status"] == "failed" @@ -236,7 +236,7 @@ def _boom(*args, **kwargs): if event["component"] == "restore" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "restore bridge boom" + assert failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" assert failure_event["details"]["catalogPath"] == "/photos/restore.txt" assert archived_file.read_text() == "restore aml bridge" diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml index 81b973a..99517bb 100644 --- a/tools/ci_ownership.toml +++ b/tools/ci_ownership.toml @@ -34,6 +34,14 @@ categories = ["integration"] glob = "tests/hardware/*" categories = ["integration"] +[[rule]] +glob = "tests/contract/*" +categories = ["integration"] + +[[rule]] +glob = "tests/test_nas_protocol_render.py" +categories = ["unit"] + [[rule]] glob = "tests/safety/*" categories = ["safety"] From caa6ae6d6ea4a376ef040ba07ba2bc5b30499f9e Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:03:42 +0000 Subject: [PATCH 104/205] ci(emulator-gates): keep the contract-pinned validation command as an explicit step The cross-repo contract pins the command to emulator-change-gates.yml; the reusable-workflow refactor had moved it out of that file (caught by test_15). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/emulator-change-gates.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/emulator-change-gates.yml b/.github/workflows/emulator-change-gates.yml index 62c6007..f04c975 100644 --- a/.github/workflows/emulator-change-gates.yml +++ b/.github/workflows/emulator-change-gates.yml @@ -50,8 +50,24 @@ permissions: contents: read jobs: + # The cross-repo contract (openblade/emulator_contract/contract.json) pins this + # exact command to this workflow file, so it stays here as an explicit step in + # addition to the reusable workflow's own validation. + contract-metadata: + name: contract-metadata + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: python3 tools/emulator_spec/validate_cross_repo_contract.py + emulator-contract: name: emulator-contract + needs: contract-metadata # './' is GitHub's documented local reusable-workflow syntax and what actionlint validates. uses: ./.github/workflows/_emulator-boot-test.yml # zizmor: ignore[self-repository] with: From 7df5cb5cfab5caa662030075003a0be540758920 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:05:32 +0000 Subject: [PATCH 105/205] docs(roadmap): verified per-item status after the 2026-10-09 fleet integration Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 2 +- docs/production-readiness-roadmap.md | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 8077f5e..efccf89 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -6,7 +6,7 @@ Status: accepted 2026-10-09. Roadmap items 7 and 8 (Phase 2), first slice. | checkout | branch | relation to `origin/master` | |---|---|---| -| `/root/ob2-final` | `feat/persistent-drive-leases` (was `integrate/gap-closure`) | branched from master (`1863348`, PR #45) for this work | +| `/root/ob2-final` | `feat/persistent-drive-leases` | integration branch for this work and the 2026-10-09 fleet (per-item status table in `docs/production-readiness-roadmap.md`) | | `/root/ob-final` | `integrate/eight-items` | tree-identical to PR #44 squash, already on master | | `/root/ob-int2` | `integrate/campaign-wiki-assistant` | fully contained in `ob-final` | | `/root/OpenBlade` | `chore/py312-baseline-and-quantum-refs` | stale (37 behind); its 3 stray commits are already on master by content | diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index c259393..9562099 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -26,6 +26,38 @@ Verified against the repo before writing (not asserted from memory): - `sharded_archive.py:234,251` calls `mark_instance_archived` per-shard inside the write loop → review #6 confirmed real. ✅ - Review #5 (scheduler `drive_id` mutation) **not reproduced**: `jobs/scheduler.py` is a plain allocator with no `_load_barcode`/mutation — re-investigate before "fixing." ⚠️ +## Status as of 2026-10-09 (fleet integration on `feat/persistent-drive-leases`) + +Verified against the tree, not asserted. "Landed" = implemented and tested in-repo; +"appliance-gated" = authored and runnable, cannot be certified without the rig. + +| # | Item | Status | Where | +|---|---|---|---| +| 1 | Relabel simplified endpoint | landed | `routes_aml_move_medium.py` | +| 2 | ScalarCoordinate / MoveClass IntFlag | landed (bit values appliance-gated) | `domain/scalar_coordinate.py` | +| 3 | Compatibility corpus + differential harness | scaffold landed; 0 captured cases (appliance-gated) | `compatibility/`, `tests/compat/` | +| 4 | Atomic sharded commit | landed: STAGING→VERIFYING→one ARCHIVED commit, job journal, no suppressed cleanup errors; resume = report-only | `jobs/sharded_archive.py`, `catalog/repository.py`, `jobs/recovery.py` | +| 5 | Protocol-agnostic backends + contract suite | landed (sim, in-process emulator, real gated); scsi+fake pairing TODO | `tests/contract/` | +| 6 | Scheduler lease regression | landed | `tests/unit/test_drive_leases.py` | +| 7 | Persistent drive leases | landed | `catalog/models.py::DriveLeaseRecord`, `jobs/scheduler.py` | +| 8 | Persistent, resumable jobs | landed (recovery + report); resume-from-staged not implemented | `jobs/recovery.py`, `GET /jobs/recovery` | +| 9 | Reconstructible catalog | landed: manifest v2, commit marker last, generation pointer, sibling tapes, uncommitted-generation rebuild rule | `nas/ltfs_manifest.py`, `nas/archive_lifecycle.py`, `nas/catalog_rebuild.py` | +| 10 | Separate the axes | landed (models + manifest/catalog carry); erasure coding modelled, unsupported | `domain/protection.py` | +| 11 | Aggregation gate + ownership check | landed | `ci.yml::ci-gate`, `tools/ci_ownership.py` | +| 12 | Full suite on master + nightly lanes | landed | `ci.yml::full-suite`, `nightly.yml` | +| 13 | Consolidated emulator workflows | landed | `_emulator-boot-test.yml` | +| 14 | Lint/type ratchet | landed (both blocking, both at zero) | `ci.yml` | +| 15 | Workflow hardening | landed: SHA pins, per-job permissions, actionlint/zizmor/shellcheck/yamllint | `workflow-lint.yml` | +| 16 | Differential + mutation | mutation lane landed (mutmut, ratchet baseline for 2 modules); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | +| 17 | Timing profiles + virtual clock | landed (10 profiles; 4 simulated client-side) | `tests/i3/timing.py` | +| 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-required pending state not modelled | `routes_aml_auth.py`, `compatibility/` | +| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 31 scenarios pass locally; SMB byte-range lock + NFS ACL skip) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | +| 20 | Tape-native NAS models | landed as models + semantic/negative tests; catalog persistence TODO | `nas/state_machine.py`, `nas/media.py`, `nas/capacity.py` | +| 21 | FUSE data plane | landed: hydrator, cache, xattr state, gateway; see `docs/fuse.md` for what is wired | `fuse/` | +| 22 | Self-hosted i3/LTFS lanes | appliance-gated, authored: read-only nightly + destructive with environment approval, allowlist, snapshots | `hardware-*.yml`, `tools/hardware/` | +| 23 | Drive-identity bridge | landed; on-rig certification test authored | `hardware/correlation.py`, `tests/hardware/readonly/` | +| 24 | Release pipeline | landed (never exercised on a tag): build-once digest, SBOM, scans, cosign, staging→approval→production, rollback | `release.yml`, `docs/runbooks/release.md` | + --- ## Phase 0 — Stop the self-referential fidelity (the meta-fix) 🟢🟡 From e10741569134650b66a734db3c01cd253254be28 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:08:50 +0000 Subject: [PATCH 106/205] feat(fuse): OPENBLADE_FUSE_CACHE_MAX_BYTES bounds the hydration cache in the CLI data plane Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/cli/fuse_and_health.py | 2 ++ openblade/config.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/openblade/cli/fuse_and_health.py b/openblade/cli/fuse_and_health.py index 2457f8f..da23a01 100644 --- a/openblade/cli/fuse_and_health.py +++ b/openblade/cli/fuse_and_health.py @@ -48,6 +48,8 @@ def build_data_plane(context: Any, filesystem: CatalogFilesystem) -> Hydrator: """ config = context.config bind = context.catalog.session.get_bind() + if config.fuse_cache_max_bytes is not None: + filesystem.cache.max_bytes = config.fuse_cache_max_bytes def _catalog() -> CatalogRepository: return CatalogRepository(Session(bind=bind, expire_on_commit=False)) diff --git a/openblade/config.py b/openblade/config.py index 27286ae..bf135dd 100644 --- a/openblade/config.py +++ b/openblade/config.py @@ -41,6 +41,7 @@ class OpenBladeConfig: fuse_mount_point: str = str(_DEFAULT_HOME / "mount") fuse_hydrate_timeout_s: float = 300.0 fuse_batch_window_ms: int = 250 + fuse_cache_max_bytes: int | None = None # None = unbounded ltfs_mount_root: str = str(_DEFAULT_HOME / "ltfs") job_timeout_seconds: int = 3600 changer_timeout_seconds: int = 60 @@ -192,6 +193,11 @@ def load_config() -> OpenBladeConfig: fuse_batch_window_ms=int( os.environ.get("OPENBLADE_FUSE_BATCH_WINDOW_MS", "").strip() or 250 ), + fuse_cache_max_bytes=( + int(os.environ["OPENBLADE_FUSE_CACHE_MAX_BYTES"]) + if os.environ.get("OPENBLADE_FUSE_CACHE_MAX_BYTES", "").strip() + else None + ), hardware_dry_run=os.environ.get("OPENBLADE_HARDWARE_DRY_RUN", "false").lower() == "true", changer_device=os.environ.get("OPENBLADE_CHANGER_DEVICE") or None, drive_devices=drive_devices, From 3f8a75137f420a0303f5601796ccd8447b864dc4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:12:34 +0000 Subject: [PATCH 107/205] feat(restore): run_restore_batch -- one lease/load/mount/unload per source tape Groups requests by the latest archived instance's barcode; per-item typed failures for unknown/offline paths; temp-name + checksum-verified rename; an unmount failure journals physical_state_unknown and skips the unload. RestoreService.enqueue_batch submits one job for the whole batch. One-load test mutation-checked (per-file loop -> (2, 2) != (1, 1)). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/restore.py | 246 ++++++++++++++++++++++++++++++- tests/unit/test_restore_batch.py | 185 +++++++++++++++++++++++ 2 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_restore_batch.py diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index dee505f..61842a1 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -8,7 +8,7 @@ from pathlib import Path, PurePosixPath from openblade.api import aml_state -from openblade.catalog.models import Job +from openblade.catalog.models import FileInstance, FileRecord, Job from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.errors import ( @@ -17,7 +17,10 @@ StaleLeaseError, safe_job_error, ) -from openblade.domain.models import JobType, MountMode +from openblade.domain.errors import ( + FileNotFoundError as CatalogPathNotFoundError, +) +from openblade.domain.models import JobType, MountHandle, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue from openblade.jobs.scheduler import DriveHandle, DriveScheduler @@ -239,6 +242,229 @@ def run_restore_job( return RestoreResult(job_id=job_id, source_barcode=instance.barcode, checksum_verified=True) +@dataclass +class RestoreItemResult: + catalog_path: str + ok: bool + error: str | None = None + + +@dataclass +class RestoreBatchResult: + job_id: str + items: list[RestoreItemResult] + tape_errors: list[str] + + @property + def ok(self) -> bool: + return not self.tape_errors and all(item.ok for item in self.items) + + +def _batch_dest(request: RestoreRequest) -> Path: + if request.dest_path.exists() and request.dest_path.is_dir(): + return request.dest_path / PurePosixPath(request.catalog_path).name + return request.dest_path + + +def _restore_one_verified( + ltfs: LTFSBackend, + mount_handle: MountHandle, + request: RestoreRequest, + record: FileRecord, + instance: FileInstance, +) -> RestoreItemResult: + """Read into a temp name; only a verified checksum renames it into place.""" + final_dest = _batch_dest(request) + temp = final_dest.with_name(f".{final_dest.name}.restoring") + try: + ltfs.read_file(mount_handle, PurePosixPath(instance.tape_path), temp) + except StaleLeaseError: + raise + except Exception as exc: # per-item: one unreadable file must not abort the tape + logger.exception("batch restore read failed for %s", request.catalog_path) + temp.unlink(missing_ok=True) + return RestoreItemResult(request.catalog_path, ok=False, error=safe_job_error(exc)) + if sha256sum(temp) != record.checksum_sha256: + temp.rename(final_dest.with_name(f"{final_dest.name}.quarantine")) + return RestoreItemResult( + request.catalog_path, ok=False, error=f"Checksum mismatch for {request.catalog_path}" + ) + temp.replace(final_dest) + return RestoreItemResult(request.catalog_path, ok=True) + + +def _restore_tape_group( + barcode: str, + group: list[tuple[RestoreRequest, FileRecord, FileInstance]], + library: LibraryBackend, + ltfs: LTFSBackend, + catalog: CatalogRepository, + job_id: str, + scheduler: DriveScheduler, +) -> tuple[list[RestoreItemResult], str | None]: + """One lease, one load, one RO mount, one unmount, one unload for ``group``.""" + results: list[RestoreItemResult] = [] + tape_error: str | None = None + handles = scheduler.acquire_drives([barcode]) + handle = handles[0] + catalog.journal(job_id, "lease_acquired", {"drive_id": handle.drive_id, "barcode": barcode}) + catalog.journal(job_id, "batch_tape_started", {"barcode": barcode, "files": len(group)}) + drive_id: int | None = None + slot_id: int | None = None + fenced_out = False + unmount_failed = False + try: + try: + drive_id, slot_id = _load_if_needed(catalog, library, ltfs, handle, job_id) + scheduler.record_physical_drives(handles) + _mark_aml_drive_busy(barcode, drive_id) + scheduler.verify(handle) # fencing: never mount on a stale lease + mount_handle = ltfs.mount(barcode, MountMode.READ_ONLY) + try: + for request, record, instance in group: + results.append( + _restore_one_verified(ltfs, mount_handle, request, record, instance) + ) + finally: + scheduler.verify(handle) # fencing: before unmount + try: + ltfs.unmount(mount_handle) + except Exception: + # Still (maybe) mounted: unloading now could strand a dirty + # index. Leave the cartridge in the drive for reconcile. + unmount_failed = True + logger.exception("batch restore unmount failed for %s", barcode) + catalog.journal( + job_id, + "physical_state_unknown", + {"barcode": barcode, "drive_id": drive_id, "reason": "unmount_failed"}, + ) + tape_error = f"Unmount of {barcode} failed; left in drive for reconcile" + except StaleLeaseError: + fenced_out = True + raise + except Exception as exc: # tape-level: load/mount failed -> fail this tape's items + logger.exception("batch restore failed on tape %s", barcode) + message = safe_job_error(exc) + done = {item.catalog_path for item in results} + results.extend( + RestoreItemResult(req.catalog_path, ok=False, error=message) + for req, _, _ in group + if req.catalog_path not in done + ) + finally: + if not fenced_out and not unmount_failed and drive_id is not None: + try: + scheduler.verify(handle) # fencing: before unload + except StaleLeaseError: + fenced_out = True + raise + if slot_id is not None: + execute_tape_request( + catalog, + library, + ltfs, + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode=barcode, + drive_id=drive_id, + slot_id=slot_id, + requested_by="restore-job", + job_id=job_id, + ), + ) + _mark_aml_drive_idle(barcode, drive_id, slot_id) + finally: + scheduler.release_drives(handles) + catalog.journal(job_id, "lease_released", {"drive_id": handle.drive_id, "barcode": barcode}) + catalog.journal( + job_id, + "batch_tape_finished", + {"barcode": barcode, "ok": sum(1 for r in results if r.ok), "files": len(group)}, + ) + return results, tape_error + + +def run_restore_batch( + requests: list[RestoreRequest], + library: LibraryBackend, + ltfs: LTFSBackend, + catalog: CatalogRepository, + job_id: str, + *, + scheduler: DriveScheduler | None = None, +) -> RestoreBatchResult: + """Restore many cataloged files with ONE load/mount/unload per source tape. + + Requests are grouped by the barcode of each path's latest archived + instance. A path with no archived instance (or on an exported cartridge) + is a per-item failure, not a batch abort. A lost lease aborts the batch + and leaves the hardware alone, exactly as ``run_restore_job`` does. + """ + catalog.update_job_state(job_id, "running") + items: list[RestoreItemResult] = [] + tape_errors: list[str] = [] + groups: dict[str, list[tuple[RestoreRequest, FileRecord, FileInstance]]] = {} + for request in requests: + try: + record, instance = catalog.get_latest_instance_for_path(request.catalog_path) + except CatalogPathNotFoundError: + items.append( + RestoreItemResult( + request.catalog_path, + ok=False, + error=f"No archived instance for {request.catalog_path}", + ) + ) + continue + cartridge = catalog.get_cartridge(instance.barcode) + if cartridge is not None and cartridge.state == "exported": + items.append( + RestoreItemResult( + request.catalog_path, + ok=False, + error=f"Cartridge {instance.barcode} is offline", + ) + ) + continue + if request.dry_run: + items.append(RestoreItemResult(request.catalog_path, ok=True)) + continue + groups.setdefault(instance.barcode, []).append((request, record, instance)) + if groups and scheduler is None: + scheduler = DriveScheduler( + num_drives=len(InventoryService(library).snapshot().drives), job_id=job_id + ) + for barcode, group in groups.items(): + assert scheduler is not None + try: + results, tape_error = _restore_tape_group( + barcode, group, library, ltfs, catalog, job_id, scheduler + ) + except StaleLeaseError as exc: + logger.error( + "batch job %s lost its lease on %s; leaving the drive untouched -- " + "physical state unknown, reconcile before reuse", + job_id, + barcode, + ) + catalog.update_job_state(job_id, "failed", safe_job_error(exc)) + raise + items.extend(results) + if tape_error is not None: + tape_errors.append(tape_error) + result = RestoreBatchResult(job_id=job_id, items=items, tape_errors=tape_errors) + if result.ok: + catalog.update_job_state(job_id, "completed") + else: + failed = sum(1 for item in items if not item.ok) + detail = f"{failed} of {len(items)} files failed to restore" + if tape_errors: + detail = f"{detail}; {'; '.join(tape_errors)}" + catalog.update_job_state(job_id, "failed", detail) + return result + + class RestoreService: def __init__( self, @@ -267,3 +493,19 @@ def enqueue(self, catalog_path: str, destination: Path) -> Job: refreshed = self.catalog.get_job(job.id) assert refreshed is not None return refreshed + + def enqueue_batch(self, requests: list[RestoreRequest]) -> tuple[Job, RestoreBatchResult]: + """One restore job covering every request; one load per source tape.""" + job = self.catalog.create_job( + JobType.RESTORE.value, + { + "batch": [ + {"catalog_path": r.catalog_path, "dest_path": str(r.dest_path)} + for r in requests + ] + }, + ) + result = run_restore_batch(requests, self.library, self.ltfs, self.catalog, job.id) + refreshed = self.catalog.get_job(job.id) + assert refreshed is not None + return refreshed, result diff --git a/tests/unit/test_restore_batch.py b/tests/unit/test_restore_batch.py new file mode 100644 index 0000000..c963b59 --- /dev/null +++ b/tests/unit/test_restore_batch.py @@ -0,0 +1,185 @@ +"""run_restore_batch: one lease/load/mount/unmount/unload per source tape.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.archive import ArchiveRequest, run_archive_job +from openblade.jobs.restore import RestoreRequest, run_restore_batch +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend +from openblade.simulator.scenarios import one_drive_twenty_slots_five_cartridges + + +class _Recorder: + """Counts library load/unload calls without changing their behaviour.""" + + def __init__(self, library: MockLibraryBackend) -> None: + self.loads = 0 + self.unloads = 0 + real_load, real_unload = library.load, library.unload + + def load(*args: Any, **kwargs: Any) -> Any: + self.loads += 1 + return real_load(*args, **kwargs) + + def unload(*args: Any, **kwargs: Any) -> Any: + self.unloads += 1 + return real_unload(*args, **kwargs) + + library.load = load # type: ignore[method-assign] + library.unload = unload # type: ignore[method-assign] + + +def _stack( + tmp_path: Path, groups: list[str] +) -> tuple[CatalogRepository, MockLibraryBackend, MockLTFSBackend]: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo = CatalogRepository(get_session()) + library, ltfs = one_drive_twenty_slots_five_cartridges() + slots = library.inventory().slots + for index, name in enumerate(groups): + slot = slots[index] + barcode = str(slot.barcode) + library.load(slot.slot_id, 0) + ltfs.format(barcode, FormatConfirmation(barcode, SafetyToken.generate("format", barcode))) + library.unload(0, slot.slot_id) + group = repo.create_volume_group(name) + repo.add_barcode_to_volume_group(group.id, barcode) + return repo, library, ltfs + + +def _archive( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + source: Path, + group: str, +) -> None: + job = repo.create_job("archive", {"source_path": str(source), "volume_group": group}) + run_archive_job( + ArchiveRequest(source_path=source, volume_group_name=group), + library, + ltfs, + repo, + job.id, + scheduler=DriveScheduler(1, store=CatalogLeaseStore(repo), job_id=job.id), + ) + + +def _write(tmp_path: Path, name: str) -> Path: + path = tmp_path / "src" / name + path.parent.mkdir(exist_ok=True) + path.write_text(f"payload of {name}") + return path + + +def _run( + repo: CatalogRepository, + library: MockLibraryBackend, + ltfs: MockLTFSBackend, + requests: list[RestoreRequest], +) -> tuple[str, Any]: + job = repo.create_job("restore", {}) + scheduler = DriveScheduler(1, store=CatalogLeaseStore(repo), job_id=job.id) + result = run_restore_batch(requests, library, ltfs, repo, job.id, scheduler=scheduler) + return job.id, result + + +def test_restore_batch_two_files_one_tape_loads_and_unloads_once(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + for name in ("a.txt", "b.txt"): + _archive(repo, library, ltfs, _write(tmp_path, name), "photos") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + job_id, result = _run( + repo, + library, + ltfs, + [RestoreRequest(f"/photos/{n}", out / n) for n in ("a.txt", "b.txt")], + ) + + assert (recorder.loads, recorder.unloads) == (1, 1) + assert result.ok and [i.ok for i in result.items] == [True, True] + assert (out / "a.txt").read_text() == "payload of a.txt" + assert (out / "b.txt").read_text() == "payload of b.txt" + assert repo.get_job(job_id).state == "completed" # type: ignore[union-attr] + + +def test_restore_batch_two_tapes_loads_and_unloads_each_once(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos", "video"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + _archive(repo, library, ltfs, _write(tmp_path, "c.txt"), "video") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + job_id, result = _run( + repo, + library, + ltfs, + [ + RestoreRequest("/photos/a.txt", out / "a.txt"), + RestoreRequest("/video/c.txt", out / "c.txt"), + ], + ) + + assert (recorder.loads, recorder.unloads) == (2, 2) + assert result.ok + events = [e.event for e in repo.job_journal(job_id)] + assert events.count("batch_tape_started") == 2 + assert events.count("batch_tape_finished") == 2 + + +def test_restore_batch_unknown_path_is_item_failure_not_abort(tmp_path: Path) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + out = tmp_path / "out" + out.mkdir() + + job_id, result = _run( + repo, + library, + ltfs, + [ + RestoreRequest("/photos/missing.txt", out / "missing.txt"), + RestoreRequest("/photos/a.txt", out / "a.txt"), + ], + ) + + by_path = {i.catalog_path: i for i in result.items} + assert not by_path["/photos/missing.txt"].ok + assert by_path["/photos/a.txt"].ok and (out / "a.txt").exists() + assert repo.get_job(job_id).state == "failed" # type: ignore[union-attr] + + +def test_restore_batch_unmount_failure_skips_unload_and_journals( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, library, ltfs = _stack(tmp_path, ["photos"]) + _archive(repo, library, ltfs, _write(tmp_path, "a.txt"), "photos") + out = tmp_path / "out" + out.mkdir() + recorder = _Recorder(library) + + def broken_unmount(handle: Any) -> Any: + raise OSError("simulated unmount failure") + + monkeypatch.setattr(ltfs, "unmount", broken_unmount) + + job_id, result = _run(repo, library, ltfs, [RestoreRequest("/photos/a.txt", out / "a.txt")]) + + assert recorder.loads == 1 + assert recorder.unloads == 0 # never unload a (maybe) still-mounted tape + assert not result.ok + assert "physical_state_unknown" in [e.event for e in repo.job_journal(job_id)] + assert repo.get_job(job_id).state == "failed" # type: ignore[union-attr] From f089e77b6866885e147eb7b505814e5c94882d52 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:14:07 +0000 Subject: [PATCH 108/205] feat(fuse): JobRestoreEngine submits one restore job per tape batch restore_batch now calls RestoreService.enqueue_batch once for the whole tape group instead of one job (and one load) per file. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 5a270b1..01a9b04 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -19,6 +19,7 @@ from openblade.domain.errors import OpenBladeError from openblade.fuse.cache import CacheChecksumError, HydrationCache +from openblade.jobs.restore import RestoreRequest logger = logging.getLogger(__name__) @@ -103,11 +104,11 @@ def resume(self, job_id: str) -> Path: class JobRestoreEngine: - """Production engine: one restore job per file via ``RestoreService``. + """Production engine: one restore job per tape batch via ``RestoreService``. - Files in one batch share a tape and are restored back to back, so the - library loads that cartridge once (scheduler/lease policy is the jobs - layer's concern, not ours). + Files in one batch share a tape; ``RestoreService.enqueue_batch`` takes one + lease and loads, mounts, unmounts and unloads that cartridge once + (scheduler/lease policy is the jobs layer's concern, not ours). """ def __init__( @@ -172,10 +173,16 @@ def _restore_batch( raise HydrationFailedError(f"{path} is not in the catalog") dest = self.staging_root / record.id dest.parent.mkdir(parents=True, exist_ok=True) - job = service.enqueue(path, dest) - if job.state != "completed": - raise HydrationFailedError(f"restore job {job.id} for {path} ended {job.state}") staged[path] = dest + # ONE job for the whole tape group: the jobs layer loads/mounts once. + job, result = service.enqueue_batch( + [RestoreRequest(catalog_path=path, dest_path=dest) for path, dest in staged.items()] + ) + if job.state != "completed": + failed = [item.catalog_path for item in result.items if not item.ok] + raise HydrationFailedError( + f"restore job {job.id} ended {job.state}; failed: {', '.join(failed) or 'tape'}" + ) return staged def find_active_job(self, catalog_path: str) -> str | None: From a21870921e908cb671e8e38a7a355b9acd9b2f41 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:14:19 +0000 Subject: [PATCH 109/205] docs(fuse): one restore job (one tape load) per hydration batch Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/fuse.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fuse.md b/docs/fuse.md index bfb8250..9f21a4e 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -7,8 +7,8 @@ Five layers, top to bottom, and what is real today: | SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`). | | Namespace/metadata | **Real.** `CatalogFuseOperations` (`openblade/fuse/mount.py`): offline files report their real size; xattr `user.openblade.state` = `offline`/`hydrating`/`online`. | | Hydration/cache | **Real.** `Hydrator` (`openblade/fuse/hydration.py`): one shared ticket per path, `OPENBLADE_FUSE_HYDRATE_TIMEOUT` (default 300 s) blocking wait, or `EAGAIN` with `blocking=False`; failure = `EIO`, file stays `offline`, reason in `last_error()`. `HydrationCache` (`cache.py`): optional byte budget with LRU eviction that never evicts a file with open handles; restores land under a temp name and are renamed only after the sha256 matches. | -| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue` (one restore job per file); never a tape backend directly. A restarted hydrator re-attaches to a pending/running restore job for the path. Every `restore_batch`/`resume` call runs on a Hydrator worker thread over its **own** catalog Session (`catalog_factory`), never the caller's. Files of one tape batch are still restored one job per file (not yet one load per tape). | -| Placement/robotics | **Partial.** Requests are batched per tape barcode for `OPENBLADE_FUSE_BATCH_WINDOW_MS` (default 250 ms) and restored back to back; whether that is one physical load is the jobs scheduler's decision. | +| Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue_batch` — **one restore job per tape batch** (`run_restore_batch`: one lease, one load, one read-only mount, one unmount, one unload per source barcode; per-file checksum verification with a temp name renamed only on a match; a failed unmount journals `physical_state_unknown` and skips the unload). Never a tape backend directly. A restarted hydrator re-attaches to a pending/running single-file restore job for the path (batch jobs are not yet re-attachable). Every `restore_batch`/`resume` call runs on a Hydrator worker thread over its **own** catalog Session (`catalog_factory`), never the caller's. | +| Placement/robotics | **Partial.** Requests are batched per tape barcode for `OPENBLADE_FUSE_BATCH_WINDOW_MS` (default 250 ms) and each batch is one restore job, so one physical load per tape per batch. | Unmount (`destroy`) cancels batches not yet started and waits for in-flight restores. `openblade fuse mount --hydrate` builds this data plane (`build_data_plane` in `openblade/cli/fuse_and_health.py`) and passes it as `data_plane=`. Timeout and batch window come from the settings fields `fuse_hydrate_timeout_s` / `fuse_batch_window_ms` (env `OPENBLADE_FUSE_HYDRATE_TIMEOUT` / `OPENBLADE_FUSE_BATCH_WINDOW_MS`). The cache byte budget (`max_bytes`) is not yet configurable from the CLI, so the mount's cache is unbounded. From 96c3b330d453dac3582527a40773591af2e35a23 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:16:17 +0000 Subject: [PATCH 110/205] chore: replace deprecated datetime.utcnow() with domain.clock.naive_utcnow Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 5 ++-- openblade/catalog/models.py | 30 +++++++++----------- openblade/catalog/repository.py | 15 +++++----- openblade/cli/main.py | 3 +- openblade/domain/clock.py | 14 +++++++++ openblade/nas/catalog_rebuild.py | 4 +-- openblade/nas/catalog_rebuild_worker.py | 4 +-- openblade/nas/catalog_shard.py | 4 +-- openblade/nas/fuse_hook.py | 4 +-- openblade/nas/health_service.py | 4 +-- openblade/nas/ltfs_manifest.py | 5 ++-- openblade/nas/manifest_validator.py | 4 +-- openblade/nas/rbac_service.py | 3 +- openblade/nas/tape_orchestrator.py | 4 +-- openblade/nas/types.py | 6 ++-- openblade/nas/virtual_fs.py | 4 +-- tests/integration/test_archive_aml_bridge.py | 20 ++++++++++--- 17 files changed, 80 insertions(+), 53 deletions(-) create mode 100644 openblade/domain/clock.py diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index 8b13efb..b687e08 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -17,6 +17,7 @@ from openblade.catalog.db import get_catalog_repository from openblade.catalog.models import AmlUser from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.domain.models import MountHandle, MountMode from openblade.domain.wire import coerce_int from openblade.nas.catalog_rebuild import CatalogRebuildPlanner @@ -220,7 +221,7 @@ def _manifest_payload(service: NasService, dataset_id: str) -> dict[str, object] ], "total_files": detail["file_count"], "total_bytes": detail["total_bytes"], - "generated_at": datetime.utcnow().isoformat() + "Z", + "generated_at": naive_utcnow().isoformat() + "Z", } @@ -237,7 +238,7 @@ def _report_payload(service: NasService, dataset_id: str) -> dict[str, object]: for record in records ], "checksums": {record.relative_path: record.checksum_sha256 for record in records}, - "generated_at": datetime.utcnow().isoformat() + "Z", + "generated_at": naive_utcnow().isoformat() + "Z", } diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index be779f7..47358fb 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -18,6 +18,8 @@ ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from openblade.domain.clock import naive_utcnow + class Base(DeclarativeBase): """Base class for catalog ORM models.""" @@ -197,8 +199,8 @@ class RbacRole(Base): name: Mapped[str] = mapped_column(String, unique=True, nullable=False, index=True) description: Mapped[str] = mapped_column(Text, default="") permissions: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) - updated_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) class RbacUser(Base): @@ -213,8 +215,8 @@ class RbacUser(Base): is_active: Mapped[bool] = mapped_column(Boolean, default=True) is_admin: Mapped[bool] = mapped_column(Boolean, default=False) api_token_ids: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) - updated_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) last_login_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -227,7 +229,7 @@ class RbacApiToken(Base): token_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) permissions: Mapped[str] = mapped_column(Text, default="[]") expires_at: Mapped[str | None] = mapped_column(Text, nullable=True) - created_at: Mapped[str] = mapped_column(Text, default=lambda: datetime.utcnow().isoformat()) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat()) last_used_at: Mapped[str | None] = mapped_column(Text, nullable=True) revoked: Mapped[bool] = mapped_column(Boolean, default=False) @@ -244,7 +246,7 @@ class RbacAuditEvent(Base): outcome: Mapped[str] = mapped_column(String, default="") details: Mapped[str] = mapped_column(Text, default="{}") created_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat(), index=True + Text, default=lambda: naive_utcnow().isoformat(), index=True ) ip_address: Mapped[str | None] = mapped_column(String, nullable=True) @@ -374,10 +376,10 @@ class PathMapping(Base): checksum: Mapped[str | None] = mapped_column(String, nullable=True) last_seen_at: Mapped[str | None] = mapped_column(String, nullable=True) created_at: Mapped[str] = mapped_column( - String, nullable=False, default=lambda: datetime.utcnow().isoformat() + "Z" + String, nullable=False, default=lambda: naive_utcnow().isoformat() + "Z" ) updated_at: Mapped[str] = mapped_column( - String, nullable=False, default=lambda: datetime.utcnow().isoformat() + "Z" + String, nullable=False, default=lambda: naive_utcnow().isoformat() + "Z" ) __table_args__ = (UniqueConstraint("logical_path", "pool_id", name="uq_path_pool"),) @@ -397,12 +399,8 @@ class CatalogRebuildRun(Base): datasets_recovered: Mapped[int] = mapped_column(Integer, default=0) path_mappings_recovered: Mapped[int] = mapped_column(Integer, default=0) error_summary: Mapped[str] = mapped_column(Text, default="[]") - created_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) - updated_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) + created_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") + updated_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") completed_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -416,9 +414,7 @@ class ManifestVersion(Base): sha256: Mapped[str] = mapped_column(String(64), nullable=False) file_count: Mapped[int] = mapped_column(Integer, default=0) is_current: Mapped[bool] = mapped_column(Boolean, default=False) - recorded_at: Mapped[str] = mapped_column( - Text, default=lambda: datetime.utcnow().isoformat() + "Z" - ) + recorded_at: Mapped[str] = mapped_column(Text, default=lambda: naive_utcnow().isoformat() + "Z") class NasRestoreJob(Base): diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index c383830..0ecdc07 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -42,6 +42,7 @@ TapeOpLog, VolumeGroup, ) +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import FileNotFoundError from openblade.domain.models import DriveLease, FileInstanceState from openblade.domain.policies import SafetyToken @@ -115,7 +116,7 @@ def _load_json_value(value: str | None, default: object) -> object: def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + return naive_utcnow().isoformat() @dataclass(frozen=True) @@ -669,7 +670,7 @@ def mark_instance_archived(self, instance_id: str, checksum_verified: bool = Tru if instance is None: raise FileNotFoundError(f"File instance {instance_id} not found") instance.state = FileInstanceState.ARCHIVED.value - instance.archived_at = datetime.utcnow() + instance.archived_at = naive_utcnow() instance.checksum_verified = checksum_verified self.session.commit() @@ -692,7 +693,7 @@ def journal( """Append one event to ``job_id``'s journal and commit.""" entry = JobJournalEntry( job_id=job_id, - at=datetime.utcnow(), + at=naive_utcnow(), event=event, detail_json=json.dumps(dict(detail or {}), sort_keys=True, default=str), ) @@ -739,7 +740,7 @@ def create_staged_instance( self.session.add( JobJournalEntry( job_id=job_id, - at=datetime.utcnow(), + at=naive_utcnow(), event=_SHARD_STAGED_EVENT, detail_json=json.dumps( { @@ -819,7 +820,7 @@ def mark_instances_archived( """ verifying = frozenset({FileInstanceState.VERIFYING.value}) instances = self._load_instances_in(instance_ids, verifying) - now = datetime.utcnow() + now = naive_utcnow() for instance in instances: instance.state = FileInstanceState.ARCHIVED.value instance.archived_at = now @@ -839,7 +840,7 @@ def update_job_state(self, job_id: str, state: str, error: str | None = None) -> raise FileNotFoundError(f"Job {job_id} not found") job.state = state job.error = error - job.updated_at = datetime.utcnow() + job.updated_at = naive_utcnow() self.session.commit() def get_job(self, job_id: str) -> Job | None: @@ -946,7 +947,7 @@ def upsert_nas_policy(self, policy_id: str, data: dict[str, object]) -> NasStora policy.name = parsed.name policy.policy_type = parsed.policy_type policy.config_json = json.dumps(payload) - policy.updated_at = datetime.utcnow() + policy.updated_at = naive_utcnow() self.session.commit() self.session.refresh(policy) return policy diff --git a/openblade/cli/main.py b/openblade/cli/main.py index 42af56a..2d3ab5d 100644 --- a/openblade/cli/main.py +++ b/openblade/cli/main.py @@ -21,6 +21,7 @@ from openblade.cli.assist import assist as assist_command from openblade.cli.fuse_and_health import register as register_fuse_and_health from openblade.config import OpenBladeConfig, load_config +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import ( CartridgeNotFoundError, DriveCorrelationError, @@ -1070,7 +1071,7 @@ def catalog_backup( """ context = _get_context() dest.mkdir(parents=True, exist_ok=True) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + timestamp = naive_utcnow().strftime("%Y%m%dT%H%M%SZ") gz_path = dest / f"openblade-{timestamp}.db.gz" workdir = Path(tempfile.mkdtemp(prefix="openblade-backup-")) try: diff --git a/openblade/domain/clock.py b/openblade/domain/clock.py new file mode 100644 index 0000000..0796272 --- /dev/null +++ b/openblade/domain/clock.py @@ -0,0 +1,14 @@ +"""Naive-UTC timestamps for catalog rows. + +The catalog stores naive datetimes that mean UTC. ``datetime.utcnow()`` is +deprecated since Python 3.12; this keeps the same value without the warning. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def naive_utcnow() -> datetime: + """Current UTC time as a naive datetime (catalog convention).""" + return datetime.now(UTC).replace(tzinfo=None) diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index 3343e79..ec9c1f0 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -4,12 +4,12 @@ import json import logging -from datetime import datetime from typing import cast from uuid import uuid4 from openblade.catalog.models import NasDataset as NasDatasetRow from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.domain.protection import ProtectionPolicy from openblade.nas.catalog_shard import ( CatalogShard, @@ -37,7 +37,7 @@ def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" # The catalog rows below come back as `dict[str, object]` (serialized from the diff --git a/openblade/nas/catalog_rebuild_worker.py b/openblade/nas/catalog_rebuild_worker.py index c680b51..7c35824 100644 --- a/openblade/nas/catalog_rebuild_worker.py +++ b/openblade/nas/catalog_rebuild_worker.py @@ -2,13 +2,13 @@ from __future__ import annotations -from datetime import datetime from typing import Any from uuid import uuid4 import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.types import CatalogRebuildRunRecord, RebuildPlanRequest, RebuildRunStatus @@ -17,7 +17,7 @@ def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" class CatalogRebuildWorker: diff --git a/openblade/nas/catalog_shard.py b/openblade/nas/catalog_shard.py index 9ee602f..09d1518 100644 --- a/openblade/nas/catalog_shard.py +++ b/openblade/nas/catalog_shard.py @@ -1,11 +1,11 @@ from __future__ import annotations import json -from datetime import datetime from typing import Any from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from openblade.domain.clock import naive_utcnow from openblade.nas.ltfs_manifest import TapeMetadataWriter from openblade.nas.types import NasFileState, PathMappingRecord @@ -94,7 +94,7 @@ def build_shard( barcode=barcode, openblade_tape_id=tape_id, volume_group=volume_group, - generated_at=datetime.utcnow().isoformat() + "Z", + generated_at=naive_utcnow().isoformat() + "Z", files=files, datasets=datasets, ) diff --git a/openblade/nas/fuse_hook.py b/openblade/nas/fuse_hook.py index 16c7d54..2f469e3 100644 --- a/openblade/nas/fuse_hook.py +++ b/openblade/nas/fuse_hook.py @@ -2,9 +2,9 @@ from __future__ import annotations -from datetime import datetime from typing import Any +from openblade.domain.clock import naive_utcnow from openblade.nas.service import NasService from openblade.nas.types import NasFileState @@ -23,7 +23,7 @@ def __init__(self, service: NasService): def _log_access(self, *, pool_id: str, logical_path: str, state: str, action: str) -> None: self._access_log.append( { - "timestamp": datetime.utcnow().isoformat() + "Z", + "timestamp": naive_utcnow().isoformat() + "Z", "pool_id": pool_id, "path": logical_path, "state": state, diff --git a/openblade/nas/health_service.py b/openblade/nas/health_service.py index bd12815..83e8757 100644 --- a/openblade/nas/health_service.py +++ b/openblade/nas/health_service.py @@ -4,7 +4,6 @@ import time from collections.abc import Callable -from datetime import datetime from typing import Any import structlog @@ -12,6 +11,7 @@ from openblade.catalog.models import Cartridge, NasDataset, NasFileRecord, PathMapping from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.types import ( CatalogStatusResponse, ComponentHealth, @@ -31,7 +31,7 @@ def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" class HealthService: diff --git a/openblade/nas/ltfs_manifest.py b/openblade/nas/ltfs_manifest.py index 9e362ce..bb0589e 100644 --- a/openblade/nas/ltfs_manifest.py +++ b/openblade/nas/ltfs_manifest.py @@ -2,12 +2,13 @@ import hashlib import json -from datetime import datetime from pathlib import PurePosixPath from typing import Any from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from openblade.domain.clock import naive_utcnow + MANIFEST_SCHEMA_V1 = "openblade.manifest.v1" MANIFEST_SCHEMA_V2 = "openblade.manifest.v2" MANIFEST_SCHEMA = MANIFEST_SCHEMA_V2 @@ -367,4 +368,4 @@ def _safe_name(value: str) -> str: @staticmethod def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/openblade/nas/manifest_validator.py b/openblade/nas/manifest_validator.py index 6facf24..35c9201 100644 --- a/openblade/nas/manifest_validator.py +++ b/openblade/nas/manifest_validator.py @@ -1,10 +1,10 @@ from __future__ import annotations import json -from datetime import datetime from pydantic import BaseModel, Field, ValidationError +from openblade.domain.clock import naive_utcnow from openblade.nas.catalog_shard import CatalogShard, CatalogShardWriter from openblade.nas.ltfs_manifest import ( MANIFEST_SCHEMA, @@ -237,7 +237,7 @@ def begin_write(self, barcode: str, manifest: ManifestJson) -> str: """ self.writer.ensure_openblade_dirs(barcode) self.writer._ensure_dir(barcode, self.VERSIONS_DIR) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + timestamp = naive_utcnow().strftime("%Y%m%dT%H%M%SZ") temp_path = f"{self.VERSIONS_DIR}/manifest.{timestamp}.tmp" self.writer._write_json(barcode, temp_path, manifest.model_dump(by_alias=True)) return temp_path diff --git a/openblade/nas/rbac_service.py b/openblade/nas/rbac_service.py index 1b4c5ec..3a1f9ad 100644 --- a/openblade/nas/rbac_service.py +++ b/openblade/nas/rbac_service.py @@ -10,6 +10,7 @@ import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.types import ( CreateTokenRequest, CreateTokenResult, @@ -279,4 +280,4 @@ def _sanitize_details(self, value: Any) -> Any: return value def _utcnow_iso(self) -> str: - return datetime.utcnow().isoformat() + return naive_utcnow().isoformat() diff --git a/openblade/nas/tape_orchestrator.py b/openblade/nas/tape_orchestrator.py index dd88064..1b0e885 100644 --- a/openblade/nas/tape_orchestrator.py +++ b/openblade/nas/tape_orchestrator.py @@ -4,7 +4,6 @@ import hashlib import threading -from datetime import datetime from pathlib import PurePosixPath from typing import Any, cast from uuid import uuid4 @@ -12,6 +11,7 @@ import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.domain.errors import ( ChecksumMismatchError, ExportRefusedError, @@ -670,4 +670,4 @@ def _required_int(extras: dict[str, Any], key: str, op_name: str) -> int: def _utcnow_iso() -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 28ea24e..1c5f18b 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -3,13 +3,13 @@ from __future__ import annotations import uuid -from datetime import datetime from enum import Enum from typing import Any, Literal from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator +from openblade.domain.clock import naive_utcnow from openblade.domain.protection import ProtectionPolicy from openblade.nas.media import Cartridge, MediaGeneration @@ -887,8 +887,8 @@ class PathMappingRecord(BaseModel): size: int = 0 checksum: str = "" last_seen_at: str = "" - created_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") - updated_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat() + "Z") + created_at: str = Field(default_factory=lambda: naive_utcnow().isoformat() + "Z") + updated_at: str = Field(default_factory=lambda: naive_utcnow().isoformat() + "Z") @field_validator("logical_path", mode="before") @classmethod diff --git a/openblade/nas/virtual_fs.py b/openblade/nas/virtual_fs.py index 62212f9..7804151 100644 --- a/openblade/nas/virtual_fs.py +++ b/openblade/nas/virtual_fs.py @@ -6,13 +6,13 @@ from __future__ import annotations -from datetime import datetime from secrets import token_hex from threading import RLock import structlog from openblade.catalog.repository import CatalogRepository +from openblade.domain.clock import naive_utcnow from openblade.nas.types import ( HydrationJob, HydrationRequest, @@ -405,4 +405,4 @@ def _split_path(self, path: str) -> list[str]: return [segment for segment in path.split("/") if segment] def _utcnow_iso(self) -> str: - return datetime.utcnow().isoformat() + "Z" + return naive_utcnow().isoformat() + "Z" diff --git a/tests/integration/test_archive_aml_bridge.py b/tests/integration/test_archive_aml_bridge.py index 8b57773..c98fcc7 100644 --- a/tests/integration/test_archive_aml_bridge.py +++ b/tests/integration/test_archive_aml_bridge.py @@ -168,7 +168,11 @@ def _boom(*args, **kwargs): monkeypatch.setattr(context.ltfs, "write_file", original_write_file) jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "Job failed (RuntimeError); see server logs for detail") + aml_job = next( + job + for job in jobs + if job["result"] == "Job failed (RuntimeError); see server logs for detail" + ) assert aml_job["type"] == "archive" assert aml_job["status"] == "failed" @@ -179,7 +183,9 @@ def _boom(*args, **kwargs): if event["component"] == "archive" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + assert ( + failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + ) drive = ( client.get("/aml/drive/DRV-001").json().get("drive") @@ -225,7 +231,11 @@ def _boom(*args, **kwargs): assert response.status_code == 500 jobs = client.get("/aml/jobs").json()["jobList"]["job"] - aml_job = next(job for job in jobs if job["result"] == "Job failed (RuntimeError); see server logs for detail") + aml_job = next( + job + for job in jobs + if job["result"] == "Job failed (RuntimeError); see server logs for detail" + ) assert aml_job["type"] == "restore" assert aml_job["status"] == "failed" @@ -236,7 +246,9 @@ def _boom(*args, **kwargs): if event["component"] == "restore" and event["details"].get("jobId") == aml_job["id"] ) assert failure_event["severity"] == "error" - assert failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + assert ( + failure_event["details"]["error"] == "Job failed (RuntimeError); see server logs for detail" + ) assert failure_event["details"]["catalogPath"] == "/photos/restore.txt" assert archived_file.read_text() == "restore aml bridge" From 6e58d5bdaf735048a5cc78ccd9f568ef247e5994 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:21:49 +0000 Subject: [PATCH 111/205] fix(safety): unload gates for real hardware and across processes RealLibraryBackend.unload had no LTFS mount gate at all; it now refuses while the drive's mount record is mounted or dirty, like the simulator and the Web Services backend. The tape orchestrator additionally refuses an unload while another job holds a live catalog lease on the drive, because mount state is per process and the lease is the only shared evidence. Both gates ship with mutation-checked regression tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/errors.py | 4 + openblade/hardware/library.py | 6 ++ openblade/nas/tape_orchestrator.py | 21 +++++ tests/unit/test_unload_gates.py | 120 +++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 tests/unit/test_unload_gates.py diff --git a/openblade/domain/errors.py b/openblade/domain/errors.py index 1d2f651..cbb8ed1 100644 --- a/openblade/domain/errors.py +++ b/openblade/domain/errors.py @@ -46,6 +46,10 @@ class TapeMountedError(SafetyViolationError): """Cannot unload a mounted tape.""" +class DriveLeasedError(SafetyViolationError): + """Another job holds a live lease on this drive; it may have LTFS mounted.""" + + class DriveBusyError(OpenBladeError): """Drive is busy.""" diff --git a/openblade/hardware/library.py b/openblade/hardware/library.py index 9ec8b56..405f425 100644 --- a/openblade/hardware/library.py +++ b/openblade/hardware/library.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from openblade.config import OpenBladeConfig +from openblade.domain.errors import TapeMountedError from openblade.domain.models import ( Barcode, CartridgeState, @@ -103,6 +104,11 @@ def load(self, source_slot: int, drive_id: int) -> OperationResult: return result def unload(self, drive_id: int, target_slot: int) -> OperationResult: + # Never unload while LTFS is mounted or dirty. The changer cannot see + # host-side mount state, so this in-process record is the gate. + mount_state = self._mount_states.get(drive_id, MountState.UNMOUNTED) + if mount_state is not MountState.UNMOUNTED: + raise TapeMountedError(f"Drive {drive_id} cannot be unloaded while {mount_state.value}") result = self.changer.unload(drive_id, target_slot) if result.success: self._mount_states[drive_id] = MountState.UNMOUNTED diff --git a/openblade/nas/tape_orchestrator.py b/openblade/nas/tape_orchestrator.py index 1b0e885..f423618 100644 --- a/openblade/nas/tape_orchestrator.py +++ b/openblade/nas/tape_orchestrator.py @@ -14,6 +14,7 @@ from openblade.domain.clock import naive_utcnow from openblade.domain.errors import ( ChecksumMismatchError, + DriveLeasedError, ExportRefusedError, ImportExportSlotError, MailslotUnsupportedError, @@ -237,11 +238,31 @@ def _unload(self, request: TapeOpRequest) -> dict[str, Any]: if drive_id is None: raise ValueError(f"Barcode {request.barcode} is not loaded in a drive") slot_id = request.slot_id if request.slot_id is not None else self._find_empty_slot() + self._refuse_if_leased_by_another_job(drive_id, request.job_id) result = self.library.unload(drive_id, slot_id) return self._operation_result( result, {"barcode": request.barcode, "drive_id": drive_id, "slot_id": slot_id} ) + def _refuse_if_leased_by_another_job(self, drive_id: int, job_id: str | None) -> None: + """Cross-process half of "never unload while mounted". + + Mount state is tracked per process, so a drive another job (possibly in + another process) still holds a live lease on may have LTFS mounted there. + The lease is the only shared evidence; an unload waits for it to be + released or to expire (recovery then reports the drive for reconciliation). + """ + if self.repo is None: + return + for lease in self.repo.live_leases(): + physical = ( + lease.physical_drive_id if lease.physical_drive_id is not None else lease.drive_id + ) + if physical == drive_id and lease.job_id != job_id: + raise DriveLeasedError( + f"Drive {drive_id} is leased by job {lease.job_id}; refusing to unload" + ) + def _format(self, request: TapeOpRequest) -> dict[str, Any]: confirmation = request.extras.get("format_confirmation") if not isinstance(confirmation, FormatConfirmation): diff --git a/tests/unit/test_unload_gates.py b/tests/unit/test_unload_gates.py new file mode 100644 index 0000000..07cf44e --- /dev/null +++ b/tests/unit/test_unload_gates.py @@ -0,0 +1,120 @@ +"""Never unload while LTFS is mounted — both halves of the gate. + +In-process: the real-hardware library adapter refuses while its mount record is +not UNMOUNTED. Cross-process: the tape orchestrator refuses while another job +holds a live catalog lease on the drive (mount state is per process, the lease +is the only shared evidence). +""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import TapeMountedError +from openblade.domain.models import MountState, OperationResult +from openblade.hardware.library import RealLibraryBackend +from openblade.nas.tape_orchestrator import TapeOperationOrchestrator +from openblade.nas.types import TapeOpRequest, TapeOpStatus, TapeOpType +from openblade.simulator.library import MockLibraryBackend +from openblade.simulator.ltfs_volume import MockLTFSBackend + + +class _RecordingChanger: + def __init__(self) -> None: + self.unloads: list[tuple[int, int]] = [] + + def unload(self, drive_id: int, target_slot: int) -> OperationResult: + self.unloads.append((drive_id, target_slot)) + return OperationResult(success=True, message="unloaded") + + +def _hardware_library() -> tuple[RealLibraryBackend, _RecordingChanger]: + # Bypass the guarded constructor: this test is about the unload gate only. + library = RealLibraryBackend.__new__(RealLibraryBackend) + changer = _RecordingChanger() + object.__setattr__(library, "changer", changer) + object.__setattr__(library, "_mount_states", {}) + return library, changer + + +def test_hardware_library_refuses_unload_while_mounted() -> None: + library, changer = _hardware_library() + library._mount_states[0] = MountState.MOUNTED_RW + with pytest.raises(TapeMountedError): + library.unload(0, 1) + assert changer.unloads == [] + + +def test_hardware_library_refuses_unload_while_dirty() -> None: + library, changer = _hardware_library() + library._mount_states[0] = MountState.DIRTY + with pytest.raises(TapeMountedError): + library.unload(0, 1) + assert changer.unloads == [] + + +def test_hardware_library_unloads_when_unmounted() -> None: + library, changer = _hardware_library() + assert library.unload(0, 1).success + assert changer.unloads == [(0, 1)] + + +def _orchestrator() -> tuple[CatalogRepository, MockLibraryBackend, TapeOperationOrchestrator]: + init_db("sqlite:///:memory:") + repo = CatalogRepository(get_session()) + library = MockLibraryBackend(num_slots=4, num_drives=1) + library.seed_slots(["OBX001L9"]) + ltfs = MockLTFSBackend(library) + library.load(1, 0) + return repo, library, TapeOperationOrchestrator(repo, library, ltfs) + + +def test_orchestrator_refuses_unload_while_another_job_holds_a_live_lease() -> None: + repo, library, orchestrator = _orchestrator() + owner = repo.create_job("archive", {}) + leases = repo.acquire_drive_leases( + job_id=owner.id, barcodes=["OBX001L9"], num_drives=1, ttl=timedelta(minutes=15) + ) + assert leases is not None + other = repo.create_job("restore", {}) + record = orchestrator.execute( + TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode="OBX001L9", + drive_id=0, + slot_id=1, + requested_by="test", + job_id=other.id, + ) + ) + assert record.status is TapeOpStatus.FAILED + assert record.error == "Tape unload operation failed" # curated; cause is logged + assert library.find_drive_by_barcode("OBX001L9") == 0 # still in the drive + + +def test_orchestrator_allows_unload_for_the_lease_owner_and_after_release() -> None: + repo, library, orchestrator = _orchestrator() + owner = repo.create_job("archive", {}) + leases = repo.acquire_drive_leases( + job_id=owner.id, barcodes=["OBX001L9"], num_drives=1, ttl=timedelta(minutes=15) + ) + assert leases is not None + request = TapeOpRequest( + op_type=TapeOpType.UNLOAD, + barcode="OBX001L9", + drive_id=0, + slot_id=1, + requested_by="test", + job_id=owner.id, + ) + orchestrator.execute(request) + assert library.find_drive_by_barcode("OBX001L9") is None + library.load(1, 0) + repo.release_leases([lease.id for lease in leases]) + other = repo.create_job("restore", {}) + orchestrator.execute(request.model_copy(update={"job_id": other.id})) + assert library.find_drive_by_barcode("OBX001L9") is None From 14347855b32ac4d08861ec0f8842bb784f316cb7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:30:35 +0000 Subject: [PATCH 112/205] fix(jobs): ArchiveService schedules through the catalog lease store The service path (assistant, API) fell back to a job-local in-memory scheduler, so its drives were invisible to every other job. Regression test observes the lease from a second session while the write runs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/archive.py | 15 ++++++++++++++- tests/unit/test_nonsharded_leases.py | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/openblade/jobs/archive.py b/openblade/jobs/archive.py index c29d9aa..d9040fb 100644 --- a/openblade/jobs/archive.py +++ b/openblade/jobs/archive.py @@ -21,7 +21,7 @@ from openblade.domain.models import JobType, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue -from openblade.jobs.scheduler import DriveHandle, DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import TapeOperationFailedError, execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -432,11 +432,23 @@ def __init__( ltfs: LTFSBackend, catalog: CatalogRepository, queue: JobQueue, + *, + lease_store: LeaseStore | None = None, ) -> None: self.library = library self.ltfs = ltfs self.catalog = catalog self.queue = queue + # Production passes the catalog-backed store so this job's drives are + # excluded from every other job, in this process or another one. + self.lease_store = lease_store if lease_store is not None else CatalogLeaseStore(catalog) + + def _scheduler(self, job_id: str) -> DriveScheduler: + return DriveScheduler( + num_drives=len(InventoryService(self.library).snapshot().drives), + store=self.lease_store, + job_id=job_id, + ) def enqueue(self, volume_group_name: str, source_path: Path) -> Job: job = self.catalog.create_job( @@ -450,6 +462,7 @@ def enqueue(self, volume_group_name: str, source_path: Path) -> Job: self.ltfs, self.catalog, job.id, + scheduler=self._scheduler(job.id), ) except Exception: for file_path in _iter_source_files(source_path): diff --git a/tests/unit/test_nonsharded_leases.py b/tests/unit/test_nonsharded_leases.py index 5768a05..448bbe9 100644 --- a/tests/unit/test_nonsharded_leases.py +++ b/tests/unit/test_nonsharded_leases.py @@ -144,3 +144,29 @@ def record_unload(*args: Any, **kwargs: Any) -> Any: assert calls == [] refreshed = repo_a.get_job(job.id) assert refreshed is not None and refreshed.state == "failed" + + +def test_archive_service_holds_a_catalog_lease_while_writing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The service path (assistant, API) must exclude other jobs, not just run_*_job.""" + from openblade.jobs.archive import ArchiveService + from openblade.jobs.queue import JobQueue + + repo_a, repo_b, library, ltfs = _stack(tmp_path) + source = tmp_path / "src" + source.mkdir() + (source / "a.bin").write_bytes(b"x" * 64) + service = ArchiveService(library, ltfs, repo_a, JobQueue()) + seen: list[list[str]] = [] + real_write = ltfs.write_file + + def write(*args: Any, **kwargs: Any) -> Any: + seen.append([lease.job_id for lease in repo_b.live_leases()]) + return real_write(*args, **kwargs) + + monkeypatch.setattr(ltfs, "write_file", write) + job = service.enqueue("photos", source) + assert job.state == "completed" + assert seen and all(owners == [job.id] for owners in seen) # seen from "another process" + assert repo_b.live_leases() == [] From 84a41efc9e651d0aca29ba337f93185969ebcf2c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:31:57 +0000 Subject: [PATCH 113/205] fix(jobs): RestoreService schedules through the catalog lease store (single and batch) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/restore.py | 24 ++++++++++++++++++++++-- tests/unit/test_fuse_cli_wiring.py | 3 +++ tests/unit/test_nonsharded_leases.py | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index 61842a1..d809b28 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -23,7 +23,7 @@ from openblade.domain.models import JobType, MountHandle, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue -from openblade.jobs.scheduler import DriveHandle, DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -472,11 +472,23 @@ def __init__( ltfs: LTFSBackend, catalog: CatalogRepository, queue: JobQueue, + *, + lease_store: LeaseStore | None = None, ) -> None: self.library = library self.ltfs = ltfs self.catalog = catalog self.queue = queue + # Catalog-backed by default so this job's drives are excluded from every + # other job, in this process or another one. + self.lease_store = lease_store if lease_store is not None else CatalogLeaseStore(catalog) + + def _scheduler(self, job_id: str) -> DriveScheduler: + return DriveScheduler( + num_drives=len(InventoryService(self.library).snapshot().drives), + store=self.lease_store, + job_id=job_id, + ) def enqueue(self, catalog_path: str, destination: Path) -> Job: job = self.catalog.create_job( @@ -489,6 +501,7 @@ def enqueue(self, catalog_path: str, destination: Path) -> Job: self.ltfs, self.catalog, job.id, + scheduler=self._scheduler(job.id), ) refreshed = self.catalog.get_job(job.id) assert refreshed is not None @@ -505,7 +518,14 @@ def enqueue_batch(self, requests: list[RestoreRequest]) -> tuple[Job, RestoreBat ] }, ) - result = run_restore_batch(requests, self.library, self.ltfs, self.catalog, job.id) + result = run_restore_batch( + requests, + self.library, + self.ltfs, + self.catalog, + job.id, + scheduler=self._scheduler(job.id), + ) refreshed = self.catalog.get_job(job.id) assert refreshed is not None return refreshed, result diff --git a/tests/unit/test_fuse_cli_wiring.py b/tests/unit/test_fuse_cli_wiring.py index 8457b87..7cbee13 100644 --- a/tests/unit/test_fuse_cli_wiring.py +++ b/tests/unit/test_fuse_cli_wiring.py @@ -66,6 +66,9 @@ def __init__(self, library: Any, ltfs: Any, catalog: Any, queue: Any) -> None: def enqueue(self, path: str, dest: Path) -> Any: return SimpleNamespace(id="j1", state="completed") + def enqueue_batch(self, requests: list[Any]) -> Any: + return SimpleNamespace(id="j1", state="completed"), SimpleNamespace(items=[]) + monkeypatch.setattr(fuse_and_health, "RestoreService", StubService) worker = threading.Thread(target=plane.engine.restore_batch, args=("T1", ["/a.bin"])) worker.start() diff --git a/tests/unit/test_nonsharded_leases.py b/tests/unit/test_nonsharded_leases.py index 448bbe9..90cc366 100644 --- a/tests/unit/test_nonsharded_leases.py +++ b/tests/unit/test_nonsharded_leases.py @@ -170,3 +170,30 @@ def write(*args: Any, **kwargs: Any) -> Any: assert job.state == "completed" assert seen and all(owners == [job.id] for owners in seen) # seen from "another process" assert repo_b.live_leases() == [] + + +def test_restore_service_holds_a_catalog_lease_while_reading( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from openblade.jobs.archive import ArchiveService + from openblade.jobs.queue import JobQueue + from openblade.jobs.restore import RestoreService + + repo_a, repo_b, library, ltfs = _stack(tmp_path) + source = tmp_path / "src" + source.mkdir() + (source / "a.bin").write_bytes(b"y" * 64) + ArchiveService(library, ltfs, repo_a, JobQueue()).enqueue("photos", source) + service = RestoreService(library, ltfs, repo_a, JobQueue()) + seen: list[list[str]] = [] + real_read = ltfs.read_file + + def read(*args: Any, **kwargs: Any) -> Any: + seen.append([lease.job_id for lease in repo_b.live_leases()]) + return real_read(*args, **kwargs) + + monkeypatch.setattr(ltfs, "read_file", read) + job = service.enqueue("/photos/a.bin", tmp_path / "out.bin") + assert job.state == "completed" + assert seen and all(owners == [job.id] for owners in seen) + assert repo_b.live_leases() == [] From e78f155f63da821108c0ca31a8a0eb0c31c282e9 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:33:07 +0000 Subject: [PATCH 114/205] fix(jobs): curated physical-state error text; failure-path journaling never masks the lane error Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/sharded_archive.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 79c5605..220805d 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -7,6 +7,7 @@ import os import shutil import uuid +from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -280,7 +281,8 @@ def _clean_unmount_and_unload( ltfs.unmount(mount) mounts.pop(barcode, None) except Exception as exc: # noqa: BLE001 - aggregated and re-raised below - failures.append(f"unmount {barcode}: {exc}") + logger.exception("job %s: unmount of %s failed", job_id, barcode) + failures.append(f"unmount {barcode}: {type(exc).__name__}") still_mounted.add(barcode) for handle in handles: slot_id = loaded_slots.get(handle.physical) @@ -311,7 +313,8 @@ def _clean_unmount_and_unload( ) loaded_slots.pop(handle.physical, None) except Exception as exc: # noqa: BLE001 - aggregated and re-raised below - failures.append(f"unload {handle.barcode}: {exc}") + logger.exception("job %s: unload of %s failed", job_id, handle.barcode) + failures.append(f"unload {handle.barcode}: {type(exc).__name__}") if failures: raise TapePhysicalStateError( "shards written but physical state is unknown (reconcile required): " @@ -340,6 +343,18 @@ def _job_error(exc: Exception) -> str: return safe_job_error(exc) +def _journal_failure( + catalog: CatalogRepository, job_id: str, event: str, detail: dict[str, object] +) -> None: + """Journal on an error path without letting a broken session mask the real error.""" + try: + catalog.journal(job_id, event, detail) + except Exception: # noqa: BLE001 - evidence only; the original exception is what the caller reports + logger.exception("job %s: could not journal %s; rolling the session back", job_id, event) + with suppress(Exception): + catalog.session.rollback() + + def record_physical_state_unknown( catalog: CatalogRepository, job_id: str, op: str, barcode: str, drive: int | None ) -> None: @@ -574,7 +589,7 @@ def _write_one( # so unmount/unload here could eject their tape mid-write. Physical # state is unknown -> reconcile (recovery report / inventory). fenced_out = True - catalog.journal(job_id, "fenced_out", {"barcodes": batch_barcodes}) + _journal_failure(catalog, job_id, "fenced_out", {"barcodes": batch_barcodes}) raise except Exception as exc: # noqa: BLE001 # Staged instances remain PENDING -> not exposed as archived; resumable. @@ -587,7 +602,7 @@ def _write_one( exc, exc_info=True, ) - catalog.journal(job_id, "failed", {"error": type(exc).__name__}) + _journal_failure(catalog, job_id, "failed", {"error": type(exc).__name__}) errors.append(_job_error(exc)) finally: if fenced_out: @@ -721,10 +736,10 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: except StaleLeaseError: # Fenced out: abort and leave the hardware alone (see _archive_stripe). fenced_out = True - catalog.journal(job_id, "fenced_out", {"barcodes": list(request.lane_barcodes)}) + _journal_failure(catalog, job_id, "fenced_out", {"barcodes": list(request.lane_barcodes)}) raise except Exception as exc: - catalog.journal(job_id, "failed", {"error": type(exc).__name__}) + _journal_failure(catalog, job_id, "failed", {"error": type(exc).__name__}) raise finally: if fenced_out: From e098d1cad36923b5c89cf9b4dcdbb72799a5968c Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:37:41 +0000 Subject: [PATCH 115/205] chore: finish the utcnow sweep (column defaults, gateway); refresh tech-debt and the stale format note Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- CLAUDE.md | 2 +- docs/tech-debt.md | 18 ++++++------------ openblade/catalog/models.py | 30 +++++++++++++++--------------- openblade/nas/protocol_gateway.py | 13 +++++++------ 4 files changed, 29 insertions(+), 34 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1ec5b26..21b338f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ Definition of done = `make all` green + relevant frontend checks + any parity ga - `ruff check .` is expected to be **clean (0)**. If you add a `# noqa`, it needs a written reason at the site — bare noqa is not accepted here. - `ruff format` changes go in their own format-only commit, never mixed with - logic. Known exception still outstanding: `openblade/assistant/tools.py`. + logic. (The former `openblade/assistant/tools.py` exception is resolved; the tree is fully formatted.) - Background + the deferred `BLE`/`S` rule families: docs/decisions/2026-09-11-pin-ruff-toolchain.md. diff --git a/docs/tech-debt.md b/docs/tech-debt.md index 23ffd29..2d84d62 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -65,21 +65,15 @@ raise. ## `datetime.utcnow` (deprecated since Python 3.12) +Paid down 2026-10-09: every call and SQLAlchemy column default now goes through +`openblade/domain/clock.py::naive_utcnow()` (same naive-UTC value, no +deprecation warning). `openblade/api/aml_state.py` keeps its own aware +`_utcnow()` helper. Verify with: + ```bash -grep -roE "datetime\.utcnow\b" --include=*.py openblade tests tools | wc -l # 54 -grep -rlE "datetime\.utcnow\b" --include=*.py openblade tests tools | wc -l # 16 files +grep -rE "datetime\.utcnow\b" --include=*.py openblade tests tools | grep -v domain/clock.py | wc -l # 0 ``` -54 references in 16 files, all under `openblade/` (0 in `tests/` and `tools/`). -They include both calls and bare references such as SQLAlchemy column defaults -(`default=datetime.utcnow` in `openblade/catalog/models.py`). The largest -groups are `openblade/catalog/models.py`, `openblade/catalog/repository.py` -and `openblade/nas/protocol_gateway.py`. These return naive datetimes. Migrate -them module by module to `datetime.now(timezone.utc)`, and check every column -or comparison that mixes naive and aware values before you switch it. -`openblade/api/aml_state.py` already uses its own aware `_utcnow()` helper and -is not counted here. - ## In-memory job queue `openblade/jobs/queue.py:23` (`class JobQueue`) keeps jobs, drive owners and the diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index 47358fb..0bd0fb9 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -36,9 +36,9 @@ class LibraryInstance(Base): role: Mapped[str] = mapped_column(String, default="primary") sort_order: Mapped[int] = mapped_column(Integer, default=0) enabled: Mapped[bool] = mapped_column(Boolean, default=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @@ -47,7 +47,7 @@ class VolumeGroup(Base): id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) name: Mapped[str] = mapped_column(String, unique=True, nullable=False, index=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) cartridges: Mapped[list[Cartridge]] = relationship( "Cartridge", back_populates="volume_group", cascade="save-update" @@ -76,7 +76,7 @@ class Cartridge(Base): used_bytes: Mapped[int] = mapped_column(Integer, default=0) state: Mapped[str] = mapped_column(String, default="in_slot") formatted: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) volume_group: Mapped[VolumeGroup | None] = relationship( "VolumeGroup", back_populates="cartridges" @@ -98,7 +98,7 @@ class FileRecord(Base): block_size: Mapped[int | None] = mapped_column(Integer, nullable=True) shard_profile: Mapped[str | None] = mapped_column(String(32), nullable=True) parent_id: Mapped[str | None] = mapped_column(String, nullable=True, index=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) volume_group: Mapped[VolumeGroup] = relationship("VolumeGroup", back_populates="file_records") instances: Mapped[list[FileInstance]] = relationship( @@ -121,7 +121,7 @@ class FileInstance(Base): state: Mapped[str] = mapped_column(String, default="pending") archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) checksum_verified: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) file_record: Mapped[FileRecord] = relationship("FileRecord", back_populates="instances") @@ -134,9 +134,9 @@ class Job(Base): state: Mapped[str] = mapped_column(String, default="pending") metadata_json: Mapped[str] = mapped_column(Text, default="{}") error: Mapped[str | None] = mapped_column(Text, nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @property @@ -182,9 +182,9 @@ class AmlUser(Base): password: Mapped[str] = mapped_column(String(64), nullable=False) role: Mapped[int] = mapped_column(Integer, nullable=False, default=1) require_password_change: Mapped[bool] = mapped_column(Boolean, default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @property @@ -258,9 +258,9 @@ class NasStoragePolicy(Base): name: Mapped[str] = mapped_column(String, nullable=False) policy_type: Mapped[str] = mapped_column(String(64), nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) updated_at: Mapped[datetime] = mapped_column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + DateTime, default=naive_utcnow, onupdate=naive_utcnow ) @@ -272,7 +272,7 @@ class NasCacheDrive(Base): root_path: Mapped[str] = mapped_column(String, nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") enabled: Mapped[bool] = mapped_column(Boolean, default=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) class NasConfig(Base): @@ -289,7 +289,7 @@ class NasShare(Base): name: Mapped[str] = mapped_column(String, nullable=False) share_type: Mapped[str] = mapped_column(String(32), nullable=False) config_json: Mapped[str] = mapped_column(Text, default="{}") - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=naive_utcnow) class NasPool(Base): @@ -484,7 +484,7 @@ class JobJournalEntry(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) job_id: Mapped[str] = mapped_column(String, nullable=False, index=True) - at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow) + at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=naive_utcnow) event: Mapped[str] = mapped_column(String(64), nullable=False) detail_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 041b585..7daca4d 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -19,6 +19,7 @@ from enum import Enum from typing import Any +from openblade.domain.clock import naive_utcnow from openblade.fuse.cache import CacheEntryInUseError from openblade.fuse.hydration import HydrationFailedError, HydrationState, Hydrator @@ -93,7 +94,7 @@ class GatewayCredential: _hashed_password: str allowed_paths: list[str] = field(default_factory=lambda: [InboxPath.GENERAL.value]) enabled: bool = True - created_at: datetime = field(default_factory=datetime.utcnow) + created_at: datetime = field(default_factory=naive_utcnow) last_seen_at: datetime | None = None @classmethod @@ -130,7 +131,7 @@ class GatewayUpload: requested_path: str routed_path: str bytes_uploaded: int - uploaded_at: datetime = field(default_factory=datetime.utcnow) + uploaded_at: datetime = field(default_factory=naive_utcnow) @dataclass @@ -214,7 +215,7 @@ def authenticate(self, username: str, password: str) -> GatewayCredential | None if not cred or not cred.enabled: return None if cred.verify_password(password): - cred.last_seen_at = datetime.utcnow() + cred.last_seen_at = naive_utcnow() return cred return None @@ -255,7 +256,7 @@ def open_session(self, username: str, remote_addr: str) -> GatewaySession: session_id=secrets.token_hex(16), username=username, remote_addr=remote_addr, - connected_at=datetime.utcnow(), + connected_at=naive_utcnow(), ) self._sessions.append(session) return session @@ -287,7 +288,7 @@ def close_session( session = self._get_session(session_id) if session is None: return - session.disconnected_at = datetime.utcnow() + session.disconnected_at = naive_utcnow() session.bytes_uploaded = max(session.bytes_uploaded, bytes_uploaded) session.files_uploaded = max(session.files_uploaded, files_uploaded) session.errors = errors @@ -337,7 +338,7 @@ def stop(self) -> None: """Mark gateway as stopped, closing any active sessions.""" if self._status is GatewayStatus.DISABLED: return - now = datetime.utcnow() + now = naive_utcnow() for session in self._sessions: if session.disconnected_at is None: with contextlib.suppress(Exception): From 7175adcc03d58d1d0e301bcb419ce47b4ef8ba36 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:40:57 +0000 Subject: [PATCH 116/205] fix(nas): hash on-tape bytes for commit markers, not a model re-dump A re-dump of the parsed manifest changes whenever ManifestJson gains a defaulted field, silently marking every older v2 generation uncommitted. The verifier now hashes the raw manifest.json / catalog-shard.json text. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/catalog_rebuild.py | 21 ++++++++++++++------- openblade/nas/ltfs_manifest.py | 15 ++++++++++----- tests/unit/test_catalog_rebuild.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/openblade/nas/catalog_rebuild.py b/openblade/nas/catalog_rebuild.py index ec9c1f0..77df65d 100644 --- a/openblade/nas/catalog_rebuild.py +++ b/openblade/nas/catalog_rebuild.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import logging from typing import cast @@ -44,6 +45,10 @@ def _utcnow_iso() -> str: # same pydantic models that are rebuilt from them), so every column read needs a # type. These are `cast` only -- no runtime narrowing, no new failure mode: # validation stays exactly where it was, in the model construction one line down. +def _sha256_text(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def _text_column(value: object) -> str | None: return cast("str | None", value) @@ -217,15 +222,17 @@ def _uncommitted_reason(self, barcode: str, manifest: ManifestJson) -> str | Non marker = self.metadata_writer.read_commit_marker(barcode, manifest.generation_id) if marker is None: return "commit marker missing" - manifest_sha = self.metadata_writer.compute_json_checksum( - manifest.model_dump(by_alias=True) - ) - shard_sha = self.metadata_writer._read_text( - barcode, self.shard_writer.SHARD_CHECKSUM_METADATA_PATH + # Hash the raw on-tape bytes, never a re-dump of the parsed model: a re-dump + # changes whenever the model gains a defaulted field, which would silently mark + # every older generation uncommitted. Markers written by the current writer hash + # these same bytes, so no compatibility shim is needed. + manifest_raw = self.metadata_writer._read_text( + barcode, self.metadata_writer._metadata_path("manifest.json") ) - if marker.manifest_sha256 != manifest_sha: + shard_raw = self.metadata_writer._read_text(barcode, self.shard_writer.SHARD_METADATA_PATH) + if manifest_raw is None or marker.manifest_sha256 != _sha256_text(manifest_raw): return "commit marker manifest hash mismatch" - if shard_sha is None or marker.catalog_shard_sha256 != shard_sha.strip(): + if shard_raw is None or marker.catalog_shard_sha256 != _sha256_text(shard_raw): return "commit marker catalog-shard hash mismatch" return None diff --git a/openblade/nas/ltfs_manifest.py b/openblade/nas/ltfs_manifest.py index bb0589e..fcb5a48 100644 --- a/openblade/nas/ltfs_manifest.py +++ b/openblade/nas/ltfs_manifest.py @@ -163,12 +163,17 @@ def write_tape_json(self, barcode: str, tape_json: TapeJson) -> None: ) def write_manifest(self, barcode: str, manifest: ManifestJson) -> str: - """Write /.openblade/manifest.json and return its sha256 checksum.""" + """Write /.openblade/manifest.json and return the sha256 of the exact bytes written. + + The checksum covers the on-tape bytes, not a re-dump of the model, so a commit + marker stays verifiable after ``ManifestJson`` gains new defaulted fields. + """ self.ensure_openblade_dirs(barcode) - payload = manifest.model_dump(by_alias=True) - checksum = self.compute_json_checksum(payload) - self._write_json(barcode, self._metadata_path("manifest.json"), payload) - return checksum + content = json.dumps( + manifest.model_dump(by_alias=True), sort_keys=True, separators=(",", ":") + ) + self._write_text(barcode, self._metadata_path("manifest.json"), content) + return hashlib.sha256(content.encode("utf-8")).hexdigest() def write_manifest_checksum(self, barcode: str, checksum: str) -> None: """Write /.openblade/manifest.sha256 for tamper-detection.""" diff --git a/tests/unit/test_catalog_rebuild.py b/tests/unit/test_catalog_rebuild.py index 76de147..3dbd2c8 100644 --- a/tests/unit/test_catalog_rebuild.py +++ b/tests/unit/test_catalog_rebuild.py @@ -541,3 +541,33 @@ def test_execute_rebuild_skips_malformed_protection_policy( assert result.datasets_recovered == 1 assert _dataset_protection_json(rebuild_env, f"dataset-{barcode}") is None + + +def test_commit_marker_survives_new_defaulted_manifest_field( + rebuild_env: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: the marker hashes on-tape bytes, so a newer ManifestJson with an extra + defaulted field (which changes model_dump()) still sees older generations as committed.""" + from openblade.nas.ltfs_manifest import ManifestJson + + class FutureManifestJson(ManifestJson): + future_field: str = "added-later" + + barcode = rebuild_env["context"].library.get_all_barcodes()[0] + _seed_tape(rebuild_env, barcode) + _stamp_generation(rebuild_env, barcode, marker="ok") + writer = rebuild_env["planner"].metadata_writer + original_read = writer.read_manifest + + def read_future(bc: str) -> ManifestJson | None: + parsed = original_read(bc) + return None if parsed is None else FutureManifestJson.model_validate(parsed.model_dump()) + + monkeypatch.setattr(writer, "read_manifest", read_future) + + result = rebuild_env["planner"].plan_rebuild( + RebuildPlanRequest(barcodes=[barcode], dry_run=True) + ) + + assert result.uncommitted_generations == [] + assert result.barcodes_to_scan == [barcode] From 312f191023fab747a60f133f2dd084e4219f08ee Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:42:20 +0000 Subject: [PATCH 117/205] fix(fuse): restarted hydrator finds and resumes batch restore jobs find_active_job/resume only understood top-level catalog_path/dest_path, so batch jobs ({"batch": [...]}) were invisible after a restart and a metadata shape with neither raised a bare KeyError. Both now resolve the matching batch entry; a missing destination is a typed HydrationFailedError. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/hydration.py | 44 ++++++++++++++++++----- tests/integration/test_fuse_data_plane.py | 41 ++++++++++++++++++++- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 01a9b04..90d68db 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -99,8 +99,8 @@ def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Pa def find_active_job(self, catalog_path: str) -> str | None: """Return the id of a pending/running restore job for the path, if any.""" - def resume(self, job_id: str) -> Path: - """Wait for an existing job to finish and return its staged file.""" + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + """Wait for an existing job to finish and return the path's staged file.""" class JobRestoreEngine: @@ -189,15 +189,15 @@ def find_active_job(self, catalog_path: str) -> str | None: for job in self.catalog.list_jobs(): if job.job_type != "restore" or job.state not in {"pending", "running"}: continue - if json.loads(job.metadata_json or "{}").get("catalog_path") == catalog_path: + if _job_entry(json.loads(job.metadata_json or "{}"), catalog_path) is not None: return str(job.id) return None - def resume(self, job_id: str) -> Path: + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: with self._worker_catalog() as catalog: - return self._resume(catalog, job_id) + return self._resume(catalog, job_id, catalog_path) - def _resume(self, catalog: Any, job_id: str) -> Path: + def _resume(self, catalog: Any, job_id: str, catalog_path: str | None = None) -> Path: deadline = time.monotonic() + self.resume_timeout_s while True: job = catalog.get_job(job_id) @@ -211,7 +211,7 @@ def _resume(self, catalog: Any, job_id: str) -> Path: if job is None: raise HydrationFailedError(f"restore job {job_id} disappeared") if job.state == "completed": - return Path(json.loads(job.metadata_json)["dest_path"]) + return _job_dest(job_id, json.loads(job.metadata_json or "{}"), catalog_path) if job.state not in {"pending", "running"}: raise HydrationFailedError(f"restore job {job_id} ended {job.state}") if time.monotonic() >= deadline: @@ -219,6 +219,34 @@ def _resume(self, catalog: Any, job_id: str) -> Path: time.sleep(self.poll_interval_s) +def _job_entry(metadata: Any, catalog_path: str | None) -> dict[str, Any] | None: + """The {catalog_path, dest_path} entry for ``catalog_path`` in single or batch job metadata. + + Single jobs store the entry at top level; batch jobs store ``{"batch": [entry, ...]}``. + With ``catalog_path=None`` a single job's top-level entry is returned. + """ + if not isinstance(metadata, dict): + return None + if "catalog_path" in metadata and catalog_path in {None, metadata["catalog_path"]}: + return metadata + batch = metadata.get("batch") + if catalog_path is not None and isinstance(batch, list): + for entry in batch: + if isinstance(entry, dict) and entry.get("catalog_path") == catalog_path: + return entry + return None + + +def _job_dest(job_id: str, metadata: Any, catalog_path: str | None) -> Path: + entry = _job_entry(metadata, catalog_path) + dest = None if entry is None else entry.get("dest_path") + if not isinstance(dest, str) or not dest: + raise HydrationFailedError( + f"restore job {job_id} has no staged destination for {catalog_path or 'its path'}" + ) + return Path(dest) + + class Hydrator: def __init__( self, @@ -378,7 +406,7 @@ def _flush(self, tape_key: str) -> None: def _resume(self, ticket: HydrationTicket) -> None: assert ticket.job_id is not None try: - self._commit(ticket, self.engine.resume(ticket.job_id)) + self._commit(ticket, self.engine.resume(ticket.job_id, ticket.catalog_path)) except HydrationTimeoutError: self._fail(ticket, f"restore job {ticket.job_id} timed out") except (HydrationFailedError, OpenBladeError, OSError, KeyError): diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 9f719e1..67f118d 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -56,7 +56,7 @@ def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Pa def find_active_job(self, catalog_path: str) -> str | None: return None - def resume(self, job_id: str) -> Path: + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: raise HydrationFailedError(job_id) @@ -327,3 +327,42 @@ def test_staging_copy_is_removed_after_verified_store(catalog, tmp_path, corrupt assert ticket.done.wait(3) assert (ticket.error is not None) is corrupt assert not (tmp_path / "s" / "c.bin").exists(), "staging copy leaked" + + +def test_restarted_engine_finds_and_resumes_batch_job(catalog, tmp_path): # type: ignore[no-untyped-def] + """Regression: batch jobs store {"batch": [...]}; a restarted hydrator must see them.""" + other = CatalogRepository(get_session()) + job = other.create_job( + "restore", + { + "batch": [ + {"catalog_path": "/a.bin", "dest_path": str(tmp_path / "a")}, + {"catalog_path": "/b.bin", "dest_path": str(tmp_path / "b")}, + ] + }, + ) + other.session.commit() + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=3.0 + ) + enqueued: list[object] = [] + engine.restore_batch = lambda *a: enqueued.append(a) or {} # type: ignore[method-assign,func-returns-value] + + assert engine.find_active_job("/b.bin") == str(job.id) + assert engine.find_active_job("/c.bin") is None + row = other.get_job(job.id) + row.state = "completed" + other.session.commit() + assert engine.resume(str(job.id), "/b.bin") == tmp_path / "b" + assert enqueued == [] + + +def test_resume_job_without_destination_raises_typed_error(catalog, tmp_path): # type: ignore[no-untyped-def] + job = catalog.create_job("restore", {"something": "else"}) + job.state = "completed" + catalog.session.commit() + engine = JobRestoreEngine( + None, catalog, tmp_path / "s", poll_interval_s=0.02, resume_timeout_s=1.0 + ) + with pytest.raises(HydrationFailedError): + engine.resume(str(job.id), "/a.bin") From 0bc5e58fbc3d47d25088cbddf0b2613746acace8 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:44:52 +0000 Subject: [PATCH 118/205] fix(nas): gateway on_open returns a handle token; on_close releases only it Closing by catalog path let a double close (or a close of a failed open) drop another client's pin, exposing an open file to budget eviction. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/protocol_gateway.py | 37 ++++++++++++++++++----- tests/integration/test_fuse_data_plane.py | 9 ++++-- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 7daca4d..5ffe011 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -14,6 +14,7 @@ import posixpath import secrets import socket +import threading from dataclasses import dataclass, field from datetime import datetime from enum import Enum @@ -147,6 +148,14 @@ class GatewaySession: uploads: list[GatewayUpload] = field(default_factory=list) +@dataclass(frozen=True) +class GatewayHandle: + """An open handle: opaque ``token`` for ``on_close`` plus the state at open.""" + + token: str + state: HydrationState + + class ProtocolGateway: """In-process protocol gateway manager for emulated SFTP/SCP ingest.""" @@ -154,6 +163,8 @@ def __init__(self) -> None: self._credentials: dict[str, GatewayCredential] = {} self._sessions: list[GatewaySession] = [] self._last_error: str | None = None + self._handles: dict[str, str] = {} # open-handle token -> pinned checksum + self._handles_lock = threading.Lock() self._bind_host = os.environ.get("OPENBLADE_SFTP_HOST", "0.0.0.0") self._bind_port = int(os.environ.get("OPENBLADE_SFTP_PORT", "2222")) self._max_sessions = int(os.environ.get("OPENBLADE_SFTP_MAX_SESSIONS", "10")) @@ -381,12 +392,12 @@ def _require_hydrator(self) -> Hydrator: raise HydrationFailedError("no hydrator attached to the protocol gateway") return hydrator - def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> HydrationState: + def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> GatewayHandle: """Pin the file and start hydration; optionally wait ``timeout`` seconds. Every successful ``on_open`` takes exactly one pin, whatever the state, - so it pairs with exactly one ``on_close`` release. Pinning only online - files let a close of a mid-hydration open drop another handle's pin. + and returns an opaque handle token; ``on_close(token)`` releases exactly + that pin, so a double close cannot drop another handle's pin. """ hydrator = self._require_hydrator() ticket = hydrator.request(catalog_path) @@ -394,10 +405,14 @@ def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> Hydratio try: if timeout: hydrator.wait(ticket, timeout) - return hydrator.status(catalog_path) + state = hydrator.status(catalog_path) except BaseException: hydrator.cache.release(ticket.checksum) raise + token = secrets.token_urlsafe(16) + with self._handles_lock: + self._handles[token] = ticket.checksum + return GatewayHandle(token=token, state=state) def on_read_range( self, catalog_path: str, offset: int, length: int, *, timeout: float | None = None @@ -406,11 +421,17 @@ def on_read_range( data = hydrator.wait(hydrator.request(catalog_path), timeout) return data[offset : offset + length] - def on_close(self, catalog_path: str) -> None: + def on_close(self, token: str) -> None: + """Release the pin taken by the ``on_open`` that returned ``token``. + + Unknown or already-closed tokens are a no-op: SMB/NFS rigs close + handles whose open failed, and a double close must not unpin others. + """ hydrator = self._require_hydrator() - record = hydrator.catalog.get_file_record(catalog_path) - if record is not None: - hydrator.cache.release(record.checksum_sha256) + with self._handles_lock: + checksum = self._handles.pop(token, None) + if checksum is not None: + hydrator.cache.release(checksum) def on_evict(self, catalog_path: str) -> bool: """Drop the cached copy; False when the file still has open handles.""" diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 67f118d..29d8603 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -186,12 +186,15 @@ def test_gateway_close_of_unpinned_open_does_not_unpin_other_handle(catalog, tmp gw.attach_hydrator(hyd) checksum = catalog.get_file_record("/a.bin").checksum_sha256 # type: ignore[union-attr] engine.gate.clear() - assert gw.on_open("/a.bin").value == "hydrating" # client B opens mid-hydration + handle_b = gw.on_open("/a.bin") # client B opens mid-hydration + assert handle_b.state.value == "hydrating" engine.gate.set() hyd.wait(hyd.request("/a.bin"), 5) - assert gw.on_open("/a.bin").value == "online" # client A opens the online file + assert gw.on_open("/a.bin").state.value == "online" # client A opens the online file - gw.on_close("/a.bin") # client B closes; A still holds the file open + gw.on_close(handle_b.token) # client B closes; A still holds the file open + gw.on_close(handle_b.token) # double close (rig retry) must not drop A's pin + gw.on_close("never-opened") # close of a failed open is a no-op hyd.cache.max_bytes = 1 gw.on_open("/c.bin", timeout=5) # budget pressure From 9a5d864c85146f81bc7084690c2827a7e48a6c81 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:47:25 +0000 Subject: [PATCH 119/205] perf(fuse): O(length) range reads via cache.read_range; typed CacheError on_read_range re-read and re-hashed the whole cached file per call. The gateway now preads only the requested range (integrity was verified at store time); retrieve keeps its full-file check. Bare ValueErrors on the protocol boundary become CacheError (OpenBladeError, still a ValueError). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 28 +++++++++++++++++++++-- openblade/fuse/hydration.py | 15 +++++++++++- openblade/nas/protocol_gateway.py | 3 +-- tests/integration/test_fuse_data_plane.py | 26 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 12d2652..45ef809 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -8,12 +8,18 @@ from collections import OrderedDict from pathlib import Path +from openblade.domain.errors import OpenBladeError + class CacheEntryInUseError(RuntimeError): """Raised when eviction is asked to drop a file that still has open handles.""" -class CacheChecksumError(ValueError): +class CacheError(OpenBladeError, ValueError): + """Typed cache failure for the protocol boundary (ValueError kept for old callers).""" + + +class CacheChecksumError(CacheError): """Raised when staged bytes do not match the catalog checksum.""" @@ -121,9 +127,27 @@ def retrieve(self, checksum: str) -> bytes: actual = hashlib.sha256(data).hexdigest() if actual != checksum: path.unlink() - raise ValueError(f"Cache integrity failure: expected {checksum}, got {actual}") + raise CacheChecksumError(f"Cache integrity failure: expected {checksum}, got {actual}") return data + def read_range(self, checksum: str, offset: int, length: int) -> bytes: + """``pread`` ``[offset, offset+length)`` of a cached file without re-hashing it. + + The whole file was hashed once by ``store_verified``/``store`` callers; per-range + reads must stay O(length), so integrity is not re-checked here (``retrieve`` does). + """ + if offset < 0 or length < 0: + raise CacheError(f"invalid range offset={offset} length={length}") + path = self.cache_key(checksum) + try: + fd = os.open(path, os.O_RDONLY) + except FileNotFoundError: + raise FileNotFoundError(f"Not in cache: {checksum}") from None + try: + return os.pread(fd, length, offset) + finally: + os.close(fd) + def evict(self, checksum: str) -> None: with self._lock: if self._refs.get(checksum, 0) > 0: diff --git a/openblade/fuse/hydration.py b/openblade/fuse/hydration.py index 90d68db..d2c2063 100644 --- a/openblade/fuse/hydration.py +++ b/openblade/fuse/hydration.py @@ -337,6 +337,19 @@ def request(self, catalog_path: str) -> HydrationTicket: return ticket def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: + return self._pinned_read(ticket, timeout, lambda: self.cache.retrieve(ticket.checksum)) + + def wait_range( + self, ticket: HydrationTicket, offset: int, length: int, timeout: float | None = None + ) -> bytes: + """Like ``wait`` but read only ``[offset, offset+length)``: O(length), no re-hash.""" + return self._pinned_read( + ticket, timeout, lambda: self.cache.read_range(ticket.checksum, offset, length) + ) + + def _pinned_read( + self, ticket: HydrationTicket, timeout: float | None, read: Callable[[], bytes] + ) -> bytes: # Each waiter pins the entry before the restore commits, so the budget # cannot evict it (e.g. while a later file of the same batch is stored) # before this waiter has read it. Released on every exit path. @@ -346,7 +359,7 @@ def wait(self, ticket: HydrationTicket, timeout: float | None = None) -> bytes: raise HydrationTimeoutError(f"{ticket.catalog_path} still hydrating") if ticket.error is not None: raise HydrationFailedError(ticket.error) - return self.cache.retrieve(ticket.checksum) + return read() finally: self.cache.release(ticket.checksum) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index 5ffe011..b94fd8d 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -418,8 +418,7 @@ def on_read_range( self, catalog_path: str, offset: int, length: int, *, timeout: float | None = None ) -> bytes: hydrator = self._require_hydrator() - data = hydrator.wait(hydrator.request(catalog_path), timeout) - return data[offset : offset + length] + return hydrator.wait_range(hydrator.request(catalog_path), offset, length, timeout) def on_close(self, token: str) -> None: """Release the pin taken by the ``on_open`` that returned ``token``. diff --git a/tests/integration/test_fuse_data_plane.py b/tests/integration/test_fuse_data_plane.py index 29d8603..a9943e4 100644 --- a/tests/integration/test_fuse_data_plane.py +++ b/tests/integration/test_fuse_data_plane.py @@ -369,3 +369,29 @@ def test_resume_job_without_destination_raises_typed_error(catalog, tmp_path): ) with pytest.raises(HydrationFailedError): engine.resume(str(job.id), "/a.bin") + + +def test_read_range_is_o_length_and_never_rehashes(tmp_path, monkeypatch): # type: ignore[no-untyped-def] + """Regression: range reads used to re-read and re-hash the whole file per call.""" + import openblade.fuse.cache as cache_mod + from openblade.fuse.cache import CacheError + + data = os.urandom(4 * 1024 * 1024) + checksum = hashlib.sha256(data).hexdigest() + cache = HydrationCache(str(tmp_path / "cache")) + cache.store(checksum, data) + calls: list[int] = [] + real = cache_mod.hashlib.sha256 + + def counting(*a, **k): # type: ignore[no-untyped-def] + calls.append(1) + return real(*a, **k) + + monkeypatch.setattr(cache_mod.hashlib, "sha256", counting) + chunk = 64 * 1024 + out = b"".join(cache.read_range(checksum, off, chunk) for off in range(0, len(data), chunk)) + + assert out == data + assert calls == [] + with pytest.raises(CacheError): + cache.read_range(checksum, -1, 10) From 1da44eb85451236cceb44385e26d493484055328 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:54:02 +0000 Subject: [PATCH 120/205] fix(jobs): restore journals physical_state_unknown via the shared helper One payload shape ({op, barcode, drive}) across producers, and a journal failure can no longer replace the original unmount exception. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/restore.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index d809b28..ad5a525 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -24,6 +24,7 @@ from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore +from openblade.jobs.sharded_archive import record_physical_state_unknown from openblade.jobs.verify import sha256sum from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType @@ -329,16 +330,13 @@ def _restore_tape_group( scheduler.verify(handle) # fencing: before unmount try: ltfs.unmount(mount_handle) - except Exception: + except Exception: # noqa: BLE001 - logged+journaled by record_physical_state_unknown # Still (maybe) mounted: unloading now could strand a dirty # index. Leave the cartridge in the drive for reconcile. unmount_failed = True - logger.exception("batch restore unmount failed for %s", barcode) - catalog.journal( - job_id, - "physical_state_unknown", - {"barcode": barcode, "drive_id": drive_id, "reason": "unmount_failed"}, - ) + # One payload shape ({op, barcode, drive}) for every producer; the + # helper swallows a journal failure so it cannot replace this one. + record_physical_state_unknown(catalog, job_id, "unmount", barcode, drive_id) tape_error = f"Unmount of {barcode} failed; left in drive for reconcile" except StaleLeaseError: fenced_out = True From 30c13540a2572bee7e75bfe60381e229bc295a63 Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 16:54:41 +0000 Subject: [PATCH 121/205] docs(fuse): gateway handle-token semantics Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/fuse.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fuse.md b/docs/fuse.md index 9f21a4e..2913a36 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -4,7 +4,7 @@ Five layers, top to bottom, and what is real today: | Layer | Status | |---|---| -| SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`). | +| SMB/NFS servers | **Not shipped in the image.** See `deploy/nas-protocols` (protocol rig). They hydrate through `ProtocolGateway.on_open/on_read_range/on_close/on_evict` (`openblade/nas/protocol_gateway.py`); `on_open` returns an opaque handle token and `on_close(token)` releases only that handle's cache pin (double or unknown closes are no-ops), so one client cannot unpin another's file. | | Namespace/metadata | **Real.** `CatalogFuseOperations` (`openblade/fuse/mount.py`): offline files report their real size; xattr `user.openblade.state` = `offline`/`hydrating`/`online`. | | Hydration/cache | **Real.** `Hydrator` (`openblade/fuse/hydration.py`): one shared ticket per path, `OPENBLADE_FUSE_HYDRATE_TIMEOUT` (default 300 s) blocking wait, or `EAGAIN` with `blocking=False`; failure = `EIO`, file stays `offline`, reason in `last_error()`. `HydrationCache` (`cache.py`): optional byte budget with LRU eviction that never evicts a file with open handles; restores land under a temp name and are renamed only after the sha256 matches. | | Archive/restore engine | **Real, via jobs.** `JobRestoreEngine` calls `RestoreService.enqueue_batch` — **one restore job per tape batch** (`run_restore_batch`: one lease, one load, one read-only mount, one unmount, one unload per source barcode; per-file checksum verification with a temp name renamed only on a match; a failed unmount journals `physical_state_unknown` and skips the unload). Never a tape backend directly. A restarted hydrator re-attaches to a pending/running single-file restore job for the path (batch jobs are not yet re-attachable). Every `restore_batch`/`resume` call runs on a Hydrator worker thread over its **own** catalog Session (`catalog_factory`), never the caller's. | From c937589a3f04b173ec50effcabbbcd69bbb4679a Mon Sep 17 00:00:00 2001 From: Amantux Date: Fri, 9 Oct 2026 17:17:21 +0000 Subject: [PATCH 122/205] fix(protocols): rig survives malformed stubs and resets offline stubs on reuse The hydrator shim is the client container's main process; a stub without a mode killed it, which turned every later SMB scenario into a skip. The session fixture now restages every offline stub in place (new inode, same path) when it reuses a running rig, so a previous run's hydrated files and recall log cannot leak into the next. 31/2 skipped, twice in a row. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/hydrator_shim.py | 13 +++++++++--- tests/e2e/protocols/conftest.py | 6 ++++++ tests/e2e/protocols/rig.py | 29 +++++++++++++++++++++++++++ tests/e2e/protocols/scenarios.py | 2 +- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/deploy/nas-protocols/hydrator_shim.py b/deploy/nas-protocols/hydrator_shim.py index 90c9fbf..d51a405 100644 --- a/deploy/nas-protocols/hydrator_shim.py +++ b/deploy/nas-protocols/hydrator_shim.py @@ -37,8 +37,9 @@ def _hydrate(root: Path, rel: str, mode: str, inflight: set[str]) -> None: target = root / rel if mode == "tape_error": (root / ".ob-control" / f"{rel}.err").write_text("OFFLINE_TAPE_UNAVAILABLE") - elif mode.startswith("delay:"): - time.sleep(float(mode.split(":", 1)[1])) + elif mode.startswith("delay:") or mode == "immediate": + if mode.startswith("delay:"): + time.sleep(float(mode.split(":", 1)[1])) tmp = target.with_name(f".{target.name}.hydrating") tmp.write_bytes((root / ".ob-tape" / rel).read_bytes()) os.replace(tmp, target) @@ -58,7 +59,13 @@ def main(root: Path) -> None: data = (root / rel).read_bytes() if (root / rel).is_file() else b"" if rel in inflight or not data.startswith(STUB_MAGIC): continue - mode = json.loads(data[len(STUB_MAGIC) :])["mode"] + # The shim is the client container's main process: one malformed or + # partially written stub must never take the whole rig down. + try: + body = json.loads(data[len(STUB_MAGIC) :] or b"{}") + mode = str(body.get("mode", "immediate")) if isinstance(body, dict) else "immediate" + except ValueError: + mode = "immediate" inflight.add(rel) with (control / "hydration.log").open("a") as log: log.write(f"recall {rel} mode={mode}\n") diff --git a/tests/e2e/protocols/conftest.py b/tests/e2e/protocols/conftest.py index 6f2592e..d603f7a 100644 --- a/tests/e2e/protocols/conftest.py +++ b/tests/e2e/protocols/conftest.py @@ -74,6 +74,12 @@ def protocol_rig() -> Iterator[Rig]: pytest.skip( f"protocol rig failed to start (docker compose up): {up.stderr.strip()[-300:]}" ) + # Always reset the offline stubs in place: a reused rig still carries the + # previous session's hydrated files and recall log. + restaged = handle.compose( + "exec", "-T", "client", "python3", "-c", rig.RESTAGE, *rig.restage_snippet_args() + ) + assert "restaged" in restaged.stdout, restaged.stderr yield handle diff --git a/tests/e2e/protocols/rig.py b/tests/e2e/protocols/rig.py index 5a31309..81b2ec4 100644 --- a/tests/e2e/protocols/rig.py +++ b/tests/e2e/protocols/rig.py @@ -51,6 +51,35 @@ ] +RESTAGE = """ +import json, os, sys, glob +root = '/rig/export/rw' +offline = json.loads(sys.argv[1]); magic = sys.argv[2].encode() +ctl = os.path.join(root, '.ob-control') +for proto in ('smb', 'nfs'): + for scenario, mode in offline.items(): + rel = f'offline/{proto}-{scenario}.bin' + tmp = os.path.join(root, rel + '.restage') + with open(tmp, 'wb') as f: + f.write(magic + json.dumps({'state': 'offline_on_tape', 'mode': mode}).encode()) + os.chmod(tmp, 0o666) + os.replace(tmp, os.path.join(root, rel)) +for stale in glob.glob(os.path.join(ctl, 'offline', '*.req')) + glob.glob(os.path.join(ctl, 'offline', '*.err')): + os.unlink(stale) +open(os.path.join(ctl, 'hydration.log'), 'w').close() +print('restaged') +""" + + +def restage_snippet_args() -> list[str]: + """argv for RESTAGE: reset every offline stub in place (new inode, same path). + + Used when a running rig is reused so an earlier run's hydrated files, stale + recall requests and log lines cannot leak into the next session. + """ + return [json.dumps(OFFLINE), STUB_MAGIC.decode()] + + def free_port() -> int: with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) diff --git a/tests/e2e/protocols/scenarios.py b/tests/e2e/protocols/scenarios.py index f64b091..788bedf 100644 --- a/tests/e2e/protocols/scenarios.py +++ b/tests/e2e/protocols/scenarios.py @@ -129,7 +129,7 @@ def test_eviction_while_open_keeps_old_bytes_or_estale(mounted: Rig, proto: str) import os, time, errno f = open('/mnt/{proto}/rw/{rel}', 'rb'); first = f.read(4) srv = '/rig/export/rw/{rel}' # server-side eviction, as the real evictor would -open(srv + '.tmp', 'wb').write(b'OPENBLADE-OFFLINE-STUB\\n{{}}'); os.replace(srv + '.tmp', srv) +open(srv + '.tmp', 'wb').write(b'OPENBLADE-OFFLINE-STUB\\n{{"state": "offline_on_tape", "mode": "immediate"}}'); os.replace(srv + '.tmp', srv) time.sleep(1) try: f.seek(0); print('OLD' if f.read(4) == first else 'NEW') From 159419aae3b26f535b9c9f2fb98f0f8853ff53a9 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:04:05 +0000 Subject: [PATCH 123/205] feat(reconcile): cross-job journal query and pending-reconciliation set Adds CatalogRepository.journal_events, openblade/jobs/reconcile.py with drives_pending_reconciliation, and DriveUnreconciledError. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 10 ++++++ openblade/domain/errors.py | 4 +++ openblade/jobs/reconcile.py | 62 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 openblade/jobs/reconcile.py diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 0ecdc07..8058a9a 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -2078,6 +2078,16 @@ def release_leases_for_job(self, job_id: str) -> int: session.commit() return len(rows) + # -- Drive reconciliation (cross-job journal query) ------------------------ + + def journal_events(self, event: str, *, since: datetime | None = None) -> list[JobJournalEntry]: + """Every ``event`` journal entry across all jobs, oldest first (ties by id).""" + stmt = select(JobJournalEntry).where(JobJournalEntry.event == event) + if since is not None: + stmt = stmt.where(JobJournalEntry.at >= since) + stmt = stmt.order_by(JobJournalEntry.at, JobJournalEntry.id) + return list(self.session.scalars(stmt)) + def _lease_now() -> datetime: """Naive UTC, matching how the catalog's DateTime columns store values.""" diff --git a/openblade/domain/errors.py b/openblade/domain/errors.py index cbb8ed1..c528615 100644 --- a/openblade/domain/errors.py +++ b/openblade/domain/errors.py @@ -50,6 +50,10 @@ class DriveLeasedError(SafetyViolationError): """Another job holds a live lease on this drive; it may have LTFS mounted.""" +class DriveUnreconciledError(SafetyViolationError): + """A drive's physical state is unknown after a failed unmount/unload; reconcile first.""" + + class DriveBusyError(OpenBladeError): """Drive is busy.""" diff --git a/openblade/jobs/reconcile.py b/openblade/jobs/reconcile.py new file mode 100644 index 0000000..f29b9f1 --- /dev/null +++ b/openblade/jobs/reconcile.py @@ -0,0 +1,62 @@ +"""Drives whose physical state is unknown after a failed unmount/unload. + +``record_physical_state_unknown`` journals ``physical_state_unknown`` when an +unmount/unload fails: the drive may still hold a (possibly LTFS-mounted) tape. +Until a later ``drive_reconciled`` entry names the same drive, no job may +acquire or load into it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from openblade.catalog.repository import CatalogRepository + +PHYSICAL_STATE_UNKNOWN = "physical_state_unknown" +DRIVE_RECONCILED = "drive_reconciled" + + +@dataclass(frozen=True) +class PendingReconciliation: + drive_id: int + barcode: str | None + op: str + job_id: str + at: datetime + + +def _drive_of(detail: dict[str, object]) -> int | None: + drive = detail.get("drive") + if drive is None or isinstance(drive, bool): + return None + try: + return int(str(drive)) + except ValueError: + return None + + +def drives_pending_reconciliation(catalog: CatalogRepository) -> dict[int, PendingReconciliation]: + """Drives whose latest ``physical_state_unknown`` has no later ``drive_reconciled``.""" + events = catalog.journal_events(PHYSICAL_STATE_UNKNOWN) + catalog.journal_events( + DRIVE_RECONCILED + ) + events.sort(key=lambda entry: (entry.at, entry.id)) + pending: dict[int, PendingReconciliation] = {} + for entry in events: + detail = entry.detail + drive = _drive_of(detail) + if drive is None: + continue + if entry.event == DRIVE_RECONCILED: + pending.pop(drive, None) + continue + barcode = detail.get("barcode") + pending[drive] = PendingReconciliation( + drive_id=drive, + barcode=None if barcode is None else str(barcode), + op=str(detail.get("op", "")), + job_id=entry.job_id, + at=entry.at, + ) + return pending From fcdd6ef8a245bbeaa070e8251a3b2e613e1b5306 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:04:21 +0000 Subject: [PATCH 124/205] fuse cache: per-entry integrity sidecar + open-time verify_entry store/store_verified write .meta.json (size, mtime_ns, one sampled 64 KiB block hash). verify_entry re-checks it and evicts + raises CacheIntegrityError on mismatch; mount.open and gateway.on_open call it when the entry is cached and fall through to re-hydration. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 86 +++++++++++++++++++++++++++++++ openblade/fuse/mount.py | 12 +++++ openblade/nas/protocol_gateway.py | 15 +++++- 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 45ef809..b08fb20 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -3,6 +3,8 @@ from __future__ import annotations import hashlib +import json +import logging import os import threading from collections import OrderedDict @@ -10,6 +12,10 @@ from openblade.domain.errors import OpenBladeError +logger = logging.getLogger(__name__) + +SAMPLE_BYTES = 64 * 1024 + class CacheEntryInUseError(RuntimeError): """Raised when eviction is asked to drop a file that still has open handles.""" @@ -23,6 +29,17 @@ class CacheChecksumError(CacheError): """Raised when staged bytes do not match the catalog checksum.""" +class CacheIntegrityError(CacheError): + """Raised when a cached entry no longer matches its integrity record (it is evicted).""" + + +def _sample_offset(checksum: str, size: int) -> int: + """Deterministic sample offset derived from the checksum; 0 when the file fits in one block.""" + if size <= SAMPLE_BYTES: + return 0 + return int(checksum[:8], 16) % max(1, size - SAMPLE_BYTES) + + class HydrationCache: """Checksum-addressed staging cache. @@ -101,9 +118,76 @@ def store_verified(self, checksum: str, staged: Path) -> Path: os.replace(tmp, final) finally: tmp.unlink(missing_ok=True) + self._write_meta(checksum) self._track(checksum, final.stat().st_size) return final + # -- integrity records --------------------------------------------------- + def meta_key(self, checksum: str) -> Path: + path = self.cache_key(checksum) + return path.with_name(f"{path.name}.meta.json") + + def _sample_digest(self, path: Path, offset: int, length: int) -> str: + with path.open("rb") as handle: + handle.seek(offset) + return hashlib.sha256(handle.read(length)).hexdigest() + + def _write_meta(self, checksum: str) -> None: + """Record size, mtime_ns and a sampled block hash beside the cached file.""" + path = self.cache_key(checksum) + st = path.stat() + offset = _sample_offset(checksum, st.st_size) + length = min(SAMPLE_BYTES, st.st_size) + record = { + "size": st.st_size, + "mtime_ns": st.st_mtime_ns, + "sample": { + "offset": offset, + "len": length, + "sha256": self._sample_digest(path, offset, length), + }, + } + meta = self.meta_key(checksum) + tmp = meta.with_name(f".{meta.name}.{os.getpid()}.{threading.get_ident()}.part") + try: + tmp.write_text(json.dumps(record), encoding="utf-8") + os.replace(tmp, meta) + finally: + tmp.unlink(missing_ok=True) + + def _entry_matches(self, checksum: str) -> bool: + path = self.cache_key(checksum) + try: + record = json.loads(self.meta_key(checksum).read_text(encoding="utf-8")) + st = path.stat() + sample = record["sample"] + if st.st_size != record["size"] or st.st_mtime_ns != record["mtime_ns"]: + return False + offset, length = int(sample["offset"]), int(sample["len"]) + if (offset, length) != ( + _sample_offset(checksum, st.st_size), + min(SAMPLE_BYTES, st.st_size), + ): + return False + return bool(self._sample_digest(path, offset, length) == sample["sha256"]) + except (OSError, ValueError, KeyError, TypeError): + return False + + def verify_entry(self, checksum: str) -> bool: + """Cheap open-time check: size + mtime_ns + one sampled 64 KiB block. + + On mismatch or a missing/unreadable integrity record the entry (file and + sidecar) is dropped and ``CacheIntegrityError`` raised, so the caller can + fall through to re-hydration. + """ + with self._lock: + if self._entry_matches(checksum): + return True + self._lru.pop(checksum, None) + self.cache_key(checksum).unlink(missing_ok=True) + self.meta_key(checksum).unlink(missing_ok=True) + raise CacheIntegrityError("cached entry failed integrity verification and was evicted") + def cache_key(self, checksum: str) -> Path: return self.cache_dir / checksum[:2] / checksum @@ -115,6 +199,7 @@ def store(self, checksum: str, data: bytes) -> Path: path.parent.mkdir(parents=True, exist_ok=True) with path.open("wb") as handle: handle.write(data) + self._write_meta(checksum) self._track(checksum, len(data)) return path @@ -156,3 +241,4 @@ def evict(self, checksum: str) -> None: path = self.cache_key(checksum) if path.exists(): path.unlink() + self.meta_key(checksum).unlink(missing_ok=True) diff --git a/openblade/fuse/mount.py b/openblade/fuse/mount.py index 8f106ac..de1d586 100644 --- a/openblade/fuse/mount.py +++ b/openblade/fuse/mount.py @@ -34,6 +34,7 @@ from pathlib import PurePosixPath from typing import Any +from openblade.fuse.cache import CacheIntegrityError, HydrationCache from openblade.fuse.filesystem import CatalogFilesystem, VirtualDirEntry from openblade.fuse.hydration import ( HydrationFailedError, @@ -249,6 +250,7 @@ def open(self, path: str, flags: int) -> int: # Pin before materialising so the entry cannot be evicted in between. record = self.filesystem.catalog.get_file_record(str(entry.path)) if record is not None: + _verify_cached_entry(self.filesystem.cache, record.checksum_sha256) self.filesystem.cache.acquire(record.checksum_sha256) try: data = self._materialise(str(entry.path)) @@ -376,3 +378,13 @@ def mount_catalog( fsname="openblade-catalog", subtype="openblade", ) + + +def _verify_cached_entry(cache: HydrationCache, checksum: str) -> None: + """Open-time integrity check; a failed entry is evicted so hydration refetches it.""" + if not cache.is_cached(checksum): + return + try: + cache.verify_entry(checksum) + except CacheIntegrityError: + logger.warning("fuse: cached entry %s failed integrity check; re-hydrating", checksum) diff --git a/openblade/nas/protocol_gateway.py b/openblade/nas/protocol_gateway.py index b94fd8d..c8bce57 100644 --- a/openblade/nas/protocol_gateway.py +++ b/openblade/nas/protocol_gateway.py @@ -10,6 +10,7 @@ import hashlib import hmac import importlib.util +import logging import os import posixpath import secrets @@ -21,9 +22,11 @@ from typing import Any from openblade.domain.clock import naive_utcnow -from openblade.fuse.cache import CacheEntryInUseError +from openblade.fuse.cache import CacheEntryInUseError, CacheIntegrityError from openblade.fuse.hydration import HydrationFailedError, HydrationState, Hydrator +logger = logging.getLogger(__name__) + class GatewayStatus(str, Enum): RUNNING = "running" @@ -400,6 +403,16 @@ def on_open(self, catalog_path: str, *, timeout: float | None = 0.0) -> GatewayH that pin, so a double close cannot drop another handle's pin. """ hydrator = self._require_hydrator() + record = hydrator.catalog.get_file_record(catalog_path) + if record is not None and hydrator.cache.is_cached(record.checksum_sha256): + # Open-time integrity check; a failed entry is evicted so request() re-hydrates. + try: + hydrator.cache.verify_entry(record.checksum_sha256) + except CacheIntegrityError: + logger.warning( + "gateway: cached entry %s failed integrity check; re-hydrating", + record.checksum_sha256, + ) ticket = hydrator.request(catalog_path) hydrator.cache.acquire(ticket.checksum) try: From b19fd7cbba170f51b6e1c4c08503a88cee4462c8 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:05:52 +0000 Subject: [PATCH 125/205] feat(reconcile): refuse unreconciled drives in scheduler and load helpers DriveScheduler excludes drives awaiting reconciliation from its candidate set and raises DriveUnreconciledError only when too few remain. The restore, sharded-archive and sharded-restore load helpers refuse a pending drive. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/reconcile.py | 11 +++++ openblade/jobs/restore.py | 3 ++ openblade/jobs/scheduler.py | 67 ++++++++++++++++++++++++++++--- openblade/jobs/sharded_archive.py | 3 ++ openblade/jobs/sharded_restore.py | 3 ++ 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/openblade/jobs/reconcile.py b/openblade/jobs/reconcile.py index f29b9f1..d4c1ddf 100644 --- a/openblade/jobs/reconcile.py +++ b/openblade/jobs/reconcile.py @@ -12,6 +12,7 @@ from datetime import datetime from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import DriveUnreconciledError PHYSICAL_STATE_UNKNOWN = "physical_state_unknown" DRIVE_RECONCILED = "drive_reconciled" @@ -60,3 +61,13 @@ def drives_pending_reconciliation(catalog: CatalogRepository) -> dict[int, Pendi at=entry.at, ) return pending + + +def ensure_drive_reconciled(catalog: CatalogRepository, drive_id: int) -> None: + """Refuse to use ``drive_id`` while its physical state is unknown.""" + item = drives_pending_reconciliation(catalog).get(drive_id) + if item is not None: + raise DriveUnreconciledError( + f"Drive {drive_id} (barcode {item.barcode or 'unknown'}) awaits reconciliation " + f"after a failed {item.op or 'operation'}; reconcile it before reuse" + ) diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index ad5a525..dd5190c 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -23,6 +23,7 @@ from openblade.domain.models import JobType, MountHandle, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.queue import JobQueue +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import CatalogLeaseStore, DriveHandle, DriveScheduler, LeaseStore from openblade.jobs.sharded_archive import record_physical_state_unknown from openblade.jobs.verify import sha256sum @@ -109,6 +110,7 @@ def _load_if_needed( barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: + ensure_drive_reconciled(catalog, drive_id) # Already in a drive: record it as the PHYSICAL drive; the lease key # (handle.drive_id) stays what the scheduler reserved. if drive_id != handle.drive_id: @@ -118,6 +120,7 @@ def _load_if_needed( if slot_id is None: raise CartridgeOfflineError(f"Cartridge {barcode} is offline") drive_id = handle.drive_id + ensure_drive_reconciled(catalog, drive_id) execute_tape_request( catalog, library, diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 4664703..319bb9e 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Protocol from uuid import uuid4 -from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.errors import DriveBusyError, DriveUnreconciledError, StaleLeaseError from openblade.domain.models import DriveLease if TYPE_CHECKING: @@ -49,9 +49,19 @@ class LeaseStore(Protocol): poll_interval: float def acquire( - self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: ... + def unreconciled_drives(self) -> dict[int, str | None]: + """Drives awaiting reconciliation (physical state unknown) -> last barcode.""" + ... + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: ... def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: ... @@ -69,17 +79,25 @@ class InMemoryLeaseStore: poll_interval = 1.0 def __init__(self) -> None: + # Drive -> barcode for drives whose physical state is unknown (tests set this). + self.unreconciled: dict[int, str | None] = {} self._lock = threading.Lock() self._leases: dict[str, DriveLease] = {} self._last_token = 0 def acquire( - self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: with self._lock: now = _now() busy = {lease.drive_id for lease in self._leases.values() if _live(lease, now)} - free = [d for d in range(num_drives) if d not in busy] + free = [d for d in range(num_drives) if d not in busy and d not in exclude] if len(free) < len(barcodes): return None acquired: list[DriveLease] = [] @@ -100,6 +118,9 @@ def acquire( acquired.append(lease) return acquired + def unreconciled_drives(self) -> dict[int, str | None]: + return dict(self.unreconciled) + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: with self._lock: now = _now() @@ -144,11 +165,31 @@ def __init__(self, repository: CatalogRepository) -> None: self._repo = repository def acquire( - self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: - return self._repo.acquire_drive_leases( + leases = self._repo.acquire_drive_leases( job_id=job_id, barcodes=barcodes, num_drives=num_drives, ttl=ttl ) + if leases is not None and any(lease.drive_id in exclude for lease in leases): + # The repository cannot yet skip drives itself: never hand out an + # unreconciled drive -- give the leases back and report "not now". + self._repo.release_leases([lease.id for lease in leases]) + return None + return leases + + def unreconciled_drives(self) -> dict[int, str | None]: + from openblade.jobs.reconcile import drives_pending_reconciliation # local: no cycle + + return { + drive_id: item.barcode + for drive_id, item in drives_pending_reconciliation(self._repo).items() + } def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: self._repo.heartbeat_leases(lease_ids, ttl) @@ -230,11 +271,25 @@ def acquire_drives( with self._lock: deadline = _monotonic() + timeout while True: + # A drive whose physical state is unknown (failed unmount/unload) + # may still hold a mounted tape: never a candidate until reconciled. + unreconciled = self._store.unreconciled_drives() + excluded = frozenset(d for d in unreconciled if 0 <= d < self._num_drives) + if self._num_drives - len(excluded) < len(barcodes): + named = ", ".join( + f"drive {d} (barcode {unreconciled[d] or 'unknown'})" + for d in sorted(excluded) + ) + raise DriveUnreconciledError( + f"Not enough reconciled drives for {len(barcodes)} tape(s): {named} " + "awaiting reconciliation after a failed unmount/unload" + ) leases = self._store.acquire( job_id=self._job_id, barcodes=barcodes, num_drives=self._num_drives, ttl=self._ttl, + exclude=excluded, ) if leases is not None: break diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 220805d..588375c 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -17,6 +17,7 @@ from openblade.domain.errors import StaleLeaseError, TapeFullError, safe_job_error from openblade.domain.models import MountHandle, MountMode from openblade.jobs.inventory import InventoryService +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import ( DEFAULT_BLOCK_SIZE, @@ -761,6 +762,7 @@ def _load_barcode( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: + ensure_drive_reconciled(catalog, loaded_drive_id) # Cartridge is already in a drive. Record it as the PHYSICAL drive without # touching the scheduler lock key (handle.drive_id) — mutating that key would # leak the reserved drive and free a drive the scheduler never held. @@ -779,6 +781,7 @@ def _load_barcode( ) if slot_id is None: raise ValueError(f"Barcode {handle.barcode} not found in any slot") + ensure_drive_reconciled(catalog, handle.drive_id) execute_tape_request( catalog, library, diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index 009e907..b9e7c0d 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -14,6 +14,7 @@ from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.errors import CartridgeOfflineError, ChecksumMismatchError, FileNotFoundError from openblade.domain.models import MountMode +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import DEFAULT_BLOCK_SIZE, compute_checksum, reassemble_block_stripe from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, record_physical_state_unknown @@ -339,6 +340,7 @@ def _ensure_loaded( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: + ensure_drive_reconciled(catalog, loaded_drive_id) # Record the physical drive without mutating the scheduler lock key # (handle.drive_id); mutating it leaks the reserved drive on release. if loaded_drive_id != handle.drive_id: @@ -348,6 +350,7 @@ def _ensure_loaded( slot_id = library.find_slot_by_barcode(handle.barcode) if slot_id is None: raise CartridgeOfflineError(f"Barcode {handle.barcode} not found in library") + ensure_drive_reconciled(catalog, handle.drive_id) execute_tape_request( catalog, library, From 9bab58f9efc4bdf2e5825ac07a717d39f73f7fee Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:06:05 +0000 Subject: [PATCH 126/205] test(protocols): real SMB byte-range lock conflict across two sessions smbprotocol 1.15.0 ships no SMB2 LOCK request (SMB2LockElement/Open.lock do not exist), so the old snippet always hit ImportError and skipped. Build the [MS-SMB2] 2.2.26 request directly, use two guest sessions (no signing), and assert STATUS_LOCK_NOT_GRANTED for the second session until the first unlocks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/test_smb.py | 80 ++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/tests/e2e/protocols/test_smb.py b/tests/e2e/protocols/test_smb.py index bd5f1ce..ebd84d3 100644 --- a/tests/e2e/protocols/test_smb.py +++ b/tests/e2e/protocols/test_smb.py @@ -37,24 +37,64 @@ def test_smb_acl_maps_posix_mode(mounted: Rig) -> None: assert "ACL:Everyone:ALLOWED/0x0/" in lines, res.stdout -def test_smb_byte_range_lock_conflicts_across_sessions(mounted: Rig) -> None: - code = """ -import smbclient, smbclient._io as io -from smbprotocol.exceptions import SMBOSError -from smbprotocol.open import SMB2LockElement, LockFlags -smbclient.ClientConfig(username='guest', password='', auth_protocol='ntlm') -p = r'\\\\samba\\rw\\lock.bin' -with smbclient.open_file(p, 'wb') as f: f.write(b'0' * 100) -a = smbclient.open_file(p, 'rb', share_access='rw'); b = smbclient.open_file(p, 'rb', share_access='rw') -el = SMB2LockElement(); el['offset'] = 0; el['length'] = 10 -el['flags'] = LockFlags.SMB2_LOCKFLAG_EXCLUSIVE_LOCK | LockFlags.SMB2_LOCKFLAG_FAIL_IMMEDIATELY -a.fd.lock([el]) -try: - b.fd.lock([el]); print('GRANTED') -except Exception as e: - print(type(e).__name__) +# smbprotocol 1.15.0 has the SMB2_LOCK command code but no LOCK request/Open.lock(), so the +# snippet builds the [MS-SMB2] 2.2.26 request itself. Guest sessions cannot sign, hence +# require_signing=False and require_secure_negotiate=False. +_LOCK_SNIPPET = r""" +import struct +from collections import OrderedDict +import smbclient +from smbprotocol.exceptions import SMBResponseException +from smbprotocol.header import Commands +from smbprotocol.structure import BytesField, IntField, Structure + +class LockReq(Structure): # [MS-SMB2] 2.2.26; smbprotocol 1.15.0 has no LOCK request + COMMAND = Commands.SMB2_LOCK + def __init__(self): + self.fields = OrderedDict([ + ("structure_size", IntField(size=2, default=48)), + ("lock_count", IntField(size=2, default=1)), + ("lock_sequence", IntField(size=4)), + ("file_id", BytesField(size=16)), + ("locks", BytesField(size=24)), + ]) + super().__init__() + +def lock(f, flags): + o = getattr(f, "raw", f).fd + req = LockReq(); req["file_id"] = o.file_id + req["locks"] = struct.pack(" dict: + # Guest sessions cannot sign; smbclient requires signing by default. + cache: dict = {} + smbclient.register_session("samba", username="guest", password="", auth_protocol="ntlm", require_signing=False, connection_cache=cache) + return cache + +smbclient.ClientConfig(require_secure_negotiate=False) # needs a signing key guests lack +ca, cb = session(), session() +with smbclient.open_file(p, "wb", connection_cache=ca) as f: + f.write(b"0" * 100) +a = smbclient.open_file(p, "rb", share_access="rw", connection_cache=ca) +b = smbclient.open_file(p, "rb", share_access="rw", connection_cache=cb) +assert getattr(a, "raw", a).fd.tree_connect.session.session_id != getattr(b, "raw", b).fd.tree_connect.session.session_id +print(lock(a, EXCL_NOW), lock(b, EXCL_NOW), lock(a, UNLOCK), lock(b, EXCL_NOW), lock(b, UNLOCK)) """ - res = mounted.py(code) - if "Error" in res.stderr and not res.stdout: - pytest.skip(f"smbprotocol lock driver unavailable: {res.stderr.strip()[-200:]}") - assert res.stdout.strip() not in {"", "GRANTED"}, res.stdout + res.stderr + + +def test_smb_byte_range_lock_conflicts_across_sessions(mounted: Rig) -> None: + """A conflicting exclusive lock from a second SMB session is refused until released.""" + res = mounted.py(_LOCK_SNIPPET) + assert res.returncode == 0, res.stderr[-800:] + # a locks, b conflicts (STATUS_LOCK_NOT_GRANTED), a unlocks, b now locks, b unlocks. + assert res.stdout.split() == ["GRANTED", "0xc0000055", "GRANTED", "GRANTED", "GRANTED"], ( + res.stdout + ) From 3919499a2d6ab439ee6ba280d6934fe35516bdc9 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:06:55 +0000 Subject: [PATCH 127/205] test(mutation): add validator/import-guard/safety/policy tests to mutmut selection Each file runs in <3 s alone; full selection: 135 passed, 31 s wall. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5b746bb..1e1907f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,4 +164,9 @@ pytest_add_cli_args_test_selection = [ "tests/unit/test_drive_leases.py", "tests/unit/test_recovery.py", "tests/unit/test_scalar_coordinate.py", + "tests/unit/test_manifest_validator.py", + "tests/unit/test_import_guard.py", + "tests/unit/test_safety_gates.py", + "tests/unit/test_hardware_safety.py", + "tests/unit/test_capacity_policy.py", ] From bd279810c4bbbb3dcb286f693801af87034210dd Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:06:55 +0000 Subject: [PATCH 128/205] feat(reconcile): reconcile_drive action, recovery report field, API route reconcile_drive refuses while the drive is still mounted and journals drive_reconciled otherwise. RecoveryReport lists pending drives (report only). POST /jobs/recovery/reconcile/{drive_id}: 200 / 409 / 404. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_jobs.py | 45 ++++++++++++++++++++++++++++++++++++ openblade/jobs/reconcile.py | 38 ++++++++++++++++++++++++++++++ openblade/jobs/recovery.py | 9 +++++++- 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/openblade/api/routes_jobs.py b/openblade/api/routes_jobs.py index 15d14f9..3e14095 100644 --- a/openblade/api/routes_jobs.py +++ b/openblade/api/routes_jobs.py @@ -2,10 +2,14 @@ from __future__ import annotations +from datetime import datetime + from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from openblade.bootstrap import AppContext, get_context +from openblade.domain.errors import DriveUnreconciledError +from openblade.jobs.reconcile import UnknownDriveError, reconcile_drive router = APIRouter() @@ -61,12 +65,28 @@ class StagedInstanceResponse(BaseModel): state: str +class PendingReconciliationResponse(BaseModel): + drive_id: int + barcode: str | None + op: str + job_id: str + at: datetime + + class RecoveryReportResponse(BaseModel): interrupted_job_ids: list[str] released_lease_ids: list[str] mismatches: list[DriveMismatchResponse] staged_instances: dict[str, list[StagedInstanceResponse]] stale_pending_job_ids: list[str] + pending_reconciliation: list[PendingReconciliationResponse] + + +class ReconcileDriveResponse(BaseModel): + drive: int + observed_barcode: str | None + job_id: str + at: datetime # Declared before /{job_id} so the literal path wins route matching. @@ -104,6 +124,31 @@ async def get_recovery_report( for job_id, instances in report.staged_instances.items() }, stale_pending_job_ids=list(report.stale_pending_job_ids), + pending_reconciliation=[ + PendingReconciliationResponse( + drive_id=p.drive_id, barcode=p.barcode, op=p.op, job_id=p.job_id, at=p.at + ) + for p in report.pending_reconciliation + ], + ) + + +@router.post("/recovery/reconcile/{drive_id}", response_model=ReconcileDriveResponse) +async def post_reconcile_drive( + drive_id: int, context: AppContext = Depends(get_context) +) -> ReconcileDriveResponse: + """Clear a drive's "physical state unknown" mark after checking it is unmounted. + + Bearer-token gated like every native route (api_auth middleware).""" + try: + done = reconcile_drive(context.catalog, context.library, drive_id) + except UnknownDriveError: + raise HTTPException(status_code=404, detail=f"Drive {drive_id} not found") from None + except DriveUnreconciledError as exc: + # Curated message from reconcile_drive (drive id, barcode, mount state). + raise HTTPException(status_code=409, detail=str(exc)) from None + return ReconcileDriveResponse( + drive=done.drive_id, observed_barcode=done.barcode, job_id=done.job_id, at=done.at ) diff --git a/openblade/jobs/reconcile.py b/openblade/jobs/reconcile.py index d4c1ddf..c4252b9 100644 --- a/openblade/jobs/reconcile.py +++ b/openblade/jobs/reconcile.py @@ -12,7 +12,10 @@ from datetime import datetime from openblade.catalog.repository import CatalogRepository +from openblade.domain.backends import LibraryBackend from openblade.domain.errors import DriveUnreconciledError +from openblade.domain.models import MountState +from openblade.jobs.inventory import InventoryService PHYSICAL_STATE_UNKNOWN = "physical_state_unknown" DRIVE_RECONCILED = "drive_reconciled" @@ -71,3 +74,38 @@ def ensure_drive_reconciled(catalog: CatalogRepository, drive_id: int) -> None: f"Drive {drive_id} (barcode {item.barcode or 'unknown'}) awaits reconciliation " f"after a failed {item.op or 'operation'}; reconcile it before reuse" ) + + +class UnknownDriveError(LookupError): + """The library reports no drive with this id.""" + + +def reconcile_drive( + catalog: CatalogRepository, + library: LibraryBackend, + drive_id: int, + *, + job_id: str = "recovery", +) -> PendingReconciliation: + """Mark ``drive_id`` reconciled against a fresh inventory snapshot. + + Refuses while the drive still reports an LTFS mount: the operator must + unmount first. Returns the ``drive_reconciled`` record (``op`` is + ``drive_reconciled``; ``barcode`` is the barcode observed in the drive). + """ + snapshot = InventoryService(library).snapshot() # SAFETY_003: via service + drive = next((d for d in snapshot.drives if d.drive_id == drive_id), None) + if drive is None: + raise UnknownDriveError(f"Drive {drive_id} not found") + observed = None if drive.barcode is None else str(drive.barcode) + if drive.mount_state != MountState.UNMOUNTED: + raise DriveUnreconciledError( + f"Drive {drive_id} (barcode {observed or 'unknown'}) is still " + f"{drive.mount_state.value}; unmount it before reconciling" + ) + entry = catalog.journal( + job_id, DRIVE_RECONCILED, {"drive": drive_id, "observed_barcode": observed} + ) + return PendingReconciliation( + drive_id=drive_id, barcode=observed, op=DRIVE_RECONCILED, job_id=job_id, at=entry.at + ) diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index 7ee016f..8068b2b 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -12,6 +12,7 @@ from openblade.catalog.repository import CatalogRepository, StagedInstance from openblade.domain.backends import LibraryBackend from openblade.jobs.inventory import InventoryService +from openblade.jobs.reconcile import PendingReconciliation, drives_pending_reconciliation from openblade.jobs.scheduler import DEFAULT_LEASE_TTL logger = logging.getLogger(__name__) @@ -43,6 +44,9 @@ class RecoveryReport: staged_instances: dict[str, list[StagedInstance]] = field(default_factory=dict) # `pending` jobs older than one lease TTL with no live lease. Report only. stale_pending_job_ids: list[str] = field(default_factory=list) + # Drives whose physical state is unknown after a failed unmount/unload. Report + # only: an operator reconciles each via POST /jobs/recovery/reconcile/{drive_id}. + pending_reconciliation: list[PendingReconciliation] = field(default_factory=list) def _is_older_than(created_at: datetime, ttl: timedelta, now: datetime) -> bool: @@ -122,14 +126,17 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - mismatches=mismatches, staged_instances=staged, stale_pending_job_ids=stale_pending, + pending_reconciliation=list(drives_pending_reconciliation(catalog).values()), ) logger.info( "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es), " - "%d staged instance(s) awaiting reconcile, %d stale pending job(s)", + "%d staged instance(s) awaiting reconcile, %d stale pending job(s), " + "%d drive(s) awaiting reconciliation", len(report.interrupted_job_ids), len(report.released_lease_ids), len(report.mismatches), sum(len(items) for items in staged.values()), len(stale_pending), + len(report.pending_reconciliation), ) return report From f11f168884fe85241f7f00c3766428419dae5a82 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:07:09 +0000 Subject: [PATCH 129/205] test: add StatefulMtxRunner fake for the mtx changer CLI Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/fakes/__init__.py | 1 + tests/fakes/mtx.py | 137 ++++++++++++++++++++++++++++++++++++ tests/unit/test_fake_mtx.py | 90 +++++++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 tests/fakes/__init__.py create mode 100644 tests/fakes/mtx.py create mode 100644 tests/unit/test_fake_mtx.py diff --git a/tests/fakes/__init__.py b/tests/fakes/__init__.py new file mode 100644 index 0000000..5ddd376 --- /dev/null +++ b/tests/fakes/__init__.py @@ -0,0 +1 @@ +"""Stateful fakes for external tools used by the hardware backends.""" diff --git a/tests/fakes/mtx.py b/tests/fakes/mtx.py new file mode 100644 index 0000000..68c57f2 --- /dev/null +++ b/tests/fakes/mtx.py @@ -0,0 +1,137 @@ +"""A stateful ``mtx`` fake so the SCSI backend can be exercised without a changer. + +``StatefulMtxRunner`` replaces :class:`SafeRunner` and answers ``mtx -f ...`` +argv the way the real tool does: ``status`` renders the text format that +``parse_mtx_status`` consumes (byte-shaped like ``SAMPLE_MTX_THREE_DRIVES``), and +``load`` / ``unload`` / ``transfer`` mutate the element state or fail with an +mtx-like ``Request Sense`` message. Element numbering follows mtx: drives are +0-based Data Transfer Elements, storage slots are 1-based, and the import/export +slots follow the storage slots. +""" + +from __future__ import annotations + +from openblade.hardware.runner import CommandResult, SafeRunner + +_ILLEGAL_REQUEST = "mtx: Request Sense: Sense Key=Illegal Request" + + +class StatefulMtxRunner(SafeRunner): + """SafeRunner that simulates one SCSI medium changer driven by ``mtx``.""" + + def __init__( + self, + device: str, + *, + drives: int = 3, + slots: int = 50, + ie_slots: int = 2, + barcodes: dict[int, str], + ) -> None: + super().__init__(dry_run=False) + self.device = device + self.drive_count = drives + self.slot_count = slots + self.ie_count = ie_slots + total = slots + ie_slots + for slot in barcodes: + if not 1 <= slot <= total: + raise ValueError(f"slot {slot} is outside 1..{total}") + self.slots: dict[int, str | None] = {s: barcodes.get(s) for s in range(1, total + 1)} + # Per drive: (barcode, source slot) when full, None when empty. + self.drives: dict[int, tuple[str, int] | None] = dict.fromkeys(range(drives)) + self.calls: list[list[str]] = [] + + def run( + self, + args: list[str], + timeout: int | None = None, + redact_args: list[int] | None = None, + ) -> CommandResult: + self.calls.append(list(args)) + if len(args) < 4 or args[:3] != ["mtx", "-f", self.device]: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + verb, operands = args[3], args[4:] + if verb == "status" and not operands: + return self._result(args, 0, stdout=self.render_status()) + numbers = _ints(operands) + if numbers is None or len(numbers) != 2: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + if verb == "load": + error = self._load(*numbers) + elif verb == "unload": + error = self._unload(*numbers) + elif verb == "transfer": + error = self._transfer(*numbers) + else: + return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") + if error is not None: + return self._result(args, 1, stderr=f"{_ILLEGAL_REQUEST} ({error})") + return self._result(args, 0) + + def render_status(self) -> str: + lines = [ + f"Storage Changer {self.device}:{self.drive_count} Drives, " + f"{self.slot_count} Slots ( {self.ie_count} Import/Export )" + ] + for drive_id, content in self.drives.items(): + if content is None: + lines.append(f"Data Transfer Element {drive_id}:Empty") + else: + barcode, source = content + lines.append( + f"Data Transfer Element {drive_id}:Full (Storage Element {source} Loaded)" + f":VolumeTag = {barcode}" + ) + for slot_id, slot_barcode in self.slots.items(): + flag = " IMPORT/EXPORT" if slot_id > self.slot_count else "" + state = "Empty" if slot_barcode is None else f"Full :VolumeTag={slot_barcode}" + lines.append(f" Storage Element {slot_id}{flag}:{state}") + return "\n" + "\n".join(lines) + "\n" + + def _load(self, slot: int, drive: int) -> str | None: + if slot not in self.slots or drive not in self.drives: + return "invalid element address" + barcode = self.slots[slot] + if barcode is None: + return f"source element {slot} is empty" + if self.drives[drive] is not None: + return f"drive {drive} is full" + self.slots[slot] = None + self.drives[drive] = (barcode, slot) + return None + + def _unload(self, slot: int, drive: int) -> str | None: + if slot not in self.slots or drive not in self.drives: + return "invalid element address" + content = self.drives[drive] + if content is None: + return f"drive {drive} is empty" + if self.slots[slot] is not None: + return f"destination element {slot} is full" + self.slots[slot] = content[0] + self.drives[drive] = None + return None + + def _transfer(self, source: int, target: int) -> str | None: + if source not in self.slots or target not in self.slots: + return "invalid element address" + if self.slots[source] is None: + return f"source element {source} is empty" + if self.slots[target] is not None: + return f"destination element {target} is full" + self.slots[target], self.slots[source] = self.slots[source], None + return None + + @staticmethod + def _result(args: list[str], code: int, *, stdout: str = "", stderr: str = "") -> CommandResult: + return CommandResult( + args=list(args), returncode=code, stdout=stdout, stderr=stderr, elapsed_seconds=0.0 + ) + + +def _ints(values: list[str]) -> list[int] | None: + try: + return [int(value) for value in values] + except ValueError: + return None diff --git a/tests/unit/test_fake_mtx.py b/tests/unit/test_fake_mtx.py new file mode 100644 index 0000000..36cf9df --- /dev/null +++ b/tests/unit/test_fake_mtx.py @@ -0,0 +1,90 @@ +"""The stateful mtx fake must speak the format ``parse_mtx_status`` consumes.""" + +from __future__ import annotations + +from openblade.hardware.mtx import parse_mtx_status +from tests.fakes.mtx import StatefulMtxRunner + +DEV = "/dev/sg3" + + +def _runner() -> StatefulMtxRunner: + return StatefulMtxRunner(DEV, barcodes={1: "VOL001L8", 2: "VOL002L8", 52: "VOL052L8"}) + + +def _status(runner: StatefulMtxRunner) -> str: + result = runner.run(["mtx", "-f", DEV, "status"]) + assert result.returncode == 0 + return result.stdout + + +def test_fake_mtx_status_round_trips_through_parser() -> None: + status = parse_mtx_status(_status(_runner())) + + assert (status.drive_count, status.slot_count) == (3, 50) + assert len(status.slots) == 50 + assert {s.slot_id: s.barcode for s in status.slots if s.occupied} == { + 1: "VOL001L8", + 2: "VOL002L8", + } + assert [(s.slot_id, s.barcode) for s in status.import_export_slots] == [ + (51, None), + (52, "VOL052L8"), + ] + + +def test_fake_mtx_load_reports_drive_barcode_and_source_slot() -> None: + runner = _runner() + + assert runner.run(["mtx", "-f", DEV, "load", "2", "1"]).returncode == 0 + + drive = next(d for d in parse_mtx_status(_status(runner)).drives if d.drive_id == 1) + assert (drive.loaded, drive.barcode, drive.source_slot) == (True, "VOL002L8", 2) + + +def test_fake_mtx_unload_and_transfer_move_cartridges() -> None: + runner = _runner() + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + assert runner.run(["mtx", "-f", DEV, "unload", "5", "0"]).returncode == 0 + assert runner.run(["mtx", "-f", DEV, "transfer", "5", "6"]).returncode == 0 + + full = {s.slot_id: s.barcode for s in parse_mtx_status(_status(runner)).slots if s.occupied} + assert full == {2: "VOL002L8", 6: "VOL001L8"} + + +def test_fake_mtx_load_from_empty_slot_fails_with_request_sense() -> None: + result = _runner().run(["mtx", "-f", DEV, "load", "3", "0"]) + + assert result.returncode == 1 + assert result.stderr.startswith("mtx: Request Sense:") + + +def test_fake_mtx_load_into_full_drive_fails_without_moving_tape() -> None: + runner = _runner() + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + result = runner.run(["mtx", "-f", DEV, "load", "2", "0"]) + + assert result.returncode == 1 + assert runner.slots[2] == "VOL002L8" + + +def test_fake_mtx_unload_empty_drive_or_into_full_slot_fails() -> None: + runner = _runner() + assert runner.run(["mtx", "-f", DEV, "unload", "5", "0"]).returncode == 1 + runner.run(["mtx", "-f", DEV, "load", "1", "0"]) + + assert runner.run(["mtx", "-f", DEV, "unload", "2", "0"]).returncode == 1 + + +def test_fake_mtx_unknown_argv_returns_2() -> None: + runner = _runner() + + for argv in ( + ["mtx", "-f", DEV, "inquiry"], + ["mtx", "-f", "/dev/other", "status"], + ["mtx", "-f", DEV, "load", "x", "0"], + ["sg_inq", DEV], + ): + assert runner.run(argv).returncode == 2, argv From 5047a030b3f4a735dcc398c792101baef53ba602 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:07:20 +0000 Subject: [PATCH 130/205] fuse cache: persist LRU index across restarts _track/evict/verify_entry write cache_dir/index.json (temp-then-rename). __init__ loads it (dropping entries whose file is gone), appends any on-disk files the index misses, and enforces max_bytes, so a restart counts existing bytes against the budget. Corrupt index -> rebuild from scan. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index b08fb20..7f85e43 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -55,6 +55,48 @@ def __init__(self, cache_dir: str, max_bytes: int | None = None) -> None: self._lock = threading.RLock() self._lru: OrderedDict[str, int] = OrderedDict() self._refs: dict[str, int] = {} + with self._lock: + # Not persisted here: a fresh cache stays empty on disk, and the scan + # re-finds any unindexed files on the next start. Evictions persist. + self._load_index() + self._enforce_budget() + + # -- persisted LRU index ------------------------------------------------- + @property + def index_path(self) -> Path: + return self.cache_dir / "index.json" + + def _load_index(self) -> None: + """Rebuild ``_lru`` from ``index.json`` plus any on-disk entries it misses. + + Entries whose file is gone are dropped; files not in the index are appended + (size from stat) so a restart always counts on-disk bytes against the budget. + """ + if self.index_path.exists(): + try: + raw = json.loads(self.index_path.read_text(encoding="utf-8")) + for checksum, size in raw: + if isinstance(checksum, str) and self.cache_key(checksum).is_file(): + self._lru[checksum] = int(size) + except (OSError, ValueError, TypeError): + logger.warning("fuse cache: index.json unreadable; rebuilding from disk scan") + self._lru.clear() + for path in sorted(self.cache_dir.glob("??/*")): + name = path.name + if name.startswith(".") or name.endswith(".meta.json") or not path.is_file(): + continue + if name not in self._lru: + self._lru[name] = path.stat().st_size + + def _persist_index(self) -> None: + tmp = self.index_path.with_name(f".index.json.{os.getpid()}.{threading.get_ident()}.part") + try: + tmp.write_text(json.dumps([[c, n] for c, n in self._lru.items()]), encoding="utf-8") + os.replace(tmp, self.index_path) + except OSError: + logger.warning("fuse cache: could not persist index.json") + finally: + tmp.unlink(missing_ok=True) # -- refcounting / LRU --------------------------------------------------- def acquire(self, checksum: str) -> None: @@ -85,6 +127,7 @@ def _track(self, checksum: str, size: int) -> None: self._lru[checksum] = size self._lru.move_to_end(checksum) self._enforce_budget() + self._persist_index() def _enforce_budget(self) -> None: if self.max_bytes is None: @@ -186,6 +229,7 @@ def verify_entry(self, checksum: str) -> bool: self._lru.pop(checksum, None) self.cache_key(checksum).unlink(missing_ok=True) self.meta_key(checksum).unlink(missing_ok=True) + self._persist_index() raise CacheIntegrityError("cached entry failed integrity verification and was evicted") def cache_key(self, checksum: str) -> Path: @@ -242,3 +286,4 @@ def evict(self, checksum: str) -> None: if path.exists(): path.unlink() self.meta_key(checksum).unlink(missing_ok=True) + self._persist_index() From 1c965f140a43c262fc07bd6f398129affb715f21 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:07:21 +0000 Subject: [PATCH 131/205] feat(emulator): env-driven AML fault profile parser and counters Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/aml_faults.py | 157 ++++++++++++++++++++++++++++++++++ tests/unit/test_aml_faults.py | 78 +++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 openblade/api/aml_faults.py create mode 100644 tests/unit/test_aml_faults.py diff --git a/openblade/api/aml_faults.py b/openblade/api/aml_faults.py new file mode 100644 index 0000000..e251f65 --- /dev/null +++ b/openblade/api/aml_faults.py @@ -0,0 +1,157 @@ +"""Env-driven AML wire-level fault injection for the emulator. + +``OPENBLADE_EMULATOR_FAULT_PROFILE`` is either a preset name (see ``PRESETS``) or a +JSON object with any of the keys ``load_fail_every`` (int), ``auth_ttl_s`` (float), +``reboot_window_s`` (float) and ``checksum_retry_every`` (int). Unset/empty means +no faults. Anything unparseable raises ``FaultProfileError``: a silent no-fault run +would be a false green for the scenario that asked for faults. + +The presets are the single source of truth for the i3 timing profiles +(``tests/i3/timing.py`` imports them). +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from dataclasses import dataclass, fields + +FAULT_PROFILE_ENV = "OPENBLADE_EMULATOR_FAULT_PROFILE" + + +class FaultProfileError(ValueError): + """``OPENBLADE_EMULATOR_FAULT_PROFILE`` could not be parsed.""" + + +@dataclass(frozen=True) +class FaultProfile: + """Fault knobs (0/None = off).""" + + load_fail_every: int = 0 + auth_ttl_s: float | None = None + reboot_window_s: float = 0.0 + checksum_retry_every: int = 0 + + @property + def active(self) -> bool: + return self != FaultProfile() + + +PRESETS: dict[str, FaultProfile] = { + "instant": FaultProfile(), + "realistic": FaultProfile(), + "hardware": FaultProfile(), + "normal": FaultProfile(), + "slow-robotics": FaultProfile(), + "busy-library": FaultProfile(), + "intermittent-drive": FaultProfile(load_fail_every=3), + "session-expiry": FaultProfile(auth_ttl_s=5.0), + "rebooting": FaultProfile(reboot_window_s=20.0), + "degraded-media": FaultProfile(checksum_retry_every=4), +} + +_INT_KEYS = ("load_fail_every", "checksum_retry_every") +_FLOAT_KEYS = ("auth_ttl_s", "reboot_window_s") + + +def parse_fault_profile(raw: str | None) -> FaultProfile: + """Strictly parse a preset name or JSON object; raise ``FaultProfileError``.""" + text = (raw or "").strip() + if not text: + return FaultProfile() + if not text.startswith("{"): + try: + return PRESETS[text.lower()] + except KeyError: + raise FaultProfileError( + f"{FAULT_PROFILE_ENV}: unknown preset {text!r}; expected one of " + f"{sorted(PRESETS)} or a JSON object" + ) from None + try: + data = json.loads(text) + except json.JSONDecodeError: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: invalid JSON") from None + if not isinstance(data, dict): + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: JSON must be an object") + known = {f.name for f in fields(FaultProfile)} + unknown = sorted(set(data) - known) + if unknown: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: unknown keys {unknown}") + values: dict[str, int | float] = {} + for key, value in data.items(): + # bool is an int subclass; reject it explicitly. + if isinstance(value, bool) or not isinstance(value, int | float) or value < 0: + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: {key} must be a number >= 0") + if key in _INT_KEYS: + if not isinstance(value, int): + raise FaultProfileError(f"{FAULT_PROFILE_ENV}: {key} must be an integer") + values[key] = value + else: + values[key] = float(value) + return FaultProfile( + load_fail_every=int(values.get("load_fail_every", 0)), + auth_ttl_s=float(values["auth_ttl_s"]) if "auth_ttl_s" in values else None, + reboot_window_s=float(values.get("reboot_window_s", 0.0)), + checksum_retry_every=int(values.get("checksum_retry_every", 0)), + ) + + +def load_fault_profile_from_env() -> FaultProfile: + return parse_fault_profile(os.environ.get(FAULT_PROFILE_ENV)) + + +class FaultState: + """Process-wide fault counters (thread-safe).""" + + def __init__(self, profile: FaultProfile) -> None: + self.profile = profile + self.started_at = time.monotonic() + self.loads = 0 + self.verifies = 0 + self._lock = threading.Lock() + + def in_reboot_window(self) -> bool: + window = self.profile.reboot_window_s + return bool(window) and time.monotonic() - self.started_at < window + + def reboot_retry_after_s(self) -> int: + remaining = self.profile.reboot_window_s - (time.monotonic() - self.started_at) + return max(1, int(remaining + 0.999)) + + def should_fail_load(self) -> bool: + every = self.profile.load_fail_every + if not every: + return False + with self._lock: + self.loads += 1 + return self.loads % every == 0 + + def should_retry_checksum(self) -> bool: + every = self.profile.checksum_retry_every + if not every: + return False + with self._lock: + self.verifies += 1 + return self.verifies % every == 0 + + +_state: FaultState | None = None +_state_lock = threading.Lock() + + +def get_fault_state() -> FaultState: + """Return the process-wide state, parsing the env on first use (may raise).""" + global _state + with _state_lock: + if _state is None: + _state = FaultState(load_fault_profile_from_env()) + return _state + + +def reset_fault_state() -> None: + """Drop the cached state so the next call re-reads the env (tests).""" + global _state + with _state_lock: + _state = None diff --git a/tests/unit/test_aml_faults.py b/tests/unit/test_aml_faults.py new file mode 100644 index 0000000..c05d7ab --- /dev/null +++ b/tests/unit/test_aml_faults.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from openblade.api import aml_faults +from openblade.api.aml_faults import ( + FaultProfile, + FaultProfileError, + FaultState, + parse_fault_profile, +) + + +def test_unset_means_no_faults() -> None: + assert parse_fault_profile(None) == FaultProfile() + assert parse_fault_profile(" ") == FaultProfile() + assert not parse_fault_profile("instant").active + + +def test_presets_resolve() -> None: + assert parse_fault_profile("intermittent-drive").load_fail_every == 3 + assert parse_fault_profile("Session-Expiry").auth_ttl_s == 5.0 + assert parse_fault_profile("rebooting").reboot_window_s == 20.0 + assert parse_fault_profile("degraded-media").checksum_retry_every == 4 + + +def test_json_object() -> None: + profile = parse_fault_profile('{"load_fail_every": 2, "auth_ttl_s": 1}') + assert profile == FaultProfile(load_fail_every=2, auth_ttl_s=1.0) + + +@pytest.mark.parametrize( + "raw", + [ + "no-such-profile", + "{not json", + "[1]", + '{"bogus": 1}', + '{"load_fail_every": 1.5}', + '{"load_fail_every": true}', + '{"auth_ttl_s": -1}', + '{"reboot_window_s": "20"}', + ], +) +def test_strict_parse_rejects(raw: str) -> None: + with pytest.raises(FaultProfileError): + parse_fault_profile(raw) + + +def test_counters() -> None: + state = FaultState(FaultProfile(load_fail_every=3, checksum_retry_every=2)) + assert [state.should_fail_load() for _ in range(6)] == [False, False, True] * 2 + assert [state.should_retry_checksum() for _ in range(4)] == [False, True] * 2 + assert not state.in_reboot_window() + + +def test_reboot_window(monkeypatch: pytest.MonkeyPatch) -> None: + now = [100.0] + monkeypatch.setattr(aml_faults.time, "monotonic", lambda: now[0]) + state = FaultState(FaultProfile(reboot_window_s=20.0)) + assert state.in_reboot_window() + assert state.reboot_retry_after_s() == 20 + now[0] = 120.0 + assert not state.in_reboot_window() + + +def test_env_state_cached_and_reset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, "rebooting") + aml_faults.reset_fault_state() + try: + assert aml_faults.get_fault_state().profile.reboot_window_s == 20.0 + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, "bad") + assert aml_faults.get_fault_state().profile.reboot_window_s == 20.0 + aml_faults.reset_fault_state() + with pytest.raises(FaultProfileError): + aml_faults.get_fault_state() + finally: + aml_faults.reset_fault_state() From de6188874fb35d6035560dbbf77a89696003af30 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:07:42 +0000 Subject: [PATCH 132/205] feat(catalog): persist NAS pool replication/protection and add NAS domain tables nas_pools gains replication_factor + protection_json (serialised like nas_datasets.protection_json); new tables nas_reservations, nas_export_sets, nas_vaults, nas_vg_replication, nas_file_spans. _migrate_schema adds them idempotently to pre-existing databases. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/db.py | 22 ++++++++ openblade/catalog/models.py | 47 ++++++++++++++++ openblade/catalog/repository.py | 8 +++ tests/unit/test_nas_persistence.py | 89 ++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 tests/unit/test_nas_persistence.py diff --git a/openblade/catalog/db.py b/openblade/catalog/db.py index b01f305..05ee776 100644 --- a/openblade/catalog/db.py +++ b/openblade/catalog/db.py @@ -113,6 +113,14 @@ def _migrate_schema(engine: Engine) -> None: library_columns = { column["name"] for column in inspector.get_columns("library_instances") } + if "nas_pools" in inspector.get_table_names(): + pool_columns = {column["name"] for column in inspector.get_columns("nas_pools")} + if "replication_factor" not in pool_columns: + connection.execute( + text("ALTER TABLE nas_pools ADD COLUMN replication_factor INTEGER DEFAULT 1") + ) + if "protection_json" not in pool_columns: + connection.execute(text("ALTER TABLE nas_pools ADD COLUMN protection_json TEXT")) if "nas_datasets" in inspector.get_table_names(): dataset_columns = {column["name"] for column in inspector.get_columns("nas_datasets")} if "protection_json" not in dataset_columns: @@ -291,6 +299,20 @@ def _migrate_schema(engine: Engine) -> None: """ ) ) + # NAS domain tables (added after initial release); create_all(checkfirst) is idempotent. + Base.metadata.create_all( + engine, + tables=[ + Base.metadata.tables[name] + for name in ( + "nas_reservations", + "nas_export_sets", + "nas_vaults", + "nas_vg_replication", + "nas_file_spans", + ) + ], + ) def init_db(db_url: str = "sqlite:///./openblade.db") -> None: diff --git a/openblade/catalog/models.py b/openblade/catalog/models.py index 0bd0fb9..b5ec24b 100644 --- a/openblade/catalog/models.py +++ b/openblade/catalog/models.py @@ -307,6 +307,9 @@ class NasPool(Base): cache_target_id: Mapped[str | None] = mapped_column(String, nullable=True) restore_target_path: Mapped[str] = mapped_column(String, default="/openblade/restore") access_mode: Mapped[str] = mapped_column(String, default="read_only") + replication_factor: Mapped[int] = mapped_column(Integer, default=1) + # Serialized ProtectionPolicy.to_dict(); None means derive from replication_factor. + protection_json: Mapped[str | None] = mapped_column(Text, nullable=True) created_at: Mapped[str | None] = mapped_column(Text, nullable=True) updated_at: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -491,3 +494,47 @@ class JobJournalEntry(Base): @property def detail(self) -> dict[str, Any]: return cast("dict[str, Any]", json.loads(self.detail_json or "{}")) + + +class NasReservation(Base): + __tablename__ = "nas_reservations" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + pool_id: Mapped[str] = mapped_column(String, nullable=False, index=True) + bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + owner: Mapped[str] = mapped_column(String, nullable=False) + # ISO-8601 with offset (SQLite DateTime would drop the tzinfo). + expires_at: Mapped[str] = mapped_column(Text, nullable=False) + + +class NasExportSet(Base): + __tablename__ = "nas_export_sets" + + id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + state: Mapped[str] = mapped_column(String, nullable=False) + + +class NasVault(Base): + __tablename__ = "nas_vaults" + + id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + location: Mapped[str] = mapped_column(String, nullable=False) + + +class NasVgReplication(Base): + __tablename__ = "nas_vg_replication" + + vg_id: Mapped[str] = mapped_column(String, primary_key=True) + barcodes_json: Mapped[str] = mapped_column(Text, default="[]") + replicas_required: Mapped[int] = mapped_column(Integer, default=1) + replicas_present: Mapped[int] = mapped_column(Integer, default=0) + + +class NasFileSpan(Base): + __tablename__ = "nas_file_spans" + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + path: Mapped[str] = mapped_column(Text, nullable=False, index=True) + segments_json: Mapped[str] = mapped_column(Text, default="[]") diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 0ecdc07..bb93e73 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -174,6 +174,8 @@ def _nas_pool_to_dict(self, row: NasPool) -> dict[str, object]: "cache_target_id": row.cache_target_id, "restore_target_path": row.restore_target_path, "access_mode": row.access_mode, + "replication_factor": row.replication_factor, + "protection": _load_json_value(row.protection_json, None), "created_at": row.created_at, "updated_at": row.updated_at, } @@ -1074,6 +1076,12 @@ def upsert_nas_pool(self, config: dict[str, object]) -> dict[str, object]: row.cache_target_id = parsed.cache_target_id row.restore_target_path = parsed.restore_target_path row.access_mode = parsed.access_mode + row.replication_factor = parsed.replication_factor + row.protection_json = ( + json.dumps(parsed.protection.to_dict(), sort_keys=True) + if parsed.protection is not None + else None + ) row.created_at = row.created_at or parsed.created_at or _utcnow_iso() row.updated_at = parsed.updated_at or _utcnow_iso() self.session.commit() diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py new file mode 100644 index 0000000..98a374d --- /dev/null +++ b/tests/unit/test_nas_persistence.py @@ -0,0 +1,89 @@ +"""Persistence of the NAS domain models (pools, reservations, spans, export sets, vaults).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, inspect, text + +from openblade.catalog.db import _migrate_schema, get_session, init_db +from openblade.catalog.repository import CatalogRepository + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()) + + +_OLD_NAS_POOLS_DDL = """ +CREATE TABLE nas_pools ( + id VARCHAR PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + description TEXT, + volume_group_ids TEXT, + default_policy_id VARCHAR, + default_ingest_mode VARCHAR, + mount_path VARCHAR, + virtual_mount_enabled BOOLEAN, + hydration_behavior VARCHAR, + cache_target_id VARCHAR, + restore_target_path VARCHAR, + access_mode VARCHAR, + created_at TEXT, + updated_at TEXT +) +""" + +_NEW_TABLES = { + "nas_reservations", + "nas_export_sets", + "nas_vaults", + "nas_vg_replication", + "nas_file_spans", +} + + +def test_migration_upgrades_pre_change_db_idempotently(tmp_path: Path) -> None: + engine = create_engine(f"sqlite:///{tmp_path / 'old.db'}") + with engine.begin() as connection: + connection.execute(text(_OLD_NAS_POOLS_DDL)) + connection.execute(text("INSERT INTO nas_pools (id, name) VALUES ('p1', 'legacy')")) + + _migrate_schema(engine) + _migrate_schema(engine) # second run must be a no-op, not an error + + inspector = inspect(engine) + pool_columns = {column["name"] for column in inspector.get_columns("nas_pools")} + assert {"replication_factor", "protection_json"} <= pool_columns + assert set(inspector.get_table_names()) >= _NEW_TABLES + with engine.connect() as connection: + row = connection.execute( + text("SELECT replication_factor, protection_json FROM nas_pools WHERE id = 'p1'") + ).one() + assert row == (1, None) + + +def test_pool_round_trips_replication_and_protection(repo: CatalogRepository) -> None: + saved = repo.upsert_nas_pool({"id": "p1", "name": "pool-a", "replication_factor": 2}) + assert saved["replication_factor"] == 2 + assert saved["protection"] is None + loaded = repo.get_nas_pool("p1") + assert loaded is not None + assert loaded["replication_factor"] == 2 + # Round trip: the dict form must be accepted back by upsert unchanged. + again = repo.upsert_nas_pool(dict(loaded)) + assert again["replication_factor"] == 2 + + +def test_pool_protection_is_serialised_like_dataset_protection(repo: CatalogRepository) -> None: + import json + + from openblade.domain.protection import ProtectionPolicy + + policy = ProtectionPolicy() + saved = repo.upsert_nas_pool({"id": "p2", "name": "pool-b", "protection": policy}) + assert saved["protection"] == json.loads(json.dumps(policy.to_dict(), sort_keys=True)) + again = repo.upsert_nas_pool(dict(saved)) + assert again["protection"] == saved["protection"] From 144746ee76f0791f65cd29bef45bea3ad0bc5131 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:08:08 +0000 Subject: [PATCH 133/205] JobQueue: persist jobs in the catalog jobs table JobQueue(catalog, lease_store, *, ttl) now reads and writes jobs through CatalogRepository so every process sharing the catalog sees them; run_job persists RUNNING before func and COMPLETED/FAILED after. Bootstrap builds the lease store before the queue; test constructor sites get a tmp sqlite catalog. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/bootstrap.py | 5 +- openblade/jobs/queue.py | 66 ++++++++++++------- .../test_three_drive_concurrency.py | 17 +++-- tests/unit/test_job_error_sanitization.py | 9 ++- tests/unit/test_nonsharded_leases.py | 8 ++- 5 files changed, 69 insertions(+), 36 deletions(-) diff --git a/openblade/bootstrap.py b/openblade/bootstrap.py index 35c16d1..c5b4dec 100644 --- a/openblade/bootstrap.py +++ b/openblade/bootstrap.py @@ -611,14 +611,15 @@ def create_context(config: OpenBladeConfig | None = None) -> AppContext: with suppress(Exception): _seed_demo_ltfs(catalog, ltfs) run_inventory_job(library, catalog) - queue = JobQueue() + lease_store = CatalogLeaseStore(catalog) + queue = JobQueue(catalog, lease_store) worker = Worker(queue) return AppContext( config=active_config, library=library, ltfs=ltfs, catalog=catalog, - lease_store=CatalogLeaseStore(catalog), + lease_store=lease_store, recovery_report=recovery_report, queue=queue, worker=worker, diff --git a/openblade/jobs/queue.py b/openblade/jobs/queue.py index 2040996..a235e55 100644 --- a/openblade/jobs/queue.py +++ b/openblade/jobs/queue.py @@ -1,12 +1,14 @@ -"""In-process job queue with resource ownership tracking.""" +"""Job queue persisted in the catalog, with resource ownership tracking.""" +import json import logging import threading from collections.abc import Callable -from dataclasses import replace -from datetime import datetime, timezone +from datetime import timedelta, timezone from typing import TypeVar +from openblade.catalog import models as catalog_models +from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import ( ChangerBusyError, DriveOccupiedError, @@ -14,48 +16,62 @@ safe_job_error, ) from openblade.domain.models import Job, JobState, JobType +from openblade.jobs.scheduler import DEFAULT_LEASE_TTL, LeaseStore ResultT = TypeVar("ResultT") logger = logging.getLogger(__name__) +def _job_from_row(row: catalog_models.Job) -> Job: + return Job( + id=row.id, + job_type=JobType(row.job_type), + state=JobState(row.state), + created_at=row.created_at.replace(tzinfo=timezone.utc), + updated_at=row.updated_at.replace(tzinfo=timezone.utc), + error=row.error, + metadata=json.loads(row.metadata_json or "{}"), + ) + + class JobQueue: - def __init__(self) -> None: + """Jobs live in the catalog ``jobs`` table, so every process sharing it sees them.""" + + def __init__( + self, + catalog: CatalogRepository, + lease_store: LeaseStore, + *, + ttl: timedelta = DEFAULT_LEASE_TTL, + ) -> None: + self._catalog = catalog + self._lease_store = lease_store + self._ttl = ttl self._lock = threading.RLock() - self._jobs: dict[str, Job] = {} self._drive_owners: dict[int, str] = {} self._changer_owner: str | None = None def create_job(self, job_type: JobType, metadata: dict[str, object]) -> Job: - job = Job(job_type=job_type, metadata=dict(metadata)) with self._lock: - self._jobs[job.id] = job - return job + row = self._catalog.create_job(JobType(job_type).value, dict(metadata)) + return _job_from_row(row) def get_job(self, job_id: str) -> Job: with self._lock: - try: - return self._jobs[job_id] - except KeyError as exc: - raise JobNotFoundError(job_id) from exc - - def _set_job(self, job: Job) -> None: - with self._lock: - self._jobs[job.id] = job + row = self._catalog.get_job(job_id) + if row is None: + raise JobNotFoundError(job_id) + self._catalog.session.refresh(row) + return _job_from_row(row) def update_job( self, job_id: str, *, state: JobState | None = None, error: str | None = None ) -> Job: - job = self.get_job(job_id) - updated = replace( - job, - state=state or job.state, - error=error, - updated_at=datetime.now(timezone.utc), - ) - self._set_job(updated) - return updated + with self._lock: + job = self.get_job(job_id) + self._catalog.update_job_state(job_id, (state or job.state).value, error) + return self.get_job(job_id) def claim_drive(self, drive_id: int, job_id: str) -> None: with self._lock: diff --git a/tests/integration/test_three_drive_concurrency.py b/tests/integration/test_three_drive_concurrency.py index 320fe9f..c7391d2 100644 --- a/tests/integration/test_three_drive_concurrency.py +++ b/tests/integration/test_three_drive_concurrency.py @@ -10,14 +10,17 @@ import threading import time from collections.abc import Callable +from pathlib import Path import pytest +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import ChangerBusyError, DriveOccupiedError from openblade.domain.models import JobType, MountMode from openblade.domain.policies import FormatConfirmation, SafetyToken from openblade.jobs.queue import JobQueue -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler from openblade.simulator.library import MockLibraryBackend from openblade.simulator.ltfs_volume import MockLTFSBackend from openblade.simulator.scenarios import scalar_i3_default @@ -25,6 +28,13 @@ BARCODES = ["TRI001L8", "TRI002L8", "TRI003L8"] +@pytest.fixture +def queue(tmp_path: Path) -> JobQueue: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + catalog = CatalogRepository(get_session()) + return JobQueue(catalog, CatalogLeaseStore(catalog)) + + @pytest.fixture def three_drive_library() -> tuple[MockLibraryBackend, MockLTFSBackend]: library = MockLibraryBackend(num_slots=10, num_drives=3) @@ -55,10 +65,10 @@ def test_simulator_drive_count_is_configurable() -> None: def test_three_jobs_run_on_three_drives_with_disjoint_ownership( three_drive_library: tuple[MockLibraryBackend, MockLTFSBackend], + queue: JobQueue, ) -> None: library, ltfs = three_drive_library scheduler = DriveScheduler(num_drives=3) - queue = JobQueue() jobs = [queue.create_job(JobType.ARCHIVE, {"barcode": barcode}) for barcode in BARCODES] started = threading.Barrier(len(jobs)) @@ -116,8 +126,7 @@ def _run(job_id: str, barcode: str, slot_id: int) -> None: assert ltfs.read_bytes(barcode, f"/{barcode}.bin") == b"x" * 1024 -def test_job_queue_refuses_to_share_a_drive_between_jobs() -> None: - queue = JobQueue() +def test_job_queue_refuses_to_share_a_drive_between_jobs(queue: JobQueue) -> None: first = queue.create_job(JobType.ARCHIVE, {}) second = queue.create_job(JobType.ARCHIVE, {}) queue.claim_drive(2, first.id) diff --git a/tests/unit/test_job_error_sanitization.py b/tests/unit/test_job_error_sanitization.py index 7b9a572..34702a4 100644 --- a/tests/unit/test_job_error_sanitization.py +++ b/tests/unit/test_job_error_sanitization.py @@ -37,10 +37,15 @@ class DemoError(OpenBladeError): ) -def test_run_job_records_sanitized_error() -> None: +def test_run_job_records_sanitized_error(tmp_path) -> None: + from openblade.catalog.db import get_session, init_db + from openblade.catalog.repository import CatalogRepository from openblade.jobs.queue import JobQueue + from openblade.jobs.scheduler import CatalogLeaseStore - queue = JobQueue() + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + catalog = CatalogRepository(get_session()) + queue = JobQueue(catalog, CatalogLeaseStore(catalog)) job = queue.create_job("archive", {}) def boom(): diff --git a/tests/unit/test_nonsharded_leases.py b/tests/unit/test_nonsharded_leases.py index 90cc366..1aa41bf 100644 --- a/tests/unit/test_nonsharded_leases.py +++ b/tests/unit/test_nonsharded_leases.py @@ -157,7 +157,7 @@ def test_archive_service_holds_a_catalog_lease_while_writing( source = tmp_path / "src" source.mkdir() (source / "a.bin").write_bytes(b"x" * 64) - service = ArchiveService(library, ltfs, repo_a, JobQueue()) + service = ArchiveService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))) seen: list[list[str]] = [] real_write = ltfs.write_file @@ -183,8 +183,10 @@ def test_restore_service_holds_a_catalog_lease_while_reading( source = tmp_path / "src" source.mkdir() (source / "a.bin").write_bytes(b"y" * 64) - ArchiveService(library, ltfs, repo_a, JobQueue()).enqueue("photos", source) - service = RestoreService(library, ltfs, repo_a, JobQueue()) + ArchiveService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))).enqueue( + "photos", source + ) + service = RestoreService(library, ltfs, repo_a, JobQueue(repo_a, CatalogLeaseStore(repo_a))) seen: list[list[str]] = [] real_read = ltfs.read_file From 21a5aab05ed07c9203893db3e7498301ce2bf9bf Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:08:21 +0000 Subject: [PATCH 134/205] test(contract): add scsi+fake-ltfs pairing over RealLibraryBackend Drives the production SCSI backend through StatefulMtxRunner (which now also answers sg_inq for declared drives so construction-time drive correlation succeeds) with MockLTFSBackend on top. Always runs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/contract/conftest.py | 38 +++++++++++++++++++++++++++++++++++++ tests/fakes/mtx.py | 16 ++++++++++++++++ tests/unit/test_fake_mtx.py | 7 +++++++ 3 files changed, 61 insertions(+) diff --git a/tests/contract/conftest.py b/tests/contract/conftest.py index c234ddc..1b35bf5 100644 --- a/tests/contract/conftest.py +++ b/tests/contract/conftest.py @@ -89,6 +89,41 @@ def _emulator_pair(tmp_path: Path) -> Iterator[BackendPair]: pair.unload_everything() +def _scsi_fake_ltfs_pair() -> Iterator[BackendPair]: + """The production SCSI backend (``RealLibraryBackend`` + ``MtxChangerBackend``) + driven by a stateful fake ``mtx``: real argv building, status parsing and the + unload-while-mounted gate, with no /dev access.""" + from openblade.config import BackendMode, OpenBladeConfig + from openblade.hardware.discovery import LibraryDiscovery + from openblade.hardware.library import RealLibraryBackend + from tests.fakes.mtx import StatefulMtxRunner + + devices = tuple(f"/dev/nst{i}" for i in range(3)) + config = OpenBladeConfig( + backend=BackendMode.REAL, + real_hardware_enabled=True, + hardware_dry_run=False, + changer_device="/dev/sg3", + drive_devices=devices, + drive_serial_map=tuple((f"FAKESER{i}", i) for i in range(len(devices))), + ) + runner = StatefulMtxRunner( + "/dev/sg3", + barcodes=dict(enumerate(SIM_BARCODES, start=1)), + drive_serials={device: f"FAKESER{i}" for i, device in enumerate(devices)}, + ) + library = RealLibraryBackend( + config=config, + runner=runner, + discovery=LibraryDiscovery(changers=[], drives=[], sg_map={}), + ) + # MockLTFSBackend consumes only get_all_barcodes / find_drive_by_barcode / + # set_drive_mount_state, which RealLibraryBackend implements; the latter is + # what arms RealLibraryBackend.unload's TapeMountedError gate. + ltfs = MockLTFSBackend(cast(MockLibraryBackend, library), capacity_bytes=CAPACITY) + yield BackendPair("scsi+fake-ltfs", library, ltfs, len(devices), list(SIM_BARCODES)) + + def _real_pair() -> Iterator[BackendPair]: from openblade.bootstrap import get_library, get_ltfs @@ -103,6 +138,7 @@ def _real_pair() -> Iterator[BackendPair]: PAIRINGS = [ pytest.param("sim+sim", id="sim+sim"), pytest.param("emulator+sim-ltfs", id="emulator+sim-ltfs"), + pytest.param("scsi+fake-ltfs", id="scsi+fake-ltfs"), pytest.param( "real+real", id="real+real", @@ -123,6 +159,8 @@ def backend_pair(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[Ba yield from _sim_pair() elif name == "emulator+sim-ltfs": yield from _emulator_pair(tmp_path) + elif name == "scsi+fake-ltfs": + yield from _scsi_fake_ltfs_pair() else: yield from _real_pair() diff --git a/tests/fakes/mtx.py b/tests/fakes/mtx.py index 68c57f2..14c0354 100644 --- a/tests/fakes/mtx.py +++ b/tests/fakes/mtx.py @@ -27,6 +27,7 @@ def __init__( slots: int = 50, ie_slots: int = 2, barcodes: dict[int, str], + drive_serials: dict[str, str] | None = None, ) -> None: super().__init__(dry_run=False) self.device = device @@ -40,6 +41,9 @@ def __init__( self.slots: dict[int, str | None] = {s: barcodes.get(s) for s in range(1, total + 1)} # Per drive: (barcode, source slot) when full, None when empty. self.drives: dict[int, tuple[str, int] | None] = dict.fromkeys(range(drives)) + # ``RealLibraryBackend`` correlates drives at construction with + # ``sg_inq ``; answer it for the tape devices named here. + self.drive_serials = dict(drive_serials or {}) self.calls: list[list[str]] = [] def run( @@ -49,6 +53,8 @@ def run( redact_args: list[int] | None = None, ) -> CommandResult: self.calls.append(list(args)) + if len(args) == 2 and args[0] == "sg_inq" and args[1] in self.drive_serials: + return self._result(args, 0, stdout=_sg_inq_output(self.drive_serials[args[1]])) if len(args) < 4 or args[:3] != ["mtx", "-f", self.device]: return self._result(args, 2, stderr=f"mtx: unsupported invocation {args!r}") verb, operands = args[3], args[4:] @@ -130,6 +136,16 @@ def _result(args: list[str], code: int, *, stdout: str = "", stderr: str = "") - ) +def _sg_inq_output(serial: str) -> str: + return ( + "standard INQUIRY:\n" + " PQual=0 PDT=1 RMB=1 LU_CONG=0 version=0x06 [SPC-4]\n" + " Vendor identification: IBM\n" + " Product identification: ULTRIUM-TD8\n" + f" Unit serial number: {serial}\n" + ) + + def _ints(values: list[str]) -> list[int] | None: try: return [int(value) for value in values] diff --git a/tests/unit/test_fake_mtx.py b/tests/unit/test_fake_mtx.py index 36cf9df..6300631 100644 --- a/tests/unit/test_fake_mtx.py +++ b/tests/unit/test_fake_mtx.py @@ -88,3 +88,10 @@ def test_fake_mtx_unknown_argv_returns_2() -> None: ["sg_inq", DEV], ): assert runner.run(argv).returncode == 2, argv + + +def test_fake_mtx_answers_sg_inq_only_for_declared_drives() -> None: + runner = StatefulMtxRunner(DEV, barcodes={}, drive_serials={"/dev/nst0": "SER0"}) + + assert "Unit serial number: SER0" in runner.run(["sg_inq", "/dev/nst0"]).stdout + assert runner.run(["sg_inq", "/dev/nst1"]).returncode == 2 From 7001c406fc4a3501832ca20a470ca9faaeb85768 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:08:54 +0000 Subject: [PATCH 135/205] fuse cache: loop os.pread in read_range until length or EOF Handles short reads and caps each call at 1 GiB. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 7f85e43..76647f0 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) SAMPLE_BYTES = 64 * 1024 +_PREAD_CAP = 1 << 30 # 1 GiB per os.pread call class CacheEntryInUseError(RuntimeError): @@ -273,7 +274,19 @@ def read_range(self, checksum: str, offset: int, length: int) -> bytes: except FileNotFoundError: raise FileNotFoundError(f"Not in cache: {checksum}") from None try: - return os.pread(fd, length, offset) + # pread may return short (and Linux caps one call near 2 GiB): loop until + # ``length`` bytes or EOF, returning exactly the bytes available. + parts: list[bytes] = [] + remaining = length + pos = offset + while remaining > 0: + chunk = os.pread(fd, min(remaining, _PREAD_CAP), pos) + if not chunk: + break # EOF + parts.append(chunk) + pos += len(chunk) + remaining -= len(chunk) + return b"".join(parts) finally: os.close(fd) From f834429fd03b8cc41d755f771ac904051ec89751 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:09:05 +0000 Subject: [PATCH 136/205] feat(catalog): CRUD for persisted NAS reservations, export sets, vaults, VG replication, file spans New '# --- NAS domain persistence ---' section after the restore-job methods; reservations_for_pool excludes expired rows; scratch thresholds live in nas_configs under 'scratch_thresholds'. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 155 +++++++++++++++++++++++++++++ tests/unit/test_nas_persistence.py | 61 ++++++++++++ 2 files changed, 216 insertions(+) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index bb93e73..868dc9f 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -4,6 +4,7 @@ import json import threading +import uuid from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass @@ -16,6 +17,7 @@ from sqlalchemy.orm import Session, selectinload from openblade.catalog.models import ( + Base, Cartridge, CatalogRebuildRun, DriveLeaseRecord, @@ -28,11 +30,16 @@ NasCacheDrive, NasConfig, NasDataset, + NasExportSet, NasFileRecord, + NasFileSpan, NasPool, + NasReservation, NasRestoreJob, NasShare, NasStoragePolicy, + NasVault, + NasVgReplication, PathMapping, RbacApiToken, RbacAuditEvent, @@ -1302,6 +1309,154 @@ def delete_nas_restore_job(self, job_id: str) -> bool: self.session.commit() return True + # --- NAS domain persistence --- + + _SCRATCH_THRESHOLDS_KEY = "scratch_thresholds" + + def _delete_nas_row(self, model: type[Base], key: str) -> bool: + row = self.session.get(model, key) + if row is None: + return False + self.session.delete(row) + self.session.commit() + return True + + @staticmethod + def _nas_reservation_to_dict(row: NasReservation) -> dict[str, object]: + return { + "id": row.id, + "pool_id": row.pool_id, + "bytes": row.bytes, + "owner": row.owner, + "expires_at": datetime.fromisoformat(row.expires_at), + } + + def list_nas_reservations(self, pool_id: str | None = None) -> list[dict[str, object]]: + stmt = select(NasReservation).order_by(NasReservation.expires_at) + if pool_id is not None: + stmt = stmt.where(NasReservation.pool_id == pool_id) + rows = self.session.execute(stmt).scalars().all() + return [self._nas_reservation_to_dict(row) for row in rows] + + def reservations_for_pool(self, pool_id: str, *, now: datetime) -> list[dict[str, object]]: + """Unexpired reservations for ``pool_id`` (a reservation expiring at ``now`` is expired).""" + return [ + item + for item in self.list_nas_reservations(pool_id) + if cast(datetime, item["expires_at"]) > now + ] + + def upsert_nas_reservation(self, data: dict[str, object]) -> dict[str, object]: + expires_at = data["expires_at"] + if not isinstance(expires_at, datetime): + expires_at = datetime.fromisoformat(str(expires_at)) + reservation_id = str(data.get("id") or uuid.uuid4()) + row = self.session.get(NasReservation, reservation_id) + if row is None: + row = NasReservation(id=reservation_id) + self.session.add(row) + row.pool_id = str(data["pool_id"]) + row.bytes = int(cast(int, data["bytes"])) + row.owner = str(data["owner"]) + row.expires_at = expires_at.isoformat() + self.session.commit() + return self._nas_reservation_to_dict(row) + + def delete_nas_reservation(self, reservation_id: str) -> bool: + return self._delete_nas_row(NasReservation, reservation_id) + + def list_nas_export_sets(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasExportSet).order_by(NasExportSet.id)).scalars() + return [ + {"id": r.id, "barcodes": _load_json_value(r.barcodes_json, []), "state": r.state} + for r in rows + ] + + def upsert_nas_export_set(self, data: dict[str, object]) -> dict[str, object]: + row = self.session.get(NasExportSet, str(data["id"])) or NasExportSet(id=str(data["id"])) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.state = str(data["state"]) + self.session.commit() + return {"id": row.id, "barcodes": json.loads(row.barcodes_json), "state": row.state} + + def delete_nas_export_set(self, export_set_id: str) -> bool: + return self._delete_nas_row(NasExportSet, export_set_id) + + def list_nas_vaults(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasVault).order_by(NasVault.id)).scalars() + return [ + {"id": r.id, "barcodes": _load_json_value(r.barcodes_json, []), "location": r.location} + for r in rows + ] + + def upsert_nas_vault(self, data: dict[str, object]) -> dict[str, object]: + row = self.session.get(NasVault, str(data["id"])) or NasVault(id=str(data["id"])) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.location = str(data["location"]) + self.session.commit() + return {"id": row.id, "barcodes": json.loads(row.barcodes_json), "location": row.location} + + def delete_nas_vault(self, vault_id: str) -> bool: + return self._delete_nas_row(NasVault, vault_id) + + @staticmethod + def _nas_vg_replication_to_dict(row: NasVgReplication) -> dict[str, object]: + return { + "vg_id": row.vg_id, + "barcodes": _load_json_value(row.barcodes_json, []), + "replicas_required": row.replicas_required, + "replicas_present": row.replicas_present, + } + + def list_nas_vg_replication(self) -> list[dict[str, object]]: + rows = self.session.execute(select(NasVgReplication).order_by(NasVgReplication.vg_id)) + return [self._nas_vg_replication_to_dict(row) for row in rows.scalars()] + + def upsert_nas_vg_replication(self, data: dict[str, object]) -> dict[str, object]: + vg_id = str(data["vg_id"]) + row = self.session.get(NasVgReplication, vg_id) or NasVgReplication(vg_id=vg_id) + self.session.add(row) + row.barcodes_json = json.dumps(list(cast(list[str], data.get("barcodes", [])))) + row.replicas_required = int(cast(int, data.get("replicas_required", 1))) + row.replicas_present = int(cast(int, data.get("replicas_present", 0))) + self.session.commit() + return self._nas_vg_replication_to_dict(row) + + def delete_nas_vg_replication(self, vg_id: str) -> bool: + return self._delete_nas_row(NasVgReplication, vg_id) + + def list_nas_file_spans(self, path: str | None = None) -> list[dict[str, object]]: + stmt = select(NasFileSpan).order_by(NasFileSpan.path, NasFileSpan.id) + if path is not None: + stmt = stmt.where(NasFileSpan.path == path) + return [ + {"id": r.id, "path": r.path, "segments": _load_json_value(r.segments_json, [])} + for r in self.session.execute(stmt).scalars() + ] + + def upsert_nas_file_span(self, data: dict[str, object]) -> dict[str, object]: + span_id = str(data.get("id") or uuid.uuid4()) + row = self.session.get(NasFileSpan, span_id) or NasFileSpan(id=span_id) + self.session.add(row) + row.path = str(data["path"]) + row.segments_json = json.dumps(list(cast(list[object], data.get("segments", [])))) + self.session.commit() + return {"id": row.id, "path": row.path, "segments": json.loads(row.segments_json)} + + def delete_nas_file_span(self, span_id: str) -> bool: + return self._delete_nas_row(NasFileSpan, span_id) + + def get_scratch_thresholds(self) -> dict[str, object] | None: + return self.get_nas_config(self._SCRATCH_THRESHOLDS_KEY) + + def set_scratch_thresholds(self, scratch_min: int, scratch_warn: int) -> dict[str, object]: + return self.set_nas_config( + self._SCRATCH_THRESHOLDS_KEY, + {"scratch_min": scratch_min, "scratch_warn": scratch_warn}, + ) + def create_rebuild_run(self, run: dict[str, object]) -> dict[str, object]: parsed = CatalogRebuildRunRecord.model_validate(run) row = self.session.get(CatalogRebuildRun, parsed.id) diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py index 98a374d..a431a25 100644 --- a/tests/unit/test_nas_persistence.py +++ b/tests/unit/test_nas_persistence.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest @@ -87,3 +88,63 @@ def test_pool_protection_is_serialised_like_dataset_protection(repo: CatalogRepo assert saved["protection"] == json.loads(json.dumps(policy.to_dict(), sort_keys=True)) again = repo.upsert_nas_pool(dict(saved)) assert again["protection"] == saved["protection"] + + +_NOW = datetime(2026, 1, 1, 12, 0, tzinfo=UTC) + + +def test_reservations_round_trip_and_expiry_filter(repo: CatalogRepository) -> None: + live = repo.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 100, "owner": "job-1", "expires_at": _NOW + timedelta(hours=1)} + ) + repo.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 50, "owner": "job-2", "expires_at": _NOW} + ) + repo.upsert_nas_reservation( + {"pool_id": "p2", "bytes": 7, "owner": "job-3", "expires_at": _NOW + timedelta(days=1)} + ) + assert live["expires_at"] == _NOW + timedelta(hours=1) + assert len(repo.list_nas_reservations("p1")) == 2 + active = repo.reservations_for_pool("p1", now=_NOW) + assert [item["owner"] for item in active] == ["job-1"] + assert repo.delete_nas_reservation(str(live["id"])) is True + assert repo.reservations_for_pool("p1", now=_NOW) == [] + assert repo.delete_nas_reservation(str(live["id"])) is False + + +def test_export_sets_vaults_and_vg_replication_round_trip(repo: CatalogRepository) -> None: + repo.upsert_nas_export_set({"id": "e1", "barcodes": ["A00001L9"], "state": "pending"}) + repo.upsert_nas_export_set({"id": "e1", "barcodes": ["A00001L9", "A00002L9"], "state": "out"}) + assert repo.list_nas_export_sets() == [ + {"id": "e1", "barcodes": ["A00001L9", "A00002L9"], "state": "out"} + ] + repo.upsert_nas_vault({"id": "v1", "barcodes": ["A00003L9"], "location": "offsite"}) + assert repo.list_nas_vaults() == [{"id": "v1", "barcodes": ["A00003L9"], "location": "offsite"}] + repo.upsert_nas_vg_replication( + {"vg_id": "vg1", "barcodes": ["A"], "replicas_required": 2, "replicas_present": 1} + ) + assert repo.list_nas_vg_replication() == [ + {"vg_id": "vg1", "barcodes": ["A"], "replicas_required": 2, "replicas_present": 1} + ] + assert repo.delete_nas_export_set("e1") + assert repo.delete_nas_vault("v1") + assert repo.delete_nas_vg_replication("vg1") + assert repo.list_nas_export_sets() == repo.list_nas_vaults() == [] + assert repo.list_nas_vg_replication() == [] + + +def test_file_spans_and_scratch_thresholds_round_trip(repo: CatalogRepository) -> None: + segments = [ + {"barcode": "A00001L9", "offset": 0, "length": 10}, + {"barcode": "A00002L9", "offset": 0, "length": 5}, + ] + span = repo.upsert_nas_file_span({"path": "/big.bin", "segments": segments}) + repo.upsert_nas_file_span({"path": "/other.bin", "segments": segments[:1]}) + assert repo.list_nas_file_spans("/big.bin") == [ + {"id": span["id"], "path": "/big.bin", "segments": segments} + ] + assert len(repo.list_nas_file_spans()) == 2 + assert repo.delete_nas_file_span(str(span["id"])) + assert repo.get_scratch_thresholds() is None + repo.set_scratch_thresholds(2, 5) + assert repo.get_scratch_thresholds() == {"scratch_min": 2, "scratch_warn": 5} From 9ed7491ac825259515cbbcd5eb9f22ef9efeede1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:09:20 +0000 Subject: [PATCH 137/205] docs(test-plan): add scsi+fake-ltfs to the contract pairing table Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/test-plan.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/test-plan.md b/docs/test-plan.md index c39e9b2..6041607 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -34,6 +34,7 @@ single-job) through `InventoryService`. |---|---|---|---| | `sim+sim` | `MockLibraryBackend` | `MockLTFSBackend` | always | | `emulator+sim-ltfs` | `ScalarHttpLibraryBackend` over in-process ASGI (`openblade.api.main:app`) | `MockLTFSBackend` | always | +| `scsi+fake-ltfs` | `RealLibraryBackend` + `MtxChangerBackend` over a stateful fake `mtx`/`sg_inq` runner (`tests/fakes/mtx.py`) | `MockLTFSBackend` | always | | `real+real` | `get_library()` | `get_ltfs()` | only with `OPENBLADE_REAL_HARDWARE_ENABLED=true` (marker `real_hardware`) | Known contract violations are `xfail(strict=True)` with the defect's file:line, so a From a9311480cca185df9ada54c84cc23285ce689282 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:09:45 +0000 Subject: [PATCH 138/205] fix(mutation): run mutmut's pytest sessions without capture bootstrap binds structlog to import-time sys.stderr; under capture that is a file closed after mutmut's first session, failing the clean-test run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1e1907f..47ccf62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,7 +154,11 @@ only_mutate = [ "openblade/domain/scalar_coordinate.py", "openblade/nas/manifest_validator.py", ] -pytest_add_cli_args = ["-p", "no:cacheprovider", "-q"] +# "-s": mutmut runs several pytest sessions in one process. openblade.bootstrap +# binds structlog to the sys.stderr seen at import (pytest's capture file in +# session 1), which is closed by session 2 -> "I/O operation on closed file" in +# the clean-test run. No capture keeps the stream valid across sessions. +pytest_add_cli_args = ["-p", "no:cacheprovider", "-q", "-s"] # tests/safety/test_assistant_read_only.py is excluded: it does not exercise the # mutated modules and stalled >60s locally (2026-10-09), which blocks mutmut stats. pytest_add_cli_args_test_selection = [ From f7b26a387c6a5de64a4cca1f3bcf87b41637cded Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:09:45 +0000 Subject: [PATCH 139/205] test(mutation): baseline policies.py at 23 survivors (37 mutants, 52 s) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mutation/baseline.txt b/mutation/baseline.txt index 9ae0bc1..8793069 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -1,5 +1,6 @@ # Mutation ratchet: . # Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-09. Only ever lower them. # Modules not yet baselined (policies, safety/*, recovery, manifest_validator) need a first run. +openblade/domain/policies.py 23 openblade/domain/scalar_coordinate.py 4 openblade/jobs/scheduler.py 87 From 2371edbeb3b6a0eb375dc5215635aaed9eb479a5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:09:59 +0000 Subject: [PATCH 140/205] test: fuse cache integrity, restart index, chunked pread Covers sampled-byte flip, truncation and missing sidecar (evict + raise), mount open re-hydrating a corrupted entry instead of EIO, restart byte accounting with/without index.json under max_bytes, corrupt index rebuild, and read_range over a pread that returns <=7 bytes per call. Mutation-checked: verify_entry -> return True fails 4 tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/unit/test_fuse_cache_integrity.py | 195 ++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 tests/unit/test_fuse_cache_integrity.py diff --git a/tests/unit/test_fuse_cache_integrity.py b/tests/unit/test_fuse_cache_integrity.py new file mode 100644 index 0000000..a92faea --- /dev/null +++ b/tests/unit/test_fuse_cache_integrity.py @@ -0,0 +1,195 @@ +"""FUSE hydration cache integrity: open-time verify, persisted LRU index, chunked pread.""" + +from __future__ import annotations + +import hashlib +import os +import threading +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.fuse import cache as cache_mod +from openblade.fuse.cache import SAMPLE_BYTES, CacheIntegrityError, HydrationCache +from openblade.fuse.filesystem import CatalogFilesystem +from openblade.fuse.hydration import HydrationFailedError, Hydrator +from openblade.fuse.mount import CatalogFuseOperations + +BIG = bytes(range(256)) * 1024 # 256 KiB, larger than one sample block +BIG_SUM = hashlib.sha256(BIG).hexdigest() + + +def _rewrite_keep_stat(path: Path, mutate: bytes) -> None: + """Replace file contents but restore mtime_ns, so only the sample hash can notice.""" + st = path.stat() + path.write_bytes(mutate) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns)) + + +def _flip_sampled_byte(cache: HydrationCache, checksum: str) -> None: + path = cache.cache_key(checksum) + data = bytearray(path.read_bytes()) + offset = cache_mod._sample_offset(checksum, len(data)) + data[offset + SAMPLE_BYTES // 2] ^= 0xFF + _rewrite_keep_stat(path, bytes(data)) + + +def test_verify_entry_passes_for_intact_entry(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + assert cache.verify_entry(BIG_SUM) is True + assert cache.is_cached(BIG_SUM) + + +def test_flipped_byte_in_sample_is_detected_and_evicted(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + _flip_sampled_byte(cache, BIG_SUM) + + with pytest.raises(CacheIntegrityError) as exc: + cache.verify_entry(BIG_SUM) + assert str(tmp_path) not in str(exc.value) # curated message, no paths + assert not cache.is_cached(BIG_SUM) + assert not cache.meta_key(BIG_SUM).exists() + assert cache.used_bytes() == 0 + + +def test_truncated_entry_is_detected_and_evicted(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + _rewrite_keep_stat(cache.cache_key(BIG_SUM), BIG[:1000]) + + with pytest.raises(CacheIntegrityError): + cache.verify_entry(BIG_SUM) + assert not cache.is_cached(BIG_SUM) + assert cache.used_bytes() == 0 + + +def test_missing_sidecar_is_treated_as_corrupt(tmp_path: Path) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + cache.meta_key(BIG_SUM).unlink() + with pytest.raises(CacheIntegrityError): + cache.verify_entry(BIG_SUM) + assert not cache.is_cached(BIG_SUM) + + +# -- (c) mount open re-hydrates a corrupted entry instead of EIO --------------- + + +class _Engine: + def __init__(self, staging: Path) -> None: + self.staging = staging + self.batches: list[list[str]] = [] + + def restore_batch(self, tape_key: str, catalog_paths: list[str]) -> dict[str, Path]: + self.batches.append(sorted(catalog_paths)) + out = {} + for path in catalog_paths: + dest = self.staging / path.strip("/") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(BIG) + out[path] = dest + return out + + def find_active_job(self, catalog_path: str) -> str | None: + return None + + def resume(self, job_id: str, catalog_path: str | None = None) -> Path: + raise HydrationFailedError(job_id) + + +def test_mount_open_rehydrates_corrupted_entry(tmp_path: Path) -> None: + init_db(f"sqlite:///{tmp_path / 'cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + rec = repo.create_file_record("/big.bin", len(BIG), BIG_SUM, group.id) + inst = repo.create_file_instance(rec.id, "TAPE01", "/big.bin") + repo.mark_instance_archived(inst.id) + fs = CatalogFilesystem(repo, cache_dir=str(tmp_path / "cache")) + engine = _Engine(tmp_path / "staging") + hyd = Hydrator(repo, fs.cache, engine, batch_window_s=0.01) # type: ignore[arg-type] + ops = CatalogFuseOperations(fs, data_plane=hyd, hydrate_timeout=5.0) + + fh = ops.open("/big.bin", os.O_RDONLY) + ops.release("/big.bin", fh) + assert len(engine.batches) == 1 + + _flip_sampled_byte(fs.cache, BIG_SUM) + + fh = ops.open("/big.bin", os.O_RDONLY) # must not raise EIO + assert ops.read("/big.bin", len(BIG), 0, fh) == BIG + ops.release("/big.bin", fh) + assert len(engine.batches) == 2 # went back through the data plane + + +# -- (d) restart keeps the byte budget honest --------------------------------- + + +def _payloads() -> list[tuple[str, bytes]]: + out = [] + for i in range(3): + data = bytes([i]) * 1000 + out.append((hashlib.sha256(data).hexdigest(), data)) + return out + + +def _on_disk_bytes(cache: HydrationCache) -> int: + return sum(cache.cache_key(c).stat().st_size for c, _ in _payloads() if cache.is_cached(c)) + + +@pytest.mark.parametrize("drop_index", [False, True]) +def test_restart_counts_on_disk_bytes_and_enforces_budget(tmp_path: Path, drop_index: bool) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + if drop_index: + first.index_path.unlink() # scan must still find every file + + reopened = HydrationCache(str(tmp_path)) + assert reopened.used_bytes() == _on_disk_bytes(reopened) == 3000 + + budgeted = HydrationCache(str(tmp_path), max_bytes=1500) + assert budgeted.used_bytes() == _on_disk_bytes(budgeted) <= 1500 + assert budgeted.used_bytes() > 0 + + +def test_corrupt_index_is_rebuilt_from_scan(tmp_path: Path) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + first.index_path.write_text("{not json", encoding="utf-8") + assert HydrationCache(str(tmp_path)).used_bytes() == 3000 + + +def test_index_drops_entries_whose_file_is_gone(tmp_path: Path) -> None: + first = HydrationCache(str(tmp_path)) + for checksum, data in _payloads(): + first.store(checksum, data) + first.cache_key(_payloads()[0][0]).unlink() + assert HydrationCache(str(tmp_path)).used_bytes() == 2000 + + +# -- (e) read_range survives short preads ------------------------------------- + + +def test_read_range_loops_over_short_preads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache = HydrationCache(str(tmp_path)) + cache.store(BIG_SUM, BIG) + real_pread = os.pread + calls = threading.local() + calls.n = 0 + + def short_pread(fd: int, length: int, offset: int) -> bytes: + calls.n += 1 + return real_pread(fd, min(length, 7), offset) + + monkeypatch.setattr(cache_mod.os, "pread", short_pread) + assert cache.read_range(BIG_SUM, 3, 100) == BIG[3:103] + assert calls.n > 1 + # Past EOF: exactly the bytes available. + assert cache.read_range(BIG_SUM, len(BIG) - 10, 100) == BIG[-10:] From 64e1b368d1aefe66617af8e83dc30acd31e87602 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:11:09 +0000 Subject: [PATCH 141/205] test(protocols): un-skip NFS ACL scenario; pin Ganesha 4.3 ACL limit The pinned nfs-ganesha 4.3 VFS FSAL does not advertise the NFSv4 acl attribute (verified: Disable_ACL=false + fresh mount still yields 'attribute not supported'; no Allow_ACL key exists in 4.3). Install nfs4-acl-tools in the client, assert that limit, and assert server-side mode-bit enforcement for a second uid instead of a blanket skip. render_ganesha_conf is unchanged: the ACL knob had no effect. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/Dockerfile | 2 +- tests/e2e/protocols/test_nfs.py | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/deploy/nas-protocols/Dockerfile b/deploy/nas-protocols/Dockerfile index 67dcee6..9b6fa6b 100644 --- a/deploy/nas-protocols/Dockerfile +++ b/deploy/nas-protocols/Dockerfile @@ -5,7 +5,7 @@ ARG ROLE RUN apt-get update \ && if [ "$ROLE" = samba ]; then pkgs="samba smbclient"; \ elif [ "$ROLE" = nfs ]; then pkgs="nfs-ganesha nfs-ganesha-vfs rpcbind dbus"; \ - else pkgs="smbclient cifs-utils nfs-common python3 python3-pip attr procps"; fi \ + else pkgs="smbclient cifs-utils nfs-common nfs4-acl-tools python3 python3-pip attr procps"; fi \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $pkgs \ && apt-get clean \ && if [ "$ROLE" = client ]; then pip3 install --break-system-packages --no-cache-dir smbprotocol==1.15.0; fi diff --git a/tests/e2e/protocols/test_nfs.py b/tests/e2e/protocols/test_nfs.py index a271561..b245d73 100644 --- a/tests/e2e/protocols/test_nfs.py +++ b/tests/e2e/protocols/test_nfs.py @@ -22,7 +22,23 @@ def test_nfs_posix_lock_granted_by_server(mounted: Rig) -> None: assert mounted.py(code).stdout.strip() == "LOCKED" -def test_nfs_acl_mapping() -> None: - pytest.skip( - "NFS ACL mapping needs nfs4_getfacl (nfs4-acl-tools) and Ganesha NFSv4 ACL support; not in rig" - ) +def test_nfs_acl_mapping(mounted: Rig) -> None: + """Access control on the rw export as the rig's Ganesha actually provides it. + + Limit (verified on the pinned debian:bookworm nfs-ganesha 4.3 + nfs-ganesha-vfs build): + the VFS FSAL does not advertise the NFSv4 ``acl`` attribute, even with + ``Disable_ACL = false`` on the export and a fresh client mount -- ``nfs4_getfacl`` + reports "Operation to request attribute not supported" and ``nfs4_setfacl`` fails + with "Failed to instantiate ACL". Ganesha 4.3 has no ``Allow_ACL`` key. So NFSv4 ACEs + cannot be set or read through this server; the test pins that limit (it fails, and + must be rewritten as a set/get round-trip, if a future image starts storing ACLs) and + asserts that POSIX mode bits are enforced server-side for a second uid instead. + """ + f = "/mnt/nfs/rw/acl.bin" + assert mounted.sh(f"echo hi > {f}").returncode == 0 + acl = mounted.sh(f"nfs4_getfacl {f}; nfs4_setfacl -a A::OWNER@:rwatTnNcCy {f}") + assert "attribute not supported" in acl.stdout + acl.stderr, acl.stdout + acl.stderr + assert acl.returncode != 0, "nfs4_setfacl succeeded: server now stores ACLs, extend this test" + assert mounted.sh(f"chmod 600 {f} && runuser -u nobody -- cat {f}").returncode != 0 + allowed = mounted.sh(f"chmod 644 {f} && runuser -u nobody -- cat {f}") + assert allowed.returncode == 0 and allowed.stdout.strip() == "hi", allowed.stderr From 481d35ac399876a23e23a720f8d0136528f01814 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:12:01 +0000 Subject: [PATCH 142/205] feat(nas): catalog-backed ReservationLedger and span-aware restore planning ReservationLedger(catalog) reads/writes through nas_reservations so reservations survive restarts; no-arg construction stays in-memory. RestorePlanner adds every persisted FileSpan segment tape to required_tapes/tape_load_order/batches_by_tape. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/capacity.py | 34 +++++++++++++++++--- openblade/nas/restore_planner.py | 23 ++++++++++++++ tests/unit/test_nas_persistence.py | 51 ++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index 4121990..cadcd47 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -10,11 +10,15 @@ from collections.abc import Iterable, Sequence from datetime import datetime, timedelta from enum import Enum +from typing import TYPE_CHECKING from pydantic import BaseModel, ConfigDict, Field, model_validator from openblade.nas.types import NasFileState +if TYPE_CHECKING: + from openblade.catalog.repository import CatalogRepository + # --- 2. scratch thresholds ------------------------------------------------- @@ -91,13 +95,26 @@ def active(self, now: datetime) -> bool: class ReservationLedger: - """In-memory ledger of capacity reservations per pool.""" + """Ledger of capacity reservations per pool. + + Pure in-memory by default; constructed with ``catalog`` it reads and writes + through the persisted ``nas_reservations`` table so reservations survive restarts. + """ - def __init__(self) -> None: + def __init__(self, catalog: CatalogRepository | None = None) -> None: self._items: list[CapacityReservation] = [] + self._catalog = catalog + + def _active(self, pool_id: str, now: datetime) -> list[CapacityReservation]: + if self._catalog is None: + return [r for r in self._items if r.pool_id == pool_id and r.active(now)] + return [ + CapacityReservation.model_validate(item) + for item in self._catalog.reservations_for_pool(pool_id, now=now) + ] def reserved(self, pool_id: str, now: datetime) -> int: - return sum(r.bytes for r in self._items if r.pool_id == pool_id and r.active(now)) + return sum(r.bytes for r in self._active(pool_id, now)) def available(self, pool_id: str, free_bytes: int, now: datetime) -> int: return max(0, free_bytes - self.reserved(pool_id, now)) @@ -115,10 +132,19 @@ def reserve( if nbytes > self.available(pool_id, free_bytes, now): raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") res = CapacityReservation(pool_id=pool_id, bytes=nbytes, owner=owner, expires_at=now + ttl) - self._items.append(res) + if self._catalog is not None: + self._catalog.upsert_nas_reservation(res.model_dump()) + else: + self._items.append(res) return res def release(self, pool_id: str, owner: str) -> int: + if self._catalog is not None: + removed = 0 + for item in self._catalog.list_nas_reservations(pool_id): + if item["owner"] == owner and self._catalog.delete_nas_reservation(str(item["id"])): + removed += 1 + return removed before = len(self._items) self._items = [r for r in self._items if not (r.pool_id == pool_id and r.owner == owner)] return before - len(self._items) diff --git a/openblade/nas/restore_planner.py b/openblade/nas/restore_planner.py index b53c918..e108927 100644 --- a/openblade/nas/restore_planner.py +++ b/openblade/nas/restore_planner.py @@ -46,6 +46,7 @@ def plan(self, request: RestorePlanRequest) -> RestorePlan: requested_paths = [self._normalize_path(path) for path in request.paths] records = self._resolve_records(request.pool_id, requested_paths) + span_barcodes = self._span_barcodes_by_path() batches_by_tape: dict[str, list[str]] = defaultdict(list) required_counts: dict[str, int] = defaultdict(int) @@ -75,6 +76,14 @@ def plan(self, request: RestorePlanRequest) -> RestorePlan: ): required_counts[record.tape_barcode] += 1 batches_by_tape[record.tape_barcode].append(logical_path) + # A file spanning several tapes needs every segment's tape loaded. + for barcode in span_barcodes.get(logical_path, []): + if ( + barcode != record.tape_barcode + and logical_path not in batches_by_tape[barcode] + ): + required_counts[barcode] += 1 + batches_by_tape[barcode].append(logical_path) elif state is NasFileState.MISSING_TAPE: missing_tapes.add(record.tape_barcode or "") elif state is NasFileState.EXPORTED and record.tape_barcode: @@ -180,6 +189,20 @@ def _build_tape_load_order( return sorted(required_counts, key=lambda barcode: (-required_counts[barcode], barcode)) + def _span_barcodes_by_path(self) -> dict[str, list[str]]: + """Persisted multi-tape spans, keyed by normalized path, barcodes in segment order.""" + spans: dict[str, list[str]] = {} + for span in self.service.repository.list_nas_file_spans(): + segments = span.get("segments") + if not isinstance(segments, list): + continue + barcodes = spans.setdefault(self._normalize_path(str(span["path"])), []) + for segment in segments: + barcode = segment.get("barcode") if isinstance(segment, dict) else None + if isinstance(barcode, str) and barcode and barcode not in barcodes: + barcodes.append(barcode) + return spans + def _normalize_path(self, path: str) -> str: normalized = str(PurePosixPath("/" + str(path or "").lstrip("/"))).lstrip("/") return "" if normalized == "." else normalized diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py index a431a25..3fffb1f 100644 --- a/tests/unit/test_nas_persistence.py +++ b/tests/unit/test_nas_persistence.py @@ -10,6 +10,10 @@ from openblade.catalog.db import _migrate_schema, get_session, init_db from openblade.catalog.repository import CatalogRepository +from openblade.nas.capacity import ReservationError, ReservationLedger +from openblade.nas.restore_planner import RestorePlanner +from openblade.nas.types import RestorePlanRequest +from tests.unit.test_nas_restore_planner import make_nas_service, seed_dataset, seed_file, seed_pool @pytest.fixture @@ -148,3 +152,50 @@ def test_file_spans_and_scratch_thresholds_round_trip(repo: CatalogRepository) - assert repo.get_scratch_thresholds() is None repo.set_scratch_thresholds(2, 5) assert repo.get_scratch_thresholds() == {"scratch_min": 2, "scratch_warn": 5} + + +def test_ledger_reservation_survives_new_ledger_instance(repo: CatalogRepository) -> None: + ReservationLedger(repo).reserve( + "p1", 400, "job-1", timedelta(hours=1), free_bytes=1_000, now=_NOW + ) + fresh = ReservationLedger(repo) + assert fresh.reserved("p1", _NOW) == 400 + assert fresh.available("p1", 1_000, _NOW) == 600 + with pytest.raises(ReservationError): + fresh.reserve("p1", 700, "job-2", timedelta(hours=1), free_bytes=1_000, now=_NOW) + assert fresh.reserved("p1", _NOW + timedelta(hours=2)) == 0 # expired + assert fresh.release("p1", "job-1") == 1 + assert ReservationLedger(repo).reserved("p1", _NOW) == 0 + + +def test_in_memory_ledger_does_not_touch_catalog(repo: CatalogRepository) -> None: + ReservationLedger().reserve("p1", 10, "job", timedelta(hours=1), free_bytes=100, now=_NOW) + assert repo.list_nas_reservations() == [] + + +def test_spanning_file_requires_every_segment_tape(tmp_path: Path) -> None: + service = make_nas_service(tmp_path) + pool = seed_pool(service) + dataset = seed_dataset(service, pool_id=pool.id) + seed_file( + service, + dataset_id=dataset.id, + pool_id=pool.id, + relative_path="video/big.mkv", + tape_barcode="VOL001L9", + ) + service.repository.upsert_nas_file_span( + { + "path": "video/big.mkv", + "segments": [ + {"barcode": "VOL001L9", "offset": 0, "length": 6}, + {"barcode": "VOL002L9", "offset": 0, "length": 4}, + ], + } + ) + + plan = RestorePlanner(service).plan(RestorePlanRequest(pool_id=pool.id)) + + assert set(plan.required_tapes) == {"VOL001L9", "VOL002L9"} + assert set(plan.tape_load_order) == {"VOL001L9", "VOL002L9"} + assert plan.batches_by_tape["VOL002L9"] == ["video/big.mkv"] From 3789b9ab54e1015cbab8a4b0f7adf297692f9b8f Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:12:01 +0000 Subject: [PATCH 143/205] test(reconcile): pending set, scheduler skip/refuse, load refusal, API; ADR note Mutation-checked: removing the scheduler exclusion or the _load_if_needed guard fails the corresponding tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 16 +++ tests/unit/test_reconcile.py | 110 ++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 tests/unit/test_reconcile.py diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index efccf89..bfd02e3 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -223,3 +223,19 @@ Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any `tests/unit tests/integration tests/safety` green, frontend untouched, reviewer subagent pass, one commit per logical change (models+repo, scheduler, fencing at call sites, recovery+route, docs). + +## Reconciliation + +A failed unmount/unload journals `physical_state_unknown` (`{"op","barcode","drive"}`): +the drive may still hold a possibly LTFS-mounted tape. `openblade/jobs/reconcile.py` +derives the pending set across all jobs (`CatalogRepository.journal_events`): a drive is +pending while its latest `physical_state_unknown` has no later `drive_reconciled` entry. + +- `DriveScheduler.acquire_drives` excludes pending drives from its candidates and raises + `DriveUnreconciledError` only when too few reconciled drives remain. +- The restore / sharded-archive / sharded-restore load helpers refuse a pending drive. +- `recover_after_restart` lists pending drives in `RecoveryReport.pending_reconciliation` + (report only; nothing is auto-reconciled). +- `POST /jobs/recovery/reconcile/{drive_id}` takes a fresh inventory snapshot, refuses + (409) while the drive still reports a mount, else journals `drive_reconciled` + (`{"drive","observed_barcode"}`); 404 for an unknown drive. diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py new file mode 100644 index 0000000..002c659 --- /dev/null +++ b/tests/unit/test_reconcile.py @@ -0,0 +1,110 @@ +"""Drives with unknown physical state (failed unmount/unload) must be reconciled before reuse.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from openblade.api import routes_jobs +from openblade.bootstrap import AppContext, create_context, get_context +from openblade.config import OpenBladeConfig +from openblade.domain.errors import DriveUnreconciledError +from openblade.domain.models import MountState +from openblade.jobs.reconcile import ( + DRIVE_RECONCILED, + PHYSICAL_STATE_UNKNOWN, + drives_pending_reconciliation, + reconcile_drive, +) +from openblade.jobs.restore import _load_if_needed +from openblade.jobs.scheduler import DriveHandle, DriveScheduler, InMemoryLeaseStore +from openblade.simulator.library import MockLibraryBackend + + +def _context(tmp_path: Path) -> AppContext: + return create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + + +def _mark_unknown(context: AppContext, drive: int | None, barcode: str = "MCK00001") -> None: + context.catalog.journal( + "job-1", PHYSICAL_STATE_UNKNOWN, {"op": "unload", "barcode": barcode, "drive": drive} + ) + + +def test_pending_set_tracks_unknown_and_reconciled(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, 1) + _mark_unknown(context, None) # no drive named: ignored + pending = drives_pending_reconciliation(context.catalog) + assert set(pending) == {1} + assert pending[1].barcode == "MCK00001" + assert pending[1].op == "unload" + assert pending[1].job_id == "job-1" + context.catalog.journal("recovery", DRIVE_RECONCILED, {"drive": 1}) + assert drives_pending_reconciliation(context.catalog) == {} + _mark_unknown(context, 1) # a later failure makes it pending again + assert set(drives_pending_reconciliation(context.catalog)) == {1} + + +def test_scheduler_skips_unreconciled_drive_and_refuses_when_none_left() -> None: + store = InMemoryLeaseStore() + store.unreconciled = {0: "MCK00001"} + scheduler = DriveScheduler(2, store=store) + [handle] = scheduler.acquire_drives(["MCK00002"], timeout=0.1) + assert handle.drive_id == 1 + scheduler.release_drives([handle]) + + store.unreconciled = {0: "MCK00001", 1: None} + with pytest.raises(DriveUnreconciledError, match="drive 0 .*MCK00001"): + scheduler.acquire_drives(["MCK00002"], timeout=0.1) + + +def test_load_if_needed_refuses_unreconciled_drive(tmp_path: Path) -> None: + context = _context(tmp_path) + barcode = next(str(slot.barcode) for slot in context.library.inventory().slots if slot.barcode) + _mark_unknown(context, 0, barcode="MCK99999") + handle = DriveHandle(drive_id=0, barcode=barcode) + with pytest.raises(DriveUnreconciledError, match="Drive 0"): + _load_if_needed(context.catalog, context.library, context.ltfs, handle, "job-2") + assert context.library.find_drive_by_barcode(barcode) is None # nothing was loaded + + +def test_reconcile_drive_refuses_while_mounted(tmp_path: Path) -> None: + context = _context(tmp_path) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + _mark_unknown(context, 1) + library._drives[1].mount_state = MountState.MOUNTED_RW + with pytest.raises(DriveUnreconciledError, match="still mounted_rw"): + reconcile_drive(context.catalog, library, 1) + assert set(drives_pending_reconciliation(context.catalog)) == {1} + + library._drives[1].mount_state = MountState.UNMOUNTED + done = reconcile_drive(context.catalog, library, 1) + assert done.drive_id == 1 + assert done.op == DRIVE_RECONCILED + assert drives_pending_reconciliation(context.catalog) == {} + + +def test_reconcile_api_round_trip(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, 1) + app = FastAPI() + app.include_router(routes_jobs.router, prefix="/jobs") + app.dependency_overrides[get_context] = lambda: context + client = TestClient(app) + + context.library._drives[1].mount_state = MountState.MOUNTED_RO # type: ignore[attr-defined] + conflict = client.post("/jobs/recovery/reconcile/1") + assert conflict.status_code == 409 + assert "Drive 1" in conflict.json()["detail"] + + context.library._drives[1].mount_state = MountState.UNMOUNTED # type: ignore[attr-defined] + ok = client.post("/jobs/recovery/reconcile/1") + assert ok.status_code == 200 + assert ok.json()["drive"] == 1 + assert drives_pending_reconciliation(context.catalog) == {} + + assert client.post("/jobs/recovery/reconcile/99").status_code == 404 From e6621bcdfe6d3b95ce9c465324dcaf4a62714277 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:12:35 +0000 Subject: [PATCH 144/205] feat(api): expose pool replication/protection and NAS reservation, export-set, vault lists GET /nas/reservations?pool_id=, /nas/export-sets, /nas/vaults (read-only, same router/deps as neighbouring NAS routes). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 22 ++++++++++++++++++++++ tests/unit/test_nas_persistence.py | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index b687e08..b7a7aca 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -879,6 +879,28 @@ async def get_pool(pool_id: str, service: NasService = Depends(get_nas_service)) return pool +@router.get("/reservations") +async def list_reservations( + pool_id: str | None = None, + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_reservations(pool_id) + + +@router.get("/export-sets") +async def list_export_sets( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_export_sets() + + +@router.get("/vaults") +async def list_vaults( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> list[dict[str, object]]: + return repo.list_nas_vaults() + + @router.put("/pools/{pool_id}", response_model=NasPool) async def update_pool( pool_id: str, diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py index 3fffb1f..fe1ca6f 100644 --- a/tests/unit/test_nas_persistence.py +++ b/tests/unit/test_nas_persistence.py @@ -199,3 +199,30 @@ def test_spanning_file_requires_every_segment_tape(tmp_path: Path) -> None: assert set(plan.required_tapes) == {"VOL001L9", "VOL002L9"} assert set(plan.tape_load_order) == {"VOL001L9", "VOL002L9"} assert plan.batches_by_tape["VOL002L9"] == ["video/big.mkv"] + + +def test_api_exposes_pool_protection_and_persisted_lists(tmp_path: Path) -> None: + from fastapi.testclient import TestClient + + from openblade.api.main import app + from openblade.bootstrap import create_context, get_context, reset_context + from openblade.config import OpenBladeConfig + + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'api.db'}"))) + catalog = get_context().catalog + catalog.upsert_nas_pool({"id": "p1", "name": "pool-a", "replication_factor": 2}) + catalog.upsert_nas_reservation( + {"pool_id": "p1", "bytes": 5, "owner": "job", "expires_at": _NOW} + ) + catalog.upsert_nas_export_set({"id": "e1", "barcodes": ["A"], "state": "out"}) + catalog.upsert_nas_vault({"id": "v1", "barcodes": ["B"], "location": "offsite"}) + client = TestClient(app) + + pool = client.get("/nas/pools/p1").json() + assert pool["replication_factor"] == 2 + assert "protection" in pool + reservations = client.get("/nas/reservations", params={"pool_id": "p1"}).json() + assert [item["owner"] for item in reservations] == ["job"] + assert client.get("/nas/reservations", params={"pool_id": "other"}).json() == [] + assert client.get("/nas/export-sets").json()[0]["id"] == "e1" + assert client.get("/nas/vaults").json()[0]["location"] == "offsite" From ba1a22fdf6ea77ba7aa03961d509e950100a6f44 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:12:40 +0000 Subject: [PATCH 145/205] test(mutation): baseline recovery.py at 43 survivors (134 mutants, 129 s) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mutation/baseline.txt b/mutation/baseline.txt index 8793069..2bfd9a7 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -3,4 +3,5 @@ # Modules not yet baselined (policies, safety/*, recovery, manifest_validator) need a first run. openblade/domain/policies.py 23 openblade/domain/scalar_coordinate.py 4 +openblade/jobs/recovery.py 43 openblade/jobs/scheduler.py 87 From 0aac146bf56f2bba6c10645cce92cbec2c8a34be Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:12:49 +0000 Subject: [PATCH 146/205] JobQueue: drive and changer claims are catalog leases claim_drive/claim_changer now take a lease through the LeaseStore, so ownership holds across processes sharing the catalog. The store allocates the lowest free ids and cannot target one, so a claim requests every free id up to the target in one BEGIN IMMEDIATE transaction, keeps the target and releases the probes, retrying briefly under contention. Changer ownership is a lease on the reserved pseudo-drive CHANGER_DRIVE_ID = 64. In-memory owner maps removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/queue.py | 107 +++++++++++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/openblade/jobs/queue.py b/openblade/jobs/queue.py index a235e55..7d5d6dc 100644 --- a/openblade/jobs/queue.py +++ b/openblade/jobs/queue.py @@ -3,6 +3,7 @@ import json import logging import threading +import time from collections.abc import Callable from datetime import timedelta, timezone from typing import TypeVar @@ -15,13 +16,41 @@ JobNotFoundError, safe_job_error, ) -from openblade.domain.models import Job, JobState, JobType +from openblade.domain.models import DriveLease, Job, JobState, JobType from openblade.jobs.scheduler import DEFAULT_LEASE_TTL, LeaseStore ResultT = TypeVar("ResultT") logger = logging.getLogger(__name__) +# Changer ownership is a lease on this reserved pseudo-drive id. The lease store +# only allocates ids in range(num_drives) (no negatives), and claiming a slot +# probes every free id below it, so the sentinel is kept small: it must exceed +# any real drive count (a Scalar i3 tops out well below it) but stay cheap. +CHANGER_DRIVE_ID = 64 +_CHANGER_KIND = "CHANGER" +_DRIVE_KIND = "QUEUE-CLAIM" +_PROBE_BARCODE = "QUEUE-PROBE" +_CLAIM_ATTEMPTS = 50 +_CLAIM_RETRY_SECONDS = 0.01 + + +def _is_transient(lease: DriveLease) -> bool: + """True for another claimer's in-flight lease, which never denotes ownership. + + Claim barcodes name their target (``KIND:slot``). When a claimer's snapshot + is stale the store can hand its claim barcode to a lower id; that lease, like + every probe, is released immediately and must not read as an owner. + """ + if lease.barcode == _PROBE_BARCODE: + return True + kind, sep, _ = lease.barcode.partition(":") + return ( + bool(sep) + and kind in (_DRIVE_KIND, _CHANGER_KIND) + and lease.barcode != f"{kind}:{lease.drive_id}" + ) + def _job_from_row(row: catalog_models.Job) -> Job: return Job( @@ -48,9 +77,9 @@ def __init__( self._catalog = catalog self._lease_store = lease_store self._ttl = ttl + # Guards the shared catalog session only; cross-process ownership is the + # catalog's BEGIN IMMEDIATE inside lease acquisition. self._lock = threading.RLock() - self._drive_owners: dict[int, str] = {} - self._changer_owner: str | None = None def create_job(self, job_type: JobType, metadata: dict[str, object]) -> Job: with self._lock: @@ -73,28 +102,70 @@ def update_job( self._catalog.update_job_state(job_id, (state or job.state).value, error) return self.get_job(job_id) + def _claim_slot( + self, slot: int, job_id: str, kind: str, busy: Callable[[str], Exception] + ) -> None: + for _attempt in range(_CLAIM_ATTEMPTS): + live = self._lease_store.live_leases() + for lease in live: + if lease.drive_id == slot and not _is_transient(lease): + if lease.job_id == job_id: + return + raise busy(lease.job_id) + # The store allocates the lowest free ids in range(num_drives); it + # cannot target one id. So request every free id up to and including + # ``slot`` in one all-or-nothing transaction, keep ``slot``, and + # release the probes. A concurrent claimer's probes make this miss; + # re-check and retry until ``slot`` has a real owner or is ours. + taken = {lease.drive_id for lease in live} + probes = sum(1 for d in range(slot) if d not in taken) + leases = self._lease_store.acquire( + job_id=job_id, + barcodes=[_PROBE_BARCODE] * probes + [f"{kind}:{slot}"], + num_drives=slot + 1, + ttl=self._ttl, + ) + if leases is not None: + extra = [lease.id for lease in leases if lease.drive_id != slot] + if extra: + self._lease_store.release(extra) + if len(extra) < len(leases): + return + time.sleep(_CLAIM_RETRY_SECONDS) + raise busy("another job (claim contended)") + + def _release_slot(self, slot: int, job_id: str) -> None: + mine = [ + lease.id + for lease in self._lease_store.live_leases() + if lease.drive_id == slot and lease.job_id == job_id + ] + if mine: + self._lease_store.release(mine) + def claim_drive(self, drive_id: int, job_id: str) -> None: - with self._lock: - owner = self._drive_owners.get(drive_id) - if owner is not None and owner != job_id: - raise DriveOccupiedError(f"Drive {drive_id} is already owned by job {owner}") - self._drive_owners[drive_id] = job_id + if drive_id < 0 or drive_id >= CHANGER_DRIVE_ID: + raise ValueError(f"drive id {drive_id} out of range") + self._claim_slot( + drive_id, + job_id, + _DRIVE_KIND, + lambda owner: DriveOccupiedError(f"Drive {drive_id} is already owned by job {owner}"), + ) def release_drive(self, drive_id: int, job_id: str) -> None: - with self._lock: - if self._drive_owners.get(drive_id) == job_id: - del self._drive_owners[drive_id] + self._release_slot(drive_id, job_id) def claim_changer(self, job_id: str) -> None: - with self._lock: - if self._changer_owner is not None and self._changer_owner != job_id: - raise ChangerBusyError(f"Changer owned by job {self._changer_owner}") - self._changer_owner = job_id + self._claim_slot( + CHANGER_DRIVE_ID, + job_id, + _CHANGER_KIND, + lambda owner: ChangerBusyError(f"Changer owned by job {owner}"), + ) def release_changer(self, job_id: str) -> None: - with self._lock: - if self._changer_owner == job_id: - self._changer_owner = None + self._release_slot(CHANGER_DRIVE_ID, job_id) def run_job(self, job: Job, func: Callable[[], ResultT]) -> tuple[Job, ResultT]: self.update_job(job.id, state=JobState.RUNNING) From 54b414a9dd8a535630c0fb5f9f8cb6b6dc223bf1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:13:53 +0000 Subject: [PATCH 147/205] Test JobQueue jobs and claims across two catalog sessions Two JobQueue instances over two CatalogRepository sessions on one sqlite file: job visibility, drive and changer exclusivity, release-then-claim, and run_job state transitions observed from the other process mid-run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/unit/test_job_queue_persistent.py | 85 +++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/unit/test_job_queue_persistent.py diff --git a/tests/unit/test_job_queue_persistent.py b/tests/unit/test_job_queue_persistent.py new file mode 100644 index 0000000..1a8998f --- /dev/null +++ b/tests/unit/test_job_queue_persistent.py @@ -0,0 +1,85 @@ +"""JobQueue state is shared by every process using the same catalog file.""" + +from __future__ import annotations + +import threading +from pathlib import Path + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import ChangerBusyError, DriveOccupiedError +from openblade.domain.models import JobState, JobType +from openblade.jobs.queue import JobQueue +from openblade.jobs.scheduler import CatalogLeaseStore + + +@pytest.fixture +def queues(tmp_path: Path) -> tuple[JobQueue, JobQueue]: + # Two sessions on one file DB model two processes sharing one catalog. + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + return JobQueue(repo_a, CatalogLeaseStore(repo_a)), JobQueue(repo_b, CatalogLeaseStore(repo_b)) + + +def test_job_created_by_one_process_is_visible_to_another( + queues: tuple[JobQueue, JobQueue], +) -> None: + a, b = queues + job = a.create_job(JobType.ARCHIVE, {"barcode": "TAPE01L8"}) + seen = b.get_job(job.id) + assert seen.job_type is JobType.ARCHIVE + assert seen.state is JobState.PENDING + assert seen.metadata == {"barcode": "TAPE01L8"} + + +def test_drive_claim_is_exclusive_across_processes(queues: tuple[JobQueue, JobQueue]) -> None: + a, b = queues + job_a = a.create_job(JobType.ARCHIVE, {}) + job_b = b.create_job(JobType.ARCHIVE, {}) + a.claim_drive(0, job_a.id) + with pytest.raises(DriveOccupiedError): + b.claim_drive(0, job_b.id) + b.claim_drive(1, job_b.id) + with pytest.raises(DriveOccupiedError): + a.claim_drive(1, job_a.id) + a.release_drive(0, job_a.id) + b.claim_drive(0, job_b.id) + + +def test_changer_claim_is_exclusive_across_processes(queues: tuple[JobQueue, JobQueue]) -> None: + a, b = queues + job_a = a.create_job(JobType.ARCHIVE, {}) + job_b = b.create_job(JobType.ARCHIVE, {}) + a.claim_changer(job_a.id) + with pytest.raises(ChangerBusyError): + b.claim_changer(job_b.id) + # The changer pseudo-drive does not occupy a real drive. + b.claim_drive(0, job_b.id) + a.release_changer(job_a.id) + b.claim_changer(job_b.id) + + +def test_run_job_transitions_are_visible_from_another_process( + queues: tuple[JobQueue, JobQueue], +) -> None: + a, b = queues + job = a.create_job(JobType.ARCHIVE, {}) + running = threading.Event() + proceed = threading.Event() + + def func() -> str: + running.set() + assert proceed.wait(timeout=5) + return "done" + + results: list[str] = [] + worker = threading.Thread(target=lambda: results.append(a.run_job(job, func)[1])) + worker.start() + assert running.wait(timeout=5) + assert b.get_job(job.id).state is JobState.RUNNING + proceed.set() + worker.join(timeout=5) + assert results == ["done"] + assert b.get_job(job.id).state is JobState.COMPLETED From 6d50b4511eb62934b13bf5009d6e2ad7bee42e54 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:14:02 +0000 Subject: [PATCH 148/205] Docs: record the JobQueue persistent-lease follow-up Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index efccf89..b175068 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -223,3 +223,22 @@ Item 4's STAGING/VERIFYING states, job resume, replacing `JobQueue`, any `tests/unit tests/integration tests/safety` green, frontend untouched, reviewer subagent pass, one commit per logical change (models+repo, scheduler, fencing at call sites, recovery+route, docs). + +## JobQueue follow-up landed + +The `JobQueue` deferral above (and the "replacing `JobQueue`" non-goal) is now +done as a persistent façade, not a replacement: +`JobQueue(catalog, lease_store, *, ttl=DEFAULT_LEASE_TTL)` keeps jobs only in the +catalog `jobs` table (`run_job` persists RUNNING before the work and +COMPLETED/FAILED after, error text still via `safe_job_error`), and drive/changer +ownership is a lease in the shared lease table, so it holds across processes. +`LeaseStore.acquire` allocates the lowest free ids and cannot target one, so a +claim requests every free id up to the target in one `BEGIN IMMEDIATE` +transaction, keeps the target, releases the probe leases, and retries briefly +under contention. Changer ownership is a lease on the reserved pseudo-drive +`CHANGER_DRIVE_ID = 64` (negative ids are unreachable through `range(num_drives)`; +a large sentinel would insert one probe row per lower id on every claim). +Known costs: each claim of id *k* writes up to *k* released probe rows; claims +are not heartbeated, so they lapse after the TTL. A repository method that +leases one specific drive id would remove both the probes and the retry loop. +Covered by `tests/unit/test_job_queue_persistent.py` (two sessions, one file). From e691a1798e1100b1d308cc63ad1da50b845b7195 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:14:07 +0000 Subject: [PATCH 149/205] test(mutation): baseline safety/import_guard.py at 20 survivors (86 mutants, 46 s) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mutation/baseline.txt b/mutation/baseline.txt index 2bfd9a7..ec0ae8e 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -5,3 +5,4 @@ openblade/domain/policies.py 23 openblade/domain/scalar_coordinate.py 4 openblade/jobs/recovery.py 43 openblade/jobs/scheduler.py 87 +openblade/safety/import_guard.py 20 From b6ba8655e1e0864585de71c3053f6da25a671b04 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:15:05 +0000 Subject: [PATCH 150/205] test(mutation): baseline nas/manifest_validator.py at 74 survivors (255 mutants, 43 s) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mutation/baseline.txt b/mutation/baseline.txt index ec0ae8e..2f5e196 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -5,4 +5,5 @@ openblade/domain/policies.py 23 openblade/domain/scalar_coordinate.py 4 openblade/jobs/recovery.py 43 openblade/jobs/scheduler.py 87 +openblade/nas/manifest_validator.py 74 openblade/safety/import_guard.py 20 From f36db679d840fce6bef2787cd3c2b2be2a5dffe0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:17:36 +0000 Subject: [PATCH 151/205] test(mutation): kill 32 survivors in policies/recovery, ratchet 23->2, 43->32 policies: assert token fields/length, exact-expiry boundary, curated operator messages for expiry, barcode mismatch and real-hardware refusal. Remaining 2 include token_urlsafe(None), equivalent (secrets.DEFAULT_ENTROPY == 32). recovery: expired lease of a non-running owner is untouched (and->or guard), two expired leases fail the job once with one journal entry + exact payload, _is_older_than TTL boundary and aware->UTC normalisation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- mutation/baseline.txt | 4 +- pyproject.toml | 2 + tests/unit/test_policies_guards.py | 68 +++++++++++++++++++++++ tests/unit/test_recovery_guards.py | 89 ++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_policies_guards.py create mode 100644 tests/unit/test_recovery_guards.py diff --git a/mutation/baseline.txt b/mutation/baseline.txt index 2f5e196..2f90b01 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -1,9 +1,9 @@ # Mutation ratchet: . # Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-09. Only ever lower them. # Modules not yet baselined (policies, safety/*, recovery, manifest_validator) need a first run. -openblade/domain/policies.py 23 +openblade/domain/policies.py 2 openblade/domain/scalar_coordinate.py 4 -openblade/jobs/recovery.py 43 +openblade/jobs/recovery.py 32 openblade/jobs/scheduler.py 87 openblade/nas/manifest_validator.py 74 openblade/safety/import_guard.py 20 diff --git a/pyproject.toml b/pyproject.toml index 47ccf62..f276bef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,4 +173,6 @@ pytest_add_cli_args_test_selection = [ "tests/unit/test_safety_gates.py", "tests/unit/test_hardware_safety.py", "tests/unit/test_capacity_policy.py", + "tests/unit/test_policies_guards.py", + "tests/unit/test_recovery_guards.py", ] diff --git a/tests/unit/test_policies_guards.py b/tests/unit/test_policies_guards.py new file mode 100644 index 0000000..c6f4a55 --- /dev/null +++ b/tests/unit/test_policies_guards.py @@ -0,0 +1,68 @@ +"""Pin the safety-token / format-confirmation / real-hardware guards in policies.py. + +Written to kill surviving mutmut mutants: every assertion here fails when the +corresponding guard, field or curated operator message is mutated. +""" + +from __future__ import annotations + +import pytest + +from openblade.domain import policies +from openblade.domain.errors import ( + BarcodeMismatchError, + RealHardwareDisabledError, + SafetyViolationError, +) +from openblade.domain.policies import FormatConfirmation, RealHardwareGuard, SafetyToken + + +def test_generate_binds_operation_barcode_and_a_32_byte_token() -> None: + tok = SafetyToken.generate("format", "ABC123L6") + assert tok.operation == "format" + assert tok.target_barcode == "ABC123L6" + assert isinstance(tok.token, str) + # secrets.token_urlsafe(32) -> 32 bytes base64url without padding = 43 chars. + assert len(tok.token) == 43 + assert tok.token != SafetyToken.generate("format", "ABC123L6").token + + +def test_token_is_invalid_at_exact_expiry_instant(monkeypatch: pytest.MonkeyPatch) -> None: + tok = SafetyToken.generate("format", "ABC123L6", ttl_seconds=10) + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at) + assert tok.is_valid() is False + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at - 0.001) + assert tok.is_valid() is True + + +def test_expired_token_raises_curated_message(monkeypatch: pytest.MonkeyPatch) -> None: + tok = SafetyToken.generate("format", "ABC123L6", ttl_seconds=10) + monkeypatch.setattr(policies.time, "time", lambda: tok.expires_at + 1) + with pytest.raises(SafetyViolationError) as exc: + tok.validate() + assert str(exc.value) == "Safety token has expired" + + +def test_barcode_mismatch_names_both_barcodes() -> None: + conf = FormatConfirmation("ABC123L6", SafetyToken.generate("format", "ABC123L6")) + conf.validate("ABC123L6") + with pytest.raises(BarcodeMismatchError) as exc: + conf.validate("XYZ999L6") + assert str(exc.value) == "Expected barcode 'ABC123L6' but got 'XYZ999L6'" + + +@pytest.mark.parametrize(("backend", "enabled"), [("sim", True), ("real", False)]) +def test_real_hardware_disabled_message_names_both_settings(backend: str, enabled: bool) -> None: + with pytest.raises(RealHardwareDisabledError) as exc: + RealHardwareGuard(backend, enabled, "operator ok").validate() + assert str(exc.value) == ( + "Real hardware operations require OPENBLADE_BACKEND=real and " + "OPENBLADE_REAL_HARDWARE_ENABLED=true" + ) + + +def test_real_hardware_requires_operator_acknowledgment() -> None: + RealHardwareGuard("real", True, "operator ok").validate() + with pytest.raises(RealHardwareDisabledError) as exc: + RealHardwareGuard("real", True, "").validate() + assert str(exc.value) == "Operator acknowledgment is required" diff --git a/tests/unit/test_recovery_guards.py b/tests/unit/test_recovery_guards.py new file mode 100644 index 0000000..f16011c --- /dev/null +++ b/tests/unit/test_recovery_guards.py @@ -0,0 +1,89 @@ +"""Pin recovery guards that mutation testing showed were unasserted. + +Each test fails when the corresponding guard in openblade/jobs/recovery.py is +mutated (and->or on the owner check, the once-per-job dedupe, the journal +payload, the stale-age boundary and its timezone normalisation). +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path + +from openblade.bootstrap import create_context +from openblade.catalog.repository import CatalogRepository +from openblade.config import OpenBladeConfig +from openblade.jobs import recovery +from openblade.jobs.recovery import INTERRUPTED_ERROR, recover_after_restart +from openblade.simulator.library import MockLibraryBackend + +EXPIRED = timedelta(seconds=-1) + + +def _catalog(tmp_path: Path) -> CatalogRepository: + return create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")).catalog + + +def _library() -> MockLibraryBackend: + return MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + +def test_recovery_expired_lease_of_pending_job_leaves_job_untouched(tmp_path: Path) -> None: + catalog = _catalog(tmp_path) + job = catalog.create_job("archive", {}) + leases = catalog.acquire_drive_leases( + job_id=job.id, barcodes=["MCK00001"], num_drives=2, ttl=EXPIRED + ) + assert leases is not None + + report = recover_after_restart(catalog, _library()) + + assert report.interrupted_job_ids == [] + refreshed = catalog.get_job(job.id) + assert refreshed is not None + assert refreshed.state == "pending" + assert [e.event for e in catalog.job_journal(job.id)].count("recovered") == 0 + + +def test_recovery_job_with_two_expired_leases_is_failed_once(tmp_path: Path) -> None: + catalog = _catalog(tmp_path) + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + leases = catalog.acquire_drive_leases( + job_id=job.id, barcodes=["MCK00001", "MCK00002"], num_drives=2, ttl=EXPIRED + ) + assert leases is not None + assert len(leases) == 2 + + report = recover_after_restart(catalog, _library()) + + assert report.interrupted_job_ids == [job.id] + recovered = [e for e in catalog.job_journal(job.id) if e.event == "recovered"] + assert len(recovered) == 1 + assert json.loads(recovered[0].detail_json) == { + "lease_id": leases[0].id, + "error": INTERRUPTED_ERROR, + } + + +def test_is_older_than_exact_ttl_boundary_is_not_stale() -> None: + now = datetime(2026, 10, 10, 12, 0, 0) + ttl = timedelta(minutes=15) + + assert recovery._is_older_than(now - ttl, ttl, now) is False + assert recovery._is_older_than(now - ttl - timedelta(microseconds=1), ttl, now) is True + + +def test_is_older_than_normalises_aware_timestamp_to_utc() -> None: + now = datetime(2026, 10, 10, 12, 0, 0) + ttl = timedelta(minutes=15) + plus_five = timezone(timedelta(hours=5)) + # 16:50+05:00 == 11:50 UTC: 10 min old -> fresh. Read naively it is in the future. + fresh = datetime(2026, 10, 10, 16, 50, tzinfo=plus_five) + # 16:40+05:00 == 11:40 UTC: 20 min old -> stale. Read naively (16:40) it is fresh. + stale = datetime(2026, 10, 10, 16, 40, tzinfo=plus_five) + + assert recovery._is_older_than(fresh, ttl, now) is False + assert recovery._is_older_than(stale, ttl, now) is True + assert recovery._is_older_than(datetime(2026, 10, 10, 11, 50, tzinfo=UTC), ttl, now) is False From 392ff62f5429db682cc78bf3124e8ad690fc684d Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:17:43 +0000 Subject: [PATCH 152/205] docs(mutation): 6/6 modules baselined (header + roadmap item 16) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/production-readiness-roadmap.md | 2 +- mutation/baseline.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index 9562099..4ff3b57 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -48,7 +48,7 @@ Verified against the tree, not asserted. "Landed" = implemented and tested in-re | 13 | Consolidated emulator workflows | landed | `_emulator-boot-test.yml` | | 14 | Lint/type ratchet | landed (both blocking, both at zero) | `ci.yml` | | 15 | Workflow hardening | landed: SHA pins, per-job permissions, actionlint/zizmor/shellcheck/yamllint | `workflow-lint.yml` | -| 16 | Differential + mutation | mutation lane landed (mutmut, ratchet baseline for 2 modules); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | +| 16 | Differential + mutation | mutation lane landed (mutmut 3.8.0, ratchet baseline for 6/6 modules as of 2026-10-10); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | | 17 | Timing profiles + virtual clock | landed (10 profiles; 4 simulated client-side) | `tests/i3/timing.py` | | 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-required pending state not modelled | `routes_aml_auth.py`, `compatibility/` | | 19 | Real SMB/NFS protocol tests | landed (containerised rig, 31 scenarios pass locally; SMB byte-range lock + NFS ACL skip) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | diff --git a/mutation/baseline.txt b/mutation/baseline.txt index 2f90b01..eb8cf2c 100644 --- a/mutation/baseline.txt +++ b/mutation/baseline.txt @@ -1,6 +1,6 @@ # Mutation ratchet: . -# Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-09. Only ever lower them. -# Modules not yet baselined (policies, safety/*, recovery, manifest_validator) need a first run. +# Real counts from tools/mutation_run.sh, mutmut 3.8.0, 2026-10-10. Only ever lower them. +# All 6 modules in [tool.mutmut].only_mutate are baselined (openblade/safety/* == import_guard.py). openblade/domain/policies.py 2 openblade/domain/scalar_coordinate.py 4 openblade/jobs/recovery.py 32 From f213c86736ad5b79cb440f824a586af90c606420 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:20:42 +0000 Subject: [PATCH 153/205] feat(emulator): AML fault hooks (reboot 503, load 409, checksum retry, session TTL) resolve_request_fault() needs a one-line call from the main.py middleware (cross-cutting); wire tests for 503/409 are strict-xfail until then. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/aml_latency.py | 46 +++++++- openblade/api/aml_state.py | 16 ++- tests/integration/test_aml_emulator_faults.py | 111 ++++++++++++++++++ 3 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_aml_emulator_faults.py diff --git a/openblade/api/aml_latency.py b/openblade/api/aml_latency.py index 5dabf8d..0fa8cee 100644 --- a/openblade/api/aml_latency.py +++ b/openblade/api/aml_latency.py @@ -9,8 +9,9 @@ from pathlib import Path from typing import Any -from fastapi import Request +from fastapi import HTTPException, Request +from openblade.api.aml_faults import get_fault_state from openblade.api.aml_state import ( get_aml_emulator_latency_config, record_aml_emulator_latency_metric, @@ -128,18 +129,53 @@ def _resolve_profile_delay_ms( return max(0, delay_ms) -def resolve_request_latency_delay_seconds(method: str, path: str) -> float: - if not _is_latency_managed_path(path): - return 0.0 +def _resolve_operation_class(method: str, path: str) -> str: operation_class = _lookup_operation_class(method, path) if operation_class is None: operation_class = _fallback_operation_class(method, path) + return operation_class + + +def resolve_request_latency_delay_seconds(method: str, path: str) -> float: + if not _is_latency_managed_path(path): + return 0.0 + operation_class = _resolve_operation_class(method, path) config = get_aml_emulator_latency_config() return _resolve_profile_delay_ms(config, operation_class) / 1000.0 +def resolve_request_fault(method: str, path: str) -> HTTPException | None: + """Return the injected AML error for this request, if a fault profile fires. + + Driven by ``OPENBLADE_EMULATOR_FAULT_PROFILE`` (see ``aml_faults``). The caller + renders it through the normal AML error envelope. + """ + if not _is_latency_managed_path(path): + return None + state = get_fault_state() + if not state.profile.active: + return None + if state.in_reboot_window(): + return HTTPException( + status_code=503, + detail="Library is rebooting; retry later", + headers={"Retry-After": str(state.reboot_retry_after_s())}, + ) + if _resolve_operation_class(method, path) == "mount" and state.should_fail_load(): + return HTTPException(status_code=409, detail="Drive load failed; retry the operation") + return None + + async def apply_request_latency(request: Request) -> float: - delay = resolve_request_latency_delay_seconds(request.method, request.url.path) + method, path = request.method, request.url.path + delay = resolve_request_latency_delay_seconds(method, path) + if ( + delay > 0 + and _resolve_operation_class(method, path) == "diagnostic" + and get_fault_state().should_retry_checksum() + ): + # Simulated checksum retry: the verify pass runs a second time. + delay *= 2 if delay > 0: await asyncio.sleep(delay) return delay diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 48b7ed6..76b99c2 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -19,6 +19,7 @@ import pyotp from sqlalchemy import select +from openblade.api.aml_faults import get_fault_state from openblade.catalog.db import get_session, init_db from openblade.catalog.models import AmlUser from openblade.simulator.i3_config import scalar_i3_active_config @@ -2479,6 +2480,15 @@ def max_sessions_per_user() -> int: return value if value >= 1 else _DEFAULT_MAX_SESSIONS_PER_USER +def _session_lifetime() -> timedelta: + # OPENBLADE_EMULATOR_FAULT_PROFILE auth_ttl_s (e.g. "session-expiry") overrides + # the configured timeout so expiry is observable within a test run. + ttl = get_fault_state().profile.auth_ttl_s + if ttl is not None: + return timedelta(seconds=ttl) + return timedelta(minutes=_STATE.session_timeout_minutes) + + def create_session(user: AmlUser) -> SessionRecord: purge_expired_sessions() # Evict the user's oldest sessions so the new login always succeeds: refusing @@ -2495,7 +2505,7 @@ def create_session(user: AmlUser) -> SessionRecord: user_name=user.name, role=user.role, created_at=now, - expires_at=now + timedelta(minutes=_STATE.session_timeout_minutes), + expires_at=now + _session_lifetime(), ) _STATE.sessions[record.token] = record return record @@ -2506,7 +2516,9 @@ def get_session_user(token: str) -> AmlUser | None: record = _STATE.sessions.get(token) if record is None: return None - record.expires_at = _utcnow() + timedelta(minutes=_STATE.session_timeout_minutes) + if get_fault_state().profile.auth_ttl_s is None: + # Sliding expiry; a fault-profile TTL is absolute from login instead. + record.expires_at = _utcnow() + timedelta(minutes=_STATE.session_timeout_minutes) return get_user(record.user_name) diff --git a/tests/integration/test_aml_emulator_faults.py b/tests/integration/test_aml_emulator_faults.py new file mode 100644 index 0000000..102b708 --- /dev/null +++ b/tests/integration/test_aml_emulator_faults.py @@ -0,0 +1,111 @@ +"""OPENBLADE_EMULATOR_FAULT_PROFILE faults are observable on the AML wire.""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import datetime, timedelta +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from openblade.api import aml_faults, aml_state +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig + +# 503/409 need the middleware in openblade/api/main.py to call +# aml_latency.resolve_request_fault (cross-cutting request, not owned by this +# workstream). strict=True: these flip to failures once wired, forcing removal. +_NEEDS_MAIN_HOOK = pytest.mark.xfail( + strict=True, reason="main.py middleware does not yet call resolve_request_fault" +) + + +def _client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, profile: str | None) -> TestClient: + if profile is None: + monkeypatch.delenv(aml_faults.FAULT_PROFILE_ENV, raising=False) + else: + monkeypatch.setenv(aml_faults.FAULT_PROFILE_ENV, profile) + aml_faults.reset_fault_state() + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'faults.db'}")) + reset_context(context) + return TestClient(app) + + +def _login(client: TestClient) -> None: + resp = client.post("/aml/users/login", json={"name": "admin", "password": "password"}) + assert resp.status_code == 200 + + +@pytest.fixture(autouse=True) +def _reset_faults() -> Iterator[None]: + yield + aml_faults.reset_fault_state() + + +def test_no_fault_profile_mount_requests_never_409( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, None) + _login(client) + + statuses = [client.post("/aml/operations/mount", json={}).status_code for _ in range(6)] + + assert 409 not in statuses + + +@_NEEDS_MAIN_HOOK +def test_intermittent_drive_every_third_load_returns_409( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, "intermittent-drive") + _login(client) + + responses = [client.post("/aml/operations/mount", json={}) for _ in range(3)] + + assert [r.status_code == 409 for r in responses] == [False, False, True] + assert responses[2].json()["code"] == "AML_CONFLICT" + + +@_NEEDS_MAIN_HOOK +def test_rebooting_returns_503_with_retry_after( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, "rebooting") + + response = client.get("/aml/system") + + assert response.status_code == 503 + assert int(response.headers["Retry-After"]) >= 1 + + +def test_session_expiry_profile_expires_session_after_ttl( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, '{"auth_ttl_s": 5}') + start = datetime(2026, 1, 1, 12, 0, 0) + now = [start] + monkeypatch.setattr(aml_state, "_utcnow", lambda: now[0]) + _login(client) + assert client.get("/aml/system").status_code == 200 + + now[0] = start + timedelta(seconds=6) + response = client.get("/aml/system") + + assert response.status_code == 401 + + +def test_no_fault_profile_session_outlives_five_seconds( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client(tmp_path, monkeypatch, None) + start = datetime(2026, 1, 1, 12, 0, 0) + now = [start] + monkeypatch.setattr(aml_state, "_utcnow", lambda: now[0]) + _login(client) + + now[0] = start + timedelta(seconds=6) + response = client.get("/aml/system") + + assert response.status_code == 200 From 24b37f8f892e40782390743ed3bcf1ffa58681d1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:21:07 +0000 Subject: [PATCH 154/205] Add targeted and exclude-aware drive lease acquisition CatalogRepository.acquire_drive_leases honours exclude; new acquire_drive_lease_at leases one specific id under BEGIN IMMEDIATE. Wired through the LeaseStore protocol; drops the release-and-retry stub. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 47 ++++++++++++++++++++++- openblade/jobs/scheduler.py | 47 +++++++++++++++++++---- tests/unit/test_drive_leases.py | 66 +++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 10 deletions(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 6769f99..7f13a2a 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -2118,7 +2118,13 @@ def _lease_session(self) -> Iterator[Session]: yield session def acquire_drive_leases( - self, *, job_id: str, barcodes: list[str], num_drives: int, ttl: timedelta + self, + *, + job_id: str, + barcodes: list[str], + num_drives: int, + ttl: timedelta, + exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: """All-or-nothing: one live lease per barcode, or None if not enough free drives.""" with self._lock, self._lease_session() as session: @@ -2135,7 +2141,7 @@ def acquire_drive_leases( ) ) ) - free = [d for d in range(num_drives) if d not in busy] + free = [d for d in range(num_drives) if d not in busy and d not in exclude] if len(free) < len(barcodes): session.rollback() return None @@ -2161,6 +2167,43 @@ def acquire_drive_leases( session.commit() return [_lease_from_row(row) for row in rows] + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + """Lease one specific drive id (negative ids allowed), or None if it is held.""" + with self._lock, self._lease_session() as session: + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + now = _lease_now() + held = session.scalar( + select(func.count()) + .select_from(DriveLeaseRecord) + .where( + DriveLeaseRecord.drive_id == drive_id, + DriveLeaseRecord.released_at.is_(None), + DriveLeaseRecord.expires_at > now, + ) + ) + if held: + session.rollback() + return None + token = int( + session.scalar(select(func.coalesce(func.max(DriveLeaseRecord.fencing_token), 0))) + or 0 + ) + row = DriveLeaseRecord( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + fencing_token=token + 1, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + session.add(row) + session.commit() + return _lease_from_row(row) + def heartbeat_leases(self, lease_ids: list[str], ttl: timedelta) -> None: with self._lock, self._lease_session() as session: now = _lease_now() diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 319bb9e..2627986 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -58,6 +58,12 @@ def acquire( exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: ... + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + """Lease one specific drive id, or None if a live lease already holds it.""" + ... + def unreconciled_drives(self) -> dict[int, str | None]: """Drives awaiting reconciliation (physical state unknown) -> last barcode.""" ... @@ -118,6 +124,30 @@ def acquire( acquired.append(lease) return acquired + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + with self._lock: + now = _now() + if any( + lease.drive_id == drive_id and _live(lease, now) for lease in self._leases.values() + ): + return None + self._last_token += 1 + lease = DriveLease( + id=str(uuid4()), + drive_id=drive_id, + job_id=job_id, + barcode=barcode, + physical_drive_id=None, + fencing_token=self._last_token, + acquired_at=now, + heartbeat_at=now, + expires_at=now + ttl, + ) + self._leases[lease.id] = lease + return lease + def unreconciled_drives(self) -> dict[int, str | None]: return dict(self.unreconciled) @@ -173,15 +203,16 @@ def acquire( ttl: timedelta, exclude: frozenset[int] = frozenset(), ) -> list[DriveLease] | None: - leases = self._repo.acquire_drive_leases( - job_id=job_id, barcodes=barcodes, num_drives=num_drives, ttl=ttl + return self._repo.acquire_drive_leases( + job_id=job_id, barcodes=barcodes, num_drives=num_drives, ttl=ttl, exclude=exclude + ) + + def acquire_drive_lease_at( + self, *, job_id: str, drive_id: int, barcode: str, ttl: timedelta + ) -> DriveLease | None: + return self._repo.acquire_drive_lease_at( + job_id=job_id, drive_id=drive_id, barcode=barcode, ttl=ttl ) - if leases is not None and any(lease.drive_id in exclude for lease in leases): - # The repository cannot yet skip drives itself: never hand out an - # unreconciled drive -- give the leases back and report "not now". - self._repo.release_leases([lease.id for lease in leases]) - return None - return leases def unreconciled_drives(self) -> dict[int, str | None]: from openblade.jobs.reconcile import drives_pending_reconciliation # local: no cycle diff --git a/tests/unit/test_drive_leases.py b/tests/unit/test_drive_leases.py index 4f12511..1e4f732 100644 --- a/tests/unit/test_drive_leases.py +++ b/tests/unit/test_drive_leases.py @@ -11,6 +11,7 @@ from openblade.catalog.db import get_session, init_db from openblade.catalog.repository import CatalogRepository from openblade.domain.errors import DriveBusyError, StaleLeaseError +from openblade.domain.models import DriveLease from openblade.jobs.scheduler import CatalogLeaseStore, DriveScheduler, InMemoryLeaseStore @@ -84,3 +85,68 @@ def test_held_leases_are_kept_alive_without_explicit_heartbeats() -> None: scheduler.release_drives(handles) time.sleep(0.4) assert store.live_leases() == [] # released, and the thread did not revive it + + +TTL = timedelta(seconds=60) + + +def test_excluded_drive_is_skipped_in_one_call(tmp_path: Path) -> None: + repo, _ = _two_repos(tmp_path) + leases = repo.acquire_drive_leases( + job_id="j", barcodes=["A00001L9"], num_drives=2, ttl=TTL, exclude=frozenset({0}) + ) + assert leases is not None and [lease.drive_id for lease in leases] == [1] + assert ( + CatalogLeaseStore(repo).acquire( + job_id="k", barcodes=["B"], num_drives=2, ttl=TTL, exclude=frozenset({0}) + ) + is None + ) + + +@pytest.mark.parametrize("drive_id", [0, -1]) +def test_acquire_drive_lease_at_free_then_held(tmp_path: Path, drive_id: int) -> None: + repo, other = _two_repos(tmp_path) + lease = repo.acquire_drive_lease_at(job_id="j", drive_id=drive_id, barcode="A", ttl=TTL) + assert lease is not None and lease.drive_id == drive_id + assert other.acquire_drive_lease_at(job_id="k", drive_id=drive_id, barcode="B", ttl=TTL) is None + repo.release_leases([lease.id]) + assert other.acquire_drive_lease_at(job_id="k", drive_id=drive_id, barcode="B", ttl=TTL) + + +def test_in_memory_acquire_drive_lease_at_is_exclusive() -> None: + store = InMemoryLeaseStore() + assert store.acquire_drive_lease_at(job_id="j", drive_id=-1, barcode="A", ttl=TTL) + assert store.acquire_drive_lease_at(job_id="k", drive_id=-1, barcode="B", ttl=TTL) is None + + +def test_two_sessions_racing_for_one_drive_id_exactly_one_wins(tmp_path: Path) -> None: + import threading + + repo_a, repo_b = _two_repos(tmp_path) + for _ in range(20): + barrier = threading.Barrier(2) + results: list[DriveLease | None] = [] + + def grab( + repo: CatalogRepository, + job: str, + barrier: threading.Barrier = barrier, + results: list[DriveLease | None] = results, + ) -> None: + barrier.wait() + results.append( + repo.acquire_drive_lease_at(job_id=job, drive_id=3, barcode=job, ttl=TTL) + ) + + threads = [ + threading.Thread(target=grab, args=(repo_a, "a")), + threading.Thread(target=grab, args=(repo_b, "b")), + ] + for t in threads: + t.start() + for t in threads: + t.join() + winners = [r for r in results if r is not None] + assert len(winners) == 1 + repo_a.release_leases([winners[0].id]) From 04c845e2222f201353a59a85414b8ab5dee28c8f Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:21:37 +0000 Subject: [PATCH 155/205] test(i3): import fault presets from emulator; skip client faults when server-side Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/i3/conftest.py | 8 +++++- tests/i3/test_timing_profiles.py | 14 +++++++++++ tests/i3/timing.py | 42 +++++++++++++++++++++++--------- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/tests/i3/conftest.py b/tests/i3/conftest.py index 136b6ad..44bf166 100644 --- a/tests/i3/conftest.py +++ b/tests/i3/conftest.py @@ -6,7 +6,13 @@ I3_AML_URL base URL for AML API (default: http://localhost:8000) I3_AML_USER AML username (default: admin) I3_AML_PASSWORD AML password (default: password) - I3_TIMING_PROFILE instant | realistic | hardware + I3_TIMING_PROFILE instant | realistic | hardware | + I3_FAULTS_SERVER_SIDE 1 = emulator injects faults (default when I3_TEST_MODE=emulator) + +Server-side faults: the emulator must be started with +OPENBLADE_EMULATOR_FAULT_PROFILE set to the SAME name as I3_TIMING_PROFILE +(presets in openblade.api.aml_faults). No AML endpoint exposes the fault +profile name, so the client cannot assert the two match -- keep them in sync. I3_REAL_HARDWARE_ENABLED safety gate — must be "true" to run real-i3 tests OPENBLADE_API_TOKEN native-REST bearer token, if the target enforces one OPENBLADE_API_TOKEN_FILE file holding that token (takes precedence) diff --git a/tests/i3/test_timing_profiles.py b/tests/i3/test_timing_profiles.py index a434ea0..409f78b 100644 --- a/tests/i3/test_timing_profiles.py +++ b/tests/i3/test_timing_profiles.py @@ -33,6 +33,7 @@ def vclock(monkeypatch: pytest.MonkeyPatch) -> Iterator[VirtualClock]: monkeypatch.delenv("I3_TIMING_PROFILE", raising=False) monkeypatch.delenv("I3_TEST_MODE", raising=False) + monkeypatch.setenv("I3_FAULTS_SERVER_SIDE", "0") clock = VirtualClock() started = time.monotonic() with use_clock(clock): @@ -246,3 +247,16 @@ def test_emulator_env_latency_override_accepted_by_emulator_hook( assert env["OPENBLADE_EMULATOR_LATENCY_PROFILE"] == "realistic" assert parsed == json.loads(env["OPENBLADE_EMULATOR_LATENCY_PROFILE_MS"]) assert parsed is not None and parsed["move"]["realistic"] == 4500 + + +def test_profile_runtime_server_side_faults_skips_client_injection( + vclock: VirtualClock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("I3_FAULTS_SERVER_SIDE", "1") + runtime = ProfileRuntime("intermittent-drive", clock=vclock) + + for _ in range(6): + runtime.perform("tape_load") + runtime.check_session(runtime.issue_session() - 3600.0) + + assert runtime.loads == 6 diff --git a/tests/i3/timing.py b/tests/i3/timing.py index deb7e5d..cdfde29 100644 --- a/tests/i3/timing.py +++ b/tests/i3/timing.py @@ -21,7 +21,8 @@ Delays are applied client-side by ``wait_for_op``. Faults (load failures, session expiry, 503 window, checksum retries) are applied client-side by a -``ProfileRuntime`` because the emulator has no env-driven fault hook for them; +``ProfileRuntime`` unless ``faults_server_side()`` -- then the emulator injects +them via ``OPENBLADE_EMULATOR_FAULT_PROFILE`` (presets in ``openblade.api.aml_faults``); ``emulator_env()`` exports the env vars the emulator DOES honour (``I3_TIMING_PROFILE``, ``OPENBLADE_EMULATOR_LATENCY_PROFILE``, ``OPENBLADE_EMULATOR_LATENCY_PROFILE_MS``) so its own latency matches. @@ -40,6 +41,8 @@ from dataclasses import dataclass, field from typing import Literal, Protocol, get_args +from openblade.api.aml_faults import PRESETS as FAULT_PRESETS + OpType = Literal[ "tape_load", "tape_unload", @@ -226,20 +229,35 @@ class ProfileFaults: checksum_retry_every: int = 0 +_FAULT_BASES: dict[ProfileName, LegacyProfileName] = { + "instant": "instant", + "hardware": "hardware", +} + +# Fault knobs come from the emulator's presets (single source of truth). PROFILE_FAULTS: dict[ProfileName, ProfileFaults] = { - "instant": ProfileFaults(base="instant"), - "realistic": ProfileFaults(base="realistic"), - "hardware": ProfileFaults(base="hardware"), - "normal": ProfileFaults(), - "slow-robotics": ProfileFaults(), - "busy-library": ProfileFaults(), - "intermittent-drive": ProfileFaults(load_fail_every=3), - "session-expiry": ProfileFaults(auth_ttl_s=5.0), - "rebooting": ProfileFaults(reboot_window_s=20.0), - "degraded-media": ProfileFaults(checksum_retry_every=4), + name: ProfileFaults( + base=_FAULT_BASES.get(name, "realistic"), + load_fail_every=FAULT_PRESETS[name].load_fail_every, + auth_ttl_s=FAULT_PRESETS[name].auth_ttl_s, + reboot_window_s=FAULT_PRESETS[name].reboot_window_s, + checksum_retry_every=FAULT_PRESETS[name].checksum_retry_every, + ) + for name in TIMING_PROFILES } +def faults_server_side() -> bool: + """True when the emulator injects the faults (``OPENBLADE_EMULATOR_FAULT_PROFILE``). + + ``I3_FAULTS_SERVER_SIDE`` wins; otherwise ON when ``I3_TEST_MODE`` is emulator (its default). + """ + raw = os.environ.get("I3_FAULTS_SERVER_SIDE") + if raw is not None and raw.strip(): + return raw.strip().lower() in {"1", "true", "yes", "on"} + return os.environ.get("I3_TEST_MODE", "emulator").strip().lower() == "emulator" + + class RetryableOpError(Exception): """A transient failure the caller is expected to retry.""" @@ -269,6 +287,8 @@ def __post_init__(self) -> None: @property def faults(self) -> ProfileFaults: + if faults_server_side(): + return ProfileFaults(base=PROFILE_FAULTS[self.name].base) return PROFILE_FAULTS[self.name] def _sleep_for(self, op_type: OpType, multiplier: float = 1.0) -> None: From bf8a4e5d510dddfe8ae2ba9eb373b4e411a0a94d Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:22:01 +0000 Subject: [PATCH 156/205] docs(emulator): document OPENBLADE_EMULATOR_FAULT_PROFILE in contract, compose, README Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/emulator/docker-compose.standalone.yml | 1 + openblade/emulator_contract/README.md | 19 +++++++++++++++++++ openblade/emulator_contract/contract.json | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/deploy/emulator/docker-compose.standalone.yml b/deploy/emulator/docker-compose.standalone.yml index fe6eeb3..74c9f58 100644 --- a/deploy/emulator/docker-compose.standalone.yml +++ b/deploy/emulator/docker-compose.standalone.yml @@ -22,6 +22,7 @@ x-runtime-defaults: &runtime-defaults EMULATOR_OCCUPANCY_PERCENT: ${EMULATOR_OCCUPANCY_PERCENT:-60} EMULATOR_LATENCY_PROFILE: ${EMULATOR_LATENCY_PROFILE:-instant} OPENBLADE_SCALAR_API_ONLY: ${OPENBLADE_SCALAR_API_ONLY:-true} + OPENBLADE_EMULATOR_FAULT_PROFILE: ${OPENBLADE_EMULATOR_FAULT_PROFILE:-} x-ui-target-defaults: &ui-target-defaults EMULATOR_UI_TARGET_LIBRARY1_URL: ${EMULATOR_UI_TARGET_LIBRARY1_URL:-http://openblade-emulator-library-1:8010} diff --git a/openblade/emulator_contract/README.md b/openblade/emulator_contract/README.md index 4da6e3a..eb21fb3 100644 --- a/openblade/emulator_contract/README.md +++ b/openblade/emulator_contract/README.md @@ -167,3 +167,22 @@ in pull requests and pushes to `master`, and also support manual dispatch: review before OpenBlade consumes it. - OpenBlade should fail fast on startup/test bring-up when contract expectations are not met. + +## Fault profiles + +`OPENBLADE_EMULATOR_FAULT_PROFILE` (runtime env; not a wire change) injects +AML wire-level faults. Value: a preset name or a JSON object with any of: + +| key | type | effect | +|---|---|---| +| `load_fail_every` | int | every Nth `mount`-class request returns 409 `AML_CONFLICT` | +| `auth_ttl_s` | float | sessions expire this many seconds after login (absolute, no sliding refresh) → 401 | +| `reboot_window_s` | float | every AML request returns 503 `AML_UNAVAILABLE` + `Retry-After` for this long after start | +| `checksum_retry_every` | int | every Nth `diagnostic`-class request takes twice its latency (simulated checksum retry) | + +Presets (`openblade/api/aml_faults.py`, shared with `tests/i3/timing.py`): +`intermittent-drive` (`load_fail_every=3`), `session-expiry` (`auth_ttl_s=5`), +`rebooting` (`reboot_window_s=20`), `degraded-media` (`checksum_retry_every=4`); +`instant`, `realistic`, `hardware`, `normal`, `slow-robotics`, `busy-library` = no faults. +Unset/empty = no faults. An unknown preset or malformed JSON raises +`FaultProfileError` (never a silent no-fault run). diff --git a/openblade/emulator_contract/contract.json b/openblade/emulator_contract/contract.json index 8f4009a..34e7b55 100644 --- a/openblade/emulator_contract/contract.json +++ b/openblade/emulator_contract/contract.json @@ -42,7 +42,8 @@ "EMULATOR_DRIVE_COUNT": "integer", "EMULATOR_OCCUPANCY_PERCENT": "integer", "EMULATOR_LATENCY_PROFILE": "instant|realistic|hardware|custom", - "OPENBLADE_SCALAR_API_ONLY": "true|false" + "OPENBLADE_SCALAR_API_ONLY": "true|false", + "OPENBLADE_EMULATOR_FAULT_PROFILE": "preset-name|json-object" }, "latency_profiles": ["instant", "realistic", "hardware", "custom"], "manual_matrix_file": "openblade/emulator_contract/quantum_i3_rev_h_matrix.json", From e8795ee231ae5fe201fe813d9766dfd6f32c8da5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:23:59 +0000 Subject: [PATCH 157/205] feat(emulator): wire AML fault profile into the latency middleware resolve_request_fault runs before routing so the reboot-window 503 and the intermittent-load 409 reach the wire; get_fault_state() at startup makes a malformed OPENBLADE_EMULATOR_FAULT_PROFILE fail at boot and anchors the reboot window at process start. Drops the strict xfail markers on the two wire tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/main.py | 11 +++++++++++ tests/integration/test_aml_emulator_faults.py | 9 --------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/openblade/api/main.py b/openblade/api/main.py index 870e5e5..b16ea43 100644 --- a/openblade/api/main.py +++ b/openblade/api/main.py @@ -59,9 +59,11 @@ routes_virtual_fs, routes_volume_groups, ) +from openblade.api.aml_faults import get_fault_state from openblade.api.aml_latency import ( apply_request_latency, capture_request_latency_metric, + resolve_request_fault, should_capture_latency_metrics, ) from openblade.api.api_auth import api_auth_middleware, log_api_auth_status @@ -187,6 +189,12 @@ async def apply_aml_emulator_latency( if not should_capture_latency_metrics(request.url.path): return await call_next(request) + # Env-driven fault profile (OPENBLADE_EMULATOR_FAULT_PROFILE): reboot-window 503 + # and intermittent load 409 are answered here, before routing, like the appliance. + fault = resolve_request_fault(request.method, request.url.path) + if fault is not None: + return await handle_http_exception(request, fault) + start = time.perf_counter() simulated_delay = await apply_request_latency(request) status_code = 500 @@ -280,6 +288,9 @@ async def initialize_aml_state() -> None: from openblade.api.aml_state import ensure_initialized log_api_auth_status() + # Parse the fault profile at boot so a typo fails loudly here, not as a 500 on + # the first request; this also anchors the reboot window at process start. + get_fault_state() context = get_context() ensure_initialized( context.config.db_url, diff --git a/tests/integration/test_aml_emulator_faults.py b/tests/integration/test_aml_emulator_faults.py index 102b708..31c2252 100644 --- a/tests/integration/test_aml_emulator_faults.py +++ b/tests/integration/test_aml_emulator_faults.py @@ -14,13 +14,6 @@ from openblade.bootstrap import create_context, reset_context from openblade.config import OpenBladeConfig -# 503/409 need the middleware in openblade/api/main.py to call -# aml_latency.resolve_request_fault (cross-cutting request, not owned by this -# workstream). strict=True: these flip to failures once wired, forcing removal. -_NEEDS_MAIN_HOOK = pytest.mark.xfail( - strict=True, reason="main.py middleware does not yet call resolve_request_fault" -) - def _client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, profile: str | None) -> TestClient: if profile is None: @@ -55,7 +48,6 @@ def test_no_fault_profile_mount_requests_never_409( assert 409 not in statuses -@_NEEDS_MAIN_HOOK def test_intermittent_drive_every_third_load_returns_409( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -68,7 +60,6 @@ def test_intermittent_drive_every_third_load_returns_409( assert responses[2].json()["code"] == "AML_CONFLICT" -@_NEEDS_MAIN_HOOK def test_rebooting_returns_503_with_retry_after( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 59be7daf7a54a373eb2885249f612f8618c30891 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:25:13 +0000 Subject: [PATCH 158/205] Claim queue drive/changer slots with targeted leases _claim_slot now uses acquire_drive_lease_at; probes, retry loop and _is_transient are gone, and the changer pseudo-drive is -1. Adds a race test that forces both claimers past the owner pre-check (mutation-checked: fails when acquire_drive_lease_at ignores live leases). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/queue.py | 78 ++++++++----------------- tests/unit/test_job_queue_persistent.py | 46 +++++++++++++++ 2 files changed, 69 insertions(+), 55 deletions(-) diff --git a/openblade/jobs/queue.py b/openblade/jobs/queue.py index 7d5d6dc..a5a4c64 100644 --- a/openblade/jobs/queue.py +++ b/openblade/jobs/queue.py @@ -3,7 +3,6 @@ import json import logging import threading -import time from collections.abc import Callable from datetime import timedelta, timezone from typing import TypeVar @@ -16,40 +15,18 @@ JobNotFoundError, safe_job_error, ) -from openblade.domain.models import DriveLease, Job, JobState, JobType +from openblade.domain.models import Job, JobState, JobType from openblade.jobs.scheduler import DEFAULT_LEASE_TTL, LeaseStore ResultT = TypeVar("ResultT") logger = logging.getLogger(__name__) -# Changer ownership is a lease on this reserved pseudo-drive id. The lease store -# only allocates ids in range(num_drives) (no negatives), and claiming a slot -# probes every free id below it, so the sentinel is kept small: it must exceed -# any real drive count (a Scalar i3 tops out well below it) but stay cheap. -CHANGER_DRIVE_ID = 64 +# Changer ownership is a lease on this reserved pseudo-drive id. Real drives +# are 0..N-1, so a negative id can never collide with one. +CHANGER_DRIVE_ID = -1 _CHANGER_KIND = "CHANGER" _DRIVE_KIND = "QUEUE-CLAIM" -_PROBE_BARCODE = "QUEUE-PROBE" -_CLAIM_ATTEMPTS = 50 -_CLAIM_RETRY_SECONDS = 0.01 - - -def _is_transient(lease: DriveLease) -> bool: - """True for another claimer's in-flight lease, which never denotes ownership. - - Claim barcodes name their target (``KIND:slot``). When a claimer's snapshot - is stale the store can hand its claim barcode to a lower id; that lease, like - every probe, is released immediately and must not read as an owner. - """ - if lease.barcode == _PROBE_BARCODE: - return True - kind, sep, _ = lease.barcode.partition(":") - return ( - bool(sep) - and kind in (_DRIVE_KIND, _CHANGER_KIND) - and lease.barcode != f"{kind}:{lease.drive_id}" - ) def _job_from_row(row: catalog_models.Job) -> Job: @@ -102,37 +79,28 @@ def update_job( self._catalog.update_job_state(job_id, (state or job.state).value, error) return self.get_job(job_id) + def _owner(self, slot: int) -> str | None: + for lease in self._lease_store.live_leases(): + if lease.drive_id == slot: + return lease.job_id + return None + def _claim_slot( self, slot: int, job_id: str, kind: str, busy: Callable[[str], Exception] ) -> None: - for _attempt in range(_CLAIM_ATTEMPTS): - live = self._lease_store.live_leases() - for lease in live: - if lease.drive_id == slot and not _is_transient(lease): - if lease.job_id == job_id: - return - raise busy(lease.job_id) - # The store allocates the lowest free ids in range(num_drives); it - # cannot target one id. So request every free id up to and including - # ``slot`` in one all-or-nothing transaction, keep ``slot``, and - # release the probes. A concurrent claimer's probes make this miss; - # re-check and retry until ``slot`` has a real owner or is ours. - taken = {lease.drive_id for lease in live} - probes = sum(1 for d in range(slot) if d not in taken) - leases = self._lease_store.acquire( - job_id=job_id, - barcodes=[_PROBE_BARCODE] * probes + [f"{kind}:{slot}"], - num_drives=slot + 1, - ttl=self._ttl, + owner = self._owner(slot) + if owner == job_id: + return + if owner is None: + lease = self._lease_store.acquire_drive_lease_at( + job_id=job_id, drive_id=slot, barcode=f"{kind}:{slot}", ttl=self._ttl ) - if leases is not None: - extra = [lease.id for lease in leases if lease.drive_id != slot] - if extra: - self._lease_store.release(extra) - if len(extra) < len(leases): - return - time.sleep(_CLAIM_RETRY_SECONDS) - raise busy("another job (claim contended)") + if lease is not None: + return + owner = self._owner(slot) + if owner == job_id: + return + raise busy(owner or "another job") def _release_slot(self, slot: int, job_id: str) -> None: mine = [ @@ -144,7 +112,7 @@ def _release_slot(self, slot: int, job_id: str) -> None: self._lease_store.release(mine) def claim_drive(self, drive_id: int, job_id: str) -> None: - if drive_id < 0 or drive_id >= CHANGER_DRIVE_ID: + if drive_id < 0: raise ValueError(f"drive id {drive_id} out of range") self._claim_slot( drive_id, diff --git a/tests/unit/test_job_queue_persistent.py b/tests/unit/test_job_queue_persistent.py index 1a8998f..688bc3f 100644 --- a/tests/unit/test_job_queue_persistent.py +++ b/tests/unit/test_job_queue_persistent.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading +from collections.abc import Callable from pathlib import Path import pytest @@ -83,3 +84,48 @@ def func() -> str: worker.join(timeout=5) assert results == ["done"] assert b.get_job(job.id).state is JobState.COMPLETED + + +@pytest.mark.parametrize("target", ["drive", "changer"]) +def test_racing_claims_past_the_owner_precheck_exactly_one_wins( + queues: tuple[JobQueue, JobQueue], monkeypatch: pytest.MonkeyPatch, target: str +) -> None: + # Both claimers pass the "nobody owns it" pre-check before either acquires, + # so exclusivity rests solely on the store's targeted acquisition. + barrier = threading.Barrier(2) + for queue in queues: + original = queue._owner + calls = {"n": 0} + + def gated( + slot: int, + original: Callable[[int], str | None] = original, + calls: dict[str, int] = calls, + ) -> str | None: + calls["n"] += 1 + result = original(slot) + if calls["n"] == 1: + barrier.wait(timeout=5) + return result + + monkeypatch.setattr(queue, "_owner", gated) + + outcomes: list[str] = [] + + def claim(queue: JobQueue) -> None: + job = queue.create_job(JobType.ARCHIVE, {}) + try: + if target == "drive": + queue.claim_drive(2, job.id) + else: + queue.claim_changer(job.id) + outcomes.append("won") + except (DriveOccupiedError, ChangerBusyError): + outcomes.append("busy") + + threads = [threading.Thread(target=claim, args=(q,)) for q in queues] + for t in threads: + t.start() + for t in threads: + t.join() + assert sorted(outcomes) == ["busy", "won"] From 7bce7bbd35d960fbdacf1f05e8deff057880b6c7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:26:46 +0000 Subject: [PATCH 159/205] Test that gateway on_open evicts a corrupt cache entry and re-hydrates Mutation-checked: fails when on_open skips verify_entry. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/unit/test_protocol_gateway.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/test_protocol_gateway.py b/tests/unit/test_protocol_gateway.py index 8e2f382..24dcfcf 100644 --- a/tests/unit/test_protocol_gateway.py +++ b/tests/unit/test_protocol_gateway.py @@ -242,3 +242,32 @@ def test_sessions_api_includes_upload_audit( assert resp.status_code == 200 payload = resp.json() assert payload[0]["uploads"][0]["requested_path"] == "/openblade/inbox/upload.iso" + + +def test_on_open_evicts_corrupt_cache_entry_and_rehydrates( + gw: ProtocolGateway, tmp_path: Path +) -> None: + from openblade.catalog.db import get_session, init_db + from openblade.catalog.repository import CatalogRepository + from openblade.fuse.cache import HydrationCache + from openblade.fuse.hydration import Hydrator + from tests.unit.test_fuse_cache_integrity import BIG, BIG_SUM, _Engine, _flip_sampled_byte + + init_db(f"sqlite:///{tmp_path / 'gw-cat.db'}") + repo = CatalogRepository(get_session()) + group = repo.create_volume_group("g") + rec = repo.create_file_record("/big.bin", len(BIG), BIG_SUM, group.id) + inst = repo.create_file_instance(rec.id, "TAPE01", "/big.bin") + repo.mark_instance_archived(inst.id) + cache = HydrationCache(str(tmp_path / "cache")) + cache.store(BIG_SUM, BIG) + cache.verify_entry(BIG_SUM) # a verified, intact entry + engine = _Engine(tmp_path / "staging") + gw.attach_hydrator(Hydrator(repo, cache, engine, batch_window_s=0.01)) # type: ignore[arg-type] # _Engine is a duck-typed test double + + _flip_sampled_byte(cache, BIG_SUM) + handle = gw.on_open("/big.bin", timeout=5.0) + + assert len(engine.batches) == 1 # a hydration was requested, not the bad bytes served + assert cache.cache_key(BIG_SUM).read_bytes() == BIG + gw.on_close(handle.token) From 02c52b4dc97859cbab4495d0fb419e10652b8d2a Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:33:38 +0000 Subject: [PATCH 160/205] chore(integration): round-2 merge fixes - atomicity test passes a real catalog to the load helpers: the already-loaded fast path now runs the reconciliation guard, so None is no longer acceptable - ci_ownership rule for tests/fakes/ - regenerate docs/wiki/reference/api.md (reconcile + NAS list routes) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/wiki/reference/api.md | 13 ++++++++++--- tests/integration/test_sharded_archive_atomicity.py | 2 +- tools/ci_ownership.toml | 4 ++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/wiki/reference/api.md b/docs/wiki/reference/api.md index 9d24438..3b66b4a 100644 --- a/docs/wiki/reference/api.md +++ b/docs/wiki/reference/api.md @@ -6,7 +6,7 @@ Introspected from the OpenAPI schema of `openblade.api.main:app`. -**1179 operations** total: 199 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. +**1186 operations** total: 206 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. The application serves two surfaces from one ASGI app. Setting `OPENBLADE_SCALAR_API_ONLY=true` puts it in emulator-only mode, where the @@ -83,12 +83,13 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/restore/` | Enqueue Restore | - | `openblade__api__routes_restore__RestoreRequest` | `EnqueuedJobResponse` | | `POST` | `/restore/tree` | Restore Tree | - | `TreeRestoreApiRequest` | `TreeRestoreResponse` | -### `jobs` (3 operations) +### `jobs` (4 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | | `GET` | `/jobs/` | List Jobs | `library_id`? (query) | - | `openblade__api__routes_jobs__JobResponse[]` | | `GET` | `/jobs/recovery` | Get Recovery Report | - | - | `RecoveryReportResponse` | +| `POST` | `/jobs/recovery/reconcile/{drive_id}` | Post Reconcile Drive | `drive_id` (path) | - | `ReconcileDriveResponse` | | `GET` | `/jobs/{job_id}` | Get Job | `job_id` (path) | - | `openblade__api__routes_jobs__JobResponse` | ### `catalog` (6 operations) @@ -113,7 +114,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/virtual/ls` | List Virtual Directory | `path`? (query) | - | `VirtualDirectoryListing` | | `GET` | `/virtual/stat` | Stat Virtual Path | `path` (query) | - | `VirtualFileEntry` | -### `NAS Config` (118 operations) +### `NAS Config` (124 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | @@ -136,6 +137,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/datasets/{dataset_id}/manifest` | Get Dataset Manifest | `dataset_id` (path) | - | `object` | | `GET` | `/nas/datasets/{dataset_id}/report` | Get Dataset Report | `dataset_id` (path) | - | `object` | | `POST` | `/nas/datasets/{dataset_id}/verify` | Verify Dataset | `dataset_id` (path) | - | `object` | +| `GET` | `/nas/export-sets` | List Export Sets | - | - | `object[]` | | `GET` | `/nas/fuse/log` | Get Fuse Log | - | - | `object[]` | | `POST` | `/nas/fuse/open` | Open Virtual File | - | `FuseOpenRequest` | `object` | | `POST` | `/nas/ingest/start` | Start Ingest | - | `StartIngestRequest` | `StartIngestResponse` | @@ -159,6 +161,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/pools/{pool_id}/browse` | Browse Pool | `pool_id` (path), `path`? (query) | - | `object` | | `GET` | `/nas/pools/{pool_id}/files/{file_path}` | Get Pool File Detail | `pool_id` (path), `file_path` (path) | - | `NasFileRecord` | | `POST` | `/nas/pools/{pool_id}/request-restore` | Request Restore | `pool_id` (path) | `RestorePlanRequest` | `NasRestoreJob` | +| `GET` | `/nas/reservations` | List Reservations | `pool_id`? (query) | - | `object[]` | | `POST` | `/nas/resolve-policy` | Resolve Policy | - | `ResolvePolicyRequest` | `EffectivePolicy` | | `GET` | `/nas/restore-jobs` | List Restore Jobs | - | - | `NasRestoreJob[]` | | `GET` | `/nas/restore-jobs/{job_id}` | Get Restore Job | `job_id` (path) | - | `NasRestoreJob` | @@ -176,6 +179,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/nas/source-stream` | Get Source Stream Config | - | - | `SourceStreamConfig` | | `PUT` | `/nas/source-stream` | Update Source Stream Config | - | `SourceStreamConfig` | `SourceStreamConfig` | | `DELETE` | `/nas/source-stream` | Delete Source Stream Config | - | - | `object` | +| `GET` | `/nas/vaults` | List Vaults | - | - | `object[]` | | `POST` | `/storage/nas/archive-plan` | Archive Plan | - | `ArchivePlanRequest` | `ArchivePlan` | | `GET` | `/storage/nas/cache-drives` | List Cache Drives | - | - | `CacheDriveConfig[]` | | `POST` | `/storage/nas/cache-drives` | Create Or Update Cache Drive | - | `CacheDriveConfig` | `CacheDriveConfig` | @@ -195,6 +199,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/datasets/{dataset_id}/manifest` | Get Dataset Manifest | `dataset_id` (path) | - | `object` | | `GET` | `/storage/nas/datasets/{dataset_id}/report` | Get Dataset Report | `dataset_id` (path) | - | `object` | | `POST` | `/storage/nas/datasets/{dataset_id}/verify` | Verify Dataset | `dataset_id` (path) | - | `object` | +| `GET` | `/storage/nas/export-sets` | List Export Sets | - | - | `object[]` | | `GET` | `/storage/nas/fuse/log` | Get Fuse Log | - | - | `object[]` | | `POST` | `/storage/nas/fuse/open` | Open Virtual File | - | `FuseOpenRequest` | `object` | | `POST` | `/storage/nas/ingest/start` | Start Ingest | - | `StartIngestRequest` | `StartIngestResponse` | @@ -218,6 +223,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/pools/{pool_id}/browse` | Browse Pool | `pool_id` (path), `path`? (query) | - | `object` | | `GET` | `/storage/nas/pools/{pool_id}/files/{file_path}` | Get Pool File Detail | `pool_id` (path), `file_path` (path) | - | `NasFileRecord` | | `POST` | `/storage/nas/pools/{pool_id}/request-restore` | Request Restore | `pool_id` (path) | `RestorePlanRequest` | `NasRestoreJob` | +| `GET` | `/storage/nas/reservations` | List Reservations | `pool_id`? (query) | - | `object[]` | | `POST` | `/storage/nas/resolve-policy` | Resolve Policy | - | `ResolvePolicyRequest` | `EffectivePolicy` | | `GET` | `/storage/nas/restore-jobs` | List Restore Jobs | - | - | `NasRestoreJob[]` | | `GET` | `/storage/nas/restore-jobs/{job_id}` | Get Restore Job | `job_id` (path) | - | `NasRestoreJob` | @@ -235,6 +241,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/storage/nas/source-stream` | Get Source Stream Config | - | - | `SourceStreamConfig` | | `PUT` | `/storage/nas/source-stream` | Update Source Stream Config | - | `SourceStreamConfig` | `SourceStreamConfig` | | `DELETE` | `/storage/nas/source-stream` | Delete Source Stream Config | - | - | `object` | +| `GET` | `/storage/nas/vaults` | List Vaults | - | - | `object[]` | ### `upload-download` (6 operations) diff --git a/tests/integration/test_sharded_archive_atomicity.py b/tests/integration/test_sharded_archive_atomicity.py index db07b01..b538988 100644 --- a/tests/integration/test_sharded_archive_atomicity.py +++ b/tests/integration/test_sharded_archive_atomicity.py @@ -76,7 +76,7 @@ def test_load_helpers_do_not_mutate_scheduler_lock_key() -> None: assert slot is not None library.load(slot, 1) # cartridge physically in drive 1 handle = DriveHandle(drive_id=0, barcode=BARCODES[0]) # scheduler reserved drive 0 - physical, _slot = helper(None, library, ltfs, handle, "job") # already-loaded fast path + physical, _slot = helper(_catalog(), library, ltfs, handle, "job") # already-loaded fast path assert handle.drive_id == 0, f"{helper.__name__} mutated the scheduler lock key" assert handle.physical == 1 and physical == 1 diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml index 99517bb..d863d95 100644 --- a/tools/ci_ownership.toml +++ b/tools/ci_ownership.toml @@ -38,6 +38,10 @@ categories = ["integration"] glob = "tests/contract/*" categories = ["integration"] +[[rule]] +glob = "tests/fakes/*" +categories = ["unit", "integration"] + [[rule]] glob = "tests/test_nas_protocol_render.py" categories = ["unit"] From 5d2cf355a32bd2b6cbd415ab9b3ca865124bc93a Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:41:38 +0000 Subject: [PATCH 161/205] fix(fuse-cache): accept any checksum string for sample offset; reject unsafe cache keys S5: seed the verify sample offset from sha256(checksum) so non-hex checksums on >64 KiB files no longer raise ValueError; cache_key raises CacheError for path separators, '..', NUL, empty. N6: document the sampled-verify blind spot. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 11 ++++++++++- tests/unit/test_fuse_cache_integrity.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 76647f0..78b3e0c 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -38,7 +38,10 @@ def _sample_offset(checksum: str, size: int) -> int: """Deterministic sample offset derived from the checksum; 0 when the file fits in one block.""" if size <= SAMPLE_BYTES: return 0 - return int(checksum[:8], 16) % max(1, size - SAMPLE_BYTES) + # Seed from a digest of the checksum so any string (legacy non-hex records, + # test presets) yields a deterministic offset instead of a ValueError. + seed = int.from_bytes(hashlib.sha256(checksum.encode()).digest()[:8], "big") + return seed % max(1, size - SAMPLE_BYTES) class HydrationCache: @@ -223,6 +226,10 @@ def verify_entry(self, checksum: str) -> bool: On mismatch or a missing/unreadable integrity record the entry (file and sidecar) is dropped and ``CacheIntegrityError`` raised, so the caller can fall through to re-hydration. + + This is a sampled check by design: bit-rot outside the sampled window on + a file whose size and mtime_ns are preserved is NOT detected here; only + a full re-hash (``store_verified``) catches that. """ with self._lock: if self._entry_matches(checksum): @@ -234,6 +241,8 @@ def verify_entry(self, checksum: str) -> bool: raise CacheIntegrityError("cached entry failed integrity verification and was evicted") def cache_key(self, checksum: str) -> Path: + if checksum in ("", ".") or any(bad in checksum for bad in ("/", "\\", "..", "\0")): + raise CacheError("checksum is not a safe cache path component") return self.cache_dir / checksum[:2] / checksum def is_cached(self, checksum: str) -> bool: diff --git a/tests/unit/test_fuse_cache_integrity.py b/tests/unit/test_fuse_cache_integrity.py index a92faea..311929d 100644 --- a/tests/unit/test_fuse_cache_integrity.py +++ b/tests/unit/test_fuse_cache_integrity.py @@ -12,7 +12,7 @@ from openblade.catalog.db import get_session, init_db from openblade.catalog.repository import CatalogRepository from openblade.fuse import cache as cache_mod -from openblade.fuse.cache import SAMPLE_BYTES, CacheIntegrityError, HydrationCache +from openblade.fuse.cache import SAMPLE_BYTES, CacheError, CacheIntegrityError, HydrationCache from openblade.fuse.filesystem import CatalogFilesystem from openblade.fuse.hydration import HydrationFailedError, Hydrator from openblade.fuse.mount import CatalogFuseOperations @@ -193,3 +193,20 @@ def short_pread(fd: int, length: int, offset: int) -> bytes: assert calls.n > 1 # Past EOF: exactly the bytes available. assert cache.read_range(BIG_SUM, len(BIG) - 10, 100) == BIG[-10:] + + +def test_non_hex_checksum_large_entry_stores_and_verifies(tmp_path: Path) -> None: + """Regression S5: a non-hex checksum on a >64 KiB file must not raise ValueError.""" + cache = HydrationCache(str(tmp_path)) + data = b"x" * (200 * 1024) + path = cache.store("preset", data) + assert path.read_bytes() == data + assert cache.verify_entry("preset") is True + + +@pytest.mark.parametrize("checksum", ["../x", "", ".", "..", "a/b", "a\\b", "a\0b"]) +def test_unsafe_checksum_path_component_is_rejected(tmp_path: Path, checksum: str) -> None: + cache = HydrationCache(str(tmp_path)) + with pytest.raises(CacheError): + cache.store(checksum, b"") + assert not (tmp_path.parent / "x").exists() From 7689ab86af0b563fdb934729a37ba5adffe1af4a Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:44:36 +0000 Subject: [PATCH 162/205] fix(job-queue): reconciliation barrier on claim_drive; heartbeat claims during run_job S3: claim_drive now calls ensure_drive_reconciled so an unreconciled drive raises DriveUnreconciledError, matching DriveScheduler.acquire. Add JobQueue.heartbeat(job_id) and a daemon heartbeat (every ttl/3) in run_job so a holder past the TTL keeps its claim. Both tests mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/queue.py | 32 +++++++++++++++++++ tests/unit/test_job_queue_persistent.py | 41 ++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/openblade/jobs/queue.py b/openblade/jobs/queue.py index a5a4c64..d8905a0 100644 --- a/openblade/jobs/queue.py +++ b/openblade/jobs/queue.py @@ -16,6 +16,7 @@ safe_job_error, ) from openblade.domain.models import Job, JobState, JobType +from openblade.jobs.reconcile import ensure_drive_reconciled from openblade.jobs.scheduler import DEFAULT_LEASE_TTL, LeaseStore ResultT = TypeVar("ResultT") @@ -114,6 +115,9 @@ def _release_slot(self, slot: int, job_id: str) -> None: def claim_drive(self, drive_id: int, job_id: str) -> None: if drive_id < 0: raise ValueError(f"drive id {drive_id} out of range") + # Same barrier as DriveScheduler.acquire: a drive whose physical state is + # unknown after a crash must not be handed out (DriveUnreconciledError). + ensure_drive_reconciled(self._catalog, drive_id) self._claim_slot( drive_id, job_id, @@ -135,13 +139,41 @@ def claim_changer(self, job_id: str) -> None: def release_changer(self, job_id: str) -> None: self._release_slot(CHANGER_DRIVE_ID, job_id) + def heartbeat(self, job_id: str) -> None: + """Renew every live drive/changer lease held by ``job_id`` for another ``ttl``.""" + with self._lock: + mine = [lease.id for lease in self._lease_store.live_leases() if lease.job_id == job_id] + if mine: + self._lease_store.heartbeat(mine, self._ttl) + + def _heartbeat_loop(self, job_id: str, stop: threading.Event) -> None: + interval = max(self._ttl.total_seconds() / 3.0, 0.05) + while not stop.wait(interval): + try: + self.heartbeat(job_id) + except Exception: # noqa: BLE001 - a missed beat is logged; the next one retries + logger.warning("job %s lease heartbeat failed", job_id, exc_info=True) + def run_job(self, job: Job, func: Callable[[], ResultT]) -> tuple[Job, ResultT]: self.update_job(job.id, state=JobState.RUNNING) + # Keep this job's claims alive while func runs; otherwise a holder past the + # TTL silently loses its slot and another job can lease the same drive. + stop = threading.Event() + beat = threading.Thread( + target=self._heartbeat_loop, + args=(job.id, stop), + name="job-lease-heartbeat", + daemon=True, + ) + beat.start() try: result = func() except Exception as exc: logger.warning("job %s failed", job.id, exc_info=True) self.update_job(job.id, state=JobState.FAILED, error=safe_job_error(exc)) raise + finally: + stop.set() + beat.join() completed = self.update_job(job.id, state=JobState.COMPLETED, error=None) return completed, result diff --git a/tests/unit/test_job_queue_persistent.py b/tests/unit/test_job_queue_persistent.py index 688bc3f..fc3a354 100644 --- a/tests/unit/test_job_queue_persistent.py +++ b/tests/unit/test_job_queue_persistent.py @@ -3,16 +3,19 @@ from __future__ import annotations import threading +import time from collections.abc import Callable +from datetime import timedelta from pathlib import Path import pytest from openblade.catalog.db import get_session, init_db from openblade.catalog.repository import CatalogRepository -from openblade.domain.errors import ChangerBusyError, DriveOccupiedError +from openblade.domain.errors import ChangerBusyError, DriveOccupiedError, DriveUnreconciledError from openblade.domain.models import JobState, JobType from openblade.jobs.queue import JobQueue +from openblade.jobs.reconcile import PHYSICAL_STATE_UNKNOWN from openblade.jobs.scheduler import CatalogLeaseStore @@ -129,3 +132,39 @@ def claim(queue: JobQueue) -> None: for t in threads: t.join() assert sorted(outcomes) == ["busy", "won"] + + +def test_claim_drive_refuses_unreconciled_drive(tmp_path: Path) -> None: + """Regression S3: the public claim API honours the reconciliation barrier.""" + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo = CatalogRepository(get_session()) + queue = JobQueue(repo, CatalogLeaseStore(repo)) + repo.journal( + "crashed-job", PHYSICAL_STATE_UNKNOWN, {"op": "unload", "barcode": "T1L8", "drive": 2} + ) + job = queue.create_job(JobType.ARCHIVE, {}) + with pytest.raises(DriveUnreconciledError): + queue.claim_drive(2, job.id) + assert all(lease.drive_id != 2 for lease in CatalogLeaseStore(repo).live_leases()) + queue.claim_drive(1, job.id) # other drives are unaffected + + +def test_run_job_heartbeats_claims_past_the_ttl(tmp_path: Path) -> None: + """Regression S3: a holder running longer than the TTL keeps its drive claim.""" + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + repo_a, repo_b = CatalogRepository(get_session()), CatalogRepository(get_session()) + queue = JobQueue(repo_a, CatalogLeaseStore(repo_a), ttl=timedelta(seconds=2)) + observer = CatalogLeaseStore(repo_b) + job = queue.create_job(JobType.ARCHIVE, {}) + queue.claim_drive(0, job.id) + seen: list[bool] = [] + + def work() -> None: + for _ in range(10): + time.sleep(0.5) + repo_b.session.expire_all() + seen.append(any(lease.job_id == job.id for lease in observer.live_leases())) + + queue.run_job(job, work) + assert len(seen) == 10 + assert all(seen), seen From 7a0c7d7c2ead84c3f629ab281d87d3d7f37e413b Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:44:54 +0000 Subject: [PATCH 163/205] fix(jobs): fail closed when physical_state_unknown cannot be journaled (B1) record_physical_state_unknown swallowed journal failures, so the job's finally released the drive lease with no pending-reconciliation entry and the next job could LOAD/UNLOAD a drive that may still have LTFS mounted. - CatalogRepository.journal_durable: fresh session over the same bind, 1 attempt + 3 retries (50/100/200 ms) on OperationalError; never touches the (possibly poisoned) shared session. - On failure the matching DriveHandle is flagged hold_unjournaled; DriveScheduler.release_drives releases every other handle, keeps the flagged lease, logs ERROR, then raises JournalWriteError. - recover_after_restart no longer auto-releases live leases of failed / failed_recoverable jobs; reports them as held_failed_lease_ids. Tests: retry-once writes the row; always-failing journal keeps the lease, cleans the other drive, and a second job cannot get the drive (mutation-checked: restoring the swallow fails it); recovery keeps a failed job's lease. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 31 +++++++ openblade/jobs/recovery.py | 19 +++- openblade/jobs/restore.py | 8 +- openblade/jobs/scheduler.py | 31 ++++++- openblade/jobs/sharded_archive.py | 51 +++++++++-- openblade/jobs/sharded_restore.py | 21 +++-- .../test_staged_sharded_archive.py | 88 ++++++++++++++++++- tests/unit/test_recovery.py | 17 ++++ 8 files changed, 245 insertions(+), 21 deletions(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 7f13a2a..eebb577 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -4,6 +4,7 @@ import json import threading +import time import uuid from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager @@ -14,6 +15,7 @@ from uuid import uuid4 from sqlalchemy import func, select +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session, selectinload from openblade.catalog.models import ( @@ -710,6 +712,35 @@ def journal( self.session.commit() return entry + # journal_durable: one attempt plus three retries after these backoffs. + _JOURNAL_RETRY_DELAYS = (0.05, 0.1, 0.2) + + def journal_durable( + self, job_id: str, event: str, detail: Mapping[str, object] | None = None + ) -> None: + """Append a journal event on a FRESH session over the same bind. + + For safety evidence written on an error path: ``self.session`` may already + be poisoned (PendingRollbackError) by the failure being recorded, so it is + never touched. Retries ``OperationalError`` (e.g. SQLite "database is + locked") with 50/100/200 ms backoff, then re-raises the last error. + """ + payload = json.dumps(dict(detail or {}), sort_keys=True, default=str) + for delay in (*self._JOURNAL_RETRY_DELAYS, None): + try: + with self._lease_session() as session: + session.add( + JobJournalEntry( + job_id=job_id, at=naive_utcnow(), event=event, detail_json=payload + ) + ) + session.commit() + return + except OperationalError: + if delay is None: + raise + time.sleep(delay) + def job_journal(self, job_id: str) -> list[JobJournalEntry]: """Every journal entry for ``job_id``, oldest first (ties broken by id).""" stmt = ( diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index 8068b2b..f38cac0 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -11,6 +11,7 @@ from openblade.catalog.repository import CatalogRepository, StagedInstance from openblade.domain.backends import LibraryBackend +from openblade.domain.models import DriveLease from openblade.jobs.inventory import InventoryService from openblade.jobs.reconcile import PendingReconciliation, drives_pending_reconciliation from openblade.jobs.scheduler import DEFAULT_LEASE_TTL @@ -18,6 +19,7 @@ logger = logging.getLogger(__name__) _NON_TERMINAL_JOB_STATES = frozenset({"pending", "running"}) +_FAILED_JOB_STATES = frozenset({"failed", "failed_recoverable"}) INTERRUPTED_ERROR = ( "interrupted: drive lease expired without a heartbeat; physical state unknown — " @@ -47,6 +49,10 @@ class RecoveryReport: # Drives whose physical state is unknown after a failed unmount/unload. Report # only: an operator reconciles each via POST /jobs/recovery/reconcile/{drive_id}. pending_reconciliation: list[PendingReconciliation] = field(default_factory=list) + # Live leases of FAILED jobs, deliberately NOT released: a failed job releases + # its leases in `finally` unless its physical_state_unknown could not be + # journaled (fail closed). Report only; the lease lapses at its TTL. + held_failed_lease_ids: list[str] = field(default_factory=list) def _is_older_than(created_at: datetime, ttl: timedelta, now: datetime) -> bool: @@ -77,12 +83,15 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - ) # Bookkeeping: a live lease whose job already reached a terminal state. live = catalog.live_leases() + held: list[DriveLease] = [] leased_job_ids = {lease.job_id for lease in live} for lease in live: owner = catalog.get_job(lease.job_id) # A pending or running owner may belong to another live process; its # lease is only reclaimed once it expires (handled above). - if owner is None or owner.state not in _NON_TERMINAL_JOB_STATES: + if owner is not None and owner.state in _FAILED_JOB_STATES: + held.append(lease) + elif owner is None or owner.state not in _NON_TERMINAL_JOB_STATES: orphaned.append(lease) catalog.release_leases([lease.id for lease in orphaned]) @@ -127,7 +136,15 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - staged_instances=staged, stale_pending_job_ids=stale_pending, pending_reconciliation=list(drives_pending_reconciliation(catalog).values()), + held_failed_lease_ids=[lease.id for lease in held], ) + for lease in held: + logger.error( + "recovery: keeping lease %s on drive %d (failed job %s); reconcile the drive", + lease.id, + lease.drive_id, + lease.job_id, + ) logger.info( "recovery: %d job(s) interrupted, %d lease(s) released, %d drive mismatch(es), " "%d staged instance(s) awaiting reconcile, %d stale pending job(s), " diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index dd5190c..d969445 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -337,9 +337,11 @@ def _restore_tape_group( # Still (maybe) mounted: unloading now could strand a dirty # index. Leave the cartridge in the drive for reconcile. unmount_failed = True - # One payload shape ({op, barcode, drive}) for every producer; the - # helper swallows a journal failure so it cannot replace this one. - record_physical_state_unknown(catalog, job_id, "unmount", barcode, drive_id) + # One payload shape ({op, barcode, drive}) for every producer. A + # journal failure flags the handle; release_drives keeps its lease. + record_physical_state_unknown( + catalog, job_id, "unmount", barcode, drive_id, handles=handles + ) tape_error = f"Unmount of {barcode} failed; left in drive for reconcile" except StaleLeaseError: fenced_out = True diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 2627986..75fb7a8 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -9,7 +9,12 @@ from typing import TYPE_CHECKING, Protocol from uuid import uuid4 -from openblade.domain.errors import DriveBusyError, DriveUnreconciledError, StaleLeaseError +from openblade.domain.errors import ( + DriveBusyError, + DriveUnreconciledError, + OpenBladeError, + StaleLeaseError, +) from openblade.domain.models import DriveLease if TYPE_CHECKING: @@ -18,6 +23,10 @@ logger = logging.getLogger(__name__) +class JournalWriteError(OpenBladeError): + """A drive's physical_state_unknown could not be journaled; its lease was kept.""" + + @dataclass class DriveHandle: # The scheduler lock key — the drive this lease reserves. IMMUTABLE for the @@ -33,6 +42,10 @@ class DriveHandle: lease_id: str = "" fencing_token: int = 0 _released: bool = field(default=False, init=False, repr=False) + # Set when this drive's physical_state_unknown could not be journaled: + # release_drives() then KEEPS the lease (fail closed) so no other job can + # LOAD/UNLOAD a drive that may still have LTFS mounted. + hold_unjournaled: bool = field(default=False, init=False, repr=False) @property def physical(self) -> int: @@ -374,7 +387,15 @@ def _heartbeat_loop(self) -> None: def release_drives(self, handles: list[DriveHandle]) -> None: """Release drives and notify waiting jobs.""" - pending = [handle for handle in handles if not handle._released] + for handle in handles: + if handle.hold_unjournaled and not handle._released: + logger.error( + "Keeping lease on drive %d (%s): physical_state_unknown was not " + "journaled; reconcile the drive before reuse", + handle.drive_id, + handle.barcode, + ) + pending = [h for h in handles if not h._released and not h.hold_unjournaled] self._store.release([handle.lease_id for handle in pending]) with self._lock: for handle in pending: @@ -384,6 +405,12 @@ def release_drives(self, handles: list[DriveHandle]) -> None: if not self._held: self._stop_heartbeat.set() self._lock.notify_all() + kept = [h.drive_id for h in handles if h.hold_unjournaled and not h._released] + if kept: + # Raised only after every other drive was released (collect, then raise). + raise JournalWriteError( + f"physical_state_unknown not journaled for drive(s) {kept}; leases kept" + ) def verify(self, handle: DriveHandle) -> None: """Raise StaleLeaseError unless the handle's lease is still live with its token.""" diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 588375c..24a68a7 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -7,6 +7,7 @@ import os import shutil import uuid +from collections.abc import Sequence from contextlib import suppress from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -18,7 +19,7 @@ from openblade.domain.models import MountHandle, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.reconcile import ensure_drive_reconciled -from openblade.jobs.scheduler import DriveHandle, DriveScheduler +from openblade.jobs.scheduler import DriveHandle, DriveScheduler, JournalWriteError from openblade.jobs.shard import ( DEFAULT_BLOCK_SIZE, ShardMode, @@ -357,9 +358,22 @@ def _journal_failure( def record_physical_state_unknown( - catalog: CatalogRepository, job_id: str, op: str, barcode: str, drive: int | None + catalog: CatalogRepository, + job_id: str, + op: str, + barcode: str, + drive: int | None, + *, + handles: Sequence[DriveHandle], ) -> None: - """Log + journal a cleanup op that raised; the drive must be reconciled before reuse.""" + """Log + journal a cleanup op that raised; the drive must be reconciled before reuse. + + Fail closed: the row is written via ``journal_durable`` (fresh session, retried). + If that still fails, the matching handle is flagged ``hold_unjournaled`` so + ``release_drives`` keeps its lease and then raises ``JournalWriteError`` -- + after every other drive was cleaned and released. With no matching handle + there is no lease to keep, so the error is raised here. + """ logger.exception( "job %s: %s of %s (drive %s) failed during cleanup; %s", job_id, @@ -369,11 +383,27 @@ def record_physical_state_unknown( PHYSICAL_STATE_UNKNOWN, ) try: - catalog.journal( + catalog.journal_durable( job_id, "physical_state_unknown", {"op": op, "barcode": barcode, "drive": drive} ) - except Exception: # noqa: BLE001 - journal is evidence, must not stop cleanup of other drives - logger.exception("job %s: could not journal physical_state_unknown", job_id) + except Exception: # noqa: BLE001 - any journal failure must fail closed, never pass + matched = [ + h + for h in handles + if h.barcode == barcode or (drive is not None and drive in (h.drive_id, h.physical)) + ] + logger.exception( + "job %s: could not journal physical_state_unknown for %s (drive %s); keeping its lease", + job_id, + barcode, + drive, + ) + if not matched: + raise JournalWriteError( + f"physical_state_unknown for {barcode} could not be journaled" + ) from None + for handle in matched: + handle.hold_unjournaled = True def _best_effort_unmount_and_unload( @@ -403,7 +433,12 @@ def _best_effort_unmount_and_unload( failed = True still_mounted.add(barcode) record_physical_state_unknown( - catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode) + catalog, + job_id, + "unmount", + barcode, + drive_by_barcode.get(barcode), + handles=handles, ) for handle in handles: slot_id = loaded_slots.get(handle.physical) @@ -430,7 +465,7 @@ def _best_effort_unmount_and_unload( except Exception: # noqa: BLE001 - recorded as physical_state_unknown, cleanup continues failed = True record_physical_state_unknown( - catalog, job_id, "unload", handle.barcode, handle.physical + catalog, job_id, "unload", handle.barcode, handle.physical, handles=handles ) if failed and errors is not None and not errors: errors.append(PHYSICAL_STATE_UNKNOWN) diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index b9e7c0d..42fae9c 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -137,10 +137,15 @@ def run_sharded_restore( def _cleanup_failed( - catalog: CatalogRepository, job_id: str, op: str, barcode: str, drive: int | None + catalog: CatalogRepository, + job_id: str, + op: str, + barcode: str, + drive: int | None, + handles: list[DriveHandle], ) -> None: """Journal a failed cleanup op; it is the job error only if none exists yet.""" - record_physical_state_unknown(catalog, job_id, op, barcode, drive) + record_physical_state_unknown(catalog, job_id, op, barcode, drive, handles=handles) job = catalog.get_job(job_id) if job is not None and not job.error: catalog.update_job_state(job_id, "failed", error=PHYSICAL_STATE_UNKNOWN) @@ -193,7 +198,7 @@ def _restore_single( except Exception: # noqa: BLE001 - recorded as physical_state_unknown cleanup_failed = True still_mounted = True - _cleanup_failed(catalog, job_id, "unmount", instance.barcode, drive_id) + _cleanup_failed(catalog, job_id, "unmount", instance.barcode, drive_id, handles) finally: # Never unload while LTFS is mounted or dirty: a failed unmount is # journaled above and the drive is left for reconciliation. @@ -214,7 +219,7 @@ def _restore_single( ) except Exception: # noqa: BLE001 - recorded as physical_state_unknown cleanup_failed = True - _cleanup_failed(catalog, job_id, "unload", instance.barcode, drive_id) + _cleanup_failed(catalog, job_id, "unload", instance.barcode, drive_id, handles) scheduler.release_drives(handles) if not cleanup_failed: @@ -297,7 +302,9 @@ def _read_shard(index: int, instance: Any) -> Path: except Exception: # noqa: BLE001 - recorded as physical_state_unknown cleanup_failed = True still_mounted_barcodes.add(barcode) - _cleanup_failed(catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode)) + _cleanup_failed( + catalog, job_id, "unmount", barcode, drive_by_barcode.get(barcode), handles + ) for handle in handles: slot_id = loaded_slots.get(handle.physical) # Never unload while LTFS is mounted or dirty (failed unmount above). @@ -318,7 +325,9 @@ def _read_shard(index: int, instance: Any) -> Path: ) except Exception: # noqa: BLE001 - recorded as physical_state_unknown cleanup_failed = True - _cleanup_failed(catalog, job_id, "unload", handle.barcode, handle.physical) + _cleanup_failed( + catalog, job_id, "unload", handle.barcode, handle.physical, handles + ) scheduler.release_drives(handles) if not cleanup_failed: diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py index 610cf5d..6d5d8dc 100644 --- a/tests/integration/test_staged_sharded_archive.py +++ b/tests/integration/test_staged_sharded_archive.py @@ -6,10 +6,13 @@ from pathlib import Path from typing import Any +import pytest from sqlalchemy import select +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session from openblade.catalog.models import FileInstance -from openblade.jobs.scheduler import DriveScheduler +from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore, JournalWriteError from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, run_sharded_archive from tests.integration.test_sharded_archive_atomicity import ( BARCODES, @@ -101,3 +104,86 @@ def unload(drive_id: int, target_slot: int) -> Any: assert BARCODES[0] in in_drives assert drives[BARCODES[0]] not in unload_requests assert "archived" not in _states(catalog) + + +def _stuck_unmount(ltfs: Any, monkeypatch: Any) -> None: + real_unmount = ltfs.unmount + + def unmount(mount: Any) -> Any: + if str(mount.barcode) == BARCODES[0]: + raise RuntimeError("stuck") + return real_unmount(mount) + + monkeypatch.setattr(ltfs, "unmount", unmount) + + +def test_journal_locked_once_is_retried_and_written(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + _stuck_unmount(ltfs, monkeypatch) + real_add = Session.add + calls = {"n": 0} + + def flaky_add(self: Session, instance: object, *args: Any, **kwargs: Any) -> None: + is_psu = getattr(instance, "event", None) == "physical_state_unknown" + if is_psu and calls["n"] == 0: + calls["n"] += 1 + raise OperationalError("INSERT", {}, Exception("database is locked")) + real_add(self, instance, *args, **kwargs) + + monkeypatch.setattr(Session, "add", flaky_add) + store = InMemoryLeaseStore() + job = catalog.create_job("archive", {}) + run_sharded_archive( + _request(_source(tmp_path)), + library, + ltfs, + catalog, + DriveScheduler(num_drives=2, store=store), + job.id, + ) + assert calls["n"] == 1 + rows = [e for e in catalog.job_journal(job.id) if e.event == "physical_state_unknown"] + assert rows and rows[0].detail["barcode"] == BARCODES[0] + assert store.live_leases() == [] # journaled -> released as before + + +def test_journal_failure_keeps_the_lease_fail_closed(tmp_path: Path, monkeypatch) -> None: + library, ltfs = _setup() + catalog = _catalog() + _stuck_unmount(ltfs, monkeypatch) + real_unload = library.unload + unloaded: list[int] = [] + + def unload(drive_id: int, target_slot: int) -> Any: + unloaded.append(drive_id) + return real_unload(drive_id, target_slot) + + monkeypatch.setattr(library, "unload", unload) + + def broken(*_args: Any, **_kwargs: Any) -> None: + raise OperationalError("INSERT", {}, Exception("database is locked")) + + monkeypatch.setattr(catalog, "journal_durable", broken) + store = InMemoryLeaseStore() + job = catalog.create_job("archive", {}) + with pytest.raises(JournalWriteError): + run_sharded_archive( + _request(_source(tmp_path)), + library, + ltfs, + catalog, + DriveScheduler(num_drives=2, store=store, job_id=job.id), + job.id, + ) + in_drives = {str(d.barcode) for d in library.inventory().drives if d.barcode} + assert BARCODES[1] not in in_drives # the other drive was still cleaned + assert unloaded # ...by a real UNLOAD + live = store.live_leases() + assert [lease.barcode for lease in live] == [BARCODES[0]] + # A second job must NOT be handed the drive that may still have LTFS mounted. + second = DriveScheduler(num_drives=2, store=store, job_id="second") + with pytest.raises(Exception): # noqa: B017,PT011 - any refusal; success is the bug + second.acquire_drives(["SECOND1", "SECOND2"], timeout=0.2) + got = second.acquire_drives(["SECOND1"], timeout=0.2) + assert got[0].drive_id != live[0].drive_id diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py index 42ba1e7..d8b2703 100644 --- a/tests/unit/test_recovery.py +++ b/tests/unit/test_recovery.py @@ -140,3 +140,20 @@ def test_recovery_reports_stale_pending_jobs_without_touching_them(tmp_path: Pat refreshed = catalog.get_job(job_id) assert refreshed is not None assert refreshed.state == "pending" + + +def test_recovery_keeps_live_lease_of_a_failed_job(tmp_path: Path) -> None: + # Fail closed: a failed job only leaves a live lease behind when it could not + # journal physical_state_unknown; auto-releasing it would hand the drive out. + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + failed = catalog.create_job("archive", {}) + catalog.update_job_state(failed.id, "failed", error="physical state unknown") + lease_id = _lease(catalog, failed.id, "MCK00001", ttl=timedelta(minutes=15)) + library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, library) + + assert report.released_lease_ids == [] + assert report.held_failed_lease_ids == [lease_id] + assert [lease.id for lease in catalog.live_leases()] == [lease_id] From 69f888b1de15075da4a7c7c67894467d958c67ed Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:46:07 +0000 Subject: [PATCH 164/205] fix(reconcile): order journal by id; exclude pending barcodes (S1, S2) - drives_pending_reconciliation sorts by autoincrement id only; `at` is the writer's wall clock, so a clock step could let a later physical_state_unknown be popped by an earlier drive_reconciled. - barcodes_pending_reconciliation: barcode-keyed pending set that also covers drive=None events (previously dropped -> excluded nothing). - ensure_drive_reconciled takes the barcode and refuses to load a pending barcode; all archive/restore load helpers pass it. Tests: clock-skewed `at` still pending (mutation-checked against the old (at, id) sort); a drive=None event blocks a load of that barcode until reconciled. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/reconcile.py | 43 ++++++++++++++++++++++++++++--- openblade/jobs/restore.py | 4 +-- openblade/jobs/sharded_archive.py | 4 +-- openblade/jobs/sharded_restore.py | 4 +-- tests/unit/test_reconcile.py | 33 ++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 9 deletions(-) diff --git a/openblade/jobs/reconcile.py b/openblade/jobs/reconcile.py index c4252b9..42d4064 100644 --- a/openblade/jobs/reconcile.py +++ b/openblade/jobs/reconcile.py @@ -45,7 +45,9 @@ def drives_pending_reconciliation(catalog: CatalogRepository) -> dict[int, Pendi events = catalog.journal_events(PHYSICAL_STATE_UNKNOWN) + catalog.journal_events( DRIVE_RECONCILED ) - events.sort(key=lambda entry: (entry.at, entry.id)) + # Order by autoincrement id only: every writer inserts into one table, while + # `at` is the writer's wall clock and a clock step would reorder events. + events.sort(key=lambda entry: entry.id) pending: dict[int, PendingReconciliation] = {} for entry in events: detail = entry.detail @@ -66,8 +68,43 @@ def drives_pending_reconciliation(catalog: CatalogRepository) -> dict[int, Pendi return pending -def ensure_drive_reconciled(catalog: CatalogRepository, drive_id: int) -> None: - """Refuse to use ``drive_id`` while its physical state is unknown.""" +def barcodes_pending_reconciliation(catalog: CatalogRepository) -> set[str]: + """Barcodes whose cartridge's physical state is unknown. + + Covers events journaled with ``drive=None`` (which no drive-keyed entry can + exclude): such an entry is cleared only by a later ``drive_reconciled`` + naming the same barcode. Drive-keyed pending entries contribute their barcode. + """ + events = catalog.journal_events(PHYSICAL_STATE_UNKNOWN) + catalog.journal_events( + DRIVE_RECONCILED + ) + events.sort(key=lambda entry: entry.id) + driveless: set[str] = set() + for entry in events: + barcode = entry.detail.get("barcode") + if barcode is None: + continue + if entry.event == DRIVE_RECONCILED: + driveless.discard(str(barcode)) + elif _drive_of(entry.detail) is None: + driveless.add(str(barcode)) + keyed = { + item.barcode + for item in drives_pending_reconciliation(catalog).values() + if item.barcode is not None + } + return driveless | keyed + + +def ensure_drive_reconciled( + catalog: CatalogRepository, drive_id: int, barcode: str | None = None +) -> None: + """Refuse to use ``drive_id`` (or load ``barcode``) while its physical state is unknown.""" + if barcode is not None and barcode in barcodes_pending_reconciliation(catalog): + raise DriveUnreconciledError( + f"Barcode {barcode} awaits reconciliation after a failed unmount/unload; " + "reconcile it before reuse" + ) item = drives_pending_reconciliation(catalog).get(drive_id) if item is not None: raise DriveUnreconciledError( diff --git a/openblade/jobs/restore.py b/openblade/jobs/restore.py index d969445..b61d1da 100644 --- a/openblade/jobs/restore.py +++ b/openblade/jobs/restore.py @@ -110,7 +110,7 @@ def _load_if_needed( barcode = handle.barcode drive_id = library.find_drive_by_barcode(barcode) if drive_id is not None: - ensure_drive_reconciled(catalog, drive_id) + ensure_drive_reconciled(catalog, drive_id, barcode) # Already in a drive: record it as the PHYSICAL drive; the lease key # (handle.drive_id) stays what the scheduler reserved. if drive_id != handle.drive_id: @@ -120,7 +120,7 @@ def _load_if_needed( if slot_id is None: raise CartridgeOfflineError(f"Cartridge {barcode} is offline") drive_id = handle.drive_id - ensure_drive_reconciled(catalog, drive_id) + ensure_drive_reconciled(catalog, drive_id, barcode) execute_tape_request( catalog, library, diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index 24a68a7..c6047f2 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -797,7 +797,7 @@ def _load_barcode( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: - ensure_drive_reconciled(catalog, loaded_drive_id) + ensure_drive_reconciled(catalog, loaded_drive_id, handle.barcode) # Cartridge is already in a drive. Record it as the PHYSICAL drive without # touching the scheduler lock key (handle.drive_id) — mutating that key would # leak the reserved drive and free a drive the scheduler never held. @@ -816,7 +816,7 @@ def _load_barcode( ) if slot_id is None: raise ValueError(f"Barcode {handle.barcode} not found in any slot") - ensure_drive_reconciled(catalog, handle.drive_id) + ensure_drive_reconciled(catalog, handle.drive_id, handle.barcode) execute_tape_request( catalog, library, diff --git a/openblade/jobs/sharded_restore.py b/openblade/jobs/sharded_restore.py index 42fae9c..0c99040 100644 --- a/openblade/jobs/sharded_restore.py +++ b/openblade/jobs/sharded_restore.py @@ -349,7 +349,7 @@ def _ensure_loaded( ) -> tuple[int, int | None]: loaded_drive_id = library.find_drive_by_barcode(handle.barcode) if loaded_drive_id is not None: - ensure_drive_reconciled(catalog, loaded_drive_id) + ensure_drive_reconciled(catalog, loaded_drive_id, handle.barcode) # Record the physical drive without mutating the scheduler lock key # (handle.drive_id); mutating it leaks the reserved drive on release. if loaded_drive_id != handle.drive_id: @@ -359,7 +359,7 @@ def _ensure_loaded( slot_id = library.find_slot_by_barcode(handle.barcode) if slot_id is None: raise CartridgeOfflineError(f"Barcode {handle.barcode} not found in library") - ensure_drive_reconciled(catalog, handle.drive_id) + ensure_drive_reconciled(catalog, handle.drive_id, handle.barcode) execute_tape_request( catalog, library, diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py index 002c659..16add34 100644 --- a/tests/unit/test_reconcile.py +++ b/tests/unit/test_reconcile.py @@ -108,3 +108,36 @@ def test_reconcile_api_round_trip(tmp_path: Path) -> None: assert drives_pending_reconciliation(context.catalog) == {} assert client.post("/jobs/recovery/reconcile/99").status_code == 404 + + +def test_pending_orders_by_id_not_wall_clock(tmp_path: Path) -> None: + from datetime import timedelta + + from openblade.jobs.reconcile import drives_pending_reconciliation + + catalog = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'c.db'}")).catalog + job = catalog.create_job("archive", {}) + catalog.journal(job.id, "drive_reconciled", {"drive": 1, "barcode": "MCK00001"}) + later = catalog.journal( + job.id, "physical_state_unknown", {"op": "unmount", "barcode": "MCK00001", "drive": 1} + ) + later.at = later.at - timedelta(hours=1) # writer clock stepped back + catalog.session.commit() + + assert 1 in drives_pending_reconciliation(catalog) + + +def test_driveless_event_still_blocks_a_load_of_that_barcode(tmp_path: Path) -> None: + from openblade.jobs.reconcile import barcodes_pending_reconciliation, ensure_drive_reconciled + + catalog = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'c.db'}")).catalog + job = catalog.create_job("archive", {}) + catalog.journal( + job.id, "physical_state_unknown", {"op": "unmount", "barcode": "MCK00002", "drive": None} + ) + assert barcodes_pending_reconciliation(catalog) == {"MCK00002"} + with pytest.raises(DriveUnreconciledError, match="MCK00002"): + ensure_drive_reconciled(catalog, 0, "MCK00002") + ensure_drive_reconciled(catalog, 0, "MCK00003") # other barcodes unaffected + catalog.journal(job.id, "drive_reconciled", {"drive": 0, "barcode": "MCK00002"}) + assert barcodes_pending_reconciliation(catalog) == set() From 29857bacd94adeda894aefa8e43cf460ef3e649f Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:46:59 +0000 Subject: [PATCH 165/205] fix(nas-capacity): atomic reservation check-and-insert under BEGIN IMMEDIATE S6: ReservationLedger.reserve (catalog mode) now goes through CatalogRepository.reserve_nas_capacity, which re-sums live reservations and inserts inside one BEGIN IMMEDIATE on a dedicated connection. Regression test: 20 racers over two sessions each want 60% -> exactly one wins (old path over-committed 2-4x, mutation-checked). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 39 ++++++++++++++++++++++++++++++ openblade/nas/capacity.py | 15 ++++++++---- tests/unit/test_nas_persistence.py | 31 ++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 7f13a2a..449658e 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -1362,6 +1362,45 @@ def upsert_nas_reservation(self, data: dict[str, object]) -> dict[str, object]: self.session.commit() return self._nas_reservation_to_dict(row) + def reserve_nas_capacity( + self, + pool_id: str, + nbytes: int, + owner: str, + expires_at: datetime, + *, + capacity_bytes: int, + now: datetime, + ) -> bool: + """Atomically insert a reservation only if it fits under ``capacity_bytes``. + + Check-and-insert run inside one ``BEGIN IMMEDIATE`` on a dedicated + connection, so two processes racing for the last bytes cannot both pass + the check (reviewer S6). Returns whether the reservation was inserted. + """ + with self._lock, self._lease_session() as session: + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + rows = session.scalars(select(NasReservation).where(NasReservation.pool_id == pool_id)) + reserved = sum( + int(row.bytes) + for row in rows + if cast(datetime, self._nas_reservation_to_dict(row)["expires_at"]) > now + ) + if nbytes > capacity_bytes - reserved: + session.rollback() + return False + session.add( + NasReservation( + id=str(uuid.uuid4()), + pool_id=pool_id, + bytes=nbytes, + owner=owner, + expires_at=expires_at.isoformat(), + ) + ) + session.commit() + return True + def delete_nas_reservation(self, reservation_id: str) -> bool: return self._delete_nas_row(NasReservation, reservation_id) diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index cadcd47..84c4435 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -129,13 +129,18 @@ def reserve( free_bytes: int, now: datetime, ) -> CapacityReservation: - if nbytes > self.available(pool_id, free_bytes, now): - raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") res = CapacityReservation(pool_id=pool_id, bytes=nbytes, owner=owner, expires_at=now + ttl) if self._catalog is not None: - self._catalog.upsert_nas_reservation(res.model_dump()) - else: - self._items.append(res) + # Check-and-insert is one BEGIN IMMEDIATE transaction in the catalog, + # so concurrent processes cannot both claim the last bytes. + if not self._catalog.reserve_nas_capacity( + pool_id, nbytes, owner, res.expires_at, capacity_bytes=free_bytes, now=now + ): + raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") + return res + if nbytes > self.available(pool_id, free_bytes, now): + raise ReservationError(f"pool {pool_id} cannot reserve {nbytes} bytes") + self._items.append(res) return res def release(self, pool_id: str, owner: str) -> int: diff --git a/tests/unit/test_nas_persistence.py b/tests/unit/test_nas_persistence.py index fe1ca6f..f28fa60 100644 --- a/tests/unit/test_nas_persistence.py +++ b/tests/unit/test_nas_persistence.py @@ -2,6 +2,7 @@ from __future__ import annotations +import threading from datetime import UTC, datetime, timedelta from pathlib import Path @@ -226,3 +227,33 @@ def test_api_exposes_pool_protection_and_persisted_lists(tmp_path: Path) -> None assert client.get("/nas/reservations", params={"pool_id": "other"}).json() == [] assert client.get("/nas/export-sets").json()[0]["id"] == "e1" assert client.get("/nas/vaults").json()[0]["location"] == "offsite" + + +def test_concurrent_reservations_across_processes_never_overcommit(tmp_path: Path) -> None: + """Regression S6: 20 racers over two catalog sessions each want 60% -> exactly one wins.""" + init_db(f"sqlite:///{tmp_path / 'race.db'}") + ledgers = [ReservationLedger(CatalogRepository(get_session())) for _ in range(2)] + start = threading.Barrier(20) + won: list[str] = [] + errors: list[BaseException] = [] + + def racer(i: int) -> None: + start.wait() + try: + ledgers[i % 2].reserve( + "p1", 600, f"owner-{i}", timedelta(minutes=5), free_bytes=1000, now=_NOW + ) + won.append(f"owner-{i}") + except ReservationError: + pass + except BaseException as exc: # noqa: BLE001 - surfaced by the assertion below + errors.append(exc) + + threads = [threading.Thread(target=racer, args=(i,)) for i in range(20)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert errors == [] + assert len(won) == 1 + assert ledgers[0].reserved("p1", _NOW) == 600 From c04239158bbb8fcef56928b4fa0a5c7a2cc6edd2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:49:16 +0000 Subject: [PATCH 166/205] fix(scheduler): refuse barcodes pending reconciliation acquire_drives raises DriveUnreconciledError naming the barcode when a requested cartridge has an unreconciled physical_state_unknown entry. Mutation-checked: disabling the guard fails both new tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/scheduler.py | 22 ++++++++++++++++++++++ tests/unit/test_reconcile.py | 27 ++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index 75fb7a8..a2ef69d 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -81,6 +81,10 @@ def unreconciled_drives(self) -> dict[int, str | None]: """Drives awaiting reconciliation (physical state unknown) -> last barcode.""" ... + def unreconciled_barcodes(self) -> set[str]: + """Barcodes whose cartridge may still sit in a drive awaiting reconciliation.""" + ... + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: ... def set_physical_drive(self, lease_id: str, physical_drive_id: int) -> None: ... @@ -100,6 +104,8 @@ class InMemoryLeaseStore: def __init__(self) -> None: # Drive -> barcode for drives whose physical state is unknown (tests set this). self.unreconciled: dict[int, str | None] = {} + # Barcodes pending reconciliation (tests set this). + self.unreconciled_barcode_set: set[str] = set() self._lock = threading.Lock() self._leases: dict[str, DriveLease] = {} self._last_token = 0 @@ -164,6 +170,9 @@ def acquire_drive_lease_at( def unreconciled_drives(self) -> dict[int, str | None]: return dict(self.unreconciled) + def unreconciled_barcodes(self) -> set[str]: + return set(self.unreconciled_barcode_set) + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: with self._lock: now = _now() @@ -235,6 +244,11 @@ def unreconciled_drives(self) -> dict[int, str | None]: for drive_id, item in drives_pending_reconciliation(self._repo).items() } + def unreconciled_barcodes(self) -> set[str]: + from openblade.jobs.reconcile import barcodes_pending_reconciliation # local: no cycle + + return barcodes_pending_reconciliation(self._repo) + def heartbeat(self, lease_ids: list[str], ttl: timedelta) -> None: self._repo.heartbeat_leases(lease_ids, ttl) @@ -317,6 +331,14 @@ def acquire_drives( while True: # A drive whose physical state is unknown (failed unmount/unload) # may still hold a mounted tape: never a candidate until reconciled. + # A cartridge awaiting reconciliation may still be mounted in some + # drive: refuse it outright rather than move it. + pending = sorted(set(barcodes) & self._store.unreconciled_barcodes()) + if pending: + raise DriveUnreconciledError( + f"Tape(s) {', '.join(pending)} awaiting reconciliation after a " + "failed unmount/unload; reconcile before reuse" + ) unreconciled = self._store.unreconciled_drives() excluded = frozenset(d for d in unreconciled if 0 <= d < self._num_drives) if self._num_drives - len(excluded) < len(barcodes): diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py index 16add34..a6fae3b 100644 --- a/tests/unit/test_reconcile.py +++ b/tests/unit/test_reconcile.py @@ -20,7 +20,12 @@ reconcile_drive, ) from openblade.jobs.restore import _load_if_needed -from openblade.jobs.scheduler import DriveHandle, DriveScheduler, InMemoryLeaseStore +from openblade.jobs.scheduler import ( + CatalogLeaseStore, + DriveHandle, + DriveScheduler, + InMemoryLeaseStore, +) from openblade.simulator.library import MockLibraryBackend @@ -141,3 +146,23 @@ def test_driveless_event_still_blocks_a_load_of_that_barcode(tmp_path: Path) -> ensure_drive_reconciled(catalog, 0, "MCK00003") # other barcodes unaffected catalog.journal(job.id, "drive_reconciled", {"drive": 0, "barcode": "MCK00002"}) assert barcodes_pending_reconciliation(catalog) == set() + + +def test_scheduler_refuses_barcode_pending_reconciliation() -> None: + store = InMemoryLeaseStore() + store.unreconciled_barcode_set = {"MCK00001"} + scheduler = DriveScheduler(3, store=store) + with pytest.raises(DriveUnreconciledError, match="MCK00001"): + scheduler.acquire_drives(["MCK00002", "MCK00001"], timeout=0.1) + # Other barcodes are unaffected, and no lease leaked from the refusal. + handles = scheduler.acquire_drives(["MCK00002", "MCK00003", "MCK00004"], timeout=0.1) + assert len(handles) == 3 + scheduler.release_drives(handles) + + +def test_catalog_store_reports_driveless_pending_barcode(tmp_path: Path) -> None: + context = _context(tmp_path) + _mark_unknown(context, None, barcode="MCK00007") + scheduler = DriveScheduler(2, store=CatalogLeaseStore(context.catalog)) + with pytest.raises(DriveUnreconciledError, match="MCK00007"): + scheduler.acquire_drives(["MCK00007"], timeout=0.1) From d722f592cea5baf2c75b3e96777b6b63fa462c2d Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:51:42 +0000 Subject: [PATCH 167/205] fix(jobs): keep cleanup and the original lane error when lease release fails release_drives raising JournalWriteError inside a finally replaced the lane's real failure. It now logs the in-flight exception at ERROR and appends its safe_job_error text; _archive_stripe (lane error already caught into errors) carries the summary into the raise. shard_dir rmtree runs even if release raises. Both message paths mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/jobs/scheduler.py | 14 +++++++--- openblade/jobs/sharded_archive.py | 19 ++++++++++--- .../test_staged_sharded_archive.py | 4 ++- tests/unit/test_reconcile.py | 27 ++++++++++++++++++- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index a2ef69d..bd555ed 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import sys import threading from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta @@ -14,6 +15,7 @@ DriveUnreconciledError, OpenBladeError, StaleLeaseError, + safe_job_error, ) from openblade.domain.models import DriveLease @@ -429,10 +431,16 @@ def release_drives(self, handles: list[DriveHandle]) -> None: self._lock.notify_all() kept = [h.drive_id for h in handles if h.hold_unjournaled and not h._released] if kept: + message = f"physical_state_unknown not journaled for drive(s) {kept}; leases kept" + # Called from a `finally` that is unwinding the lane's real failure: + # this raise replaces it (it survives only as __context__), so log it + # and name it in the message the job records. + original = sys.exception() + if isinstance(original, Exception) and not isinstance(original, JournalWriteError): + logger.error("Lane failed before lease release", exc_info=original) + message += f"; original failure: {safe_job_error(original)}" # Raised only after every other drive was released (collect, then raise). - raise JournalWriteError( - f"physical_state_unknown not journaled for drive(s) {kept}; leases kept" - ) + raise JournalWriteError(message) def verify(self, handle: DriveHandle) -> None: """Raise StaleLeaseError unless the handle's lease is still live with its token.""" diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index c6047f2..e085b19 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -647,7 +647,18 @@ def _write_one( _best_effort_unmount_and_unload( catalog, library, ltfs, mounts, handles, loaded_slots, job_id, errors ) - scheduler.release_drives(handles) + try: + scheduler.release_drives(handles) + except JournalWriteError as release_exc: + # The lane error was caught above, so it is not in flight here and + # would be lost with `errors`: carry it into the raised message. + if not errors: + raise + summary = _summarize_errors(errors) + logger.error("Lane failed before lease release: %s", summary) + raise JournalWriteError( + f"{release_exc}; original failure: {summary}" + ) from release_exc return files_archived, bytes_archived @@ -784,8 +795,10 @@ def _write_shard(spec: ShardSpec, shard_tmp: Path) -> tuple[int, str, int]: _best_effort_unmount_and_unload( catalog, library, ltfs, mounts, handles, loaded_slots, job_id, None ) - scheduler.release_drives(handles) - shutil.rmtree(shard_dir, ignore_errors=True) + try: + scheduler.release_drives(handles) # may raise JournalWriteError + finally: + shutil.rmtree(shard_dir, ignore_errors=True) def _load_barcode( diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py index 6d5d8dc..67b50c8 100644 --- a/tests/integration/test_staged_sharded_archive.py +++ b/tests/integration/test_staged_sharded_archive.py @@ -167,7 +167,7 @@ def broken(*_args: Any, **_kwargs: Any) -> None: monkeypatch.setattr(catalog, "journal_durable", broken) store = InMemoryLeaseStore() job = catalog.create_job("archive", {}) - with pytest.raises(JournalWriteError): + with pytest.raises(JournalWriteError) as caught: run_sharded_archive( _request(_source(tmp_path)), library, @@ -176,6 +176,8 @@ def broken(*_args: Any, **_kwargs: Any) -> None: DriveScheduler(num_drives=2, store=store, job_id=job.id), job.id, ) + # The job's recorded error still names the lane failure, not only the journal. + assert "original failure: " in str(caught.value) in_drives = {str(d.barcode) for d in library.inventory().drives if d.barcode} assert BARCODES[1] not in in_drives # the other drive was still cleaned assert unloaded # ...by a real UNLOAD diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py index a6fae3b..f670132 100644 --- a/tests/unit/test_reconcile.py +++ b/tests/unit/test_reconcile.py @@ -11,7 +11,7 @@ from openblade.api import routes_jobs from openblade.bootstrap import AppContext, create_context, get_context from openblade.config import OpenBladeConfig -from openblade.domain.errors import DriveUnreconciledError +from openblade.domain.errors import DriveUnreconciledError, TapeFullError, safe_job_error from openblade.domain.models import MountState from openblade.jobs.reconcile import ( DRIVE_RECONCILED, @@ -25,6 +25,7 @@ DriveHandle, DriveScheduler, InMemoryLeaseStore, + JournalWriteError, ) from openblade.simulator.library import MockLibraryBackend @@ -166,3 +167,27 @@ def test_catalog_store_reports_driveless_pending_barcode(tmp_path: Path) -> None scheduler = DriveScheduler(2, store=CatalogLeaseStore(context.catalog)) with pytest.raises(DriveUnreconciledError, match="MCK00007"): scheduler.acquire_drives(["MCK00007"], timeout=0.1) + + +def test_release_failure_in_finally_names_the_original_lane_error() -> None: + scheduler = DriveScheduler(2, store=InMemoryLeaseStore()) + [handle] = scheduler.acquire_drives(["MCK00001"], timeout=0.1) + handle.hold_unjournaled = True + with pytest.raises(JournalWriteError) as caught: + try: + raise TapeFullError("tape MCK00001 is full") + finally: + scheduler.release_drives([handle]) + recorded = safe_job_error(caught.value) # what the job queue stores + assert "not journaled" in recorded + assert "original failure: tape MCK00001 is full" in recorded + assert isinstance(caught.value.__context__, TapeFullError) + + +def test_release_failure_without_lane_error_keeps_plain_message() -> None: + scheduler = DriveScheduler(2, store=InMemoryLeaseStore()) + [handle] = scheduler.acquire_drives(["MCK00001"], timeout=0.1) + handle.hold_unjournaled = True + with pytest.raises(JournalWriteError) as caught: + scheduler.release_drives([handle]) + assert "original failure" not in str(caught.value) From d3c1d75833237e8668e3699b101745012f760de5 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:53:43 +0000 Subject: [PATCH 168/205] fix(recovery): skip queue-claim leases in mismatch check; JournalWriteError is a SafetyViolationError recover_after_restart no longer reports JobQueue slot claims (drive_id < 0, QUEUE-CLAIM:/CHANGER: barcodes) as drive mismatches (S4). JournalWriteError moves to domain.errors as a SafetyViolationError subclass; imports updated. Mutation-checked: removing the skip fails the new test. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/domain/errors.py | 4 ++++ openblade/jobs/recovery.py | 11 ++++++++++ openblade/jobs/scheduler.py | 6 +---- openblade/jobs/sharded_archive.py | 9 ++++++-- .../test_staged_sharded_archive.py | 3 ++- tests/unit/test_reconcile.py | 8 +++++-- tests/unit/test_recovery.py | 22 +++++++++++++++++++ 7 files changed, 53 insertions(+), 10 deletions(-) diff --git a/openblade/domain/errors.py b/openblade/domain/errors.py index c528615..cb42aa3 100644 --- a/openblade/domain/errors.py +++ b/openblade/domain/errors.py @@ -54,6 +54,10 @@ class DriveUnreconciledError(SafetyViolationError): """A drive's physical state is unknown after a failed unmount/unload; reconcile first.""" +class JournalWriteError(SafetyViolationError): + """A drive's physical_state_unknown could not be journaled; its lease was kept.""" + + class DriveBusyError(OpenBladeError): """Drive is busy.""" diff --git a/openblade/jobs/recovery.py b/openblade/jobs/recovery.py index f38cac0..b8162bb 100644 --- a/openblade/jobs/recovery.py +++ b/openblade/jobs/recovery.py @@ -16,6 +16,15 @@ from openblade.jobs.reconcile import PendingReconciliation, drives_pending_reconciliation from openblade.jobs.scheduler import DEFAULT_LEASE_TTL +# Synthetic barcodes JobQueue stores on its slot-claim leases (`f"{kind}:{slot}"`, +# kinds _DRIVE_KIND / _CHANGER_KIND in jobs/queue.py). They never name a cartridge. +_QUEUE_CLAIM_PREFIXES = ("QUEUE-CLAIM:", "CHANGER:") + + +def _is_queue_claim(lease: DriveLease) -> bool: + return lease.drive_id < 0 or lease.barcode.startswith(_QUEUE_CLAIM_PREFIXES) + + logger = logging.getLogger(__name__) _NON_TERMINAL_JOB_STATES = frozenset({"pending", "running"}) @@ -101,6 +110,8 @@ def recover_after_restart(catalog: CatalogRepository, library: LibraryBackend) - } mismatches: list[DriveMismatch] = [] for lease in orphaned: + if _is_queue_claim(lease): + continue # a JobQueue slot claim, not a tape in a drive: nothing to compare physical = ( lease.physical_drive_id if lease.physical_drive_id is not None else lease.drive_id ) diff --git a/openblade/jobs/scheduler.py b/openblade/jobs/scheduler.py index bd555ed..0ec60e3 100644 --- a/openblade/jobs/scheduler.py +++ b/openblade/jobs/scheduler.py @@ -13,7 +13,7 @@ from openblade.domain.errors import ( DriveBusyError, DriveUnreconciledError, - OpenBladeError, + JournalWriteError, StaleLeaseError, safe_job_error, ) @@ -25,10 +25,6 @@ logger = logging.getLogger(__name__) -class JournalWriteError(OpenBladeError): - """A drive's physical_state_unknown could not be journaled; its lease was kept.""" - - @dataclass class DriveHandle: # The scheduler lock key — the drive this lease reserves. IMMUTABLE for the diff --git a/openblade/jobs/sharded_archive.py b/openblade/jobs/sharded_archive.py index e085b19..a8e9029 100644 --- a/openblade/jobs/sharded_archive.py +++ b/openblade/jobs/sharded_archive.py @@ -15,11 +15,16 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.capacity import has_room_for -from openblade.domain.errors import StaleLeaseError, TapeFullError, safe_job_error +from openblade.domain.errors import ( + JournalWriteError, + StaleLeaseError, + TapeFullError, + safe_job_error, +) from openblade.domain.models import MountHandle, MountMode from openblade.jobs.inventory import InventoryService from openblade.jobs.reconcile import ensure_drive_reconciled -from openblade.jobs.scheduler import DriveHandle, DriveScheduler, JournalWriteError +from openblade.jobs.scheduler import DriveHandle, DriveScheduler from openblade.jobs.shard import ( DEFAULT_BLOCK_SIZE, ShardMode, diff --git a/tests/integration/test_staged_sharded_archive.py b/tests/integration/test_staged_sharded_archive.py index 67b50c8..c52652b 100644 --- a/tests/integration/test_staged_sharded_archive.py +++ b/tests/integration/test_staged_sharded_archive.py @@ -12,7 +12,8 @@ from sqlalchemy.orm import Session from openblade.catalog.models import FileInstance -from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore, JournalWriteError +from openblade.domain.errors import JournalWriteError +from openblade.jobs.scheduler import DriveScheduler, InMemoryLeaseStore from openblade.jobs.sharded_archive import PHYSICAL_STATE_UNKNOWN, run_sharded_archive from tests.integration.test_sharded_archive_atomicity import ( BARCODES, diff --git a/tests/unit/test_reconcile.py b/tests/unit/test_reconcile.py index f670132..14bb0c3 100644 --- a/tests/unit/test_reconcile.py +++ b/tests/unit/test_reconcile.py @@ -11,7 +11,12 @@ from openblade.api import routes_jobs from openblade.bootstrap import AppContext, create_context, get_context from openblade.config import OpenBladeConfig -from openblade.domain.errors import DriveUnreconciledError, TapeFullError, safe_job_error +from openblade.domain.errors import ( + DriveUnreconciledError, + JournalWriteError, + TapeFullError, + safe_job_error, +) from openblade.domain.models import MountState from openblade.jobs.reconcile import ( DRIVE_RECONCILED, @@ -25,7 +30,6 @@ DriveHandle, DriveScheduler, InMemoryLeaseStore, - JournalWriteError, ) from openblade.simulator.library import MockLibraryBackend diff --git a/tests/unit/test_recovery.py b/tests/unit/test_recovery.py index d8b2703..a61c2ec 100644 --- a/tests/unit/test_recovery.py +++ b/tests/unit/test_recovery.py @@ -157,3 +157,25 @@ def test_recovery_keeps_live_lease_of_a_failed_job(tmp_path: Path) -> None: assert report.released_lease_ids == [] assert report.held_failed_lease_ids == [lease_id] assert [lease.id for lease in catalog.live_leases()] == [lease_id] + + +def test_recovery_ignores_queue_claim_leases_but_reports_real_mismatches(tmp_path: Path) -> None: + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'r.db'}")) + catalog = context.catalog + job = catalog.create_job("archive", {}) + catalog.update_job_state(job.id, "running") + expired = timedelta(seconds=-1) + # JobQueue's slot claims (jobs/queue.py): the changer at -1 and drive 1. + for drive_id, barcode in ((-1, "CHANGER:-1"), (1, "QUEUE-CLAIM:1")): + claim = catalog.acquire_drive_lease_at( + job_id=job.id, drive_id=drive_id, barcode=barcode, ttl=expired + ) + assert claim is not None + _lease(catalog, job.id, "MCK00001", ttl=expired) # a real tape lease + empty_library = MockLibraryBackend(num_slots=4, num_drives=2, num_import_export_slots=1) + + report = recover_after_restart(catalog, empty_library) + + assert [(m.expected_barcode, m.observed_barcode) for m in report.mismatches] == [ + ("MCK00001", None) + ] From e5ee7048978270a2eeb44db683d71e5b6a15edf4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:53:48 +0000 Subject: [PATCH 169/205] style: ruff format test_sharded_archive_atomicity.py (format-only) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_sharded_archive_atomicity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_sharded_archive_atomicity.py b/tests/integration/test_sharded_archive_atomicity.py index b538988..0758161 100644 --- a/tests/integration/test_sharded_archive_atomicity.py +++ b/tests/integration/test_sharded_archive_atomicity.py @@ -76,7 +76,9 @@ def test_load_helpers_do_not_mutate_scheduler_lock_key() -> None: assert slot is not None library.load(slot, 1) # cartridge physically in drive 1 handle = DriveHandle(drive_id=0, barcode=BARCODES[0]) # scheduler reserved drive 0 - physical, _slot = helper(_catalog(), library, ltfs, handle, "job") # already-loaded fast path + physical, _slot = helper( + _catalog(), library, ltfs, handle, "job" + ) # already-loaded fast path assert handle.drive_id == 0, f"{helper.__name__} mutated the scheduler lock key" assert handle.physical == 1 and physical == 1 From c13cf248f399a2ea6c5395a622cd5ad6782cbd6e Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:53:58 +0000 Subject: [PATCH 170/205] docs(decisions): reconciliation fail-closed rule, barcode exclusion, TTL residual Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index e0502f6..a97aa0c 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -239,6 +239,21 @@ pending while its latest `physical_state_unknown` has no later `drive_reconciled - `POST /jobs/recovery/reconcile/{drive_id}` takes a fresh inventory snapshot, refuses (409) while the drive still reports a mount, else journals `drive_reconciled` (`{"drive","observed_barcode"}`); 404 for an unknown drive. +- **Barcode-level exclusion.** A pending entry also blocks its *barcode*: + `acquire_drives` raises `DriveUnreconciledError` naming any requested barcode in + `barcodes_pending_reconciliation` (this covers entries with no drive id). +- **Fail closed on journal failure.** If `physical_state_unknown` itself cannot be + journaled, the drive's lease is KEPT (`DriveHandle.hold_unjournaled`) and + `release_drives` raises `JournalWriteError` (a `SafetyViolationError`) after + releasing every other drive. The message names the original lane failure so the + job's recorded error is not only "journal write failed". Recovery likewise keeps + live leases of failed jobs and reports them in `held_failed_lease_ids`. + JobQueue slot claims (`drive_id < 0`, `QUEUE-CLAIM:`/`CHANGER:` barcodes) are not + compared against inventory as drive mismatches. +- **Residual risk.** A held lease lives only as long as the process heartbeats it. + Once the process dies, the lease lapses at its TTL and, because the journal write + failed, no pending entry exists. The lease TTL is therefore the reconciliation + window for that case: an operator must reconcile before it expires. ## JobQueue follow-up landed From 6db8d738a947b87e960b7a15c95e75df4165d2f8 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 14:59:07 +0000 Subject: [PATCH 171/205] docs: roadmap status 2026-10-10 and tech-debt after round-2 fleet Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/production-readiness-roadmap.md | 16 ++++++++-------- docs/tech-debt.md | 27 +++++++++++++++++---------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index 4ff3b57..2a42716 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -26,7 +26,7 @@ Verified against the repo before writing (not asserted from memory): - `sharded_archive.py:234,251` calls `mark_instance_archived` per-shard inside the write loop → review #6 confirmed real. ✅ - Review #5 (scheduler `drive_id` mutation) **not reproduced**: `jobs/scheduler.py` is a plain allocator with no `_load_barcode`/mutation — re-investigate before "fixing." ⚠️ -## Status as of 2026-10-09 (fleet integration on `feat/persistent-drive-leases`) +## Status as of 2026-10-10 (fleet integration on `feat/persistent-drive-leases`) Verified against the tree, not asserted. "Landed" = implemented and tested in-repo; "appliance-gated" = authored and runnable, cannot be certified without the rig. @@ -36,11 +36,11 @@ Verified against the tree, not asserted. "Landed" = implemented and tested in-re | 1 | Relabel simplified endpoint | landed | `routes_aml_move_medium.py` | | 2 | ScalarCoordinate / MoveClass IntFlag | landed (bit values appliance-gated) | `domain/scalar_coordinate.py` | | 3 | Compatibility corpus + differential harness | scaffold landed; 0 captured cases (appliance-gated) | `compatibility/`, `tests/compat/` | -| 4 | Atomic sharded commit | landed: STAGING→VERIFYING→one ARCHIVED commit, job journal, no suppressed cleanup errors; resume = report-only | `jobs/sharded_archive.py`, `catalog/repository.py`, `jobs/recovery.py` | -| 5 | Protocol-agnostic backends + contract suite | landed (sim, in-process emulator, real gated); scsi+fake pairing TODO | `tests/contract/` | +| 4 | Atomic sharded commit | landed: STAGING→VERIFYING→one ARCHIVED commit, job journal, no suppressed cleanup errors; `physical_state_unknown` now has a consumer (drive/barcode refusal, `reconcile_drive`, fail-closed when the journal write fails); resume = report-only | `jobs/sharded_archive.py`, `catalog/repository.py`, `jobs/recovery.py` | +| 5 | Protocol-agnostic backends + contract suite | landed (sim, in-process emulator, scsi+fake-ltfs over a stateful mtx fake, real gated) | `tests/contract/` | | 6 | Scheduler lease regression | landed | `tests/unit/test_drive_leases.py` | | 7 | Persistent drive leases | landed | `catalog/models.py::DriveLeaseRecord`, `jobs/scheduler.py` | -| 8 | Persistent, resumable jobs | landed (recovery + report); resume-from-staged not implemented | `jobs/recovery.py`, `GET /jobs/recovery` | +| 8 | Persistent, resumable jobs | landed (recovery + report, `JobQueue` persistent over catalog jobs + leases, `POST /jobs/recovery/reconcile/{drive}`); resume-from-staged not implemented | `jobs/recovery.py`, `jobs/queue.py`, `GET /jobs/recovery` | | 9 | Reconstructible catalog | landed: manifest v2, commit marker last, generation pointer, sibling tapes, uncommitted-generation rebuild rule | `nas/ltfs_manifest.py`, `nas/archive_lifecycle.py`, `nas/catalog_rebuild.py` | | 10 | Separate the axes | landed (models + manifest/catalog carry); erasure coding modelled, unsupported | `domain/protection.py` | | 11 | Aggregation gate + ownership check | landed | `ci.yml::ci-gate`, `tools/ci_ownership.py` | @@ -49,11 +49,11 @@ Verified against the tree, not asserted. "Landed" = implemented and tested in-re | 14 | Lint/type ratchet | landed (both blocking, both at zero) | `ci.yml` | | 15 | Workflow hardening | landed: SHA pins, per-job permissions, actionlint/zizmor/shellcheck/yamllint | `workflow-lint.yml` | | 16 | Differential + mutation | mutation lane landed (mutmut 3.8.0, ratchet baseline for 6/6 modules as of 2026-10-10); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | -| 17 | Timing profiles + virtual clock | landed (10 profiles; 4 simulated client-side) | `tests/i3/timing.py` | +| 17 | Timing profiles + virtual clock | landed (10 profiles; the 4 fault profiles are injected by the emulator via `OPENBLADE_EMULATOR_FAULT_PROFILE`) | `tests/i3/timing.py` | | 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-required pending state not modelled | `routes_aml_auth.py`, `compatibility/` | -| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 31 scenarios pass locally; SMB byte-range lock + NFS ACL skip) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | -| 20 | Tape-native NAS models | landed as models + semantic/negative tests; catalog persistence TODO | `nas/state_machine.py`, `nas/media.py`, `nas/capacity.py` | -| 21 | FUSE data plane | landed: hydrator, cache, xattr state, gateway; see `docs/fuse.md` for what is wired | `fuse/` | +| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 33/33 pass locally incl. real SMB byte-range lock conflict; NFSv4 ACL storage is unsupported by Debian's Ganesha VFS FSAL — test pins that limit) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | +| 20 | Tape-native NAS models | landed; pool protection/replication, reservations (atomic reserve), export sets, vaults, VG replication and file spans persisted in the catalog; scratch-threshold health and per-barcode `reserved_bytes` mapping still TODO | `nas/state_machine.py`, `nas/media.py`, `nas/capacity.py` | +| 21 | FUSE data plane | landed: hydrator, cache (open-time integrity check, persisted LRU index, chunked pread), xattr state, gateway; see `docs/fuse.md` | `fuse/` | | 22 | Self-hosted i3/LTFS lanes | appliance-gated, authored: read-only nightly + destructive with environment approval, allowlist, snapshots | `hardware-*.yml`, `tools/hardware/` | | 23 | Drive-identity bridge | landed; on-rig certification test authored | `hardware/correlation.py`, `tests/hardware/readonly/` | | 24 | Release pipeline | landed (never exercised on a tag): build-once digest, SBOM, scans, cosign, staging→approval→production, rollback | `release.yml`, `docs/runbooks/release.md` | diff --git a/docs/tech-debt.md b/docs/tech-debt.md index 2d84d62..422695e 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -74,13 +74,20 @@ deprecation warning). `openblade/api/aml_state.py` keeps its own aware grep -rE "datetime\.utcnow\b" --include=*.py openblade tests tools | grep -v domain/clock.py | wc -l # 0 ``` -## In-memory job queue - -`openblade/jobs/queue.py:23` (`class JobQueue`) keeps jobs, drive owners and the -changer owner in process memory (`dict`s guarded by a `threading.RLock`). It is -created once in `openblade/bootstrap.py:614`. Nothing in `queue.py` persists -anything, so a process restart loses queued and running job records and the -ownership map. Any durable recovery has to come from elsewhere (for example, -drive leases and `openblade/jobs/recovery.py`), not from the queue. The queue -also assumes a single process, so running more than one worker process would -give each one its own independent queue. +## Job queue (resolved 2026-10-10) + +`openblade/jobs/queue.py` is now a façade over the catalog `jobs` table and the +`LeaseStore` (`acquire_drive_lease_at`, changer = pseudo-drive `-1`); claims are +heartbeated during `run_job` and refuse drives pending reconciliation. Remaining +gap: `DriveScheduler` leases and queue claims share one drive-id space by design, +and no production caller uses `claim_drive`/`claim_changer` yet. + +## NAS persistence gaps + +`ArchivePlanRequest.reserved_bytes` is keyed by barcode while `nas_reservations` +are per pool; mapping one to the other needs a design decision before the planner +can consume the ledger. Scratch-threshold health (`get_scratch_thresholds`) has no +consumer because the catalog has no scratch-cartridge count to evaluate. +`nas_file_spans` has no `pool_id`, so identical relative paths in two pools share +spans. A real NFSv4 ACL protocol scenario needs a Ganesha image built with VFS +ACL support; the shipped Debian package cannot store ACLs. \ No newline at end of file From 7435dd110bd223ed7e9d8b8f6f6d243533153e0c Mon Sep 17 00:00:00 2001 From: Amantux Date: Sat, 10 Oct 2026 15:05:58 +0000 Subject: [PATCH 172/205] fix(fuse-cache): only a whole-component ".." is an unsafe cache key The key is a single path component, so a ".." substring inside a longer token cannot traverse; the broad check rejected a truncated test checksum. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/fuse/cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openblade/fuse/cache.py b/openblade/fuse/cache.py index 78b3e0c..fed1b44 100644 --- a/openblade/fuse/cache.py +++ b/openblade/fuse/cache.py @@ -241,7 +241,10 @@ def verify_entry(self, checksum: str) -> bool: raise CacheIntegrityError("cached entry failed integrity verification and was evicted") def cache_key(self, checksum: str) -> Path: - if checksum in ("", ".") or any(bad in checksum for bad in ("/", "\\", "..", "\0")): + # The key is one path component, so only a whole-component "." / ".." or an + # embedded separator/NUL can escape cache_dir; a ".." substring inside a + # longer token (truncated test fixtures use "...") is harmless. + if checksum in ("", ".", "..") or any(bad in checksum for bad in ("/", "\\", "\0")): raise CacheError("checksum is not a safe cache path component") return self.cache_dir / checksum[:2] / checksum From dfbd8f99e9d782a8c902b62c39b77fe2e426f50f Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:19:30 +0000 Subject: [PATCH 173/205] feat(nas): planner refuses plans exceeding pool reservation budget ArchivePlanRequest.pool_reserved_bytes is subtracted from the summed tape capacity; a plan placing more bytes (x copies) is blocked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/planner.py | 15 ++++++++ openblade/nas/types.py | 3 ++ tests/unit/test_capacity_reservations.py | 46 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/unit/test_capacity_reservations.py diff --git a/openblade/nas/planner.py b/openblade/nas/planner.py index 876c816..e5d0058 100644 --- a/openblade/nas/planner.py +++ b/openblade/nas/planner.py @@ -16,6 +16,8 @@ TapeAssignment, ) +POOL_BUDGET_BLOCKER = "Insufficient pool capacity after reservations" + class ArchivePlanner: DEFAULT_TAPE_CAPACITY = 12_000_000_000_000 @@ -350,6 +352,19 @@ def _apply_common_checks(self, request: ArchivePlanRequest, plan: ArchivePlan) - "Requested copies exceed number of available tapes", field="copies", ) + self._check_pool_budget(request, plan) + + def _check_pool_budget(self, request: ArchivePlanRequest, plan: ArchivePlan) -> None: + """Refuse a plan whose placed bytes exceed capacity left after pool reservations.""" + if not request.available_tapes: + return + total_capacity = sum( + self._tape_capacity(request, barcode) for barcode in request.available_tapes + ) + budget = total_capacity - request.pool_reserved_bytes + placed = plan.total_bytes * plan.copies_required + if placed > budget: + self._block_plan(plan, POOL_BUDGET_BLOCKER) def _ordered_tapes(self, request: ArchivePlanRequest) -> list[str]: preferred = [ diff --git a/openblade/nas/types.py b/openblade/nas/types.py index 1c5f18b..9f51d71 100644 --- a/openblade/nas/types.py +++ b/openblade/nas/types.py @@ -455,6 +455,9 @@ class ArchivePlanRequest(BaseModel): cartridges: dict[str, Cartridge] = Field(default_factory=dict) drive_generation: MediaGeneration | None = None reserved_bytes: dict[str, int] = Field(default_factory=dict) + # Pool-level budget held by other jobs' NasReservation rows (the source of + # truth); subtracted from the summed tape capacity, not spread per barcode. + pool_reserved_bytes: int = Field(default=0, ge=0) class CacheDriveConfig(BaseModel): diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py new file mode 100644 index 0000000..b0572d4 --- /dev/null +++ b/tests/unit/test_capacity_reservations.py @@ -0,0 +1,46 @@ +"""Pool reservations are a budget the archive planner must respect.""" + +from __future__ import annotations + +from openblade.nas.planner import POOL_BUDGET_BLOCKER, ArchivePlanner +from openblade.nas.types import ArchivePlanRequest + +TAPE_CAPACITY = 1_000 + + +def _request(pool_reserved_bytes: int, *, copies: int = 1) -> ArchivePlanRequest: + return ArchivePlanRequest( + files=["/a", "/b"], + file_sizes={"/a": 300, "/b": 300}, + available_tapes=["T00001L8", "T00002L8"], + tape_capacities={"T00001L8": TAPE_CAPACITY, "T00002L8": TAPE_CAPACITY}, + copies=copies, + pool_reserved_bytes=pool_reserved_bytes, + ) + + +def test_plan_fits_exactly_within_pool_budget() -> None: + # 2 tapes x 1000 = 2000; 600 placed -> exactly 1400 may be reserved by others. + plan = ArchivePlanner().plan(_request(1_400)) + assert POOL_BUDGET_BLOCKER not in plan.enqueue_blockers + assert plan.is_safe_to_enqueue + + +def test_same_plan_refused_when_budget_shrinks_by_one_byte() -> None: + plan = ArchivePlanner().plan(_request(1_401)) + assert POOL_BUDGET_BLOCKER in plan.enqueue_blockers + assert not plan.is_safe_to_enqueue + + +def test_budget_counts_every_copy() -> None: + # Two copies place 1200 bytes; 801 reserved leaves only 1199. + assert POOL_BUDGET_BLOCKER in ArchivePlanner().plan(_request(801, copies=2)).enqueue_blockers + assert ( + POOL_BUDGET_BLOCKER not in ArchivePlanner().plan(_request(800, copies=2)).enqueue_blockers + ) + + +def test_default_budget_is_unchanged_behaviour() -> None: + request = _request(0) + assert request.pool_reserved_bytes == 0 + assert ArchivePlanner().plan(request).is_safe_to_enqueue From 47470e5e3292fa7b77b857fc8dfd9fa7990b4c01 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:20:31 +0000 Subject: [PATCH 174/205] feat(compat): add compat_capture replay/capture/diff tool with sanitizer Replays corpus request blocks against a base URL, writes sanitized cases with expected filled from the observed response, and refuses source=captured without --appliance-firmware and --attest. Routes tools/compat_capture* to the compat CI category. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tools/ci_ownership.toml | 4 + tools/compat_capture.py | 295 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 tools/compat_capture.py diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml index d863d95..024b683 100644 --- a/tools/ci_ownership.toml +++ b/tools/ci_ownership.toml @@ -58,6 +58,10 @@ categories = ["i3"] glob = "tests/compat/*" categories = ["compat"] +[[rule]] +glob = "tools/compat_capture*" +categories = ["compat"] + [[rule]] glob = "tests/fixtures/*" categories = ["unit", "compat"] diff --git a/tools/compat_capture.py b/tools/compat_capture.py new file mode 100644 index 0000000..0c3fb37 --- /dev/null +++ b/tools/compat_capture.py @@ -0,0 +1,295 @@ +"""Compatibility capture + sanitize pipeline for the ``compatibility/`` corpus. + +Replays each case's ``request`` block against a live base URL (the OpenBlade +emulator or, on-site, a real Scalar i3) and writes a new case whose +``expected`` is filled from the observed response. Every written case passes +through :func:`sanitize` first so tokens, serials, hosts, timestamps and +passwords never reach the repository. + + capture --base-url URL --recipe --out + [--source self-captured|captured] [--profile scalar-i3-341g] + [--login user:pass] [--appliance-firmware VER --attest NAME] + capture --base-url URL --recipe --diff + +``source: captured`` is appliance-only and ENFORCED by the corpus harness, so +the tool refuses to write it without ``--appliance-firmware`` AND ``--attest``. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import httpx + +TOKEN = "" +SERIAL = "" +HOST = "" +TIMESTAMP = "" +REDACTED = "" + +SOURCES = ("self-captured", "captured") +_COOKIE_ATTRS = frozenset({"path", "domain", "expires", "max-age", "samesite"}) + + +class CaptureRefusedError(Exception): + """The requested capture would mislabel provenance; nothing was written.""" + + +@dataclass(frozen=True) +class Rules: + # Key rules match on the lower-cased key with "_"/"-" removed, by substring. + token_keys: tuple[str, ...] = ("session", "token", "authorization", "apikey") + serial_keys: tuple[str, ...] = ("serial",) + password_keys: tuple[str, ...] = ("password", "passwd") + cookie_keys: tuple[str, ...] = ("cookie", "setcookie") + bearer: re.Pattern[str] = re.compile(r"(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+") + url_host: re.Pattern[str] = re.compile(r"(?i)\b(https?|wss?)://[^/\s\"'?#]+") + ipv4: re.Pattern[str] = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b") + timestamp: re.Pattern[str] = re.compile( + r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?" + r"|\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT" + ) + + +DEFAULT_RULES = Rules() + + +def _norm(key: str) -> str: + return key.lower().replace("_", "").replace("-", "") + + +def _key_hit(key: str, needles: Iterable[str]) -> bool: + k = _norm(key) + return any(n in k for n in needles) + + +def _scrub_cookie(value: str) -> str: + def repl(m: re.Match[str]) -> str: + name = m.group(1) + return m.group(0) if name.strip().lower() in _COOKIE_ATTRS else f"{name}={TOKEN}" + + return re.sub(r"([^=;,\s]+)=([^;,]*)", repl, value) + + +def _scrub_str(value: str, rules: Rules) -> str: + value = rules.bearer.sub(lambda m: f"{m.group(1)} {TOKEN}", value) + value = rules.url_host.sub(lambda m: f"{m.group(1)}://{HOST}", value) + value = rules.ipv4.sub(HOST, value) + return rules.timestamp.sub(TIMESTAMP, value) + + +def _walk(node: Any, rules: Rules) -> Any: + if isinstance(node, dict): + out: dict[str, Any] = {} + for key, value in node.items(): + if value is None or isinstance(value, bool): + out[key] = value + elif _key_hit(key, rules.password_keys): + out[key] = REDACTED + elif _key_hit(key, rules.cookie_keys) and isinstance(value, str): + out[key] = _scrub_cookie(value) + elif _key_hit(key, rules.token_keys) and not isinstance(value, (dict, list)): + out[key] = TOKEN + elif _key_hit(key, rules.serial_keys) and not isinstance(value, (dict, list)): + out[key] = SERIAL + else: + out[key] = _walk(value, rules) + return out + if isinstance(node, list): + return [_walk(item, rules) for item in node] + if isinstance(node, str): + return _scrub_str(node, rules) + return node + + +def sanitize(case: dict[str, Any], rules: Rules = DEFAULT_RULES) -> dict[str, Any]: + """Pure: return a scrubbed deep copy of ``case``; the input is not mutated.""" + result: dict[str, Any] = _walk(copy.deepcopy(case), rules) + return result + + +def dumps_case(case: dict[str, Any]) -> str: + """Deterministic serialisation: same case -> byte-identical file.""" + return json.dumps(case, indent=2, sort_keys=True, ensure_ascii=True) + "\n" + + +def load_recipes(path: Path) -> list[dict[str, Any]]: + files = sorted(path.rglob("*.json")) if path.is_dir() else [path] + return [json.loads(p.read_text()) for p in files] + + +def _login(client: httpx.Client, login: str | None) -> None: + if login is None: + raise CaptureRefusedError("case needs auth but --login user:pass was not given") + user, _, password = login.partition(":") + resp = client.post("/aml/users/login", json={"name": user, "password": password}) + if resp.status_code != 200: + raise CaptureRefusedError(f"login as {user!r} failed with status {resp.status_code}") + + +def replay(client: httpx.Client, case: dict[str, Any], login: str | None) -> httpx.Response: + # Mirrors tests/compat/test_compatibility_corpus.py::_run so captures replay identically. + if case.get("auth"): + _login(client, login) + req = case["request"] + headers = dict(req.get("headers") or {}) + if "content_type" in req: + headers["Content-Type"] = req["content_type"] + kwargs: dict[str, Any] = {"headers": headers or None} + if "body" in req: + kwargs["content"] = req["body"] + else: + kwargs["json"] = req.get("json") + for _ in range(int(req.get("repeat", 0))): + client.request(req["method"], req["path"], **kwargs) + return client.request(req["method"], req["path"], **kwargs) + + +def _body(resp: httpx.Response) -> Any: + try: + return resp.json() + except ValueError: + return resp.text + + +def build_case( + recipe: dict[str, Any], + resp: httpx.Response, + *, + source: str, + profile: str | None = None, + appliance_firmware: str | None = None, + attest: str | None = None, +) -> dict[str, Any]: + if source not in SOURCES: + raise CaptureRefusedError(f"source must be one of {SOURCES}, not {source!r}") + if source == "captured" and not (appliance_firmware and attest): + raise CaptureRefusedError( + "source=captured is appliance-only: --appliance-firmware and --attest are both required" + ) + raw = _body(resp) + clean = sanitize({"b": raw})["b"] + expected: dict[str, Any] = {"status": resp.status_code} + ctype = resp.headers.get("content-type") + if ctype: + expected["headers"] = {"content-type": ctype.split(";")[0].strip()} + if isinstance(raw, dict): + # Only values the sanitizer left untouched are enforceable. + expected["json_contains"] = {k: v for k, v in clean.items() if raw.get(k) == v} + observed_headers = {k.lower(): v for k, v in resp.headers.items() if k.lower() != "date"} + case: dict[str, Any] = { + key: recipe[key] + for key in ("id", "profile", "firmware", "auth", "manual_ref", "request") + if key in recipe + } + case.update( + source=source, + expected=expected, + observed={"headers": observed_headers, "body": clean}, + ) + if profile: + case["profile"] = profile + if source == "captured": + case.update(firmware=appliance_firmware, attested_by=attest) + else: + case["captured_from"] = "openblade-emulator" + case["fidelity_notes"] = ( + "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression " + "guard of emulator behaviour, NOT evidence of appliance fidelity." + ) + return sanitize(case) + + +def drift(case: dict[str, Any], resp: httpx.Response) -> list[str]: + exp = case["expected"] + problems: list[str] = [] + if resp.status_code != exp["status"]: + problems.append(f"status {resp.status_code} != {exp['status']}") + for name, needle in exp.get("headers", {}).items(): + value = resp.headers.get(name) + if value is None or needle not in value: + problems.append(f"header {name}={value!r} lacks {needle!r}") + if "json_contains" in exp: + body = _body(resp) + if not isinstance(body, dict): + problems.append(f"body is not a JSON object: {str(body)[:80]!r}") + else: + for key, value in exp["json_contains"].items(): + if body.get(key, KeyError) != value: + problems.append(f"{key}: expected {value!r}, got {body.get(key)!r}") + return problems + + +ClientFactory = Callable[[], httpx.Client] + + +def run_diff(factory: ClientFactory, cases: list[dict[str, Any]], login: str | None) -> int: + total = 0 + for case in cases: + with factory() as client: + problems = drift(case, replay(client, case, login)) + total += len(problems) + for p in problems: + print(f"DRIFT {case['id']}: {p}") + print(f"[compat_capture] diff: {len(cases)} cases, {total} drift findings") + return 1 if total else 0 + + +def run_capture(factory: ClientFactory, cases: list[dict[str, Any]], out: Path, **opts: Any) -> int: + login = opts.pop("login", None) + built = [] + for recipe in cases: # build everything first: a refusal writes nothing + with factory() as client: + built.append(build_case(recipe, replay(client, recipe, login), **opts)) + out.mkdir(parents=True, exist_ok=True) + for case in built: + (out / f"{case['id']}.json").write_text(dumps_case(case)) + print(f"[compat_capture] wrote {len(built)} cases to {out}") + return 0 + + +def main(argv: list[str] | None = None, factory: ClientFactory | None = None) -> int: + ap = argparse.ArgumentParser(prog="compat_capture", description=__doc__.split("\n")[0]) + ap.add_argument("command", choices=["capture"]) + ap.add_argument("--base-url", required=True) + ap.add_argument("--recipe", required=True, type=Path) + ap.add_argument("--out", type=Path) + ap.add_argument("--diff", action="store_true") + ap.add_argument("--source", choices=SOURCES, default="self-captured") + ap.add_argument("--profile") + ap.add_argument("--login") + ap.add_argument("--appliance-firmware") + ap.add_argument("--attest") + args = ap.parse_args(argv) + make = factory or (lambda: httpx.Client(base_url=args.base_url, timeout=30.0)) + cases = load_recipes(args.recipe) + try: + if args.diff: + return run_diff(make, cases, args.login) + if args.out is None: + ap.error("--out is required unless --diff is given") + return run_capture( + make, + cases, + args.out, + login=args.login, + source=args.source, + profile=args.profile, + appliance_firmware=args.appliance_firmware, + attest=args.attest, + ) + except CaptureRefusedError as exc: + print(f"[compat_capture] REFUSED: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From 111b7e383e7fc4977b817955113ccddcdef75d6c Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:21:15 +0000 Subject: [PATCH 175/205] test(compat): sanitizer rules, provenance refusal and determinism for compat_capture Mutation-checked: dropping the session/token key rule fails 2 tests; disabling the captured-attestation refusal fails 4. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/compat/test_compat_capture.py | 137 ++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/compat/test_compat_capture.py diff --git a/tests/compat/test_compat_capture.py b/tests/compat/test_compat_capture.py new file mode 100644 index 0000000..a1faf1d --- /dev/null +++ b/tests/compat/test_compat_capture.py @@ -0,0 +1,137 @@ +"""Unit tests for tools/compat_capture.py: sanitizer rules, provenance refusal, determinism.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + +import httpx +import pytest + +_PATH = Path(__file__).resolve().parents[2] / "tools" / "compat_capture.py" +_spec = importlib.util.spec_from_file_location("compat_capture", _PATH) +assert _spec and _spec.loader +cc = importlib.util.module_from_spec(_spec) +sys.modules["compat_capture"] = cc +_spec.loader.exec_module(cc) + +SECRET = "s3cr3t-T0KEN-abcdef0123456789" + + +def _dirty_case() -> dict[str, Any]: + return { + "id": "x", + "request": { + "method": "POST", + "path": "/aml/users/login", + "headers": {"Authorization": f"Bearer {SECRET}", "Cookie": f"sessionID={SECRET}"}, + "json": {"name": "admin", "password": "hunter2-plain", "newPassword": "pw2"}, + }, + "observed": { + "headers": {"set-cookie": f"sessionID={SECRET}; Path=/; HttpOnly; SameSite=lax"}, + "body": { + "sessionId": SECRET, + "access_token": SECRET, + "hardwareSerialNumber": "QTM-SN-998877", + "partnerBladeSerial": "PB-1234", + "link": "https://i3-lab.example.com:8443/aml/x?y=1", + "peer": "connect to 10.20.30.40:3260 ok", + "when": "2026-10-11T09:15:02.123Z", + "httpDate": "Sun, 11 Oct 2026 09:15:02 GMT", + "nested": [{"session_token": SECRET, "count": 3, "ok": True}], + }, + }, + } + + +def test_sanitize_scrubs_every_secret_class() -> None: + clean = cc.sanitize(_dirty_case()) + text = json.dumps(clean) + for leaked in (SECRET, "hunter2-plain", "pw2", "QTM-SN-998877", "PB-1234"): + assert leaked not in text, f"leaked {leaked!r}" + for leaked in ("i3-lab.example.com", "10.20.30.40", "2026-10-11T09", "09:15:02 GMT"): + assert leaked not in text, f"leaked {leaked!r}" + body = clean["observed"]["body"] + assert clean["request"]["json"]["password"] == cc.REDACTED + assert clean["request"]["json"]["name"] == "admin" + assert clean["request"]["headers"]["Authorization"] == cc.TOKEN + assert body["link"] == f"https://{cc.HOST}/aml/x?y=1" + assert body["when"] == cc.TIMESTAMP + assert body["nested"][0] == {"session_token": cc.TOKEN, "count": 3, "ok": True} + # Cookie attributes survive; only the value is replaced. + assert clean["observed"]["headers"]["set-cookie"] == ( + f"sessionID={cc.TOKEN}; Path=/; HttpOnly; SameSite=lax" + ) + + +def test_sanitize_is_pure_and_idempotent() -> None: + dirty = _dirty_case() + before = json.dumps(dirty, sort_keys=True) + once = cc.sanitize(dirty) + assert json.dumps(dirty, sort_keys=True) == before, "input was mutated" + assert cc.sanitize(once) == once + + +def _resp(body: dict[str, Any], status: int = 200) -> httpx.Response: + return httpx.Response( + status, + json=body, + headers={"set-cookie": f"sessionID={SECRET}; Path=/", "date": "Sun, 11 Oct 2026"}, + request=httpx.Request("GET", "http://appliance.local/aml/x"), + ) + + +RECIPE = {"id": "r1", "profile": "scalar-i3-341g", "source": "inferred", "request": {}} + + +def test_build_case_is_deterministic_and_enforces_only_unsanitized_keys() -> None: + body = {"status": "ok", "sessionId": SECRET, "serialNumber": "SN1"} + a = cc.dumps_case(cc.build_case(RECIPE, _resp(body), source="self-captured")) + b = cc.dumps_case(cc.build_case(RECIPE, _resp(body), source="self-captured")) + assert a == b + case = json.loads(a) + assert SECRET not in a and "SN1" not in a + assert case["source"] == "self-captured" + assert case["expected"]["json_contains"] == {"status": "ok"} + assert "date" not in case["observed"]["headers"] + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"appliance_firmware": "341G"}, {"attest": "Jane Operator"}], + ids=["neither", "firmware-only", "attest-only"], +) +def test_captured_without_attestation_is_refused(kwargs: dict[str, str]) -> None: + with pytest.raises(cc.CaptureRefusedError): + cc.build_case(RECIPE, _resp({}), source="captured", **kwargs) + + +def test_captured_with_attestation_records_provenance() -> None: + case = cc.build_case( + RECIPE, _resp({}), source="captured", appliance_firmware="341G", attest="Jane Operator" + ) + assert (case["source"], case["firmware"], case["attested_by"]) == ( + "captured", + "341G", + "Jane Operator", + ) + + +def test_cli_refusal_writes_nothing(tmp_path: Path) -> None: + recipe = tmp_path / "r.json" + recipe.write_text(json.dumps({**RECIPE, "request": {"method": "GET", "path": "/x"}})) + out = tmp_path / "out" + + def factory() -> httpx.Client: + return httpx.Client( + base_url="http://t", transport=httpx.MockTransport(lambda r: httpx.Response(200)) + ) + + argv = ["capture", "--base-url", "http://t", "--recipe", str(recipe), "--out", str(out)] + assert cc.main([*argv, "--source", "captured"], factory=factory) == 2 + assert not out.exists() + assert cc.main(argv, factory=factory) == 0 + assert (out / "r1.json").exists() From 59066d210a0e8c4248544bf766318099048f8044 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:21:38 +0000 Subject: [PATCH 176/205] feat(aml): mark MFA-required logins as pending sessions (inferred) A login by a user with an enabled MFA record now issues a session flagged mfa_pending, adds Warning: 199 - "MFA Required" and, under OPENBLADE_AML_STRICT_PRECONDITIONS=true, answers 412 "MFA verification required". The documented login body is unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/aml_state.py | 10 +++- openblade/api/routes_aml_auth.py | 13 ++++- tests/integration/test_aml_auth_fidelity.py | 61 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 76b99c2..613430b 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -235,6 +235,9 @@ class SessionRecord: role: int created_at: datetime expires_at: datetime + # Inferred (not in the Rev D manual): a login by a user with an enabled MFA + # record is pending until POST /users/login/mfa succeeds. Never serialised. + mfa_pending: bool = False def as_dict(self) -> dict[str, Any]: return { @@ -2489,7 +2492,7 @@ def _session_lifetime() -> timedelta: return timedelta(minutes=_STATE.session_timeout_minutes) -def create_session(user: AmlUser) -> SessionRecord: +def create_session(user: AmlUser, *, mfa_pending: bool = False) -> SessionRecord: purge_expired_sessions() # Evict the user's oldest sessions so the new login always succeeds: refusing # would let abandoned sessions lock an operator (incl. admin) out until expiry. @@ -2506,6 +2509,7 @@ def create_session(user: AmlUser) -> SessionRecord: role=user.role, created_at=now, expires_at=now + _session_lifetime(), + mfa_pending=mfa_pending, ) _STATE.sessions[record.token] = record return record @@ -2612,6 +2616,10 @@ def set_user_mfa(user_name: str, mfa_type: str, enabled: bool) -> dict[str, Any] return existing.as_dict() +def user_has_enabled_mfa(user_name: str) -> bool: + return any(record.enabled for record in _STATE.mfa_configs.get(user_name, {}).values()) + + def get_user_mfa(user_name: str, mfa_type: str) -> MFARecord | None: return _STATE.mfa_configs.setdefault(user_name, _default_user_mfa()).get(mfa_type) diff --git a/openblade/api/routes_aml_auth.py b/openblade/api/routes_aml_auth.py index 1bd3c8b..dcd1fb2 100644 --- a/openblade/api/routes_aml_auth.py +++ b/openblade/api/routes_aml_auth.py @@ -603,7 +603,8 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> _clear_rate_limit(remote_address) if user.role == 2 and not aml_state.get_service_access().get("enabled", True): raise HTTPException(status_code=503, detail="Service access is disabled") - session_record = aml_state.create_session(user) + mfa_pending = aml_state.user_has_enabled_mfa(user.name) + session_record = aml_state.create_session(user, mfa_pending=mfa_pending) aml_state.record_login_activity(user.name, success=True, remote_address=remote_address) body = _ws_result("Login successful").model_dump() # Provide session token in response body for API clients/tests that expect it @@ -612,9 +613,13 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> # OPENBLADE_AML_STRICT_PRECONDITIONS=true the same login answers 412 instead of # 200; the session is still issued so the client can change the password. strict = os.environ.get("OPENBLADE_AML_STRICT_PRECONDITIONS", "false").lower() == "true" - precondition_failed = strict and user.require_password_change + # Inferred likewise for MFA: the challenge wire shape is not in Rev D, so a + # pending login is signalled by a Warning header (412 when strict). MFA wins + # the summary because the pending session cannot change a password anyway. + precondition_failed = strict and (user.require_password_change or mfa_pending) if precondition_failed: - body.update(code=412, description="Error", summary="Default password must be changed") + summary = "MFA verification required" if mfa_pending else "Default password must be changed" + body.update(code=412, description="Error", summary=summary) response = _negotiated(request, 412 if precondition_failed else 200, body) _is_production = os.environ.get("OPENBLADE_ENV", "development").lower() == "production" response.set_cookie( @@ -627,6 +632,8 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> ) if user.require_password_change: response.headers["Warning"] = "Default Password Supplied" + if mfa_pending: + response.headers.append("Warning", '199 - "MFA Required"') return response diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py index 3ef0a3a..7cc3473 100644 --- a/tests/integration/test_aml_auth_fidelity.py +++ b/tests/integration/test_aml_auth_fidelity.py @@ -298,3 +298,64 @@ def test_error_envelope_shape( assert "Traceback" not in text and LEAK not in text and "RuntimeError" not in text message = body.get("summary", body.get("detail", body.get("message"))) assert isinstance(message, str) and message + + +# --- MFA-required pending session (inferred; no Rev D challenge wire shape) ----- + +MFA_WARNING = '199 - "MFA Required"' + + +def _enable_totp(client: TestClient) -> str: + """Enable TOTP for admin through the documented routes; return the secret.""" + import pyotp + + token = _login(client) + auth = {"Authorization": f"Bearer {token}"} + secret = client.get("/aml/users/login/mfa/totp/key", headers=auth).json()["key"] + enabled = client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": True, "authenticationCode": pyotp.TOTP(secret).now()}, + headers=auth, + ) + assert enabled.status_code == 200, enabled.text + client.cookies.clear() + return str(secret) + + +def test_mfa_login_lenient_is_200_with_warning_and_pending( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENBLADE_AML_STRICT_PRECONDITIONS", raising=False) + _enable_totp(client) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + assert MFA_WARNING in response.headers.get_list("Warning") + assert "Default Password Supplied" in response.headers.get_list("Warning") + # No new JSON fields: same keys as an ordinary (non-pending) login body. + assert set(response.json()) == { + "action", + "code", + "customCode", + "description", + "summary", + "token", + } + assert aml_state._STATE.sessions[response.json()["token"]].mfa_pending is True + + +def test_mfa_login_strict_is_412_and_session_issued( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _enable_totp(client) + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 412 + body = response.json() + assert body["code"] == 412 and body["summary"] == "MFA verification required" + assert MFA_WARNING in response.headers.get_list("Warning") + assert aml_state._STATE.sessions[body["token"]].mfa_pending is True + + +def test_login_without_enabled_mfa_is_not_pending(client: TestClient) -> None: + token = _login(client) + assert aml_state._STATE.sessions[token].mfa_pending is False From eeb52912f9fe3c9681f7764001b973e416ededbc Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:22:39 +0000 Subject: [PATCH 177/205] build(nas-protocols): serve NFS from Fedora 41 Ganesha 5.9 with VFS ACL mapping Debian bookworm's nfs-ganesha 4.3 has no VFS POSIX-ACL mapping, so nfs4_setfacl failed with "attribute not supported". Option (a) worked first: Fedora 41 nfs-ganesha-5.9-4.fc41 links libacl into libfsalvfs.so (acl_get_fd, acl_set_fd) and libganesha_nfsd exports fsal_acl_2_posix_acl. On the rig, nfs4_setfacl -a A::OWNER@:rwatTnNcCy /mnt/nfs/rw/x returned 0. No source build was needed. samba/client roles keep the pinned Debian base (default target). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/Dockerfile | 15 ++++++++++++--- deploy/nas-protocols/docker-compose.yml | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deploy/nas-protocols/Dockerfile b/deploy/nas-protocols/Dockerfile index 9b6fa6b..6fa7400 100644 --- a/deploy/nas-protocols/Dockerfile +++ b/deploy/nas-protocols/Dockerfile @@ -1,10 +1,19 @@ -# One pinned base for all three rig roles; the role is chosen by build arg. +# The NFS server role needs a Ganesha built with VFS POSIX-ACL mapping so NFSv4 +# ACLs (nfs4_setfacl) work; Debian bookworm's nfs-ganesha 4.3 is built without it. +# Fedora 41's nfs-ganesha 5.9 links libacl into libfsalvfs.so and exports +# fsal_acl_2_posix_acl. fedora:41 pinned by digest (docker inspect RepoDigests). +FROM fedora@sha256:f1a3fab47bcb3c3ddf3135d5ee7ba8b7b25f2e809a47440936212a3a50957f3d AS nfs-ganesha +RUN dnf -y install --setopt=install_weak_deps=False nfs-ganesha nfs-ganesha-vfs rpcbind \ + && dnf clean all +COPY hydrator_shim.py /usr/local/bin/hydrator_shim.py + +# samba and client roles: one pinned Debian base, the role chosen by build arg. # debian:bookworm-slim pinned by digest (docker inspect RepoDigests). -FROM debian@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 +# Last stage, so it stays the default build target. +FROM debian@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 AS debian-roles ARG ROLE RUN apt-get update \ && if [ "$ROLE" = samba ]; then pkgs="samba smbclient"; \ - elif [ "$ROLE" = nfs ]; then pkgs="nfs-ganesha nfs-ganesha-vfs rpcbind dbus"; \ else pkgs="smbclient cifs-utils nfs-common nfs4-acl-tools python3 python3-pip attr procps"; fi \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $pkgs \ && apt-get clean \ diff --git a/deploy/nas-protocols/docker-compose.yml b/deploy/nas-protocols/docker-compose.yml index 94ac072..2232fdb 100644 --- a/deploy/nas-protocols/docker-compose.yml +++ b/deploy/nas-protocols/docker-compose.yml @@ -14,7 +14,7 @@ services: interval: 2s retries: 30 nfs-ganesha: - build: { context: ., args: { ROLE: nfs } } + build: { context: ., target: nfs-ganesha, args: { ROLE: nfs } } # Ganesha's VFS FSAL needs open_by_handle_at (CAP_DAC_READ_SEARCH) etc. privileged: true command: ["sh", "-c", "rpcbind && exec ganesha.nfsd -F -L STDOUT -p /tmp/ganesha.pid -f /rig/ganesha.conf"] From 81effc5710915a5f0ce5eadd690b2826e8c75578 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:23:59 +0000 Subject: [PATCH 178/205] feat(nas): archive plans book a pool reservation against the budget /nas/archive-plan sums unexpired pool reservations into pool_reserved_bytes, then atomically books the plan's bytes (owner=plan_id, 1h TTL). A lost race is a typed ReservationError -> 409. Release is by expiry: ingest never builds a plan request, so no completion hook owns the plan id. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 10 ++- openblade/nas/capacity.py | 49 +++++++++++++- openblade/nas/planner.py | 9 +-- tests/unit/test_capacity_reservations.py | 83 +++++++++++++++++++++++- 4 files changed, 142 insertions(+), 9 deletions(-) diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index b7a7aca..7971225 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -20,6 +20,7 @@ from openblade.domain.clock import naive_utcnow from openblade.domain.models import MountHandle, MountMode from openblade.domain.wire import coerce_int +from openblade.nas.capacity import ReservationError, plan_with_pool_reservation from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.catalog_rebuild_worker import SAFE_REBUILD_PREFLIGHT_ERROR, CatalogRebuildWorker from openblade.nas.catalog_shard import CatalogShardWriter @@ -39,7 +40,6 @@ from openblade.nas.ltfs_manifest import TapeMetadataWriter from openblade.nas.manifest_validator import ManifestValidator from openblade.nas.path_mapping import PathMappingService -from openblade.nas.planner import ArchivePlanner from openblade.nas.restore_planner import RestorePlan, RestorePlanner from openblade.nas.service import NasService from openblade.nas.sidecar import SidecarResolver @@ -1164,7 +1164,13 @@ async def archive_plan( updates["max_parallelism"] = policy.max_parallelism request = request.model_copy(update=updates) - plan = ArchivePlanner().plan(request) + try: + plan = plan_with_pool_reservation( + service.repository, request, now=datetime.now(timezone.utc) + ) + except ReservationError as exc: + # Curated domain message (pool id only); two plans raced for the same bytes. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc if policy is not None: plan.policy_name = policy.name plan.policy_type = policy.policy_type diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index 84c4435..fc9ffc4 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -10,11 +10,12 @@ from collections.abc import Iterable, Sequence from datetime import datetime, timedelta from enum import Enum -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from pydantic import BaseModel, ConfigDict, Field, model_validator -from openblade.nas.types import NasFileState +from openblade.nas.planner import ArchivePlanner +from openblade.nas.types import ArchivePlan, ArchivePlanRequest, NasFileState if TYPE_CHECKING: from openblade.catalog.repository import CatalogRepository @@ -82,6 +83,50 @@ class ReservationError(ValueError): """A reservation cannot be satisfied from the pool's free capacity.""" +# No lease-TTL setting is in scope where plans are built, so a booked plan holds +# its pool bytes for one hour; expiry is the release path for plans never started. +PLAN_RESERVATION_TTL = timedelta(hours=1) + + +def plan_with_pool_reservation( + repository: CatalogRepository, + request: ArchivePlanRequest, + *, + now: datetime, + planner: ArchivePlanner | None = None, +) -> ArchivePlan: + """Plan against the pool's reservation budget, then book the plan's own bytes. + + Unexpired ``NasReservation`` rows for ``request.pool`` are the source of truth: + their sum becomes ``pool_reserved_bytes``. A safe plan books + ``total_bytes * copies`` under ``owner=plan_id``; if the atomic booking loses + a race, ``ReservationError`` is raised. Requests without a pool are planned + unchanged. + """ + planner = planner or ArchivePlanner() + pool_id = request.pool + if pool_id is None: + return planner.plan(request) + reserved = sum( + int(cast(int, row["bytes"])) for row in repository.reservations_for_pool(pool_id, now=now) + ) + request = request.model_copy(update={"pool_reserved_bytes": reserved}) + plan = planner.plan(request) + if not plan.is_safe_to_enqueue: + return plan + booked = repository.reserve_nas_capacity( + pool_id, + plan.total_bytes * plan.copies_required, + plan.plan_id, + now + PLAN_RESERVATION_TTL, + capacity_bytes=planner.pool_capacity_bytes(request), + now=now, + ) + if not booked: + raise ReservationError(f"pool {pool_id} capacity was reserved by a concurrent plan") + return plan + + class CapacityReservation(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/openblade/nas/planner.py b/openblade/nas/planner.py index e5d0058..79ed7f8 100644 --- a/openblade/nas/planner.py +++ b/openblade/nas/planner.py @@ -358,14 +358,15 @@ def _check_pool_budget(self, request: ArchivePlanRequest, plan: ArchivePlan) -> """Refuse a plan whose placed bytes exceed capacity left after pool reservations.""" if not request.available_tapes: return - total_capacity = sum( - self._tape_capacity(request, barcode) for barcode in request.available_tapes - ) - budget = total_capacity - request.pool_reserved_bytes + budget = self.pool_capacity_bytes(request) - request.pool_reserved_bytes placed = plan.total_bytes * plan.copies_required if placed > budget: self._block_plan(plan, POOL_BUDGET_BLOCKER) + def pool_capacity_bytes(self, request: ArchivePlanRequest) -> int: + """Summed usable capacity of the request's tapes (per-barcode holds already removed).""" + return sum(self._tape_capacity(request, barcode) for barcode in request.available_tapes) + def _ordered_tapes(self, request: ArchivePlanRequest) -> list[str]: preferred = [ barcode for barcode in request.available_tapes if not self._is_scratch_tape(barcode) diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index b0572d4..6a57f02 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -2,14 +2,28 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from openblade.catalog.repository import CatalogRepository +from openblade.nas.capacity import ( + PLAN_RESERVATION_TTL, + ReservationError, + plan_with_pool_reservation, +) from openblade.nas.planner import POOL_BUDGET_BLOCKER, ArchivePlanner from openblade.nas.types import ArchivePlanRequest TAPE_CAPACITY = 1_000 -def _request(pool_reserved_bytes: int, *, copies: int = 1) -> ArchivePlanRequest: +def _request( + pool_reserved_bytes: int, *, copies: int = 1, pool: str | None = None +) -> ArchivePlanRequest: return ArchivePlanRequest( + pool=pool, files=["/a", "/b"], file_sizes={"/a": 300, "/b": 300}, available_tapes=["T00001L8", "T00002L8"], @@ -44,3 +58,70 @@ def test_default_budget_is_unchanged_behaviour() -> None: request = _request(0) assert request.pool_reserved_bytes == 0 assert ArchivePlanner().plan(request).is_safe_to_enqueue + + +NOW = datetime(2026, 10, 11, 12, 0, tzinfo=UTC) + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + from tests.unit.test_nas_restore_planner import make_nas_service + + return make_nas_service(tmp_path).repository + + +def test_booked_plan_consumes_pool_so_second_plan_is_refused(repo: CatalogRepository) -> None: + # Arrange: pool of 2000 bytes; another job already holds 800. + assert repo.reserve_nas_capacity( + "pool-1", 800, "other-job", NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + ) + + # Act: first plan places 600 and books it (1400 held); second places 601 > 600 left. + first = plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=NOW) + second_request = _request(0, pool="pool-1").model_copy( + update={"file_sizes": {"/a": 300, "/b": 301}} + ) + second = plan_with_pool_reservation(repo, second_request, now=NOW) + + # Assert + assert first.is_safe_to_enqueue + assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)} == { + "other-job", + first.plan_id, + } + assert POOL_BUDGET_BLOCKER in second.enqueue_blockers + + +def test_booking_expires_after_ttl(repo: CatalogRepository) -> None: + plan_with_pool_reservation(repo, _request(0, pool="pool-1", copies=3), now=NOW) + + later = plan_with_pool_reservation( + repo, _request(0, pool="pool-1", copies=3), now=NOW + PLAN_RESERVATION_TTL + ) + + assert later.is_safe_to_enqueue + + +def test_expired_reservation_is_ignored(repo: CatalogRepository) -> None: + assert repo.reserve_nas_capacity( + "pool-1", 2_000, "stale-job", NOW + timedelta(seconds=1), capacity_bytes=2_000, now=NOW + ) + + plan = plan_with_pool_reservation( + repo, _request(0, pool="pool-1"), now=NOW + timedelta(minutes=1) + ) + + assert plan.is_safe_to_enqueue + + +def test_lost_booking_race_raises_typed_error(repo: CatalogRepository) -> None: + # Simulate a concurrent writer landing between the budget read and the booking. + class RacingRepo: + def reservations_for_pool(self, pool_id: str, *, now: datetime) -> list[dict[str, object]]: + return [] + + def reserve_nas_capacity(self, *args: object, **kwargs: object) -> bool: + return False + + with pytest.raises(ReservationError): + plan_with_pool_reservation(RacingRepo(), _request(0, pool="pool-1"), now=NOW) # type: ignore[arg-type] From 7c0d3955cdee969dfb3d46a9e8b27aa8cff58001 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:24:26 +0000 Subject: [PATCH 179/205] ci(mhvtl-weekly): run drive-identity bridge test in the no-LTFS suite cut The LTFS branch already ran it via tests/hardware/; the documented cut omitted it although it is a mount DRY. It skips (visible via -rs) unless OPENBLADE_APPLIANCE_* are set, which the mhvtl rig does not provide. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .github/workflows/mhvtl-weekly.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 5be2a08..97b46c1 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -358,7 +358,11 @@ jobs: # the scratch cartridges. Without it those suites fail on mount # rather than skip, which would make the canary permanently red for # a reason that is not a product defect. - suites=(tests/hardware/test_device_discovery.py + # The drive-identity bridge is a mount DRY (no mkltfs needed). It + # skips (reported via -rs) unless OPENBLADE_APPLIANCE_* point at an + # AML endpoint, which this rig does not provide. + suites=(tests/hardware/readonly/test_drive_identity_bridge.py + tests/hardware/test_device_discovery.py tests/hardware/test_changer_operations.py tests/hardware/test_drive_health.py tests/hardware/test_catalog_integrity.py) From 6f88b184d5969dba64091399a6d3022e8ab9f365 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:25:17 +0000 Subject: [PATCH 180/205] docs(roadmap): item 23 bridge is implemented, not yet certified on metal Drop the stale scalar_http NotImplementedError sentence (only nas/ingest.py raises it now) and point at correlation.py and the read-only bridge test. The test skips on mhvtl because it needs an AML endpoint, so the item stays uncertified. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/production-readiness-roadmap.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index 2a42716..d4e1239 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -183,10 +183,14 @@ emulator. (GitHub Environment approval, allowlisted sacrificial barcodes, snapshot before/after, load/unload, LTFS format, write/read/checksum, reboot persistence, sharded write+restore, safe failure injection, guaranteed recovery). -23. 🔴 **#E Physical drive-identity bridge:** certify AML coordinate ↔ serial ↔ SCSI - inquiry ↔ `/dev/tape/by-id` ↔ LTFS mount on the rig before enabling any real - write path. Never rely on transient `/dev/nst0` ordering. (`scalar_http` currently - raises `NotImplementedError` here — correct until proven on metal.) +23. 🟡 **#E Physical drive-identity bridge:** AML coordinate ↔ serial ↔ SCSI + inquiry ↔ `/dev/tape/by-id` ↔ LTFS mount (dry) is implemented in + `openblade/hardware/correlation.py` (`resolve_tape_by_id`) and covered by + `tests/hardware/readonly/test_drive_identity_bridge.py`. Never rely on transient + `/dev/nst0` ordering. Not yet certified: the test needs an AML endpoint + (`OPENBLADE_APPLIANCE_*`), so on the mhvtl rig it skips (2026-10-11: discovery + passes, bridge skipped). Certify on the appliance before enabling any real write + path. ## Phase 8 — Release pipeline (P2) 🟢🟡 From ca6e47aa4e8401fe169ec0b52319091f8866e109 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:25:49 +0000 Subject: [PATCH 181/205] test(protocols): assert a real NFSv4 DENY ACE is enforced over NFS test_nfs_acl_mapping no longer pins "attribute not supported". It adds D::65534:r (nobody) to a mode-644 file via nfs4_setfacl, asserts the ACE round-trips through nfs4_getfacl, mode stays 644, nobody's read is refused and the owner's is not; removing the ACE restores nobody's read. The existing cross-uid mode-bit assertions are kept. Mutation-checked: replacing the nfs4_setfacl call with a no-op fails the test at the getfacl assertion, and with that assertion also removed it fails at the "nobody is refused" assertion. make test-protocols: 33 passed, twice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/e2e/protocols/test_nfs.py | 42 ++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/e2e/protocols/test_nfs.py b/tests/e2e/protocols/test_nfs.py index b245d73..a06b49c 100644 --- a/tests/e2e/protocols/test_nfs.py +++ b/tests/e2e/protocols/test_nfs.py @@ -23,22 +23,36 @@ def test_nfs_posix_lock_granted_by_server(mounted: Rig) -> None: def test_nfs_acl_mapping(mounted: Rig) -> None: - """Access control on the rw export as the rig's Ganesha actually provides it. - - Limit (verified on the pinned debian:bookworm nfs-ganesha 4.3 + nfs-ganesha-vfs build): - the VFS FSAL does not advertise the NFSv4 ``acl`` attribute, even with - ``Disable_ACL = false`` on the export and a fresh client mount -- ``nfs4_getfacl`` - reports "Operation to request attribute not supported" and ``nfs4_setfacl`` fails - with "Failed to instantiate ACL". Ganesha 4.3 has no ``Allow_ACL`` key. So NFSv4 ACEs - cannot be set or read through this server; the test pins that limit (it fails, and - must be rewritten as a set/get round-trip, if a future image starts storing ACLs) and - asserts that POSIX mode bits are enforced server-side for a second uid instead. + """A real NFSv4 ACL is enforced server-side, beyond what the mode bits say. + + The rig's Ganesha (Fedora 41 nfs-ganesha 5.9, VFS FSAL) maps NFSv4 ACLs onto POSIX + ACLs. A DENY ACE for a named user (``nobody``, uid 65534 -- numeric principals, the + kernel client's default without idmapping) is stored as a ``user:65534:---`` POSIX + entry, so ``nobody`` is refused a read that mode 644 alone would allow, while the + owner keeps reading. The mapping is lossy (POSIX has no DENY and no + ``n``/``N`` named-attribute bits), so only the deny ACE itself is asserted to + round-trip, not the whole ACL text. Mode-bit enforcement for a second uid is kept. """ f = "/mnt/nfs/rw/acl.bin" - assert mounted.sh(f"echo hi > {f}").returncode == 0 - acl = mounted.sh(f"nfs4_getfacl {f}; nfs4_setfacl -a A::OWNER@:rwatTnNcCy {f}") - assert "attribute not supported" in acl.stdout + acl.stderr, acl.stdout + acl.stderr - assert acl.returncode != 0, "nfs4_setfacl succeeded: server now stores ACLs, extend this test" + deny_ace = "D::65534:r" + assert mounted.sh(f"rm -f {f} && echo hi > {f} && chmod 644 {f}").returncode == 0 + assert mounted.sh(f"runuser -u nobody -- cat {f}").returncode == 0 + + set_ = mounted.sh(f"nfs4_setfacl -a {deny_ace} {f}") + assert set_.returncode == 0, set_.stdout + set_.stderr + assert deny_ace in mounted.sh(f"nfs4_getfacl {f}").stdout.splitlines() + assert mounted.sh(f"stat -c %a {f}").stdout.strip() == "644" + assert mounted.sh(f"runuser -u nobody -- cat {f}").returncode != 0 + owner = mounted.sh(f"cat {f}") + assert owner.returncode == 0 and owner.stdout.strip() == "hi", owner.stderr + + unset = mounted.sh(f"nfs4_setfacl -x {deny_ace} {f}") + assert unset.returncode == 0, unset.stdout + unset.stderr + assert deny_ace not in mounted.sh(f"nfs4_getfacl {f}").stdout.splitlines() + readable = mounted.sh(f"runuser -u nobody -- cat {f}") + assert readable.returncode == 0 and readable.stdout.strip() == "hi", readable.stderr + + assert mounted.sh(f"rm -f {f} && echo hi > {f}").returncode == 0 assert mounted.sh(f"chmod 600 {f} && runuser -u nobody -- cat {f}").returncode != 0 allowed = mounted.sh(f"chmod 644 {f} && runuser -u nobody -- cat {f}") assert allowed.returncode == 0 and allowed.stdout.strip() == "hi", allowed.stderr From a41eb43c438abe72164925b90fba0818e33934ea Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:26:16 +0000 Subject: [PATCH 182/205] docs: NFSv4 ACL enforcement is now a real protocol-rig scenario tech-debt: replace the "needs a Ganesha image built with VFS ACL support" sentence with the image that provides it and what the NFSv4->POSIX mapping loses (observed on the rig). test-plan: the protocol-rig paragraph names the real ACL scenario. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/tech-debt.md | 7 +++++-- docs/test-plan.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/tech-debt.md b/docs/tech-debt.md index 422695e..432de19 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -89,5 +89,8 @@ are per pool; mapping one to the other needs a design decision before the planne can consume the ledger. Scratch-threshold health (`get_scratch_thresholds`) has no consumer because the catalog has no scratch-cartridge count to evaluate. `nas_file_spans` has no `pool_id`, so identical relative paths in two pools share -spans. A real NFSv4 ACL protocol scenario needs a Ganesha image built with VFS -ACL support; the shipped Debian package cannot store ACLs. \ No newline at end of file +spans. The protocol rig's NFS server is Fedora 41 nfs-ganesha 5.9 (VFS FSAL, built +with POSIX-ACL mapping); NFSv4→POSIX ACL mapping is lossy: POSIX has no DENY, so a +special-principal deny (e.g. `D::EVERYONE@:r`) is folded into the mode bits and strips +that permission from the owner too, the `n`/`N` named-attribute bits are dropped, and +removing a named-user ACE leaves a residual `A:::...` entry behind. \ No newline at end of file diff --git a/docs/test-plan.md b/docs/test-plan.md index 6041607..f21565f 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -44,7 +44,7 @@ fix flips them to XPASS and fails the run until the marker is removed. `OPENBLADE_BACKEND`.** Add it as a named pairing in `tests/contract/conftest.py`. ## Protocol tests -Run the SMB and NFS protocol rig with `make protocols-up && make test-protocols && make protocols-down` (tests are marked `protocols`, live in `tests/e2e/protocols`, and skip with a capability-naming reason when the rig is unavailable). They are not part of the default suite; `.github/workflows/nas-protocols.yml` runs them nightly and on manual dispatch, and CI records the results. The rig uses a hydrator shim to simulate tape recall, so it does not cover the real FUSE hydrator. See `docs/wiki/guides/fuse-and-nas.md`. +Run the SMB and NFS protocol rig with `make protocols-up && make test-protocols && make protocols-down` (tests are marked `protocols`, live in `tests/e2e/protocols`, and skip with a capability-naming reason when the rig is unavailable). They are not part of the default suite; `.github/workflows/nas-protocols.yml` runs them nightly and on manual dispatch, and CI records the results. The rig uses a hydrator shim to simulate tape recall, so it does not cover the real FUSE hydrator. NFSv4 ACL enforcement is a real scenario: `test_nfs_acl_mapping` sets a named-user DENY ACE with `nfs4_setfacl` and asserts the server refuses that uid a read its mode bits allow. See `docs/wiki/guides/fuse-and-nas.md`. ## CI lanes Lane definitions live in the README section "Layered CI/CD". This section lists which tests each lane runs. From 17b4e51f3e19e6af6b0f1776b38bb86b394265e4 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:30:14 +0000 Subject: [PATCH 183/205] feat(nas): scratch-stock health component and threshold API count_scratch_cartridges (no volume group, in_slot); health gains a scratch component only when thresholds are configured (WARN -> OK + message, CRITICAL -> DEGRADED, never UNHEALTHY); GET/PUT /nas/scratch-thresholds. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 28 ++++++++++++- openblade/catalog/repository.py | 9 +++++ openblade/nas/health_service.py | 32 +++++++++++++++ tests/unit/test_capacity_reservations.py | 51 ++++++++++++++++++++++++ tests/unit/test_health_service.py | 1 + 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index 7971225..9c0d36f 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -20,7 +20,11 @@ from openblade.domain.clock import naive_utcnow from openblade.domain.models import MountHandle, MountMode from openblade.domain.wire import coerce_int -from openblade.nas.capacity import ReservationError, plan_with_pool_reservation +from openblade.nas.capacity import ( + ReservationError, + ScratchThresholds, + plan_with_pool_reservation, +) from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.catalog_rebuild_worker import SAFE_REBUILD_PREFLIGHT_ERROR, CatalogRebuildWorker from openblade.nas.catalog_shard import CatalogShardWriter @@ -887,6 +891,28 @@ async def list_reservations( return repo.list_nas_reservations(pool_id) +@router.get("/scratch-thresholds", response_model=ScratchThresholds) +async def get_scratch_thresholds( + repo: CatalogRepository = Depends(get_catalog_repository), +) -> ScratchThresholds: + stored = repo.get_scratch_thresholds() + if stored is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Scratch thresholds not configured" + ) + return ScratchThresholds.model_validate(stored) + + +@router.put("/scratch-thresholds", response_model=ScratchThresholds) +async def put_scratch_thresholds( + thresholds: ScratchThresholds, + repo: CatalogRepository = Depends(get_catalog_repository), +) -> ScratchThresholds: + # The body is validated by ScratchThresholds itself (warn < min -> 422). + repo.set_scratch_thresholds(thresholds.scratch_min, thresholds.scratch_warn) + return thresholds + + @router.get("/export-sets") async def list_export_sets( repo: CatalogRepository = Depends(get_catalog_repository), diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 45f6851..6c9f2b8 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -1518,6 +1518,15 @@ def upsert_nas_file_span(self, data: dict[str, object]) -> dict[str, object]: def delete_nas_file_span(self, span_id: str) -> bool: return self._delete_nas_row(NasFileSpan, span_id) + def count_scratch_cartridges(self) -> int: + """Cartridges with no volume group that are sitting in a slot (not loaded/exported).""" + stmt = ( + select(func.count()) + .select_from(Cartridge) + .where(Cartridge.volume_group_id.is_(None), Cartridge.state == "in_slot") + ) + return int(self.session.execute(stmt).scalar_one()) + def get_scratch_thresholds(self) -> dict[str, object] | None: return self.get_nas_config(self._SCRATCH_THRESHOLDS_KEY) diff --git a/openblade/nas/health_service.py b/openblade/nas/health_service.py index 83e8757..99915b3 100644 --- a/openblade/nas/health_service.py +++ b/openblade/nas/health_service.py @@ -12,6 +12,7 @@ from openblade.catalog.models import Cartridge, NasDataset, NasFileRecord, PathMapping from openblade.catalog.repository import CatalogRepository from openblade.domain.clock import naive_utcnow +from openblade.nas.capacity import ScratchHealth, ScratchThresholds, evaluate_scratch from openblade.nas.types import ( CatalogStatusResponse, ComponentHealth, @@ -51,6 +52,9 @@ def check_health(self) -> HealthResponse: self._check_library_component(), self._check_ltfs_component(), ] + scratch = self._check_scratch_component() + if scratch is not None: + components.append(scratch) status = max(components, key=lambda component: _STATUS_PRIORITY[component.status]).status return HealthResponse( status=status, @@ -59,6 +63,34 @@ def check_health(self) -> HealthResponse: version=get_version_info()["version"], ) + def _check_scratch_component(self) -> ComponentHealth | None: + """Scratch-stock component; absent when no thresholds are configured. + + CRITICAL maps to DEGRADED, never UNHEALTHY: tape stock must not fail a probe. + """ + try: + stored = self.repo.get_scratch_thresholds() + if stored is None: + return None + thresholds = ScratchThresholds.model_validate(stored) + count = self.repo.count_scratch_cartridges() + except Exception: # noqa: BLE001 - health must not raise; DB failure is reported by the database component + logger.warning("scratch health probe failed", exc_info=True) + return None + level = evaluate_scratch(count, thresholds) + if level is ScratchHealth.CRITICAL: + return ComponentHealth( + name="scratch", + status=HealthStatus.DEGRADED, + message=f"scratch count {count} below min {thresholds.scratch_min}", + ) + message = ( + f"scratch count {count} below warn {thresholds.scratch_warn}" + if level is ScratchHealth.WARN + else "" + ) + return ComponentHealth(name="scratch", status=HealthStatus.OK, message=message) + def check_ready(self) -> ReadyResponse: """Return ready=True only when the database is reachable and the library is connected.""" checked_at = _utcnow_iso() diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 6a57f02..95542bf 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -125,3 +125,54 @@ def reserve_nas_capacity(self, *args: object, **kwargs: object) -> bool: with pytest.raises(ReservationError): plan_with_pool_reservation(RacingRepo(), _request(0, pool="pool-1"), now=NOW) # type: ignore[arg-type] + + +def _seed_cartridges(repo: CatalogRepository, scratch: int) -> int: + """Add cartridges on top of the simulator's seeded stock; return that baseline.""" + baseline = repo.count_scratch_cartridges() + from openblade.catalog.models import Cartridge + + for i in range(scratch): + repo.session.add(Cartridge(barcode=f"S{i:05d}L8", state="in_slot")) + repo.session.add(Cartridge(barcode="LOADED1L8", state="loaded")) + repo.session.add(Cartridge(barcode="ASSIGN1L8", state="in_slot", volume_group_id="vg-1")) + repo.session.commit() + return baseline + + +def test_count_scratch_excludes_loaded_and_assigned(repo: CatalogRepository) -> None: + baseline = _seed_cartridges(repo, scratch=2) + + assert repo.count_scratch_cartridges() == baseline + 2 + + +def _scratch_component(tmp_path: Path, scratch: int, thresholds: tuple[int, int] | None): # type: ignore[no-untyped-def] + from tests.unit.test_health_service import make_health_service + + service, _ = make_health_service(tmp_path, "scratch.db") + base = _seed_cartridges(service.repo, scratch) + if thresholds is not None: + service.repo.set_scratch_thresholds(base + thresholds[0], base + thresholds[1]) + result = service.check_health() + return result, next((c for c in result.components if c.name == "scratch"), None) + + +def test_scratch_component_absent_when_thresholds_unset(tmp_path: Path) -> None: + _, component = _scratch_component(tmp_path, 0, None) + assert component is None + + +def test_scratch_warn_is_ok_with_message(tmp_path: Path) -> None: + _, component = _scratch_component(tmp_path, 3, (2, 5)) + assert component is not None + assert component.status.value == "ok" + assert component.message.startswith("scratch count ") + assert " below warn " in component.message + + +def test_scratch_critical_is_degraded_not_unhealthy(tmp_path: Path) -> None: + from openblade.nas.types import HealthStatus + + _, component = _scratch_component(tmp_path, 1, (2, 5)) + assert component is not None + assert component.status is HealthStatus.DEGRADED diff --git a/tests/unit/test_health_service.py b/tests/unit/test_health_service.py index 3d98d87..dad1731 100644 --- a/tests/unit/test_health_service.py +++ b/tests/unit/test_health_service.py @@ -72,6 +72,7 @@ def test_check_health_all_ok(tmp_path) -> None: result = service.check_health() assert result.status is HealthStatus.OK + # Thresholds unset -> no scratch component, so the set stays exact. assert {component.name for component in result.components} == {"database", "library", "ltfs"} From e9b2ff50d2de903983e396d286f109c2a9ad3109 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:32:11 +0000 Subject: [PATCH 184/205] fix(compat): scrub XML-body tokens and cookie expiry; keep bad-password cases replayable The end-to-end capture leaked a session token in an application/xml login body and an expires= timestamp in Set-Cookie. Redaction also collapsed right/wrong passwords; non-login passwords now become . Directory recipes skip the tool's own outputs. XML rule mutation-checked (removing it fails 1 test). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/compat/test_compat_capture.py | 19 ++++++++++-- tools/compat_capture.py | 48 ++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/tests/compat/test_compat_capture.py b/tests/compat/test_compat_capture.py index a1faf1d..f70fd71 100644 --- a/tests/compat/test_compat_capture.py +++ b/tests/compat/test_compat_capture.py @@ -31,7 +31,9 @@ def _dirty_case() -> dict[str, Any]: "json": {"name": "admin", "password": "hunter2-plain", "newPassword": "pw2"}, }, "observed": { - "headers": {"set-cookie": f"sessionID={SECRET}; Path=/; HttpOnly; SameSite=lax"}, + "headers": { + "set-cookie": f"sessionID={SECRET}; expires=Sun, 11 Oct 2026 03:23:02 GMT; Path=/" + }, "body": { "sessionId": SECRET, "access_token": SECRET, @@ -42,6 +44,7 @@ def _dirty_case() -> dict[str, Any]: "when": "2026-10-11T09:15:02.123Z", "httpDate": "Sun, 11 Oct 2026 09:15:02 GMT", "nested": [{"session_token": SECRET, "count": 3, "ok": True}], + "xml": f"0{SECRET}SN9", }, }, } @@ -50,7 +53,7 @@ def _dirty_case() -> dict[str, Any]: def test_sanitize_scrubs_every_secret_class() -> None: clean = cc.sanitize(_dirty_case()) text = json.dumps(clean) - for leaked in (SECRET, "hunter2-plain", "pw2", "QTM-SN-998877", "PB-1234"): + for leaked in (SECRET, "hunter2-plain", "pw2", "QTM-SN-998877", "PB-1234", "03:23:02"): assert leaked not in text, f"leaked {leaked!r}" for leaked in ("i3-lab.example.com", "10.20.30.40", "2026-10-11T09", "09:15:02 GMT"): assert leaked not in text, f"leaked {leaked!r}" @@ -63,8 +66,9 @@ def test_sanitize_scrubs_every_secret_class() -> None: assert body["nested"][0] == {"session_token": cc.TOKEN, "count": 3, "ok": True} # Cookie attributes survive; only the value is replaced. assert clean["observed"]["headers"]["set-cookie"] == ( - f"sessionID={cc.TOKEN}; Path=/; HttpOnly; SameSite=lax" + f"sessionID={cc.TOKEN}; expires={cc.TIMESTAMP}; Path=/" ) + assert "0" in body["xml"] and "SN9" not in body["xml"] def test_sanitize_is_pure_and_idempotent() -> None: @@ -135,3 +139,12 @@ def factory() -> httpx.Client: assert not out.exists() assert cc.main(argv, factory=factory) == 0 assert (out / "r1.json").exists() + + +def test_wrong_password_stays_distinguishable_after_redaction() -> None: + recipe = {**RECIPE, "request": {"json": {"name": "a", "password": "wrong"}}} + bad = cc.build_case(recipe, _resp({}), source="self-captured", login="a:right") + recipe["request"]["json"]["password"] = "right" + good = cc.build_case(recipe, _resp({}), source="self-captured", login="a:right") + assert bad["request"]["json"]["password"] == cc.REDACTED_INVALID + assert good["request"]["json"]["password"] == cc.REDACTED diff --git a/tools/compat_capture.py b/tools/compat_capture.py index 0c3fb37..3d65aad 100644 --- a/tools/compat_capture.py +++ b/tools/compat_capture.py @@ -34,6 +34,8 @@ HOST = "" TIMESTAMP = "" REDACTED = "" +# A request password that was NOT the --login credential (a deliberate bad-password case). +REDACTED_INVALID = "" SOURCES = ("self-captured", "captured") _COOKIE_ATTRS = frozenset({"path", "domain", "expires", "max-age", "samesite"}) @@ -79,7 +81,26 @@ def repl(m: re.Match[str]) -> str: return re.sub(r"([^=;,\s]+)=([^;,]*)", repl, value) +def _scrub_xml(value: str, rules: Rules) -> str: + # XML bodies (Accept: application/xml) carry the same secrets as JSON keys. + def repl(m: re.Match[str]) -> str: + tag, text = m.group(1), m.group(2) + if not text: # already a placeholder (or empty): keep idempotent + return m.group(0) + for needles, mark in ( + (rules.password_keys, REDACTED), + (rules.token_keys, TOKEN), + (rules.serial_keys, SERIAL), + ): + if _key_hit(tag, needles): + return f"<{tag}>{mark}" + return m.group(0) + + return re.sub(r"<([A-Za-z_][\w.-]*)>([^<]*)", repl, value) + + def _scrub_str(value: str, rules: Rules) -> str: + value = _scrub_xml(value, rules) value = rules.bearer.sub(lambda m: f"{m.group(1)} {TOKEN}", value) value = rules.url_host.sub(lambda m: f"{m.group(1)}://{HOST}", value) value = rules.ipv4.sub(HOST, value) @@ -93,9 +114,9 @@ def _walk(node: Any, rules: Rules) -> Any: if value is None or isinstance(value, bool): out[key] = value elif _key_hit(key, rules.password_keys): - out[key] = REDACTED + out[key] = value if value in (REDACTED, REDACTED_INVALID) else REDACTED elif _key_hit(key, rules.cookie_keys) and isinstance(value, str): - out[key] = _scrub_cookie(value) + out[key] = _scrub_str(_scrub_cookie(value), rules) elif _key_hit(key, rules.token_keys) and not isinstance(value, (dict, list)): out[key] = TOKEN elif _key_hit(key, rules.serial_keys) and not isinstance(value, (dict, list)): @@ -122,8 +143,11 @@ def dumps_case(case: dict[str, Any]) -> str: def load_recipes(path: Path) -> list[dict[str, Any]]: - files = sorted(path.rglob("*.json")) if path.is_dir() else [path] - return [json.loads(p.read_text()) for p in files] + if not path.is_dir(): + return [json.loads(path.read_text())] + # A directory's own earlier outputs are not recipes: their passwords are redacted. + cases = [json.loads(p.read_text()) for p in sorted(path.rglob("*.json"))] + return [c for c in cases if c.get("source") not in SOURCES] def _login(client: httpx.Client, login: str | None) -> None: @@ -168,6 +192,7 @@ def build_case( profile: str | None = None, appliance_firmware: str | None = None, attest: str | None = None, + login: str | None = None, ) -> dict[str, Any]: if source not in SOURCES: raise CaptureRefusedError(f"source must be one of {SOURCES}, not {source!r}") @@ -195,6 +220,19 @@ def build_case( expected=expected, observed={"headers": observed_headers, "body": clean}, ) + good = login.partition(":")[2] if login else None + payload = case.get("request", {}).get("json") + if isinstance(payload, dict): + # Keep "right vs wrong password" replayable after redaction. + case["request"] = { + **case["request"], + "json": { + k: REDACTED_INVALID + if _key_hit(k, DEFAULT_RULES.password_keys) and isinstance(v, str) and v != good + else v + for k, v in payload.items() + }, + } if profile: case["profile"] = profile if source == "captured": @@ -248,7 +286,7 @@ def run_capture(factory: ClientFactory, cases: list[dict[str, Any]], out: Path, built = [] for recipe in cases: # build everything first: a refusal writes nothing with factory() as client: - built.append(build_case(recipe, replay(client, recipe, login), **opts)) + built.append(build_case(recipe, replay(client, recipe, login), login=login, **opts)) out.mkdir(parents=True, exist_ok=True) for case in built: (out / f"{case['id']}.json").write_text(dumps_case(case)) From f84846449562c44fd18b0eb6845d4e55537f275a Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:32:11 +0000 Subject: [PATCH 185/205] test(compat): add 12 self-captured aml cases and self-captured source to harness self-captured is treated like inferred (regression guard, never fidelity); redacted request passwords replay with the emulator default / a wrong value. --diff over the 15 existing inferred cases showed 0 drift. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../aml-auth-expired-cookie.json | 48 + .../aml-auth-login-malformed-xml.json | 51 + .../aml-auth-login-missing-content-type.json | 44 + .../aml-auth-login-rate-limited.json | 50 + .../aml-auth-login-unsupported-accept.json | 51 + .../aml-auth-login-xml-response.json | 40 + .../aml-auth-login-xxe-body.json | 46 + .../aml-login-invalid-credentials.json | 48 + .../aml/self-captured/aml-login-success.json | 51 + .../aml/self-captured/aml-logout-authed.json | 46 + .../aml-physical-elements-authed.json | 1513 +++++++++++++++++ .../aml-physical-elements-requires-auth.json | 44 + tests/compat/test_compatibility_corpus.py | 23 +- 13 files changed, 2051 insertions(+), 4 deletions(-) create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json create mode 100644 compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json new file mode 100644 index 0000000..7c099b9 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-expired-cookie.json @@ -0,0 +1,48 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-expired-cookie", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "headers": { + "content-length": "169", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; Max-Age=0; Path=/; HttpOnly; SameSite=lax", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Cookie": "sessionID=" + }, + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json new file mode 100644 index 0000000..8de538b --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-malformed-xml.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "Malformed XML login payload" + }, + "status": 400 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-malformed-xml", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "Malformed XML login payload" + }, + "headers": { + "content-length": "103", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Content-Type": "application/xml" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json new file mode 100644 index 0000000..5439cd2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-missing-content-type.json @@ -0,0 +1,44 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 415, + "customCode": 0, + "description": "Error", + "summary": "Unsupported or missing Content-Type" + }, + "status": 415 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-missing-content-type", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 415, + "customCode": 0, + "description": "Error", + "summary": "Unsupported or missing Content-Type" + }, + "headers": { + "content-length": "111", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json new file mode 100644 index 0000000..92b8db2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-rate-limited.json @@ -0,0 +1,50 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Review the error details and retry.", + "code": "AML_ERROR", + "customCode": null, + "description": "Too many login attempts. Try again in 5 minutes.", + "summary": "Too many login attempts. Try again in 5 minutes." + }, + "status": 429 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-rate-limited", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": "Review the error details and retry.", + "code": "AML_ERROR", + "customCode": null, + "description": "Too many login attempts. Try again in 5 minutes.", + "summary": "Too many login attempts. Try again in 5 minutes." + }, + "headers": { + "content-length": "211", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "retry-after": "297", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login", + "repeat": 10 + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json new file mode 100644 index 0000000..74438c5 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-unsupported-accept.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 406, + "customCode": 0, + "description": "Error", + "summary": "Unsupported Accept media type" + }, + "status": 406 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-unsupported-accept", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 406, + "customCode": 0, + "description": "Error", + "summary": "Unsupported Accept media type" + }, + "headers": { + "content-length": "105", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Accept": "image/png" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json new file mode 100644 index 0000000..af077a2 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xml-response.json @@ -0,0 +1,40 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/xml" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-xml-response", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": "0OKLogin successful0", + "headers": { + "content-length": "184", + "content-type": "application/xml", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; HttpOnly; Max-Age=86400; Path=/; SameSite=lax", + "vary": "Origin", + "warning": "Default Password Supplied", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "headers": { + "Accept": "application/xml" + }, + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json new file mode 100644 index 0000000..e52c712 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-auth-login-xxe-body.json @@ -0,0 +1,46 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "XML DTD/entity declarations are not allowed" + }, + "status": 400 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-auth-login-xxe-body", + "manual_ref": "Not in Rev D manual (inferred)", + "observed": { + "body": { + "action": null, + "code": 400, + "customCode": 0, + "description": "Error", + "summary": "XML DTD/entity declarations are not allowed" + }, + "headers": { + "content-length": "119", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "body": "]>&x;", + "content_type": "application/xml", + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json new file mode 100644 index 0000000..9e59e8a --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-invalid-credentials.json @@ -0,0 +1,48 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 401, + "customCode": 0, + "description": "Error", + "summary": "Invalid credentials" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-login-invalid-credentials", + "manual_ref": "Web Services Guide Rev D \u00a72.5", + "observed": { + "body": { + "action": null, + "code": 401, + "customCode": 0, + "description": "Error", + "summary": "Invalid credentials" + }, + "headers": { + "content-length": "95", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json new file mode 100644 index 0000000..0711bf5 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-login-success.json @@ -0,0 +1,51 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Login successful" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-login-success", + "manual_ref": "Web Services Guide Rev D \u00a72.5 (users/login)", + "observed": { + "body": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Login successful", + "token": "" + }, + "headers": { + "content-length": "134", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; HttpOnly; Max-Age=86400; Path=/; SameSite=lax", + "vary": "Origin", + "warning": "Default Password Supplied", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "json": { + "name": "admin", + "password": "" + }, + "method": "POST", + "path": "/aml/users/login" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json new file mode 100644 index 0000000..738f910 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-logout-authed.json @@ -0,0 +1,46 @@ +{ + "auth": "admin", + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Logged out admin" + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-logout-authed", + "manual_ref": "Web Services Guide Rev D \u00a72.5 (logout)", + "observed": { + "body": { + "action": null, + "code": 0, + "customCode": 0, + "description": "OK", + "summary": "Logged out admin" + }, + "headers": { + "content-length": "87", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "set-cookie": "sessionID=; expires=; Max-Age=0; Path=/; SameSite=lax", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "POST", + "path": "/aml/auth/logout" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json new file mode 100644 index 0000000..74cd745 --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-authed.json @@ -0,0 +1,1513 @@ +{ + "auth": "admin", + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "elementList": { + "element": [ + { + "address": 1, + "barcode": "VOL001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 2, + "barcode": "VOL002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 3, + "barcode": "VOL003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 4, + "barcode": "VOL004L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 5, + "barcode": "VOL005L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 6, + "barcode": "VOL006L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 7, + "barcode": "VOL007L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 8, + "barcode": "VOL008L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 9, + "barcode": "VOL009L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 10, + "barcode": "VOL010L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 11, + "barcode": "VOL011L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 12, + "barcode": "VOL012L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 13, + "barcode": "VOL013L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 14, + "barcode": "VOL014L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 15, + "barcode": "VOL015L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 16, + "barcode": "VOL016L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 17, + "barcode": "VOL017L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 18, + "barcode": "VOL018L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 19, + "barcode": "VOL019L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 20, + "barcode": "VOL020L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 21, + "barcode": "VOL021L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 22, + "barcode": "VOL022L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 23, + "barcode": "VOL023L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 24, + "barcode": "VOL024L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 25, + "barcode": "VOL025L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 26, + "barcode": "VOL026L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 27, + "barcode": "VOL027L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 28, + "barcode": "VOL028L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 29, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 30, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 31, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 32, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 33, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 34, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 35, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 36, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 37, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 38, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 39, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 40, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 41, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 42, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 43, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 44, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 45, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 46, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 47, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 48, + "barcode": "CLN001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 49, + "barcode": "CLN002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 50, + "barcode": "CLN003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 0, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + } + ] + } + }, + "status": 200 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-physical-elements-authed", + "manual_ref": "Web Services Guide Rev D, Table 212 (physicalLibrary/elements)", + "observed": { + "body": { + "elementList": { + "element": [ + { + "address": 1, + "barcode": "VOL001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 2, + "barcode": "VOL002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 3, + "barcode": "VOL003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 4, + "barcode": "VOL004L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 5, + "barcode": "VOL005L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 6, + "barcode": "VOL006L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 7, + "barcode": "VOL007L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 8, + "barcode": "VOL008L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 9, + "barcode": "VOL009L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 10, + "barcode": "VOL010L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 11, + "barcode": "VOL011L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 12, + "barcode": "VOL012L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 13, + "barcode": "VOL013L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 14, + "barcode": "VOL014L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 15, + "barcode": "VOL015L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 16, + "barcode": "VOL016L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 17, + "barcode": "VOL017L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 18, + "barcode": "VOL018L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 19, + "barcode": "VOL019L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 20, + "barcode": "VOL020L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 21, + "barcode": "VOL021L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 22, + "barcode": "VOL022L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 23, + "barcode": "VOL023L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 24, + "barcode": "VOL024L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 25, + "barcode": "VOL025L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 1, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 26, + "barcode": "VOL026L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 1, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 27, + "barcode": "VOL027L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 2, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 28, + "barcode": "VOL028L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 3, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 29, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 4, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 30, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 5, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 31, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 6, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 32, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 7, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 33, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 8, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 34, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 9, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 35, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 10, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 36, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 11, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 37, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 12, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 38, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 13, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 39, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 14, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 40, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 15, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 41, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 16, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 42, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 17, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 43, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 18, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 44, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 19, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 45, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 20, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 46, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 21, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 47, + "barcode": null, + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 22, + "section": 2, + "type": 2 + }, + "state": "empty", + "type": "slot" + }, + { + "address": 48, + "barcode": "CLN001L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 23, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 49, + "barcode": "CLN002L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 24, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 50, + "barcode": "CLN003L9", + "coordinate": { + "column": 1, + "frame": 1, + "rack": 1, + "row": 25, + "section": 2, + "type": 2 + }, + "state": "occupied", + "type": "slot" + }, + { + "address": 0, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "drive" + }, + { + "address": 1, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + }, + { + "address": 2, + "barcode": null, + "coordinate": null, + "state": "empty", + "type": "ieSlot" + } + ] + } + }, + "headers": { + "content-length": "7423", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json new file mode 100644 index 0000000..dbbbfcc --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/self-captured/aml-physical-elements-requires-auth.json @@ -0,0 +1,44 @@ +{ + "captured_from": "openblade-emulator", + "expected": { + "headers": { + "content-type": "application/json" + }, + "json_contains": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "status": 401 + }, + "fidelity_notes": "Self-captured from the OpenBlade emulator by tools/compat_capture.py: a regression guard of emulator behaviour, NOT evidence of appliance fidelity.", + "firmware": "341G", + "id": "aml-physical-elements-requires-auth", + "manual_ref": "Web Services Guide Rev D, Table 212 (physicalLibrary/elements)", + "observed": { + "body": { + "action": "Authenticate and retry the request.", + "code": "AML_AUTH_REQUIRED", + "customCode": null, + "description": "Authentication required", + "summary": "Authentication required" + }, + "headers": { + "content-length": "169", + "content-type": "application/json", + "permissions-policy": "geolocation=(), microphone=(), camera=()", + "referrer-policy": "strict-origin-when-cross-origin", + "vary": "Origin", + "x-content-type-options": "nosniff", + "x-frame-options": "DENY" + } + }, + "profile": "scalar-i3-341g", + "request": { + "method": "GET", + "path": "/aml/physicalLibrary/elements" + }, + "source": "self-captured" +} diff --git a/tests/compat/test_compatibility_corpus.py b/tests/compat/test_compatibility_corpus.py index fb536e3..6d888fa 100644 --- a/tests/compat/test_compatibility_corpus.py +++ b/tests/compat/test_compatibility_corpus.py @@ -34,7 +34,14 @@ def _load_cases() -> list[dict]: CASES = _load_cases() CAPTURED = [c for c in CASES if c.get("source") == "captured"] -INFERRED = [c for c in CASES if c.get("source") == "inferred"] +# self-captured = recorded from the OpenBlade emulator by tools/compat_capture.py; +# treated exactly like inferred (regression guard, never fidelity evidence). +INFERRED = [c for c in CASES if c.get("source") in {"inferred", "self-captured"}] +SELF_CAPTURED = [c for c in CASES if c.get("source") == "self-captured"] +# compat_capture redacts request passwords; replay them with the emulator's default. +_REDACTED = "" +_EMULATOR_PASSWORD = "password" +_PLACEHOLDERS = {_REDACTED: _EMULATOR_PASSWORD, "": "wrong-password"} @pytest.fixture() @@ -65,7 +72,12 @@ def _run(client: TestClient, case: dict): if "body" in req: kwargs["content"] = req["body"] else: - kwargs["json"] = req.get("json") + payload = req.get("json") + if isinstance(payload, dict): + payload = { + k: _PLACEHOLDERS.get(v, v) if isinstance(v, str) else v for k, v in payload.items() + } + kwargs["json"] = payload # request.repeat: send the request this many extra times first (e.g. to trip a limiter). for _ in range(int(req.get("repeat", 0))): client.request(req["method"], req["path"], **kwargs) @@ -94,7 +106,7 @@ def _subset_matches(expected: dict, actual: dict) -> list[str]: def test_corpus_is_non_empty_and_wellformed() -> None: assert CASES, "no compatibility cases found" for case in CASES: - assert case.get("source") in {"captured", "inferred"}, case.get("id") + assert case.get("source") in {"captured", "inferred", "self-captured"}, case.get("id") assert case.get("profile") and "request" in case and "expected" in case @@ -134,7 +146,10 @@ def test_captured_case_matches_appliance(case, client: TestClient) -> None: def test_fidelity_coverage_is_reported(capsys) -> None: # Make the gap visible in test output rather than silently green. with capsys.disabled(): - print(f"\n[compatibility corpus] captured={len(CAPTURED)} inferred={len(INFERRED)}") + print( + f"\n[compatibility corpus] captured={len(CAPTURED)} " + f"inferred={len(INFERRED) - len(SELF_CAPTURED)} self-captured={len(SELF_CAPTURED)}" + ) if not CAPTURED: print( "[compatibility corpus] WARNING: i3 wire fidelity is UNVERIFIED (no appliance captures)." From 949781772e01a7dbc2ed40a1ef78c8c4936e384e Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:32:48 +0000 Subject: [PATCH 186/205] docs(compat): appliance capture procedure, redaction list and self-capture loop Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- compatibility/README.md | 36 +++++++++++++++++++++++++++++++----- docs/test-plan.md | 1 + 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/compatibility/README.md b/compatibility/README.md index 2c048af..3fe7b2a 100644 --- a/compatibility/README.md +++ b/compatibility/README.md @@ -54,11 +54,37 @@ exact body match (use for `captured` cases where the full body is authoritative) ## Adding a real appliance capture (the goal) -1. Capture the sanitized request/response on the rig (headers, status, body). -2. Save it here with `"source": "captured"` and the firmware. -3. Run the harness. If the emulator diverges, either the emulator is wrong (fix it) - or the capture reveals a real contract we hadn't modeled (fix the emulator to - match). The emulator is required to satisfy the corpus — not the reverse. +Use `tools/compat_capture.py` (replays each recipe's `request` block, fills +`expected` from the observed response, sanitizes, writes `.json`). + +On site, against the appliance: + +```bash +python tools/compat_capture.py capture --base-url https:// \ + --recipe compatibility/scalar-i3-341g/aml --out compatibility/scalar-i3-341g/aml/captured \ + --source captured --appliance-firmware --attest "" --login : +``` + +- `--source captured` is REFUSED (exit 2, nothing written) unless BOTH + `--appliance-firmware` and `--attest` are given; they are recorded as + `firmware` / `attested_by`. Never label emulator output `captured`. +- Redacted before writing (`sanitize()`, pure + unit-tested): session tokens + (Bearer/Basic, cookie values, `session`/`token`-like JSON keys and XML tags), + serials (`serial*` keys/tags), hosts/IPs in URLs and text, ISO/HTTP-date + timestamps, and request passwords (`` = the `--login` credential, + `` = any other). Review the diff anyway before committing. +- Only response keys the sanitizer left untouched go into + `expected.json_contains`; the full scrubbed response is kept under `observed`. +- `captured` cases are ENFORCED by `tests/compat/test_compatibility_corpus.py`: + if the emulator diverges, fix the emulator (or record a contract we hadn't + modeled). The emulator must satisfy the corpus — not the reverse. Flip the + `len(CAPTURED) == 0` guard in that file when the first capture lands. + +Self-capture loop (emulator, `source: self-captured`, regression guard only): +`--diff` replays existing cases and prints drift without writing (exit 1 on +drift); then capture into `//self-captured/`. Against a live +base URL, the module-global login limiter persists between cases — prefer a +fresh process per run. ## iblade-rev-a diff --git a/docs/test-plan.md b/docs/test-plan.md index 6041607..604dd39 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -36,6 +36,7 @@ single-job) through `InventoryService`. | `emulator+sim-ltfs` | `ScalarHttpLibraryBackend` over in-process ASGI (`openblade.api.main:app`) | `MockLTFSBackend` | always | | `scsi+fake-ltfs` | `RealLibraryBackend` + `MtxChangerBackend` over a stateful fake `mtx`/`sg_inq` runner (`tests/fakes/mtx.py`) | `MockLTFSBackend` | always | | `real+real` | `get_library()` | `get_ltfs()` | only with `OPENBLADE_REAL_HARDWARE_ENABLED=true` (marker `real_hardware`) | +| `compat-capture` | in-process `TestClient` (or `--base-url`) via `tools/compat_capture.py` | n/a | always (`tests/compat`); `self-captured` cases are regression-only, `captured` enforced | Known contract violations are `xfail(strict=True)` with the defect's file:line, so a fix flips them to XPASS and fails the run until the marker is removed. From 28a90ffc9f3caa6c618425e8becc0ff78dd014b2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:33:41 +0000 Subject: [PATCH 187/205] feat(upload): share-push plans book a pool reservation like /nas/archive-plan Route the upload plan through plan_with_pool_reservation and map ReservationError to 409. Mutation-checked: bypassing the helper fails both new tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/routes_upload.py | 25 ++++++++++++------ tests/unit/test_capacity_reservations.py | 32 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/openblade/api/routes_upload.py b/openblade/api/routes_upload.py index 541cfef..453bd9d 100644 --- a/openblade/api/routes_upload.py +++ b/openblade/api/routes_upload.py @@ -53,15 +53,16 @@ def _safe_resolve(base: Path, file_id: str) -> Path: from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.wire import coerce_int +from openblade.nas.capacity import ReservationError, plan_with_pool_reservation from openblade.nas.ingest import ( get_ingest_job, register_archive_plan, run_ingest_job, start_ingest_job, ) -from openblade.nas.planner import ArchivePlanner from openblade.nas.service import NasService from openblade.nas.types import ( + ArchivePlan, ArchivePlanRequest, DatasetStatus, IngestMode, @@ -199,6 +200,21 @@ def _resolve_pool_for_share( return pool +def _plan_share_push(repo: CatalogRepository, request: ArchivePlanRequest) -> ArchivePlan: + """Plan a share push against the pool budget and book it, like ``/nas/archive-plan``.""" + try: + plan = plan_with_pool_reservation(repo, request, now=datetime.now(timezone.utc)) + except ReservationError as exc: + # Curated domain message (pool id only); two plans raced for the same bytes. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + if not plan.is_safe_to_enqueue: + raise HTTPException( + status_code=400, + detail=f"Archive plan is not safe to enqueue: {', '.join(plan.enqueue_blockers)}", + ) + return plan + + def _available_data_tapes(repo: CatalogRepository) -> list[str]: barcodes: set[str] = set() # The physical library holds the writable media (scratch/loaded tapes); the @@ -608,14 +624,9 @@ async def push_staged_files_to_share( if policy.shard_strategy is not None: request_data["shard_strategy"] = policy.shard_strategy - plan = ArchivePlanner().plan(ArchivePlanRequest.model_validate(request_data)) + plan = _plan_share_push(repo, ArchivePlanRequest.model_validate(request_data)) plan.policy_name = policy.name plan.policy_type = policy.policy_type - if not plan.is_safe_to_enqueue: - raise HTTPException( - status_code=400, - detail=f"Archive plan is not safe to enqueue: {', '.join(plan.enqueue_blockers)}", - ) plan = register_archive_plan(plan) dataset_name = request.dataset_name or ( diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 95542bf..86293f1 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -176,3 +176,35 @@ def test_scratch_critical_is_degraded_not_unhealthy(tmp_path: Path) -> None: _, component = _scratch_component(tmp_path, 1, (2, 5)) assert component is not None assert component.status is HealthStatus.DEGRADED + + +def test_upload_share_push_plan_refused_when_pool_budget_consumed( + repo: CatalogRepository, +) -> None: + from fastapi import HTTPException + + from openblade.api.routes_upload import _plan_share_push + + # Arrange: another job holds 1500 of the pool's 2000 bytes; the push needs 600. + assert repo.reserve_nas_capacity( + "pool-1", 1_500, "other-job", NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + ) + + # Act + with pytest.raises(HTTPException) as exc_info: + _plan_share_push(repo, _request(0, pool="pool-1")) + + # Assert: refused on the budget, and nothing new was booked. + assert exc_info.value.status_code == 400 + assert POOL_BUDGET_BLOCKER in str(exc_info.value.detail) + assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)] == ["other-job"] + + +def test_upload_share_push_plan_books_its_reservation(repo: CatalogRepository) -> None: + from openblade.api.routes_upload import _plan_share_push + + plan = _plan_share_push(repo, _request(0, pool="pool-1")) + + assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=datetime.now(UTC))} == { + plan.plan_id + } From 62ece7b8dc1ab09b6505144b6c38d72e9c31b8ce Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:33:55 +0000 Subject: [PATCH 188/205] feat(aml): gate MFA-pending sessions to the MFA challenge routes and logout require_auth now rejects an MFA-pending session with the existing 401 (cookie kept); POST /users/login/mfa, GET /users/login/mfa/{type}/key and logout use require_auth_mfa_pending_ok. A valid TOTP code clears the flag; an invalid one keeps it. Inferred behaviour, mutation-checked. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/aml_state.py | 11 +++++ openblade/api/routes_aml_auth.py | 53 +++++++++++++++------ tests/integration/test_aml_auth_fidelity.py | 47 ++++++++++++++++++ 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 613430b..0665544 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -2526,6 +2526,17 @@ def get_session_user(token: str) -> AmlUser | None: return get_user(record.user_name) +def is_session_mfa_pending(token: str) -> bool: + record = _STATE.sessions.get(token) + return record is not None and record.mfa_pending + + +def clear_session_mfa_pending(token: str) -> None: + record = _STATE.sessions.get(token) + if record is not None: + record.mfa_pending = False + + def clear_session(token: str) -> None: _STATE.sessions.pop(token, None) diff --git a/openblade/api/routes_aml_auth.py b/openblade/api/routes_aml_auth.py index dcd1fb2..8354473 100644 --- a/openblade/api/routes_aml_auth.py +++ b/openblade/api/routes_aml_auth.py @@ -335,18 +335,28 @@ def _verify_totp(secret: str, code: str) -> bool: return pyotp.TOTP(secret).verify(code, valid_window=1) -async def require_auth(request: Request, context: AppContext = Depends(get_context)) -> AmlUser: - _ensure_state(context) +def _session_token(request: Request) -> str | None: + """Return the token that authenticates this request (cookie first, then Bearer).""" + session_id = request.cookies.get("sessionID") + if session_id and aml_state.get_session_user(session_id) is not None: + return session_id + auth = request.headers.get("authorization", "") + if auth.lower().startswith("bearer "): + token = auth.split(" ", 1)[1] + if aml_state.get_session_user(token) is not None: + return token + return None + + +def _authenticate(request: Request, *, allow_mfa_pending: bool) -> AmlUser: # Prefer session cookie, fall back to Bearer token for API clients/tests session_id = request.cookies.get("sessionID") - user = None - if session_id: - user = aml_state.get_session_user(session_id) - if user is None: - auth = request.headers.get("authorization", "") - if auth.lower().startswith("bearer "): - token = auth.split(" ", 1)[1] - user = aml_state.get_session_user(token) + token = _session_token(request) + user = aml_state.get_session_user(token) if token is not None else None + if user is not None and not allow_mfa_pending and aml_state.is_session_mfa_pending(token or ""): + # Inferred: an MFA-pending session authenticates nothing but the MFA + # challenge routes and logout; same 401 as unauthenticated, cookie kept. + raise HTTPException(status_code=401, detail="Authentication required") if user is None: headers = None if session_id: @@ -356,6 +366,19 @@ async def require_auth(request: Request, context: AppContext = Depends(get_conte return user +async def require_auth(request: Request, context: AppContext = Depends(get_context)) -> AmlUser: + _ensure_state(context) + return _authenticate(request, allow_mfa_pending=False) + + +async def require_auth_mfa_pending_ok( + request: Request, context: AppContext = Depends(get_context) +) -> AmlUser: + """Auth for MFA challenge routes and logout, which an MFA-pending session may use.""" + _ensure_state(context) + return _authenticate(request, allow_mfa_pending=True) + + _XML_MEDIA_TYPES = frozenset({"application/xml", "text/xml"}) _FORM_MEDIA_TYPES = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) _ACCEPTABLE_MEDIA_TYPES = frozenset({"*/*", "application/*", "text/*", "application/json"}) @@ -647,7 +670,7 @@ async def auth_login(request: Request, context: AppContext = Depends(get_context @router.delete("/users/login", response_model=WSResultCode) async def logout( request: Request, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> Response: _ensure_state(context) @@ -665,7 +688,7 @@ async def logout( @router.post("/auth/logout", response_model=WSResultCode) async def auth_logout( request: Request, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> Response: return await logout(request, current_user, context) @@ -673,8 +696,9 @@ async def auth_logout( @router.post("/users/login/mfa", response_model=WSResultCode) async def validate_mfa( + request: Request, payload: MFAAuthentication, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> WSResultCode: _ensure_state(context) @@ -684,13 +708,14 @@ async def validate_mfa( code = _validate_mfa_code(payload.authenticationCode) if not _verify_totp(config.secret, code): raise HTTPException(status_code=401, detail="Invalid authentication code") + aml_state.clear_session_mfa_pending(_session_token(request) or "") return _ws_result(f"MFA validated for {payload.type}") @router.get("/users/login/mfa/{type}/key", response_model=MFASharedData) async def get_mfa_key( type: str, - current_user: AmlUser = Depends(require_auth), + current_user: AmlUser = Depends(require_auth_mfa_pending_ok), context: AppContext = Depends(get_context), ) -> MFASharedData: _ensure_state(context) diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py index 7cc3473..4a9dde1 100644 --- a/tests/integration/test_aml_auth_fidelity.py +++ b/tests/integration/test_aml_auth_fidelity.py @@ -359,3 +359,50 @@ def test_mfa_login_strict_is_412_and_session_issued( def test_login_without_enabled_mfa_is_not_pending(client: TestClient) -> None: token = _login(client) assert aml_state._STATE.sessions[token].mfa_pending is False + + +def _pending_login(client: TestClient) -> tuple[str, dict[str, str]]: + secret = _enable_totp(client) + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + client.cookies.clear() + return secret, {"Authorization": f"Bearer {response.json()['token']}"} + + +def test_mfa_pending_session_is_401_until_valid_code(client: TestClient) -> None: + import pyotp + + secret, auth = _pending_login(client) + assert client.get("/aml/users", headers=auth).status_code == 401 + assert client.get("/aml/physicalLibrary/elements", headers=auth).status_code == 401 + bad = client.post( + "/aml/users/login/mfa", json={"type": "totp", "authenticationCode": "000001"}, headers=auth + ) + assert bad.status_code == 401 + assert client.get("/aml/users", headers=auth).status_code == 401 # still pending + good = client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": pyotp.TOTP(secret).now()}, + headers=auth, + ) + assert good.status_code == 200, good.text + assert client.get("/aml/users", headers=auth).status_code == 200 + + +def test_mfa_pending_cookie_session_is_gated_without_dropping_cookie( + client: TestClient, +) -> None: + _enable_totp(client) + assert client.post("/aml/users/login", json=CREDS).status_code == 200 + blocked = client.get("/aml/users") + assert blocked.status_code == 401 + assert "set-cookie" not in blocked.headers + + +def test_mfa_pending_session_can_fetch_key_route_and_logout(client: TestClient) -> None: + _, auth = _pending_login(client) + # Key route is reachable (403 because TOTP is already confirmed, not 401). + assert client.get("/aml/users/login/mfa/totp/key", headers=auth).status_code == 403 + client.cookies.set("sessionID", auth["Authorization"].split(" ", 1)[1]) + assert client.delete("/aml/users/login").status_code == 200 + assert client.get("/aml/users", headers=auth).status_code == 401 From 3764b7826f2d338dd270c4e7097345dc5de35664 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:37:13 +0000 Subject: [PATCH 189/205] test(i3): emulator-only MFA-pending session round trip Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/i3/test_aml_mfa_pending.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/i3/test_aml_mfa_pending.py diff --git a/tests/i3/test_aml_mfa_pending.py b/tests/i3/test_aml_mfa_pending.py new file mode 100644 index 0000000..5ba25ee --- /dev/null +++ b/tests/i3/test_aml_mfa_pending.py @@ -0,0 +1,70 @@ +"""MFA-required pending session (emulator only; inferred, not in the Rev D manual). + +A login by a user with an enabled MFA record answers the documented login body +plus ``Warning: 199 - "MFA Required"``; until ``POST /aml/users/login/mfa`` +succeeds the session is refused (401) everywhere except the MFA challenge routes +and logout. The challenge wire shape must be certified by appliance capture. +""" + +from __future__ import annotations + +import httpx +import pyotp +import pytest + +pytestmark = pytest.mark.i3 + +MFA_WARNING = '199 - "MFA Required"' + + +def _bearer(client: httpx.Client, user: str, password: str) -> dict[str, str]: + resp = client.post("/aml/users/login", json={"name": user, "password": password}) + assert resp.status_code in (200, 412), resp.text + return {"Authorization": f"Bearer {resp.json()['token']}"} + + +def test_mfa_pending_session_gated_until_totp( + i3_mode: str, i3_base_url: str, i3_credentials: tuple[str, str] +) -> None: + if i3_mode != "emulator": + pytest.skip("enabling MFA on a real appliance account is out of scope") + user, password = i3_credentials + with httpx.Client(base_url=i3_base_url, timeout=30.0) as client: + setup = _bearer(client, user, password) + secret = client.get("/aml/users/login/mfa/totp/key", headers=setup).json()["key"] + totp = pyotp.TOTP(secret) + enabled = client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": True, "authenticationCode": totp.now()}, + headers=setup, + ) + assert enabled.status_code == 200, enabled.text + client.cookies.clear() + pending: dict[str, str] = {} + try: + login = client.post("/aml/users/login", json={"name": user, "password": password}) + assert login.status_code in (200, 412) + assert MFA_WARNING in login.headers.get_list("Warning") + pending = {"Authorization": f"Bearer {login.json()['token']}"} + client.cookies.clear() + assert client.get("/aml/physicalLibrary/elements", headers=pending).status_code == 401 + ok = client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": totp.now()}, + headers=pending, + ) + assert ok.status_code == 200, ok.text + assert client.get("/aml/physicalLibrary/elements", headers=pending).status_code == 200 + finally: + if pending: + client.post( + "/aml/users/login/mfa", + json={"type": "totp", "authenticationCode": totp.now()}, + headers=pending, + ) + client.cookies.clear() + client.put( + "/aml/users/mfa", + json={"type": "totp", "enabled": False, "authenticationCode": totp.now()}, + headers=pending or setup, + ) From 4419c6058ae8fcfabb57ca4189264e8889979e9d Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:37:25 +0000 Subject: [PATCH 190/205] docs(aml): describe the inferred MFA-pending login state Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/reference/i3-contract-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/i3-contract-notes.md b/docs/reference/i3-contract-notes.md index 9d00ffd..fbc653c 100644 --- a/docs/reference/i3-contract-notes.md +++ b/docs/reference/i3-contract-notes.md @@ -75,4 +75,4 @@ inferred (OpenBlade-chosen) and unverified against an appliance. | Unsupported Accept on login/logout: 406 "Unsupported Accept media type" | inferred | | Login rate limit 429 carries `Retry-After` (seconds) | inferred | | Expired/unknown `sessionID` cookie: 401 and `Set-Cookie` clearing it (Max-Age=0); expired Bearer: 401 | inferred | -| MFA-required login state | not implemented | +| MFA pending (inferred): a login by a user with an enabled MFA record keeps the documented body and adds `Warning: 199 - "MFA Required"`; with `OPENBLADE_AML_STRICT_PRECONDITIONS=true` it is 412 `"summary":"MFA verification required"` (session still issued). Until a valid code is posted to `POST /aml/users/login/mfa`, the session gets the ordinary 401 (cookie kept) on every route except that one, `GET /aml/users/login/mfa/{type}/key` and logout; an invalid code leaves it pending. Challenge wire shape not documented in Rev D manual; certify by appliance capture | inferred | From 754fdc621c0dee38f87e63db473daf1a79ef062f Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:37:28 +0000 Subject: [PATCH 191/205] feat(nas): release a plan's pool reservation when its ingest job ends Add CatalogRepository.delete_nas_reservations_for_owner and call it from run_ingest_job once the job reaches archived/failed/cancelled (the job carries the plan, whose plan_id is the reservation owner). A non-terminal exit keeps the booking until its 1h expiry. Mutation-checked: removing the release fails test_finished_job_releases_its_plan_reservation. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 13 ++++++++ openblade/nas/ingest.py | 13 +++++++- tests/unit/test_capacity_reservations.py | 42 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 6c9f2b8..99a5e1f 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -1435,6 +1435,19 @@ def reserve_nas_capacity( def delete_nas_reservation(self, reservation_id: str) -> bool: return self._delete_nas_row(NasReservation, reservation_id) + def delete_nas_reservations_for_owner(self, owner: str) -> int: + """Delete every reservation booked by ``owner`` (a plan id); return how many.""" + rows = list( + self.session.execute( + select(NasReservation).where(NasReservation.owner == owner) + ).scalars() + ) + for row in rows: + self.session.delete(row) + if rows: + self.session.commit() + return len(rows) + def list_nas_export_sets(self) -> list[dict[str, object]]: rows = self.session.execute(select(NasExportSet).order_by(NasExportSet.id)).scalars() return [ diff --git a/openblade/nas/ingest.py b/openblade/nas/ingest.py index 9f1ca42..5bcd713 100644 --- a/openblade/nas/ingest.py +++ b/openblade/nas/ingest.py @@ -793,7 +793,18 @@ def run_ingest_job( ltfs=ltfs, config=nas_service.get_source_stream_config(), ) - return executor.run() + try: + return executor.run() + finally: + # The plan's pool booking (owner=plan_id) is held only while the job can still + # write; any terminal state frees it. A non-terminal exit keeps it until expiry. + if job.status in _TERMINAL_JOB_STATES: + nas_service.repository.delete_nas_reservations_for_owner(job.plan.plan_id) + + +_TERMINAL_JOB_STATES = frozenset( + {DatasetStatus.ARCHIVED, DatasetStatus.FAILED, DatasetStatus.CANCELLED} +) def _build_archive_lifecycle_manager( diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 86293f1..82305d5 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -208,3 +208,45 @@ def test_upload_share_push_plan_books_its_reservation(repo: CatalogRepository) - assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=datetime.now(UTC))} == { plan.plan_id } + + +def test_finished_job_releases_its_plan_reservation(tmp_path: Path) -> None: + from openblade.nas.ingest import register_archive_plan, start_ingest_job + from tests.unit.test_nas_ingest import _make_plan, _run_job, _setup_service + + # Arrange: the plan's own booking fills the 2000-byte pool; another job holds none. + service, cache_root = _setup_service(tmp_path) + repo = service.repository + now = datetime.now(UTC) + plan = register_archive_plan(_make_plan(cache_root)) + assert repo.reserve_nas_capacity( + "pool-1", 2_000, plan.plan_id, now + PLAN_RESERVATION_TTL, capacity_bytes=2_000, now=now + ) + assert POOL_BUDGET_BLOCKER in ( + plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=now).enqueue_blockers + ) + job = start_ingest_job( + plan=plan, + dataset_name="dataset-a", + pool_id="pool-1", + nas_service=service, + cache_drive_id="cache-1", + ) + + # Act + finished = _run_job(service, job.job_id) + + # Assert: terminal job freed the booking, so the same plan now fits. + assert finished.status.value in {"archived", "failed", "cancelled"} + assert repo.reservations_for_pool("pool-1", now=now) == [] + assert plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=now).is_safe_to_enqueue + + +def test_delete_reservations_for_owner_keeps_other_owners(repo: CatalogRepository) -> None: + for owner in ("plan-a", "plan-a", "plan-b"): + assert repo.reserve_nas_capacity( + "pool-1", 10, owner, NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + ) + + assert repo.delete_nas_reservations_for_owner("plan-a") == 2 + assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)] == ["plan-b"] From 56635cba2dee1a131e0d9d3d5902275fca2ebe54 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:38:17 +0000 Subject: [PATCH 192/205] docs: capacity reservations decision record, tech-debt, wiki api regen Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 24 +++++++++++++++++++ docs/tech-debt.md | 12 ++++++---- docs/wiki/reference/api.md | 8 +++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index a97aa0c..080e9ab 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -273,3 +273,27 @@ Known costs: each claim of id *k* writes up to *k* released probe rows; claims are not heartbeated, so they lapse after the TTL. A repository method that leases one specific drive id would remove both the probes and the retry loop. Covered by `tests/unit/test_job_queue_persistent.py` (two sessions, one file). + +## Capacity reservations + +Decisions (WS-capacity, fleet round 3): + +1. **Pool reservations are the source of truth.** The planner takes + `pool_reserved_bytes` (sum of unexpired `nas_reservations` for the pool) and adds + the `POOL_BUDGET_BLOCKER` when placed bytes exceed total tape capacity minus that + budget. `reserved_bytes` stays per barcode for explicit holds. +2. **A plan books its own reservation.** `capacity.plan_with_pool_reservation` books + `total_bytes * copies` under `owner=plan_id` for 1h through the atomic + `reserve_nas_capacity`; a lost race raises `ReservationError` (HTTP 409). Both + `/nas/archive-plan` and the upload share-push use it. `run_ingest_job` deletes the + owner's rows once the job is archived, failed or cancelled; expiry covers the rest. +3. **Scratch** = `volume_group_id IS NULL AND state = 'in_slot'` + (`count_scratch_cartridges`); the planner's barcode heuristic is unchanged. +4. **Health**: a `scratch` component only when thresholds are configured; WARN is OK + with a message, CRITICAL is DEGRADED, never UNHEALTHY. Thresholds are managed by + `GET`/`PUT /nas/scratch-thresholds` (404 unset, 422 on `warn < min`). + +Reviewer risks still open: `request.pool` is used as the pool id without checking it +names a configured pool; client-supplied `tape_capacities` set the capacity the +booking is checked against, so a caller can inflate it; and an abandoned plan holds +its bytes for the full hour. diff --git a/docs/tech-debt.md b/docs/tech-debt.md index 422695e..3e2467d 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -84,10 +84,14 @@ and no production caller uses `claim_drive`/`claim_changer` yet. ## NAS persistence gaps -`ArchivePlanRequest.reserved_bytes` is keyed by barcode while `nas_reservations` -are per pool; mapping one to the other needs a design decision before the planner -can consume the ledger. Scratch-threshold health (`get_scratch_thresholds`) has no -consumer because the catalog has no scratch-cartridge count to evaluate. +Resolved: unexpired `nas_reservations` rows are a pool-level budget +(`ArchivePlanRequest.pool_reserved_bytes`) that every plan from `/nas/archive-plan` +and the upload share-push must fit under; the per-barcode `reserved_bytes` dict +stays for explicit holds. A safe plan books its own bytes (`owner=plan_id`) with a +1h TTL, released when its ingest job reaches a terminal state and otherwise by +expiry (abandoned plans hold the pool for up to an hour). Scratch health counts +cartridges with `volume_group_id IS NULL AND state='in_slot'`; the `scratch` +component appears only once thresholds are set and degrades, never fails readiness. `nas_file_spans` has no `pool_id`, so identical relative paths in two pools share spans. A real NFSv4 ACL protocol scenario needs a Ganesha image built with VFS ACL support; the shipped Debian package cannot store ACLs. \ No newline at end of file diff --git a/docs/wiki/reference/api.md b/docs/wiki/reference/api.md index 3b66b4a..fb4e7cf 100644 --- a/docs/wiki/reference/api.md +++ b/docs/wiki/reference/api.md @@ -6,7 +6,7 @@ Introspected from the OpenAPI schema of `openblade.api.main:app`. -**1186 operations** total: 206 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. +**1190 operations** total: 210 on the native OpenBlade control plane, 980 on the Quantum AML / iBlade emulator surface. The application serves two surfaces from one ASGI app. Setting `OPENBLADE_SCALAR_API_ONLY=true` puts it in emulator-only mode, where the @@ -114,7 +114,7 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `GET` | `/virtual/ls` | List Virtual Directory | `path`? (query) | - | `VirtualDirectoryListing` | | `GET` | `/virtual/stat` | Stat Virtual Path | `path` (query) | - | `VirtualFileEntry` | -### `NAS Config` (124 operations) +### `NAS Config` (128 operations) | Method | Path | Summary | Parameters | Request | Response | | --- | --- | --- | --- | --- | --- | @@ -172,6 +172,8 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/nas/restore-jobs/{job_id}/retry` | Retry Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/nas/restore-jobs/{job_id}/run` | Run Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/nas/restore-plan` | Restore Plan | - | `RestorePlanRequest` | `RestorePlan` | +| `GET` | `/nas/scratch-thresholds` | Get Scratch Thresholds | - | - | `ScratchThresholds` | +| `PUT` | `/nas/scratch-thresholds` | Put Scratch Thresholds | - | `ScratchThresholds` | `ScratchThresholds` | | `GET` | `/nas/shares` | List Shares | - | - | `NasShareDefinition[]` | | `POST` | `/nas/shares` | Create Or Update Share | - | `NasShareDefinition` | `NasShareDefinition` | | `GET` | `/nas/shares/{share_id}` | Get Share | `share_id` (path) | - | `NasShareDefinition` | @@ -234,6 +236,8 @@ Interactive docs for a running instance are at `/docs` and `/redoc`. | `POST` | `/storage/nas/restore-jobs/{job_id}/retry` | Retry Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/storage/nas/restore-jobs/{job_id}/run` | Run Restore Job Endpoint | `job_id` (path) | - | `NasRestoreJob` | | `POST` | `/storage/nas/restore-plan` | Restore Plan | - | `RestorePlanRequest` | `RestorePlan` | +| `GET` | `/storage/nas/scratch-thresholds` | Get Scratch Thresholds | - | - | `ScratchThresholds` | +| `PUT` | `/storage/nas/scratch-thresholds` | Put Scratch Thresholds | - | `ScratchThresholds` | `ScratchThresholds` | | `GET` | `/storage/nas/shares` | List Shares | - | - | `NasShareDefinition[]` | | `POST` | `/storage/nas/shares` | Create Or Update Share | - | `NasShareDefinition` | `NasShareDefinition` | | `GET` | `/storage/nas/shares/{share_id}` | Get Share | `share_id` (path) | - | `NasShareDefinition` | From d6ee36596cb6cb33b4fc517c4ec2d8a1ea05db43 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:41:55 +0000 Subject: [PATCH 193/205] test(compat): MFA setup hook + inferred MFA-pending login case Corpus cases may declare "setup": {"mfa": {"user", "type"}}; the harness enables TOTP for that user via GET /aml/users/login/mfa/{type}/key + PUT /aml/users/mfa before replaying the request. Adds the inferred aml-auth-login-mfa-pending case (200 + Warning "MFA Required") and corrects logout.json fidelity_notes, which still called MFA, session-expiry, max-sessions and default-password states unmodelled. Mutation-checked: with the setup call disabled the case fails (Warning lacks "MFA Required"). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../aml/auth-login-mfa-pending.json | 11 +++++++++ compatibility/scalar-i3-341g/aml/logout.json | 2 +- tests/compat/test_compatibility_corpus.py | 24 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json diff --git a/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json new file mode 100644 index 0000000..f1a7b7a --- /dev/null +++ b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json @@ -0,0 +1,11 @@ +{ + "id": "aml-auth-login-mfa-pending", + "profile": "scalar-i3-341g", + "source": "inferred", + "firmware": "341G", + "manual_ref": "Not in Rev D manual (inferred)", + "setup": { "mfa": { "user": "admin", "type": "totp" } }, + "fidelity_notes": "Login by a user with an enabled MFA record returns the documented login body plus Warning: 199 - \"MFA Required\" (412 in strict mode); the session is pending (401 everywhere except POST /users/login/mfa, GET /users/login/mfa/{type}/key and logout) until a valid code is posted. The challenge wire shape is not documented in the Rev D manual; certify by appliance capture.", + "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" } }, + "expected": { "status": 200, "headers": { "Warning": "MFA Required" } } +} diff --git a/compatibility/scalar-i3-341g/aml/logout.json b/compatibility/scalar-i3-341g/aml/logout.json index be77cee..c4695f4 100644 --- a/compatibility/scalar-i3-341g/aml/logout.json +++ b/compatibility/scalar-i3-341g/aml/logout.json @@ -5,7 +5,7 @@ "firmware": "341G", "auth": "admin", "manual_ref": "Web Services Guide Rev D §2.5 (logout)", - "fidelity_notes": "Emulator supports BOTH cookie-session (sessionID) and Bearer, with logout at POST /aml/auth/logout AND RESTful DELETE /aml/users/login, clearing the session cookie — closer to the real contract than the review assumed. Session-expiry, MFA, max-sessions, and default-password states remain unmodeled (roadmap Phase 5); certify the exact logout path/verb against a capture.", + "fidelity_notes": "Emulator supports BOTH cookie-session (sessionID) and Bearer, with logout at POST /aml/auth/logout AND RESTful DELETE /aml/users/login, clearing the session cookie — closer to the real contract than the review assumed. Session expiry (sliding timeout), per-user max-sessions eviction, the default-password Warning, and an inferred MFA-pending session (logout stays reachable while pending) are now modelled; all are emulator-inferred, not appliance-captured. Certify the exact logout path/verb against a capture.", "request": { "method": "POST", "path": "/aml/auth/logout" }, "expected": { "status": 200 } } diff --git a/tests/compat/test_compatibility_corpus.py b/tests/compat/test_compatibility_corpus.py index 6d888fa..b8ac436 100644 --- a/tests/compat/test_compatibility_corpus.py +++ b/tests/compat/test_compatibility_corpus.py @@ -61,7 +61,31 @@ def _authenticate(client: TestClient) -> None: assert resp.status_code == 200, f"corpus auth setup failed: {resp.status_code}" +def _enable_mfa(client: TestClient, user: str, mfa_type: str) -> None: + # Same documented route pair as tests/i3/test_aml_mfa_pending.py: fetch the + # user's key, then PUT /aml/users/mfa with a valid code to enable it. + import pyotp + + login = client.post("/aml/users/login", json={"name": user, "password": _EMULATOR_PASSWORD}) + assert login.status_code == 200, f"corpus mfa setup login failed: {login.status_code}" + bearer = {"Authorization": f"Bearer {login.json()['token']}"} + key = client.get(f"/aml/users/login/mfa/{mfa_type}/key", headers=bearer) + assert key.status_code == 200, f"corpus mfa setup key failed: {key.status_code}" + code = pyotp.TOTP(key.json()["key"]).now() + enabled = client.put( + "/aml/users/mfa", + json={"type": mfa_type, "enabled": True, "authenticationCode": code}, + headers=bearer, + ) + assert enabled.status_code == 200, f"corpus mfa setup enable failed: {enabled.status_code}" + client.cookies.clear() + + def _run(client: TestClient, case: dict): + # Optional "setup": {"mfa": {"user": ..., "type": "totp"}} enables MFA first. + mfa = (case.get("setup") or {}).get("mfa") + if mfa: + _enable_mfa(client, mfa["user"], mfa["type"]) if case.get("auth") == "admin": _authenticate(client) req = case["request"] From 18b9faf37c108841a40444b12594dfcaaf053277 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 03:42:11 +0000 Subject: [PATCH 194/205] docs(roadmap): refresh status rows 3, 18, 19, 22, 23 (as of 2026-10-11) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- docs/production-readiness-roadmap.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/production-readiness-roadmap.md b/docs/production-readiness-roadmap.md index d4e1239..9dcdf4d 100644 --- a/docs/production-readiness-roadmap.md +++ b/docs/production-readiness-roadmap.md @@ -26,7 +26,7 @@ Verified against the repo before writing (not asserted from memory): - `sharded_archive.py:234,251` calls `mark_instance_archived` per-shard inside the write loop → review #6 confirmed real. ✅ - Review #5 (scheduler `drive_id` mutation) **not reproduced**: `jobs/scheduler.py` is a plain allocator with no `_load_barcode`/mutation — re-investigate before "fixing." ⚠️ -## Status as of 2026-10-10 (fleet integration on `feat/persistent-drive-leases`) +## Status as of 2026-10-11 (fleet integration on `feat/persistent-drive-leases`) Verified against the tree, not asserted. "Landed" = implemented and tested in-repo; "appliance-gated" = authored and runnable, cannot be certified without the rig. @@ -35,7 +35,7 @@ Verified against the tree, not asserted. "Landed" = implemented and tested in-re |---|---|---|---| | 1 | Relabel simplified endpoint | landed | `routes_aml_move_medium.py` | | 2 | ScalarCoordinate / MoveClass IntFlag | landed (bit values appliance-gated) | `domain/scalar_coordinate.py` | -| 3 | Compatibility corpus + differential harness | scaffold landed; 0 captured cases (appliance-gated) | `compatibility/`, `tests/compat/` | +| 3 | Compatibility corpus + differential harness | scaffold + `tools/compat_capture.py` capture/sanitize/diff; 12 self-captured emulator regression cases (0 drift vs inferred set); 0 appliance captures — fidelity unverified | `compatibility/`, `tests/compat/` | | 4 | Atomic sharded commit | landed: STAGING→VERIFYING→one ARCHIVED commit, job journal, no suppressed cleanup errors; `physical_state_unknown` now has a consumer (drive/barcode refusal, `reconcile_drive`, fail-closed when the journal write fails); resume = report-only | `jobs/sharded_archive.py`, `catalog/repository.py`, `jobs/recovery.py` | | 5 | Protocol-agnostic backends + contract suite | landed (sim, in-process emulator, scsi+fake-ltfs over a stateful mtx fake, real gated) | `tests/contract/` | | 6 | Scheduler lease regression | landed | `tests/unit/test_drive_leases.py` | @@ -50,12 +50,12 @@ Verified against the tree, not asserted. "Landed" = implemented and tested in-re | 15 | Workflow hardening | landed: SHA pins, per-job permissions, actionlint/zizmor/shellcheck/yamllint | `workflow-lint.yml` | | 16 | Differential + mutation | mutation lane landed (mutmut 3.8.0, ratchet baseline for 6/6 modules as of 2026-10-10); differential appliance-gated | `tools/mutation_run.sh`, `mutation/baseline.txt` | | 17 | Timing profiles + virtual clock | landed (10 profiles; the 4 fault profiles are injected by the emulator via `OPENBLADE_EMULATOR_FAULT_PROFILE`) | `tests/i3/timing.py` | -| 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-required pending state not modelled | `routes_aml_auth.py`, `compatibility/` | -| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 33/33 pass locally incl. real SMB byte-range lock conflict; NFSv4 ACL storage is unsupported by Debian's Ganesha VFS FSAL — test pins that limit) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | +| 18 | Auth fidelity | landed as inferred behaviour + corpus cases: session cap, strict 412, XML, 415/406, 429 Retry-After, error matrix; MFA-pending session modelled (inferred): enabled-MFA login → 200 + `Warning: 199 - "MFA Required"` (412 strict); 401 except MFA challenge/key/logout until a valid TOTP; certify by appliance capture | `routes_aml_auth.py`, `compatibility/` | +| 19 | Real SMB/NFS protocol tests | landed (containerised rig, 33/33 pass locally incl. real SMB byte-range lock conflict; NFSv4 named-user DENY ACE enforced across uids (Fedora 41 Ganesha 5.9 stage, VFS ACL mapping; EVERYONE@ deny folds into mode bits)) | `deploy/nas-protocols/`, `tests/e2e/protocols/` | | 20 | Tape-native NAS models | landed; pool protection/replication, reservations (atomic reserve), export sets, vaults, VG replication and file spans persisted in the catalog; scratch-threshold health and per-barcode `reserved_bytes` mapping still TODO | `nas/state_machine.py`, `nas/media.py`, `nas/capacity.py` | | 21 | FUSE data plane | landed: hydrator, cache (open-time integrity check, persisted LRU index, chunked pread), xattr state, gateway; see `docs/fuse.md` | `fuse/` | -| 22 | Self-hosted i3/LTFS lanes | appliance-gated, authored: read-only nightly + destructive with environment approval, allowlist, snapshots | `hardware-*.yml`, `tools/hardware/` | -| 23 | Drive-identity bridge | landed; on-rig certification test authored | `hardware/correlation.py`, `tests/hardware/readonly/` | +| 22 | Self-hosted i3/LTFS lanes | appliance-gated, authored: read-only nightly + destructive with environment approval, allowlist, snapshots; reboot persistence still manual (soft-reboot automation drafted, held for operator decision) | `hardware-*.yml`, `tools/hardware/` | +| 23 | Drive-identity bridge | implemented + test authored; uncertified — skips on mhvtl (needs an AML endpoint with matching serials); certify on appliance | `hardware/correlation.py`, `tests/hardware/readonly/` | | 24 | Release pipeline | landed (never exercised on a tag): build-once digest, SBOM, scans, cosign, staging→approval→production, rollback | `release.yml`, `docs/runbooks/release.md` | --- From eb39446af73b5ad6c6c003761c71e9d885eba30d Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:01:55 +0000 Subject: [PATCH 195/205] fix(compat): sanitizer masks nested secret leaves, XML attrs, tagged elements - token/serial/password-class keys now mask every scalar leaf beneath them (a list/dict value was previously walked and leaked, e.g. {"x-auth-tokens": ["eyJ..."]}). - _scrub_xml matches elements that carry attributes and scrubs attribute values whose name hits a key rule (sessionId="..."). - Basic credentials were already covered by the Bearer|Basic rule; now tested. Mutation-checked: reverting the list masking, the attribute pass, the element-attrs pattern, or dropping Basic each fails the new tests. Self-capture re-run over the 12 aml recipes: no sanitizer-attributable changes (sanitize() is identity on all 12 committed files), so the corpus is not regenerated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/compat/test_compat_capture.py | 28 ++++++++++++ tools/compat_capture.py | 66 +++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/tests/compat/test_compat_capture.py b/tests/compat/test_compat_capture.py index f70fd71..ebbd30a 100644 --- a/tests/compat/test_compat_capture.py +++ b/tests/compat/test_compat_capture.py @@ -148,3 +148,31 @@ def test_wrong_password_stays_distinguishable_after_redaction() -> None: good = cc.build_case(recipe, _resp({}), source="self-captured", login="a:right") assert bad["request"]["json"]["password"] == cc.REDACTED_INVALID assert good["request"]["json"]["password"] == cc.REDACTED + + +@pytest.mark.parametrize( + ("dirty", "secret"), + [ + # Secret-class key holding a list: every leaf is masked, not just scalars. + ({"x-auth-tokens": ["eyJhbGciOi.secret"]}, "eyJhbGciOi.secret"), + ({"serials": {"drives": ["SN-LEAK-1"]}}, "SN-LEAK-1"), + ({"password": ["hunter2-list"]}, "hunter2-list"), + # XML attribute whose name hits a key rule. + ({"body": 'x'}, "S3CR3T"), + # XML element that carries attributes. + ({"body": 'S3CR3T2'}, "S3CR3T2"), + # Basic credentials in a key that is not token-class. + ({"note": "Basic dXNlcjpwYXNz"}, "dXNlcjpwYXNz"), + ], +) +def test_sanitize_scrubs_nested_xml_attr_and_basic_shapes( + dirty: dict[str, Any], secret: str +) -> None: + clean = cc.sanitize(dirty) + assert secret not in json.dumps(clean) + assert cc.sanitize(clean) == clean # still idempotent + + +def test_sanitize_xml_keeps_non_secret_attributes() -> None: + clean = cc.sanitize({"body": 'x'}) + assert clean["body"] == f'{cc.TOKEN}' diff --git a/tools/compat_capture.py b/tools/compat_capture.py index 3d65aad..865cb0a 100644 --- a/tools/compat_capture.py +++ b/tools/compat_capture.py @@ -64,6 +64,11 @@ class Rules: DEFAULT_RULES = Rules() +_XML_START_TAG = re.compile(r"<[A-Za-z_][\w.-]*\s[^<>]*>") +_XML_ATTR = re.compile(r"([A-Za-z_][\w.:-]*)\s*=\s*([\"'])(.*?)\2") +_XML_ELEMENT = re.compile(r"<([A-Za-z_][\w.-]*)((?:\s[^<>]*)?)>([^<]*)") + + def _norm(key: str) -> str: return key.lower().replace("_", "").replace("-", "") @@ -81,22 +86,38 @@ def repl(m: re.Match[str]) -> str: return re.sub(r"([^=;,\s]+)=([^;,]*)", repl, value) +def _secret_mark(name: str, rules: Rules) -> str | None: + for needles, mark in ( + (rules.password_keys, REDACTED), + (rules.token_keys, TOKEN), + (rules.serial_keys, SERIAL), + ): + if _key_hit(name, needles): + return mark + return None + + def _scrub_xml(value: str, rules: Rules) -> str: - # XML bodies (Accept: application/xml) carry the same secrets as JSON keys. - def repl(m: re.Match[str]) -> str: - tag, text = m.group(1), m.group(2) + # XML bodies (Accept: application/xml) carry the same secrets as JSON keys, + # both as element text and as attribute values (``sessionId="..."``). + def attr_repl(m: re.Match[str]) -> str: + mark = _secret_mark(m.group(1), rules) + return m.group(0) if mark is None else f"{m.group(1)}={m.group(2)}{mark}{m.group(2)}" + + def tag_repl(m: re.Match[str]) -> str: + return _XML_ATTR.sub(attr_repl, m.group(0)) + + def elem_repl(m: re.Match[str]) -> str: + tag, attrs, text = m.group(1), m.group(2), m.group(3) if not text: # already a placeholder (or empty): keep idempotent return m.group(0) - for needles, mark in ( - (rules.password_keys, REDACTED), - (rules.token_keys, TOKEN), - (rules.serial_keys, SERIAL), - ): - if _key_hit(tag, needles): - return f"<{tag}>{mark}" - return m.group(0) + mark = _secret_mark(tag, rules) + return m.group(0) if mark is None else f"<{tag}{attrs}>{mark}" - return re.sub(r"<([A-Za-z_][\w.-]*)>([^<]*)", repl, value) + # Element text first: once an attribute holds a placeholder the start tag + # no longer matches _XML_ELEMENT (that is what keeps a re-run idempotent). + value = _XML_ELEMENT.sub(elem_repl, value) + return _XML_START_TAG.sub(tag_repl, value) def _scrub_str(value: str, rules: Rules) -> str: @@ -107,6 +128,17 @@ def _scrub_str(value: str, rules: Rules) -> str: return rules.timestamp.sub(TIMESTAMP, value) +def _mask_leaves(node: Any, mark: str, keep: tuple[str, ...] = ()) -> Any: + # A secret-class key masks EVERY scalar beneath it, however it is nested. + if node is None or isinstance(node, bool): + return node + if isinstance(node, dict): + return {k: _mask_leaves(v, mark, keep) for k, v in node.items()} + if isinstance(node, list): + return [_mask_leaves(item, mark, keep) for item in node] + return node if node in keep else mark + + def _walk(node: Any, rules: Rules) -> Any: if isinstance(node, dict): out: dict[str, Any] = {} @@ -114,13 +146,13 @@ def _walk(node: Any, rules: Rules) -> Any: if value is None or isinstance(value, bool): out[key] = value elif _key_hit(key, rules.password_keys): - out[key] = value if value in (REDACTED, REDACTED_INVALID) else REDACTED + out[key] = _mask_leaves(value, REDACTED, (REDACTED, REDACTED_INVALID)) elif _key_hit(key, rules.cookie_keys) and isinstance(value, str): out[key] = _scrub_str(_scrub_cookie(value), rules) - elif _key_hit(key, rules.token_keys) and not isinstance(value, (dict, list)): - out[key] = TOKEN - elif _key_hit(key, rules.serial_keys) and not isinstance(value, (dict, list)): - out[key] = SERIAL + elif _key_hit(key, rules.token_keys): + out[key] = _mask_leaves(value, TOKEN) + elif _key_hit(key, rules.serial_keys): + out[key] = _mask_leaves(value, SERIAL) else: out[key] = _walk(value, rules) return out From e4d8d06ecddac220f9cad38255828e1f3e8522d2 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:04:16 +0000 Subject: [PATCH 196/205] fix(nas): derive pool budget server-side; refuse client capacity inputs (B1) With `pool` set, /nas/archive-plan and share-push now source available_tapes and tape_capacities from library inventory (InventoryService) + catalog via nas.capacity.server_budget_request. Body-supplied available_tapes / tape_capacities / reserved_bytes / pool_reserved_bytes -> 422; unknown pool -> 404. The planner's pool budget check keys on `pool`, not on a non-empty tape list, so an empty server tape set blocks instead of skipping. Replaces routes_upload._available_data_tapes (which used library.inventory() directly and silently dropped all catalog barcodes via a dead dict branch). Mutation-checked: disabling the field refusal fails the 4 x 422 tests; disabling the pool lookup fails the 404 test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 19 +++++++ openblade/api/routes_upload.py | 28 ++-------- openblade/nas/capacity.py | 70 ++++++++++++++++++++++++ openblade/nas/planner.py | 3 +- tests/unit/test_capacity_budget_api.py | 61 +++++++++++++++++++++ tests/unit/test_capacity_reservations.py | 32 ++++++++--- 6 files changed, 183 insertions(+), 30 deletions(-) create mode 100644 tests/unit/test_capacity_budget_api.py diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index 9c0d36f..f8af80e 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -21,9 +21,12 @@ from openblade.domain.models import MountHandle, MountMode from openblade.domain.wire import coerce_int from openblade.nas.capacity import ( + ClientBudgetFieldsError, + PoolNotFoundError, ReservationError, ScratchThresholds, plan_with_pool_reservation, + server_budget_request, ) from openblade.nas.catalog_rebuild import CatalogRebuildPlanner from openblade.nas.catalog_rebuild_worker import SAFE_REBUILD_PREFLIGHT_ERROR, CatalogRebuildWorker @@ -1161,6 +1164,14 @@ async def archive_plan( request: ArchivePlanRequest, service: NasService = Depends(get_nas_service), ) -> ArchivePlan: + """Plan an archive. + + With ``pool`` set, tapes and capacities are derived server-side from the library + inventory + catalog: body-supplied ``available_tapes``/``tape_capacities``/ + ``reserved_bytes``/``pool_reserved_bytes`` are refused (422) and an unknown pool + is 404. Without ``pool`` the body-supplied inputs drive a preview-only plan that + gets no pool budget and books nothing. + """ policy = None if request.policy_id is not None: policy = service.get_policy(request.policy_id) @@ -1190,6 +1201,14 @@ async def archive_plan( updates["max_parallelism"] = policy.max_parallelism request = request.model_copy(update=updates) + try: + request = server_budget_request(service.repository, request, get_context().library) + except ClientBudgetFieldsError as exc: + # Curated domain message naming only the refused field names. + raise HTTPException(status_code=422, detail=str(exc)) from exc + except PoolNotFoundError as exc: + # Curated domain message (pool id only). + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc try: plan = plan_with_pool_reservation( service.repository, request, now=datetime.now(timezone.utc) diff --git a/openblade/api/routes_upload.py b/openblade/api/routes_upload.py index 453bd9d..47a4a78 100644 --- a/openblade/api/routes_upload.py +++ b/openblade/api/routes_upload.py @@ -53,7 +53,11 @@ def _safe_resolve(base: Path, file_id: str) -> Path: from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.wire import coerce_int -from openblade.nas.capacity import ReservationError, plan_with_pool_reservation +from openblade.nas.capacity import ( + ReservationError, + plan_with_pool_reservation, + server_budget_request, +) from openblade.nas.ingest import ( get_ingest_job, register_archive_plan, @@ -202,6 +206,7 @@ def _resolve_pool_for_share( def _plan_share_push(repo: CatalogRepository, request: ArchivePlanRequest) -> ArchivePlan: """Plan a share push against the pool budget and book it, like ``/nas/archive-plan``.""" + request = server_budget_request(repo, request, get_context().library) try: plan = plan_with_pool_reservation(repo, request, now=datetime.now(timezone.utc)) except ReservationError as exc: @@ -215,25 +220,6 @@ def _plan_share_push(repo: CatalogRepository, request: ArchivePlanRequest) -> Ar return plan -def _available_data_tapes(repo: CatalogRepository) -> list[str]: - barcodes: set[str] = set() - # The physical library holds the writable media (scratch/loaded tapes); the - # catalog only tracks already-archived cartridges, so it can be empty even - # when tapes are available. Source from both. - try: - for slot in get_context().library.inventory().slots: - if slot.barcode is not None: - barcodes.add(str(slot.barcode)) - except Exception: # noqa: BLE001 - library unavailable falls back to the catalog - pass - for cartridge in repo.list_cartridges(): - barcode = cartridge.get("barcode") if isinstance(cartridge, dict) else None - value = str(barcode or "").strip() - if value: - barcodes.add(value) - return sorted(value for value in barcodes if not value.upper().startswith("CLN")) - - def _safe_workspace_path(workspace: Path, relative_path: str) -> Path: target = (workspace / relative_path).resolve() if not str(target).startswith(str(workspace.resolve())): @@ -604,7 +590,6 @@ async def push_staged_files_to_share( file_sizes[candidate] = max(int(destination_path.stat().st_size), 0) staged_file_ids.append(file_id) - available_tapes = _available_data_tapes(repo) request_data = { "policy_id": policy.id, "policy_type": policy.policy_type, @@ -613,7 +598,6 @@ async def push_staged_files_to_share( "pool": pool_id, "files": relative_files, "file_sizes": file_sizes, - "available_tapes": available_tapes, "copies": max(int(policy.copies_required), 1), "verify_before_archive": bool(policy.verify_before_archive), "verify_after_archive": bool(policy.verify_after_archive), diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index fc9ffc4..5f08fda 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -7,6 +7,7 @@ from __future__ import annotations +import logging from collections.abc import Iterable, Sequence from datetime import datetime, timedelta from enum import Enum @@ -14,11 +15,13 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from openblade.jobs.inventory import InventoryService from openblade.nas.planner import ArchivePlanner from openblade.nas.types import ArchivePlan, ArchivePlanRequest, NasFileState if TYPE_CHECKING: from openblade.catalog.repository import CatalogRepository + from openblade.domain.backends import LibraryBackend # --- 2. scratch thresholds ------------------------------------------------- @@ -79,6 +82,9 @@ def needs_consolidation(self, threshold: float = DEFAULT_CONSOLIDATION_RATIO) -> # --- 7. capacity reservations --------------------------------------------- +logger = logging.getLogger(__name__) + + class ReservationError(ValueError): """A reservation cannot be satisfied from the pool's free capacity.""" @@ -88,6 +94,70 @@ class ReservationError(ValueError): PLAN_RESERVATION_TTL = timedelta(hours=1) +class PoolNotFoundError(LookupError): + """A pool-budgeted plan names a pool id the catalog does not know.""" + + +class ClientBudgetFieldsError(ValueError): + """A pool-budgeted request carried capacity inputs only the server may supply.""" + + +# With ``pool`` set these come from the library + catalog; a client value is refused +# (never silently overwritten) so an inflated budget cannot be smuggled in. +SERVER_BUDGET_FIELDS = ( + "available_tapes", + "tape_capacities", + "reserved_bytes", + "pool_reserved_bytes", +) + + +def server_data_tapes( + repository: CatalogRepository, library: LibraryBackend | None +) -> dict[str, int]: + """Writable data tapes (barcode -> usable bytes) from library inventory + catalog. + + The library holds blank/scratch media the catalog has never seen, so slots count + at the planner's default capacity; catalogued cartridges use their recorded + ``capacity_bytes - used_bytes``. Cleaning cartridges are excluded. + """ + capacities: dict[str, int] = {} + if library is not None: + try: + for slot in InventoryService(library).snapshot().slots: + if slot.barcode is not None: + capacities[str(slot.barcode)] = ArchivePlanner.DEFAULT_TAPE_CAPACITY + except Exception as exc: # noqa: BLE001 - library offline: fall back to the catalog alone + logger.warning("library inventory unavailable for pool budget", exc_info=exc) + for cartridge in repository.list_cartridges(): + barcode = str(cartridge.barcode or "").strip() + if barcode: + capacities[barcode] = max(0, int(cartridge.capacity_bytes) - int(cartridge.used_bytes)) + return {b: capacities[b] for b in sorted(capacities) if not b.upper().startswith("CLN")} + + +def server_budget_request( + repository: CatalogRepository, request: ArchivePlanRequest, library: LibraryBackend | None +) -> ArchivePlanRequest: + """Replace a pool request's capacity inputs with server-derived ones. + + Pool-less requests are returned unchanged (they get no pool budget). Raises + ``ClientBudgetFieldsError`` if the body supplied any ``SERVER_BUDGET_FIELDS`` and + ``PoolNotFoundError`` for an unknown pool id. + """ + if request.pool is None: + return request + supplied = [name for name in SERVER_BUDGET_FIELDS if name in request.model_fields_set] + if supplied: + raise ClientBudgetFieldsError( + f"{', '.join(supplied)} cannot be supplied when pool is set; the server derives them" + ) + if repository.get_nas_pool(request.pool) is None: + raise PoolNotFoundError(f"Pool {request.pool} not found") + tapes = server_data_tapes(repository, library) + return request.model_copy(update={"available_tapes": list(tapes), "tape_capacities": tapes}) + + def plan_with_pool_reservation( repository: CatalogRepository, request: ArchivePlanRequest, diff --git a/openblade/nas/planner.py b/openblade/nas/planner.py index 79ed7f8..e2d3906 100644 --- a/openblade/nas/planner.py +++ b/openblade/nas/planner.py @@ -356,8 +356,9 @@ def _apply_common_checks(self, request: ArchivePlanRequest, plan: ArchivePlan) - def _check_pool_budget(self, request: ArchivePlanRequest, plan: ArchivePlan) -> None: """Refuse a plan whose placed bytes exceed capacity left after pool reservations.""" - if not request.available_tapes: + if request.pool is None: return + # An empty server-derived tape set yields budget 0 -> blocked, never skipped. budget = self.pool_capacity_bytes(request) - request.pool_reserved_bytes placed = plan.total_bytes * plan.copies_required if placed > budget: diff --git a/tests/unit/test_capacity_budget_api.py b/tests/unit/test_capacity_budget_api.py new file mode 100644 index 0000000..b7edf2f --- /dev/null +++ b/tests/unit/test_capacity_budget_api.py @@ -0,0 +1,61 @@ +"""API-level guards: a pool-budgeted plan's capacity inputs come from the server only.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from openblade.api.main import app +from openblade.bootstrap import create_context, reset_context +from openblade.config import OpenBladeConfig +from openblade.nas.planner import POOL_BUDGET_BLOCKER + +client = TestClient(app) + + +@pytest.fixture(autouse=True) +def _context(tmp_path: Path) -> None: + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'b.db'}"))) + assert client.post("/nas/pools", json={"id": "pool-1", "name": "P1"}).status_code in {200, 201} + + +def _body(size: int, **extra: object) -> dict[str, object]: + return {"files": ["a.bin"], "file_sizes": {"a.bin": size}, "pool": "pool-1", **extra} + + +@pytest.mark.parametrize( + "field", + [ + {"tape_capacities": {"T1": 10**18}}, + {"available_tapes": ["T1"]}, + {"reserved_bytes": {"T1": 0}}, + {"pool_reserved_bytes": 0}, + ], +) +def test_client_capacity_inputs_with_pool_are_refused(field: dict[str, object]) -> None: + response = client.post("/nas/archive-plan", json=_body(1, **field)) + assert response.status_code == 422 + assert "server derives" in response.json()["detail"] + + +def test_unknown_pool_is_404() -> None: + response = client.post("/nas/archive-plan", json={**_body(1), "pool": "nope"}) + assert response.status_code == 404 + + +def test_honest_over_budget_is_blocked_by_server_capacity() -> None: + plan = client.post("/nas/archive-plan", json=_body(10**18)).json() + assert POOL_BUDGET_BLOCKER in plan["enqueue_blockers"] + + +def test_poolless_request_keeps_body_inputs_and_no_budget() -> None: + body = { + "files": ["a.bin"], + "file_sizes": {"a.bin": 10}, + "available_tapes": ["T1"], + "tape_capacities": {"T1": 100}, + } + plan = client.post("/nas/archive-plan", json=body).json() + assert POOL_BUDGET_BLOCKER not in plan["enqueue_blockers"] diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 82305d5..4c3268c 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -20,7 +20,7 @@ def _request( - pool_reserved_bytes: int, *, copies: int = 1, pool: str | None = None + pool_reserved_bytes: int, *, copies: int = 1, pool: str | None = "pool-1" ) -> ArchivePlanRequest: return ArchivePlanRequest( pool=pool, @@ -33,6 +33,13 @@ def _request( ) +def _server_request() -> ArchivePlanRequest: + """A pool request as the share-push route builds it: no client capacity inputs.""" + return ArchivePlanRequest( + files=["a.bin", "b.bin"], file_sizes={"a.bin": 300, "b.bin": 300}, pool="pool-1" + ) + + def test_plan_fits_exactly_within_pool_budget() -> None: # 2 tapes x 1000 = 2000; 600 placed -> exactly 1400 may be reserved by others. plan = ArchivePlanner().plan(_request(1_400)) @@ -184,26 +191,37 @@ def test_upload_share_push_plan_refused_when_pool_budget_consumed( from fastapi import HTTPException from openblade.api.routes_upload import _plan_share_push + from openblade.bootstrap import get_context + from openblade.nas.capacity import server_data_tapes - # Arrange: another job holds 1500 of the pool's 2000 bytes; the push needs 600. + # Arrange: another job holds all but 100 bytes of the SERVER-derived capacity. + repo.upsert_nas_pool({"id": "pool-1", "name": "P1"}) + capacity = sum(server_data_tapes(repo, get_context().library).values()) + now = datetime.now(UTC) assert repo.reserve_nas_capacity( - "pool-1", 1_500, "other-job", NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW + "pool-1", + capacity - 100, + "other-job", + now + timedelta(hours=1), + capacity_bytes=capacity, + now=now, ) - # Act + # Act: a body-free push (the route never sends capacity inputs) needing 600 bytes. with pytest.raises(HTTPException) as exc_info: - _plan_share_push(repo, _request(0, pool="pool-1")) + _plan_share_push(repo, _server_request()) # Assert: refused on the budget, and nothing new was booked. assert exc_info.value.status_code == 400 assert POOL_BUDGET_BLOCKER in str(exc_info.value.detail) - assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)] == ["other-job"] + assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=now)] == ["other-job"] def test_upload_share_push_plan_books_its_reservation(repo: CatalogRepository) -> None: from openblade.api.routes_upload import _plan_share_push - plan = _plan_share_push(repo, _request(0, pool="pool-1")) + repo.upsert_nas_pool({"id": "pool-1", "name": "P1"}) + plan = _plan_share_push(repo, _server_request()) assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=datetime.now(UTC))} == { plan.plan_id From 6f3a44ed129164d99658624a9dfdb27d3b2f3672 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:07:08 +0000 Subject: [PATCH 197/205] fix(aml): MFA-pending login never changes status or body, strict mode too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "MFA verification required" was invented text on an undocumented surface. Strict mode now answers 412 only for the documented default-password condition, else 200; MFA is signalled solely by Warning: 199 - "MFA Required". Tests: strict+default-password keeps the documented 412 summary; strict without it is 200 with only the MFA warning (fails if MFA re-enters the 412 condition — mutation-checked). Compat note and contract-notes row updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../aml/auth-login-mfa-pending.json | 2 +- docs/reference/i3-contract-notes.md | 2 +- openblade/api/routes_aml_auth.py | 10 +++++----- tests/integration/test_aml_auth_fidelity.py | 19 +++++++++++++++++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json index f1a7b7a..72e8fc2 100644 --- a/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json +++ b/compatibility/scalar-i3-341g/aml/auth-login-mfa-pending.json @@ -5,7 +5,7 @@ "firmware": "341G", "manual_ref": "Not in Rev D manual (inferred)", "setup": { "mfa": { "user": "admin", "type": "totp" } }, - "fidelity_notes": "Login by a user with an enabled MFA record returns the documented login body plus Warning: 199 - \"MFA Required\" (412 in strict mode); the session is pending (401 everywhere except POST /users/login/mfa, GET /users/login/mfa/{type}/key and logout) until a valid code is posted. The challenge wire shape is not documented in the Rev D manual; certify by appliance capture.", + "fidelity_notes": "Login by a user with an enabled MFA record returns the documented login body plus Warning: 199 - \"MFA Required\"; MFA never changes the status or body, in strict mode too (strict 412 only for the documented default-password condition); the session is pending (401 everywhere except POST /users/login/mfa, GET /users/login/mfa/{type}/key and logout) until a valid code is posted. The challenge wire shape is not documented in the Rev D manual; certify by appliance capture.", "request": { "method": "POST", "path": "/aml/users/login", "json": { "name": "admin", "password": "password" } }, "expected": { "status": 200, "headers": { "Warning": "MFA Required" } } } diff --git a/docs/reference/i3-contract-notes.md b/docs/reference/i3-contract-notes.md index fbc653c..eab43e8 100644 --- a/docs/reference/i3-contract-notes.md +++ b/docs/reference/i3-contract-notes.md @@ -75,4 +75,4 @@ inferred (OpenBlade-chosen) and unverified against an appliance. | Unsupported Accept on login/logout: 406 "Unsupported Accept media type" | inferred | | Login rate limit 429 carries `Retry-After` (seconds) | inferred | | Expired/unknown `sessionID` cookie: 401 and `Set-Cookie` clearing it (Max-Age=0); expired Bearer: 401 | inferred | -| MFA pending (inferred): a login by a user with an enabled MFA record keeps the documented body and adds `Warning: 199 - "MFA Required"`; with `OPENBLADE_AML_STRICT_PRECONDITIONS=true` it is 412 `"summary":"MFA verification required"` (session still issued). Until a valid code is posted to `POST /aml/users/login/mfa`, the session gets the ordinary 401 (cookie kept) on every route except that one, `GET /aml/users/login/mfa/{type}/key` and logout; an invalid code leaves it pending. Challenge wire shape not documented in Rev D manual; certify by appliance capture | inferred | +| MFA pending (inferred): a login by a user with an enabled MFA record keeps the documented body and adds `Warning: 199 - "MFA Required"`; MFA never changes the status code or body, with `OPENBLADE_AML_STRICT_PRECONDITIONS=true` too: strict mode answers 412 only for the documented default-password condition (`"Default password must be changed"`), else 200; the Warning header is the only MFA signal (session still issued). Until a valid code is posted to `POST /aml/users/login/mfa`, the session gets the ordinary 401 (cookie kept) on every route except that one, `GET /aml/users/login/mfa/{type}/key` and logout; an invalid code leaves it pending. Challenge wire shape not documented in Rev D manual; certify by appliance capture | inferred | diff --git a/openblade/api/routes_aml_auth.py b/openblade/api/routes_aml_auth.py index 8354473..056f516 100644 --- a/openblade/api/routes_aml_auth.py +++ b/openblade/api/routes_aml_auth.py @@ -637,12 +637,12 @@ async def login(request: Request, context: AppContext = Depends(get_context)) -> # 200; the session is still issued so the client can change the password. strict = os.environ.get("OPENBLADE_AML_STRICT_PRECONDITIONS", "false").lower() == "true" # Inferred likewise for MFA: the challenge wire shape is not in Rev D, so a - # pending login is signalled by a Warning header (412 when strict). MFA wins - # the summary because the pending session cannot change a password anyway. - precondition_failed = strict and (user.require_password_change or mfa_pending) + # pending login is signalled ONLY by the Warning header below. MFA never + # changes the status or body, in strict mode too: there is no documented + # MFA summary text to return. + precondition_failed = strict and user.require_password_change if precondition_failed: - summary = "MFA verification required" if mfa_pending else "Default password must be changed" - body.update(code=412, description="Error", summary=summary) + body.update(code=412, description="Error", summary="Default password must be changed") response = _negotiated(request, 412 if precondition_failed else 200, body) _is_production = os.environ.get("OPENBLADE_ENV", "development").lower() == "production" response.set_cookie( diff --git a/tests/integration/test_aml_auth_fidelity.py b/tests/integration/test_aml_auth_fidelity.py index 4a9dde1..b7af767 100644 --- a/tests/integration/test_aml_auth_fidelity.py +++ b/tests/integration/test_aml_auth_fidelity.py @@ -343,19 +343,34 @@ def test_mfa_login_lenient_is_200_with_warning_and_pending( assert aml_state._STATE.sessions[response.json()["token"]].mfa_pending is True -def test_mfa_login_strict_is_412_and_session_issued( +def test_mfa_login_strict_default_password_keeps_documented_412( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: + # MFA never invents a status or summary: the 412 is the default-password one. _enable_totp(client) monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") response = client.post("/aml/users/login", json=CREDS) assert response.status_code == 412 body = response.json() - assert body["code"] == 412 and body["summary"] == "MFA verification required" + assert body["code"] == 412 and body["summary"] == "Default password must be changed" assert MFA_WARNING in response.headers.get_list("Warning") assert aml_state._STATE.sessions[body["token"]].mfa_pending is True +def test_mfa_login_strict_without_other_precondition_is_200_with_warning( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _enable_totp(client) + aml_state.update_user(CREDS["name"], password=CREDS["password"]) # clears default-pw flag + monkeypatch.setenv("OPENBLADE_AML_STRICT_PRECONDITIONS", "true") + response = client.post("/aml/users/login", json=CREDS) + assert response.status_code == 200 + body = response.json() + assert body["code"] == 0 and body["summary"] == "Login successful" + assert response.headers.get_list("Warning") == [MFA_WARNING] + assert aml_state._STATE.sessions[body["token"]].mfa_pending is True + + def test_login_without_enabled_mfa_is_not_pending(client: TestClient) -> None: token = _login(client) assert aml_state._STATE.sessions[token].mfa_pending is False From 2f77cc2fdf4992561d297d59937d957bba5d11ee Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:08:26 +0000 Subject: [PATCH 198/205] build(nas): pin nfs-ganesha 5.9 in the Fedora stage Verified on the pinned digest: docker run --rm fedora@sha256:f1a3fab4... dnf list --available nfs-ganesha nfs-ganesha-vfs nfs-ganesha.x86_64 5.9-4.fc41 fedora nfs-ganesha-vfs.x86_64 5.9-4.fc41 fedora Globs are quoted so the RUN shell cannot expand them. docker build --target nfs-ganesha deploy/nas-protocols succeeds. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- deploy/nas-protocols/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/nas-protocols/Dockerfile b/deploy/nas-protocols/Dockerfile index 6fa7400..4df915f 100644 --- a/deploy/nas-protocols/Dockerfile +++ b/deploy/nas-protocols/Dockerfile @@ -3,7 +3,7 @@ # Fedora 41's nfs-ganesha 5.9 links libacl into libfsalvfs.so and exports # fsal_acl_2_posix_acl. fedora:41 pinned by digest (docker inspect RepoDigests). FROM fedora@sha256:f1a3fab47bcb3c3ddf3135d5ee7ba8b7b25f2e809a47440936212a3a50957f3d AS nfs-ganesha -RUN dnf -y install --setopt=install_weak_deps=False nfs-ganesha nfs-ganesha-vfs rpcbind \ +RUN dnf -y install --setopt=install_weak_deps=False "nfs-ganesha-5.9*" "nfs-ganesha-vfs-5.9*" rpcbind \ && dnf clean all COPY hydrator_shim.py /usr/local/bin/hydrator_shim.py From 201ee40bdfc3fca0e6f6131a1853f8e0d439c9e0 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:11:05 +0000 Subject: [PATCH 199/205] fix(nas): book pool capacity at ingest start, not at plan preview (S2) plan_with_pool_reservation -> plan_with_pool_budget (budget only). New book_plan_capacity books total*copies under owner=plan_id with a server-derived capacity ceiling; /nas/ingest/start and share-push call it before start_ingest_job and map ReservationError to 409. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/api/nas_config.py | 18 ++++---- openblade/api/routes_upload.py | 16 ++++--- openblade/nas/capacity.py | 39 ++++++++++------ tests/unit/test_capacity_budget_api.py | 36 +++++++++++++++ tests/unit/test_capacity_reservations.py | 57 +++++++++++++++--------- 5 files changed, 116 insertions(+), 50 deletions(-) diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index f8af80e..9d2bc7b 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -25,7 +25,8 @@ PoolNotFoundError, ReservationError, ScratchThresholds, - plan_with_pool_reservation, + book_plan_capacity, + plan_with_pool_budget, server_budget_request, ) from openblade.nas.catalog_rebuild import CatalogRebuildPlanner @@ -1209,13 +1210,7 @@ async def archive_plan( except PoolNotFoundError as exc: # Curated domain message (pool id only). raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc - try: - plan = plan_with_pool_reservation( - service.repository, request, now=datetime.now(timezone.utc) - ) - except ReservationError as exc: - # Curated domain message (pool id only); two plans raced for the same bytes. - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + plan = plan_with_pool_budget(service.repository, request, now=datetime.now(timezone.utc)) if policy is not None: plan.policy_name = policy.name plan.policy_type = policy.policy_type @@ -1255,6 +1250,13 @@ async def start_ingest( policy = service.get_policy(plan.policy_name) if policy is not None and not policy.auto_clean_before_archive: auto_clean_drives = False + try: + book_plan_capacity( + service.repository, plan, context.library, now=datetime.now(timezone.utc) + ) + except ReservationError as exc: + # Curated domain message (pool + plan id only); the pool budget is exhausted. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc if auto_clean_drives: cleaned_drives = _auto_clean_required_drives() job = start_ingest_job( diff --git a/openblade/api/routes_upload.py b/openblade/api/routes_upload.py index 47a4a78..27bd8e9 100644 --- a/openblade/api/routes_upload.py +++ b/openblade/api/routes_upload.py @@ -55,7 +55,8 @@ def _safe_resolve(base: Path, file_id: str) -> Path: from openblade.domain.wire import coerce_int from openblade.nas.capacity import ( ReservationError, - plan_with_pool_reservation, + book_plan_capacity, + plan_with_pool_budget, server_budget_request, ) from openblade.nas.ingest import ( @@ -205,13 +206,9 @@ def _resolve_pool_for_share( def _plan_share_push(repo: CatalogRepository, request: ArchivePlanRequest) -> ArchivePlan: - """Plan a share push against the pool budget and book it, like ``/nas/archive-plan``.""" + """Plan a share push against the server-derived pool budget (booked at ingest start).""" request = server_budget_request(repo, request, get_context().library) - try: - plan = plan_with_pool_reservation(repo, request, now=datetime.now(timezone.utc)) - except ReservationError as exc: - # Curated domain message (pool id only); two plans raced for the same bytes. - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + plan = plan_with_pool_budget(repo, request, now=datetime.now(timezone.utc)) if not plan.is_safe_to_enqueue: raise HTTPException( status_code=400, @@ -616,6 +613,11 @@ async def push_staged_files_to_share( dataset_name = request.dataset_name or ( f"share-push-{pool_id}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}" ) + try: + book_plan_capacity(repo, plan, get_context().library, now=datetime.now(timezone.utc)) + except ReservationError as exc: + # Curated domain message (pool + plan id only); the pool budget is exhausted. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc job = start_ingest_job( plan=plan, dataset_name=dataset_name, diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index 5f08fda..7d3a89b 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -158,20 +158,19 @@ def server_budget_request( return request.model_copy(update={"available_tapes": list(tapes), "tape_capacities": tapes}) -def plan_with_pool_reservation( +def plan_with_pool_budget( repository: CatalogRepository, request: ArchivePlanRequest, *, now: datetime, planner: ArchivePlanner | None = None, ) -> ArchivePlan: - """Plan against the pool's reservation budget, then book the plan's own bytes. + """Plan against the pool's reservation budget without booking anything. Unexpired ``NasReservation`` rows for ``request.pool`` are the source of truth: - their sum becomes ``pool_reserved_bytes``. A safe plan books - ``total_bytes * copies`` under ``owner=plan_id``; if the atomic booking loses - a race, ``ReservationError`` is raised. Requests without a pool are planned - unchanged. + their sum becomes ``pool_reserved_bytes``. Previews never consume budget; the + plan's bytes are booked only when ingest starts (``book_plan_capacity``). + Requests without a pool are planned unchanged. """ planner = planner or ArchivePlanner() pool_id = request.pool @@ -180,21 +179,35 @@ def plan_with_pool_reservation( reserved = sum( int(cast(int, row["bytes"])) for row in repository.reservations_for_pool(pool_id, now=now) ) - request = request.model_copy(update={"pool_reserved_bytes": reserved}) - plan = planner.plan(request) - if not plan.is_safe_to_enqueue: - return plan + return planner.plan(request.model_copy(update={"pool_reserved_bytes": reserved})) + + +def book_plan_capacity( + repository: CatalogRepository, + plan: ArchivePlan, + library: LibraryBackend | None, + *, + now: datetime, +) -> None: + """Book a pool plan's bytes at ingest start (owner=plan_id, TTL 1h). + + The capacity ceiling is server-derived (``server_data_tapes``), never taken from + the stored request. Pool-less plans book nothing. Raises ``ReservationError`` + when the atomic booking is refused, so the caller must not create the job. + """ + pool_id = plan.pool + if pool_id is None: + return booked = repository.reserve_nas_capacity( pool_id, plan.total_bytes * plan.copies_required, plan.plan_id, now + PLAN_RESERVATION_TTL, - capacity_bytes=planner.pool_capacity_bytes(request), + capacity_bytes=sum(server_data_tapes(repository, library).values()), now=now, ) if not booked: - raise ReservationError(f"pool {pool_id} capacity was reserved by a concurrent plan") - return plan + raise ReservationError(f"pool {pool_id} cannot reserve capacity for plan {plan.plan_id}") class CapacityReservation(BaseModel): diff --git a/tests/unit/test_capacity_budget_api.py b/tests/unit/test_capacity_budget_api.py index b7edf2f..d644dc5 100644 --- a/tests/unit/test_capacity_budget_api.py +++ b/tests/unit/test_capacity_budget_api.py @@ -59,3 +59,39 @@ def test_poolless_request_keeps_body_inputs_and_no_budget() -> None: } plan = client.post("/nas/archive-plan", json=body).json() assert POOL_BUDGET_BLOCKER not in plan["enqueue_blockers"] + + +def test_preview_books_nothing_but_second_start_is_409(monkeypatch: pytest.MonkeyPatch) -> None: + from datetime import UTC, datetime, timedelta + + from openblade.bootstrap import get_context + from openblade.nas.capacity import server_data_tapes + + # Arrange: keep started jobs running (no release) and leave 1500 bytes of pool budget. + monkeypatch.setattr("openblade.api.nas_config.run_ingest_job", lambda *a, **k: None) + context = get_context() + repo = context.catalog + capacity = sum(server_data_tapes(repo, context.library).values()) + now = datetime.now(UTC) + assert repo.reserve_nas_capacity( + "pool-1", + capacity - 1_500, + "other-job", + now + timedelta(hours=1), + capacity_bytes=capacity, + now=now, + ) + + # Act: two previews of 1000 bytes each, then start both. + first = client.post("/nas/archive-plan", json=_body(1_000, ingest_mode="source_stream")).json() + second = client.post("/nas/archive-plan", json=_body(1_000, ingest_mode="source_stream")).json() + start = {"dataset_name": "ds", "pool_id": "pool-1", "auto_clean_drives": False} + first_start = client.post("/nas/ingest/start", json={**start, "plan_id": first["plan_id"]}) + second_start = client.post("/nas/ingest/start", json={**start, "plan_id": second["plan_id"]}) + + # Assert: previews never consumed budget; the second START lost to the first booking. + assert first["is_safe_to_enqueue"] and second["is_safe_to_enqueue"] + assert first_start.status_code == 200, first_start.text + assert second_start.status_code == 409 + owners = {r["owner"] for r in repo.reservations_for_pool("pool-1", now=now)} + assert owners == {"other-job", first["plan_id"]} diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 4c3268c..b418051 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -11,7 +11,8 @@ from openblade.nas.capacity import ( PLAN_RESERVATION_TTL, ReservationError, - plan_with_pool_reservation, + book_plan_capacity, + plan_with_pool_budget, ) from openblade.nas.planner import POOL_BUDGET_BLOCKER, ArchivePlanner from openblade.nas.types import ArchivePlanRequest @@ -77,34 +78,49 @@ def repo(tmp_path: Path) -> CatalogRepository: return make_nas_service(tmp_path).repository -def test_booked_plan_consumes_pool_so_second_plan_is_refused(repo: CatalogRepository) -> None: +@pytest.fixture +def two_tapes(monkeypatch: pytest.MonkeyPatch) -> None: + """Server-derived pool capacity of 2000 bytes for ``book_plan_capacity``.""" + monkeypatch.setattr( + "openblade.nas.capacity.server_data_tapes", + lambda repo, library: {"T00001L8": TAPE_CAPACITY, "T00002L8": TAPE_CAPACITY}, + ) + + +@pytest.mark.usefixtures("two_tapes") +def test_preview_books_nothing_but_second_start_is_refused(repo: CatalogRepository) -> None: # Arrange: pool of 2000 bytes; another job already holds 800. assert repo.reserve_nas_capacity( "pool-1", 800, "other-job", NOW + timedelta(hours=1), capacity_bytes=2_000, now=NOW ) - # Act: first plan places 600 and books it (1400 held); second places 601 > 600 left. - first = plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=NOW) + # Act: 600- and 601-byte previews both fit (nothing booked); only one START can book. + first = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW) second_request = _request(0, pool="pool-1").model_copy( update={"file_sizes": {"/a": 300, "/b": 301}} ) - second = plan_with_pool_reservation(repo, second_request, now=NOW) + second = plan_with_pool_budget(repo, second_request, now=NOW) + book_plan_capacity(repo, first, None, now=NOW) # Assert - assert first.is_safe_to_enqueue + assert first.is_safe_to_enqueue and second.is_safe_to_enqueue + with pytest.raises(ReservationError): + book_plan_capacity(repo, second, None, now=NOW) assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)} == { "other-job", first.plan_id, } - assert POOL_BUDGET_BLOCKER in second.enqueue_blockers +@pytest.mark.usefixtures("two_tapes") def test_booking_expires_after_ttl(repo: CatalogRepository) -> None: - plan_with_pool_reservation(repo, _request(0, pool="pool-1", copies=3), now=NOW) + first = plan_with_pool_budget(repo, _request(0, pool="pool-1", copies=3), now=NOW) + book_plan_capacity(repo, first, None, now=NOW) - later = plan_with_pool_reservation( + later = plan_with_pool_budget( repo, _request(0, pool="pool-1", copies=3), now=NOW + PLAN_RESERVATION_TTL ) + book_plan_capacity(repo, later, None, now=NOW + PLAN_RESERVATION_TTL) assert later.is_safe_to_enqueue @@ -114,24 +130,22 @@ def test_expired_reservation_is_ignored(repo: CatalogRepository) -> None: "pool-1", 2_000, "stale-job", NOW + timedelta(seconds=1), capacity_bytes=2_000, now=NOW ) - plan = plan_with_pool_reservation( - repo, _request(0, pool="pool-1"), now=NOW + timedelta(minutes=1) - ) + plan = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW + timedelta(minutes=1)) assert plan.is_safe_to_enqueue +@pytest.mark.usefixtures("two_tapes") def test_lost_booking_race_raises_typed_error(repo: CatalogRepository) -> None: # Simulate a concurrent writer landing between the budget read and the booking. - class RacingRepo: - def reservations_for_pool(self, pool_id: str, *, now: datetime) -> list[dict[str, object]]: - return [] + plan = plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=NOW) + class RacingRepo: def reserve_nas_capacity(self, *args: object, **kwargs: object) -> bool: return False with pytest.raises(ReservationError): - plan_with_pool_reservation(RacingRepo(), _request(0, pool="pool-1"), now=NOW) # type: ignore[arg-type] + book_plan_capacity(RacingRepo(), plan, None, now=NOW) # type: ignore[arg-type] def _seed_cartridges(repo: CatalogRepository, scratch: int) -> int: @@ -217,15 +231,14 @@ def test_upload_share_push_plan_refused_when_pool_budget_consumed( assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=now)] == ["other-job"] -def test_upload_share_push_plan_books_its_reservation(repo: CatalogRepository) -> None: +def test_upload_share_push_plan_books_nothing_until_start(repo: CatalogRepository) -> None: from openblade.api.routes_upload import _plan_share_push repo.upsert_nas_pool({"id": "pool-1", "name": "P1"}) plan = _plan_share_push(repo, _server_request()) - assert {r["owner"] for r in repo.reservations_for_pool("pool-1", now=datetime.now(UTC))} == { - plan.plan_id - } + assert plan.is_safe_to_enqueue + assert repo.reservations_for_pool("pool-1", now=datetime.now(UTC)) == [] def test_finished_job_releases_its_plan_reservation(tmp_path: Path) -> None: @@ -241,7 +254,7 @@ def test_finished_job_releases_its_plan_reservation(tmp_path: Path) -> None: "pool-1", 2_000, plan.plan_id, now + PLAN_RESERVATION_TTL, capacity_bytes=2_000, now=now ) assert POOL_BUDGET_BLOCKER in ( - plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=now).enqueue_blockers + plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=now).enqueue_blockers ) job = start_ingest_job( plan=plan, @@ -257,7 +270,7 @@ def test_finished_job_releases_its_plan_reservation(tmp_path: Path) -> None: # Assert: terminal job freed the booking, so the same plan now fits. assert finished.status.value in {"archived", "failed", "cancelled"} assert repo.reservations_for_pool("pool-1", now=now) == [] - assert plan_with_pool_reservation(repo, _request(0, pool="pool-1"), now=now).is_safe_to_enqueue + assert plan_with_pool_budget(repo, _request(0, pool="pool-1"), now=now).is_safe_to_enqueue def test_delete_reservations_for_owner_keeps_other_owners(repo: CatalogRepository) -> None: From 47e8e6903e97a7890b93113932be54e569f625c7 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:13:37 +0000 Subject: [PATCH 200/205] fix(nas): count scratch from library inventory; exclude VG members from pool budget (S1) count_scratch_cartridges counted catalog rows, but blank tapes are not rows. count_scratch_tapes counts inventory slots not in a volume group and not catalogued as written; server_data_tapes also skips VG members. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/catalog/repository.py | 9 ---- openblade/nas/capacity.py | 36 +++++++++++++-- openblade/nas/health_service.py | 11 +++-- tests/unit/test_capacity_reservations.py | 58 ++++++++++++++---------- 4 files changed, 75 insertions(+), 39 deletions(-) diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 99a5e1f..4e4008d 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -1531,15 +1531,6 @@ def upsert_nas_file_span(self, data: dict[str, object]) -> dict[str, object]: def delete_nas_file_span(self, span_id: str) -> bool: return self._delete_nas_row(NasFileSpan, span_id) - def count_scratch_cartridges(self) -> int: - """Cartridges with no volume group that are sitting in a slot (not loaded/exported).""" - stmt = ( - select(func.count()) - .select_from(Cartridge) - .where(Cartridge.volume_group_id.is_(None), Cartridge.state == "in_slot") - ) - return int(self.session.execute(stmt).scalar_one()) - def get_scratch_thresholds(self) -> dict[str, object] | None: return self.get_nas_config(self._SCRATCH_THRESHOLDS_KEY) diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index 7d3a89b..4580f55 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -119,7 +119,8 @@ def server_data_tapes( The library holds blank/scratch media the catalog has never seen, so slots count at the planner's default capacity; catalogued cartridges use their recorded - ``capacity_bytes - used_bytes``. Cleaning cartridges are excluded. + ``capacity_bytes - used_bytes``. Cleaning cartridges and volume-group members + (already assigned elsewhere) are excluded. """ capacities: dict[str, int] = {} if library is not None: @@ -129,11 +130,38 @@ def server_data_tapes( capacities[str(slot.barcode)] = ArchivePlanner.DEFAULT_TAPE_CAPACITY except Exception as exc: # noqa: BLE001 - library offline: fall back to the catalog alone logger.warning("library inventory unavailable for pool budget", exc_info=exc) + assigned: set[str] = set() for cartridge in repository.list_cartridges(): barcode = str(cartridge.barcode or "").strip() - if barcode: - capacities[barcode] = max(0, int(cartridge.capacity_bytes) - int(cartridge.used_bytes)) - return {b: capacities[b] for b in sorted(capacities) if not b.upper().startswith("CLN")} + if not barcode: + continue + if cartridge.volume_group_id is not None: + assigned.add(barcode) + capacities[barcode] = max(0, int(cartridge.capacity_bytes) - int(cartridge.used_bytes)) + return { + b: capacities[b] + for b in sorted(capacities) + if b not in assigned and not b.upper().startswith("CLN") + } + + +def count_scratch_tapes(repository: CatalogRepository, library: LibraryBackend) -> int: + """Blank tapes in library slots: not in any volume group, not catalogued as written. + + The catalog only holds catalogued/archived cartridges, so scratch stock must be + counted from the inventory snapshot. Cleaning cartridges are excluded. + """ + excluded = { + str(c.barcode).strip() + for c in repository.list_cartridges() + if c.barcode and (c.volume_group_id is not None or int(c.used_bytes) > 0) + } + barcodes = { + str(slot.barcode) + for slot in InventoryService(library).snapshot().slots + if slot.barcode is not None + } + return sum(1 for b in barcodes if b not in excluded and not b.upper().startswith("CLN")) def server_budget_request( diff --git a/openblade/nas/health_service.py b/openblade/nas/health_service.py index 99915b3..a6dd1d2 100644 --- a/openblade/nas/health_service.py +++ b/openblade/nas/health_service.py @@ -12,7 +12,12 @@ from openblade.catalog.models import Cartridge, NasDataset, NasFileRecord, PathMapping from openblade.catalog.repository import CatalogRepository from openblade.domain.clock import naive_utcnow -from openblade.nas.capacity import ScratchHealth, ScratchThresholds, evaluate_scratch +from openblade.nas.capacity import ( + ScratchHealth, + ScratchThresholds, + count_scratch_tapes, + evaluate_scratch, +) from openblade.nas.types import ( CatalogStatusResponse, ComponentHealth, @@ -73,8 +78,8 @@ def _check_scratch_component(self) -> ComponentHealth | None: if stored is None: return None thresholds = ScratchThresholds.model_validate(stored) - count = self.repo.count_scratch_cartridges() - except Exception: # noqa: BLE001 - health must not raise; DB failure is reported by the database component + count = count_scratch_tapes(self.repo, self.library) + except Exception: # noqa: BLE001 - health must not raise; DB/library failures are reported by their own components logger.warning("scratch health probe failed", exc_info=True) return None level = evaluate_scratch(count, thresholds) diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index b418051..5fc8acb 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -12,6 +12,7 @@ PLAN_RESERVATION_TTL, ReservationError, book_plan_capacity, + count_scratch_tapes, plan_with_pool_budget, ) from openblade.nas.planner import POOL_BUDGET_BLOCKER, ArchivePlanner @@ -148,30 +149,41 @@ def reserve_nas_capacity(self, *args: object, **kwargs: object) -> bool: book_plan_capacity(RacingRepo(), plan, None, now=NOW) # type: ignore[arg-type] -def _seed_cartridges(repo: CatalogRepository, scratch: int) -> int: - """Add cartridges on top of the simulator's seeded stock; return that baseline.""" - baseline = repo.count_scratch_cartridges() - from openblade.catalog.models import Cartridge - - for i in range(scratch): - repo.session.add(Cartridge(barcode=f"S{i:05d}L8", state="in_slot")) - repo.session.add(Cartridge(barcode="LOADED1L8", state="loaded")) - repo.session.add(Cartridge(barcode="ASSIGN1L8", state="in_slot", volume_group_id="vg-1")) - repo.session.commit() - return baseline - - -def test_count_scratch_excludes_loaded_and_assigned(repo: CatalogRepository) -> None: - baseline = _seed_cartridges(repo, scratch=2) +def _scratch_service(tmp_path: Path): # type: ignore[no-untyped-def] + from tests.unit.test_health_service import make_health_service - assert repo.count_scratch_cartridges() == baseline + 2 + service, _ = make_health_service(tmp_path, "scratch.db") + return service, count_scratch_tapes(service.repo, service.library) -def _scratch_component(tmp_path: Path, scratch: int, thresholds: tuple[int, int] | None): # type: ignore[no-untyped-def] - from tests.unit.test_health_service import make_health_service +def test_scratch_counts_blank_inventory_minus_assigned_and_written(tmp_path: Path) -> None: + from openblade.catalog.models import Cartridge + from openblade.jobs.inventory import InventoryService - service, _ = make_health_service(tmp_path, "scratch.db") - base = _seed_cartridges(service.repo, scratch) + # Arrange: N blank tapes from the simulator inventory; none are catalog rows yet. + service, blank = _scratch_service(tmp_path) + slots = InventoryService(service.library).snapshot().slots + barcodes = sorted( + str(s.barcode) for s in slots if s.barcode and not str(s.barcode).startswith("CLN") + ) + assert blank == len(barcodes) >= 2 + + # Act + Assert: a volume-group member is no longer scratch... + group = service.repo.create_volume_group("vg-scratch") + service.repo.add_barcode_to_volume_group(group.id, barcodes[0]) + assert count_scratch_tapes(service.repo, service.library) == blank - 1 + # ...nor is a tape catalogued as written. + written = service.repo.session.query(Cartridge).filter_by(barcode=barcodes[1]).one_or_none() + if written is None: + service.repo.session.add(Cartridge(barcode=barcodes[1], used_bytes=10)) + else: + written.used_bytes = 10 + service.repo.session.commit() + assert count_scratch_tapes(service.repo, service.library) == blank - 2 + + +def _scratch_component(tmp_path: Path, thresholds: tuple[int, int] | None): # type: ignore[no-untyped-def] + service, base = _scratch_service(tmp_path) if thresholds is not None: service.repo.set_scratch_thresholds(base + thresholds[0], base + thresholds[1]) result = service.check_health() @@ -179,12 +191,12 @@ def _scratch_component(tmp_path: Path, scratch: int, thresholds: tuple[int, int] def test_scratch_component_absent_when_thresholds_unset(tmp_path: Path) -> None: - _, component = _scratch_component(tmp_path, 0, None) + _, component = _scratch_component(tmp_path, None) assert component is None def test_scratch_warn_is_ok_with_message(tmp_path: Path) -> None: - _, component = _scratch_component(tmp_path, 3, (2, 5)) + _, component = _scratch_component(tmp_path, (-1, 2)) assert component is not None assert component.status.value == "ok" assert component.message.startswith("scratch count ") @@ -194,7 +206,7 @@ def test_scratch_warn_is_ok_with_message(tmp_path: Path) -> None: def test_scratch_critical_is_degraded_not_unhealthy(tmp_path: Path) -> None: from openblade.nas.types import HealthStatus - _, component = _scratch_component(tmp_path, 1, (2, 5)) + _, component = _scratch_component(tmp_path, (1, 3)) assert component is not None assert component.status is HealthStatus.DEGRADED From 3b4722a9acba0793f726f0c26c3445f92480d801 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:14:17 +0000 Subject: [PATCH 201/205] fix(nas): never let a failed reservation release mask the ingest error (S3) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/ingest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openblade/nas/ingest.py b/openblade/nas/ingest.py index 5bcd713..ac10e20 100644 --- a/openblade/nas/ingest.py +++ b/openblade/nas/ingest.py @@ -799,7 +799,12 @@ def run_ingest_job( # The plan's pool booking (owner=plan_id) is held only while the job can still # write; any terminal state frees it. A non-terminal exit keeps it until expiry. if job.status in _TERMINAL_JOB_STATES: - nas_service.repository.delete_nas_reservations_for_owner(job.plan.plan_id) + try: + nas_service.repository.delete_nas_reservations_for_owner(job.plan.plan_id) + except Exception as exc: # noqa: BLE001 - a failed release (e.g. session in PendingRollbackError) must not replace the executor's exception; the booking expires by TTL + logger.warning( + "reservation release failed for plan %s", job.plan.plan_id, exc_info=exc + ) _TERMINAL_JOB_STATES = frozenset( From ccf71076fcc767013502438cda33aef8a2520a29 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:17:42 +0000 Subject: [PATCH 202/205] fix(nas): release the plan booking when the ingest executor raises (S3 test) run() converts failures to FAILED, but if it raises anyway (e.g. _mark_dataset_failed on a session in PendingRollbackError) the job can still be ARCHIVING, so the terminal-status check skipped the release and the booking sat until TTL. An escaping exception now releases too. Tests: executor raises + release raises -> RuntimeError surfaces and the "reservation release failed" warning is logged; executor raises with a non-terminal status -> booking released. Mutation-checked: removing the try/except fails the first; dropping the raised-path fails both. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- openblade/nas/ingest.py | 12 ++- tests/unit/test_capacity_reservations.py | 106 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/openblade/nas/ingest.py b/openblade/nas/ingest.py index ac10e20..6227de9 100644 --- a/openblade/nas/ingest.py +++ b/openblade/nas/ingest.py @@ -793,12 +793,18 @@ def run_ingest_job( ltfs=ltfs, config=nas_service.get_source_stream_config(), ) + raised = True try: - return executor.run() + result = executor.run() + raised = False + return result finally: # The plan's pool booking (owner=plan_id) is held only while the job can still - # write; any terminal state frees it. A non-terminal exit keeps it until expiry. - if job.status in _TERMINAL_JOB_STATES: + # write; any terminal state frees it. run() normally converts failures into + # FAILED, but if it raises anyway (e.g. _mark_dataset_failed hit a broken + # session) the status can still be ARCHIVING and nothing will ever resume this + # job, so an escaping exception releases too. A non-terminal *return* keeps it. + if raised or job.status in _TERMINAL_JOB_STATES: try: nas_service.repository.delete_nas_reservations_for_owner(job.plan.plan_id) except Exception as exc: # noqa: BLE001 - a failed release (e.g. session in PendingRollbackError) must not replace the executor's exception; the booking expires by TTL diff --git a/tests/unit/test_capacity_reservations.py b/tests/unit/test_capacity_reservations.py index 5fc8acb..9ccd8bd 100644 --- a/tests/unit/test_capacity_reservations.py +++ b/tests/unit/test_capacity_reservations.py @@ -293,3 +293,109 @@ def test_delete_reservations_for_owner_keeps_other_owners(repo: CatalogRepositor assert repo.delete_nas_reservations_for_owner("plan-a") == 2 assert [r["owner"] for r in repo.reservations_for_pool("pool-1", now=NOW)] == ["plan-b"] + + +# --- S3: releasing the booking must never mask the executor's exception --------- + + +def _raising_ingest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def] + """A started cache-drive ingest whose executor raises past run()'s own handler.""" + from openblade.bootstrap import create_context, reset_context + from openblade.config import OpenBladeConfig + from openblade.nas import ingest as ingest_mod + from openblade.nas.service import NasService + from openblade.nas.types import ( + ArchivePlan, + CacheDriveConfig, + IngestMode, + NasPool, + TapeAssignment, + ) + + context = create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 's3.db'}")) + reset_context(context) + ingest_mod.clear_ingest_state() + service = NasService(context.catalog) + cache_root = tmp_path / "cache" + (cache_root / "dataset").mkdir(parents=True) + (cache_root / "dataset" / "a.txt").write_bytes(b"alpha") + service.upsert_pool(NasPool(id="pool-1", name="pool-1")) + service.upsert_cache_drive( + CacheDriveConfig( + id="cache-1", name="c", root_path=str(cache_root), max_bytes=10**6, min_free_bytes=0 + ) + ) + plan = ingest_mod.register_archive_plan( + ArchivePlan( + plan_id="plan-s3", + ingest_mode=IngestMode.CACHE_DRIVE, + source_path=str(cache_root / "dataset"), + pool="pool-1", + volume_group="vg-1", + files=[str(cache_root / "dataset" / "a.txt")], + total_files=1, + total_bytes=5, + tape_assignments=[ + TapeAssignment(barcode="VOL001L9", files=["a.txt"], estimated_bytes=5) + ], + ) + ) + job = ingest_mod.start_ingest_job( + plan=plan, + dataset_name="ds", + pool_id="pool-1", + nas_service=service, + cache_drive_id="cache-1", + ) + + def executor_boom(self: object) -> object: + raise RuntimeError("executor exploded") + + # Raises past run()'s boundary without reaching a terminal status, like + # _mark_dataset_failed failing on a session in PendingRollbackError. + monkeypatch.setattr(ingest_mod.CacheDriveIngest, "run", executor_boom) + + def run() -> object: + return ingest_mod.run_ingest_job( + job.job_id, + nas_service=service, + library=context.library, + ltfs=context.ltfs, + cache_drive_id="cache-1", + ) + + return service, run + + +def test_failed_release_does_not_mask_executor_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + service, run = _raising_ingest(tmp_path, monkeypatch) + + def release_boom(owner: str) -> int: + raise ValueError("session is in PendingRollbackError") + + monkeypatch.setattr(service.repository, "delete_nas_reservations_for_owner", release_boom) + + with ( + caplog.at_level("WARNING", logger="openblade.nas.ingest"), + pytest.raises(RuntimeError, match="executor exploded"), + ): + run() + assert "reservation release failed for plan plan-s3" in caplog.text + + +def test_executor_exception_releases_booking_even_if_status_not_terminal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, run = _raising_ingest(tmp_path, monkeypatch) + released: list[str] = [] + monkeypatch.setattr( + service.repository, + "delete_nas_reservations_for_owner", + lambda owner: released.append(owner), + ) + + with pytest.raises(RuntimeError, match="executor exploded"): + run() + assert released == ["plan-s3"] From 63761c83c9ce70425f7512ac1df70953dab1b4f1 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:18:34 +0000 Subject: [PATCH 203/205] docs(nas): record server-derived budget, booking at start, inventory scratch Capacity reservations decision: client capacity fields 422 / unknown pool 404 when pool is set, booking at ingest start (409, no job) not preview, release on terminal state or executor exception, scratch counted from the library inventory. Open risks: booking uses plan.pool not the start request's pool; share-push 409 leaves a registered plan with no job. tech-debt adjusted to match. Wiki API reference regenerated: unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- .../2026-10-09-persistent-drive-leases.md | 43 +++++++++++++------ docs/tech-debt.md | 12 ++++-- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 080e9ab..1273934 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -282,18 +282,37 @@ Decisions (WS-capacity, fleet round 3): `pool_reserved_bytes` (sum of unexpired `nas_reservations` for the pool) and adds the `POOL_BUDGET_BLOCKER` when placed bytes exceed total tape capacity minus that budget. `reserved_bytes` stays per barcode for explicit holds. -2. **A plan books its own reservation.** `capacity.plan_with_pool_reservation` books - `total_bytes * copies` under `owner=plan_id` for 1h through the atomic - `reserve_nas_capacity`; a lost race raises `ReservationError` (HTTP 409). Both - `/nas/archive-plan` and the upload share-push use it. `run_ingest_job` deletes the - owner's rows once the job is archived, failed or cancelled; expiry covers the rest. -3. **Scratch** = `volume_group_id IS NULL AND state = 'in_slot'` - (`count_scratch_cartridges`); the planner's barcode heuristic is unchanged. -4. **Health**: a `scratch` component only when thresholds are configured; WARN is OK +2. **The budget is server-derived.** When `pool` is set on `/nas/archive-plan` (and + always on the upload share-push), `capacity.server_budget_request` derives + `available_tapes` and `tape_capacities` from the library inventory + catalog + (`server_data_tapes`: `capacity_bytes - used_bytes`, excluding cleaning cartridges + and volume-group members). A body that also supplies `tape_capacities`, + `available_tapes` or `reserved_bytes` is rejected with 422 (never silently + overwritten); an unknown pool id is 404. An empty server-derived tape set blocks + the plan instead of skipping the budget check. Pool-less requests keep the + body-driven path but get no pool budget and no booking. +3. **Book at consumption, not at preview.** `capacity.plan_with_pool_budget` only + computes the budget; previews book nothing, so repeating a preview is never + refused. `book_plan_capacity` books `total_bytes * copies` under `owner=plan_id` + for 1h through the atomic `reserve_nas_capacity`, called from + `POST /nas/ingest/start` and the share-push before the job exists; a lost race or + exhausted budget raises `ReservationError` -> 409 and no job is created. +4. **Release.** `run_ingest_job` deletes the owner's rows when the job ends archived, + failed or cancelled, and also whenever the executor raises (the status may still + be non-terminal and nothing will resume the job). A failing release is logged + ("reservation release failed") and never replaces the executor's exception; the + row then expires by TTL. +5. **Scratch** is counted from the library inventory + (`InventoryService(library).snapshot()`, `count_scratch_tapes`): slots holding a + barcode that is not a volume-group member, not catalogued with `used_bytes > 0`, + and not a `CLN` cartridge. Blank tapes have no catalog row, so a catalog count + would report a library full of scratch as CRITICAL. +6. **Health**: a `scratch` component only when thresholds are configured; WARN is OK with a message, CRITICAL is DEGRADED, never UNHEALTHY. Thresholds are managed by `GET`/`PUT /nas/scratch-thresholds` (404 unset, 422 on `warn < min`). -Reviewer risks still open: `request.pool` is used as the pool id without checking it -names a configured pool; client-supplied `tape_capacities` set the capacity the -booking is checked against, so a caller can inflate it; and an abandoned plan holds -its bytes for the full hour. +Reviewer risks still open: `book_plan_capacity` books against `plan.pool`, not the +`pool_id` of the start request, so a plan previewed for one pool and started into +another is budgeted against the first; the share-push registers its plan before +booking, so a 409 there leaves a registered plan with no job; and a started job that +never reaches a terminal state without raising holds its bytes for the full hour. diff --git a/docs/tech-debt.md b/docs/tech-debt.md index f56a721..4dac5c8 100644 --- a/docs/tech-debt.md +++ b/docs/tech-debt.md @@ -87,10 +87,14 @@ and no production caller uses `claim_drive`/`claim_changer` yet. Resolved: unexpired `nas_reservations` rows are a pool-level budget (`ArchivePlanRequest.pool_reserved_bytes`) that every plan from `/nas/archive-plan` and the upload share-push must fit under; the per-barcode `reserved_bytes` dict -stays for explicit holds. A safe plan books its own bytes (`owner=plan_id`) with a -1h TTL, released when its ingest job reaches a terminal state and otherwise by -expiry (abandoned plans hold the pool for up to an hour). Scratch health counts -cartridges with `volume_group_id IS NULL AND state='in_slot'`; the `scratch` +stays for explicit holds. With `pool` set, the budget's tape set and capacities are +derived server-side (client-supplied capacity fields are 422, unknown pool 404). +Previews book nothing; ingest start books `owner=plan_id` with a 1h TTL (409 and no +job when the budget is exhausted), released when the job reaches a terminal state or +the executor raises, and otherwise by expiry. Open: the booking uses `plan.pool` +rather than the start request's pool, and a 409 on share-push leaves a registered +plan with no job. Scratch health counts inventory slots whose barcode is not a +volume-group member, not written (`used_bytes > 0`) and not `CLN`; the `scratch` component appears only once thresholds are set and degrades, never fails readiness. `nas_file_spans` has no `pool_id`, so identical relative paths in two pools share spans. The protocol rig's NFS server is Fedora 41 nfs-ganesha 5.9 (VFS FSAL, built From 7d82885eb8075521dd35d4c76093d064eeaeef23 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:30:19 +0000 Subject: [PATCH 204/205] test(nas): stop sending available_tapes with pool in the e2e archive test B1 makes /nas/archive-plan derive the tape set server-side when `pool` is set and reject client-supplied tape fields with 422; the end-to-end archive/recall test still posted `available_tapes`, so it failed 422. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- tests/integration/test_nas_i3_end_to_end_archive_recall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_nas_i3_end_to_end_archive_recall.py b/tests/integration/test_nas_i3_end_to_end_archive_recall.py index df07fd5..869ecae 100644 --- a/tests/integration/test_nas_i3_end_to_end_archive_recall.py +++ b/tests/integration/test_nas_i3_end_to_end_archive_recall.py @@ -207,7 +207,7 @@ def test_nas_i3_end_to_end_archive_recall_hash_verified(iteration: int, tmp_path "pool": "archive-pool", "files": relative_paths, "file_sizes": file_sizes, - "available_tapes": tape_candidates[:2], + # With `pool` set the server derives the tape set; sending it is a 422. "copies": 1, "verify_before_archive": True, "verify_after_archive": True, From 3841c599d8d3bcae7a1bcf7ecd90f5b2aed4eda9 Mon Sep 17 00:00:00 2001 From: Amantux Date: Sun, 11 Oct 2026 04:44:03 +0000 Subject: [PATCH 205/205] fix(demo): let the server derive the pool tape set in the MVP plan request With `pool` set, /nas/archive-plan now rejects client-supplied available_tapes/tape_capacities (422) so the budget cannot be inflated; the demo was still sending the tape list and failed on the first run of the gate. The unused `get_context` import that fed it is dropped too. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016uyKrkqBY5KxLuSjotaVGD --- scripts/mvp_e2e_demo.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/mvp_e2e_demo.py b/scripts/mvp_e2e_demo.py index 7c2dc61..20746d6 100644 --- a/scripts/mvp_e2e_demo.py +++ b/scripts/mvp_e2e_demo.py @@ -278,7 +278,7 @@ def policy_driven_nas_flow() -> None: from fastapi.testclient import TestClient from openblade.api.main import app - from openblade.bootstrap import create_context, get_context, reset_context + from openblade.bootstrap import create_context, reset_context from openblade.config import OpenBladeConfig from openblade.nas.ingest import clear_ingest_state @@ -347,11 +347,8 @@ def policy_driven_nas_flow() -> None: ) ok("policy + pool + cache-drive configured", "policy=nas-sharded -> pool=archive-pool") - tapes = sorted( - str(s.barcode) - for s in get_context().library.inventory().slots - if s.barcode is not None and not str(s.barcode).upper().startswith("CLN") - ) + # With ``pool`` set the server derives the tape set and capacities itself; + # client-supplied ``available_tapes`` is rejected (422) by design. plan = client.post( "/nas/archive-plan", json={ @@ -360,7 +357,6 @@ def policy_driven_nas_flow() -> None: "pool": "archive-pool", "files": relative_paths, "file_sizes": file_sizes, - "available_tapes": tapes[:3], "copies": 1, "max_parallelism": 2, },