diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98f4cd7..80d1ba6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,64 @@ jobs: - uses: docker://rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 with: args: -color + rust: + name: Rust quality gates (fmt, clippy, test) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1 + with: + toolchain: 1.98.1 + components: rustfmt, clippy + - run: cargo fmt --all -- --check + - run: RUSTFLAGS="-D warnings" cargo clippy --workspace --all-targets + - run: cargo test --workspace --locked + + component-package: + name: Component package (linux x86_64) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1 + with: + toolchain: 1.98.1 + - name: Build release component + run: cargo build --release --locked --target x86_64-unknown-linux-gnu --bin bitty-storage + - name: Pack target archive and checksums + run: | + scripts/package-component.sh \ + --version "$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)" \ + --target x86_64-unknown-linux-gnu \ + --binary target/x86_64-unknown-linux-gnu/release/bitty-storage \ + --output dist/x86_64-unknown-linux-gnu.tar.gz + - name: Verify package shape + run: scripts/test-package-component.sh + - name: Smoke-test the packed executable + run: | + set -euo pipefail + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)" + mkdir -p dist/smoke + tar -xzf dist/x86_64-unknown-linux-gnu.tar.gz -C dist/smoke + dist/smoke/bitty-storage --version | grep -Fx "bitty-storage $version" + dist/smoke/bitty-storage --check-manifest dist/smoke/bitty-component.toml + test -s dist/SHA256SUMS + (cd dist && sha256sum -c SHA256SUMS) + - name: Run the live protocol suite against the packed executable + env: + BITTY_STORAGE_COMPONENT_BIN: ${{ github.workspace }}/dist/smoke/bitty-storage + run: cargo test --test component_server --locked + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: storage-component-linux-x86_64 + path: | + dist/*.tar.gz + dist/*.sha256 + dist/SHA256SUMS + if-no-files-found: error + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index a3dad4d..c822c5c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,3 +3,5 @@ Metadata-only candidate; no product migration authorized. Read repo.toml, TODO, CarryCtx and shared security corpus. W-131 must separate transcript, command history, session snapshots and sandboxed plugin KV. No generic database, arbitrary filesystem authority or default raw stdout persistence. Terminal Truth remains volatile and Core-owned. Require opt-in capture, isolation, retention/deletion, secret minimization, recovery and bounded schemas. English only; no hardcoded host values. Rust starts 0.0.1, edition 2024, Core MSRV. Run just gates; metadata checks are not product evidence. CTX-0001 -> 0002 -> 0003 -> 0004 orders bootstrap, contracts, implementation, independent verification. Named sessions, narrowed scopes, task worktrees after first commit; managed hooks required. Direct bootstrap is authorized, not independent acceptance. No commit/push/release without authority; publish redacted snapshots only. Preserve unrelated work; no silent installs, destructive cleanup or unowned process kills. Record negative tests and canonical docs synchronization. + +Executable component boundary (CTX-0008, bitty#2495): the `storage` component is a stdio coprocess around journaled namespaces, not a second database. Preserve protocol v1 framing and bounds, atomic journal-before-snapshot commits, previous-revision rollback semantics, and the rule that the wire carries namespaces/keys/values but never a filesystem path. The host chooses the root at spawn time and owns installation, digest verification, capability grants, consent, safe-mode denial, XDG migration, and restore re-derivation; the component commits exactly the batch it is handed. On Linux/Android state operations must stay directory-fd anchored (`openat`/`renameat`/`unlinkat`) with `O_NOFOLLOW` final components and `O_CREAT|O_EXCL` temps; never reintroduce path-based state access, symlink following, or a non-Linux fallback. Do not add an installer, download source, catalog row, or Core dependency here. `just component-package` builds and packs locally; no tag or R2 upload belongs to this task. diff --git a/Cargo.lock b/Cargo.lock index 6fa4fa2..04a9d34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,65 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + [[package]] name = "bitty-storage" version = "0.0.1" +dependencies = [ + "rustix", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml index 9e51c34..da678d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,18 @@ publish = false name = "bitty_storage" path = "src/lib.rs" +[[bin]] +name = "bitty-storage" +path = "src/bin/bitty-storage.rs" + [lints.rust] unsafe_code = "forbid" [lints.clippy] all = { level = "warn", priority = -1 } + +# Namespace locks are OS advisory `flock`s, safe API only (no `unsafe`). +# Same platform split as the Core/execution claim lock; the Linux x86_64 +# component is the packaged target for this task. +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] +rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } diff --git a/README.md b/README.md index b076079..c95231e 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,232 @@ Bounded isolated storage mechanics crate (landed CTX-0003, pending independent v Prerequisite: W-131 / bitty-docs CTX-0267 accepted (commit 21d63dc); W-137 plugin contract accepted (bitty-plugins-docs PR #138, commit 002c7ce). Separate transcript, command history, session snapshots and plugin KV. Terminal Truth remains volatile; capture is opt-in and secret-minimizing. CTX-0001 -> CTX-0002 -> CTX-0003 -> CTX-0004 maps to Issues #4 -> #3 -> #2 -> #1. CTX-0001 (bootstrap) is complete: metadata gates, independent review, first publication, redacted CarryCtx snapshot and branch protection are recorded. CTX-0002 (contract readiness) is accepted on the W-131/W-137 acceptances plus recorded privacy review; CTX-0003 implementation is landed here. + +## Executable component boundary (bitty#2495) + +The crate is also the installable `storage` native component. It is a stdio +coprocess in the DIR-030 shape: the host spawns `bitty-storage --root PATH`, +sends length-prefixed frames on stdin, and reads frames from stdout. stderr +carries diagnostics only. The component wraps journaled namespaces under the +host-selected root; it carries no installer, no download source, no capability +policy, no XDG migration, and no Core dependency. + +- Descriptor: [`bitty-component.toml`](bitty-component.toml) — name `storage`, + version `0.0.1`, protocol `[1, 1]`, executable `bitty-storage`. +- Parser: `bitty_storage::component::descriptor` — closed `[component]` table, + strict `X.Y.Z`, `[a-z][a-z0-9-]{0,31}` name, `bitty-` executable, 64 + lowercase hex digits, and a `validate_for_install` check that refuses the + packaging placeholder digest. +- Store: `bitty_storage::component::store` — one locked directory per + namespace with `snapshot`, `previous`, `journal`, and `lock` members. +- Server: `bitty_storage::component::server` — handshake, request dispatch, + bounded pages, atomic commits, previous-revision rollback, clean shutdown. +- Binary: `bitty-storage` (`--root PATH`; `--version`; `--check-manifest PATH`). + +### Wire protocol v1 + +`frame = length:u32be payload[length]`, `payload = tag:u8 fields...`; integers +are big-endian, strings/bytes/list lengths are `u32be` and bounded. The frame +cap is 256 KiB and is checked from the four-byte header before any payload +read or allocation. Tags are fixed; unknown tags, trailing bytes, zero request +ids, and invalid namespace/key grammar fail closed. + +| Tag | Message | Direction | Fields | +| ------ | ----------------- | --------- | --------------------------------------------------------------------------------------------------------------------------- | +| `0x01` | `Hello` | host→comp | `min:u16 max:u16 component:str version:str` | +| `0x02` | `HelloAck` | comp→host | `protocol:u16 component:str version:str` | +| `0x10` | `OpenNamespace` | host→comp | `request:u64 namespace:str` | +| `0x11` | `ReadSnapshot` | host→comp | `request:u64 namespace:str start_row:u32 max_rows:u32 max_bytes:u32` | +| `0x12` | `CommitBatch` | host→comp | `request:u64 namespace:str expected_revision:u64 upserts:list deletes:list` | +| `0x13` | `Rollback` | host→comp | `request:u64 namespace:str expected_revision:u64` | +| `0x3F` | `Shutdown` | host→comp | none | +| `0x20` | `NamespaceOpened` | comp→host | `request:u64 namespace:str revision:u64 rows:u32 bytes:u64` | +| `0x21` | `SnapshotChunk` | comp→host | `request:u64 namespace:str revision:u64 start_row:u32 rows:list total_rows:u32 total_bytes:u64 done:bool` | +| `0x22` | `Committed` | comp→host | `request:u64 namespace:str revision:u64 rows_written:u32 rows_removed:u32 bytes_written:u64 total_rows:u32 total_bytes:u64` | +| `0x23` | `RolledBack` | comp→host | `request:u64 namespace:str revision:u64 restored_revision:u64 total_rows:u32 total_bytes:u64` | +| `0x2F` | `Error` | comp→host | `request:u64 kind:u8 message:str` | +| `0x3E` | `ShutdownAck` | comp→host | none | + +A `Row` is `key:str value:bytes`; rows are ordered by key in snapshots and +responses. Requests are synchronous. `Committed` is a success response only: it +means the batch is durable (journal and snapshot renames plus parent-directory +fsync) and the namespace still named the opened directory at the final +attachment check. A request rejected before publication returns `Error`, never +`Committed`, and performs no mutation: validation, stale-revision, bounds, +locking, and pre-rename I/O failures leave the committed revision, its rows, +and the previous-rollback image unchanged. A failure after publication is +reported with the distinct `uncertain` kind (code 11): a rename may already be +visible, its directory entry may not be confirmed durable, cleanup, recovery, +or rollback-image restoration may be incomplete, or the namespace path may no +longer name the opened directory. The host must reopen the namespace and read +its revision before retrying; retrying blindly can apply a second revision. +`rollback` is a one-step restore of the previous revision as a new monotonic +revision, not an abort of an in-flight request; repeating it toggles. It +accepts only a `previous` image whose revision is exactly `current - 1`: an +absent slot returns `no_previous`, and an image whose revision is equal to or +newer than the current snapshot, or skips a revision, returns `corrupt_state`. +A rollback therefore always publishes older content as a new revision and can +never acknowledge a no-op. `expected_revision` fences every commit and +rollback, so a stale host cannot overwrite newer state. + +Wire error kinds are stable within protocol v1 and additive: `protocol` 1, +`invalid_request` 2, `budget` 3, `unsupported` 4, `unknown_namespace` 5, +`stale_revision` 6, `locked` 7, `corrupt_state` 8, `io` 9, `internal` 10, and +`uncertain` 11. `io` from a commit or rollback means a pre-rename failure whose +state was fully rolled back, so retrying the same revision is safe (from +open/read it is an ordinary I/O failure). `uncertain` is the only kind that +requires reopen-and-reread before a retry. Existing codes never move: a v1 host +that predates `uncertain` receives 11 (or any newer kind) as unknown and must +fail closed as a generic failure instead of retrying from a cached revision. + +Properties the implementation enforces: + +- **No path surface.** A request names a namespace, a key, and opaque bytes. + Namespace and key grammars reject `/`, `\`, `:`, controls, non-ASCII, and + `..`; the host chooses the root at spawn time. On Linux/Android the root and + namespace directories are opened once and every state operation is + `openat`/`renameat`/`unlinkat` relative to that directory fd, with + `O_NOFOLLOW` on the final component and `O_CREAT|O_EXCL` temp creation. + `snapshot`, `previous`, `journal`, and `lock` are each validated as regular + files before use; symlinks, FIFOs, devices, and directories are refused. + A replaced namespace or racing ancestor therefore cannot redirect a read, + write, rename, or unlink outside the opened directory. +- **Attachment identity is the opened inode; the path check is detection + only.** After open, the server's authority is the namespace directory fd. + It stats the root-relative namespace path and compares `(st_dev, st_ino)` + with that fd before acknowledging recovery and after publication. Those + stats detect a rename, removal, or replacement visible at that instant. They + are best-effort detection, not an atomic binding between the pathname and + the stdio acknowledgement: a replacement that races the final check or the + response write is not detected, and the response still describes only the + opened inode. A detected detached namespace returns `uncertain`, never a + success, and the server evicts the cached handle; the client must reopen + (which re-resolves the pathname) and reread the revision before retrying. A + success therefore guarantees the operation ran against the opened directory + inode; it does not guarantee the pathname still names that inode when the + host reads the response. +- **Atomic commit with a journal.** The candidate snapshot is published as + `journal` first (temp+fsync+rename), then `previous`, then `snapshot`; the + journal is removed last. The prior `previous` image (or its absence) is + captured before the `previous` rewrite. A pre-rename failure in the + `previous`/`snapshot` steps restores that image, removes the journal durably, + and leaves the old snapshot intact, so a request reported as `io` was not + applied and the N-1 rollback image is unchanged. If the restoration cannot + be confirmed, the result is `uncertain` (never `io`) and the journal is + retained so recovery completes the commit. A crash after journal publication + is replayed on the next open. Once a rename may have happened, any failed + cleanup, recovery, restoration, or attachment check returns `uncertain` and + retains recovery evidence where possible. Reopen and inspect the revision + before retrying; no power-loss behavior beyond successful file and directory + fsync is claimed. +- **Crash recovery.** `snapshot` is replaced by atomic rename, so a partial + image is never observed. A journal whose target is newer than the snapshot + is replayed; recovery first completes the `previous` rewrite, so the restored + rollback slot is exactly the pre-commit revision (N-1). An equal-or-older + journal is discarded; malformed state fails closed with `corrupt_state`. A + failure while replaying a valid journal (writing `previous`/`snapshot`, + attachment, or journal removal) returns `uncertain` and keeps the journal, + because the target may become visible on the next open. A corrupt snapshot + with no valid journal is not auto-repaired or silently reset: the namespace + stays unavailable with a typed `corrupt_state`, and the host must restore it + from backup or delete it under its own policy. +- **Cross-process locking.** Opening a namespace takes a non-blocking + exclusive OS `flock` held for the connection lifetime. A second component + gets a typed `locked` error and the kernel releases the lock on crash. The + lock member is opened `O_NOFOLLOW`, verified regular with a single link, and + compared by device/inode with the current `lock` entry after locking, so a + symlinked or hardlinked lock and a replacement between `openat` and `flock` + are refused. Those checks are detection points; a later replacement cannot + retarget the inode this handle already locked. The namespace directory is + the lock's trust boundary: a process that can replace members inside it can + also replace the lock. +- **Bounded everything.** 64 open namespaces, 4096 rows per namespace, 1 MiB + per snapshot, 128 KiB per value, 128 upserts/deletes and 240 KiB of batch + bytes per commit, 128 rows and 240 KiB per read page, and a 256 KiB frame. + Reads are paged, so a full snapshot never needs an over-frame response. The + commit path rewrites the whole bounded snapshot per batch; hosts should + batch hot-path changes (the issue's performance rule), and no benchmark is + claimed by this task. +- **Clean shutdown.** `Shutdown` or stdin EOF drops every namespace handle + (releasing locks), writes `ShutdownAck` for an explicit `Shutdown`, and + exits 0. Protocol violations exit 3; a root/server failure exits 4. +- **Platform posture.** The anchored backend is Linux/Android only. Other + platforms fail closed at `StorageRoot::new` with `unsupported`; there is no + path-based fallback. Linux x86_64 is the packaged target for this task, and + Windows namespace-grammar hardening (trailing-dot aliasing) is tracked + before any Windows enablement. + +Generic (non-component) atomic helpers keep normal path semantics: a symlinked +final state directory is followed, and a missing parent on load is +`NotFound` (clean start), not an I/O error. After opening that parent once, +every operation is still fd-relative, so an ancestor replaced after the open +cannot redirect the commit. + +### Packaging + +```bash +just component-package # build Linux x86_64 and pack dist/x86_64-unknown-linux-gnu.tar.gz +just component-package-test # package-shape test without cargo, network, or R2 +``` + +`scripts/package-component.sh` builds (or accepts `--binary`) one Rust target +and writes exactly `.tar.gz`, `.tar.gz.sha256`, and +`SHA256SUMS`. The archive contains exactly `bitty-component.toml` (with the +target binary's real digest) and `bitty-storage` (`bitty-storage.exe` on +Windows). It validates the canonical record, the Cargo version, the target +name, and every existing archive's full descriptor plus its exact member list +(two regular files: descriptor and platform executable) before writing, so a +mixed-version, mixed-identity, duplicate-member, or non-regular archive cannot +be published. Only Linux x86_64 is built and tested in this task; #2491 owns +the per-platform matrix and catalog rows. No tag is created and nothing is +uploaded to R2. + +### Verification + +```bash +just product # fmt, clippy -D warnings, all tests +cargo test --test component_server --locked # live protocol lifecycle +cargo test --test component_store --locked # journal recovery and locking +cargo test --test component_descriptor --locked +``` + +The live suite covers a real commit/read/rollback lifecycle, a clean restart, +a SIGKILL after an acknowledged commit, stale revisions, bounded paging, +namespace/row/byte limits, oversize and malformed frames, path-shaped wire +names, two components contending for one namespace lock, a symlinked lock +(both existing and absent outside targets), and a namespace replaced after +open. Set `BITTY_STORAGE_COMPONENT_BIN` to run the same suite against a packed +release binary. The store suite constructs snapshot and journal bytes +independently to prove recovery across a crash between journal publication and +the snapshot rename, proves a failed replay is `uncertain` and retriable after +the filesystem is repaired, and exercises symlinks and hardlinks for the state +members and lock. It also proves that a failed snapshot publication restores +the N-1 rollback image (or keeps it absent), that an unconfirmed restoration is +`uncertain` with the journal retained, and that a `previous` image which is +equal to, newer than, or skips the current revision is `corrupt_state` rather +than a no-op rollback risk. The atomic-write suite proves a pre-planted temp +symlink is never followed and cannot create or modify an outside fixture; it +also covers post-rename directory-sync uncertainty, symlinked generic state +directories, and missing-parent loads. Windows path-commit behavior is covered +by a Windows-only test; the local and CI matrix still builds only Linux +x86_64. + +### Remaining for the Core host client + +- Resolve/install the descriptor, verify name/version/protocol/executable and + the streamed SHA-256, choose the storage root, and spawn the binary from its + version directory. +- Own the handshake and request deadlines, capability grants, consent, + safe-mode denial, XDG-state migration, and restore re-derivation (none live + in this component). +- Return a request-scoped `Error` (nonzero request id) to its caller with the + typed kind; it applies to that request only. Treat a connection-level + `Error` (request id 0 / `CONNECTION_ID`), unexpected EOF, or exit 3 as + `component_lost`: fail in-flight work, restart with the existing backoff + policy, and place the component in its own process group/session so a crash + is cleaned up by killing the group. Treat the `uncertain` kind (11) as + reopen-and-reread before any retry, and treat unknown error kinds as a + generic failure rather than retryable. +- Run parity tests against the in-process crate and remove the direct linked + dependency after parity; synchronize the canonical docs and add the + remaining target matrix (#2491). diff --git a/bitty-component.toml b/bitty-component.toml new file mode 100644 index 0000000..33b9f77 --- /dev/null +++ b/bitty-component.toml @@ -0,0 +1,16 @@ +# bitty-component.toml — canonical component descriptor record for `storage` +# (bitty#2495 split). Owner naming: component name `storage` (repository +# `bitty-storage`); executable `bitty-storage` (the `bitty-` rule). +# +# This file records the FIXED descriptor fields (name, version, protocol, +# executable). The `sha256` below is a parser-valid placeholder: every target +# binary has its own digest, so `scripts/package-component.sh` renders a +# per-target stage copy with the real digest and refuses to pack when any +# other line drifts from this file. Never publish this placeholder file itself +# as a release descriptor. +[component] +name = "storage" +version = "0.0.1" +protocol = [1, 1] +executable = "bitty-storage" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" diff --git a/justfile b/justfile index 2e11542..9ec1a52 100644 --- a/justfile +++ b/justfile @@ -14,6 +14,21 @@ metadata: test -s .carryctx/config.toml python3 -c 'import tomllib; from pathlib import Path; [tomllib.loads(p.read_text()) for p in [Path("repo.toml"), Path(".carryctx/config.toml")]]' +component-metadata: + #!/usr/bin/env bash + set -euo pipefail + test -s bitty-component.toml + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)" + test -n "$version" + grep -Fx '[component]' bitty-component.toml >/dev/null + grep -Fx 'name = "storage"' bitty-component.toml >/dev/null + grep -Fx "version = \"$version\"" bitty-component.toml >/dev/null + grep -Fx 'protocol = [1, 1]' bitty-component.toml >/dev/null + grep -Fx 'executable = "bitty-storage"' bitty-component.toml >/dev/null + grep -Fx 'sha256 = "0000000000000000000000000000000000000000000000000000000000000000"' bitty-component.toml >/dev/null + test -x scripts/package-component.sh + test -x scripts/test-package-component.sh + hygiene: #!/usr/bin/env bash set -euo pipefail @@ -62,9 +77,18 @@ check: just fmt-check just markdownlint just metadata + just component-metadata just hygiene just paths +# Build the Linux x86_64 executable component and pack its release archive. +component-package: + ./scripts/package-component.sh --target x86_64-unknown-linux-gnu --output dist/x86_64-unknown-linux-gnu.tar.gz + +# Shape test for the packager (no cargo build, network, or R2). +component-package-test: + ./scripts/test-package-component.sh + workflows: actionlint act -n diff --git a/repo.toml b/repo.toml index f14f788..0634b13 100644 --- a/repo.toml +++ b/repo.toml @@ -2,7 +2,7 @@ name = "bitty-storage" kind = "extension" lang = "rust" -role = "candidate bounded storage extension; landed crate, pending independent verification" +role = "bounded storage extension; landed crate + installable stdio component (protocol v1)" owner = "bitty-core" docs = "bitty-terminal-docs" @@ -10,7 +10,7 @@ docs = "bitty-terminal-docs" version = "0.0.1" edition = "2024" rust_version = "1.85" -status = "landed" +status = "landed; executable component boundary CTX-0008" [contracts] -status = "W-131 accepted (bitty-docs 21d63dc) + W-137 accepted (bitty-plugins-docs 002c7ce); CTX-0002 accepted; CTX-0003 implementation landed, CTX-0004 verification pending" +status = "W-131 accepted (bitty-docs 21d63dc) + W-137 accepted (bitty-plugins-docs 002c7ce); CTX-0002 accepted; CTX-0003 implementation landed, CTX-0004 verification pending; storage component protocol v1 packed (host client pending)" diff --git a/scripts/package-component.sh b/scripts/package-component.sh new file mode 100755 index 0000000..f6083c1 --- /dev/null +++ b/scripts/package-component.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# package-component.sh — build (optionally) and pack one `storage` component +# target in the DIR-030 layout. +# +# The archive contains exactly two root members: `bitty-component.toml` with +# the target executable's real SHA-256, and the executable `bitty-storage` +# (`bitty-storage.exe` on Windows). Beside it the script writes +# `.tar.gz.sha256` and the aggregate `SHA256SUMS`; nothing is +# uploaded, tagged, or published. +# +# This script is the Linux x86_64 build/test path and also accepts any other +# Rust target triple for the #2491 per-platform matrix. A target whose binary +# was not built on the matching platform stays unverified and must not be +# added to a catalog. +# +# Usage: +# scripts/package-component.sh --target TRIPLE --output PATH [--version X.Y.Z] +# scripts/package-component.sh --target TRIPLE --output PATH --binary PATH +# +# Needs: cargo unless --binary is given, plus tar, gzip, and sha256sum or +# shasum. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" +readonly REPO_ROOT +readonly DESCRIPTOR_FILE="bitty-component.toml" +readonly COMPONENT_NAME="storage" +readonly EXECUTABLE="bitty-storage" +readonly ZERO_DIGEST="0000000000000000000000000000000000000000000000000000000000000000" +readonly MAX_DESCRIPTOR_BYTES=4096 +readonly MAX_BINARY_BYTES=268435456 +readonly CDN_LAYOUT="extensions/storage" + +VERSION="" +TARGET="" +BINARY="" +OUTPUT="" + +die() { + printf 'package-component: ERROR: %s\n' "$1" >&2 + exit 1 +} + +usage() { + cat <<'EOF' +Usage: scripts/package-component.sh --target TRIPLE --output PATH [--version X.Y.Z] [--binary PATH] + + --target TRIPLE Rust target triple; output basename must be TRIPLE.tar.gz + --output PATH output archive path + --version X.Y.Z version (defaults to bitty-component.toml; must match) + --binary PATH already-built executable (skips cargo build) + +Environment: + CARGO_TARGET_DIR respected when building (default: /target) + GITHUB_REF_TYPE/GITHUB_REF_NAME on a tag, the tag must be v + +The archive layout is stable: bitty-component.toml first, then the +executable. Upload key layout (when a release task authorizes publication): +/extensions/storage//.tar.gz and SHA256SUMS. +EOF +} + +valid_version() { + local raw="$1" + [[ "${#raw}" -le 64 ]] || return 1 + [[ "$raw" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || return 1 + local part + for part in "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}"; do + [[ "${#part}" -gt 10 ]] && return 1 + [[ "${#part}" -eq 10 && "$part" -gt 4294967295 ]] && return 1 + done + return 0 +} + +valid_target() { + [[ "$1" =~ ^[a-z0-9_]+(-[a-z0-9_]+)+$ ]] +} + +descriptor_value() { + awk -F'"' -v key="$1" ' + $1 ~ "^[[:space:]]*" key "[[:space:]]*=" { print $2; exit } + ' "$REPO_ROOT/$DESCRIPTOR_FILE" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --target) + TARGET="${2:-}" + shift 2 + ;; + --output) + OUTPUT="${2:-}" + shift 2 + ;; + --version) + VERSION="${2:-}" + shift 2 + ;; + --binary) + BINARY="${2:-}" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + die "unknown argument: $1" + ;; + esac +done + +[[ -f "$REPO_ROOT/$DESCRIPTOR_FILE" && ! -L "$REPO_ROOT/$DESCRIPTOR_FILE" ]] || die "root descriptor is missing" +descriptor_size="$(wc -c <"$REPO_ROOT/$DESCRIPTOR_FILE" | tr -d ' ')" +[[ "$descriptor_size" -le "$MAX_DESCRIPTOR_BYTES" ]] || die "root descriptor exceeds $MAX_DESCRIPTOR_BYTES bytes" + +cargo_version="$(sed -n 's/^version = "\(.*\)"$/\1/p' "$REPO_ROOT/Cargo.toml" | head -n 1)" +valid_version "$cargo_version" || die "Cargo.toml version must be strict X.Y.Z: $cargo_version" +descriptor_version="$(descriptor_value version)" +valid_version "$descriptor_version" || die "descriptor version must be strict X.Y.Z: $descriptor_version" +[[ -z "$VERSION" ]] || valid_version "$VERSION" || die "version must be strict X.Y.Z: $VERSION" +[[ -z "$VERSION" ]] || [[ "$VERSION" == "$cargo_version" ]] || die "version $VERSION differs from Cargo.toml $cargo_version" +[[ -z "$VERSION" ]] || [[ "$VERSION" == "$descriptor_version" ]] || die "version $VERSION differs from descriptor $descriptor_version" +if [[ "${GITHUB_REF_TYPE:-}" == "tag" && -n "${GITHUB_REF_NAME:-}" ]]; then + [[ "$GITHUB_REF_NAME" == "v$cargo_version" ]] || die "tag $GITHUB_REF_NAME does not match version $cargo_version" +fi + +grep -Fx '[component]' "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor must contain [component]" +grep -Fx "name = \"$COMPONENT_NAME\"" "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor name must be $COMPONENT_NAME" +grep -Fx 'protocol = [1, 1]' "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor protocol must be [1, 1]" +grep -Fx "executable = \"$EXECUTABLE\"" "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor executable must be $EXECUTABLE" +grep -Fx "version = \"$cargo_version\"" "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor version must be $cargo_version" +grep -Fx "sha256 = \"$ZERO_DIGEST\"" "$REPO_ROOT/$DESCRIPTOR_FILE" >/dev/null || die "root descriptor must use the zero digest sentinel" + +valid_target "$TARGET" || die "target must look like a Rust target triple: $TARGET" +[[ -n "$OUTPUT" ]] || die "--output is required" +[[ "$(basename -- "$OUTPUT")" == "$TARGET.tar.gz" ]] || die "output basename must be $TARGET.tar.gz" +if [[ "$OUTPUT" != /* ]]; then + OUTPUT="$PWD/${OUTPUT#./}" +fi +output_dir="$(dirname -- "$OUTPUT")" +mkdir -p -- "$output_dir" + +archive_executable="$EXECUTABLE" +if [[ "$TARGET" == *-windows-* ]]; then + archive_executable="$EXECUTABLE.exe" +fi + +if [[ -z "$BINARY" ]]; then + command -v cargo >/dev/null 2>&1 || die "cargo is required to build the component" + target_root="${CARGO_TARGET_DIR:-$REPO_ROOT/target}" + # Cargo resolves an environment-supplied relative target directory + # against its working directory, which is REPO_ROOT below. Resolve it + # here with the same base so the binary lookup cannot miss. + if [[ "$target_root" != /* ]]; then + target_root="$REPO_ROOT/$target_root" + fi + BINARY="$target_root/$TARGET/release/$archive_executable" + printf 'package-component: building %s %s for %s\n' "$EXECUTABLE" "$cargo_version" "$TARGET" >&2 + (cd -- "$REPO_ROOT" && cargo build --release --locked --target "$TARGET" --bin "$EXECUTABLE") +fi +[[ -f "$BINARY" && ! -L "$BINARY" ]] || die "binary is missing or is a symlink: $BINARY" +binary_size="$(wc -c <"$BINARY" | tr -d ' ')" +[[ "$binary_size" -gt 0 ]] || die "binary is empty" +[[ "$binary_size" -le "$MAX_BINARY_BYTES" ]] || die "binary exceeds $MAX_BINARY_BYTES bytes" + +if command -v sha256sum >/dev/null 2>&1; then + SHA256SUM=(sha256sum) +elif command -v shasum >/dev/null 2>&1; then + SHA256SUM=(shasum -a 256) +else + die "sha256sum or shasum is required" +fi +command -v tar >/dev/null 2>&1 || die "tar is required" +command -v gzip >/dev/null 2>&1 || die "gzip is required" + +binary_digest="$("${SHA256SUM[@]}" "$BINARY" | awk '{print $1}')" +[[ "$binary_digest" =~ ^[0-9a-f]{64}$ ]] || die "digest tool returned an invalid SHA-256" + +# Validate existing target archives so a mixed-version directory can never +# publish two incompatible descriptors under one SHA256SUMS. +shopt -s nullglob +existing_archives=("$output_dir"/*.tar.gz) +shopt -u nullglob +for candidate in "${existing_archives[@]}"; do + [[ -f "$candidate" && ! -L "$candidate" ]] || die "archive is not a regular file: $candidate" + candidate_name="$(basename -- "$candidate")" + valid_target "${candidate_name%.tar.gz}" || die "archive basename is not a target triple: $candidate_name" + [[ "$candidate" == "$OUTPUT" ]] && continue + candidate_target="${candidate_name%.tar.gz}" + candidate_executable="bitty-storage" + [[ "$candidate_target" == *-windows-* ]] && candidate_executable="bitty-storage.exe" + mapfile -t candidate_members < <(tar -tzf "$candidate" 2>/dev/null) || die "archive is unreadable: $candidate" + expected_members=("$DESCRIPTOR_FILE" "$candidate_executable") + [[ "${#candidate_members[@]}" -eq 2 ]] || die "archive $candidate_name must contain exactly two members" + expected_member_list="$(printf '%s\n' "${expected_members[@]}" | LC_ALL=C sort)" + actual_member_list="$(printf '%s\n' "${candidate_members[@]}" | LC_ALL=C sort)" + [[ "$actual_member_list" == "$expected_member_list" ]] || + die "archive $candidate_name members are not exactly descriptor plus executable" + mapfile -t candidate_types < <(tar -tvzf "$candidate" 2>/dev/null | awk '{print substr($1, 1, 1)}') || die "archive member metadata is unreadable: $candidate" + [[ "${#candidate_types[@]}" -eq 2 && "${candidate_types[0]}" == "-" && "${candidate_types[1]}" == "-" ]] || + die "archive $candidate_name members must be regular files" + candidate_descriptor="$(tar -xOzf "$candidate" "$DESCRIPTOR_FILE" 2>/dev/null)" || die "archive has no readable $DESCRIPTOR_FILE: $candidate" + # Only the per-binary sha256 line may differ from the canonical record; + # name, version, protocol, executable, and comments must match. + if ! diff <(grep -v '^sha256 = ' "$REPO_ROOT/$DESCRIPTOR_FILE") <(grep -v '^sha256 = ' <<<"$candidate_descriptor") >/dev/null; then + die "archive $candidate_name descriptor drifted from $DESCRIPTOR_FILE" + fi + [[ "$(grep -c '^sha256 = ' <<<"$candidate_descriptor")" == "1" ]] || die "archive $candidate_name has no unique sha256 line" + candidate_digest="$(awk -F'"' '$1 ~ /^[[:space:]]*sha256[[:space:]]*=/ { print $2; exit }' <<<"$candidate_descriptor")" + [[ "$(awk -F'"' '$1 ~ /^[[:space:]]*executable[[:space:]]*=/ { print $2; exit }' <<<"$candidate_descriptor")" == "bitty-storage" ]] || die "archive $candidate_name has unexpected executable" + candidate_binary_digest="$(tar -xOzf "$candidate" "$candidate_executable" 2>/dev/null | "${SHA256SUM[@]}" | awk '{print $1}')" || die "archive $candidate_name executable is unreadable" + [[ "$candidate_digest" =~ ^[0-9a-f]{64}$ && "$candidate_digest" == "$candidate_binary_digest" ]] || die "archive $candidate_name executable digest does not match its descriptor" +done + +scratch_root="${TMPDIR:-/tmp/bitty}" +mkdir -p -- "$scratch_root" +stage_dir="$(mktemp -d "$scratch_root/storage-component.XXXXXX")" +cleanup() { rm -rf -- "$stage_dir"; } +trap cleanup EXIT + +# Render the per-target descriptor: only the sha256 line may differ from the +# canonical record, so descriptor drift fails here instead of shipping. +awk -v digest="$binary_digest" ' + /^[[:space:]]*sha256[[:space:]]*=/ { print "sha256 = \"" digest "\""; next } + { print } +' "$REPO_ROOT/$DESCRIPTOR_FILE" >"$stage_dir/$DESCRIPTOR_FILE" +if ! diff <(grep -v '^sha256 = ' "$REPO_ROOT/$DESCRIPTOR_FILE") <(grep -v '^sha256 = ' "$stage_dir/$DESCRIPTOR_FILE") >/dev/null; then + die "rendered descriptor drifted from $DESCRIPTOR_FILE (non-sha256 lines)" +fi +grep -Fx "sha256 = \"$binary_digest\"" "$stage_dir/$DESCRIPTOR_FILE" >/dev/null || die "rendered digest is missing" + +cp -- "$BINARY" "$stage_dir/$archive_executable" +chmod 755 "$stage_dir/$archive_executable" + +rm -f -- "$OUTPUT" +tar -czf "$OUTPUT" -C "$stage_dir" "$DESCRIPTOR_FILE" "$archive_executable" +expected_member_list="$(printf '%s\n%s\n' "$DESCRIPTOR_FILE" "$archive_executable" | LC_ALL=C sort)" +actual_member_list="$(tar -tzf "$OUTPUT" | LC_ALL=C sort)" +[[ "$actual_member_list" == "$expected_member_list" ]] || die "archive members are not exactly descriptor plus executable" + +# Read the packed descriptor back; the digest must describe the packed bytes. +packed_descriptor="$(tar -xOzf "$OUTPUT" "$DESCRIPTOR_FILE")" +packed_digest="$(awk -F'"' '$1 ~ /^[[:space:]]*sha256[[:space:]]*=/ { print $2; exit }' <<<"$packed_descriptor")" +[[ "$packed_digest" == "$binary_digest" ]] || die "packed descriptor digest does not match the executable" + +# Recompute every archive's sidecar and the aggregate manifest. +shopt -s nullglob +tarballs=() +for candidate in "$output_dir"/*.tar.gz; do + candidate_name="$(basename -- "$candidate")" + candidate_target="${candidate_name%.tar.gz}" + if valid_target "$candidate_target"; then + tarballs+=("$candidate_name") + fi +done +shopt -u nullglob +[[ "${#tarballs[@]}" -gt 0 ]] || die "no target archives found in $output_dir" +printf '%s\n' "${tarballs[@]}" | LC_ALL=C sort >"$output_dir/.component-targets" +sorted_archives=() +while IFS= read -r archive_name; do + sorted_archives+=("$archive_name") +done <"$output_dir/.component-targets" +for archive_name in "${sorted_archives[@]}"; do + (cd -- "$output_dir" && "${SHA256SUM[@]}" "$archive_name" >"$archive_name.sha256") +done +(cd -- "$output_dir" && "${SHA256SUM[@]}" "${sorted_archives[@]}" | LC_ALL=C sort -k2 >SHA256SUMS) +rm -f -- "$output_dir/.component-targets" +(cd -- "$output_dir" && "${SHA256SUM[@]}" -c SHA256SUMS >/dev/null) || die "SHA256SUMS verification failed" +for archive_name in "${sorted_archives[@]}"; do + cmp -s <(cd -- "$output_dir" && "${SHA256SUM[@]}" "$archive_name") "$output_dir/$archive_name.sha256" || + die "sidecar digest does not match archive: $archive_name.sha256" +done + +printf 'package-component: PASS (name=%s, target=%s, version=%s, output=%s, key=%s/%s/%s.tar.gz)\n' \ + "$COMPONENT_NAME" "$TARGET" "$cargo_version" "$OUTPUT" "$CDN_LAYOUT" "$cargo_version" "$TARGET" diff --git a/scripts/test-package-component.sh b/scripts/test-package-component.sh new file mode 100755 index 0000000..7cfcbc3 --- /dev/null +++ b/scripts/test-package-component.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Shape test for scripts/package-component.sh: no Rust build, network, or R2. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" +readonly SCRIPT="$REPO_ROOT/scripts/package-component.sh" +scratch_root="${TMPDIR:-/tmp/bitty}" +mkdir -p -- "$scratch_root" +tmp="$(mktemp -d "$scratch_root/package-component-test.XXXXXX")" +cleanup() { rm -rf -- "$tmp"; } +trap cleanup EXIT + +version="$(awk -F'"' '$1 ~ /^[[:space:]]*version[[:space:]]*=/ { print $2; exit }' "$REPO_ROOT/bitty-component.toml")" +[[ -n "$version" ]] || { + printf 'descriptor version is missing\n' >&2 + exit 1 +} +printf '#!/bin/sh\necho bitty-storage %s\n' "$version" >"$tmp/bitty-storage" +chmod 755 "$tmp/bitty-storage" +printf '#!/bin/sh\necho bitty-storage %s arm\n' "$version" >"$tmp/bitty-storage-arm" +chmod 755 "$tmp/bitty-storage-arm" + +output_dir="$tmp/dist" +mkdir -p -- "$output_dir" +"$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$output_dir/x86_64-unknown-linux-gnu.tar.gz" +"$SCRIPT" --version "$version" --target aarch64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage-arm" --output "$output_dir/aarch64-unknown-linux-gnu.tar.gz" + +expected_files=$'SHA256SUMS\naarch64-unknown-linux-gnu.tar.gz\naarch64-unknown-linux-gnu.tar.gz.sha256\nx86_64-unknown-linux-gnu.tar.gz\nx86_64-unknown-linux-gnu.tar.gz.sha256' +actual_files="$(find "$output_dir" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort)" +[[ "$actual_files" == "$expected_files" ]] || { + printf 'unexpected package files:\n%s\n' "$actual_files" >&2 + exit 1 +} + +(cd "$output_dir" && sha256sum -c SHA256SUMS >/dev/null) +for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do + archive="$output_dir/$target.tar.gz" + members="$(tar -tzf "$archive" | LC_ALL=C sort)" + [[ "$members" == $'bitty-component.toml\nbitty-storage' ]] || { + printf 'unexpected archive members for %s:\n%s\n' "$target" "$members" >&2 + exit 1 + } + extract="$tmp/$target" + mkdir -p "$extract" + tar -xzf "$archive" -C "$extract" + grep -Fx 'name = "storage"' "$extract/bitty-component.toml" >/dev/null + grep -Fx "version = \"$version\"" "$extract/bitty-component.toml" >/dev/null + grep -Fx 'protocol = [1, 1]' "$extract/bitty-component.toml" >/dev/null + grep -Fx 'executable = "bitty-storage"' "$extract/bitty-component.toml" >/dev/null + digest="$(sha256sum "$extract/bitty-storage" | awk '{print $1}')" + grep -Fx "sha256 = \"$digest\"" "$extract/bitty-component.toml" >/dev/null +done + +# Windows target triples must use the platform executable member name. +windows_dir="$tmp/windows" +mkdir -p -- "$windows_dir" +"$SCRIPT" --version "$version" --target x86_64-pc-windows-gnu \ + --binary "$tmp/bitty-storage" --output "$windows_dir/x86_64-pc-windows-gnu.tar.gz" +windows_members="$(tar -tzf "$windows_dir/x86_64-pc-windows-gnu.tar.gz" | LC_ALL=C sort)" +[[ "$windows_members" == $'bitty-component.toml\nbitty-storage.exe' ]] || { + printf 'unexpected Windows archive members:\n%s\n' "$windows_members" >&2 + exit 1 +} + +# Duplicate and non-regular archive members are rejected before digest checks. +for shape in duplicate symlink; do + shape_dir="$tmp/$shape" + shape_extract="$tmp/$shape-extract" + mkdir -p -- "$shape_dir" "$shape_extract" + tar -xzf "$output_dir/x86_64-unknown-linux-gnu.tar.gz" -C "$shape_extract" + if [[ "$shape" == duplicate ]]; then + (cd "$shape_extract" && tar -czf "$shape_dir/aarch64-unknown-linux-gnu.tar.gz" bitty-component.toml bitty-storage bitty-storage) + else + cp "$shape_extract/bitty-storage" "$shape_extract/bitty-storage-copy" + rm "$shape_extract/bitty-storage" + ln -s bitty-storage-copy "$shape_extract/bitty-storage" + tar -czf "$shape_dir/aarch64-unknown-linux-gnu.tar.gz" -C "$shape_extract" bitty-component.toml bitty-storage + fi + if "$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$shape_dir/x86_64-unknown-linux-gnu.tar.gz" \ + >/dev/null 2>&1; then + printf 'packager accepted malformed %s archive\n' "$shape" >&2 + exit 1 + fi +done + +# Existing archives must have a descriptor digest that matches the executable, +# even when the descriptor's non-digest metadata is otherwise canonical. +digest_dir="$tmp/digest-mismatch" +digest_extract="$tmp/digest-mismatch-extract" +mkdir -p -- "$digest_dir" "$digest_extract" +tar -xzf "$output_dir/x86_64-unknown-linux-gnu.tar.gz" -C "$digest_extract" +printf '#!/bin/sh\necho substituted\n' >"$digest_extract/bitty-storage" +chmod 755 "$digest_extract/bitty-storage" +tar -czf "$digest_dir/aarch64-unknown-linux-gnu.tar.gz" -C "$digest_extract" bitty-component.toml bitty-storage +if "$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$digest_dir/x86_64-unknown-linux-gnu.tar.gz" \ + >/dev/null 2>&1; then + printf 'packager accepted an existing archive with a mismatched executable digest\n' >&2 + exit 1 +fi + +# A stale archive with a different descriptor version in the same output +# directory must stop the pack instead of producing mixed metadata. +stale_dir="$tmp/stale" +stale_extract="$tmp/stale-extract" +mkdir -p -- "$stale_dir" "$stale_extract" +tar -xzf "$output_dir/x86_64-unknown-linux-gnu.tar.gz" -C "$stale_extract" +sed -i 's/^version = ".*"$/version = "9.9.9"/' "$stale_extract/bitty-component.toml" +tar -czf "$stale_dir/aarch64-unknown-linux-gnu.tar.gz" -C "$stale_extract" bitty-component.toml bitty-storage +if "$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$stale_dir/x86_64-unknown-linux-gnu.tar.gz" \ + >/dev/null 2>&1; then + printf 'packager accepted a mixed-version output directory\n' >&2 + exit 1 +fi + +if "$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$tmp/wrong-name.tgz" >/dev/null 2>&1; then + printf 'packager accepted a non-target output name\n' >&2 + exit 1 +fi + +# A same-version archive whose name/executable/protocol drifted must also be +# rejected; only the per-binary sha256 line may differ. +drift_dir="$tmp/drift" +drift_extract="$tmp/drift-extract" +mkdir -p -- "$drift_dir" "$drift_extract" +tar -xzf "$output_dir/x86_64-unknown-linux-gnu.tar.gz" -C "$drift_extract" +sed -i 's/^executable = ".*"$/executable = "bitty-other"/' "$drift_extract/bitty-component.toml" +tar -czf "$drift_dir/aarch64-unknown-linux-gnu.tar.gz" -C "$drift_extract" bitty-component.toml bitty-storage +if "$SCRIPT" --version "$version" --target x86_64-unknown-linux-gnu \ + --binary "$tmp/bitty-storage" --output "$drift_dir/x86_64-unknown-linux-gnu.tar.gz" \ + >/dev/null 2>&1; then + printf 'packager accepted a same-version archive with a drifted descriptor\n' >&2 + exit 1 +fi + +printf 'package-component.test: PASS\n' diff --git a/src/atomic_io.rs b/src/atomic_io.rs index e1d4edb..5f6413a 100644 --- a/src/atomic_io.rs +++ b/src/atomic_io.rs @@ -2,8 +2,8 @@ //! //! Ported from the Core session save path and the plugin-store durable //! commit: temp-plus-rename writes with fsync, stale-temp hygiene, and -//! capped loads. All errors are content-free (kinds, counts, and paths -//! only) so they are safe for logs. +//! capped loads. All errors are content-free (kinds and counts only) so they +//! are safe for logs. //! //! # Core duplication decision (storage#13, CTX-0006): option 2 //! @@ -22,6 +22,36 @@ //! the seam without removing the function duplication, so option 1 (Core owns //! a generic atomic-write trait implemented here) is rejected. //! +//! # Path-containment hardening (component server) +//! +//! On Linux/Android the commit path opens the destination parent once as a +//! directory fd and performs `openat`/`renameat` relative to it: +//! +//! - the temp is created `O_CREAT|O_EXCL|O_NOFOLLOW` (an existing symlink, +//! regular file, FIFO, or device is refused, never followed/truncated); +//! - `write_atomic_durably` requires the temp to share the destination parent; +//! - the pre-write sweep and capped load are also `*at`-anchored, and a load +//! rejects a symlink/non-regular final component; +//! - a racing namespace/ancestor replacement therefore cannot redirect the +//! commit outside the opened directory. The component store additionally +//! verifies the namespace fd is still the directory named under the root +//! before serving or mutating it (best-effort detection, never an atomic +//! acknowledgement). +//! +//! The generic helpers used by `FileBackend`/session/KV follow a symlinked +//! final parent directory (caller-owned state paths such as a symlinked +//! `$XDG_STATE_HOME`), but they still open that parent once as a directory fd +//! and perform every subsequent operation `*at` relative to it. Replacing an +//! ancestor after the open cannot redirect the commit; only the symlink +//! already present at open time is followed. The generic helpers are not a +//! component boundary and make no claim about a pathname that changes +//! underneath the caller. +//! +//! On other platforms the low-level helper uses exclusive creation +//! (`create_new`) but remains path-based; the executable component store is +//! Linux/Android-only and fails closed elsewhere, so this path is limited to +//! the crate's generic FileBackend/KV users. +//! //! # Temp-sweep parity analysis against the Core copy //! //! The commit contract is identical on both sides: write the temp, fsync the @@ -31,11 +61,12 @@ //! //! - Filesystem abstraction: Core commits through `FileSystem` so tests can //! inject write, sync, and rename failures via `FakeFileSystem`; this crate -//! commits through `std::fs` directly because it takes no Core dependency -//! and needs no injection seam. Identical behavior is pinned from opposite -//! sides: Core asserts operation ordering through recorded writes, syncs, -//! renames, and removals, while this crate asserts the resulting filesystem -//! outcomes (destination preserved, temp gone). +//! commits through `std::fs` plus `*at` directory-fd calls on Linux/Android +//! because it takes no Core dependency and needs no injection seam. +//! Identical behavior is pinned from opposite sides: Core asserts operation +//! ordering through recorded writes, syncs, renames, and removals, while +//! this crate asserts the resulting filesystem outcomes (destination +//! preserved, temp gone). //! - Temp naming: Core uses one transaction temp per index write with a //! process-global sequence; this crate uses [`unique_temp_sibling_for`] //! (process id plus thread id plus sequence) so concurrent savers of one @@ -53,12 +84,13 @@ //! explicit install-path decision. Any Core sweep must run before writing, //! never after the rename. //! - Durability: this crate syncs the temp file before the rename and then -//! best-effort syncs the parent directory after the rename; Core syncs the +//! syncs the parent directory after the rename; Core syncs the //! temp file before the rename through `FileSystem::sync_file` with no //! directory sync. Success on both sides means the temp bytes reached disk //! before the rename. -//! - Permissions: this crate creates the temp with mode 0600 on Unix and -//! re-asserts it before the first byte; Core writes through +//! - Permissions: this crate creates the temp with mode 0600 (Linux/Android +//! through the anchored `openat`); the non-Linux helper re-asserts 0600 +//! before the first byte. Core writes through //! `NativeFileSystem::write_file` under the store directory permissions. //! This side holds secret-capable session and KV payloads, so the tighter //! mode stays here. @@ -66,13 +98,19 @@ //! before touching the filesystem; Core rejects over-ceiling indexes //! during encoding before calling its commit. Both fail closed pre-write. //! - Errors: this crate reports [`IoError`] and [`LoadError`] without -//! contents; Core maps failures to content-free denials. A failed commit -//! on either side leaves the previous destination intact. +//! contents; Core maps failures to content-free denials. A pre-rename +//! failure leaves the destination intact. Once rename succeeds, +//! a later directory-sync failure on Unix-like targets is reported as +//! uncertain because the new destination may already be visible. +//! Windows has no std-only directory fsync; it skips that sync and makes no +//! directory-entry durability claim. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use crate::ceiling::{MAX_SESSION_FILE_BYTES, SESSION_FILE_NAME}; +#[cfg(any(target_os = "linux", target_os = "android"))] +use crate::fs_at; /// Monotonic suffix for concurrent temp siblings of one destination. static WRITE_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -88,12 +126,25 @@ pub const STALE_TEMP_AGE_SECS: u64 = 3_600; pub struct IoError { context: &'static str, message: String, + uncertain: bool, } impl IoError { #[must_use] pub fn new(context: &'static str, message: String) -> Self { - Self { context, message } + Self { + context, + message, + uncertain: false, + } + } + + fn uncertain(context: &'static str, message: String) -> Self { + Self { + context, + message, + uncertain: true, + } } #[must_use] @@ -105,6 +156,13 @@ impl IoError { pub fn message(&self) -> &str { &self.message } + + /// Whether rename may have installed the new destination but durability + /// could not be confirmed. Do not blindly retry from a cached snapshot. + #[must_use] + pub const fn is_uncertain(&self) -> bool { + self.uncertain + } } impl std::fmt::Display for IoError { @@ -119,6 +177,28 @@ fn io_error(context: &'static str, err: std::io::Error) -> IoError { IoError::new(context, err.to_string()) } +#[cfg(any(target_os = "linux", target_os = "android"))] +fn atomic_error(err: std::io::Error) -> IoError { + if fs_at::is_uncertain_durability_error(&err) { + IoError::uncertain("atomic commit", err.to_string()) + } else { + io_error("atomic commit", err) + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn parent_dir(path: &Path) -> &Path { + match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn parent_dir_owned(path: &Path) -> PathBuf { + parent_dir(path).to_path_buf() +} + /// Temp sibling for an atomic save (`.tmp.`). #[must_use] pub fn temp_sibling_for(path: &Path) -> PathBuf { @@ -160,6 +240,26 @@ pub fn unique_temp_sibling_for(path: &Path) -> PathBuf { /// BEFORE writing, never after the rename: a post-rename sweep would race /// a concurrent saver's live temp sibling. pub fn clean_temp_siblings(path: &Path) { + #[cfg(any(target_os = "linux", target_os = "android"))] + { + let parent = parent_dir(path); + let Ok(dir) = fs_at::open_dir_path_following(parent) else { + return; + }; + let stem = path.file_name().map_or_else( + || SESSION_FILE_NAME.into(), + |name| name.to_string_lossy().into_owned(), + ); + let _ = fs_at::clean_temp_siblings_at(&dir, &stem); + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + clean_temp_siblings_path(path); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +fn clean_temp_siblings_path(path: &Path) { let Some(parent) = path.parent() else { return; }; @@ -195,32 +295,74 @@ pub fn clean_temp_siblings(path: &Path) { } /// Writes `bytes` to `temp`, fsyncs, then atomically renames onto -/// `destination`. A failed commit removes the temp and leaves the previous -/// destination untouched (partial writes are never observable). +/// `destination`. A pre-rename failure preserves the old destination. A +/// post-rename directory-sync failure on Unix-like targets returns an error +/// with [`IoError::is_uncertain`] set because the rename may already be +/// visible. Windows skips the parent-directory sync (std cannot open a +/// directory there) and therefore claims no directory-entry durability. +/// +/// On Linux/Android the destination parent is opened as a directory fd and +/// the temp is created `O_CREAT|O_EXCL|O_NOFOLLOW`, so a pre-planted symlink +/// or a racing parent replacement cannot redirect the write. `temp` must +/// share the destination parent. Other platforms use exclusive creation only +/// (the component store itself is Linux/Android-only and fails closed +/// elsewhere). pub fn write_atomic_durably(destination: &Path, data: &[u8], temp: &Path) -> Result<(), IoError> { if let Some(parent) = destination.parent() { if !parent.as_os_str().is_empty() { std::fs::create_dir_all(parent).map_err(|e| io_error("create destination dir", e))?; } } + #[cfg(any(target_os = "linux", target_os = "android"))] + { + write_atomic_durably_at(destination, data, temp) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + write_atomic_durably_path(destination, data, temp) + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn write_atomic_durably_at(destination: &Path, data: &[u8], temp: &Path) -> Result<(), IoError> { + let destination_parent = parent_dir_owned(destination); + let temp_parent = parent_dir_owned(temp); + if destination_parent != temp_parent { + return Err(IoError::new( + "atomic commit", + "temp must share destination parent".to_owned(), + )); + } + let dir = fs_at::open_dir_path_following(&destination_parent) + .map_err(|error| io_error("open destination dir", error))?; + let destination_name = destination + .file_name() + .ok_or_else(|| IoError::new("atomic commit", "destination has no file name".to_owned()))?; + let temp_name = temp + .file_name() + .ok_or_else(|| IoError::new("atomic commit", "temp has no file name".to_owned()))?; + fs_at::write_atomic_named(&dir, destination_name, temp_name, data, usize::MAX) + .map_err(atomic_error) +} + +#[cfg(any(not(any(target_os = "linux", target_os = "android")), test))] +fn write_atomic_path_with_sync( + destination: &Path, + data: &[u8], + temp: &Path, + sync_parent: impl FnOnce(&Path) -> std::io::Result<()>, +) -> Result<(), IoError> { + let mut renamed = false; let write_result = (|| -> Result<(), std::io::Error> { use std::io::Write as _; - #[cfg(unix)] - let mut file = { - use std::os::unix::fs::OpenOptionsExt as _; - std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(temp)? - }; - #[cfg(not(unix))] - let mut file = std::fs::File::create(temp)?; + // `create_new` is O_CREAT|O_EXCL, so an existing symlink or regular + // file at the temp name is refused instead of followed/truncated. + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(temp)?; #[cfg(unix)] { - // Defense in depth: the mode is already 0600 from create; - // re-assert before the first byte so the sync covers both. use std::os::unix::fs::PermissionsExt as _; file.set_permissions(std::fs::Permissions::from_mode(0o600))?; } @@ -228,22 +370,46 @@ pub fn write_atomic_durably(destination: &Path, data: &[u8], temp: &Path) -> Res file.sync_all()?; drop(file); std::fs::rename(temp, destination)?; + renamed = true; if let Some(parent) = destination.parent() { if !parent.as_os_str().is_empty() { - if let Ok(dir) = std::fs::File::open(parent) { - let _ = dir.sync_all(); - } + sync_parent(parent)?; } } Ok(()) })(); if let Err(err) = write_result { let _ = std::fs::remove_file(temp); - return Err(io_error("atomic commit", err)); + return Err(if renamed { + IoError::uncertain("atomic commit", err.to_string()) + } else { + io_error("atomic commit", err) + }); } Ok(()) } +#[cfg(not(any(target_os = "linux", target_os = "android")))] +fn write_atomic_durably_path(destination: &Path, data: &[u8], temp: &Path) -> Result<(), IoError> { + write_atomic_path_with_sync(destination, data, temp, |parent| { + // Windows has no std-only directory fsync: `File::open` on a directory + // needs `FILE_FLAG_BACKUP_SEMANTICS`. Skip the parent sync there rather + // than failing after the rename has already published the data; that + // platform therefore makes no claim about directory-entry durability + // across power loss. Unix-like non-Linux targets sync the parent and + // report a post-rename failure as uncertain. + #[cfg(unix)] + { + std::fs::File::open(parent)?.sync_all() + } + #[cfg(not(unix))] + { + let _ = parent; + Ok(()) + } + }) +} + /// Writes `bytes` atomically to `path` (temp + fsync + rename, mode 0600). /// /// Rejects over-cap payloads before touching the filesystem, sweeps @@ -262,16 +428,28 @@ pub fn save_bytes_atomic(path: &Path, bytes: &[u8], cap: usize) -> Result<(), Io std::fs::create_dir_all(parent).map_err(|e| io_error("create destination dir", e))?; } } - clean_temp_siblings(path); - let temp = unique_temp_sibling_for(path); - // Our name is unique among live writers; dropping a stale twin only - // covers a pid-reusing predecessor's litter under the same name. - let _ = std::fs::remove_file(&temp); - let result = write_atomic_durably(path, bytes, &temp); - if result.is_err() { - let _ = std::fs::remove_file(&temp); + #[cfg(any(target_os = "linux", target_os = "android"))] + { + let parent = parent_dir_owned(path); + let dir = fs_at::open_dir_path_following(&parent) + .map_err(|error| io_error("open destination dir", error))?; + let name = path.file_name().ok_or_else(|| { + IoError::new("atomic commit", "destination has no file name".to_owned()) + })?; + let stem = name.to_string_lossy().into_owned(); + let _ = fs_at::clean_temp_siblings_at(&dir, &stem); + fs_at::write_atomic_unique(&dir, name, bytes, cap, &stem).map_err(atomic_error) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + clean_temp_siblings_path(path); + let temp = unique_temp_sibling_for(path); + let result = write_atomic_durably_path(path, bytes, &temp); + if result.is_err() { + let _ = std::fs::remove_file(&temp); + } + result } - result } /// Reads a file with a hard size cap. @@ -282,6 +460,42 @@ pub fn save_bytes_atomic(path: &Path, bytes: &[u8], cap: usize) -> Result<(), Io /// fail-closed after the first excess byte; the reported `actual` /// saturates at `cap + 1` and means "over cap", never an exact size. pub fn load_bytes_capped(path: &Path, cap: usize) -> Result, LoadError> { + #[cfg(any(target_os = "linux", target_os = "android"))] + { + load_bytes_capped_at(path, cap) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + load_bytes_capped_path(path, cap) + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn load_bytes_capped_at(path: &Path, cap: usize) -> Result, LoadError> { + let parent = parent_dir_owned(path); + let dir = fs_at::open_dir_path_following(&parent).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + LoadError::NotFound + } else { + LoadError::Io(error.to_string()) + } + })?; + let name = path + .file_name() + .ok_or_else(|| LoadError::Io("path has no file name".to_owned()))?; + match fs_at::read_file_at(&dir, name, cap) { + Ok(Some(bytes)) => Ok(bytes), + Ok(None) => Err(LoadError::NotFound), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => Err(LoadError::TooLarge { + actual: cap.saturating_add(1), + limit: cap, + }), + Err(error) => Err(LoadError::Io(error.to_string())), + } +} + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +fn load_bytes_capped_path(path: &Path, cap: usize) -> Result, LoadError> { use std::io::Read as _; let file = std::fs::File::open(path).map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { @@ -345,3 +559,84 @@ pub fn save_session_bytes(path: &Path, bytes: &[u8]) -> Result<(), IoError> { pub fn load_session_bytes(path: &Path) -> Result, LoadError> { load_bytes_capped(path, MAX_SESSION_FILE_BYTES) } + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bitty-storage-atomic-path-{}-{name}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn path_commit_post_rename_sync_failure_is_uncertain() { + let dir = scratch("post-rename"); + let destination = dir.join("session"); + let temp = dir.join("session.tmp.inject"); + let error = write_atomic_path_with_sync(&destination, b"new", &temp, |_| { + Err(std::io::Error::other("injected directory sync failure")) + }) + .expect_err("post-rename sync failure must be reported"); + assert!( + error.is_uncertain(), + "a rename that may already be visible must be uncertain" + ); + assert_eq!(std::fs::read(&destination).unwrap(), b"new"); + assert!( + !temp.exists(), + "the temp must not survive the failed commit" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn path_commit_pre_rename_failure_is_not_uncertain() { + let dir = scratch("pre-rename"); + let destination = dir.join("session"); + std::fs::create_dir(&destination).unwrap(); + let temp = dir.join("session.tmp.inject"); + let error = write_atomic_path_with_sync(&destination, b"new", &temp, |_| { + panic!("directory sync must not run before a successful rename") + }) + .expect_err("rename onto a directory must fail"); + assert!( + !error.is_uncertain(), + "a pre-rename failure leaves the destination intact" + ); + assert!(destination.is_dir()); + assert!(!temp.exists()); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn path_commit_success_is_not_uncertain() { + let dir = scratch("success"); + let destination = dir.join("session"); + let temp = dir.join("session.tmp.inject"); + write_atomic_path_with_sync(&destination, b"new", &temp, |parent| { + assert_eq!(parent, dir.as_path()); + Ok(()) + }) + .expect("commit"); + assert_eq!(std::fs::read(&destination).unwrap(), b"new"); + assert!(!temp.exists()); + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(windows)] + #[test] + fn windows_atomic_save_publishes_without_directory_sync() { + let dir = scratch("windows"); + let path = dir.join("session"); + save_session_bytes(&path, b"windows-bytes") + .expect("Windows commit must not fail after rename"); + assert_eq!(load_session_bytes(&path).unwrap(), b"windows-bytes"); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/src/bin/bitty-storage.rs b/src/bin/bitty-storage.rs new file mode 100644 index 0000000..34feb42 --- /dev/null +++ b/src/bin/bitty-storage.rs @@ -0,0 +1,172 @@ +//! `bitty-storage` executable component entry point. +//! +//! A bare invocation serves the v1 stdio protocol +//! ([`bitty_storage::component`]) over the host-selected storage root. +//! `--root PATH` selects the root explicitly; without it the component +//! derives `/bitty/storage` (or +//! `$HOME/.local/state/bitty/storage`) and refuses to run when neither is +//! available. `--version` reports the component version used by the +//! descriptor and release tag, and `--check-manifest` validates a packaged +//! `bitty-component.toml` with the same fail-closed parser the component +//! ships. The binary carries no installer and never publishes itself. + +#![forbid(unsafe_code)] + +use std::ffi::{OsStr, OsString}; +use std::fs::File; +use std::io::Read as _; +use std::path::PathBuf; +use std::process::ExitCode; + +use bitty_storage::ceiling::SESSION_APP_DIR_NAME; +use bitty_storage::component::descriptor::{ComponentDescriptor, MAX_DESCRIPTOR_BYTES}; +use bitty_storage::component::server::{COMPONENT_VERSION, SERVER_EXIT_INTERNAL, serve_stdio}; +use bitty_storage::component::store::StorageRoot; + +/// Usage refusal (the operator asked for something this binary does not +/// serve); mirrors the other component binaries' `EXIT_REFUSED`. +const EXIT_REFUSED: u8 = 2; + +fn usage() { + eprintln!( + "usage: bitty-storage [--root PATH] [--version | --check-manifest PATH]\n\ + \n\ + With no arguments (or only --root), serves the storage component\n\ + protocol v1 on stdin/stdout over //. stderr carries\n\ + diagnostics only. Without --root, the root is derived from\n\ + XDG_STATE_HOME (or HOME)." + ); +} + +fn check_manifest(path: &std::ffi::OsStr) -> Result<(), String> { + let file = File::open(path).map_err(|error| format!("cannot open descriptor: {error}"))?; + let mut bytes = Vec::new(); + file.take((MAX_DESCRIPTOR_BYTES as u64) + 1) + .read_to_end(&mut bytes) + .map_err(|error| format!("cannot read descriptor: {error}"))?; + let descriptor = ComponentDescriptor::parse_bytes(&bytes) + .map_err(|error| format!("invalid descriptor: {error}"))?; + descriptor + .validate_for_install() + .map_err(|error| format!("invalid descriptor: {error}"))?; + println!( + "ok {} {} protocol [{}, {}] {}", + descriptor.name, + descriptor.version, + descriptor.protocol_min, + descriptor.protocol_max, + descriptor.executable + ); + Ok(()) +} + +fn state_home_from(xdg_state_home: Option<&OsStr>, home: Option<&OsStr>) -> Option { + if let Some(value) = xdg_state_home { + let path = PathBuf::from(value); + // XDG requires absolute values; a relative one is ignored rather + // than resolved against the component's CWD. + if path.is_absolute() { + return Some(path); + } + } + let home = PathBuf::from(home?); + if !home.is_absolute() { + return None; + } + Some(home.join(".local").join("state")) +} + +fn default_root() -> Result { + let xdg = std::env::var_os("XDG_STATE_HOME"); + let home = std::env::var_os("HOME"); + let state = state_home_from(xdg.as_deref(), home.as_deref()).ok_or_else(|| { + "no usable absolute XDG_STATE_HOME or HOME for the storage root".to_owned() + })?; + Ok(state.join(SESSION_APP_DIR_NAME).join("storage")) +} + +fn serve(root: PathBuf) -> ExitCode { + match StorageRoot::new(root) { + Ok(root) => ExitCode::from(serve_stdio(root)), + Err(error) => { + eprintln!("bitty-storage: cannot open storage root: {error}"); + ExitCode::from(SERVER_EXIT_INTERNAL) + } + } +} + +fn main() -> ExitCode { + let mut args = std::env::args_os().skip(1); + match (args.next(), args.next(), args.next()) { + (None, _, _) => match default_root() { + Ok(root) => serve(root), + Err(error) => { + eprintln!("bitty-storage: {error}"); + ExitCode::from(EXIT_REFUSED) + } + }, + (Some(flag), None, _) if flag == "--version" || flag == "-V" => { + println!("bitty-storage {COMPONENT_VERSION}"); + ExitCode::SUCCESS + } + (Some(flag), None, _) if flag == "--help" || flag == "-h" => { + usage(); + ExitCode::SUCCESS + } + (Some(flag), Some(path), None) if flag == "--check-manifest" => { + match check_manifest(&path) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("bitty-storage: {error}"); + ExitCode::from(EXIT_REFUSED) + } + } + } + (Some(flag), Some(path), None) if flag == "--root" => serve(PathBuf::from(path)), + (Some(flag), None, _) if flag.to_string_lossy().starts_with("--root=") => { + let value = flag.to_string_lossy(); + let path = value.trim_start_matches("--root=").to_owned(); + if path.is_empty() { + usage(); + ExitCode::from(EXIT_REFUSED) + } else { + serve(PathBuf::from(OsString::from(path))) + } + } + _ => { + usage(); + ExitCode::from(EXIT_REFUSED) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relative_xdg_state_home_is_ignored() { + assert_eq!( + state_home_from( + Some(OsStr::new("relative/state")), + Some(OsStr::new("/home/u")) + ), + Some(PathBuf::from("/home/u/.local/state")) + ); + assert_eq!( + state_home_from( + Some(OsStr::new("relative/state")), + Some(OsStr::new("relative/home")) + ), + None + ); + } + + #[test] + fn absolute_xdg_state_home_wins() { + assert_eq!( + state_home_from(Some(OsStr::new("/var/state")), Some(OsStr::new("/home/u"))), + Some(PathBuf::from("/var/state")) + ); + } +} diff --git a/src/component/descriptor.rs b/src/component/descriptor.rs new file mode 100644 index 0000000..9bf81b1 --- /dev/null +++ b/src/component/descriptor.rs @@ -0,0 +1,589 @@ +//! `bitty-component.toml` parsing and validation (component descriptor v1). +//! +//! The descriptor is the same closed record Core installs for every native +//! component (DIR-030): one `[component]` table with exactly `name`, +//! `version`, `protocol`, `executable`, and `sha256`. This module parses it +//! with a hand-written, fail-closed reader (no TOML dependency, no escapes, +//! no multi-line values) so the component can validate the descriptor shipped +//! beside it and packaging can validate the canonical repository record +//! before rendering a per-target digest. +//! +//! # Pack policy versus install policy +//! +//! - [`ComponentDescriptor::parse`] accepts the zero-digest sentinel used by +//! the canonical repository record; packaging replaces it with the real +//! target digest. +//! - [`ComponentDescriptor::validate_for_install`] refuses the sentinel and +//! requires the identity of this component (`storage`, protocol 1, +//! executable `bitty-storage`). +//! +//! Versions are the strict `X.Y.Z` core only. Core's installer accepts full +//! semver; packaging and this component deliberately refuse prerelease or +//! build metadata for the 0.0.x line, matching the other extension +//! packagers. + +use std::fmt; + +use super::COMPONENT_VERSION; + +/// Component name carried by the descriptor, the handshake, and the release +/// layout (`extensions/storage//`). +pub const COMPONENT_NAME: &str = "storage"; + +/// Executable base name (no platform suffix in the descriptor). +pub const COMPONENT_EXECUTABLE: &str = "bitty-storage"; + +/// Lowest wire protocol version this component speaks. +pub const COMPONENT_PROTOCOL_MIN: u16 = 1; + +/// Highest wire protocol version this component speaks. +pub const COMPONENT_PROTOCOL_MAX: u16 = 1; + +/// Maximum descriptor size accepted from disk or an archive (matches Core's +/// `COMPONENT_DESCRIPTOR_MAX_BYTES`). +pub const MAX_DESCRIPTOR_BYTES: usize = 4096; + +/// Parser-valid placeholder digest used by the canonical repository record. +/// +/// Packaging renders a per-target copy with the real executable digest; the +/// placeholder itself must never be installed or published. +pub const PLACEHOLDER_SHA256: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; + +const SHA256_HEX_LEN: usize = 64; + +/// Executable base-name prefix every component descriptor must carry. +const EXECUTABLE_PREFIX: &str = "bitty-"; + +/// A validated `bitty-component.toml` record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ComponentDescriptor { + /// Component name (`[a-z][a-z0-9-]{0,31}`). + pub name: String, + /// Strict `X.Y.Z` version. + pub version: String, + /// Lowest supported wire protocol version. + pub protocol_min: u16, + /// Highest supported wire protocol version. + pub protocol_max: u16, + /// Executable base name (`bitty-`, no path separators). + pub executable: String, + /// Lowercase hex SHA-256 of the executable. + pub sha256: String, +} + +/// Descriptor-level validation failure (owned; no TOML type escapes). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DescriptorError { + /// The descriptor exceeds [`MAX_DESCRIPTOR_BYTES`]. + TooLarge { + /// Observed byte length. + bytes: usize, + }, + /// The bytes are not UTF-8. + NotUtf8, + /// Malformed descriptor syntax at a 1-based line. + Syntax { + /// Line number. + line: usize, + /// What was wrong. + reason: &'static str, + }, + /// A required key is missing. + MissingKey(&'static str), + /// The name violates `[a-z][a-z0-9-]{0,31}`. + InvalidName, + /// The version is not strict `X.Y.Z`. + InvalidVersion, + /// The executable name is not `bitty-` or contains separators. + InvalidExecutable, + /// The digest is not 64 lowercase hex characters. + InvalidDigest, + /// `protocol` is not `[min, max]` with `1 <= min <= max`. + InvalidProtocol, + /// The record describes a different component. + IdentityMismatch { + /// Field that disagreed. + field: &'static str, + }, + /// The record still carries the packaging placeholder digest. + PlaceholderDigest, +} + +impl fmt::Display for DescriptorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TooLarge { bytes } => { + write!( + f, + "descriptor is {bytes} bytes (limit {MAX_DESCRIPTOR_BYTES})" + ) + } + Self::NotUtf8 => f.write_str("descriptor is not UTF-8"), + Self::Syntax { line, reason } => write!(f, "descriptor line {line}: {reason}"), + Self::MissingKey(key) => write!(f, "descriptor is missing '{key}'"), + Self::InvalidName => f.write_str("component name must match [a-z][a-z0-9-]{0,31}"), + Self::InvalidVersion => f.write_str("component version must be strict X.Y.Z"), + Self::InvalidExecutable => { + f.write_str("executable must be 'bitty-' without path separators") + } + Self::InvalidDigest => f.write_str("sha256 must be 64 lowercase hex characters"), + Self::InvalidProtocol => { + f.write_str("protocol must be [min, max] with 1 <= min <= max") + } + Self::IdentityMismatch { field } => { + write!(f, "descriptor {field} does not match this component") + } + Self::PlaceholderDigest => { + f.write_str("descriptor still carries the packaging placeholder digest") + } + } + } +} + +impl std::error::Error for DescriptorError {} + +impl ComponentDescriptor { + /// Parses descriptor text (fail closed). + /// + /// # Errors + /// + /// Returns [`DescriptorError`] for oversize input, malformed syntax, + /// unknown or duplicate keys, invalid field values, or a missing key. + pub fn parse(text: &str) -> Result { + if text.len() > MAX_DESCRIPTOR_BYTES { + return Err(DescriptorError::TooLarge { bytes: text.len() }); + } + let mut in_component = false; + let mut seen_table = false; + let mut name = None; + let mut version = None; + let mut protocol = None; + let mut executable = None; + let mut sha256 = None; + for (index, raw_line) in text.lines().enumerate() { + let line_no = index + 1; + let syntax = |reason| DescriptorError::Syntax { + line: line_no, + reason, + }; + let line = strip_comment(raw_line).map_err(syntax)?.trim(); + if line.is_empty() { + continue; + } + if line.starts_with('[') { + if line != "[component]" { + return Err(syntax("only the [component] table is allowed")); + } + if seen_table { + return Err(syntax("duplicate [component] table")); + } + seen_table = true; + in_component = true; + continue; + } + if !in_component { + return Err(syntax("key outside the [component] table")); + } + let (key, value) = line + .split_once('=') + .ok_or_else(|| syntax("expected key = value"))?; + let key = key.trim(); + let value = value.trim(); + let slot = match key { + "name" => &mut name, + "version" => &mut version, + "executable" => &mut executable, + "sha256" => &mut sha256, + "protocol" => { + if protocol.is_some() { + return Err(syntax("duplicate key")); + } + protocol = Some(parse_protocol(value).map_err(syntax)?); + continue; + } + _ => return Err(syntax("unknown key")), + }; + if slot.is_some() { + return Err(syntax("duplicate key")); + } + *slot = Some(parse_basic_string(value).map_err(syntax)?); + } + let name = name.ok_or(DescriptorError::MissingKey("name"))?; + let version = version.ok_or(DescriptorError::MissingKey("version"))?; + let (protocol_min, protocol_max) = + protocol.ok_or(DescriptorError::MissingKey("protocol"))?; + let executable = executable.ok_or(DescriptorError::MissingKey("executable"))?; + let sha256 = sha256.ok_or(DescriptorError::MissingKey("sha256"))?; + + validate_component_name(&name)?; + validate_version(&version)?; + if protocol_min == 0 || protocol_min > protocol_max { + return Err(DescriptorError::InvalidProtocol); + } + if executable != format!("{EXECUTABLE_PREFIX}{name}") || !is_plain_file_name(&executable) { + return Err(DescriptorError::InvalidExecutable); + } + if sha256.len() != SHA256_HEX_LEN + || !sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(DescriptorError::InvalidDigest); + } + Ok(Self { + name, + version, + protocol_min, + protocol_max, + executable, + sha256, + }) + } + + /// Parses descriptor bytes, rejecting oversize input and invalid UTF-8. + /// + /// # Errors + /// + /// Returns [`DescriptorError`] for the same reasons as + /// [`ComponentDescriptor::parse`], plus [`DescriptorError::NotUtf8`]. + pub fn parse_bytes(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DESCRIPTOR_BYTES { + return Err(DescriptorError::TooLarge { bytes: bytes.len() }); + } + let text = std::str::from_utf8(bytes).map_err(|_| DescriptorError::NotUtf8)?; + Self::parse(text) + } + + /// Whether the descriptor's protocol range includes `version`. + #[must_use] + pub fn supports_protocol(&self, version: u16) -> bool { + (self.protocol_min..=self.protocol_max).contains(&version) + } + + /// Whether the digest is the packaging placeholder. + #[must_use] + pub fn has_placeholder_digest(&self) -> bool { + self.sha256 == PLACEHOLDER_SHA256 + } + + /// Validates that this record describes this component (`storage`, + /// executable `bitty-storage`, protocol 1). + /// + /// # Errors + /// + /// Returns [`DescriptorError::IdentityMismatch`] naming the first + /// disagreeing field, or [`DescriptorError::InvalidProtocol`] when the + /// range excludes protocol version 1. + pub fn validate_identity(&self) -> Result<(), DescriptorError> { + if self.name != COMPONENT_NAME { + return Err(DescriptorError::IdentityMismatch { field: "name" }); + } + if self.version != COMPONENT_VERSION { + return Err(DescriptorError::IdentityMismatch { field: "version" }); + } + if self.executable != COMPONENT_EXECUTABLE { + return Err(DescriptorError::IdentityMismatch { + field: "executable", + }); + } + if self.protocol_min != COMPONENT_PROTOCOL_MIN + || self.protocol_max != COMPONENT_PROTOCOL_MAX + { + return Err(DescriptorError::InvalidProtocol); + } + Ok(()) + } + + /// Validates the record for installation: identity plus a real digest. + /// + /// # Errors + /// + /// Returns the identity failures of + /// [`ComponentDescriptor::validate_identity`] and + /// [`DescriptorError::PlaceholderDigest`] while the sentinel remains. + pub fn validate_for_install(&self) -> Result<(), DescriptorError> { + self.validate_identity()?; + if self.has_placeholder_digest() { + return Err(DescriptorError::PlaceholderDigest); + } + Ok(()) + } +} + +/// Validates a component name against `[a-z][a-z0-9-]{0,31}`. +/// +/// # Errors +/// +/// Returns [`DescriptorError::InvalidName`] when the grammar is violated. +pub fn validate_component_name(name: &str) -> Result<(), DescriptorError> { + let bytes = name.as_bytes(); + let valid = !bytes.is_empty() + && bytes.len() <= 32 + && bytes[0].is_ascii_lowercase() + && bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-'); + if valid { + Ok(()) + } else { + Err(DescriptorError::InvalidName) + } +} + +/// Strict `X.Y.Z` version check: no leading zeros, no prerelease, no build +/// metadata, each part within `u32`. +fn validate_version(version: &str) -> Result<(), DescriptorError> { + if version.len() > 64 { + return Err(DescriptorError::InvalidVersion); + } + let parts: Vec<&str> = version.split('.').collect(); + if parts.len() != 3 { + return Err(DescriptorError::InvalidVersion); + } + for part in parts { + if part.is_empty() || !part.bytes().all(|b| b.is_ascii_digit()) { + return Err(DescriptorError::InvalidVersion); + } + if part.len() > 1 && part.starts_with('0') { + return Err(DescriptorError::InvalidVersion); + } + if part.len() > 10 || part.parse::().is_err() { + return Err(DescriptorError::InvalidVersion); + } + } + Ok(()) +} + +/// Non-empty, not `.`/`..`, and free of separators, NUL, and controls. +fn is_plain_file_name(value: &str) -> bool { + !value.is_empty() + && value != "." + && value != ".." + && !value + .chars() + .any(|c| c == '/' || c == '\\' || c == ':' || c.is_control() || c.is_whitespace()) +} + +fn strip_comment(line: &str) -> Result<&str, &'static str> { + let mut in_string = false; + for (index, ch) in line.char_indices() { + match ch { + '"' => in_string = !in_string, + '#' if !in_string => return Ok(&line[..index]), + _ => {} + } + } + if in_string { + return Err("unterminated string"); + } + Ok(line) +} + +/// `"..."` with no escapes and no embedded quotes. +fn parse_basic_string(value: &str) -> Result { + let inner = value + .strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + .ok_or("expected a double-quoted string")?; + if inner.contains('"') || inner.contains('\\') { + return Err("escapes and embedded quotes are not allowed"); + } + if inner.chars().any(char::is_control) { + return Err("control characters are not allowed"); + } + Ok(inner.to_owned()) +} + +/// `[min, max]` with two unsigned 16-bit integers. +fn parse_protocol(value: &str) -> Result<(u16, u16), &'static str> { + let inner = value + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .ok_or("protocol must be an array [min, max]")?; + let mut parts = inner.split(',').map(str::trim); + let (Some(min), Some(max), None) = (parts.next(), parts.next(), parts.next()) else { + return Err("protocol must have exactly two entries"); + }; + let parse = |raw: &str| { + if raw.is_empty() || !raw.bytes().all(|b| b.is_ascii_digit()) { + return Err("protocol entries must be unsigned integers"); + } + raw.parse::() + .map_err(|_| "protocol entry out of range") + }; + Ok((parse(min)?, parse(max)?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DIGEST: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + fn text(name: &str, version: &str, protocol: &str, executable: &str, sha: &str) -> String { + format!( + "# record\n[component]\nname = \"{name}\" # grammar\nversion = \"{version}\"\nprotocol = {protocol}\nexecutable = \"{executable}\"\nsha256 = \"{sha}\"\n" + ) + } + + #[test] + fn parses_the_component_record() { + let descriptor = ComponentDescriptor::parse(&text( + "storage", + "0.0.1", + "[1, 1]", + "bitty-storage", + DIGEST, + )) + .expect("valid"); + assert_eq!(descriptor.name, "storage"); + assert_eq!(descriptor.version, "0.0.1"); + assert_eq!((descriptor.protocol_min, descriptor.protocol_max), (1, 1)); + assert!(descriptor.supports_protocol(1)); + assert!(!descriptor.has_placeholder_digest()); + descriptor.validate_for_install().expect("installable"); + } + + #[test] + fn repository_record_uses_the_placeholder_digest() { + let descriptor = ComponentDescriptor::parse(&text( + "storage", + "0.0.1", + "[1, 1]", + "bitty-storage", + PLACEHOLDER_SHA256, + )) + .expect("placeholder parses"); + assert!(descriptor.has_placeholder_digest()); + assert_eq!( + descriptor.validate_for_install(), + Err(DescriptorError::PlaceholderDigest) + ); + } + + #[test] + fn identity_is_checked() { + let wrong_name = + ComponentDescriptor::parse(&text("net", "0.0.1", "[1, 1]", "bitty-net", DIGEST)) + .expect("parses"); + assert_eq!( + wrong_name.validate_identity(), + Err(DescriptorError::IdentityMismatch { field: "name" }) + ); + + let wrong_protocol = ComponentDescriptor::parse(&text( + "storage", + "0.0.1", + "[2, 3]", + "bitty-storage", + DIGEST, + )) + .expect("parses"); + assert_eq!( + wrong_protocol.validate_identity(), + Err(DescriptorError::InvalidProtocol) + ); + + let wrong_version = ComponentDescriptor::parse(&text( + "storage", + "9.9.9", + "[1, 1]", + "bitty-storage", + DIGEST, + )) + .expect("parses"); + assert_eq!( + wrong_version.validate_identity(), + Err(DescriptorError::IdentityMismatch { field: "version" }) + ); + } + + #[test] + fn rejects_unknown_duplicate_and_missing_keys() { + let unknown = + text("storage", "0.0.1", "[1, 1]", "bitty-storage", DIGEST) + "extra = \"x\"\n"; + assert!(matches!( + ComponentDescriptor::parse(&unknown), + Err(DescriptorError::Syntax { .. }) + )); + + let duplicate = "[component]\nname = \"storage\"\nname = \"storage\"\n"; + assert!(matches!( + ComponentDescriptor::parse(duplicate), + Err(DescriptorError::Syntax { .. }) + )); + + let missing = "[component]\nname = \"storage\"\n"; + assert_eq!( + ComponentDescriptor::parse(missing), + Err(DescriptorError::MissingKey("version")) + ); + } + + #[test] + fn rejects_invalid_fields() { + assert_eq!( + ComponentDescriptor::parse(&text( + "Storage", + "0.0.1", + "[1, 1]", + "bitty-Storage", + DIGEST + )), + Err(DescriptorError::InvalidName) + ); + assert_eq!( + ComponentDescriptor::parse(&text("storage", "0.0", "[1, 1]", "bitty-storage", DIGEST)), + Err(DescriptorError::InvalidVersion) + ); + assert_eq!( + ComponentDescriptor::parse(&text( + "storage", + "00.0.1", + "[1, 1]", + "bitty-storage", + DIGEST + )), + Err(DescriptorError::InvalidVersion) + ); + assert_eq!( + ComponentDescriptor::parse(&text( + "storage", + "0.0.1", + "[1, 1]", + "../bitty-storage", + DIGEST + )), + Err(DescriptorError::InvalidExecutable) + ); + assert_eq!( + ComponentDescriptor::parse(&text( + "storage", + "0.0.1", + "[2, 1]", + "bitty-storage", + DIGEST + )), + Err(DescriptorError::InvalidProtocol) + ); + assert_eq!( + ComponentDescriptor::parse(&text("storage", "0.0.1", "[1, 1]", "bitty-storage", "ABC")), + Err(DescriptorError::InvalidDigest) + ); + } + + #[test] + fn rejects_oversize_and_non_utf8_input() { + let mut oversize = String::from("#"); + oversize.push_str(&"x".repeat(MAX_DESCRIPTOR_BYTES)); + assert!(matches!( + ComponentDescriptor::parse(&oversize), + Err(DescriptorError::TooLarge { .. }) + )); + assert_eq!( + ComponentDescriptor::parse_bytes(&[0xff, 0xfe]), + Err(DescriptorError::NotUtf8) + ); + } +} diff --git a/src/component/mod.rs b/src/component/mod.rs new file mode 100644 index 0000000..91b452b --- /dev/null +++ b/src/component/mod.rs @@ -0,0 +1,46 @@ +//! Executable-component boundary for bounded storage mechanics. +//! +//! The crate still exposes the in-process [`StorageBackend`](crate::StorageBackend), +//! [`KvStore`](crate::KvStore), and session codec for Core's linked use; this +//! module adds the installable form required by the component split +//! (bitty#2495): a validated `bitty-component.toml`, a bounded v1 stdio wire +//! protocol, a journaled namespace store, and a server binary wrapping that +//! store. It carries no installer, no download source, no capability policy, +//! no XDG migration, and no platform packaging logic beyond the repository +//! script. +//! +//! ```text +//! Core host client ──stdio frames──▶ bitty-storage ──▶ journaled namespaces +//! (remaining work) (this module) (atomic temp+rename) +//! ``` +//! +//! The wire schema names namespaces, keys, and opaque values only. It never +//! carries a filesystem path: the host chooses the storage root when it spawns +//! the component, and every namespace path is derived from that root after +//! strict name validation. Core owns installation, digest verification, +//! capability grants, consent, safe-mode denial, migration, and restore +//! re-derivation; this component commits exactly the batch it is handed. + +pub mod descriptor; +pub mod protocol; +pub mod server; +pub mod store; + +/// Component version reported by `HelloAck`, `--version`, and the descriptor. +pub const COMPONENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub use descriptor::{ + COMPONENT_EXECUTABLE, COMPONENT_NAME, COMPONENT_PROTOCOL_MAX, COMPONENT_PROTOCOL_MIN, + ComponentDescriptor, DescriptorError, MAX_DESCRIPTOR_BYTES, PLACEHOLDER_SHA256, +}; +pub use protocol::{ + ErrorKind, FrameReader, MAX_BATCH_BYTES, MAX_BATCH_ROWS, MAX_FRAME_BYTES, MAX_KEY_BYTES, + MAX_NAMESPACE_NAME_BYTES, MAX_NAMESPACES, MAX_READ_BYTES, MAX_READ_ROWS, MAX_ROWS, + MAX_SNAPSHOT_BYTES, MAX_VALUE_BYTES, Message, PROTOCOL_VERSION, Row, WireError, decode_message, + encode_frame, encode_message, negotiate, write_frame, +}; +pub use server::{SERVER_EXIT_INTERNAL, SERVER_EXIT_OK, SERVER_EXIT_PROTOCOL, serve_stdio}; +pub use store::{ + CommitOutcome, Namespace, NamespaceName, RollbackOutcome, SnapshotPage, StorageRoot, + StoreError, StoreErrorKind, +}; diff --git a/src/component/protocol.rs b/src/component/protocol.rs new file mode 100644 index 0000000..185c3e6 --- /dev/null +++ b/src/component/protocol.rs @@ -0,0 +1,1801 @@ +//! Storage component wire protocol v1 (core host <-> `bitty-storage`). +//! +//! The component is a stdio coprocess: the host writes requests to its stdin +//! and reads framed messages from its stdout. stderr stays diagnostic and is +//! never protocol. This module is the authoritative schema and codec; it is +//! hand-written (`std` only), forbids panics on hostile input, and fails +//! closed on every bound. +//! +//! # Framing +//! +//! ```text +//! frame = length:u32be payload[length] +//! payload = tag:u8 fields... +//! ``` +//! +//! `length` is 1..=[`MAX_FRAME_BYTES`]. A larger or zero length is rejected +//! from the four-byte header alone; no payload byte is read or allocated. +//! +//! # Field encoding +//! +//! - Integers are big-endian fixed width (`u8`, `u16`, `u32`, `u64`). +//! - `bool` is one byte, `0x00` or `0x01`; any other value is rejected. +//! - `str` is `len:u32be` + UTF-8 bytes, every length bounded per field. +//! - `bytes` is `len:u32be` + raw bytes, bounded per field. +//! - Lists are `count:u32be` + elements, count bounded per field. +//! +//! # Messages +//! +//! | Tag | Message | Fields (in order) | Direction | +//! |--------|-----------------|--------------------------------------------------------------------------------|------------| +//! | `0x01` | `Hello` | `min:u16 max:u16 component:str version:str` | host->comp | +//! | `0x02` | `HelloAck` | `protocol:u16 component:str version:str` | comp->host | +//! | `0x10` | `OpenNamespace` | `request:u64 namespace:str` | host->comp | +//! | `0x11` | `ReadSnapshot` | `request:u64 namespace:str start_row:u32 max_rows:u32 max_bytes:u32` | host->comp | +//! | `0x12` | `CommitBatch` | `request:u64 namespace:str expected_revision:u64 upserts:list deletes:list` | host->comp | +//! | `0x13` | `Rollback` | `request:u64 namespace:str expected_revision:u64` | host->comp | +//! | `0x3F` | `Shutdown` | (none) | host->comp | +//! | `0x20` | `NamespaceOpened` | `request:u64 namespace:str revision:u64 rows:u32 bytes:u64` | comp->host | +//! | `0x21` | `SnapshotChunk` | `request:u64 namespace:str revision:u64 start_row:u32 rows:list total_rows:u32 total_bytes:u64 done:bool` | comp->host | +//! | `0x22` | `Committed` | `request:u64 namespace:str revision:u64 rows_written:u32 rows_removed:u32 bytes_written:u64 total_rows:u32 total_bytes:u64` | comp->host | +//! | `0x23` | `RolledBack` | `request:u64 namespace:str revision:u64 restored_revision:u64 total_rows:u32 total_bytes:u64` | comp->host | +//! | `0x2F` | `Error` | `request:u64 kind:u8 message:str` | comp->host | +//! | `0x3E` | `ShutdownAck` | (none) | comp->host | +//! +//! A `Row` is `key:str value:bytes`. Rows are ordered by key in snapshots and +//! in every response; a batch names the rows it replaces or removes, never a +//! whole-file image. `request` ids are non-zero and identify one exchange; +//! `Error.request == 0` marks a connection-level failure. +//! +//! # Authority and capability boundary +//! +//! This protocol carries **no filesystem paths, no descriptor passing, and no +//! command language**. A request names a namespace, a key, and opaque bytes; +//! the host chooses the storage root when it spawns the component. Namespace +//! and key grammars reject separators, dot segments, controls, and non-ASCII +//! bytes, so a wire value can never escape the storage root. Error messages +//! are bounded static diagnostics, never echoed keys, values, or paths. The +//! host owns installation, digest verification, capability grants, consent, +//! migration, and safe-mode denial; the component stores exactly what it is +//! handed. +//! +//! # Bounds +//! +//! [`MAX_FRAME_BYTES`], [`MAX_SNAPSHOT_BYTES`], [`MAX_ROWS`], +//! [`MAX_BATCH_ROWS`], [`MAX_BATCH_BYTES`], [`MAX_VALUE_BYTES`], +//! [`MAX_READ_ROWS`], [`MAX_READ_BYTES`], and [`MAX_NAMESPACES`] are enforced +//! on both encode and decode. Reads are paged so a 1 MiB snapshot never needs +//! a frame larger than 256 KiB. + +use std::collections::BTreeSet; +use std::fmt; +use std::io::{self, Read, Write}; + +use crate::component::descriptor::COMPONENT_NAME; + +/// Wire protocol version implemented by this crate. +pub const PROTOCOL_VERSION: u16 = 1; + +/// Size of the big-endian `u32` length prefix in front of every payload. +pub const FRAME_HEADER_BYTES: usize = 4; + +/// Maximum payload length of one frame (256 KiB, matching the other +/// component codecs). +pub const MAX_FRAME_BYTES: usize = 256 * 1024; + +/// Maximum namespaces one component process holds open (and locks). +pub const MAX_NAMESPACES: usize = 64; + +/// Maximum namespace-name bytes (`[a-z0-9][a-z0-9._-]{0,63}`). +pub const MAX_NAMESPACE_NAME_BYTES: usize = 64; + +/// Maximum key bytes (`[a-z0-9][a-z0-9._-]{0,127}`). +pub const MAX_KEY_BYTES: usize = 128; + +/// Maximum bytes of one row value (one frame chunk, not a whole snapshot). +pub const MAX_VALUE_BYTES: usize = 128 * 1024; + +/// Maximum rows in one namespace snapshot. +pub const MAX_ROWS: usize = 4096; + +/// Maximum serialized snapshot bytes (matches Core's session-file ceiling). +pub const MAX_SNAPSHOT_BYTES: usize = 1_048_576; + +/// Maximum upserts or deletes in one `CommitBatch`. +pub const MAX_BATCH_ROWS: usize = 128; + +/// Maximum encoded row/delete payload bytes in one `CommitBatch`. +pub const MAX_BATCH_BYTES: usize = 240 * 1024; + +/// Maximum rows one `SnapshotChunk` carries. +pub const MAX_READ_ROWS: usize = 128; + +/// Maximum row bytes one `SnapshotChunk` carries. +pub const MAX_READ_BYTES: usize = 240 * 1024; + +/// Maximum component name bytes (`[a-z][a-z0-9-]{0,31}`). +pub const MAX_COMPONENT_NAME_BYTES: usize = 32; + +/// Maximum version string bytes in `Hello` / `HelloAck`. +pub const MAX_VERSION_BYTES: usize = 64; + +/// Maximum `Error.message` bytes. +pub const MAX_ERROR_MESSAGE_BYTES: usize = 256; + +// ── tags ──────────────────────────────────────────────────────────────────── + +/// `Hello` tag. +pub const TAG_HELLO: u8 = 0x01; +/// `HelloAck` tag. +pub const TAG_HELLO_ACK: u8 = 0x02; +/// `OpenNamespace` tag. +pub const TAG_OPEN_NAMESPACE: u8 = 0x10; +/// `ReadSnapshot` tag. +pub const TAG_READ_SNAPSHOT: u8 = 0x11; +/// `CommitBatch` tag. +pub const TAG_COMMIT_BATCH: u8 = 0x12; +/// `Rollback` tag. +pub const TAG_ROLLBACK: u8 = 0x13; +/// `Shutdown` tag. +pub const TAG_SHUTDOWN: u8 = 0x3F; +/// `NamespaceOpened` tag. +pub const TAG_NAMESPACE_OPENED: u8 = 0x20; +/// `SnapshotChunk` tag. +pub const TAG_SNAPSHOT_CHUNK: u8 = 0x21; +/// `Committed` tag. +pub const TAG_COMMITTED: u8 = 0x22; +/// `RolledBack` tag. +pub const TAG_ROLLED_BACK: u8 = 0x23; +/// `Error` tag. +pub const TAG_ERROR: u8 = 0x2F; +/// `ShutdownAck` tag. +pub const TAG_SHUTDOWN_ACK: u8 = 0x3E; + +/// Connection-level `Error` frame request id. +pub const CONNECTION_ID: u64 = 0; + +// ── values ────────────────────────────────────────────────────────────────── + +/// One namespace row: an identifier plus opaque bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + /// Row key (`[a-z0-9][a-z0-9._-]{0,127}`). + pub key: String, + /// Opaque value bytes (at most [`MAX_VALUE_BYTES`]). + pub value: Vec, +} + +/// Typed error category carried by `Error`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ErrorKind { + /// The peer violated the protocol (code 1). + Protocol, + /// A request was malformed or violated a bound (code 2). + InvalidRequest, + /// A capacity or byte budget is exhausted (code 3). + Budget, + /// The operation is not supported on this component/platform (code 4). + Unsupported, + /// The namespace was never opened on this connection (code 5). + UnknownNamespace, + /// `expected_revision` does not match the current revision (code 6). + StaleRevision, + /// Another process holds the namespace lock (code 7). + Locked, + /// A snapshot or journal is structurally invalid (code 8). + CorruptState, + /// A filesystem operation failed (code 9). A commit or rollback `io` is a + /// pre-rename failure, so retrying the same revision is safe; an + /// open/read `io` is an ordinary I/O failure. + Io, + /// A commit may have completed but its durable outcome could not be + /// confirmed, or the namespace path stopped naming the opened directory; + /// the host must reopen and reread before retrying (code 11). + Uncertain, + /// Internal component failure (code 10). + Internal, +} + +// Stable wire codes are 1 protocol, 2 invalid_request, 3 budget, 4 +// unsupported, 5 unknown_namespace, 6 stale_revision, 7 locked, 8 +// corrupt_state, 9 io, 10 internal, and 11 uncertain. +// +// The table is additive within protocol v1: existing codes never move, so a +// v1 host that predates `uncertain` still decodes every code it knows. Such a +// host receives 11 as an unknown kind and must fail closed (treat it as +// non-retryable) rather than retrying from a cached revision; a host that +// knows `uncertain` reopens the namespace and rereads its revision first. +// `internal` stays at 10 so this addition cannot silently renumber it. + +impl ErrorKind { + /// Wire code. + #[must_use] + pub const fn code(self) -> u8 { + match self { + Self::Protocol => 1, + Self::InvalidRequest => 2, + Self::Budget => 3, + Self::Unsupported => 4, + Self::UnknownNamespace => 5, + Self::StaleRevision => 6, + Self::Locked => 7, + Self::CorruptState => 8, + Self::Io => 9, + Self::Internal => 10, + Self::Uncertain => 11, + } + } + + /// Kind for a wire code, or `None` for an unknown code. + #[must_use] + pub const fn from_code(code: u8) -> Option { + match code { + 1 => Some(Self::Protocol), + 2 => Some(Self::InvalidRequest), + 3 => Some(Self::Budget), + 4 => Some(Self::Unsupported), + 5 => Some(Self::UnknownNamespace), + 6 => Some(Self::StaleRevision), + 7 => Some(Self::Locked), + 8 => Some(Self::CorruptState), + 9 => Some(Self::Io), + 10 => Some(Self::Internal), + 11 => Some(Self::Uncertain), + _ => None, + } + } + + /// Stable snake_case name. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Protocol => "protocol", + Self::InvalidRequest => "invalid_request", + Self::Budget => "budget", + Self::Unsupported => "unsupported", + Self::UnknownNamespace => "unknown_namespace", + Self::StaleRevision => "stale_revision", + Self::Locked => "locked", + Self::CorruptState => "corrupt_state", + Self::Io => "io", + Self::Uncertain => "uncertain", + Self::Internal => "internal", + } + } +} + +impl fmt::Display for ErrorKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +// ── messages ──────────────────────────────────────────────────────────────── + +/// One protocol v1 message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Message { + /// Handshake offer; must be the first frame (host -> component). + Hello { + /// Lowest protocol version the host supports. + min: u16, + /// Highest protocol version the host supports. + max: u16, + /// Component name the host expects. + component: String, + /// Host version string. + version: String, + }, + /// Handshake answer (component -> host). + HelloAck { + /// Negotiated protocol version. + protocol: u16, + /// Component name. + component: String, + /// Component version. + version: String, + }, + /// Open (create when absent) and lock one namespace (host -> component). + OpenNamespace { + /// Non-zero request id for this exchange. + request: u64, + /// Namespace name. + namespace: String, + }, + /// Read one page of the current committed snapshot (host -> component). + ReadSnapshot { + /// Non-zero request id for this exchange. + request: u64, + /// Open namespace name. + namespace: String, + /// Zero-based first row index in key order. + start_row: u32, + /// Maximum rows to return; `0` means [`MAX_READ_ROWS`]. + max_rows: u32, + /// Maximum row-value bytes to return; `0` means [`MAX_READ_BYTES`]. + max_bytes: u32, + }, + /// Atomically apply one batch to a namespace (host -> component). + CommitBatch { + /// Non-zero request id for this exchange. + request: u64, + /// Open namespace name. + namespace: String, + /// Revision the host read before building the batch; `0` for a + /// namespace that has never been committed. + expected_revision: u64, + /// Rows to insert or replace. + upserts: Vec, + /// Keys to remove. + deletes: Vec, + }, + /// Restore the previous revision as a new revision (host -> component). + Rollback { + /// Non-zero request id for this exchange. + request: u64, + /// Open namespace name. + namespace: String, + /// Current revision the host observed. + expected_revision: u64, + }, + /// Flush, release namespace locks, and exit (host -> component). + Shutdown, + /// Namespace opened or already open (component -> host). + NamespaceOpened { + /// Request id from `OpenNamespace`. + request: u64, + /// Namespace name. + namespace: String, + /// Current committed revision (`0` when empty). + revision: u64, + /// Current row count. + rows: u32, + /// Current row-value bytes. + bytes: u64, + }, + /// One page of a committed snapshot (component -> host). + SnapshotChunk { + /// Request id from `ReadSnapshot`. + request: u64, + /// Namespace name. + namespace: String, + /// Revision the rows belong to. + revision: u64, + /// Zero-based row index of the first returned row. + start_row: u32, + /// Rows in key order. + rows: Vec, + /// Rows in the whole snapshot. + total_rows: u32, + /// Row-value bytes in the whole snapshot. + total_bytes: u64, + /// Whether this page reaches the end of the snapshot. + done: bool, + }, + /// Batch committed atomically (component -> host). + Committed { + /// Request id from `CommitBatch`. + request: u64, + /// Namespace name. + namespace: String, + /// New committed revision. + revision: u64, + /// Upserts applied. + rows_written: u32, + /// Deletes applied. + rows_removed: u32, + /// Row-value bytes written by the batch. + bytes_written: u64, + /// Row count after the commit. + total_rows: u32, + /// Row-value bytes after the commit. + total_bytes: u64, + }, + /// Previous revision restored as a new revision (component -> host). + RolledBack { + /// Request id from `Rollback`. + request: u64, + /// Namespace name. + namespace: String, + /// New committed revision. + revision: u64, + /// Revision whose contents were restored. + restored_revision: u64, + /// Row count after the rollback. + total_rows: u32, + /// Row-value bytes after the rollback. + total_bytes: u64, + }, + /// Typed error (component -> host). + Error { + /// Request id, or [`CONNECTION_ID`] for a connection-level failure. + request: u64, + /// Error category. + kind: ErrorKind, + /// Bounded static diagnostic; never echoed keys, values, or paths. + message: String, + }, + /// Namespaces released; the component exits after this frame + /// (component -> host). + ShutdownAck, +} + +impl Message { + /// Wire tag of this message. + #[must_use] + pub const fn tag(&self) -> u8 { + match self { + Self::Hello { .. } => TAG_HELLO, + Self::HelloAck { .. } => TAG_HELLO_ACK, + Self::OpenNamespace { .. } => TAG_OPEN_NAMESPACE, + Self::ReadSnapshot { .. } => TAG_READ_SNAPSHOT, + Self::CommitBatch { .. } => TAG_COMMIT_BATCH, + Self::Rollback { .. } => TAG_ROLLBACK, + Self::Shutdown => TAG_SHUTDOWN, + Self::NamespaceOpened { .. } => TAG_NAMESPACE_OPENED, + Self::SnapshotChunk { .. } => TAG_SNAPSHOT_CHUNK, + Self::Committed { .. } => TAG_COMMITTED, + Self::RolledBack { .. } => TAG_ROLLED_BACK, + Self::Error { .. } => TAG_ERROR, + Self::ShutdownAck => TAG_SHUTDOWN_ACK, + } + } +} + +// ── grammar ───────────────────────────────────────────────────────────────── + +/// Whether `name` matches `[a-z0-9][a-z0-9._-]{0,63}` with no dot segment. +#[must_use] +pub fn valid_namespace_name(name: &str) -> bool { + valid_name(name, MAX_NAMESPACE_NAME_BYTES) +} + +/// Whether `key` matches `[a-z0-9][a-z0-9._-]{0,127}` with no dot segment. +#[must_use] +pub fn valid_key(key: &str) -> bool { + valid_name(key, MAX_KEY_BYTES) +} + +fn valid_name(value: &str, max: usize) -> bool { + let bytes = value.as_bytes(); + if bytes.is_empty() || bytes.len() > max { + return false; + } + let first = bytes[0]; + if !(first.is_ascii_lowercase() || first.is_ascii_digit()) { + return false; + } + if !bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) + { + return false; + } + // Reject empty dot segments (`..`) so a name can never read as a path + // traversal even before it is joined to the storage root. + !value.contains("..") +} + +// ── errors ────────────────────────────────────────────────────────────────── + +/// Wire codec failure (owned; no `io::Error` escapes). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WireError { + /// The underlying stream failed. + Io(io::ErrorKind), + /// A frame exceeds [`MAX_FRAME_BYTES`] (or an encoder field exceeds its + /// bound). + TooLarge { + /// Observed length. + len: usize, + }, + /// A zero-length frame. + EmptyFrame, + /// A frame ended before a field was complete. + Truncated, + /// A frame carried bytes after the last field. + TrailingBytes, + /// The tag is not assigned in protocol v1. + UnknownTag(u8), + /// A string field is not UTF-8. + InvalidUtf8, + /// A field violates its grammar or bound. + InvalidField(&'static str), +} + +impl fmt::Display for WireError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(kind) => write!(f, "stream i/o error: {kind}"), + Self::TooLarge { len } => { + write!(f, "frame is {len} bytes (limit {MAX_FRAME_BYTES})") + } + Self::EmptyFrame => f.write_str("empty frame"), + Self::Truncated => f.write_str("truncated frame"), + Self::TrailingBytes => f.write_str("trailing bytes after the last field"), + Self::UnknownTag(tag) => write!(f, "unknown tag 0x{tag:02x}"), + Self::InvalidUtf8 => f.write_str("string field is not UTF-8"), + Self::InvalidField(reason) => write!(f, "invalid field: {reason}"), + } + } +} + +impl std::error::Error for WireError {} + +// ── framing ───────────────────────────────────────────────────────────────── + +/// Reads length-prefixed frames from a stream. +pub struct FrameReader { + reader: R, +} + +impl FrameReader { + /// Wraps `reader`. + pub const fn new(reader: R) -> Self { + Self { reader } + } + + /// Reads one frame payload, or `None` at a clean EOF between frames. + /// + /// The four-byte header is rejected before any payload allocation or + /// read when the declared length is zero or over [`MAX_FRAME_BYTES`]. + /// + /// # Errors + /// + /// Returns [`WireError`] for I/O failures, a partial header, an empty or + /// oversize frame, or a truncated payload. + pub fn read_frame(&mut self) -> Result>, WireError> { + let mut header = [0u8; FRAME_HEADER_BYTES]; + let mut filled = 0; + while filled < header.len() { + match self.reader.read(&mut header[filled..]) { + Ok(0) if filled == 0 => return Ok(None), + Ok(0) => return Err(WireError::Truncated), + Ok(n) => filled += n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(WireError::Io(error.kind())), + } + } + let len = u32::from_be_bytes(header) as usize; + if len == 0 { + return Err(WireError::EmptyFrame); + } + if len > MAX_FRAME_BYTES { + return Err(WireError::TooLarge { len }); + } + let mut payload = vec![0u8; len]; + self.reader + .read_exact(&mut payload) + .map_err(|error| match error.kind() { + io::ErrorKind::UnexpectedEof => WireError::Truncated, + kind => WireError::Io(kind), + })?; + Ok(Some(payload)) + } + + /// Reads and decodes one message, or `None` at a clean EOF. + /// + /// # Errors + /// + /// Returns [`WireError`] for framing failures (see + /// [`FrameReader::read_frame`]) and decode failures. + pub fn read_message(&mut self) -> Result, WireError> { + self.read_frame()? + .map(|payload| decode_message(&payload)) + .transpose() + } +} + +/// Encodes one complete frame (length prefix included). +/// +/// # Errors +/// +/// Returns [`WireError`] when a field violates its bound or the payload +/// exceeds [`MAX_FRAME_BYTES`]. +pub fn encode_frame(message: &Message) -> Result, WireError> { + let payload = encode_message(message)?; + let len = + u32::try_from(payload.len()).map_err(|_| WireError::TooLarge { len: payload.len() })?; + let mut frame = Vec::with_capacity(FRAME_HEADER_BYTES + payload.len()); + frame.extend_from_slice(&len.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +/// Writes one complete frame in a single `write_all`. +/// +/// # Errors +/// +/// Returns [`WireError`] for encode or I/O failures. +pub fn write_frame(writer: &mut W, message: &Message) -> Result<(), WireError> { + let frame = encode_frame(message)?; + writer + .write_all(&frame) + .map_err(|error| WireError::Io(error.kind())) +} + +/// Encodes one message payload (tag included, no length prefix). +/// +/// # Errors +/// +/// Returns [`WireError`] when a field violates its bound or the payload +/// exceeds [`MAX_FRAME_BYTES`]. +pub fn encode_message(message: &Message) -> Result, WireError> { + let mut encoder = Encoder::default(); + match message { + Message::Hello { + min, + max, + component, + version, + } => { + encoder.u8(TAG_HELLO); + encoder.u16(*min); + encoder.u16(*max); + encoder.component_name(component)?; + encoder.version(version)?; + } + Message::HelloAck { + protocol, + component, + version, + } => { + encoder.u8(TAG_HELLO_ACK); + encoder.u16(*protocol); + encoder.component_name(component)?; + encoder.version(version)?; + } + Message::OpenNamespace { request, namespace } => { + encoder.u8(TAG_OPEN_NAMESPACE); + encoder.request(*request)?; + encoder.namespace(namespace)?; + } + Message::ReadSnapshot { + request, + namespace, + start_row, + max_rows, + max_bytes, + } => { + if *max_rows as usize > MAX_READ_ROWS { + return Err(WireError::InvalidField("max_rows")); + } + if *max_bytes as usize > MAX_READ_BYTES { + return Err(WireError::InvalidField("max_bytes")); + } + encoder.u8(TAG_READ_SNAPSHOT); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u32(*start_row); + encoder.u32(*max_rows); + encoder.u32(*max_bytes); + } + Message::CommitBatch { + request, + namespace, + expected_revision, + upserts, + deletes, + } => { + encoder.u8(TAG_COMMIT_BATCH); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*expected_revision); + encoder.rows(upserts)?; + encoder.delete_keys(deletes)?; + } + Message::Rollback { + request, + namespace, + expected_revision, + } => { + encoder.u8(TAG_ROLLBACK); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*expected_revision); + } + Message::Shutdown => encoder.u8(TAG_SHUTDOWN), + Message::NamespaceOpened { + request, + namespace, + revision, + rows, + bytes, + } => { + if *rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("rows")); + } + if *bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("bytes")); + } + encoder.u8(TAG_NAMESPACE_OPENED); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*revision); + encoder.u32(*rows); + encoder.u64(*bytes); + } + Message::SnapshotChunk { + request, + namespace, + revision, + start_row, + rows, + total_rows, + total_bytes, + done, + } => { + if *total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if *total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + encoder.u8(TAG_SNAPSHOT_CHUNK); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*revision); + encoder.u32(*start_row); + encoder.response_rows(rows)?; + encoder.u32(*total_rows); + encoder.u64(*total_bytes); + encoder.u8(u8::from(*done)); + } + Message::Committed { + request, + namespace, + revision, + rows_written, + rows_removed, + bytes_written, + total_rows, + total_bytes, + } => { + if *total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if *rows_written as usize > MAX_BATCH_ROWS || *rows_removed as usize > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("batch rows")); + } + if *bytes_written > MAX_BATCH_BYTES as u64 { + return Err(WireError::InvalidField("bytes_written")); + } + if *total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + encoder.u8(TAG_COMMITTED); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*revision); + encoder.u32(*rows_written); + encoder.u32(*rows_removed); + encoder.u64(*bytes_written); + encoder.u32(*total_rows); + encoder.u64(*total_bytes); + } + Message::RolledBack { + request, + namespace, + revision, + restored_revision, + total_rows, + total_bytes, + } => { + if *total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if *total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + encoder.u8(TAG_ROLLED_BACK); + encoder.request(*request)?; + encoder.namespace(namespace)?; + encoder.u64(*revision); + encoder.u64(*restored_revision); + encoder.u32(*total_rows); + encoder.u64(*total_bytes); + } + Message::Error { + request, + kind, + message, + } => { + encoder.u8(TAG_ERROR); + encoder.u64(*request); + encoder.u8(kind.code()); + encoder.diagnostic(message)?; + } + Message::ShutdownAck => encoder.u8(TAG_SHUTDOWN_ACK), + } + let payload = encoder.finish(); + if payload.len() > MAX_FRAME_BYTES { + return Err(WireError::TooLarge { len: payload.len() }); + } + Ok(payload) +} + +/// Decodes one message payload (tag included, no length prefix). +/// +/// # Errors +/// +/// Returns [`WireError`] for an unknown tag, truncation, trailing bytes, +/// invalid UTF-8, unknown enum codes, zero request ids, invalid namespace or +/// key grammar, or any bound violation. +pub fn decode_message(payload: &[u8]) -> Result { + let mut decoder = Decoder::new(payload); + let tag = decoder.u8()?; + let message = match tag { + TAG_HELLO => Message::Hello { + min: decoder.u16()?, + max: decoder.u16()?, + component: decoder.component_name()?, + version: decoder.version()?, + }, + TAG_HELLO_ACK => Message::HelloAck { + protocol: decoder.u16()?, + component: decoder.component_name()?, + version: decoder.version()?, + }, + TAG_OPEN_NAMESPACE => Message::OpenNamespace { + request: decoder.request()?, + namespace: decoder.namespace()?, + }, + TAG_READ_SNAPSHOT => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let start_row = decoder.u32()?; + let max_rows = decoder.u32()?; + let max_bytes = decoder.u32()?; + if max_rows as usize > MAX_READ_ROWS { + return Err(WireError::InvalidField("max_rows")); + } + if max_bytes as usize > MAX_READ_BYTES { + return Err(WireError::InvalidField("max_bytes")); + } + Message::ReadSnapshot { + request, + namespace, + start_row, + max_rows, + max_bytes, + } + } + TAG_COMMIT_BATCH => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let expected_revision = decoder.u64()?; + let upserts = decoder.rows()?; + let deletes = decoder.delete_keys()?; + Message::CommitBatch { + request, + namespace, + expected_revision, + upserts, + deletes, + } + } + TAG_ROLLBACK => Message::Rollback { + request: decoder.request()?, + namespace: decoder.namespace()?, + expected_revision: decoder.u64()?, + }, + TAG_SHUTDOWN => Message::Shutdown, + TAG_NAMESPACE_OPENED => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let revision = decoder.u64()?; + let rows = decoder.u32()?; + let bytes = decoder.u64()?; + if rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("rows")); + } + if bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("bytes")); + } + Message::NamespaceOpened { + request, + namespace, + revision, + rows, + bytes, + } + } + TAG_SNAPSHOT_CHUNK => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let revision = decoder.u64()?; + let start_row = decoder.u32()?; + let rows = decoder.response_rows()?; + let total_rows = decoder.u32()?; + let total_bytes = decoder.u64()?; + let done = decoder.bool()?; + if total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + Message::SnapshotChunk { + request, + namespace, + revision, + start_row, + rows, + total_rows, + total_bytes, + done, + } + } + TAG_COMMITTED => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let revision = decoder.u64()?; + let rows_written = decoder.u32()?; + let rows_removed = decoder.u32()?; + let bytes_written = decoder.u64()?; + let total_rows = decoder.u32()?; + let total_bytes = decoder.u64()?; + if total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if rows_written as usize > MAX_BATCH_ROWS || rows_removed as usize > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("batch rows")); + } + if bytes_written > MAX_BATCH_BYTES as u64 { + return Err(WireError::InvalidField("bytes_written")); + } + if total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + Message::Committed { + request, + namespace, + revision, + rows_written, + rows_removed, + bytes_written, + total_rows, + total_bytes, + } + } + TAG_ROLLED_BACK => { + let request = decoder.request()?; + let namespace = decoder.namespace()?; + let revision = decoder.u64()?; + let restored_revision = decoder.u64()?; + let total_rows = decoder.u32()?; + let total_bytes = decoder.u64()?; + if total_rows as usize > MAX_ROWS { + return Err(WireError::InvalidField("total_rows")); + } + if total_bytes > MAX_SNAPSHOT_BYTES as u64 { + return Err(WireError::InvalidField("total_bytes")); + } + Message::RolledBack { + request, + namespace, + revision, + restored_revision, + total_rows, + total_bytes, + } + } + TAG_ERROR => { + let request = decoder.u64()?; + let kind_code = decoder.u8()?; + let kind = + ErrorKind::from_code(kind_code).ok_or(WireError::InvalidField("error kind"))?; + Message::Error { + request, + kind, + message: decoder.diagnostic()?, + } + } + TAG_SHUTDOWN_ACK => Message::ShutdownAck, + other => return Err(WireError::UnknownTag(other)), + }; + if !decoder.is_empty() { + return Err(WireError::TrailingBytes); + } + Ok(message) +} + +/// Component name this binary answers to in `Hello`. +#[must_use] +pub fn expected_component_name() -> &'static str { + COMPONENT_NAME +} + +/// Protocol version range this binary supports. +#[must_use] +pub const fn supported_protocol_range() -> (u16, u16) { + (PROTOCOL_VERSION, PROTOCOL_VERSION) +} + +/// Negotiates `[min, max]` against this component's range. +/// +/// Returns the highest overlapping version, or `None` when the ranges do +/// not intersect. +#[must_use] +pub fn negotiate(min: u16, max: u16) -> Option { + if min > max { + return None; + } + let low = min.max(PROTOCOL_VERSION); + let high = max.min(PROTOCOL_VERSION); + (low <= high).then_some(high) +} + +// ── encoder / decoder ─────────────────────────────────────────────────────── + +#[derive(Default)] +struct Encoder { + buffer: Vec, +} + +impl Encoder { + fn u8(&mut self, value: u8) { + self.buffer.push(value); + } + + fn u16(&mut self, value: u16) { + self.buffer.extend_from_slice(&value.to_be_bytes()); + } + + fn u32(&mut self, value: u32) { + self.buffer.extend_from_slice(&value.to_be_bytes()); + } + + fn u64(&mut self, value: u64) { + self.buffer.extend_from_slice(&value.to_be_bytes()); + } + + fn bytes(&mut self, data: &[u8]) { + self.u32(u32::try_from(data.len()).unwrap_or(u32::MAX)); + self.buffer.extend_from_slice(data); + } + + fn request(&mut self, request: u64) -> Result<(), WireError> { + if request == 0 { + return Err(WireError::InvalidField("zero request id")); + } + self.u64(request); + Ok(()) + } + + fn component_name(&mut self, value: &str) -> Result<(), WireError> { + let bytes = value.as_bytes(); + let valid = !bytes.is_empty() + && bytes.len() <= MAX_COMPONENT_NAME_BYTES + && bytes[0].is_ascii_lowercase() + && bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-'); + if !valid { + return Err(WireError::InvalidField("component name")); + } + self.bytes(bytes); + Ok(()) + } + + fn version(&mut self, value: &str) -> Result<(), WireError> { + let valid = !value.is_empty() + && value.len() <= MAX_VERSION_BYTES + && value.bytes().all(|b| b.is_ascii_graphic() || b == b' '); + if !valid { + return Err(WireError::InvalidField("version")); + } + self.bytes(value.as_bytes()); + Ok(()) + } + + fn namespace(&mut self, value: &str) -> Result<(), WireError> { + if !valid_namespace_name(value) { + return Err(WireError::InvalidField("namespace")); + } + self.bytes(value.as_bytes()); + Ok(()) + } + + fn rows(&mut self, rows: &[Row]) -> Result<(), WireError> { + if rows.len() > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("upserts")); + } + self.u32(u32::try_from(rows.len()).unwrap_or(u32::MAX)); + let mut total = 0usize; + let mut seen = BTreeSet::new(); + for row in rows { + if !valid_key(&row.key) { + return Err(WireError::InvalidField("row key")); + } + if !seen.insert(row.key.as_str()) { + return Err(WireError::InvalidField("duplicate row key")); + } + if row.value.len() > MAX_VALUE_BYTES { + return Err(WireError::InvalidField("row value")); + } + total = total.saturating_add(row.key.len() + row.value.len() + 8); + self.bytes(row.key.as_bytes()); + self.bytes(&row.value); + } + if total > MAX_BATCH_BYTES { + return Err(WireError::InvalidField("batch bytes")); + } + Ok(()) + } + + fn response_rows(&mut self, rows: &[Row]) -> Result<(), WireError> { + if rows.len() > MAX_READ_ROWS { + return Err(WireError::InvalidField("rows")); + } + self.u32(u32::try_from(rows.len()).unwrap_or(u32::MAX)); + let mut total = 0usize; + for row in rows { + if !valid_key(&row.key) { + return Err(WireError::InvalidField("row key")); + } + if row.value.len() > MAX_VALUE_BYTES { + return Err(WireError::InvalidField("row value")); + } + total = total.saturating_add(row.key.len() + row.value.len()); + self.bytes(row.key.as_bytes()); + self.bytes(&row.value); + } + if total > MAX_READ_BYTES { + return Err(WireError::InvalidField("read bytes")); + } + Ok(()) + } + + fn delete_keys(&mut self, keys: &[String]) -> Result<(), WireError> { + if keys.len() > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("deletes")); + } + self.u32(u32::try_from(keys.len()).unwrap_or(u32::MAX)); + let mut total = 0usize; + let mut seen = BTreeSet::new(); + for key in keys { + if !valid_key(key) { + return Err(WireError::InvalidField("delete key")); + } + if !seen.insert(key.as_str()) { + return Err(WireError::InvalidField("duplicate delete key")); + } + total = total.saturating_add(key.len() + 4); + self.bytes(key.as_bytes()); + } + if total > MAX_BATCH_BYTES { + return Err(WireError::InvalidField("batch bytes")); + } + Ok(()) + } + + fn diagnostic(&mut self, message: &str) -> Result<(), WireError> { + let valid = !message.is_empty() + && message.len() <= MAX_ERROR_MESSAGE_BYTES + && message.bytes().all(|b| (0x20..=0x7e).contains(&b)); + if !valid { + return Err(WireError::InvalidField("error message")); + } + self.bytes(message.as_bytes()); + Ok(()) + } + + fn finish(self) -> Vec { + self.buffer + } +} + +struct Decoder<'a> { + buffer: &'a [u8], + position: usize, +} + +impl<'a> Decoder<'a> { + const fn new(buffer: &'a [u8]) -> Self { + Self { + buffer, + position: 0, + } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], WireError> { + let end = self.position.checked_add(len).ok_or(WireError::Truncated)?; + if end > self.buffer.len() { + return Err(WireError::Truncated); + } + let slice = &self.buffer[self.position..end]; + self.position = end; + Ok(slice) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn bool(&mut self) -> Result { + match self.u8()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(WireError::InvalidField("bool")), + } + } + + fn u16(&mut self) -> Result { + let bytes: [u8; 2] = self.take(2)?.try_into().map_err(|_| WireError::Truncated)?; + Ok(u16::from_be_bytes(bytes)) + } + + fn u32(&mut self) -> Result { + let bytes: [u8; 4] = self.take(4)?.try_into().map_err(|_| WireError::Truncated)?; + Ok(u32::from_be_bytes(bytes)) + } + + fn u64(&mut self) -> Result { + let bytes: [u8; 8] = self.take(8)?.try_into().map_err(|_| WireError::Truncated)?; + Ok(u64::from_be_bytes(bytes)) + } + + fn request(&mut self) -> Result { + let request = self.u64()?; + if request == 0 { + return Err(WireError::InvalidField("zero request id")); + } + Ok(request) + } + + fn bytes(&mut self, max: usize) -> Result, WireError> { + let len = self.u32()? as usize; + if len > max { + return Err(WireError::InvalidField("byte length")); + } + Ok(self.take(len)?.to_vec()) + } + + fn str_bounded(&mut self, max: usize, field: &'static str) -> Result { + let len = self.u32()? as usize; + if len > max { + return Err(WireError::InvalidField(field)); + } + let bytes = self.take(len)?; + std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|_| WireError::InvalidUtf8) + } + + fn component_name(&mut self) -> Result { + let name = self.str_bounded(MAX_COMPONENT_NAME_BYTES, "component name")?; + let valid = !name.is_empty() + && name.as_bytes()[0].is_ascii_lowercase() + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if !valid { + return Err(WireError::InvalidField("component name")); + } + Ok(name) + } + + fn version(&mut self) -> Result { + let version = self.str_bounded(MAX_VERSION_BYTES, "version")?; + let valid = + !version.is_empty() && version.bytes().all(|b| b.is_ascii_graphic() || b == b' '); + if !valid { + return Err(WireError::InvalidField("version")); + } + Ok(version) + } + + fn namespace(&mut self) -> Result { + let namespace = self.str_bounded(MAX_NAMESPACE_NAME_BYTES, "namespace")?; + if !valid_namespace_name(&namespace) { + return Err(WireError::InvalidField("namespace")); + } + Ok(namespace) + } + + fn rows(&mut self) -> Result, WireError> { + let count = self.u32()? as usize; + if count > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("upserts")); + } + let mut rows = Vec::with_capacity(count); + let mut total = 0usize; + let mut seen = BTreeSet::new(); + for _ in 0..count { + let key = self.str_bounded(MAX_KEY_BYTES, "row key")?; + if !valid_key(&key) { + return Err(WireError::InvalidField("row key")); + } + if !seen.insert(key.clone()) { + return Err(WireError::InvalidField("duplicate row key")); + } + let value = self.bytes(MAX_VALUE_BYTES)?; + total = total.saturating_add(key.len() + value.len() + 8); + rows.push(Row { key, value }); + } + if total > MAX_BATCH_BYTES { + return Err(WireError::InvalidField("batch bytes")); + } + Ok(rows) + } + + fn response_rows(&mut self) -> Result, WireError> { + let count = self.u32()? as usize; + if count > MAX_READ_ROWS { + return Err(WireError::InvalidField("rows")); + } + let mut rows = Vec::with_capacity(count); + let mut total = 0usize; + for _ in 0..count { + let key = self.str_bounded(MAX_KEY_BYTES, "row key")?; + if !valid_key(&key) { + return Err(WireError::InvalidField("row key")); + } + let value = self.bytes(MAX_VALUE_BYTES)?; + total = total.saturating_add(key.len() + value.len()); + rows.push(Row { key, value }); + } + if total > MAX_READ_BYTES { + return Err(WireError::InvalidField("read bytes")); + } + Ok(rows) + } + + fn delete_keys(&mut self) -> Result, WireError> { + let count = self.u32()? as usize; + if count > MAX_BATCH_ROWS { + return Err(WireError::InvalidField("deletes")); + } + let mut keys = Vec::with_capacity(count); + let mut total = 0usize; + let mut seen = BTreeSet::new(); + for _ in 0..count { + let key = self.str_bounded(MAX_KEY_BYTES, "delete key")?; + if !valid_key(&key) { + return Err(WireError::InvalidField("delete key")); + } + if !seen.insert(key.clone()) { + return Err(WireError::InvalidField("duplicate delete key")); + } + total = total.saturating_add(key.len() + 4); + keys.push(key); + } + if total > MAX_BATCH_BYTES { + return Err(WireError::InvalidField("batch bytes")); + } + Ok(keys) + } + + fn diagnostic(&mut self) -> Result { + let message = self.str_bounded(MAX_ERROR_MESSAGE_BYTES, "error message")?; + let valid = !message.is_empty() && message.bytes().all(|b| (0x20..=0x7e).contains(&b)); + if !valid { + return Err(WireError::InvalidField("error message")); + } + Ok(message) + } + + fn is_empty(&self) -> bool { + self.position == self.buffer.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(message: &Message) -> Vec { + encode_frame(message).expect("encodes") + } + + fn roundtrip(message: Message) { + let encoded = frame(&message); + let mut reader = FrameReader::new(encoded.as_slice()); + assert_eq!(reader.read_message().expect("decodes"), Some(message)); + assert_eq!(reader.read_message().expect("clean eof"), None); + } + + fn hello() -> Message { + Message::Hello { + min: 1, + max: 1, + component: "storage".to_owned(), + version: "0.0.1".to_owned(), + } + } + + fn rows() -> Vec { + vec![ + Row { + key: "alpha".to_owned(), + value: b"one".to_vec(), + }, + Row { + key: "beta".to_owned(), + value: Vec::new(), + }, + ] + } + + #[test] + fn every_message_roundtrips() { + roundtrip(hello()); + roundtrip(Message::HelloAck { + protocol: 1, + component: "storage".to_owned(), + version: "0.0.1".to_owned(), + }); + roundtrip(Message::OpenNamespace { + request: 1, + namespace: "session-1".to_owned(), + }); + roundtrip(Message::ReadSnapshot { + request: 2, + namespace: "session-1".to_owned(), + start_row: 0, + max_rows: 32, + max_bytes: 1024, + }); + roundtrip(Message::CommitBatch { + request: 3, + namespace: "session-1".to_owned(), + expected_revision: 4, + upserts: rows(), + deletes: vec!["gamma".to_owned()], + }); + roundtrip(Message::Rollback { + request: 4, + namespace: "session-1".to_owned(), + expected_revision: 5, + }); + roundtrip(Message::Shutdown); + roundtrip(Message::NamespaceOpened { + request: 1, + namespace: "session-1".to_owned(), + revision: 2, + rows: 2, + bytes: 6, + }); + roundtrip(Message::SnapshotChunk { + request: 2, + namespace: "session-1".to_owned(), + revision: 2, + start_row: 0, + rows: rows(), + total_rows: 2, + total_bytes: 6, + done: true, + }); + roundtrip(Message::Committed { + request: 3, + namespace: "session-1".to_owned(), + revision: 5, + rows_written: 2, + rows_removed: 1, + bytes_written: 6, + total_rows: 2, + total_bytes: 6, + }); + roundtrip(Message::RolledBack { + request: 4, + namespace: "session-1".to_owned(), + revision: 6, + restored_revision: 4, + total_rows: 2, + total_bytes: 6, + }); + roundtrip(Message::Error { + request: 3, + kind: ErrorKind::StaleRevision, + message: "revision conflict".to_owned(), + }); + roundtrip(Message::ShutdownAck); + } + + #[test] + fn oversized_frame_is_rejected_from_the_header() { + let mut stream = Vec::new(); + stream.extend_from_slice(&u32::try_from(MAX_FRAME_BYTES + 1).unwrap().to_be_bytes()); + let mut reader = FrameReader::new(stream.as_slice()); + assert_eq!( + reader.read_frame(), + Err(WireError::TooLarge { + len: MAX_FRAME_BYTES + 1 + }) + ); + } + + #[test] + fn zero_length_frame_and_partial_header_are_rejected() { + let mut reader = FrameReader::new([0u8, 0, 0, 0].as_slice()); + assert_eq!(reader.read_frame(), Err(WireError::EmptyFrame)); + + let mut reader = FrameReader::new([0u8, 0].as_slice()); + assert_eq!(reader.read_frame(), Err(WireError::Truncated)); + } + + #[test] + fn truncated_payload_is_rejected() { + let encoded = frame(&hello()); + let mut reader = FrameReader::new(&encoded[..encoded.len() - 1]); + assert_eq!(reader.read_frame(), Err(WireError::Truncated)); + } + + #[test] + fn unknown_tags_and_trailing_bytes_are_rejected() { + assert_eq!(decode_message(&[0x7f]), Err(WireError::UnknownTag(0x7f))); + let mut encoded = encode_message(&Message::Shutdown).unwrap(); + encoded.push(0); + assert_eq!(decode_message(&encoded), Err(WireError::TrailingBytes)); + } + + #[test] + fn zero_request_ids_are_rejected() { + let mut encoded = encode_message(&Message::OpenNamespace { + request: 1, + namespace: "session-1".to_owned(), + }) + .unwrap(); + encoded[1..9].copy_from_slice(&0u64.to_be_bytes()); + assert_eq!( + decode_message(&encoded), + Err(WireError::InvalidField("zero request id")) + ); + + let zero = Message::OpenNamespace { + request: 0, + namespace: "session-1".to_owned(), + }; + assert_eq!( + encode_message(&zero), + Err(WireError::InvalidField("zero request id")) + ); + } + + #[test] + fn namespaces_and_keys_reject_path_shaped_values() { + for bad in [ + "", + "Upper", + "-dash", + ".dot", + "has/slash", + "has\\slash", + "has space", + "double..dot", + "../escape", + "with:colon", + "x".repeat(MAX_NAMESPACE_NAME_BYTES + 1).as_str(), + ] { + assert!(!valid_namespace_name(bad), "namespace: {bad}"); + } + for bad in [ + "", + "Upper", + "-dash", + ".dot", + "has/slash", + "has\\slash", + "has space", + "double..dot", + "../escape", + "with:colon", + "x".repeat(MAX_KEY_BYTES + 1).as_str(), + ] { + assert!(!valid_key(bad), "key: {bad}"); + } + for good in ["a", "0", "session-1", "plugin.store_v2"] { + assert!(valid_namespace_name(good), "namespace: {good}"); + assert!(valid_key(good), "key: {good}"); + } + let message = Message::OpenNamespace { + request: 1, + namespace: "../escape".to_owned(), + }; + assert_eq!( + encode_message(&message), + Err(WireError::InvalidField("namespace")) + ); + } + + #[test] + fn batch_and_value_bounds_are_enforced() { + let too_many = vec![ + Row { + key: "k".to_owned(), + value: Vec::new(), + }; + MAX_BATCH_ROWS + 1 + ]; + let message = Message::CommitBatch { + request: 1, + namespace: "ns".to_owned(), + expected_revision: 0, + upserts: too_many, + deletes: Vec::new(), + }; + assert_eq!( + encode_message(&message), + Err(WireError::InvalidField("upserts")) + ); + + let too_big = Message::CommitBatch { + request: 1, + namespace: "ns".to_owned(), + expected_revision: 0, + upserts: vec![Row { + key: "big".to_owned(), + value: vec![0; MAX_VALUE_BYTES + 1], + }], + deletes: Vec::new(), + }; + assert_eq!( + encode_message(&too_big), + Err(WireError::InvalidField("row value")) + ); + + let duplicate = Message::CommitBatch { + request: 1, + namespace: "ns".to_owned(), + expected_revision: 0, + upserts: vec![ + Row { + key: "same".to_owned(), + value: Vec::new(), + }, + Row { + key: "same".to_owned(), + value: Vec::new(), + }, + ], + deletes: Vec::new(), + }; + assert_eq!( + encode_message(&duplicate), + Err(WireError::InvalidField("duplicate row key")) + ); + } + + #[test] + fn response_counters_are_bounded() { + let too_many_rows = Message::Committed { + request: 1, + namespace: "ns".to_owned(), + revision: 2, + rows_written: u32::try_from(MAX_BATCH_ROWS + 1).unwrap(), + rows_removed: 0, + bytes_written: 0, + total_rows: 1, + total_bytes: 0, + }; + assert_eq!( + encode_message(&too_many_rows), + Err(WireError::InvalidField("batch rows")) + ); + + let too_many_bytes = Message::SnapshotChunk { + request: 1, + namespace: "ns".to_owned(), + revision: 2, + start_row: 0, + rows: Vec::new(), + total_rows: 0, + total_bytes: u64::try_from(MAX_SNAPSHOT_BYTES + 1).unwrap(), + done: true, + }; + assert_eq!( + encode_message(&too_many_bytes), + Err(WireError::InvalidField("total_bytes")) + ); + } + + #[test] + fn read_bounds_are_enforced() { + let too_many_rows = Message::ReadSnapshot { + request: 1, + namespace: "ns".to_owned(), + start_row: 0, + max_rows: u32::try_from(MAX_READ_ROWS + 1).unwrap(), + max_bytes: 0, + }; + assert_eq!( + encode_message(&too_many_rows), + Err(WireError::InvalidField("max_rows")) + ); + + let too_many_bytes = Message::ReadSnapshot { + request: 1, + namespace: "ns".to_owned(), + start_row: 0, + max_rows: 1, + max_bytes: u32::try_from(MAX_READ_BYTES + 1).unwrap(), + }; + assert_eq!( + encode_message(&too_many_bytes), + Err(WireError::InvalidField("max_bytes")) + ); + } + + #[test] + fn error_messages_are_bounded_and_ascii() { + let bad = Message::Error { + request: 0, + kind: ErrorKind::Protocol, + message: "bad\nframe".to_owned(), + }; + assert_eq!( + encode_message(&bad), + Err(WireError::InvalidField("error message")) + ); + let too_long = Message::Error { + request: 0, + kind: ErrorKind::Protocol, + message: "x".repeat(MAX_ERROR_MESSAGE_BYTES + 1), + }; + assert_eq!( + encode_message(&too_long), + Err(WireError::InvalidField("error message")) + ); + } + + #[test] + fn handshake_negotiation_is_exact() { + assert_eq!(negotiate(1, 1), Some(1)); + assert_eq!(negotiate(0, 1), Some(1)); + assert_eq!(negotiate(1, 9), Some(1)); + assert_eq!(negotiate(2, 9), None); + assert_eq!(negotiate(2, 1), None); + } + + #[test] + fn error_codes_are_stable() { + for kind in [ + ErrorKind::Protocol, + ErrorKind::InvalidRequest, + ErrorKind::Budget, + ErrorKind::Unsupported, + ErrorKind::UnknownNamespace, + ErrorKind::StaleRevision, + ErrorKind::Locked, + ErrorKind::CorruptState, + ErrorKind::Io, + ErrorKind::Internal, + ErrorKind::Uncertain, + ] { + assert_eq!(ErrorKind::from_code(kind.code()), Some(kind)); + } + // v1 codes are additive: the added `uncertain` kind is 11 and the + // pre-existing `internal` kind stays 10, so a v1 host never has a + // known code silently change meaning. + assert_eq!(ErrorKind::Protocol.code(), 1); + assert_eq!(ErrorKind::InvalidRequest.code(), 2); + assert_eq!(ErrorKind::Budget.code(), 3); + assert_eq!(ErrorKind::Unsupported.code(), 4); + assert_eq!(ErrorKind::UnknownNamespace.code(), 5); + assert_eq!(ErrorKind::StaleRevision.code(), 6); + assert_eq!(ErrorKind::Locked.code(), 7); + assert_eq!(ErrorKind::CorruptState.code(), 8); + assert_eq!(ErrorKind::Io.code(), 9); + assert_eq!(ErrorKind::Internal.code(), 10); + assert_eq!(ErrorKind::Uncertain.code(), 11); + assert_eq!(ErrorKind::from_code(10), Some(ErrorKind::Internal)); + assert_eq!(ErrorKind::from_code(11), Some(ErrorKind::Uncertain)); + assert_eq!(ErrorKind::Uncertain.as_str(), "uncertain"); + assert_eq!(ErrorKind::from_code(0), None); + assert_eq!(ErrorKind::from_code(12), None); + } + + #[test] + fn uncertain_error_roundtrips_on_the_wire() { + let message = Message::Error { + request: 7, + kind: ErrorKind::Uncertain, + message: "storage outcome uncertain; reopen namespace".to_owned(), + }; + let encoded = encode_message(&message).expect("encode"); + assert_eq!(encoded[0], TAG_ERROR); + assert_eq!(encoded[9], 11, "uncertain wire code must be 11"); + assert_eq!(decode_message(&encoded).expect("decode"), message); + } +} diff --git a/src/component/server.rs b/src/component/server.rs new file mode 100644 index 0000000..ccbdd9d --- /dev/null +++ b/src/component/server.rs @@ -0,0 +1,537 @@ +//! Component server: exposes journaled namespaces over the v1 stdio protocol. +//! +//! The binary entry point (`bitty-storage` with `--root PATH`) calls +//! [`serve_stdio`]. Tests call [`serve`] with explicit streams. The server is +//! a thin, policy-free adapter: Core owns installation, digest verification, +//! capability grants, consent, safe-mode denial, XDG migration, and restore +//! re-derivation. This process accepts only what the wire schema permits +//! (namespaces, keys, opaque values) and delegates every durable decision to +//! the journaled store. +//! +//! # Request model +//! +//! Requests are synchronous and single-threaded: a `CommitBatch` is fully +//! durable (or rejected before publication) before `Committed` is written. +//! A failure after a possible publication returns the distinct `uncertain` +//! kind, evicts the cached handle, and requires the host to reopen and +//! reread the revision before retrying. There is no per-connection +//! transaction staging, so `Rollback` is the explicit one-step restore of +//! the previous revision, not an abort of an in-flight request. Namespace +//! locks are held for the connection lifetime, which gives one component +//! process exclusive write access to each open namespace even when another +//! host starts a second component. +//! +//! # Shutdown +//! +//! `Shutdown` (or stdin EOF) drops every namespace handle, releasing the OS +//! locks, writes `ShutdownAck` for an explicit `Shutdown`, and exits 0. +//! Protocol violations exit 3; a server that cannot start exits 4. A +//! component crash cannot corrupt acknowledged state because every commit is +//! journaled before the snapshot rename and replayed on the next open. + +use std::collections::BTreeMap; +use std::io::{self, Read, Write}; + +use super::descriptor::COMPONENT_NAME; +use super::protocol::{ + CONNECTION_ID, ErrorKind, FrameReader, MAX_NAMESPACES, Message, Row, WireError, negotiate, + write_frame, +}; +use super::store::{Namespace, NamespaceName, StorageRoot, StoreError, StoreErrorKind}; + +/// Version reported in `HelloAck` and by `--version` (the crate version). +pub use super::COMPONENT_VERSION; + +/// Clean shutdown (explicit `Shutdown` or stdin EOF). +pub const SERVER_EXIT_OK: u8 = 0; +/// Handshake or protocol violation; the peer is told why first when the +/// stream still permits a write. +pub const SERVER_EXIT_PROTOCOL: u8 = 3; +/// The server could not open its root or write the handshake. +pub const SERVER_EXIT_INTERNAL: u8 = 4; + +const ERR_HANDSHAKE_REQUIRED: &str = "handshake required"; +const ERR_PROTOCOL_VERSION: &str = "protocol version unsupported"; +const ERR_COMPONENT_MISMATCH: &str = "component name mismatch"; +const ERR_UNEXPECTED_MESSAGE: &str = "unexpected message"; +const ERR_FRAME_TOO_LARGE: &str = "frame too large"; +const ERR_INVALID_FRAME: &str = "invalid protocol frame"; +const ERR_INVALID_NAMESPACE: &str = "invalid namespace"; +const ERR_NAMESPACE_LIMIT: &str = "namespace limit reached"; +const ERR_UNKNOWN_NAMESPACE: &str = "unknown namespace"; +const ERR_STALE_REVISION: &str = "revision conflict"; +const ERR_LOCKED: &str = "namespace is locked"; +const ERR_CORRUPT_STATE: &str = "storage state is corrupt"; +const ERR_BOUND: &str = "storage bound exceeded"; +const ERR_NO_PREVIOUS: &str = "no previous revision"; +const ERR_UNSUPPORTED: &str = "operation unsupported"; +const ERR_IO: &str = "storage i/o failed"; +const ERR_UNCERTAIN: &str = "storage outcome uncertain; reopen namespace"; +const ERR_INTERNAL: &str = "internal error"; + +/// Serves the protocol on stdin/stdout over `root`. +#[must_use] +pub fn serve_stdio(root: StorageRoot) -> u8 { + serve(root, io::stdin(), io::stdout()) +} + +/// Serves the protocol on explicit streams (tests and embedders). +/// +/// Returns [`SERVER_EXIT_OK`], [`SERVER_EXIT_PROTOCOL`], or +/// [`SERVER_EXIT_INTERNAL`]. +pub fn serve(root: StorageRoot, input: R, output: W) -> u8 +where + R: Read, + W: Write, +{ + let mut writer = FrameWriter::new(output); + let mut reader = FrameReader::new(input); + + let protocol = match reader.read_message() { + Ok(Some(Message::Hello { + min, + max, + component, + version: _, + })) => { + if component != COMPONENT_NAME { + writer.connection_error(ErrorKind::Protocol, ERR_COMPONENT_MISMATCH); + return SERVER_EXIT_PROTOCOL; + } + match negotiate(min, max) { + Some(protocol) => protocol, + None => { + writer.connection_error(ErrorKind::Protocol, ERR_PROTOCOL_VERSION); + return SERVER_EXIT_PROTOCOL; + } + } + } + Ok(Some(_)) => { + writer.connection_error(ErrorKind::Protocol, ERR_HANDSHAKE_REQUIRED); + return SERVER_EXIT_PROTOCOL; + } + Ok(None) => return SERVER_EXIT_OK, + Err(error) => { + writer.wire_error(&error); + return SERVER_EXIT_PROTOCOL; + } + }; + + if !writer.write(&Message::HelloAck { + protocol, + component: COMPONENT_NAME.to_owned(), + version: COMPONENT_VERSION.to_owned(), + }) { + return SERVER_EXIT_INTERNAL; + } + + let mut namespaces: BTreeMap = BTreeMap::new(); + let mut exit_code = SERVER_EXIT_OK; + loop { + match reader.read_message() { + Ok(Some(message)) => match message { + Message::OpenNamespace { request, namespace } => { + handle_open(&root, &mut namespaces, &mut writer, request, namespace) + } + Message::ReadSnapshot { + request, + namespace, + start_row, + max_rows, + max_bytes, + } => handle_read( + &mut namespaces, + &mut writer, + request, + &namespace, + start_row, + max_rows, + max_bytes, + ), + Message::CommitBatch { + request, + namespace, + expected_revision, + upserts, + deletes, + } => handle_commit( + &mut namespaces, + &mut writer, + request, + &namespace, + expected_revision, + upserts, + deletes, + ), + Message::Rollback { + request, + namespace, + expected_revision, + } => handle_rollback( + &mut namespaces, + &mut writer, + request, + &namespace, + expected_revision, + ), + Message::Shutdown => { + namespaces.clear(); + let _ = writer.write(&Message::ShutdownAck); + return SERVER_EXIT_OK; + } + Message::Hello { .. } => { + writer.connection_error(ErrorKind::Protocol, ERR_UNEXPECTED_MESSAGE); + exit_code = SERVER_EXIT_PROTOCOL; + break; + } + _ => { + writer.connection_error(ErrorKind::Protocol, ERR_UNEXPECTED_MESSAGE); + exit_code = SERVER_EXIT_PROTOCOL; + break; + } + }, + Ok(None) => break, + Err(error) => { + writer.wire_error(&error); + exit_code = SERVER_EXIT_PROTOCOL; + break; + } + } + } + + // Dropping handles releases the OS locks even on an EOF path. + namespaces.clear(); + exit_code +} + +fn handle_open( + root: &StorageRoot, + namespaces: &mut BTreeMap, + writer: &mut FrameWriter, + request: u64, + namespace: String, +) { + let name = match NamespaceName::parse(&namespace) { + Ok(name) => name, + Err(_) => { + writer.request_error(request, ErrorKind::InvalidRequest, ERR_INVALID_NAMESPACE); + return; + } + }; + if let Some(open) = namespaces.get(name.as_str()) { + if open.ensure_attached().is_err() { + namespaces.remove(name.as_str()); + } else { + writer.write(&Message::NamespaceOpened { + request, + namespace, + revision: open.revision(), + rows: u32::try_from(open.row_count()).unwrap_or(u32::MAX), + bytes: open.value_bytes(), + }); + return; + } + } + if namespaces.len() >= MAX_NAMESPACES { + writer.request_error(request, ErrorKind::Budget, ERR_NAMESPACE_LIMIT); + return; + } + match root.open_namespace(&name) { + Ok(open) => { + let opened = Message::NamespaceOpened { + request, + namespace: name.as_str().to_owned(), + revision: open.revision(), + rows: u32::try_from(open.row_count()).unwrap_or(u32::MAX), + bytes: open.value_bytes(), + }; + namespaces.insert(name.as_str().to_owned(), open); + writer.write(&opened); + } + Err(error) => writer.store_error(request, &error), + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_read( + namespaces: &mut BTreeMap, + writer: &mut FrameWriter, + request: u64, + namespace: &str, + start_row: u32, + max_rows: u32, + max_bytes: u32, +) { + let Some(open) = namespaces.get(namespace) else { + writer.request_error(request, ErrorKind::UnknownNamespace, ERR_UNKNOWN_NAMESPACE); + return; + }; + let result = open.read_page(start_row, max_rows, max_bytes); + evict_uncertain(namespaces, namespace, &result); + match result { + Ok(page) => { + writer.write(&Message::SnapshotChunk { + request, + namespace: namespace.to_owned(), + revision: page.revision, + start_row: page.start_row, + rows: page.rows, + total_rows: page.total_rows, + total_bytes: page.total_bytes, + done: page.done, + }); + } + Err(error) => writer.store_error(request, &error), + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_commit( + namespaces: &mut BTreeMap, + writer: &mut FrameWriter, + request: u64, + namespace: &str, + expected_revision: u64, + upserts: Vec, + deletes: Vec, +) { + let Some(open) = namespaces.get_mut(namespace) else { + writer.request_error(request, ErrorKind::UnknownNamespace, ERR_UNKNOWN_NAMESPACE); + return; + }; + let result = open.commit(expected_revision, upserts, deletes); + evict_uncertain(namespaces, namespace, &result); + match result { + Ok(outcome) => { + writer.write(&Message::Committed { + request, + namespace: namespace.to_owned(), + revision: outcome.revision, + rows_written: outcome.rows_written, + rows_removed: outcome.rows_removed, + bytes_written: outcome.bytes_written, + total_rows: outcome.total_rows, + total_bytes: outcome.total_bytes, + }); + } + Err(error) => writer.store_error(request, &error), + } +} + +fn handle_rollback( + namespaces: &mut BTreeMap, + writer: &mut FrameWriter, + request: u64, + namespace: &str, + expected_revision: u64, +) { + let Some(open) = namespaces.get_mut(namespace) else { + writer.request_error(request, ErrorKind::UnknownNamespace, ERR_UNKNOWN_NAMESPACE); + return; + }; + let result = open.rollback(expected_revision); + evict_uncertain(namespaces, namespace, &result); + match result { + Ok(outcome) => { + writer.write(&Message::RolledBack { + request, + namespace: namespace.to_owned(), + revision: outcome.revision, + restored_revision: outcome.restored_revision, + total_rows: outcome.total_rows, + total_bytes: outcome.total_bytes, + }); + } + Err(error) => writer.store_error(request, &error), + } +} + +fn store_error(error: &StoreError) -> (ErrorKind, &'static str) { + match error.kind() { + StoreErrorKind::InvalidName => (ErrorKind::InvalidRequest, ERR_INVALID_NAMESPACE), + StoreErrorKind::Bound => (ErrorKind::Budget, ERR_BOUND), + StoreErrorKind::Locked => (ErrorKind::Locked, ERR_LOCKED), + StoreErrorKind::Corrupt => (ErrorKind::CorruptState, ERR_CORRUPT_STATE), + StoreErrorKind::Io => (ErrorKind::Io, ERR_IO), + StoreErrorKind::Uncertain => (ErrorKind::Uncertain, ERR_UNCERTAIN), + StoreErrorKind::NoPrevious => (ErrorKind::InvalidRequest, ERR_NO_PREVIOUS), + StoreErrorKind::StaleRevision => (ErrorKind::StaleRevision, ERR_STALE_REVISION), + StoreErrorKind::Unsupported => (ErrorKind::Unsupported, ERR_UNSUPPORTED), + StoreErrorKind::Internal => (ErrorKind::Internal, ERR_INTERNAL), + } +} + +fn evict_uncertain( + namespaces: &mut BTreeMap, + namespace: &str, + result: &Result, +) { + if result + .as_ref() + .is_err_and(|error| error.kind() == StoreErrorKind::Uncertain) + { + namespaces.remove(namespace); + } +} + +struct FrameWriter { + writer: W, + failed: bool, +} + +impl FrameWriter { + fn new(writer: W) -> Self { + Self { + writer, + failed: false, + } + } + + fn write(&mut self, message: &Message) -> bool { + if self.failed { + return false; + } + match write_frame(&mut self.writer, message).and_then(|()| { + self.writer + .flush() + .map_err(|error| WireError::Io(error.kind())) + }) { + Ok(()) => true, + Err(_) => { + self.failed = true; + false + } + } + } + + fn request_error(&mut self, request: u64, kind: ErrorKind, message: &'static str) { + self.write(&Message::Error { + request, + kind, + message: message.to_owned(), + }); + } + + fn connection_error(&mut self, kind: ErrorKind, message: &'static str) { + self.request_error(CONNECTION_ID, kind, message); + } + + fn store_error(&mut self, request: u64, error: &StoreError) { + let (kind, message) = store_error(error); + self.request_error(request, kind, message); + } + + fn wire_error(&mut self, error: &WireError) { + let (kind, message) = match error { + WireError::TooLarge { .. } => (ErrorKind::Protocol, ERR_FRAME_TOO_LARGE), + _ => (ErrorKind::Protocol, ERR_INVALID_FRAME), + }; + self.connection_error(kind, message); + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use crate::component::protocol::decode_message; + + #[test] + fn open_after_path_replacement_does_not_acknowledge_cached_inode() { + let root_path = + std::env::temp_dir().join(format!("server-open-replace-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root_path); + std::fs::create_dir_all(&root_path).unwrap(); + let root = StorageRoot::new(&root_path).unwrap(); + let mut namespaces = BTreeMap::new(); + let mut output = Vec::new(); + { + let mut writer = FrameWriter::new(&mut output); + handle_open(&root, &mut namespaces, &mut writer, 1, "test".to_owned()); + } + namespaces + .get_mut("test") + .unwrap() + .commit(0, Vec::new(), Vec::new()) + .unwrap(); + + std::fs::rename(root_path.join("test"), root_path.join("detached")).unwrap(); + std::fs::create_dir(root_path.join("test")).unwrap(); + output.clear(); + { + let mut writer = FrameWriter::new(&mut output); + handle_open(&root, &mut namespaces, &mut writer, 2, "test".to_owned()); + } + + assert_eq!(namespaces.get("test").unwrap().revision(), 0); + let message = decode_message(&output[4..]).unwrap(); + assert!(matches!( + message, + Message::NamespaceOpened { + request: 2, + revision: 0, + .. + } + )); + assert_eq!( + namespaces.len(), + 1, + "detached cached handle must be evicted" + ); + let _ = std::fs::remove_dir_all(root_path); + } + + #[test] + fn uncertain_result_evicts_cached_snapshot_before_reopen() { + let root_path = + std::env::temp_dir().join(format!("server-uncertain-reopen-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root_path); + std::fs::create_dir_all(&root_path).unwrap(); + let root = StorageRoot::new(&root_path).unwrap(); + let name = NamespaceName::parse("test").unwrap(); + let mut cached = root.open_namespace(&name).unwrap(); + cached + .commit( + 0, + vec![Row { + key: "committed".to_owned(), + value: b"on-disk".to_vec(), + }], + Vec::new(), + ) + .unwrap(); + assert_eq!(cached.revision(), 1); + let mut namespaces = BTreeMap::from([("test".to_owned(), cached)]); + + // An uncertain result must drop the cached handle (releasing its + // lock) instead of serving its possibly stale in-memory snapshot. + let uncertain: Result<(), StoreError> = Err(StoreError::uncertain()); + evict_uncertain(&mut namespaces, "test", &uncertain); + assert!(!namespaces.contains_key("test")); + + // The reopen reloads the disk state, not the evicted snapshot. + let reopened = root + .open_namespace(&name) + .expect("reopen must reload disk state"); + assert_eq!(reopened.revision(), 1); + let page = reopened.read_page(0, 0, 0).expect("read after reopen"); + assert_eq!(page.revision, 1); + assert_eq!( + page.rows, + vec![Row { + key: "committed".to_owned(), + value: b"on-disk".to_vec(), + }] + ); + let _ = std::fs::remove_dir_all(root_path); + } + + #[test] + fn uncertain_maps_to_a_dedicated_wire_kind() { + let error = StoreError::uncertain(); + let (kind, message) = store_error(&error); + assert_eq!(kind, ErrorKind::Uncertain); + assert_eq!(message, ERR_UNCERTAIN); + assert_eq!(kind.code(), 11); + assert_eq!(ErrorKind::Internal.code(), 10); + } +} diff --git a/src/component/store.rs b/src/component/store.rs new file mode 100644 index 0000000..342e2c9 --- /dev/null +++ b/src/component/store.rs @@ -0,0 +1,268 @@ +//! Journaled, path-contained namespace storage for the component server. +//! +//! Each namespace is one directory under the host-chosen storage root with +//! four fixed members: +//! +//! ```text +//! // +//! snapshot current committed state (atomic temp+rename, mode 0600) +//! previous state before the last commit (rollback source) +//! journal write-ahead target for the in-flight commit +//! lock OS advisory lock held for this namespace's lifetime +//! ``` +//! +//! # Commit protocol +//! +//! 1. Serialize the candidate snapshot and publish it as `journal` +//! (temp+fsync+rename). The journal is the durable intent record. +//! 2. Atomically rewrite `previous` from the current snapshot. The prior +//! `previous` image (or its absence) is captured first so a failed step 3 +//! can restore it instead of leaving revision N in both slots. +//! 3. Atomically rewrite `snapshot` from the candidate. +//! 4. Remove `journal` durably (fsync of the namespace directory). +//! +//! A crash after step 1 but before step 3 is repaired on the next open by +//! replaying the journal. A crash after step 3 leaves a journal that is +//! older than or equal to the snapshot, so recovery removes it without +//! rewriting state. A pre-rename failure in steps 2 or 3 restores the prior +//! `previous` image (or its absence), removes the journal durably, and leaves +//! the previous snapshot in place, so the request is reported as `io` and was +//! not applied: the committed revision and its N-1 rollback image are both +//! unchanged. If that restoration cannot be confirmed, the result is +//! `uncertain`, never `io`, and the journal is retained for recovery. Once a +//! rename may have happened, every failure to confirm cleanup, recovery, +//! restoration, or attachment is `uncertain`: the new snapshot may already be +//! visible, and the caller must reopen and reread the revision before retrying. +//! +//! `rollback` accepts only a `previous` image whose revision is exactly +//! `current - 1`. An absent slot is `no_previous`; an equal, newer, or +//! skipping revision is `corrupt_state`, so a rollback can never acknowledge +//! a no-op or restore an image older than one step. +//! +//! # Authority boundary +//! +//! The wire protocol never carries a path. Namespace names are validated to +//! `[a-z0-9][a-z0-9._-]{0,63}` with no `..`. On Linux/Android the root and +//! namespace directories are opened once and every state operation is +//! `openat`/`renameat`/`unlinkat` relative to that directory fd, with +//! `O_NOFOLLOW` on the final component and `O_CREAT|O_EXCL` temp creation. +//! A racing namespace or ancestor replacement therefore cannot redirect a +//! read, write, rename, or unlink outside the opened directory. State files +//! must be regular files; symlinks, FIFOs, devices, and directories are +//! refused before use. Other platforms fail closed with `Unsupported` rather +//! than falling back to path-based state operations. +//! +//! Namespace identity is the inode of the namespace directory opened by +//! `OpenNamespace`; every state operation stays relative to that fd. The +//! server additionally stats the root-relative namespace path and compares +//! `(st_dev, st_ino)` with the opened fd before acknowledging a recovery and +//! after publication. That check detects a rename, removal, or replacement +//! that is visible at the stat instant. It is **best-effort detection, not an +//! atomic binding** between the pathname and the stdio acknowledgement: a +//! replacement that races the final check or the response write is not +//! detected, and the response then still describes only the opened inode. A +//! detected detached namespace returns `uncertain` and the server evicts the +//! cached handle; the client must reopen, which re-resolves the pathname, and +//! reread the revision before retrying. A successful response therefore +//! guarantees that the operation ran against the opened directory inode; it +//! never guarantees that the pathname still names that inode when the host +//! reads the response. + +use crate::component::protocol::valid_namespace_name; + +/// Store failure category (stable; the server maps it to a wire error). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoreErrorKind { + /// A namespace or key violates its grammar. + InvalidName, + /// A row, byte, snapshot, or journal bound was exceeded. + Bound, + /// Another process holds the namespace lock. + Locked, + /// Snapshot or journal bytes are malformed or fail their checksum, or the + /// rollback slot violates its previous-revision invariant. + Corrupt, + /// A filesystem operation failed (no path or content in the message). + Io, + /// A post-rename commit, recovery, restoration, cleanup, or attachment + /// result could not be confirmed. Callers must reopen and reread the + /// revision before retrying. + Uncertain, + /// Rollback was requested but no previous revision exists. + NoPrevious, + /// `expected_revision` does not match the current revision. + StaleRevision, + /// The platform has no fs-at/anchored backend here. + Unsupported, + /// Internal invariant failure. + Internal, +} + +/// Content-free store error (safe for stderr and bounded wire diagnostics). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreError { + kind: StoreErrorKind, + message: &'static str, +} + +impl StoreError { + const fn new(kind: StoreErrorKind, message: &'static str) -> Self { + Self { kind, message } + } + + /// Failure category. + #[must_use] + pub const fn kind(&self) -> StoreErrorKind { + self.kind + } + + /// Bounded static diagnostic. + #[must_use] + pub const fn message(&self) -> &'static str { + self.message + } + + pub(crate) fn invalid_name() -> Self { + Self::new(StoreErrorKind::InvalidName, "invalid namespace or key name") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn bound() -> Self { + Self::new(StoreErrorKind::Bound, "storage bound exceeded") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn corrupt() -> Self { + Self::new(StoreErrorKind::Corrupt, "storage state is corrupt") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn io() -> Self { + Self::new(StoreErrorKind::Io, "storage i/o failed") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn uncertain() -> Self { + Self::new( + StoreErrorKind::Uncertain, + "storage outcome uncertain; reopen namespace", + ) + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn locked() -> Self { + Self::new(StoreErrorKind::Locked, "namespace is locked") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn no_previous() -> Self { + Self::new(StoreErrorKind::NoPrevious, "no previous revision") + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + pub(crate) fn stale() -> Self { + Self::new(StoreErrorKind::StaleRevision, "revision conflict") + } + + #[cfg(not(any(target_os = "linux", target_os = "android")))] + pub(crate) fn unsupported() -> Self { + Self::new( + StoreErrorKind::Unsupported, + "storage backend is unsupported on this platform", + ) + } +} + +impl std::fmt::Display for StoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message) + } +} + +impl std::error::Error for StoreError {} + +/// A validated namespace name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamespaceName(String); + +impl NamespaceName { + /// Parses a namespace name (same grammar as the wire codec). + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::InvalidName`] for an empty, over-long, + /// non-ASCII, separator-bearing, or dot-segment name. + pub fn parse(value: &str) -> Result { + if !valid_namespace_name(value) { + return Err(StoreError::invalid_name()); + } + Ok(Self(value.to_owned())) + } + + /// Name as a string slice. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// One page of a committed snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotPage { + /// Revision the rows belong to. + pub revision: u64, + /// Zero-based row index of the first returned row. + pub start_row: u32, + /// Rows in key order. + pub rows: Vec, + /// Rows in the whole snapshot. + pub total_rows: u32, + /// Row-value bytes in the whole snapshot. + pub total_bytes: u64, + /// Whether this page reaches the end. + pub done: bool, +} + +/// Result of one atomic batch commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CommitOutcome { + /// New revision. + pub revision: u64, + /// Upserts applied. + pub rows_written: u32, + /// Deletes applied (including absent keys, which are no-ops). + pub rows_removed: u32, + /// Row-value bytes written by the batch. + pub bytes_written: u64, + /// Row count after the commit. + pub total_rows: u32, + /// Row-value bytes after the commit. + pub total_bytes: u64, +} + +/// Result of restoring the previous revision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RollbackOutcome { + /// New revision created by the rollback. + pub revision: u64, + /// Revision whose contents were restored. + pub restored_revision: u64, + /// Row count after the rollback. + pub total_rows: u32, + /// Row-value bytes after the rollback. + pub total_bytes: u64, +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +#[path = "store/anchored.rs"] +mod anchored; + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub use anchored::{Namespace, StorageRoot}; + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +#[path = "store/unsupported.rs"] +mod unsupported; + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +pub use unsupported::{Namespace, StorageRoot}; diff --git a/src/component/store/anchored.rs b/src/component/store/anchored.rs new file mode 100644 index 0000000..376cc63 --- /dev/null +++ b/src/component/store/anchored.rs @@ -0,0 +1,1045 @@ +//! Anchored namespace backend (Linux/Android). +//! +//! The root and each namespace directory stay open as directory fds. All +//! state operations are `*at` relative to the namespace fd, final components +//! use `O_NOFOLLOW`, and temp files use `O_CREAT|O_EXCL`; see `crate::fs_at`. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io; +use std::path::PathBuf; + +use crate::component::protocol::{ + MAX_BATCH_BYTES, MAX_BATCH_ROWS, MAX_KEY_BYTES, MAX_READ_BYTES, MAX_READ_ROWS, MAX_ROWS, + MAX_SNAPSHOT_BYTES, MAX_VALUE_BYTES, Row, valid_key, +}; +use crate::fs_at; + +use super::{ + CommitOutcome, NamespaceName, RollbackOutcome, SnapshotPage, StoreError, StoreErrorKind, +}; + +/// Snapshot file magic (format version follows separately). +const SNAPSHOT_MAGIC: &[u8] = b"bitty-storage-snapshot\n"; +/// Journal file magic (format version follows separately). +const JOURNAL_MAGIC: &[u8] = b"bitty-storage-journal\n"; +/// On-disk format version for both files. +const FORMAT_VERSION: u32 = 1; +/// Fixed snapshot member name. +const SNAPSHOT_FILE: &str = "snapshot"; +/// Fixed previous-revision member name. +const PREVIOUS_FILE: &str = "previous"; +/// Fixed journal member name. +const JOURNAL_FILE: &str = "journal"; +/// Fixed lock member name. +const LOCK_FILE: &str = "lock"; + +/// Maximum journal bytes: journal header, a whole snapshot, and checksum. +pub const JOURNAL_MAX_BYTES: usize = JOURNAL_MAGIC.len() + 4 + 8 + 4 + MAX_SNAPSHOT_BYTES + 8; + +// Test-only fault injection: forces the next rollback-image restoration to +// fail so unit tests can prove an unconfirmed restore reports `Uncertain`. +#[cfg(test)] +thread_local! { + static FAIL_PREVIOUS_RESTORE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Closed namespace snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Snapshot { + revision: u64, + rows: BTreeMap>, +} + +impl Snapshot { + /// Empty state at revision `0`. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + /// Current revision (`0` means never committed). + #[must_use] + pub const fn revision(&self) -> u64 { + self.revision + } + + /// Row count. + #[must_use] + pub fn row_count(&self) -> usize { + self.rows.len() + } + + /// Sum of row-value bytes. + #[must_use] + pub fn value_bytes(&self) -> u64 { + self.rows.values().map(|value| value.len() as u64).sum() + } + + /// Encodes the snapshot with magic, version, revision, rows, checksum. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::Bound`] when the encoded image exceeds + /// [`MAX_SNAPSHOT_BYTES`]. + pub fn encode(&self) -> Result, StoreError> { + let mut out = Vec::new(); + out.extend_from_slice(SNAPSHOT_MAGIC); + out.extend_from_slice(&FORMAT_VERSION.to_be_bytes()); + out.extend_from_slice(&self.revision.to_be_bytes()); + out.extend_from_slice( + &u32::try_from(self.rows.len()) + .map_err(|_| StoreError::bound())? + .to_be_bytes(), + ); + for (key, value) in &self.rows { + let key_len = u32::try_from(key.len()).map_err(|_| StoreError::bound())?; + let value_len = u32::try_from(value.len()).map_err(|_| StoreError::bound())?; + out.extend_from_slice(&key_len.to_be_bytes()); + out.extend_from_slice(key.as_bytes()); + out.extend_from_slice(&value_len.to_be_bytes()); + out.extend_from_slice(value); + } + let checksum = checksum(&out); + out.extend_from_slice(&checksum.to_be_bytes()); + if out.len() > MAX_SNAPSHOT_BYTES { + return Err(StoreError::bound()); + } + Ok(out) + } + + /// Decodes and fully validates a snapshot image. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::Corrupt`] for bad magic, version, lengths, + /// ordering, grammar, trailing bytes, or checksum, and + /// [`StoreErrorKind::Bound`] for an over-bound image or row. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_SNAPSHOT_BYTES { + return Err(StoreError::bound()); + } + let mut cursor = Cursor::new(bytes); + if cursor.take(SNAPSHOT_MAGIC.len())? != SNAPSHOT_MAGIC { + return Err(StoreError::corrupt()); + } + if cursor.u32()? != FORMAT_VERSION { + return Err(StoreError::corrupt()); + } + let revision = cursor.u64()?; + if revision == 0 { + return Err(StoreError::corrupt()); + } + let count = cursor.u32()? as usize; + if count > MAX_ROWS { + return Err(StoreError::bound()); + } + let mut rows = BTreeMap::new(); + let mut previous_key: Option = None; + let mut value_bytes = 0usize; + for _ in 0..count { + let key_len = cursor.u32()? as usize; + if key_len == 0 || key_len > MAX_KEY_BYTES { + return Err(StoreError::corrupt()); + } + let key = std::str::from_utf8(cursor.take(key_len)?) + .map_err(|_| StoreError::corrupt())? + .to_owned(); + if !valid_key(&key) { + return Err(StoreError::corrupt()); + } + if let Some(previous) = &previous_key { + if key <= *previous { + return Err(StoreError::corrupt()); + } + } + previous_key = Some(key.clone()); + let value_len = cursor.u32()? as usize; + if value_len > MAX_VALUE_BYTES { + return Err(StoreError::bound()); + } + let value = cursor.take(value_len)?.to_vec(); + value_bytes = value_bytes.saturating_add(value_len); + if value_bytes > MAX_SNAPSHOT_BYTES { + return Err(StoreError::bound()); + } + rows.insert(key, value); + } + let stored = cursor.u64()?; + if !cursor.is_empty() { + return Err(StoreError::corrupt()); + } + if checksum(&bytes[..bytes.len() - 8]) != stored { + return Err(StoreError::corrupt()); + } + Ok(Self { revision, rows }) + } +} + +/// Canonical storage root chosen by the host, held as a directory fd. +#[derive(Debug)] +pub struct StorageRoot { + root: File, +} + +impl StorageRoot { + /// Creates and opens the storage root by walking each path component + /// relative to an already-open directory fd. Symlinked ancestors and + /// parent-directory components are refused. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::Io`] when the directory cannot be created or + /// resolved, and [`StoreErrorKind::InvalidName`] when the resolved root + /// is not a directory or its final component is a symlink. + pub fn new(path: impl Into) -> Result { + let path = path.into(); + let root = fs_at::create_and_open_dir_tree(&path).map_err(map_namespace_error)?; + Ok(Self { root }) + } + + /// Opens and exclusively locks one namespace, recovering a published + /// journal first. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::InvalidName`] for a bad name or symlinked + /// namespace, [`StoreErrorKind::Corrupt`] for a symlinked or non-regular + /// state member, [`StoreErrorKind::Locked`] when another process holds + /// the lock, [`StoreErrorKind::Io`] for pre-replay filesystem failures, + /// and [`StoreErrorKind::Uncertain`] when a valid journal could not be + /// fully replayed or the namespace stopped being attached. An uncertain + /// open must be retried as a fresh open after the filesystem condition is + /// repaired. + pub fn open_namespace(&self, name: &NamespaceName) -> Result { + let namespace = name.as_str(); + fs_at::create_dir_at(&self.root, namespace).map_err(map_namespace_error)?; + let dir = fs_at::open_dir_at(&self.root, namespace).map_err(map_namespace_error)?; + // Validate every fixed state member before any content is read. A + // symlink, FIFO, device, or directory is refused even for `previous`, + // which is not read on the open path. + for member in [SNAPSHOT_FILE, PREVIOUS_FILE, JOURNAL_FILE] { + fs_at::regular_or_absent(&dir, member).map_err(map_state_error)?; + } + let lock = fs_at::open_lock_at(&dir, LOCK_FILE).map_err(map_lock_error)?; + for stem in [JOURNAL_FILE, PREVIOUS_FILE, SNAPSHOT_FILE] { + fs_at::clean_temp_siblings_at(&dir, stem).map_err(|_| StoreError::io())?; + } + let snapshot = recover_then_check_attached(&dir, &self.root, namespace, || {})?; + let root = self.root.try_clone().map_err(|_| StoreError::io())?; + Ok(Namespace { + name: name.clone(), + root, + dir, + _lock: lock, + snapshot, + }) + } +} + +/// One open, locked namespace anchored to its directory fd. +#[derive(Debug)] +pub struct Namespace { + name: NamespaceName, + root: File, + dir: File, + _lock: File, + snapshot: Snapshot, +} + +impl Namespace { + /// Namespace name. + #[must_use] + pub fn name(&self) -> &NamespaceName { + &self.name + } + + /// Current revision. + #[must_use] + pub const fn revision(&self) -> u64 { + self.snapshot.revision() + } + + /// Current row count. + #[must_use] + pub fn row_count(&self) -> usize { + self.snapshot.row_count() + } + + /// Current row-value bytes. + #[must_use] + pub fn value_bytes(&self) -> u64 { + self.snapshot.value_bytes() + } + + /// Best-effort attachment check: returns `Uncertain` when the namespace + /// directory fd is no longer the directory currently named under the + /// storage root (renamed, removed, or replaced by a symlink/other entry). + /// The stat pair is a detection point, not a lease: the pathname can be + /// replaced again immediately after this call returns. The caller must + /// reopen and reread rather than retrying against this handle. + pub(crate) fn ensure_attached(&self) -> Result<(), StoreError> { + match fs_at::directory_matches(&self.root, self.name.as_str(), &self.dir) { + Ok(true) => Ok(()), + Ok(false) => Err(StoreError::uncertain()), + Err(_) => Err(StoreError::uncertain()), + } + } + + /// Reads one bounded page of the committed snapshot. + /// + /// `max_rows` and `max_bytes` of `0` mean the protocol defaults. A page + /// that cannot fit one row inside `max_bytes` is a [`StoreErrorKind::Bound`] + /// rather than a silent empty page. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::Bound`] when a request parameter exceeds + /// [`MAX_READ_ROWS`] / [`MAX_READ_BYTES`] or the next row cannot fit. + pub fn read_page( + &self, + start_row: u32, + max_rows: u32, + max_bytes: u32, + ) -> Result { + self.ensure_attached()?; + let row_cap = if max_rows == 0 { + MAX_READ_ROWS + } else { + max_rows as usize + }; + if row_cap > MAX_READ_ROWS { + return Err(StoreError::bound()); + } + let byte_cap = if max_bytes == 0 { + MAX_READ_BYTES + } else { + max_bytes as usize + }; + if byte_cap > MAX_READ_BYTES { + return Err(StoreError::bound()); + } + + let total_rows = + u32::try_from(self.snapshot.rows.len()).map_err(|_| StoreError::bound())?; + let total_bytes = self.snapshot.value_bytes(); + let start = start_row as usize; + let mut rows = Vec::new(); + let mut used = 0usize; + let mut more = false; + for (index, (key, value)) in self.snapshot.rows.iter().enumerate().skip(start) { + if rows.len() >= row_cap { + more = true; + break; + } + let cost = key.len() + value.len(); + if used + cost > byte_cap { + if rows.is_empty() { + return Err(StoreError::bound()); + } + more = true; + break; + } + rows.push(Row { + key: key.clone(), + value: value.clone(), + }); + used += cost; + more = index + 1 != self.snapshot.rows.len(); + } + Ok(SnapshotPage { + revision: self.snapshot.revision(), + start_row, + rows, + total_rows, + total_bytes, + done: !more, + }) + } + + /// Atomically applies one batch at `expected_revision`. + /// + /// Deletes run before upserts, so a key present in both lists ends up + /// with the upserted value. Present and absent deletes both count as + /// applied; absent keys are successful no-ops. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::StaleRevision`] when the expected revision + /// differs, [`StoreErrorKind::InvalidName`] for a bad key, + /// [`StoreErrorKind::Bound`] for a row/byte cap, including an over-bound + /// prior rollback image, [`StoreErrorKind::Corrupt`] for a malformed prior + /// rollback image, [`StoreErrorKind::Io`] for a pre-mutation state read or + /// a pre-rename durable-write failure that was fully rolled back, and + /// [`StoreErrorKind::Uncertain`] when publication, restoration, cleanup, + /// recovery, or attachment durability cannot be confirmed. The public + /// server evicts an uncertain namespace; callers must reopen before + /// retrying. + pub fn commit( + &mut self, + expected_revision: u64, + upserts: Vec, + deletes: Vec, + ) -> Result { + self.ensure_attached()?; + if expected_revision != self.snapshot.revision { + return Err(StoreError::stale()); + } + if upserts.len() > MAX_BATCH_ROWS || deletes.len() > MAX_BATCH_ROWS { + return Err(StoreError::bound()); + } + let rows_written = u32::try_from(upserts.len()).map_err(|_| StoreError::bound())?; + let rows_removed = u32::try_from(deletes.len()).map_err(|_| StoreError::bound())?; + let mut batch_bytes = 0usize; + for row in &upserts { + if !valid_key(&row.key) { + return Err(StoreError::invalid_name()); + } + if row.value.len() > MAX_VALUE_BYTES { + return Err(StoreError::bound()); + } + batch_bytes = batch_bytes.saturating_add(row.key.len() + row.value.len() + 8); + } + for key in &deletes { + if !valid_key(key) { + return Err(StoreError::invalid_name()); + } + batch_bytes = batch_bytes.saturating_add(key.len() + 4); + } + if batch_bytes > MAX_BATCH_BYTES { + return Err(StoreError::bound()); + } + + let mut candidate_rows = self.snapshot.rows.clone(); + for key in &deletes { + candidate_rows.remove(key); + } + let mut bytes_written = 0u64; + for row in upserts { + bytes_written = bytes_written.saturating_add(row.value.len() as u64); + candidate_rows.insert(row.key, row.value); + } + if candidate_rows.len() > MAX_ROWS { + return Err(StoreError::bound()); + } + let revision = self + .snapshot + .revision + .checked_add(1) + .ok_or_else(StoreError::bound)?; + let candidate = Snapshot { + revision, + rows: candidate_rows, + }; + let candidate_bytes = candidate.encode()?; + let current_bytes = self.snapshot.encode()?; + self.publish_then_check_attached( + &candidate_bytes, + ¤t_bytes, + self.snapshot.revision == 0, + || {}, + )?; + self.snapshot = candidate; + Ok(CommitOutcome { + revision, + rows_written, + rows_removed, + bytes_written, + total_rows: u32::try_from(self.snapshot.rows.len()).map_err(|_| StoreError::bound())?, + total_bytes: self.snapshot.value_bytes(), + }) + } + + /// Restores the previous revision as a new revision. + /// + /// One-step undo only: after a rollback the previous slot holds the state + /// that was current before the rollback, so repeating rollback toggles. + /// + /// # Errors + /// + /// Returns [`StoreErrorKind::StaleRevision`] on a revision mismatch, + /// [`StoreErrorKind::NoPrevious`] when no previous revision exists, + /// [`StoreErrorKind::Corrupt`] for an invalid previous image or one whose + /// revision is not exactly `current - 1`, [`StoreErrorKind::Io`] for a + /// pre-mutation state read or a pre-rename durable-write failure that was + /// fully rolled back, and [`StoreErrorKind::Uncertain`] when publication, + /// restoration, cleanup, recovery, or attachment durability cannot be + /// confirmed. Callers must reopen before retrying. + pub fn rollback(&mut self, expected_revision: u64) -> Result { + self.ensure_attached()?; + if expected_revision != self.snapshot.revision { + return Err(StoreError::stale()); + } + let previous_bytes = read_state(&self.dir, PREVIOUS_FILE, MAX_SNAPSHOT_BYTES)? + .ok_or_else(StoreError::no_previous)?; + let previous = Snapshot::decode(&previous_bytes)?; + // The rollback slot must name the immediate predecessor. An equal, + // newer, or skipping revision would acknowledge a no-op or restore + // the wrong image, so it is corrupt state rather than a clean + // rollback. + if previous.revision.checked_add(1) != Some(self.snapshot.revision) { + return Err(StoreError::corrupt()); + } + let revision = self + .snapshot + .revision + .checked_add(1) + .ok_or_else(StoreError::bound)?; + let candidate = Snapshot { + revision, + rows: previous.rows.clone(), + }; + let candidate_bytes = candidate.encode()?; + let current_bytes = self.snapshot.encode()?; + self.publish_then_check_attached(&candidate_bytes, ¤t_bytes, false, || {})?; + self.snapshot = candidate; + Ok(RollbackOutcome { + revision, + restored_revision: previous.revision, + total_rows: u32::try_from(self.snapshot.rows.len()).map_err(|_| StoreError::bound())?, + total_bytes: self.snapshot.value_bytes(), + }) + } + + /// Publishes `journal -> previous -> snapshot`. + /// + /// The prior rollback image (bytes or absence) is captured before step 2 + /// so a pre-publication failure can put it back and report `Io` truthfully: + /// the snapshot and its rollback history are unchanged. A failure after a + /// rename may already be visible, so it stays `Uncertain` and keeps the + /// journal for recovery. + fn publish( + &self, + candidate_bytes: &[u8], + current_bytes: &[u8], + first_commit: bool, + ) -> Result<(), StoreError> { + let journal = Journal::encode(candidate_bytes)?; + // Capture before any mutation: a failed snapshot publication must + // restore this image (or its absence) instead of leaving revision N in + // both slots and silently destroying the true N-1 rollback image. + let prior_previous = read_state(&self.dir, PREVIOUS_FILE, MAX_SNAPSHOT_BYTES)?; + fs_at::write_atomic_unique( + &self.dir, + JOURNAL_FILE, + &journal, + JOURNAL_MAX_BYTES, + "journal", + ) + .map_err(map_write_error)?; + let previous_result = if first_commit { + // A prior namespace lifetime may have left a previous image; + // revision 0 must not be rollback-able. + match fs_at::remove_file_durably_at(&self.dir, PREVIOUS_FILE) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(_) => Err(StoreError::uncertain()), + } + } else { + fs_at::write_atomic_unique( + &self.dir, + PREVIOUS_FILE, + current_bytes, + MAX_SNAPSHOT_BYTES, + "previous", + ) + .map_err(map_write_error) + }; + if let Err(error) = previous_result { + if error.kind() == StoreErrorKind::Uncertain { + return Err(error); + } + // A non-uncertain step-2 failure is pre-rename, so `previous` is + // still the prior image; only the journal has to go. + return self.abort_publish(error); + } + if let Err(source) = fs_at::write_atomic_unique( + &self.dir, + SNAPSHOT_FILE, + candidate_bytes, + MAX_SNAPSHOT_BYTES, + "snapshot", + ) { + let error = map_write_error(source); + if error.kind() == StoreErrorKind::Uncertain { + // The target may already be visible, so `previous` must stay + // the N-1 image written by step 2; recovery completes it. + return Err(error); + } + self.restore_previous(prior_previous.as_deref())?; + return self.abort_publish(error); + } + self.ensure_attached() + .map_err(|_| StoreError::uncertain())?; + match fs_at::remove_file_durably_at(&self.dir, JOURNAL_FILE) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err(StoreError::uncertain()), + } + self.ensure_attached().map_err(|_| StoreError::uncertain()) + } + + /// Restores the pre-commit `previous` image, or removes the member when it + /// was absent. Failure to confirm the restoration is `Uncertain`, never a + /// plain `Io`, because the rollback slot may now name the current snapshot. + fn restore_previous(&self, prior_previous: Option<&[u8]>) -> Result<(), StoreError> { + #[cfg(test)] + if FAIL_PREVIOUS_RESTORE.with(std::cell::Cell::get) { + FAIL_PREVIOUS_RESTORE.with(|flag| flag.set(false)); + return Err(StoreError::uncertain()); + } + match prior_previous { + Some(bytes) => fs_at::write_atomic_unique( + &self.dir, + PREVIOUS_FILE, + bytes, + MAX_SNAPSHOT_BYTES, + "previous", + ) + .map_err(|_| StoreError::uncertain()), + None => match fs_at::remove_file_durably_at(&self.dir, PREVIOUS_FILE) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(_) => Err(StoreError::uncertain()), + }, + } + } + + /// Removes the aborted journal durably and returns the original error. A + /// cleanup failure is `Uncertain`: recovery may still replay the target. + fn abort_publish(&self, error: StoreError) -> Result<(), StoreError> { + match fs_at::remove_file_durably_at(&self.dir, JOURNAL_FILE) { + Ok(()) => Err(error), + Err(source) if source.kind() == io::ErrorKind::NotFound => Err(error), + Err(_) => Err(StoreError::uncertain()), + } + } + + fn publish_then_check_attached( + &self, + candidate_bytes: &[u8], + current_bytes: &[u8], + first_commit: bool, + after_publish: impl FnOnce(), + ) -> Result<(), StoreError> { + self.publish(candidate_bytes, current_bytes, first_commit)?; + after_publish(); + self.ensure_attached().map_err(|_| StoreError::uncertain()) + } +} + +// ── journal ───────────────────────────────────────────────────────────────── + +struct Journal; + +impl Journal { + fn encode(snapshot_bytes: &[u8]) -> Result, StoreError> { + let mut out = Vec::new(); + out.extend_from_slice(JOURNAL_MAGIC); + out.extend_from_slice(&FORMAT_VERSION.to_be_bytes()); + let revision = snapshot_revision(snapshot_bytes)?; + out.extend_from_slice(&revision.to_be_bytes()); + out.extend_from_slice( + &u32::try_from(snapshot_bytes.len()) + .map_err(|_| StoreError::bound())? + .to_be_bytes(), + ); + out.extend_from_slice(snapshot_bytes); + let checksum = checksum(&out); + out.extend_from_slice(&checksum.to_be_bytes()); + if out.len() > JOURNAL_MAX_BYTES { + return Err(StoreError::bound()); + } + Ok(out) + } + + fn decode(bytes: &[u8]) -> Result { + if bytes.len() > JOURNAL_MAX_BYTES { + return Err(StoreError::bound()); + } + let mut cursor = Cursor::new(bytes); + if cursor.take(JOURNAL_MAGIC.len())? != JOURNAL_MAGIC { + return Err(StoreError::corrupt()); + } + if cursor.u32()? != FORMAT_VERSION { + return Err(StoreError::corrupt()); + } + let target_revision = cursor.u64()?; + let payload_len = cursor.u32()? as usize; + if payload_len == 0 || payload_len > MAX_SNAPSHOT_BYTES { + return Err(StoreError::corrupt()); + } + let payload = cursor.take(payload_len)?; + let stored = cursor.u64()?; + if !cursor.is_empty() { + return Err(StoreError::corrupt()); + } + if checksum(&bytes[..bytes.len() - 8]) != stored { + return Err(StoreError::corrupt()); + } + let snapshot = Snapshot::decode(payload)?; + if snapshot.revision != target_revision { + return Err(StoreError::corrupt()); + } + Ok(snapshot) + } +} + +fn snapshot_revision(bytes: &[u8]) -> Result { + let offset = SNAPSHOT_MAGIC.len() + 4; + if bytes.len() < offset + 8 { + return Err(StoreError::corrupt()); + } + let revision = u64::from_be_bytes( + bytes[offset..offset + 8] + .try_into() + .map_err(|_| StoreError::corrupt())?, + ); + Ok(revision) +} + +// ── recovery ──────────────────────────────────────────────────────────────── + +fn recover(dir: &File, root: &File, namespace: &str) -> Result { + let journal_bytes = read_state(dir, JOURNAL_FILE, JOURNAL_MAX_BYTES)?; + let snapshot_bytes = match read_state(dir, SNAPSHOT_FILE, MAX_SNAPSHOT_BYTES) { + Ok(bytes) => bytes, + // A durable journal names a target that may already be installed. + // Without the snapshot bytes we cannot tell whether it became + // visible, so an I/O failure must not invite a blind retry from a + // cached revision. Structural failures keep their typed kind. + Err(error) if journal_bytes.is_some() && error.kind() == StoreErrorKind::Io => { + return Err(StoreError::uncertain()); + } + Err(error) => return Err(error), + }; + + let current = snapshot_bytes.as_deref().map(Snapshot::decode).transpose(); + + let Some(journal_bytes) = journal_bytes else { + return match current { + Ok(snapshot) => Ok(snapshot.unwrap_or_else(Snapshot::empty)), + Err(error) => Err(error), + }; + }; + let target = Journal::decode(&journal_bytes)?; + match ¤t { + Ok(Some(snapshot)) if snapshot.revision >= target.revision => { + ensure_namespace_attached(root, namespace, dir)?; + fs_at::remove_file_durably_at(dir, JOURNAL_FILE) + .map_err(|_| StoreError::uncertain())?; + return Ok(snapshot.clone()); + } + _ => {} + } + // Complete step 2 of the interrupted commit before publishing step 3, so + // `previous` names the pre-commit state after recovery and a later + // rollback cannot skip a revision. The write is idempotent when step 2 + // already ran. With no snapshot (first commit), a stale `previous` from a + // prior namespace lifetime must be removed so revision 1 is not + // rollback-able to unrelated state. An unreadable snapshot leaves + // `previous` untouched because the pre-commit state is unknowable. + // + // The journal is already the durable intent, so every failure while + // completing it is uncertain: the target may become visible on the next + // open even though this recovery attempt failed. Only pre-replay reads + // and journal decoding keep their typed corrupt/io classification. + match (¤t, snapshot_bytes.as_deref()) { + (Ok(Some(_)), Some(bytes)) => { + fs_at::write_atomic_unique(dir, PREVIOUS_FILE, bytes, MAX_SNAPSHOT_BYTES, "previous") + .map_err(|_| StoreError::uncertain())?; + } + (Ok(None), _) => match fs_at::remove_file_at(dir, PREVIOUS_FILE) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err(StoreError::uncertain()), + }, + _ => {} + } + // A valid journal is the durable intent, so complete it even when the + // snapshot is missing or unreadable (atomic renames prevent a torn + // snapshot, but a corrupt file must not strand a committed target). + let target_bytes = target.encode()?; + fs_at::write_atomic_unique( + dir, + SNAPSHOT_FILE, + &target_bytes, + MAX_SNAPSHOT_BYTES, + "snapshot", + ) + .map_err(|_| StoreError::uncertain())?; + ensure_namespace_attached(root, namespace, dir)?; + fs_at::remove_file_durably_at(dir, JOURNAL_FILE).map_err(|_| StoreError::uncertain())?; + Ok(target) +} + +fn recover_then_check_attached( + dir: &File, + root: &File, + namespace: &str, + after_recovery: impl FnOnce(), +) -> Result { + let snapshot = recover(dir, root, namespace)?; + after_recovery(); + ensure_namespace_attached(root, namespace, dir)?; + Ok(snapshot) +} + +fn ensure_namespace_attached(root: &File, namespace: &str, dir: &File) -> Result<(), StoreError> { + match fs_at::directory_matches(root, namespace, dir) { + Ok(true) => Ok(()), + Ok(false) => Err(StoreError::uncertain()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Err(StoreError::uncertain()), + Err(_) => Err(StoreError::uncertain()), + } +} + +// ── error mapping ─────────────────────────────────────────────────────────── + +fn map_namespace_error(error: io::Error) -> StoreError { + match error.kind() { + io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => StoreError::invalid_name(), + _ => StoreError::io(), + } +} + +fn map_state_error(error: io::Error) -> StoreError { + match error.kind() { + io::ErrorKind::InvalidInput => StoreError::corrupt(), + io::ErrorKind::InvalidData => StoreError::bound(), + _ => StoreError::io(), + } +} + +fn map_lock_error(error: io::Error) -> StoreError { + match error.kind() { + io::ErrorKind::WouldBlock => StoreError::locked(), + io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => StoreError::corrupt(), + _ => StoreError::io(), + } +} + +fn map_write_error(error: io::Error) -> StoreError { + if fs_at::is_uncertain_durability_error(&error) { + return StoreError::uncertain(); + } + match error.kind() { + io::ErrorKind::InvalidInput => StoreError::bound(), + _ => StoreError::io(), + } +} + +fn read_state(dir: &File, name: &str, cap: usize) -> Result>, StoreError> { + fs_at::read_file_at(dir, name, cap).map_err(map_state_error) +} + +// ── codec helpers ─────────────────────────────────────────────────────────── + +fn checksum(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +struct Cursor<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> Cursor<'a> { + const fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], StoreError> { + let end = self + .position + .checked_add(len) + .ok_or_else(StoreError::corrupt)?; + if end > self.bytes.len() { + return Err(StoreError::corrupt()); + } + let slice = &self.bytes[self.position..end]; + self.position = end; + Ok(slice) + } + + fn u32(&mut self) -> Result { + let bytes: [u8; 4] = self + .take(4)? + .try_into() + .map_err(|_| StoreError::corrupt())?; + Ok(u32::from_be_bytes(bytes)) + } + + fn u64(&mut self) -> Result { + let bytes: [u8; 8] = self + .take(8)? + .try_into() + .map_err(|_| StoreError::corrupt())?; + Ok(u64::from_be_bytes(bytes)) + } + + fn is_empty(&self) -> bool { + self.position == self.bytes.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::component::store::NamespaceName; + + #[test] + fn unconfirmed_previous_restore_is_uncertain_and_recovery_completes_the_commit() { + let root_path = + std::env::temp_dir().join(format!("restore-failure-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root_path); + std::fs::create_dir_all(&root_path).unwrap(); + let root = StorageRoot::new(&root_path).unwrap(); + let mut namespace = root + .open_namespace(&NamespaceName::parse("ns").unwrap()) + .unwrap(); + namespace + .commit( + 0, + vec![Row { + key: "a".to_owned(), + value: b"1".to_vec(), + }], + Vec::new(), + ) + .unwrap(); + namespace + .commit( + 1, + vec![Row { + key: "b".to_owned(), + value: b"2".to_vec(), + }], + Vec::new(), + ) + .unwrap(); + + // Block the snapshot rename so publication fails after `previous` was + // rewritten, then force the rollback-image restoration to fail too. + let dir = root_path.join("ns"); + std::fs::remove_file(dir.join("snapshot")).unwrap(); + std::fs::create_dir(dir.join("snapshot")).unwrap(); + FAIL_PREVIOUS_RESTORE.with(|flag| flag.set(true)); + let error = namespace + .commit( + 2, + vec![Row { + key: "c".to_owned(), + value: b"3".to_vec(), + }], + Vec::new(), + ) + .expect_err("an unconfirmed restoration must not be reported as io"); + assert_eq!(error.kind(), StoreErrorKind::Uncertain); + assert!( + dir.join("journal").exists(), + "recovery evidence must survive an unconfirmed restoration" + ); + + // Repair the synthetic snapshot fault; the real pre-commit snapshot is + // revision 2 and the retained journal must complete revision 3. + std::fs::remove_dir(dir.join("snapshot")).unwrap(); + let current = Snapshot { + revision: 2, + rows: [ + ("a".to_owned(), b"1".to_vec()), + ("b".to_owned(), b"2".to_vec()), + ] + .into_iter() + .collect(), + }; + std::fs::write(dir.join("snapshot"), current.encode().unwrap()).unwrap(); + drop(namespace); + + let mut reopened = root + .open_namespace(&NamespaceName::parse("ns").unwrap()) + .expect("recovery must complete once the filesystem is repaired"); + assert_eq!(reopened.revision(), 3); + let rolled = reopened.rollback(3).expect("rollback after recovery"); + assert_eq!(rolled.restored_revision, 2); + let _ = std::fs::remove_dir_all(&root_path); + } + + #[test] + fn replacement_after_publish_is_uncertain_not_acknowledged() { + let root_path = std::env::temp_dir().join(format!("attached-race-{}", std::process::id())); + let detached = std::env::temp_dir().join(format!("detached-race-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root_path); + let _ = std::fs::remove_dir_all(&detached); + std::fs::create_dir_all(&root_path).unwrap(); + std::fs::create_dir_all(&detached).unwrap(); + let root = StorageRoot::new(&root_path).unwrap(); + let mut namespace = root + .open_namespace(&NamespaceName::parse("race").unwrap()) + .unwrap(); + namespace + .commit(0, Vec::new(), Vec::new()) + .expect("initial commit"); + let candidate = Snapshot { + revision: 2, + rows: BTreeMap::new(), + } + .encode() + .unwrap(); + let current = Snapshot { + revision: 1, + rows: BTreeMap::new(), + } + .encode() + .unwrap(); + + let error = namespace + .publish_then_check_attached(&candidate, ¤t, false, || { + std::fs::rename(root_path.join("race"), root_path.join("race-detached")).unwrap(); + std::os::unix::fs::symlink(&detached, root_path.join("race")).unwrap(); + }) + .expect_err("replacement after publish must not be acknowledged"); + assert_eq!(error.kind(), StoreErrorKind::Uncertain); + assert!(root_path.join("race-detached/snapshot").exists()); + let _ = std::fs::remove_dir_all(root_path); + let _ = std::fs::remove_dir_all(detached); + } + + #[test] + fn replacement_after_open_recovery_is_uncertain_not_acknowledged() { + let root_path = std::env::temp_dir().join(format!("recovery-race-{}", std::process::id())); + let detached_path = + std::env::temp_dir().join(format!("recovery-detached-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root_path); + let _ = std::fs::remove_dir_all(&detached_path); + std::fs::create_dir_all(&root_path).unwrap(); + std::fs::create_dir_all(&detached_path).unwrap(); + let root = StorageRoot::new(&root_path).unwrap(); + let name = NamespaceName::parse("recovery").unwrap(); + let opened = root.open_namespace(&name).unwrap(); + drop(opened); + let dir = fs_at::open_dir_at(&root.root, name.as_str()).unwrap(); + + let result = recover_then_check_attached(&dir, &root.root, name.as_str(), || { + std::fs::rename( + root_path.join("recovery"), + root_path.join("recovery-detached"), + ) + .unwrap(); + std::os::unix::fs::symlink(&detached_path, root_path.join("recovery")).unwrap(); + }); + assert_eq!( + result + .expect_err("must not acknowledge replaced namespace") + .kind(), + StoreErrorKind::Uncertain + ); + let _ = std::fs::remove_dir_all(root_path); + let _ = std::fs::remove_dir_all(detached_path); + } +} diff --git a/src/component/store/unsupported.rs b/src/component/store/unsupported.rs new file mode 100644 index 0000000..efc9af1 --- /dev/null +++ b/src/component/store/unsupported.rs @@ -0,0 +1,103 @@ +//! Fail-closed namespace backend for platforms without the anchored +//! directory-fd implementation. +//! +//! Linux x86_64 is the packaged target for this task. Returning +//! `Unsupported` from construction is deliberate: a path-based fallback would +//! re-introduce the symlink/ancestor races this backend exists to remove. + +use std::path::PathBuf; + +use crate::component::protocol::Row; + +use super::{CommitOutcome, NamespaceName, RollbackOutcome, SnapshotPage, StoreError}; + +/// Unsupported storage root. +#[derive(Debug)] +pub struct StorageRoot; + +impl StorageRoot { + /// Fails closed on this platform. + /// + /// # Errors + /// + /// Always returns [`StoreErrorKind::Unsupported`](super::StoreErrorKind::Unsupported). + pub fn new(_path: impl Into) -> Result { + Err(StoreError::unsupported()) + } + + /// Fails closed on this platform. + /// + /// # Errors + /// + /// Always returns [`StoreErrorKind::Unsupported`](super::StoreErrorKind::Unsupported). + pub fn open_namespace(&self, _name: &NamespaceName) -> Result { + Err(StoreError::unsupported()) + } +} + +/// Unsupported namespace handle. +#[derive(Debug)] +pub struct Namespace; + +impl Namespace { + /// Always zero; no namespace can be opened on this platform. + #[must_use] + pub const fn revision(&self) -> u64 { + 0 + } + + /// Always zero; no namespace can be opened on this platform. + #[must_use] + pub const fn row_count(&self) -> usize { + 0 + } + + /// Always zero; no namespace can be opened on this platform. + #[must_use] + pub const fn value_bytes(&self) -> u64 { + 0 + } + + /// Fails closed on this platform; no cached handle can exist because + /// [`StorageRoot::open_namespace`] always returns `Unsupported`. + pub(crate) fn ensure_attached(&self) -> Result<(), StoreError> { + Err(StoreError::unsupported()) + } + + /// Fails closed on this platform. + /// + /// # Errors + /// + /// Always returns [`StoreErrorKind::Unsupported`](super::StoreErrorKind::Unsupported). + pub fn read_page( + &self, + _start_row: u32, + _max_rows: u32, + _max_bytes: u32, + ) -> Result { + Err(StoreError::unsupported()) + } + + /// Fails closed on this platform. + /// + /// # Errors + /// + /// Always returns [`StoreErrorKind::Unsupported`](super::StoreErrorKind::Unsupported). + pub fn commit( + &mut self, + _expected_revision: u64, + _upserts: Vec, + _deletes: Vec, + ) -> Result { + Err(StoreError::unsupported()) + } + + /// Fails closed on this platform. + /// + /// # Errors + /// + /// Always returns [`StoreErrorKind::Unsupported`](super::StoreErrorKind::Unsupported). + pub fn rollback(&mut self, _expected_revision: u64) -> Result { + Err(StoreError::unsupported()) + } +} diff --git a/src/fs_at.rs b/src/fs_at.rs new file mode 100644 index 0000000..05c2d4b --- /dev/null +++ b/src/fs_at.rs @@ -0,0 +1,493 @@ +//! Directory-fd anchored filesystem operations (Linux/Android). +//! +//! The component store never trusts a path after the namespace directory has +//! been opened. Every operation below is relative to an already-open directory +//! fd, uses `O_NOFOLLOW` on the final component, and creates temporary files +//! with `O_CREAT|O_EXCL` so a pre-planted symlink is never followed. A racing +//! rename or replacement of an ancestor therefore cannot redirect a read, +//! write, rename, or unlink outside the opened directory. +//! +//! Pathname checks here (`directory_matches`, the lock replacement check) are +//! detection points, not leases or acknowledgements. They prove the +//! device/inode identity at the instant of the `statat` call; a racing +//! rename/replacement after that call is not prevented by them. The fd +//! anchoring remains in force either way, so a late replacement can only +//! detach the pathname, never redirect already-issued operations. +//! +//! This module is deliberately Linux/Android only. Other platforms fail +//! closed in `component::store` rather than falling back to path-based state +//! operations for the component. + +use std::fs::File; +use std::io::{self, Read as _, Write as _}; +use std::mem::MaybeUninit; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt as _; +use std::path::{Component, Path}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use rustix::fs::{ + AtFlags, FileType, FlockOperation, Mode, OFlags, RawDir, flock, fstat, fsync, mkdirat, open, + openat, renameat, statat, unlinkat, +}; +use rustix::io::Errno; +use rustix::path::Arg; + +/// Minimum temp-sibling age before the anchored sweep treats it as crash +/// litter. Live writers hold their temps for milliseconds. +pub(crate) const STALE_TEMP_AGE_SECS: u64 = 3_600; + +/// Bounded retries when an attacker has pre-created a candidate temp name. +const TEMP_ATTEMPTS: usize = 16; + +static TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug)] +struct DirectorySyncFailure; + +impl std::fmt::Display for DirectorySyncFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("directory fsync failed after mutation") + } +} + +impl std::error::Error for DirectorySyncFailure {} + +/// Returns whether a mutation (rename or unlink) completed but the following +/// directory fsync failed, so its durability could not be confirmed. +pub(crate) fn is_uncertain_durability_error(error: &io::Error) -> bool { + error + .get_ref() + .is_some_and(|source| source.is::()) +} + +fn invalid(reason: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, reason) +} + +/// Opens a directory path itself with `O_DIRECTORY|O_NOFOLLOW`. Used once for +/// the host-chosen storage root; every later operation is `*at` relative. +pub(crate) fn open_dir_path(path: &Path) -> io::Result { + let fd = open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW, + Mode::empty(), + ) + .map_err(map_final_component_error)?; + Ok(File::from(fd)) +} + +/// Opens a directory path for the crate's generic (non-component) helpers. +/// +/// A final symlink to a directory is followed, matching normal path semantics +/// for caller-owned state directories (`FileBackend`, session, KV). Every +/// later operation is still fd-relative, so replacing an ancestor after this +/// open cannot redirect those operations. The component store never uses this +/// opener; it stays no-follow and anchored through [`open_dir_path`]. +pub(crate) fn open_dir_path_following(path: &Path) -> io::Result { + let fd = open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| match error { + Errno::LOOP | Errno::MLINK | Errno::NOTDIR => invalid("symlink or non-directory"), + other => other.into(), + })?; + Ok(File::from(fd)) +} + +/// Creates and opens a directory path by walking each component from an +/// already-open `/` or `.` directory. Symlinked components and `..` are +/// refused; no canonicalize-then-open window exists. +pub(crate) fn create_and_open_dir_tree(path: &Path) -> io::Result { + if path + .components() + .any(|component| matches!(component, Component::ParentDir | Component::Prefix(_))) + { + return Err(invalid("unsafe root path")); + } + let mut current = if path.is_absolute() { + open_dir_path(Path::new("/"))? + } else { + open_dir_path(Path::new("."))? + }; + for component in path.components() { + match component { + Component::RootDir | Component::CurDir => {} + Component::Normal(name) => { + create_dir_at(¤t, name)?; + current = open_dir_at(¤t, name)?; + } + Component::ParentDir | Component::Prefix(_) => unreachable!("validated above"), + } + } + Ok(current) +} + +/// Opens one child directory without following a symlink. +pub(crate) fn open_dir_at(parent: &File, name: P) -> io::Result { + let fd = openat( + parent, + name, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW, + Mode::empty(), + ) + .map_err(map_final_component_error)?; + Ok(File::from(fd)) +} + +/// Creates one child directory mode 0700; an existing entry is accepted and +/// validated by the following `open_dir_at`. +pub(crate) fn create_dir_at(parent: &File, name: P) -> io::Result<()> { + match mkdirat(parent, name, Mode::from_raw_mode(0o700)) { + Ok(()) | Err(Errno::EXIST) => Ok(()), + Err(error) => Err(error.into()), + } +} + +/// Returns `true` when `name` is a regular file, `false` when absent, and +/// `InvalidInput` for a symlink or any other non-regular entry. No open is +/// performed, so a FIFO or device cannot block or have side effects here. +pub(crate) fn regular_or_absent(parent: &File, name: P) -> io::Result { + match statat(parent, name, AtFlags::SYMLINK_NOFOLLOW) { + Ok(stat) => { + if FileType::from_raw_mode(stat.st_mode).is_file() { + Ok(true) + } else { + Err(invalid("state member is not a regular file")) + } + } + Err(Errno::NOENT) => Ok(false), + Err(error) => Err(error.into()), + } +} + +/// Opens and exclusively `flock`s the namespace lock member. +/// +/// `O_NOFOLLOW` (without `O_EXCL`) rejects a symlink without creating its +/// target. The opened object is verified as a regular file with one link +/// before the lock is taken. After locking, the opened fd and the current +/// `lock` entry are compared by device/inode so a replacement between `openat` +/// and `flock` is refused instead of locking a detached inode. That pair is a +/// best-effort detection point: a later replacement is not prevented, but the +/// lock already held on the opened inode keeps its exclusion semantics for +/// this namespace handle. Contention is reported as `WouldBlock`. +pub(crate) fn open_lock_at(parent: &File, name: P) -> io::Result { + open_lock_at_with(parent, name, || {}) +} + +fn open_lock_at_with( + parent: &File, + name: P, + after_open: impl FnOnce(), +) -> io::Result { + let fd = openat( + parent, + name, + OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK, + Mode::from_raw_mode(0o600), + ) + .map_err(map_final_component_error)?; + let file = File::from(fd); + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(invalid("lock member is not a regular file")); + } + #[cfg(unix)] + if metadata.nlink() != 1 { + return Err(invalid("lock member must not have hard links")); + } + after_open(); + match flock(&file, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => { + let opened = fstat(&file)?; + let named = statat(parent, name, AtFlags::SYMLINK_NOFOLLOW)?; + if opened.st_dev != named.st_dev || opened.st_ino != named.st_ino { + return Err(invalid("lock member was replaced while opening")); + } + Ok(file) + } + Err(error) => Err(error.into()), + } +} + +/// Reads one regular file relative to `parent`, capped at `cap` bytes. +/// +/// `O_NOFOLLOW|O_NONBLOCK` prevents symlink following and FIFO blocking; the +/// fd metadata check rejects every non-regular object before any byte is +/// consumed. `None` means the entry is absent. +pub(crate) fn read_file_at( + parent: &File, + name: P, + cap: usize, +) -> io::Result>> { + let fd = match openat( + parent, + name, + OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK, + Mode::empty(), + ) { + Ok(fd) => fd, + Err(Errno::NOENT) => return Ok(None), + Err(error) => return Err(map_final_component_error(error)), + }; + let file = File::from(fd); + if !file.metadata()?.is_file() { + return Err(invalid("state member is not a regular file")); + } + let limit = u64::try_from(cap).unwrap_or(u64::MAX).saturating_add(1); + let mut reader = file.take(limit); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; + if bytes.len() > cap { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "state member exceeds ceiling", + )); + } + Ok(Some(bytes)) +} + +/// Removes one child entry without following it. `NotFound` is returned as +/// such so callers can distinguish an absent member. +pub(crate) fn remove_file_at(parent: &File, name: P) -> io::Result<()> { + unlinkat(parent, name, AtFlags::empty()).map_err(Into::into) +} + +/// Removes one child and confirms the directory entry update with fsync. +pub(crate) fn remove_file_durably_at(parent: &File, name: P) -> io::Result<()> { + remove_file_at(parent, name)?; + fsync(parent).map_err(|_| io::Error::other(DirectorySyncFailure)) +} + +/// Whether the directory fd `dir` is still the directory currently named +/// `name` under `root` (same device and inode, still a real directory). +/// +/// This is a best-effort detection point, not a guarantee about when a +/// response is written: a rename/replacement after the `statat` returns is not +/// observed. A detected mismatch (including a symlink replacement, which +/// `statat` does not follow) lets the caller fail closed instead of +/// acknowledging a write against an orphaned directory. +pub(crate) fn directory_matches( + root: &File, + name: P, + dir: &File, +) -> io::Result { + let named = statat(root, name, AtFlags::SYMLINK_NOFOLLOW)?; + let actual = fstat(dir)?; + Ok(FileType::from_raw_mode(named.st_mode).is_dir() + && named.st_dev == actual.st_dev + && named.st_ino == actual.st_ino) +} + +/// Atomically writes `data` to `name` through an explicitly supplied temp +/// name. The temp is created `O_CREAT|O_EXCL|O_NOFOLLOW`; an existing entry, +/// including a symlink, is never followed or truncated. +pub(crate) fn write_atomic_named( + parent: &File, + name: P, + temp: Q, + data: &[u8], + cap: usize, +) -> io::Result<()> { + write_atomic_named_with_sync(parent, name, temp, data, cap, |directory| { + fsync(directory).map_err(io::Error::from) + }) +} + +fn write_atomic_named_with_sync( + parent: &File, + name: P, + temp: Q, + data: &[u8], + cap: usize, + sync_directory: impl FnOnce(&File) -> io::Result<()>, +) -> io::Result<()> { + if data.len() > cap { + return Err(invalid("payload exceeds ceiling")); + } + let fd = openat( + parent, + temp, + OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW, + Mode::from_raw_mode(0o600), + ) + .map_err(map_final_component_error)?; + let mut file = File::from(fd); + let result = (|| -> io::Result<()> { + file.write_all(data)?; + file.sync_all()?; + drop(file); + renameat(parent, temp, parent, name)?; + // If this fails the rename may already be visible, so report the + // error and let callers classify the commit as uncertain. + sync_directory(parent).map_err(|_| io::Error::other(DirectorySyncFailure))?; + Ok(()) + })(); + if result.is_err() { + let _ = unlinkat(parent, temp, AtFlags::empty()); + } + result +} + +/// Atomically writes `data` to `name` using a fresh exclusive temp name. +/// `AlreadyExists` from a pre-created candidate is retried a bounded number +/// of times; every other error is returned. +pub(crate) fn write_atomic_unique( + parent: &File, + name: P, + data: &[u8], + cap: usize, + stem: &str, +) -> io::Result<()> { + let mut last = None; + for _ in 0..TEMP_ATTEMPTS { + let temp = temp_name(stem); + match write_atomic_named(parent, name, temp.as_str(), data, cap) { + Ok(()) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + last = Some(error); + } + Err(error) => return Err(error), + } + } + Err(last.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::AlreadyExists, + "temp name candidates exhausted", + ) + })) +} + +/// Removes aged `.tmp.*` siblings relative to `parent`, never following +/// a symlink and never deleting a directory. Age is checked with `statat` +/// `AT_SYMLINK_NOFOLLOW`; unknown or future mtimes are kept. +pub(crate) fn clean_temp_siblings_at(parent: &File, stem: &str) -> io::Result<()> { + let dot_prefix = format!("{stem}.tmp."); + let dash_prefix = format!("{stem}.tmp-"); + let now = now_secs(); + let mut buffer = vec![MaybeUninit::::uninit(); 8192]; + let mut entries = RawDir::new(parent, &mut buffer); + while let Some(entry) = entries.next() { + let entry = entry?; + let name = entry.file_name().to_string_lossy().into_owned(); + if !(name.starts_with(&dot_prefix) || name.starts_with(&dash_prefix)) { + continue; + } + let Ok(stat) = statat(parent, entry.file_name(), AtFlags::SYMLINK_NOFOLLOW) else { + continue; + }; + let Ok(mtime) = u64::try_from(stat.st_mtime) else { + continue; + }; + if now >= mtime && now - mtime >= STALE_TEMP_AGE_SECS { + let _ = unlinkat(parent, entry.file_name(), AtFlags::empty()); + } + } + Ok(()) +} + +fn temp_name(stem: &str) -> String { + format!( + "{stem}.tmp-{}-{:?}-{}", + std::process::id(), + std::thread::current().id(), + TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ) +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()) +} + +fn map_final_component_error(error: Errno) -> io::Error { + match error { + Errno::LOOP | Errno::MLINK | Errno::NOTDIR => invalid("symlink or non-directory"), + other => other.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_dir(name: &str) -> std::path::PathBuf { + let root = std::env::temp_dir().join(format!( + "fs-at-{name}-{}-{}", + std::process::id(), + TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir(&root).unwrap(); + root + } + + #[test] + fn directory_sync_failure_reports_renamed_but_uncertain_commit() { + let root = scratch_dir("sync"); + let dir = open_dir_path(&root).unwrap(); + let error = write_atomic_named_with_sync( + &dir, + "snapshot", + "snapshot.tmp", + b"committed bytes", + 64, + |_| Err(io::Error::other("injected directory fsync failure")), + ) + .expect_err("directory sync error must be returned"); + assert!(is_uncertain_durability_error(&error)); + assert_eq!( + std::fs::read(root.join("snapshot")).unwrap(), + b"committed bytes" + ); + assert!(!root.join("snapshot.tmp").exists()); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn hardlinked_lock_member_is_refused() { + let root = scratch_dir("lock-hardlink"); + std::fs::write(root.join("lock"), b"lock").unwrap(); + std::fs::hard_link(root.join("lock"), root.join("lock-copy")).unwrap(); + let dir = open_dir_path(&root).unwrap(); + let error = + open_lock_at(&dir, "lock").expect_err("a lock member with two links must be refused"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!( + error.to_string().contains("hard links"), + "diagnostic must name the hard-link refusal: {error}" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn lock_replaced_while_opening_is_refused() { + let root = scratch_dir("lock-replaced"); + let dir = open_dir_path(&root).unwrap(); + let error = open_lock_at_with(&dir, "lock", || { + std::fs::rename(root.join("lock"), root.join("lock-detached")).unwrap(); + std::fs::write(root.join("lock"), b"replacement").unwrap(); + }) + .expect_err("a lock replaced between openat and flock must be refused"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!( + error.to_string().contains("replaced"), + "diagnostic must name the replacement refusal: {error}" + ); + assert_eq!( + std::fs::read(root.join("lock")).unwrap(), + b"replacement", + "the check must not follow or rewrite the replacement" + ); + assert_eq!( + std::fs::read(root.join("lock-detached")).unwrap(), + b"", + "the originally opened lock must stay untouched" + ); + let _ = std::fs::remove_dir_all(root); + } +} diff --git a/src/kv.rs b/src/kv.rs index e08af0d..74e9945 100644 --- a/src/kv.rs +++ b/src/kv.rs @@ -56,7 +56,10 @@ pub enum StoreErrorCode { ValueInvalid, /// Entry-count or total-bytes quota exceeded. Quota, - /// Durable commit failed; previous state is intact. + /// Durable commit failed. A pre-rename failure leaves the previous state + /// intact; a post-rename durability failure is reported as this same code + /// even though the new image may already be visible, so callers that need + /// exactness reload before retrying. Io, } @@ -317,6 +320,10 @@ impl KvStore { } buffer.push('}'); let temp = unique_temp_sibling_for(path); + // The KV contract has a single `E_STORE_IO` failure code. A + // post-rename directory-durability failure is reported through the + // same code, so a caller that needs exactness reloads this file + // before retrying even though the in-memory entries were not updated. write_atomic_durably(path, buffer.as_bytes(), &temp).map_err(|_| { StoreError::new(StoreErrorCode::Io, "could not commit the plugin state file") })?; diff --git a/src/lib.rs b/src/lib.rs index 7cae7a3..6278e57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,9 @@ pub mod atomic_io; pub mod ceiling; +pub mod component; +#[cfg(any(target_os = "linux", target_os = "android"))] +mod fs_at; pub mod history; pub mod kv; pub mod session_codec; @@ -44,11 +47,15 @@ pub use transcript::{RetentionPolicy, SegmentDescriptor, TranscriptDeletionEvide /// /// A backend implements the byte commit for one storage object; Core runs /// its own validation, capability, and safe/headless gates first and calls -/// through this seam only afterwards. Implementations must be atomic (a -/// failed commit leaves the previous state intact), bounded (every -/// [`ceiling`] bound enforced fail-closed), and content-free in errors. -/// Deletion must remove bytes and produce evidence; a post-purge -/// export/query returns nothing. +/// through this seam only afterwards. Implementations are atomic before +/// rename: a pre-rename failure leaves the previous state intact. After a +/// rename, an implementation may have published the new bytes while failing +/// to confirm durability; it exposes that through its error type (for +/// [`FileBackend`], [`atomic_io::IoError::is_uncertain`]) and the caller must +/// reload before retrying. Implementations are bounded (every [`ceiling`] +/// bound enforced fail-closed) and content-free in errors. Deletion must +/// remove bytes and produce evidence; a post-purge export/query returns +/// nothing. pub trait StorageBackend { /// Error type with content-free display (safe for logs). type Error: std::error::Error; diff --git a/tests/atomic_io.rs b/tests/atomic_io.rs index 13c0c3f..fc2d60c 100644 --- a/tests/atomic_io.rs +++ b/tests/atomic_io.rs @@ -43,6 +43,41 @@ fn save_and_load_round_trip() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn missing_parent_load_is_not_found_not_io() { + let dir = scratch_dir("missing-parent"); + let path = dir.join("absent").join("session"); + assert_eq!( + load_session_bytes(&path), + Err(LoadError::NotFound), + "a clean store without its parent directory must load as empty" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg(unix)] +fn save_and_load_follow_a_symlinked_state_directory() { + let dir = scratch_dir("symlinked-state"); + let actual = dir.join("actual"); + std::fs::create_dir(&actual).unwrap(); + let link = dir.join("state"); + std::os::unix::fs::symlink(&actual, &link).unwrap(); + let path = link.join("session"); + save_session_bytes(&path, b"through-link").unwrap(); + assert_eq!( + std::fs::read(actual.join("session")).unwrap(), + b"through-link", + "the commit must land in the symlink target" + ); + assert_eq!(load_session_bytes(&path).unwrap(), b"through-link"); + assert!( + std::fs::symlink_metadata(&link).unwrap().is_symlink(), + "the state-directory symlink must stay a symlink" + ); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn over_cap_save_rejected_before_any_write() { let dir = scratch_dir("overcap"); @@ -72,6 +107,33 @@ fn stale_temp_siblings_are_swept_before_writing() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn temp_sweep_only_removes_siblings_in_destination_parent() { + let root = scratch_dir("sweep-scope"); + let parent = root.join("parent"); + let namespace = parent.join("namespace"); + std::fs::create_dir_all(&namespace).unwrap(); + let target = namespace.join("session"); + let parent_litter = parent.join("session.tmp.parent"); + let root_litter = root.join("session.tmp.grandparent"); + for litter in [&parent_litter, &root_litter] { + std::fs::write(litter, b"unrelated").unwrap(); + backdate_as_litter(litter); + } + + clean_temp_siblings(&target); + + assert!( + parent_litter.exists(), + "parent litter is outside the sweep namespace" + ); + assert!( + root_litter.exists(), + "grandparent litter is outside the sweep namespace" + ); + let _ = std::fs::remove_dir_all(root); +} + #[test] fn sweep_never_deletes_live_temps() { let dir = scratch_dir("live"); @@ -437,3 +499,80 @@ fn low_level_commit_does_not_sweep_while_save_sweeps() { assert_eq!(load_session_bytes(&path).unwrap(), b"via-save"); let _ = std::fs::remove_dir_all(&dir); } + +#[test] +#[cfg(unix)] +fn temp_symlink_to_an_existing_outside_file_is_never_followed() { + let dir = scratch_dir("temp-symlink-existing"); + let outside = scratch_dir("temp-symlink-existing-outside"); + let destination = dir.join("session"); + std::fs::write(&destination, b"previous").unwrap(); + let victim = outside.join("victim"); + std::fs::write(&victim, b"outside-keep").unwrap(); + let temp = dir.join("session.tmp.attack"); + std::os::unix::fs::symlink(&victim, &temp).unwrap(); + + let result = write_atomic_durably(&destination, b"new-bytes", &temp); + assert!(result.is_err(), "a symlinked temp must be refused"); + assert_eq!( + std::fs::read(&victim).unwrap(), + b"outside-keep", + "outside target must never be written" + ); + assert_eq!( + std::fs::read(&destination).unwrap(), + b"previous", + "destination must be preserved on refusal" + ); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +#[cfg(unix)] +fn temp_symlink_to_an_absent_outside_file_creates_nothing() { + let dir = scratch_dir("temp-symlink-absent"); + let outside = scratch_dir("temp-symlink-absent-outside"); + let destination = dir.join("session"); + std::fs::write(&destination, b"previous").unwrap(); + let victim = outside.join("victim"); + let temp = dir.join("session.tmp.attack"); + std::os::unix::fs::symlink(&victim, &temp).unwrap(); + + let result = write_atomic_durably(&destination, b"new-bytes", &temp); + assert!(result.is_err(), "a symlinked temp must be refused"); + assert!( + !victim.exists(), + "an absent outside target must never be created" + ); + assert_eq!( + std::fs::read(&destination).unwrap(), + b"previous", + "destination must be preserved on refusal" + ); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +fn existing_regular_temp_is_never_truncated() { + let dir = scratch_dir("temp-existing-regular"); + let destination = dir.join("session"); + std::fs::write(&destination, b"previous").unwrap(); + let temp = dir.join("session.tmp.existing"); + std::fs::write(&temp, b"temp-keep").unwrap(); + + let result = write_atomic_durably(&destination, b"new-bytes", &temp); + assert!(result.is_err(), "an existing temp must be refused"); + assert_eq!( + std::fs::read(&temp).unwrap(), + b"temp-keep", + "an existing temp must never be truncated" + ); + assert_eq!( + std::fs::read(&destination).unwrap(), + b"previous", + "destination must be preserved on refusal" + ); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/tests/component_descriptor.rs b/tests/component_descriptor.rs new file mode 100644 index 0000000..b3a7d6a --- /dev/null +++ b/tests/component_descriptor.rs @@ -0,0 +1,111 @@ +//! Component descriptor record and binary identity checks. +//! +//! The repository's `bitty-component.toml` is the canonical record packaging +//! renders per target; these tests keep the parser, the record, the crate +//! version, and the binary name in agreement. + +use bitty_storage::component::descriptor::{ + COMPONENT_EXECUTABLE, COMPONENT_NAME, COMPONENT_PROTOCOL_MAX, COMPONENT_PROTOCOL_MIN, + ComponentDescriptor, PLACEHOLDER_SHA256, +}; +use bitty_storage::component::server::{ + COMPONENT_VERSION, SERVER_EXIT_INTERNAL, SERVER_EXIT_OK, SERVER_EXIT_PROTOCOL, +}; +use bitty_storage::component::{MAX_NAMESPACES, PROTOCOL_VERSION}; + +const RECORD: &str = include_str!("../bitty-component.toml"); + +#[test] +fn repository_record_matches_the_component_identity() { + assert_eq!(COMPONENT_VERSION, env!("CARGO_PKG_VERSION")); + let descriptor = ComponentDescriptor::parse(RECORD).expect("repository record parses"); + assert_eq!(descriptor.name, COMPONENT_NAME); + assert_eq!(descriptor.version, COMPONENT_VERSION); + assert_eq!( + (descriptor.protocol_min, descriptor.protocol_max), + (COMPONENT_PROTOCOL_MIN, COMPONENT_PROTOCOL_MAX) + ); + assert_eq!(descriptor.executable, COMPONENT_EXECUTABLE); + assert_eq!(descriptor.sha256, PLACEHOLDER_SHA256); + assert!(descriptor.has_placeholder_digest()); + descriptor + .validate_identity() + .expect("identity matches the binary"); + assert_eq!( + descriptor.validate_for_install(), + Err(bitty_storage::component::DescriptorError::PlaceholderDigest) + ); +} + +#[test] +fn packaged_digest_is_installable() { + let rendered = RECORD.replace(PLACEHOLDER_SHA256, &"ab".repeat(32)); + let descriptor = ComponentDescriptor::parse(&rendered).expect("rendered record parses"); + descriptor + .validate_for_install() + .expect("real digest is installable"); + assert!(!descriptor.has_placeholder_digest()); +} + +#[test] +fn protocol_and_capacity_claims_are_consistent() { + assert_eq!(PROTOCOL_VERSION, 1); + assert_eq!(COMPONENT_PROTOCOL_MIN, PROTOCOL_VERSION); + assert_eq!(COMPONENT_PROTOCOL_MAX, PROTOCOL_VERSION); + assert_eq!(MAX_NAMESPACES, 64); +} + +#[test] +fn server_exit_codes_are_stable() { + assert_eq!(SERVER_EXIT_OK, 0); + assert_eq!(SERVER_EXIT_PROTOCOL, 3); + assert_eq!(SERVER_EXIT_INTERNAL, 4); +} + +#[test] +fn binary_reports_its_version() { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_bitty-storage")) + .arg("--version") + .output() + .expect("run --version"); + assert!(output.status.success()); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + format!("bitty-storage {COMPONENT_VERSION}") + ); +} + +#[test] +fn binary_validates_packaged_and_rejects_placeholder_manifests() { + let binary = env!("CARGO_BIN_EXE_bitty-storage"); + let path = std::env::temp_dir().join(format!( + "bitty-storage-manifest-{}-{}.toml", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()) + )); + let rendered = RECORD.replace(PLACEHOLDER_SHA256, &"cd".repeat(32)); + std::fs::write(&path, rendered).expect("write rendered descriptor"); + + let accepted = std::process::Command::new(binary) + .arg("--check-manifest") + .arg(&path) + .output() + .expect("run --check-manifest"); + assert!(accepted.status.success(), "{accepted:?}"); + assert!(String::from_utf8_lossy(&accepted.stdout).starts_with("ok storage 0.0.1")); + let _ = std::fs::remove_file(&path); + + let repository = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("bitty-component.toml"); + let rejected = std::process::Command::new(binary) + .arg("--check-manifest") + .arg(repository) + .output() + .expect("run --check-manifest on repository record"); + assert_eq!(rejected.status.code(), Some(2)); + assert!( + String::from_utf8_lossy(&rejected.stderr).contains("placeholder"), + "{rejected:?}" + ); +} diff --git a/tests/component_server.rs b/tests/component_server.rs new file mode 100644 index 0000000..9bbc9ca --- /dev/null +++ b/tests/component_server.rs @@ -0,0 +1,723 @@ +//! Live component-server integration tests. +//! +//! The real `bitty-storage` binary is spawned as a stdio coprocess and driven +//! with the public protocol codec. Every test uses a fresh temporary storage +//! root, so no host state is read or written. +//! +//! Covered acceptance paths: handshake, commit durability across a clean +//! restart and a SIGKILL, rollback, bounded paging, stale revisions, +//! namespace limits, oversize frames, malformed frames, path-shaped wire +//! names, and cross-process lock contention. + +#![cfg(any(target_os = "linux", target_os = "android"))] + +use std::io::{BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::time::Duration; + +use bitty_storage::component::protocol::{ + ErrorKind, FrameReader, MAX_FRAME_BYTES, MAX_NAMESPACES, MAX_READ_ROWS, Message, + PROTOCOL_VERSION, Row, write_frame, +}; +use bitty_storage::component::server::{COMPONENT_VERSION, SERVER_EXIT_OK, SERVER_EXIT_PROTOCOL}; + +/// Upper bound for one protocol message. +const WAIT: Duration = Duration::from_secs(20); + +static SEQ: AtomicU64 = AtomicU64::new(0); + +fn scratch_root(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bitty-storage-component-{}-{}-{}", + name, + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create scratch root"); + dir +} + +fn row(key: &str, value: &[u8]) -> Row { + Row { + key: key.to_owned(), + value: value.to_vec(), + } +} + +/// One spawned component process plus its decoded message stream. +struct Component { + child: Child, + stdin: std::process::ChildStdin, + messages: Receiver>, + exited: bool, +} + +impl Component { + fn start(root: &Path) -> Self { + let binary: PathBuf = std::env::var_os("BITTY_STORAGE_COMPONENT_BIN") + .map_or_else(|| env!("CARGO_BIN_EXE_bitty-storage").into(), Into::into); + let mut child = Command::new(binary) + .arg("--root") + .arg(root) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn bitty-storage"); + let stdin = child.stdin.take().expect("component stdin"); + let stdout = child.stdout.take().expect("component stdout"); + let (sender, messages) = mpsc::channel(); + std::thread::Builder::new() + .name("component-reader".to_owned()) + .spawn(move || { + let mut reader = FrameReader::new(BufReader::new(stdout)); + loop { + match reader.read_message() { + Ok(Some(message)) => { + if sender.send(Ok(message)).is_err() { + break; + } + } + Ok(None) => break, + Err(error) => { + let _ = sender.send(Err(error.to_string())); + break; + } + } + } + }) + .expect("reader thread"); + let mut component = Self { + child, + stdin, + messages, + exited: false, + }; + component.handshake(); + component + } + + fn handshake(&mut self) { + self.send(&Message::Hello { + min: PROTOCOL_VERSION, + max: PROTOCOL_VERSION, + component: "storage".to_owned(), + version: "test".to_owned(), + }); + match self.recv() { + Message::HelloAck { + protocol, + component, + version, + } => { + assert_eq!(protocol, PROTOCOL_VERSION); + assert_eq!(component, "storage"); + assert_eq!(version, COMPONENT_VERSION); + } + other => panic!("expected HelloAck, got {other:?}"), + } + } + + fn send(&mut self, message: &Message) { + write_frame(&mut self.stdin, message).expect("write frame"); + self.stdin.flush().expect("flush frame"); + } + + fn send_raw(&mut self, bytes: &[u8]) { + self.stdin.write_all(bytes).expect("write raw"); + self.stdin.flush().expect("flush raw"); + } + + fn recv(&self) -> Message { + match self.messages.recv_timeout(WAIT) { + Ok(Ok(message)) => message, + Ok(Err(error)) => panic!("component protocol error: {error}"), + Err(RecvTimeoutError::Timeout) => panic!("timed out waiting for a component frame"), + Err(RecvTimeoutError::Disconnected) => panic!("component closed its stdout"), + } + } + + fn open(&mut self, request: u64, namespace: &str) -> Message { + self.send(&Message::OpenNamespace { + request, + namespace: namespace.to_owned(), + }); + self.recv() + } + + fn commit( + &mut self, + request: u64, + namespace: &str, + expected_revision: u64, + upserts: Vec, + deletes: Vec, + ) -> Message { + self.send(&Message::CommitBatch { + request, + namespace: namespace.to_owned(), + expected_revision, + upserts, + deletes, + }); + self.recv() + } + + fn read_page( + &mut self, + request: u64, + namespace: &str, + start_row: u32, + max_rows: u32, + max_bytes: u32, + ) -> Message { + self.send(&Message::ReadSnapshot { + request, + namespace: namespace.to_owned(), + start_row, + max_rows, + max_bytes, + }); + self.recv() + } + + fn read_all(&mut self, namespace: &str) -> (u64, Vec) { + let mut rows = Vec::new(); + let mut start = 0u32; + let mut revision: u64; + let mut request = 1000u64; + loop { + let page = self.read_page(request, namespace, start, MAX_READ_ROWS as u32, 0); + request += 1; + match page { + Message::SnapshotChunk { + revision: seen, + rows: page_rows, + total_rows, + done, + .. + } => { + revision = seen; + assert!( + !page_rows.is_empty() || done, + "empty page before the snapshot end" + ); + start += u32::try_from(page_rows.len()).expect("page length"); + rows.extend(page_rows); + if done { + assert_eq!(rows.len(), total_rows as usize); + break; + } + } + other => panic!("expected SnapshotChunk, got {other:?}"), + } + } + (revision, rows) + } + + fn shutdown(mut self) { + self.send(&Message::Shutdown); + match self.recv() { + Message::ShutdownAck => {} + other => panic!("expected ShutdownAck, got {other:?}"), + } + let status = self.child.wait().expect("wait for component"); + self.exited = true; + assert_eq!(status.code(), Some(i32::from(SERVER_EXIT_OK))); + } + + fn kill_and_wait(&mut self) -> std::process::ExitStatus { + self.child.kill().expect("kill component"); + let status = self.child.wait().expect("wait for killed component"); + self.exited = true; + status + } +} + +impl Drop for Component { + fn drop(&mut self) { + if self.exited { + return; + } + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn expect_opened(message: Message, namespace: &str) -> (u64, u32, u64) { + match message { + Message::NamespaceOpened { + namespace: seen, + revision, + rows, + bytes, + .. + } => { + assert_eq!(seen, namespace); + (revision, rows, bytes) + } + other => panic!("expected NamespaceOpened, got {other:?}"), + } +} + +fn expect_error(message: Message, kind: ErrorKind) -> String { + match message { + Message::Error { + kind: seen, + message, + .. + } => { + assert_eq!(seen, kind); + message + } + other => panic!("expected Error, got {other:?}"), + } +} + +fn expect_committed(message: Message) -> u64 { + match message { + Message::Committed { revision, .. } => revision, + other => panic!("expected Committed, got {other:?}"), + } +} + +#[test] +fn host_error_kind_codes_stay_stable_and_additive() { + // Core host expectation: protocol v1 codes never move. `uncertain` is an + // additive kind at 11; `internal` remains 10 so a host that only knows + // v1.0 keeps its existing mapping. + assert_eq!(ErrorKind::Internal.code(), 10); + assert_eq!(ErrorKind::Uncertain.code(), 11); + assert_eq!(ErrorKind::from_code(10), Some(ErrorKind::Internal)); + assert_eq!(ErrorKind::from_code(11), Some(ErrorKind::Uncertain)); + assert_eq!(ErrorKind::from_code(12), None); + assert_eq!(ErrorKind::Uncertain.as_str(), "uncertain"); +} + +#[test] +fn handshake_then_shutdown_is_clean() { + let root = scratch_root("handshake"); + let component = Component::start(&root); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn commit_survives_a_clean_restart() { + let root = scratch_root("restart"); + let mut first = Component::start(&root); + let (revision, rows, bytes) = expect_opened(first.open(1, "session-1"), "session-1"); + assert_eq!((revision, rows, bytes), (0, 0, 0)); + + let revision = expect_committed(first.commit( + 2, + "session-1", + 0, + vec![row("alpha", b"one"), row("beta", b"two")], + Vec::new(), + )); + assert_eq!(revision, 1); + let (revision, rows) = first.read_all("session-1"); + assert_eq!(revision, 1); + assert_eq!(rows, vec![row("alpha", b"one"), row("beta", b"two")]); + first.shutdown(); + + let mut second = Component::start(&root); + let (revision, count, _) = expect_opened(second.open(1, "session-1"), "session-1"); + assert_eq!((revision, count), (1, 2)); + let (revision, rows) = second.read_all("session-1"); + assert_eq!(revision, 1); + assert_eq!(rows, vec![row("alpha", b"one"), row("beta", b"two")]); + second.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn commit_is_atomic_and_rejects_a_stale_revision() { + let root = scratch_root("stale"); + let mut component = Component::start(&root); + assert_eq!(expect_opened(component.open(1, "ns"), "ns").0, 0); + assert_eq!( + expect_committed(component.commit(2, "ns", 0, vec![row("a", b"1")], Vec::new())), + 1 + ); + let text = expect_error( + component.commit(3, "ns", 0, vec![row("a", b"2"), row("b", b"3")], Vec::new()), + ErrorKind::StaleRevision, + ); + assert_eq!(text, "revision conflict"); + let (revision, rows) = component.read_all("ns"); + assert_eq!(revision, 1); + assert_eq!(rows, vec![row("a", b"1")]); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn rollback_restores_the_previous_revision_as_a_new_revision() { + let root = scratch_root("rollback"); + let mut component = Component::start(&root); + assert_eq!(expect_opened(component.open(1, "history"), "history").0, 0); + assert_eq!( + expect_committed(component.commit( + 2, + "history", + 0, + vec![row("a", b"1"), row("b", b"2")], + Vec::new(), + )), + 1 + ); + assert_eq!( + expect_committed(component.commit( + 3, + "history", + 1, + vec![row("a", b"changed"), row("c", b"3")], + vec!["b".to_owned()], + )), + 2 + ); + + component.send(&Message::Rollback { + request: 4, + namespace: "history".to_owned(), + expected_revision: 2, + }); + match component.recv() { + Message::RolledBack { + revision, + restored_revision, + total_rows, + .. + } => { + assert_eq!(revision, 3); + assert_eq!(restored_revision, 1); + assert_eq!(total_rows, 2); + } + other => panic!("expected RolledBack, got {other:?}"), + } + let (revision, rows) = component.read_all("history"); + assert_eq!(revision, 3); + assert_eq!(rows, vec![row("a", b"1"), row("b", b"2")]); + + // A second rollback toggles back to the state before the first one. + component.send(&Message::Rollback { + request: 5, + namespace: "history".to_owned(), + expected_revision: 3, + }); + match component.recv() { + Message::RolledBack { + revision, + restored_revision, + .. + } => { + assert_eq!(revision, 4); + assert_eq!(restored_revision, 2); + } + other => panic!("expected RolledBack, got {other:?}"), + } + let (_, rows) = component.read_all("history"); + assert_eq!(rows, vec![row("a", b"changed"), row("c", b"3")]); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn rollback_without_a_previous_revision_is_typed() { + let root = scratch_root("no-previous"); + let mut component = Component::start(&root); + assert_eq!(expect_opened(component.open(1, "fresh"), "fresh").0, 0); + component.send(&Message::Rollback { + request: 2, + namespace: "fresh".to_owned(), + expected_revision: 0, + }); + let text = expect_error(component.recv(), ErrorKind::InvalidRequest); + assert_eq!(text, "no previous revision"); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn crash_after_commit_preserves_acknowledged_state() { + let root = scratch_root("crash"); + let mut component = Component::start(&root); + assert_eq!(expect_opened(component.open(1, "session"), "session").0, 0); + assert_eq!( + expect_committed(component.commit( + 2, + "session", + 0, + vec![row("alpha", b"durable"), row("beta", b"also")], + Vec::new(), + )), + 1 + ); + let status = component.kill_and_wait(); + assert!(!status.success(), "SIGKILL must not report success"); + + let mut restarted = Component::start(&root); + let (revision, rows, _) = expect_opened(restarted.open(1, "session"), "session"); + assert_eq!((revision, rows), (1, 2)); + let (revision, rows) = restarted.read_all("session"); + assert_eq!(revision, 1); + assert_eq!(rows, vec![row("alpha", b"durable"), row("beta", b"also")]); + restarted.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn reads_are_paged_and_ordered() { + let root = scratch_root("paging"); + let mut component = Component::start(&root); + expect_opened(component.open(1, "rows"), "rows"); + let first: Vec = (0..100) + .map(|index| row(&format!("k{index:04}"), format!("v{index}").as_bytes())) + .collect(); + let second: Vec = (100..200) + .map(|index| row(&format!("k{index:04}"), format!("v{index}").as_bytes())) + .collect(); + assert_eq!( + expect_committed(component.commit(2, "rows", 0, first.clone(), Vec::new())), + 1 + ); + assert_eq!( + expect_committed(component.commit(3, "rows", 1, second.clone(), Vec::new())), + 2 + ); + + let mut seen = Vec::new(); + let mut start = 0u32; + let mut pages = 0; + loop { + match component.read_page(10 + pages, "rows", start, 32, 0) { + Message::SnapshotChunk { + rows: page, + total_rows, + done, + .. + } => { + assert_eq!(total_rows, 200); + assert!(page.len() <= 32); + pages += 1; + start += u32::try_from(page.len()).expect("page length"); + seen.extend(page); + if done { + break; + } + } + other => panic!("expected SnapshotChunk, got {other:?}"), + } + } + assert_eq!(pages, 7); + assert_eq!(seen.len(), 200); + let mut expected = first; + expected.extend(second); + expected.sort_by(|left, right| left.key.cmp(&right.key)); + assert_eq!(seen, expected); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn namespace_row_and_byte_bounds_fail_closed() { + let root = scratch_root("bounds"); + let mut component = Component::start(&root); + expect_opened(component.open(1, "bounded"), "bounded"); + + // 32 full batches fill MAX_ROWS (4096); the next row is rejected and the + // committed revision does not advance. + let mut revision = 0; + for batch in 0..32 { + let rows: Vec = (0..128) + .map(|offset| row(&format!("k{:05}", batch * 128 + offset), b"x")) + .collect(); + revision = + expect_committed(component.commit(100 + batch, "bounded", revision, rows, Vec::new())); + } + assert_eq!(revision, 32); + let text = expect_error( + component.commit( + 200, + "bounded", + revision, + vec![row("overflow", b"x")], + Vec::new(), + ), + ErrorKind::Budget, + ); + assert_eq!(text, "storage bound exceeded"); + let (seen, rows) = component.read_all("bounded"); + assert_eq!(seen, 32); + assert_eq!(rows.len(), 4096); + + // Namespace capacity is bounded independently. + for index in 0..MAX_NAMESPACES - 1 { + let name = format!("ns-{index:02}"); + expect_opened(component.open(1000 + index as u64, &name), &name); + } + let text = expect_error(component.open(2000, "one-too-many"), ErrorKind::Budget); + assert_eq!(text, "namespace limit reached"); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn lock_contention_between_components_is_typed_and_recovers() { + let root = scratch_root("lock"); + let mut first = Component::start(&root); + expect_opened(first.open(1, "shared"), "shared"); + + let mut second = Component::start(&root); + let text = expect_error(second.open(1, "shared"), ErrorKind::Locked); + assert_eq!(text, "namespace is locked"); + second.shutdown(); + + // The kernel releases the lock when the first component goes away. + first.shutdown(); + let mut third = Component::start(&root); + expect_opened(third.open(1, "shared"), "shared"); + third.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn unknown_namespace_is_typed() { + let root = scratch_root("unknown"); + let mut component = Component::start(&root); + let text = expect_error( + component.read_page(1, "never-opened", 0, 0, 0), + ErrorKind::UnknownNamespace, + ); + assert_eq!(text, "unknown namespace"); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn oversized_frame_is_rejected_and_the_component_exits_protocol() { + let root = scratch_root("oversize-frame"); + let mut component = Component::start(&root); + let oversized = u32::try_from(MAX_FRAME_BYTES + 1).expect("fits u32"); + component.send_raw(&oversized.to_be_bytes()); + let text = expect_error(component.recv(), ErrorKind::Protocol); + assert_eq!(text, "frame too large"); + let status = component.child.wait().expect("wait for component"); + assert_eq!(status.code(), Some(i32::from(SERVER_EXIT_PROTOCOL))); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn malformed_frames_exit_protocol_without_panicking() { + let root = scratch_root("malformed"); + let mut component = Component::start(&root); + component.send_raw(&[0u8, 0, 0, 0]); + let text = expect_error(component.recv(), ErrorKind::Protocol); + assert_eq!(text, "invalid protocol frame"); + let status = component.child.wait().expect("wait for component"); + assert_eq!(status.code(), Some(i32::from(SERVER_EXIT_PROTOCOL))); + let _ = std::fs::remove_dir_all(&root); + + let root = scratch_root("unknown-tag"); + let mut component = Component::start(&root); + let mut frame = Vec::new(); + frame.extend_from_slice(&1u32.to_be_bytes()); + frame.push(0x7f); + component.send_raw(&frame); + let text = expect_error(component.recv(), ErrorKind::Protocol); + assert_eq!(text, "invalid protocol frame"); + let status = component.child.wait().expect("wait for component"); + assert_eq!(status.code(), Some(i32::from(SERVER_EXIT_PROTOCOL))); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn path_shaped_namespace_on_the_wire_is_rejected() { + let root = scratch_root("wire-path"); + let mut component = Component::start(&root); + // `../escape` cannot be encoded through the typed codec, so build the + // exact wire bytes an untrusted peer could send. + let namespace = b"../escape"; + let mut payload = Vec::new(); + payload.push(0x10u8); + payload.extend_from_slice(&7u64.to_be_bytes()); + payload.extend_from_slice(&(namespace.len() as u32).to_be_bytes()); + payload.extend_from_slice(namespace); + let mut frame = Vec::new(); + frame.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + frame.extend_from_slice(&payload); + component.send_raw(&frame); + let text = expect_error(component.recv(), ErrorKind::Protocol); + assert_eq!(text, "invalid protocol frame"); + let status = component.child.wait().expect("wait for component"); + assert_eq!(status.code(), Some(i32::from(SERVER_EXIT_PROTOCOL))); + assert!(!root.join("..").join("escape").exists()); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn packed_server_refuses_a_symlinked_lock_and_leaves_outside_absent() { + let root = scratch_root("server-lock-symlink"); + let outside = scratch_root("server-lock-symlink-outside"); + std::fs::create_dir(root.join("evil")).expect("namespace dir"); + let target = outside.join("lock"); + std::os::unix::fs::symlink(&target, root.join("evil").join("lock")).expect("symlink"); + let mut component = Component::start(&root); + let text = expect_error(component.open(1, "evil"), ErrorKind::CorruptState); + assert_eq!(text, "storage state is corrupt"); + assert!( + !target.exists(), + "absent outside lock target must never be created" + ); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +fn packed_server_refuses_commit_after_namespace_replacement() { + let root = scratch_root("server-ancestor-swap"); + let outside = scratch_root("server-ancestor-swap-outside"); + let mut component = Component::start(&root); + expect_opened(component.open(1, "ns"), "ns"); + assert_eq!( + expect_committed(component.commit(2, "ns", 0, vec![row("a", b"1")], Vec::new())), + 1 + ); + std::fs::rename(root.join("ns"), root.join("ns-original")).expect("rename namespace"); + std::os::unix::fs::symlink(&outside, root.join("ns")).expect("ancestor symlink"); + let text = expect_error( + component.commit(3, "ns", 1, vec![row("b", b"2")], Vec::new()), + ErrorKind::Uncertain, + ); + assert_eq!(text, "storage outcome uncertain; reopen namespace"); + assert!( + outside.read_dir().expect("outside read").next().is_none(), + "packed commit must not create anything in the outside fixture" + ); + + // The uncertain result must have evicted the cached handle: reopening + // re-resolves the pathname and refuses the symlinked replacement instead + // of acknowledging the detached inode. + let text = expect_error(component.open(4, "ns"), ErrorKind::InvalidRequest); + assert_eq!(text, "invalid namespace"); + + // Once the pathname names the original directory again, a reopen loads + // the on-disk revision rather than any cached snapshot. + std::fs::remove_file(root.join("ns")).expect("remove replacement symlink"); + std::fs::rename(root.join("ns-original"), root.join("ns")).expect("restore namespace"); + let (revision, rows, _) = expect_opened(component.open(5, "ns"), "ns"); + assert_eq!((revision, rows), (1, 1)); + let (revision, rows) = component.read_all("ns"); + assert_eq!(revision, 1); + assert_eq!(rows, vec![row("a", b"1")]); + component.shutdown(); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); +} diff --git a/tests/component_store.rs b/tests/component_store.rs new file mode 100644 index 0000000..ce4fe80 --- /dev/null +++ b/tests/component_store.rs @@ -0,0 +1,829 @@ +//! Store-level durability, journal recovery, containment, and bound tests. +//! +//! These tests deliberately construct the on-disk snapshot and journal bytes +//! themselves (rather than calling private encoders) so the format is proven +//! from an independent implementation, including a crash between journal +//! publication and snapshot replacement. + +#![cfg(any(target_os = "linux", target_os = "android"))] + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use bitty_storage::component::protocol::{MAX_ROWS, Row}; +use bitty_storage::component::store::{NamespaceName, StorageRoot, StoreErrorKind}; + +const SNAPSHOT_MAGIC: &[u8] = b"bitty-storage-snapshot\n"; +const JOURNAL_MAGIC: &[u8] = b"bitty-storage-journal\n"; +const FORMAT_VERSION: u32 = 1; + +static SEQ: AtomicU64 = AtomicU64::new(0); + +fn scratch_root(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bitty-storage-store-{}-{}-{}", + name, + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create scratch root"); + dir +} + +fn checksum(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +fn snapshot_bytes(revision: u64, rows: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(SNAPSHOT_MAGIC); + out.extend_from_slice(&FORMAT_VERSION.to_be_bytes()); + out.extend_from_slice(&revision.to_be_bytes()); + out.extend_from_slice(&(rows.len() as u32).to_be_bytes()); + let mut sorted = rows.to_vec(); + sorted.sort_by(|left, right| left.0.cmp(right.0)); + for (key, value) in sorted { + out.extend_from_slice(&(key.len() as u32).to_be_bytes()); + out.extend_from_slice(key.as_bytes()); + out.extend_from_slice(&(value.len() as u32).to_be_bytes()); + out.extend_from_slice(value); + } + let checksum = checksum(&out); + out.extend_from_slice(&checksum.to_be_bytes()); + out +} + +fn journal_bytes(revision: u64, rows: &[(&str, &[u8])]) -> Vec { + let payload = snapshot_bytes(revision, rows); + let mut out = Vec::new(); + out.extend_from_slice(JOURNAL_MAGIC); + out.extend_from_slice(&FORMAT_VERSION.to_be_bytes()); + out.extend_from_slice(&revision.to_be_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + out.extend_from_slice(&payload); + let checksum = checksum(&out); + out.extend_from_slice(&checksum.to_be_bytes()); + out +} + +fn name(value: &str) -> NamespaceName { + NamespaceName::parse(value).expect("valid namespace") +} + +fn row(key: &str, value: &[u8]) -> Row { + Row { + key: key.to_owned(), + value: value.to_vec(), + } +} + +#[test] +fn commit_persists_and_rollback_toggles() { + let root = scratch_root("commit"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("session")).expect("open"); + assert_eq!(namespace.revision(), 0); + let outcome = namespace + .commit(0, vec![row("a", b"1"), row("b", b"2")], Vec::new()) + .expect("commit"); + assert_eq!(outcome.revision, 1); + assert_eq!(outcome.total_rows, 2); + drop(namespace); + + let mut reopened = storage.open_namespace(&name("session")).expect("reopen"); + assert_eq!(reopened.revision(), 1); + let page = reopened.read_page(0, 0, 0).expect("read"); + assert_eq!(page.total_rows, 2); + assert_eq!(page.rows, vec![row("a", b"1"), row("b", b"2")]); + + let outcome = reopened + .commit( + 1, + vec![row("a", b"changed"), row("c", b"3")], + vec!["b".to_owned()], + ) + .expect("second commit"); + assert_eq!(outcome.revision, 2); + let rolled = reopened.rollback(2).expect("rollback"); + assert_eq!(rolled.revision, 3); + assert_eq!(rolled.restored_revision, 1); + let page = reopened.read_page(0, 0, 0).expect("read"); + assert_eq!(page.rows, vec![row("a", b"1"), row("b", b"2")]); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn published_journal_is_replayed_after_a_crash() { + let root = scratch_root("journal"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("session")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("commit"); + drop(namespace); + + // Simulate a crash after the journal rename but before the snapshot + // rename: the next open must complete the published target. + std::fs::write( + root.join("session").join("journal"), + journal_bytes(2, &[("a", b"1"), ("b", b"2")]), + ) + .expect("write journal"); + + let reopened = storage.open_namespace(&name("session")).expect("recover"); + assert_eq!(reopened.revision(), 2); + let page = reopened.read_page(0, 0, 0).expect("read"); + assert_eq!(page.rows, vec![row("a", b"1"), row("b", b"2")]); + assert!( + !root.join("session").join("journal").exists(), + "replayed journal must be removed" + ); + + // Recovery must also complete step 2 (`previous` = revision 1) before + // publishing the journal target, or this rollback would skip a revision. + let mut reopened = reopened; + let rolled = reopened.rollback(2).expect("rollback after replay"); + assert_eq!(rolled.restored_revision, 1); + let page = reopened.read_page(0, 0, 0).expect("read"); + assert_eq!(page.rows, vec![row("a", b"1")]); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn first_commit_recovery_removes_stale_previous() { + let root = scratch_root("stale-previous"); + let storage = StorageRoot::new(&root).expect("root"); + let namespace = storage.open_namespace(&name("fresh")).expect("open"); + drop(namespace); + + // A previous image from an earlier namespace lifetime plus a journal for + // the first commit: recovery must remove the stale image, not make + // revision 1 rollback-able to unrelated state. + std::fs::write( + root.join("fresh").join("previous"), + snapshot_bytes(7, &[("old", b"stale")]), + ) + .expect("write previous"); + std::fs::write( + root.join("fresh").join("journal"), + journal_bytes(1, &[("a", b"1")]), + ) + .expect("write journal"); + + let mut reopened = storage.open_namespace(&name("fresh")).expect("recover"); + assert_eq!(reopened.revision(), 1); + assert!(!root.join("fresh").join("previous").exists()); + assert_eq!( + reopened.rollback(1).expect_err("no previous").kind(), + StoreErrorKind::NoPrevious + ); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +#[cfg(unix)] +fn journal_replay_failure_is_uncertain_and_retriable_after_repair() { + use std::os::unix::fs::PermissionsExt as _; + + let root = scratch_root("recover-uncertain"); + let namespace_dir = root.join("ns"); + std::fs::create_dir_all(&namespace_dir).expect("namespace dir"); + std::fs::write( + namespace_dir.join("journal"), + journal_bytes(1, &[("a", b"1")]), + ) + .expect("journal fixture"); + // Pre-create the lock so opening only needs RDWR, not directory write. + std::fs::write(namespace_dir.join("lock"), b"").expect("lock fixture"); + + let mut permissions = std::fs::metadata(&namespace_dir) + .expect("namespace metadata") + .permissions(); + permissions.set_mode(0o500); + std::fs::set_permissions(&namespace_dir, permissions.clone()).expect("restrict namespace"); + let enforced = std::fs::write(namespace_dir.join("probe"), b"").is_err(); + let _ = std::fs::remove_file(namespace_dir.join("probe")); + if !enforced { + let mut restore = permissions; + restore.set_mode(0o700); + let _ = std::fs::set_permissions(&namespace_dir, restore); + let _ = std::fs::remove_dir_all(&root); + return; // privileged runner: directory modes are not enforced + } + + let storage = StorageRoot::new(&root).expect("root"); + let error = storage + .open_namespace(&name("ns")) + .expect_err("failed journal replay must not be acknowledged"); + assert_eq!( + error.kind(), + StoreErrorKind::Uncertain, + "post-write recovery failure must be uncertain" + ); + assert!( + namespace_dir.join("journal").exists(), + "recovery evidence must survive a failed replay" + ); + + let mut writable = std::fs::metadata(&namespace_dir) + .expect("namespace metadata") + .permissions(); + writable.set_mode(0o700); + std::fs::set_permissions(&namespace_dir, writable).expect("restore namespace"); + let reopened = storage + .open_namespace(&name("ns")) + .expect("recovery must be retriable after the filesystem is repaired"); + assert_eq!(reopened.revision(), 1); + assert!( + !namespace_dir.join("journal").exists(), + "successful replay must consume the journal" + ); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +#[cfg(unix)] +fn journal_with_unreadable_snapshot_is_uncertain_not_io() { + use std::os::unix::fs::PermissionsExt as _; + + let root = scratch_root("recover-unreadable-snapshot"); + let namespace_dir = root.join("ns"); + std::fs::create_dir_all(&namespace_dir).expect("namespace dir"); + let snapshot = namespace_dir.join("snapshot"); + std::fs::write(&snapshot, snapshot_bytes(0, &[])).expect("snapshot fixture"); + std::fs::write( + namespace_dir.join("journal"), + journal_bytes(1, &[("a", b"1")]), + ) + .expect("journal fixture"); + std::fs::write(namespace_dir.join("lock"), b"").expect("lock fixture"); + + let mut permissions = std::fs::metadata(&snapshot) + .expect("snapshot metadata") + .permissions(); + permissions.set_mode(0o000); + std::fs::set_permissions(&snapshot, permissions.clone()).expect("hide snapshot"); + if std::fs::read(&snapshot).is_ok() { + let mut restore = permissions; + restore.set_mode(0o600); + let _ = std::fs::set_permissions(&snapshot, restore); + let _ = std::fs::remove_dir_all(&root); + return; // privileged runner: file modes are not enforced + } + + let storage = StorageRoot::new(&root).expect("root"); + let error = storage + .open_namespace(&name("ns")) + .expect_err("an unreadable snapshot with a durable journal must not be acknowledged"); + assert_eq!( + error.kind(), + StoreErrorKind::Uncertain, + "a journal-present snapshot read failure must be uncertain" + ); + assert!( + namespace_dir.join("journal").exists(), + "the durable target must remain for the next recovery attempt" + ); + + let mut readable = std::fs::metadata(&snapshot) + .expect("snapshot metadata") + .permissions(); + readable.set_mode(0o600); + std::fs::set_permissions(&snapshot, readable).expect("restore snapshot"); + let reopened = storage + .open_namespace(&name("ns")) + .expect("recovery must succeed once the snapshot is readable"); + assert_eq!(reopened.revision(), 1); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn corrupt_journal_and_snapshot_fail_closed() { + let root = scratch_root("corrupt"); + let storage = StorageRoot::new(&root).expect("root"); + let namespace = storage.open_namespace(&name("bad-journal")).expect("open"); + drop(namespace); + std::fs::write(root.join("bad-journal").join("journal"), b"garbage").expect("write"); + let error = storage + .open_namespace(&name("bad-journal")) + .expect_err("corrupt journal"); + assert_eq!(error.kind(), StoreErrorKind::Corrupt); + + let namespace = storage.open_namespace(&name("bad-snapshot")).expect("open"); + drop(namespace); + std::fs::write(root.join("bad-snapshot").join("snapshot"), b"garbage").expect("write"); + let error = storage + .open_namespace(&name("bad-snapshot")) + .expect_err("corrupt snapshot"); + assert_eq!(error.kind(), StoreErrorKind::Corrupt); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn path_shaped_names_and_symlinked_directories_are_refused() { + for bad in ["../escape", "/absolute", "a/b", "..", ".hidden", "Upper"] { + assert_eq!( + NamespaceName::parse(bad).expect_err("invalid").kind(), + StoreErrorKind::InvalidName, + "name: {bad}" + ); + } + + let root = scratch_root("symlink"); + let outside = scratch_root("outside"); + let storage = StorageRoot::new(&root).expect("root"); + std::os::unix::fs::symlink(&outside, root.join("evil")).expect("symlink"); + let error = storage + .open_namespace(&name("evil")) + .expect_err("symlink must be refused"); + assert_eq!(error.kind(), StoreErrorKind::InvalidName); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +fn a_second_open_is_locked_while_the_first_lives() { + let root = scratch_root("lock"); + let first_root = StorageRoot::new(&root).expect("root"); + let second_root = StorageRoot::new(&root).expect("root"); + let first = first_root.open_namespace(&name("shared")).expect("first"); + let error = second_root + .open_namespace(&name("shared")) + .expect_err("lock contention"); + assert_eq!(error.kind(), StoreErrorKind::Locked); + drop(first); + second_root + .open_namespace(&name("shared")) + .expect("lock released"); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn rollback_without_previous_and_stale_revision_are_typed() { + let root = scratch_root("typed"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + assert_eq!( + namespace.rollback(0).expect_err("no previous").kind(), + StoreErrorKind::NoPrevious + ); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("commit"); + assert_eq!( + namespace + .commit(0, vec![row("b", b"2")], Vec::new()) + .expect_err("stale") + .kind(), + StoreErrorKind::StaleRevision + ); + let page = namespace.read_page(0, 0, 0).expect("read"); + assert_eq!(page.rows, vec![row("a", b"1")]); + let _ = std::fs::remove_dir_all(&root); +} + +/// Replaces `snapshot` with a directory so the next publication fails at the +/// rename after `previous` was already rewritten. This is the exact ordering +/// the rollback-image restoration must survive. +fn block_snapshot_publication(root: &Path) { + let snapshot = root.join("ns").join("snapshot"); + std::fs::remove_file(&snapshot).expect("remove snapshot"); + std::fs::create_dir(&snapshot).expect("block snapshot rename"); +} + +/// Removes the artificial blocker and puts back the snapshot that was current +/// before the injected failure. +fn repair_snapshot(root: &Path, snapshot: &[u8]) { + let path = root.join("ns").join("snapshot"); + std::fs::remove_dir(&path).expect("remove snapshot blocker"); + std::fs::write(&path, snapshot).expect("restore snapshot"); +} + +#[test] +fn failed_snapshot_write_restores_the_n_minus_one_rollback_image() { + let root = scratch_root("publish-restores-previous"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("revision 1"); + namespace + .commit(1, vec![row("b", b"2")], Vec::new()) + .expect("revision 2"); + + let previous_before = snapshot_bytes(1, &[("a", b"1")]); + block_snapshot_publication(&root); + let error = namespace + .commit(2, vec![row("c", b"3")], Vec::new()) + .expect_err("snapshot publication must fail"); + assert_eq!( + error.kind(), + StoreErrorKind::Io, + "a fully rolled back pre-publication failure is io" + ); + assert_eq!( + namespace.revision(), + 2, + "failed commit must not advance state" + ); + assert_eq!( + std::fs::read(root.join("ns").join("previous")).expect("previous"), + previous_before, + "the true N-1 rollback image must be restored" + ); + assert!( + !root.join("ns").join("journal").exists(), + "a rolled back commit must drop its journal" + ); + + repair_snapshot(&root, &snapshot_bytes(2, &[("a", b"1"), ("b", b"2")])); + drop(namespace); + + let mut reopened = storage.open_namespace(&name("ns")).expect("reopen"); + assert_eq!(reopened.revision(), 2); + let rolled = reopened.rollback(2).expect("rollback"); + assert_eq!( + rolled.restored_revision, 1, + "rollback must restore N-1, never the failed N" + ); + let page = reopened.read_page(0, 0, 0).expect("read"); + assert_eq!(page.rows, vec![row("a", b"1")]); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn failed_first_commit_restores_the_absence_of_previous() { + let root = scratch_root("publish-restores-absence"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("revision 1"); + + block_snapshot_publication(&root); + let error = namespace + .commit(1, vec![row("b", b"2")], Vec::new()) + .expect_err("snapshot publication must fail"); + assert_eq!(error.kind(), StoreErrorKind::Io); + assert!( + !root.join("ns").join("previous").exists(), + "a failed commit must not invent a rollback image when none existed" + ); + assert!(!root.join("ns").join("journal").exists()); + + repair_snapshot(&root, &snapshot_bytes(1, &[("a", b"1")])); + drop(namespace); + let mut reopened = storage.open_namespace(&name("ns")).expect("reopen"); + assert_eq!(reopened.revision(), 1); + assert_eq!( + reopened + .rollback(1) + .expect_err("no previous must survive the failed commit") + .kind(), + StoreErrorKind::NoPrevious + ); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn rollback_rejects_previous_at_or_ahead_of_current() { + let root = scratch_root("previous-not-older"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("revision 1"); + namespace + .commit(1, vec![row("b", b"2")], Vec::new()) + .expect("revision 2"); + + for revision in [2, 3] { + std::fs::write( + root.join("ns").join("previous"), + snapshot_bytes(revision, &[("a", b"1"), ("b", b"2")]), + ) + .expect("overwrite previous"); + assert_eq!( + namespace + .rollback(2) + .expect_err("previous must precede the current revision") + .kind(), + StoreErrorKind::Corrupt, + "previous revision {revision} must be corrupt" + ); + assert_eq!(namespace.revision(), 2, "rejection must not mutate state"); + } + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn rollback_rejects_a_previous_that_skips_a_revision() { + let root = scratch_root("previous-skip"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + for (revision, key) in [(0, "a"), (1, "b"), (2, "c")] { + namespace + .commit(revision, vec![row(key, b"x")], Vec::new()) + .expect("commit"); + } + + std::fs::write( + root.join("ns").join("previous"), + snapshot_bytes(1, &[("a", b"x")]), + ) + .expect("skip a revision"); + assert_eq!( + namespace + .rollback(3) + .expect_err("N-2 is not the immediate predecessor") + .kind(), + StoreErrorKind::Corrupt + ); + assert_eq!(namespace.revision(), 3); + + std::fs::write( + root.join("ns").join("previous"), + snapshot_bytes(2, &[("a", b"x"), ("b", b"x")]), + ) + .expect("restore the immediate predecessor"); + let rolled = namespace + .rollback(3) + .expect("immediate predecessor is valid"); + assert_eq!(rolled.restored_revision, 2); + assert_eq!(rolled.revision, 4); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn no_op_rollback_is_impossible() { + let root = scratch_root("noop-rollback"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("revision 1"); + namespace + .commit(1, vec![row("b", b"2")], Vec::new()) + .expect("revision 2"); + + // The historical failure mode left revision N in both `snapshot` and + // `previous`; that must be corrupt state, never a "restored N" success. + std::fs::write( + root.join("ns").join("previous"), + snapshot_bytes(2, &[("a", b"1"), ("b", b"2")]), + ) + .expect("corrupt previous fixture"); + assert_eq!( + namespace + .rollback(2) + .expect_err("a no-op rollback must not be acknowledged") + .kind(), + StoreErrorKind::Corrupt + ); + assert_eq!(namespace.revision(), 2, "no new revision may be created"); + let page = namespace.read_page(0, 0, 0).expect("read"); + assert_eq!(page.revision, 2); + assert_eq!(page.rows, vec![row("a", b"1"), row("b", b"2")]); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn snapshot_and_row_bounds_reject_without_mutation() { + let root = scratch_root("bounds"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("bytes")).expect("open"); + + let chunk = vec![b'x'; 128 * 1024]; + let mut revision = 0; + let mut committed = 0; + for index in 0..10 { + match namespace.commit( + revision, + vec![row(&format!("k{index:02}"), &chunk)], + Vec::new(), + ) { + Ok(outcome) => { + revision = outcome.revision; + committed += 1; + } + Err(error) => { + assert_eq!(error.kind(), StoreErrorKind::Bound); + break; + } + } + } + assert!((7..10).contains(&committed), "committed {committed}"); + assert_eq!(namespace.revision(), revision); + assert_eq!(namespace.row_count(), committed); + + let mut rows = storage.open_namespace(&name("rows")).expect("open"); + let mut revision = 0; + for batch in 0..32 { + let upserts: Vec = (0..128) + .map(|offset| row(&format!("k{:05}", batch * 128 + offset), b"x")) + .collect(); + revision = rows + .commit(revision, upserts, Vec::new()) + .expect("batch") + .revision; + } + assert_eq!(rows.row_count(), MAX_ROWS); + let error = rows + .commit(revision, vec![row("overflow", b"x")], Vec::new()) + .expect_err("row bound"); + assert_eq!(error.kind(), StoreErrorKind::Bound); + assert_eq!(rows.row_count(), MAX_ROWS); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn journal_target_equal_to_the_snapshot_is_discarded() { + let root = scratch_root("stale-journal"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("commit"); + drop(namespace); + + // A crash after the snapshot rename leaves a journal at the same + // revision; recovery must remove it without rewriting state. + std::fs::write( + root.join("ns").join("journal"), + journal_bytes(1, &[("a", b"1")]), + ) + .expect("write journal"); + let reopened = storage.open_namespace(&name("ns")).expect("open"); + assert_eq!(reopened.revision(), 1); + assert!(!root.join("ns").join("journal").exists()); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn root_containment_keeps_every_member_inside() { + let root = scratch_root("containment"); + let storage = StorageRoot::new(&root).expect("root"); + let namespace = storage.open_namespace(&name("session-1")).expect("open"); + drop(namespace); + let dir = Path::new(&root).join("session-1"); + for member in ["snapshot", "previous", "journal", "lock"] { + let path = dir.join(member); + if path.exists() { + assert!(path.starts_with(&root), "member escaped: {path:?}"); + } + } + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn symlinked_lock_is_refused_without_outside_creation_or_write() { + for existing in [false, true] { + let label = if existing { "existing" } else { "absent" }; + let root = scratch_root(&format!("lock-{label}")); + let outside = scratch_root(&format!("lock-{label}-outside")); + let storage = StorageRoot::new(&root).expect("root"); + std::fs::create_dir(root.join("evil")).expect("namespace dir"); + let target = outside.join("lock"); + if existing { + std::fs::write(&target, b"keep").expect("outside target"); + } + std::os::unix::fs::symlink(&target, root.join("evil").join("lock")).expect("symlink"); + let error = storage + .open_namespace(&name("evil")) + .expect_err("symlinked lock must be refused"); + assert_eq!(error.kind(), StoreErrorKind::Corrupt); + if existing { + assert_eq!( + std::fs::read(&target).expect("outside target read"), + b"keep", + "existing outside target must be untouched" + ); + } else { + assert!( + !target.exists(), + "absent outside target must never be created" + ); + } + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); + } +} + +#[test] +fn hardlinked_lock_is_refused_without_touching_either_name() { + let root = scratch_root("lock-hardlink"); + let storage = StorageRoot::new(&root).expect("root"); + let namespace_dir = root.join("evil"); + std::fs::create_dir(&namespace_dir).expect("namespace dir"); + let lock = namespace_dir.join("lock"); + let other = namespace_dir.join("lock-copy"); + std::fs::write(&lock, b"keep").expect("lock fixture"); + std::fs::hard_link(&lock, &other).expect("hard link"); + let error = storage + .open_namespace(&name("evil")) + .expect_err("hardlinked lock must be refused"); + assert_eq!(error.kind(), StoreErrorKind::Corrupt); + assert_eq!(std::fs::read(&lock).expect("lock read"), b"keep"); + assert_eq!(std::fs::read(&other).expect("copy read"), b"keep"); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn every_state_member_symlink_is_refused_at_open() { + for (index, member) in ["snapshot", "previous", "journal", "lock"] + .into_iter() + .enumerate() + { + for existing in [false, true] { + let label = if existing { "existing" } else { "absent" }; + let root = scratch_root(&format!("member-{index}-{label}")); + let outside = scratch_root(&format!("member-{index}-{label}-outside")); + let storage = StorageRoot::new(&root).expect("root"); + let namespace_dir = root.join("evil"); + std::fs::create_dir(&namespace_dir).expect("namespace dir"); + let target = outside.join(member); + if existing { + std::fs::write(&target, b"keep").expect("outside target"); + } + std::os::unix::fs::symlink(&target, namespace_dir.join(member)) + .expect("member symlink"); + let error = storage + .open_namespace(&name("evil")) + .expect_err("state member symlink must be refused"); + assert_eq!(error.kind(), StoreErrorKind::Corrupt, "member: {member}"); + if existing { + assert_eq!( + std::fs::read(&target).expect("outside target read"), + b"keep", + "outside target must be untouched for {member}" + ); + } else { + assert!( + !target.exists(), + "outside target must not be created for {member}" + ); + } + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); + } + } +} + +#[test] +fn namespace_ancestor_replacement_does_not_redirect_commit() { + let root = scratch_root("ancestor-swap"); + let outside = scratch_root("ancestor-swap-outside"); + let storage = StorageRoot::new(&root).expect("root"); + let mut namespace = storage.open_namespace(&name("ns")).expect("open"); + namespace + .commit(0, vec![row("a", b"1")], Vec::new()) + .expect("first commit"); + + // Replace the namespace directory with a symlink to an isolated outside + // fixture after the handle is open. The open directory fd must keep the + // commit anchored in the original directory, never in the fixture. + std::fs::rename(root.join("ns"), root.join("ns-original")).expect("rename namespace"); + std::os::unix::fs::symlink(&outside, root.join("ns")).expect("ancestor symlink"); + let error = namespace + .commit(1, vec![row("b", b"2")], Vec::new()) + .expect_err("a replaced namespace must fail closed"); + assert_eq!( + error.kind(), + StoreErrorKind::Uncertain, + "a detected detached namespace must require reopen, not report corruption" + ); + assert!( + outside.read_dir().expect("outside read").next().is_none(), + "commit must not create anything in the outside fixture" + ); + assert!( + root.join("ns-original").join("snapshot").exists(), + "the original namespace directory must remain intact" + ); + drop(namespace); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +#[cfg(unix)] +fn root_initialization_refuses_symlinked_ancestor() { + let parent = scratch_root("root-parent"); + let actual = scratch_root("root-actual"); + let link = parent.join("link"); + std::os::unix::fs::symlink(&actual, &link).expect("symlink ancestor"); + let result = StorageRoot::new(link.join("nested")); + assert!(result.is_err(), "root walk must refuse symlink ancestors"); + assert!( + !actual.join("nested").exists(), + "refusal must not create through symlink" + ); + let _ = std::fs::remove_dir_all(parent); + let _ = std::fs::remove_dir_all(actual); +}