diff --git a/.coveragerc b/.coveragerc index f0149e9..e0a33c5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ source = zeus [report] -fail_under = 70 +fail_under = 79 precision = 2 show_missing = True skip_covered = True diff --git a/.env.example b/.env.example index 7351323..f43cadf 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,10 @@ ZEUS_API_LOG_ENABLED=1 # Optional workspace-local runtime path. ZEUS_STATE_DIR=.zeus +# SQLite commit durability. NORMAL is the compatibility/default local setting; +# FULL adds power-loss durability at the cost of higher commit latency. +ZEUS_SQLITE_SYNCHRONOUS=NORMAL + # Optional: send SIGKILL if SIGTERM does not stop Hermes before the grace timeout. # Disabled by default to avoid interrupting a still-shutting-down gateway. ZEUS_STOP_KILL_AFTER_TIMEOUT=0 @@ -50,7 +54,12 @@ ZEUS_ALLOW_LEGACY_PID_MARKERS=1 # Keep empty unless required for proxies or certificate bundles. ZEUS_ENV_PASSTHROUGH= -# Provider and gateway secrets referenced by bundled templates. +# Provider and gateway secrets referenced by bundled templates. Copy this file +# to the trusted workspace ./.env, then import only the names a bot needs with +# `zeus bot create ... --env-from NAME`. A process environment value takes +# precedence over ./.env. Do not pass secrets with legacy `--env NAME=VALUE`; +# command arguments may be exposed in shell history and process listings. After +# copying this file, run `chmod 0600 .env` before adding real secrets. OPENROUTER_API_KEY= NOUS_API_KEY= OPENAI_API_KEY= diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index af86253..d0a3bb1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -7,7 +7,7 @@ body: id: zeus-version attributes: label: Zeus version - description: Output from `zeus --version` or the installed package version. + description: Run `zeus --version` and paste the complete output. validations: required: true - type: input diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b05889c..94c9b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,8 @@ concurrency: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -50,8 +51,23 @@ jobs: coverage run -m unittest discover -s tests coverage report + python-3-14: + continue-on-error: true + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.14" + - name: Install package and test tools + run: python -m pip install -e ".[dev]" + - name: Run Zeus test suite + run: sh scripts/test.sh + lifecycle-subprocess: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -62,8 +78,60 @@ jobs: - name: Run subprocess lifecycle tests run: python -m unittest tests.test_subprocess_lifecycle + macos-process-lifecycle: + runs-on: macos-26 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + - name: Install package and test tools + run: python -m pip install -e ".[dev]" + - name: Run focused process lifecycle tests + run: | + python -m unittest -v \ + tests.test_subprocess_lifecycle \ + tests.test_fake_hermes_integration \ + tests.test_crash_recovery.GatewayLauncherTests + + real-hermes: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + - name: Prepare sanitized failure evidence + run: | + mkdir -p .tmp/real-hermes-evidence + printf '%s\n' 'result=failed' 'failure_stage=ci_setup' > .tmp/real-hermes-evidence/summary.txt + - name: Install pinned Hermes compatibility environment + run: |- + python -m pip install --require-hashes --only-binary=:all: -r requirements-hermes-ci.txt + - name: Install Zeus + run: python -m pip install -e . + - name: Check installed dependencies + run: python -m pip check + - name: Verify real Hermes without provider credentials + run: | + ZEUS_VERIFY_START_GATEWAY=1 \ + ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 \ + ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence \ + sh scripts/verify_real_hermes.sh + - name: Upload sanitized failure evidence + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: real-hermes-failure-evidence + path: .tmp/real-hermes-evidence/summary.txt + if-no-files-found: error + retention-days: 7 + package: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -71,11 +139,13 @@ jobs: python-version: "3.11" - name: Install package build tools run: python -m pip install -e ".[dev]" + - name: Check installed dependencies + run: python -m pip check - name: Build package run: | rm -rf dist python -m build - - name: Smoke test built wheel + - name: Verify installed wheel behavior run: ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh - name: Check package metadata run: twine check dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a8de793..b01dc7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,10 @@ jobs: exit 1 } python scripts/check_version_tag.py "${GITHUB_REF_NAME}" --require-changelog + - name: Require GitHub-verified release ref + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/check_verified_release_ref.py - name: Install shellcheck run: | sudo apt-get update diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bcea3..f27aa20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Added descriptive CLI help and a state-free `zeus --version` command backed by the package + version. +- Required future release tags and their referenced commits to be signed and GitHub-verified + before the privileged publishing job can run. + ## 0.3.0 - Relaunched the public repository from an audited, single-root history on the `main` branch while diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba3bf58..3a9986b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,10 +7,15 @@ Zeus is intentionally small and workspace-local. Changes should keep the project ```bash python3 -m venv .venv . .venv/bin/activate -pip install -e . -sh scripts/test.sh +python -m pip install -e ".[dev]" +make check ``` +See the [compatibility policy](docs/COMPATIBILITY.md) for the operating systems, +Python versions, and Hermes boundary covered by committed automation. The +committed Ubuntu/Python 3.11 gate uses the hash-locked Hermes Agent 0.19.0 +environment; the manual check below covers the operator's installed version. + ## Quality Bar - Keep the core runtime dependency-free unless there is a clear operational reason. @@ -25,7 +30,8 @@ sh scripts/test.sh ## Real Hermes Verification -The default test suite uses a fake Hermes executable to verify Zeus process handling. Before release, run: +The default test suite uses a fake Hermes executable to verify Zeus process handling. +Before release, separately run: ```bash sh scripts/verify_real_hermes.sh diff --git a/Makefile b/Makefile index fc9a699..2ec7e7a 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,10 @@ check: ruff check . mypy zeus bandit -r zeus + shellcheck scripts/*.sh build: + python -m pip check python -m build twine check dist/* @@ -44,6 +46,7 @@ release-check: mypy zeus bandit -r zeus shellcheck scripts/*.sh + python -m pip check rm -rf dist python -m build ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh diff --git a/README.md b/README.md index 50c8632..a0878ae 100644 --- a/README.md +++ b/README.md @@ -48,18 +48,59 @@ run summaries at that boundary. ## Quick Start +### 1. Credential-free offline demo + +The fastest first success needs neither Hermes nor provider credentials. From a +checkout: + ```bash python3 -m venv .venv . .venv/bin/activate -pip install -e . +python -m pip install -e . + +zeus demo up +zeus demo status +zeus demo down +``` + +The demo uses Zeus' packaged fake-Hermes executable and stores its disposable +runtime under `ZEUS_STATE_DIR` (the workspace-local `.zeus/` directory by +default). It exercises real profile rendering and process lifecycle behavior +without contacting a provider. + +### 2. Real Hermes setup + +Check the installed Hermes version, then prepare a private workspace secret +file: + +```bash +hermes version cp .env.example .env +chmod 0600 .env +``` + +`.env.example` contains empty placeholders and is not ready to import. Stop here +until `.env` contains a real, non-empty provider key required by the selected +template, such as `OPENROUTER_API_KEY` for `coding-bot`. As an alternative, +provide the same named secret through a secure process-environment mechanism. + +Then validate Zeus and render the real Hermes profile: +```bash zeus doctor zeus template list -zeus bot create coder --template coding-bot +zeus bot create coder --template coding-bot --env-from OPENROUTER_API_KEY zeus bot doctor coder ``` +`--env-from NAME` imports a named value from the process environment first and +then the trusted workspace `./.env`; the value never enters the Zeus argument +list or command output. A present but empty process value is an error and does +not fall back to `.env`. Keep the workspace `.env` private with `chmod 0600 .env`. +The legacy `--env NAME=VALUE` form remains available +for non-secret compatibility values, but is unsafe for secrets because command +arguments can be retained in shell history and exposed in process listings. + Safety model: Zeus is a local process orchestrator, not a sandbox. Use Docker or another Hermes terminal backend for untrusted tasks. Do not expose the API directly to a network; keep it on loopback or behind a separately hardened @@ -74,8 +115,10 @@ ZEUS_API_KEY=change-me sh scripts/start.sh ## 60-Second Demo -The asciinema recording in [docs/assets/demo.cast](docs/assets/demo.cast) mirrors the local -operator flow: +The pre-recorded asciinema cast in [docs/assets/demo.cast](docs/assets/demo.cast) +illustrates the local operator flow. It is not evidence that the current Zeus +checkout is compatible with whichever Hermes version is installed today; use +the live verification steps below for that evidence. ```bash zeus doctor @@ -98,6 +141,7 @@ zeus bot stop coder - [Operations](docs/OPERATIONS.md) - [Reconcile scheduling](docs/RECONCILE.md) - [Release process](docs/RELEASE.md) +- [Compatibility policy](docs/COMPATIBILITY.md) - [Roadmap](docs/ROADMAP.md) - [Contributing](CONTRIBUTING.md) - [Code of conduct](CODE_OF_CONDUCT.md) @@ -112,16 +156,10 @@ Zeus is maintained by [BrainX](https://github.com/brainx). See [Credits](CREDITS - Hermes Agent installed as `hermes` for real bot startup - Optional Docker or another Hermes terminal backend for stronger execution isolation -No Python package dependencies are required for the current MVP. - -## Setup - -```bash -python3 -m venv .venv -. .venv/bin/activate -pip install -e . -cp .env.example .env -``` +Zeus has no required third-party Python runtime dependencies. Development and +build tools are available separately through the optional `dev` dependency +group; see [Contributing](CONTRIBUTING.md). The exact automated platform and +Python matrix is recorded in the [compatibility policy](docs/COMPATIBILITY.md). ## Install Modes @@ -197,12 +235,17 @@ ZEUS_VERIFY_START_GATEWAY=1 sh scripts/verify_real_hermes.sh ``` The gateway check enables Hermes' local `api_server` platform on loopback, -passes a random per-run API key, probes `/health`, and then stops the bot. +passes an isolated local API key, starts with readiness waiting, verifies process +ownership, probes `/health`, and then stops the bot. Committed CI runs this flow +without provider credentials against the fully hash-locked Hermes Agent 0.19.0 +environment documented in the compatibility policy. For a clean Debian/Ubuntu host, use the fresh VPS harness: ```bash -ZEUS_VPS_INSTALL_PACKAGES=1 ZEUS_VPS_INSTALL_HERMES=1 bash scripts/fresh_vps_verify.sh +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ +ZEUS_VPS_INSTALL_PACKAGES=1 ZEUS_VPS_INSTALL_HERMES=1 \ +bash scripts/fresh_vps_verify.sh ``` See [Fresh VPS test](docs/FRESH_VPS_TEST.md) for gateway and async-delegation probes. @@ -286,6 +329,20 @@ source tree. Rendered `.env` values are serialized with quoting when needed so whitespace, `#`, quotes, and backslashes cannot create extra assignments. +Import template secrets by name instead of placing their values in argv: + +```bash +zeus bot create coder \ + --template coding-bot \ + --env-from OPENROUTER_API_KEY +``` + +Zeus looks for each imported name in the process environment and then the +trusted workspace `./.env`. Process values take precedence. Missing and empty +values fail bot creation without printing the value. Keep `./.env` mode `0600`; +Zeus also writes the imported values only to the selected profile's mode-`0600` +`.env` file. + Built-in templates include OpenRouter-backed bots and `deepseek-coding-bot`, which uses Hermes' native DeepSeek provider with `DEEPSEEK_API_KEY`. Example templates also cover gateway operations, log triage, and documentation writing. Each template should set a bounded async delegation cap: diff --git a/docs/API.md b/docs/API.md index 1460a85..e0e7d9f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -38,7 +38,7 @@ Known error codes are `invalid_request`, `invalid_bot_id`, `unknown_bot`, `idempotency_key_conflict`, `idempotency_in_progress`, `idempotency_indeterminate`, `idempotency_response_too_large`, `idempotency_store_unavailable`, -`server_busy`, `server_draining`, and `internal_error`. +`server_busy`, `server_draining`, `not_ready`, and `internal_error`. JSON responses include `cache-control: no-store`. Mutating endpoints that accept request bodies require an `application/json` content type and reject missing or invalid media @@ -159,12 +159,37 @@ permission, or serialization failures never change the HTTP response. Setting ### `GET /health` +Public process-liveness check. It does not access SQLite or authenticate the +caller, so a successful response does not mean the state store is ready. + Returns: ```json {"status":"ok"} ``` +### `GET /ready` + +Authenticated state-store readiness check, also available as `GET /v1/ready`. +It opens the existing SQLite database in read-only mode, requires schema version +6, and executes `SELECT 1`; it does not inspect or start bots. A stopped bot does +not make Zeus unready. + +The route uses the normal read-endpoint authentication policy. It requires +`x-zeus-api-key` unless loopback-only development explicitly enables +`ZEUS_ALLOW_UNAUTH_READS=1`. Query parameters are rejected before the database +probe. + +Success returns: + +```json +{"schema_version":6,"status":"ready"} +``` + +An unavailable, missing, malformed, older, or newer state database returns +`503` with `error.code=not_ready`. Failure to initialize the state store before +the API binds remains a startup failure rather than an HTTP readiness response. + ### `GET /doctor` Returns the same readiness report as `zeus doctor --json`. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 23bbcd4..9d1b5fe 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,18 +32,51 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.models`: Template, bot, and status validation. - `zeus.templates`: Bundled plus local TOML template discovery with duplicate ID checks. - `zeus.renderer`: Hermes profile rendering. -- `zeus.state`: SQLite bot projection and authoritative lifecycle ledger. +- `zeus.sqlite_db`: Shared SQLite connection factory and per-connection durability policy. +- `zeus.schema`: Schema-v6 initialization, compatibility guards, and forward migrations. +- `zeus.idempotency_store`: Durable API mutation claims and replay responses. +- `zeus.reconcile_store`: Persisted fleet reconciliation runs and ordered results. +- `zeus.bot_lifecycle_store`: Bot projection, intent, lifecycle ledger, history, and audit mirror. +- `zeus.state`: Compatibility facade composing the shared database and persistence stores. - `zeus.lifecycle`: Bounded lifecycle event types and recursively redacted details. - `zeus.request_context`: Locally generated request IDs and normalized route templates. +- `zeus.api_errors`: Transport-neutral API exception classification. +- `zeus.api_request`: Strict path, query, and JSON request parsing. +- `zeus.api_server`: Bounded HTTP concurrency and graceful server lifecycle. - `zeus.api_logging`: Locked, fail-open, secret-safe API JSONL output. - `zeus.idempotency`: Key validation and canonical request hashing. +- `zeus.process_identity`: Standard-library process observation and trusted-command checks. +- `zeus.gateway_marker`: Effect-free schema-v3 marker parsing and exact-generation values. - `zeus.hermes_adapter`: Subprocess command construction for Hermes. - `zeus.gateway_launcher`: Descriptor-only marker-before-exec helper. -- `zeus.supervisor`: Gateway lifecycle, PID ownership markers, logs, and status. -- `zeus.api`: Local HTTP API. +- `zeus.profile_manager`: Atomic profile installation, delete, archive, and rollback transactions. +- `zeus.gateway_runtime`: Process, marker, readiness, signal, and cleanup effects. +- `zeus.intent_recovery`: Store-free pending-intent recovery decisions through a structural host. +- `zeus.supervisor`: Public lifecycle facade, locks, durable intent transitions, and audit ordering. +- `zeus.api`: Local HTTP routes and compatibility facade. - `zeus.cli`: Operator CLI. - `zeus.doctor`: Readiness diagnostics. +## SQLite Durability + +Zeus uses persistent SQLite WAL mode. `SQLiteDatabase` installs the selected +`ZEUS_SQLITE_SYNCHRONOUS` policy on every returned operational connection after +both newer-schema guards, foreign-key enforcement, and WAL setup. The raw +read-only schema preflight and readiness probe cannot commit and are +intentionally not configured with durability PRAGMAs. + +Committed transactions survive an application or Zeus process crash under both +NORMAL and FULL. With NORMAL, SQLite omits a WAL sync on most commits, so a host +OS crash, hard reset, or power loss can roll back recently reported commits +after recovery while retaining WAL consistency. With FULL, SQLite syncs the WAL +at each commit to provide durability across OS crash or power loss, at the cost +of commit latency. + +The setting is per connection: every process that writes the same database must +select the intended mode. It covers SQLite only. Rendered profile files, PID +markers, locks, and the best-effort audit JSONL remain separate filesystem +state, and neither mode replaces backup and restore procedures. + ## Process Lifecycle 1. `zeus bot create` precomputes a profile, stages it under the profiles directory, @@ -63,6 +96,20 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. 7. Stop commits its stopped intent before verifying ownership and signaling; SIGTERM/SIGKILL authorization is rechecked against the exact process identity. +`Supervisor` owns the per-bot locks, lifecycle correlation context, `StateStore` +calls, and authoritative transition/audit ordering. `ProfileManager` changes only +profile and archive paths. `GatewayRuntime` owns the mutable in-process gateway +registry and host effects but never reads or writes SQLite. `PendingIntentRecovery` +contains the pending start, stop, and restart decision flow and resolves facade +methods dynamically through its structural host; it does not acquire locks, mint +operation IDs, or import the concrete facade or persistence layer. + +Status never spawns or signals; it may adopt exact evidence and repair the durable +projection. Reconcile may recover one effect per pass: a pending restart first +stops or cleans its exact schema-v3 predecessor and persists that observation, +then a later pass adopts or launches the replacement with the same pending +operation ID. + Hermes child processes receive a minimal host environment plus profile `.env` values. Operators can allow specific host variables with `ZEUS_ENV_PASSTHROUGH`. @@ -130,7 +177,7 @@ The v2-to-v3 migration is also one transaction. It creates a the projection/event invariant, and advances the schema version only after all steps succeed. Additive v3-to-v4 and v4-to-v5 upgrades add durable idempotency and desired/pending intent in forward-only transactions. Databases newer than -schema v5 are rejected rather than downgraded. +schema v6 are rejected rather than downgraded. `$ZEUS_STATE_DIR/logs/audit.jsonl` remains a best-effort compatibility mirror. It is written only after the SQLite transaction commits and is not imported into diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md new file mode 100644 index 0000000..14a5aa1 --- /dev/null +++ b/docs/COMPATIBILITY.md @@ -0,0 +1,85 @@ +# Compatibility Policy + +This document records compatibility evidence produced by the current committed +automation. It distinguishes repeatable CI from manual checks and does not turn +an untested platform or external Hermes release into a support claim. + +## Automated matrix + +| Gate | Committed runner | Python | Scope | +| --- | --- | --- | --- | +| Main CI matrix | Linux `ubuntu-24.04` | Python 3.11, 3.12, and 3.13 | Unit and integration tests, repository contracts, source-and-branch coverage, formatting, lint, typing, Bandit, and ShellCheck | +| Provisional Python compatibility | Linux `ubuntu-24.04` | Python 3.14 | Full Zeus test suite; non-required and Zeus-only because the pinned Hermes baseline requires Python below 3.14 | +| Subprocess lifecycle | Linux `ubuntu-24.04` | Python 3.11 | Focused multi-process lifecycle and locking behavior | +| macOS process lifecycle | macOS `macos-26` | Python 3.13 | Focused process, fake-Hermes integration, and gateway-launcher recovery tests | +| Real Hermes compatibility | Linux `ubuntu-24.04` | Python 3.11 | Hash-locked Hermes Agent 0.19.0 profile rendering, strict diagnostics, loopback gateway readiness, process ownership, and clean shutdown without a model-provider credential | +| Package build | Linux `ubuntu-24.04` | Python 3.11 | Wheel and source build, installed-wheel smoke test, dependency consistency, and metadata checks | +| Tagged release build | Linux `ubuntu-latest` | Python 3.11 | Full release gate, artifact checksums, and GitHub release artifacts | + +In short, the focused Linux lifecycle and package jobs use Python 3.11. Main CI +uses the explicit `ubuntu-24.04` image, while the separate tagged-release +workflow still uses `ubuntu-latest`. The focused macOS lane uses `macos-26` and +Python 3.13. Windows is not currently automated. GitHub manages the contents of +all hosted runner images and may update them over time; results from an +individual developer machine remain local evidence rather than an automated +platform guarantee. + +Python 3.14 is a provisional Zeus-only lane with `continue-on-error` behavior. +It does not promote Python 3.14 to required Hermes compatibility: the repository +pins Hermes Agent 0.19.0, whose package metadata requires Python 3.11 through +3.13, and runs that compatibility gate only on Python 3.11. + +The package metadata declares `requires-python = ">=3.11"`, while committed CI +currently tests the versions listed above. A version absent from that matrix is +not covered by the current automated compatibility claim. + +## SQLite durability compatibility + +Unset or empty `ZEUS_SQLITE_SYNCHRONOUS` configuration remains NORMAL, as do +direct `StateStore(path)` and `SQLiteDatabase(path)` calls. Upgrading therefore +does not silently change local commit latency. FULL is an explicit +higher-durability option for deployments that accept its additional commit +latency. + +This policy does not change database structure or require a migration: the +schema remains schema v6 and every existing v6 database stays compatible. + +## Manual clean-host evidence + +[`scripts/fresh_vps_verify.sh`](../scripts/fresh_vps_verify.sh) provides a manual +clean-host runbook for Debian and Ubuntu. It can bootstrap OS packages, install +Zeus into a virtual environment, run local gates, render multiple profiles, and +exercise the loopback API. Optional Hermes installation and live probes cross an +external network and credential boundary, so their logs are evidence for that +specific host and invocation rather than the committed CI environment. + +Local development checks such as `make check` and `sh scripts/wheel_smoke.sh` +remain useful evidence, but they do not add the developer's operating system to +the automated matrix. + +## Hermes boundary + +The deterministic CI baseline is Hermes Agent 0.19.0 on Ubuntu 24.04 with Python +3.11. [`requirements-hermes-ci.txt`](../requirements-hermes-ci.txt) pins the +complete 60-package wheel closure and its selected SHA-256 hashes. CI installs it +with pip hash checking and binary-only resolution; it never runs the remote +Hermes installer. The lock also carries Linux arm64 hashes for native local +container verification without changing the CI package versions. + +The gate uses no model-provider credential or paid request. It renders a profile, +runs strict Zeus and Hermes diagnostics, starts the loopback gateway with +`--wait`, checks Zeus process ownership and Hermes `/health`, then stops the bot +and removes runtime state. On failure it uploads only a two-line sanitized stage +summary, never the rendered profile, environment, logs, or process arguments. + +The manual [`scripts/verify_real_hermes.sh`](../scripts/verify_real_hermes.sh) +check still uses whichever `hermes` executable is installed on `PATH` unless +`ZEUS_VERIFY_EXPECTED_HERMES_VERSION` is set. Record `hermes version` with manual +evidence. Passing the pinned baseline does not establish compatibility with every +Hermes release or optional integration. + +## Updating this policy + +Update this file in the same change that adds or removes a CI runner, Python +version, package gate, or reproducible Hermes baseline. Aspirational platforms +belong in the [roadmap](ROADMAP.md), not in the automated matrix. diff --git a/docs/FRESH_VPS_TEST.md b/docs/FRESH_VPS_TEST.md index 5769a9e..4a3968c 100644 --- a/docs/FRESH_VPS_TEST.md +++ b/docs/FRESH_VPS_TEST.md @@ -15,25 +15,33 @@ Use this runbook to verify Zeus on a clean Debian or Ubuntu VPS with a real Herm ## Threat Model -The VPS test downloads and executes the official Hermes installer only when `ZEUS_VPS_INSTALL_HERMES=1` is set. That crosses the network boundary and trusts the Hermes installer HTTPS endpoint. Run it only on a disposable host or after reviewing the downloaded installer saved in the evidence directory. +The VPS test downloads the official Hermes installer only when `ZEUS_VPS_INSTALL_HERMES=1` is set. Remote bootstrap also requires `ZEUS_VPS_HERMES_INSTALLER_SHA256` to contain the exact 64-hex-character SHA-256 digest of an installer you reviewed. The script verifies the downloaded bytes with a constant-time comparison and refuses to execute a missing, malformed, or mismatched digest. This still crosses the network boundary, so use a digest obtained through a trusted review process and run the bootstrap only on a disposable host. -Do not put provider tokens in command history. Configure Hermes credentials through Hermes' own setup flow or a secure environment mechanism. Review `.tmp/fresh-vps-verify/.../run.log` before sharing it, because Hermes tools may print environment-specific diagnostics. +Do not put provider tokens in command history. Configure Hermes credentials through Hermes' own setup flow or a secure environment mechanism. The API smoke key is generated ephemerally by default; authenticated curl commands use a redacted transcript label and receive the key through standard input instead of an argument. Review `.tmp/fresh-vps-verify/.../run.log` before sharing it, because Hermes tools may print environment-specific diagnostics. + +`ZEUS_VPS_ASYNC_PROMPT` is also replaced by a fixed label in the transcript. Hermes still receives that operator-supplied prompt through its established `-z ` argument, so it may be visible to local process inspection while that optional probe runs. ## Fresh Host Flow Clone or copy this repository onto the VPS, then run from the repository root: ```bash +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ ZEUS_VPS_INSTALL_PACKAGES=1 \ ZEUS_VPS_INSTALL_HERMES=1 \ bash scripts/fresh_vps_verify.sh ``` -This installs basic Debian/Ubuntu packages, creates `.venv/`, installs Zeus editable, installs Hermes if missing, runs the local gates, checks Hermes, renders all templates, and runs an API smoke test. +Obtain the pinned digest separately: download the installer without executing it, review the saved file, and calculate its SHA-256 with a trusted local tool such as `sha256sum`. The verifier never learns or trusts a digest from the download endpoint itself. + +The command installs basic Debian/Ubuntu packages, creates `.venv/`, installs Zeus editable, installs the digest-verified Hermes payload if missing, runs the local gates, checks Hermes, renders all templates, and runs an API smoke test. + +On a minimal supported apt host where `python3` is not yet available, `ZEUS_VPS_INSTALL_PACKAGES=1` permits a minimal Python 3 prerequisite install before the private evidence log can be initialized. That early package-manager output is console-only; the normal package step is still recorded afterward. Without that explicit package-bootstrap opt-in, the verifier fails with a prerequisite error. To also start a real Hermes gateway for the default real-Hermes bot: ```bash +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ ZEUS_VPS_INSTALL_PACKAGES=1 \ ZEUS_VPS_INSTALL_HERMES=1 \ ZEUS_VPS_START_GATEWAY=1 \ @@ -66,7 +74,7 @@ The pass criteria are: ## Evidence -The script writes logs under: +The script writes logs under a workspace-relative directory with mode `0700`: ```text .tmp/fresh-vps-verify// @@ -74,10 +82,12 @@ The script writes logs under: Important files: -- `run.log`: full command transcript, including `git rev-parse HEAD` and `git status --short` when run from a Git checkout. -- `hermes-install.sh`: downloaded installer, when Hermes installation is enabled. +- `run.log`: command transcript, including `git rev-parse HEAD` and `git status --short` when run from a Git checkout; sensitive invocations use fixed redacted labels. +- `hermes-install.sh`: downloaded and digest-verified installer, when Hermes installation is enabled. - `zeus-api.log`: API server log from the loopback smoke test. +Evidence files use mode `0600`. The verifier rejects absolute, escaping, symlinked, or otherwise unsafe evidence paths instead of changing permissions on their targets. `ZEUS_VPS_LOG_DIR` must remain a workspace-relative scratch directory. + Runtime state is created under ignored workspace directories such as `.zeus-real-hermes-check/`, `.zeus-vps-multi/`, and `.zeus-vps-api/`. Clean up after collecting evidence: diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0405af3..84bae68 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,5 +1,55 @@ # Operations +## SQLite Durability + +Zeus uses SQLite WAL mode and applies `ZEUS_SQLITE_SYNCHRONOUS` to every +operational connection. Unset or empty configuration defaults to NORMAL for +upgrade compatibility. The bundled API and reconcile services select FULL. + +Committed transactions survive an application or Zeus process crash under both +NORMAL and FULL. With NORMAL, SQLite omits a WAL sync on most commits; a host OS +crash, hard reset, or power loss can roll back recently reported commits after +recovery while WAL consistency is retained. With FULL, SQLite syncs the WAL at +each commit to provide durability across OS crash or power loss, at the cost of +commit latency. + +The policy is per connection. Every process that writes the same database, +including manual CLI and doctor commands capable of migration, must select the +intended mode. A NORMAL writer does not change an existing FULL connection, but +its own commits retain NORMAL power-loss semantics. For hosted manual work where +FULL durability is expected, carry the setting explicitly: + +```bash +sudo -u zeus env \ + ZEUS_STATE_DIR=/var/lib/zeus \ + ZEUS_SQLITE_SYNCHRONOUS=FULL \ + /opt/zeus/.venv/bin/zeus bot reconcile +sudo -u zeus env \ + ZEUS_STATE_DIR=/var/lib/zeus \ + ZEUS_SQLITE_SYNCHRONOUS=FULL \ + /opt/zeus/.venv/bin/zeus doctor --strict +``` + +The setting covers SQLite only. It does not make rendered profile files, PID +markers, locks, or the best-effort audit JSONL atomic with the database, replace +backups, or overcome storage that lies about flushes. Continue the backup and +restore procedures below under either mode. + +## API Liveness and Readiness + +Use `GET /health` for public process liveness. It does not authenticate or touch +SQLite, so it can remain healthy while persistent state is unavailable. + +Use `GET /ready` (or `GET /v1/ready`) for load-balancer and service readiness. +This ordinary read endpoint requires `x-zeus-api-key`, except on a loopback bind +when `ZEUS_ALLOW_UNAUTH_READS=1` is explicitly enabled. It opens the existing +database read-only, requires the current schema version, and runs `SELECT 1`. +It never creates or migrates a database and does not require bots to be running. + +A ready service returns `{"schema_version":6,"status":"ready"}`. State-store +failures return `503` with `error.code=not_ready`. If state initialization fails +before the API binds, the process exits instead of serving `/ready`. + ## Backup Back up `ZEUS_STATE_DIR` regularly. For the sample systemd deployment, that is @@ -59,7 +109,7 @@ Start services and verify the restored host: ```bash sudo systemctl start zeus-api sudo systemctl start zeus-reconcile.timer -sudo -u zeus env ZEUS_STATE_DIR=/var/lib/zeus \ +sudo -u zeus env ZEUS_STATE_DIR=/var/lib/zeus ZEUS_SQLITE_SYNCHRONOUS=FULL \ /opt/zeus/.venv/bin/zeus doctor --strict ``` @@ -354,6 +404,23 @@ Hermes child processes receive a minimal environment by default plus variables rendered into the bot profile `.env`. Zeus does not pass the full API service or operator shell environment to child processes. +Import bot secrets without putting their values in command arguments: + +```bash +zeus bot create coder \ + --template coding-bot \ + --env-from OPENROUTER_API_KEY +``` + +For every `--env-from NAME`, Zeus reads the process environment first and then +the trusted workspace `./.env`. A present but empty process value fails closed; +it does not fall back to `.env`. Missing and empty errors identify only the +variable name, and imported values are not printed. Keep the trusted source +private with `chmod 0600 .env`. Zeus persists imported values only in the +selected bot profile's `.env`, which Zeus writes with mode `0600`. The legacy +`--env NAME=VALUE` form remains compatible for non-secret values but is unsafe +for secrets because argv can be visible in shell history and process listings. + To pass selected host variables, set an explicit allowlist: ```dotenv diff --git a/docs/REAL_HERMES_VERIFICATION.md b/docs/REAL_HERMES_VERIFICATION.md index 54788ff..664c5ed 100644 --- a/docs/REAL_HERMES_VERIFICATION.md +++ b/docs/REAL_HERMES_VERIFICATION.md @@ -1,6 +1,11 @@ # Real Hermes Verification -The normal test suite uses a fake Hermes executable so the repository can be tested without external credentials or a Hermes install. +The normal test suite uses a fake Hermes executable so the repository can be +tested without external credentials or a Hermes install. CI separately installs +the fully hash-locked Hermes Agent 0.19.0 environment from +`requirements-hermes-ci.txt` on Ubuntu/Python 3.11. The lock contains Linux +x86_64 and arm64 wheel hashes; the gate does not run the remote installer or make +a model-provider request. Before a release, verify against a real Hermes install: @@ -27,19 +32,26 @@ before stopping the bot: ZEUS_VERIFY_START_GATEWAY=1 sh scripts/verify_real_hermes.sh ``` -When gateway startup verification is enabled, the script waits for Zeus to report -the bot as running, then polls Hermes `/health` until the local `api_server` +When gateway startup verification is enabled, the script starts Zeus with +`--wait`, confirms Zeus reports the bot as running, then polls Hermes `/health` +until the local `api_server` reports `{"status":"ok","platform":"hermes-agent"}` or the health timeout expires. This avoids false negatives when Hermes binds the loopback API shortly after the process becomes visible to Zeus. +Successful and failed runs stop the bot and remove the isolated runtime tree. +Failures retain only a sanitized `summary.txt` containing the fixed result and +failure-stage labels. Raw logs, rendered profiles, environments, and command +arguments are never copied into that evidence directory. + Useful overrides: ```bash ZEUS_VERIFY_BOT_ID=my-check-bot ZEUS_VERIFY_TEMPLATE=research-bot ZEUS_VERIFY_STATE_DIR=.zeus-real-hermes-check -ZEUS_VERIFY_GATEWAY_SECONDS=5 +ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence +ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 ZEUS_VERIFY_API_KEY=real-hermes-local-check ZEUS_VERIFY_API_SERVER_HOST=127.0.0.1 ZEUS_VERIFY_API_SERVER_PORT=4312 @@ -47,6 +59,10 @@ ZEUS_VERIFY_HEALTH_TIMEOUT_SECONDS=30 ZEUS_VERIFY_HEALTH_INTERVAL_SECONDS=0.5 ``` +Leave `ZEUS_VERIFY_EXPECTED_HERMES_VERSION` unset for an intentional manual +compatibility check against another installed version. Such a run is local +evidence, not an update to the committed baseline. + Expected failure when Hermes is not installed: ```text diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 324667a..f062d92 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -2,8 +2,9 @@ Zeus releases are built from annotated version tags. Keep releases small, verify the repository locally first, and publish GitHub artifacts before considering -package-index distribution. Maintainers are encouraged to sign tags, but the -workflow does not cryptographically verify signer identity. +package-index distribution. Future releases require both the annotated tag and +its referenced commit to carry signatures that GitHub marks verified. The +historical v0.3.0 release predates this policy and remains unchanged. 1. Ensure CI is green on the commit to release. 2. Run the full local release gate: @@ -45,16 +46,18 @@ workflow does not cryptographically verify signer identity. python scripts/check_version_tag.py vX.Y.Z --require-changelog ``` -6. Create and push an annotated tag: +6. Confirm the release commit is signed and GitHub-verified, then create and push + a signed annotated tag: ```bash - git tag -a vX.Y.Z -m "Zeus vX.Y.Z" + git log --show-signature -1 + git tag -s vX.Y.Z -m "Zeus vX.Y.Z" git push origin vX.Y.Z ``` - If the maintainer has a configured signing identity, `git tag -s` may be - used instead. The release gate enforces an annotated tag but does not verify - the signature or signer identity. + A merely annotated or locally valid signature is insufficient: GitHub must + report both the tag and commit verification objects as `verified` with reason + `valid`. Configure the signing identity with GitHub before pushing the tag. 7. Confirm the GitHub release workflow completed and attached the generated `dist/*` artifacts plus `dist/SHA256SUMS.txt` to the GitHub Release. @@ -63,13 +66,17 @@ workflow does not cryptographically verify signer identity. `.github/workflows/release.yml` builds and checks distribution artifacts for `v*.*.*` tags, rejects lightweight tags and tags that do not match -`zeus.__version__`, and requires a matching changelog section. Its read-only -build job runs `make release-check`, including tests, source-and-branch coverage, -repository contracts, formatting, lint, type checks, Bandit, ShellCheck, package -build, wheel smoke verification, metadata checks, and checksum generation. Only -after that job succeeds does a separate privileged job download the checked -artifacts, verify their checksums, create GitHub artifact attestations, and attach -the assets to the GitHub Release. It intentionally does not publish to PyPI. +`zeus.__version__`, requires a matching changelog section, and calls GitHub's API +to require a GitHub-verified annotated tag and referenced commit. The checker +binds the tag target to the workflow's `GITHUB_SHA`, reads `GITHUB_TOKEN` only +from the environment, rejects redirects and malformed responses, and never logs +the token or raw response bodies. Its read-only build job runs +`make release-check`, including tests, source-and-branch coverage, repository +contracts, formatting, lint, type checks, Bandit, ShellCheck, package build, +wheel smoke verification, metadata checks, and checksum generation. Only after +that job succeeds does a separate privileged job download the checked artifacts, +verify their checksums, create GitHub artifact attestations, and attach the assets +to the GitHub Release. It intentionally does not publish to PyPI. Coverage configuration lives in `.coveragerc`, measures only the `zeus` package, and includes branch coverage. The threshold records the honest current baseline; diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 11aff9a..fc26d45 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,20 +1,37 @@ # Roadmap -## v0.1.x +## Current status -- Stable local CLI -- Template validation -- Real Hermes verification -- API hardening -- Restart policies +Zeus v0.3.0 is alpha software with a local-first, host-local scope. It owns +profiles, processes, lifecycle safety, and reconciliation evidence on one host. -## v0.2.x +## Shipped -- Export/import -- TUI dashboard +- Workspace-local CLI and loopback API. +- Bundled and custom Hermes template validation and rendering. +- PID ownership checks, lifecycle locking, crash recovery, and restart policy. +- SQLite lifecycle and reconciliation evidence. +- Credential-free fake-Hermes demo, hash-locked real-Hermes CI, and manual + real-Hermes verification scripts. +- Wheel builds, installed-wheel smoke checks, and GitHub release artifacts. -## v0.3.x +## Near term -- Plugin registry -- Distributed agent groups -- Remote-safe API mode +- Keep local and CI quality gates aligned with the measured coverage baseline. +- Strengthen installed-package behavior and compatibility evidence. +- Decompose large internal modules without changing the public CLI, API, or + persisted schemas. +- Improve local operational readiness, backup guidance, and health evidence. + +## Under evaluation + +- Workspace-local configuration export and import that never exports secrets. +- A local TUI for lifecycle status and reconciliation history. +- Local plugin discovery with explicit trust and compatibility boundaries. + +## Out of scope + +The out-of-scope responsibilities are cross-host placement, distributed +approvals, fleet rollout policy, and control-plane ownership. They belong to +[Olymp](https://github.com/brainx/olymp), not Zeus. Zeus will keep a narrow +host-local API and durable evidence boundary that Olymp can consume. diff --git a/docs/SYSTEMD.md b/docs/SYSTEMD.md index 4523728..d69b280 100644 --- a/docs/SYSTEMD.md +++ b/docs/SYSTEMD.md @@ -28,6 +28,7 @@ ZEUS_HERMES_BIN=/usr/local/bin/hermes ZEUS_HOST=127.0.0.1 ZEUS_PORT=4311 ZEUS_STATE_DIR=/var/lib/zeus +ZEUS_SQLITE_SYNCHRONOUS=FULL ``` Protect the file: @@ -39,6 +40,20 @@ sudo chmod 0640 /etc/zeus/zeus.env Provider keys such as `DEEPSEEK_API_KEY` can also live in this env file. Do not commit real keys. +Both bundled writer units, `zeus-api.service` and +`zeus-reconcile.service`, select `ZEUS_SQLITE_SYNCHRONOUS=FULL`. Keep the +setting consistent for every process that writes `/var/lib/zeus/zeus.db`, +including manual CLI and doctor commands. After changing the mode, restart both +writer units so newly opened connections receive it: + +```bash +sudo systemctl restart zeus-api zeus-reconcile.service +``` + +FULL applies to SQLite commits only and can add commit latency. It does not make +rendered profiles, PID markers, locks, or audit JSONL writes power-loss atomic; +retain the backup plan in [Operations](OPERATIONS.md). + Leave `ZEUS_ENV_PASSTHROUGH` unset unless Hermes needs selected proxy or certificate variables from the service environment. Profile `.env` files remain the preferred place for provider keys used by a bot. diff --git a/docs/openapi.json b/docs/openapi.json index 6c28129..4ed730f 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -83,6 +83,7 @@ "idempotency_response_too_large", "server_busy", "server_draining", + "not_ready", "internal_error" ] }, @@ -96,6 +97,20 @@ } } }, + "ReadinessResponse": { + "type": "object", + "required": ["schema_version", "status"], + "properties": { + "schema_version": { + "type": "integer", + "const": 6 + }, + "status": { + "type": "string", + "const": "ready" + } + } + }, "Bot": { "type": "object", "required": [ @@ -650,6 +665,83 @@ } } }, + "/ready": { + "get": { + "summary": "Check API state-store readiness", + "description": "Authenticates the request, then opens the Zeus database read-only, requires the current schema version, and executes SELECT 1. The /v1/ready alias is provided by the versioned server URL.", + "security": [ + { + "ZeusApiKey": [] + } + ], + "responses": { + "200": { + "description": "The state store is readable at the current schema version", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReadinessResponse" + } + } + } + }, + "400": { + "description": "Invalid query parameters", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "401": { + "description": "Missing API key, invalid API key, or API key not configured", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "429": { + "description": "Authentication rate limit exceeded", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"}, + "Retry-After": { + "description": "Seconds to wait before retrying authentication.", + "schema": { + "type": "integer", + "minimum": 1 + } + } + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "503": { + "description": "The API key is unconfigured or the state store is not ready", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + } + } + } + }, "/doctor": { "get": { "responses": { diff --git a/pyproject.toml b/pyproject.toml index 672ebc3..d615921 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ dev = [ "build>=1.2.1", "coverage>=7.0.0", "twine>=5.1.1", - "pytest>=8.3.0", ] [project.scripts] diff --git a/requirements-hermes-ci.txt b/requirements-hermes-ci.txt new file mode 100644 index 0000000..44fc467 --- /dev/null +++ b/requirements-hermes-ci.txt @@ -0,0 +1,138 @@ +# Hermes Agent 0.19.0 compatibility lock for CPython 3.11 on Ubuntu x86_64 +# and arm64. +# Resolved from PyPI on 2026-07-22. CI installs this file with --require-hashes +# and --only-binary=:all:. Re-resolve the complete closure when updating Hermes. +annotated-doc==0.0.4 \ + --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 +cffi==2.1.0 \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 +charset-normalizer==3.4.9 \ + --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ + --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 +click==8.4.2 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 +croniter==6.0.0 \ + --hash=sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368 +cryptography==46.0.7 \ + --hash=sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308 \ + --hash=sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325 +distro==1.9.0 \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 +fastapi==0.139.2 \ + --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c +fire==0.7.1 \ + --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 +h11==0.16.0 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +hermes-agent==0.19.0 \ + --hash=sha256:bd0bac012aee38a60894781f4597dc29ee7bedb3448540249921f10d3bef327f +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 +httptools==0.8.0 \ + --hash=sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376 \ + --hash=sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d +httpx==0.28.1 \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +jiter==0.16.0 \ + --hash=sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037 \ + --hash=sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee +markdown==3.10.2 \ + --hash=sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36 +markdown-it-py==4.2.0 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +markupsafe==3.0.3 \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +openai==2.24.0 \ + --hash=sha256:fed30480d7d6c884303287bde864980a4b137b60553ffbcf9ab4a233b7a73d94 +packaging==26.0 \ + --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 +pathspec==1.1.1 \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 +pillow==12.2.0 \ + --hash=sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e \ + --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 +prompt-toolkit==3.0.52 \ + --hash=sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955 +psutil==7.2.2 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e +ptyprocess==0.7.0 \ + --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 +pycparser==3.0 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba +pydantic-core==2.46.4 \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 +pygments==2.20.0 \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 +pyjwt==2.13.0 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 +python-dateutil==2.9.0.post0 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +python-dotenv==1.2.2 \ + --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a +python-multipart==0.0.32 \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 +pytz==2026.2 \ + --hash=sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 +pyyaml==6.0.3 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b +rich==14.3.3 \ + --hash=sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d +ruamel.yaml==0.18.17 \ + --hash=sha256:9c8ba9eb3e793efdf924b60d521820869d5bf0cb9c6f1b82d82de8295e290b9d +ruamel.yaml.clib==0.2.15 \ + --hash=sha256:617d35dc765715fa86f8c3ccdae1e4229055832c452d4ec20856136acc75053f \ + --hash=sha256:ef71831bd61fbdb7aa0399d5c4da06bea37107ab5c79ff884cc07f2450910262 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 +socksio==1.0.0 \ + --hash=sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3 +starlette==1.3.1 \ + --hash=sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6 +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 +termcolor==3.3.0 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 +tqdm==4.69.0 \ + --hash=sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 +urllib3==2.7.0 \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 +uvicorn==0.51.0 \ + --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b +uvloop==0.22.1 \ + --hash=sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21 \ + --hash=sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702 +watchfiles==1.2.0 \ + --hash=sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a \ + --hash=sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e +wcwidth==0.8.2 \ + --hash=sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85 +websockets==15.0.1 \ + --hash=sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413 \ + --hash=sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562 diff --git a/scripts/check_verified_release_ref.py b/scripts/check_verified_release_ref.py new file mode 100644 index 0000000..9fa613e --- /dev/null +++ b/scripts/check_verified_release_ref.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Fail closed unless a release tag and its commit are GitHub-verified.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from collections.abc import Callable, Mapping +from typing import Any, TextIO +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import HTTPRedirectHandler, Request, build_opener + +API_ROOT = "https://api.github.com" +API_VERSION = "2026-03-10" +MAX_RESPONSE_BYTES = 128 * 1024 +REQUEST_TIMEOUT_SECONDS = 15 +REPOSITORY_RE = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9_.-]{0,99}/[A-Za-z0-9][A-Za-z0-9_.-]{0,99}\Z") +TAG_RE = re.compile(r"\Av[0-9][A-Za-z0-9._+-]{0,127}\Z") +SHA_RE = re.compile(r"\A[0-9a-f]{40}\Z") +VERIFIED_AT_RE = re.compile(r"\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\Z") + +JsonObject = dict[str, Any] +FetchJson = Callable[[str, str], JsonObject] + + +class ReleaseVerificationError(RuntimeError): + """A release reference did not satisfy the verification policy.""" + + +class _NoRedirects(HTTPRedirectHandler): + def redirect_request( + self, + req: Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +def _reject_constant(_value: str) -> None: + raise ValueError("non-standard JSON constant") + + +def _strict_object(pairs: list[tuple[str, Any]]) -> JsonObject: + result: JsonObject = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + +def _decode_json(body: bytes) -> JsonObject: + try: + value = json.loads( + body.decode("utf-8"), + object_pairs_hook=_strict_object, + parse_constant=_reject_constant, + ) + except (UnicodeError, ValueError): + raise ReleaseVerificationError("GitHub API returned malformed JSON") from None + if not isinstance(value, dict): + raise ReleaseVerificationError("GitHub API returned an invalid response shape") + return value + + +def _fetch_github_json(path: str, token: str, *, opener: Any = None) -> JsonObject: + request = Request( + f"{API_ROOT}{path}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "User-Agent": "zeus-release-ref-verifier", + "X-GitHub-Api-Version": API_VERSION, + }, + method="GET", + ) + active_opener = opener if opener is not None else build_opener(_NoRedirects()) + try: + with active_opener.open(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + if getattr(response, "status", None) != 200: + raise ReleaseVerificationError("GitHub API returned an unexpected status") + body = response.read(MAX_RESPONSE_BYTES + 1) + except ReleaseVerificationError: + raise + except (HTTPError, URLError, TimeoutError, OSError): + raise ReleaseVerificationError("GitHub API request failed") from None + + if len(body) > MAX_RESPONSE_BYTES: + raise ReleaseVerificationError("GitHub API response exceeded the size limit") + return _decode_json(body) + + +def _required_environment(environ: Mapping[str, str], name: str) -> str: + value = environ.get(name) + if not value: + raise ReleaseVerificationError(f"required workflow environment is missing: {name}") + return value + + +def _object(value: Any, label: str) -> JsonObject: + if not isinstance(value, dict): + raise ReleaseVerificationError(f"GitHub API returned malformed {label} metadata") + return value + + +def _sha(value: Any, label: str) -> str: + if not isinstance(value, str) or SHA_RE.fullmatch(value) is None: + raise ReleaseVerificationError(f"GitHub API returned malformed {label} SHA") + return value + + +def _require_verified(value: Any, label: str) -> None: + verification = _object(value, f"{label} verification") + verified_at = verification.get("verified_at") + if ( + verification.get("verified") is not True + or verification.get("reason") != "valid" + or not isinstance(verified_at, str) + or VERIFIED_AT_RE.fullmatch(verified_at) is None + ): + raise ReleaseVerificationError(f"{label} is not GitHub-verified") + + +def verify_release_ref( + environ: Mapping[str, str], + *, + fetch_json: FetchJson | None = None, +) -> tuple[str, str]: + token = _required_environment(environ, "GITHUB_TOKEN") + if token.strip() != token or any( + ord(character) < 33 or ord(character) > 126 for character in token + ): + raise ReleaseVerificationError("GITHUB_TOKEN is malformed") + + repository = _required_environment(environ, "GITHUB_REPOSITORY") + if REPOSITORY_RE.fullmatch(repository) is None: + raise ReleaseVerificationError("GITHUB_REPOSITORY is malformed") + + if _required_environment(environ, "GITHUB_EVENT_NAME") != "push": + raise ReleaseVerificationError("release verification requires a push event") + if _required_environment(environ, "GITHUB_REF_TYPE") != "tag": + raise ReleaseVerificationError("release verification requires a tag ref") + + tag_name = _required_environment(environ, "GITHUB_REF_NAME") + if TAG_RE.fullmatch(tag_name) is None: + raise ReleaseVerificationError("GITHUB_REF_NAME is not a safe release tag") + full_ref = _required_environment(environ, "GITHUB_REF") + if full_ref != f"refs/tags/{tag_name}": + raise ReleaseVerificationError("GITHUB_REF does not match GITHUB_REF_NAME") + + event_sha = _required_environment(environ, "GITHUB_SHA") + if SHA_RE.fullmatch(event_sha) is None: + raise ReleaseVerificationError("GITHUB_SHA is malformed") + + fetch = fetch_json if fetch_json is not None else _fetch_github_json + encoded_tag = quote(tag_name, safe="") + ref_path = f"/repos/{repository}/git/ref/tags/{encoded_tag}" + ref_data = _object(fetch(ref_path, token), "tag ref") + if ref_data.get("ref") != full_ref: + raise ReleaseVerificationError("GitHub tag ref does not match the workflow ref") + ref_object = _object(ref_data.get("object"), "tag ref") + if ref_object.get("type") != "tag": + raise ReleaseVerificationError("release tag is not annotated") + tag_object_sha = _sha(ref_object.get("sha"), "tag object") + + tag_path = f"/repos/{repository}/git/tags/{tag_object_sha}" + tag_data = _object(fetch(tag_path, token), "tag object") + if tag_data.get("tag") != tag_name or tag_data.get("sha") != tag_object_sha: + raise ReleaseVerificationError("GitHub tag object does not match the workflow ref") + _require_verified(tag_data.get("verification"), "release tag") + tag_target = _object(tag_data.get("object"), "tag target") + if tag_target.get("type") != "commit": + raise ReleaseVerificationError("release tag does not reference a commit") + commit_sha = _sha(tag_target.get("sha"), "tag target") + if commit_sha != event_sha: + raise ReleaseVerificationError("release tag commit does not match GITHUB_SHA") + + commit_path = f"/repos/{repository}/git/commits/{commit_sha}" + commit_data = _object(fetch(commit_path, token), "commit") + if commit_data.get("sha") != commit_sha: + raise ReleaseVerificationError("GitHub commit does not match the release tag") + _require_verified(commit_data.get("verification"), "release commit") + return tag_name, commit_sha + + +def main( + *, + environ: Mapping[str, str] | None = None, + fetch_json: FetchJson | None = None, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, +) -> int: + active_environment = os.environ if environ is None else environ + try: + tag_name, _commit_sha = verify_release_ref( + active_environment, + fetch_json=fetch_json, + ) + except ReleaseVerificationError as error: + print(f"release ref verification failed: {error}", file=stderr) + return 1 + + print( + f"Verified GitHub release ref {tag_name}: annotated tag and commit signatures are valid.", + file=stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fresh_vps_verify.sh b/scripts/fresh_vps_verify.sh index 73e9955..7551aad 100755 --- a/scripts/fresh_vps_verify.sh +++ b/scripts/fresh_vps_verify.sh @@ -2,15 +2,235 @@ # Zeus Hermes Orchestrator # Maintained by BrainX: https://github.com/brainx set -Eeuo pipefail +umask 077 + +fail() { + echo "fresh VPS verification failed: $*" >&2 + exit 1 +} + +safe_relative_dir() { + case "$1" in + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) + fail "$2 must be a workspace-relative scratch directory" + ;; + esac +} repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" +sudo_prefix=() +if [ "${EUID:-$(id -u)}" -ne 0 ]; then + sudo_prefix=(sudo) +fi + +if ! command -v python3 >/dev/null 2>&1; then + if [ "${ZEUS_VPS_INSTALL_PACKAGES:-0}" != "1" ] || ! command -v apt-get >/dev/null 2>&1; then + fail "Python 3 is required; install it first or enable the supported apt package bootstrap" + fi + echo "Bootstrapping the Python 3 prerequisite before private evidence logging." + "${sudo_prefix[@]}" apt-get update + "${sudo_prefix[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y python3 + hash -r + command -v python3 >/dev/null 2>&1 || fail "Python 3 bootstrap completed without a python3 executable" +fi + timestamp="$(date -u '+%Y%m%dT%H%M%SZ')" log_dir="${ZEUS_VPS_LOG_DIR:-.tmp/fresh-vps-verify/$timestamp}" -mkdir -p "$log_dir" +safe_relative_dir "$log_dir" "ZEUS_VPS_LOG_DIR" + +private_evidence() { + python3 -I -B - "$repo_root" "$@" <<'PY' +import os +import stat +import sys + + +def fail(message: str) -> None: + raise ValueError(message) + + +def relative_parts(value: str) -> list[str]: + if not value or os.path.isabs(value): + fail("path must be relative") + parts = value.split("/") + if any(part in {"", ".", ".."} for part in parts): + fail("path contains an unsafe component") + if any("\r" in part or "\n" in part for part in parts): + fail("path contains a line break") + return parts + + +def same_identity(left: os.stat_result, right: os.stat_result) -> bool: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + + +def validate_directory_bindings( + bindings: list[tuple[int, str, int]], + *, + exact_final_mode: bool, +) -> None: + for index, (parent_fd, name, child_fd) in enumerate(bindings): + opened = os.fstat(child_fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if not stat.S_ISDIR(opened.st_mode) or not same_identity(opened, current): + fail("directory binding changed") + if opened.st_uid != os.geteuid(): + fail("directory is not owned by the current user") + if exact_final_mode and index == len(bindings) - 1: + if stat.S_IMODE(opened.st_mode) != 0o700: + fail("evidence directory is not private") + + +def open_directory_chain( + root: str, + parts: list[str], + *, + create: bool, +) -> tuple[list[int], list[tuple[int, str, int]]]: + if not hasattr(os, "O_NOFOLLOW"): + fail("O_NOFOLLOW is required for private evidence") + directory_flags = os.O_RDONLY | os.O_DIRECTORY + directory_flags |= os.O_NOFOLLOW + root_fd = os.open(root, directory_flags) + descriptors = [root_fd] + bindings: list[tuple[int, str, int]] = [] + parent_fd = root_fd + try: + for index, name in enumerate(parts): + created = False + try: + child_fd = os.open(name, directory_flags, dir_fd=parent_fd) + except FileNotFoundError: + if not create: + raise + os.mkdir(name, 0o700, dir_fd=parent_fd) + created = True + child_fd = os.open(name, directory_flags, dir_fd=parent_fd) + descriptors.append(child_fd) + bindings.append((parent_fd, name, child_fd)) + opened = os.fstat(child_fd) + if not stat.S_ISDIR(opened.st_mode) or opened.st_uid != os.geteuid(): + fail("unsafe evidence directory") + if created or index == len(parts) - 1: + validate_directory_bindings(bindings, exact_final_mode=False) + os.fchmod(child_fd, 0o700) + parent_fd = child_fd + validate_directory_bindings(bindings, exact_final_mode=True) + return descriptors, bindings + except BaseException: + for descriptor in reversed(descriptors): + os.close(descriptor) + raise + + +def validate_file(opened: os.stat_result, current: os.stat_result) -> None: + if not stat.S_ISREG(opened.st_mode) or not same_identity(opened, current): + fail("unsafe evidence file binding") + if opened.st_uid != os.geteuid() or opened.st_nlink != 1: + fail("unsafe evidence file ownership") + if stat.S_IMODE(opened.st_mode) != 0o600: + fail("evidence file is not private") + + +def prepare(root: str, relative_directory: str, names: list[str]) -> None: + descriptors, bindings = open_directory_chain( + root, + relative_parts(relative_directory), + create=True, + ) + directory_fd = descriptors[-1] + if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_NONBLOCK"): + fail("O_NOFOLLOW and O_NONBLOCK are required for private evidence") + file_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + file_flags |= os.O_NOFOLLOW | os.O_NONBLOCK + try: + for name in names: + if "/" in name or name in {"", ".", ".."}: + fail("invalid evidence filename") + file_fd = os.open(name, file_flags, 0o600, dir_fd=directory_fd) + try: + opened = os.fstat(file_fd) + current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if not stat.S_ISREG(opened.st_mode) or not same_identity(opened, current): + fail("unsafe evidence file binding") + if opened.st_uid != os.geteuid() or opened.st_nlink != 1: + fail("unsafe evidence file ownership") + os.fchmod(file_fd, 0o600) + validate_file( + os.fstat(file_fd), + os.stat(name, dir_fd=directory_fd, follow_symlinks=False), + ) + validate_directory_bindings(bindings, exact_final_mode=True) + finally: + os.close(file_fd) + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def validate_descriptor( + root: str, + relative_file: str, + descriptor: int, + *, + truncate: bool, +) -> None: + parts = relative_parts(relative_file) + if len(parts) < 2: + fail("evidence file must be inside an evidence directory") + descriptors, bindings = open_directory_chain(root, parts[:-1], create=False) + directory_fd = descriptors[-1] + try: + opened = os.fstat(descriptor) + current = os.stat(parts[-1], dir_fd=directory_fd, follow_symlinks=False) + validate_file(opened, current) + validate_directory_bindings(bindings, exact_final_mode=True) + if truncate: + os.ftruncate(descriptor, 0) + validate_file( + os.fstat(descriptor), + os.stat(parts[-1], dir_fd=directory_fd, follow_symlinks=False), + ) + validate_directory_bindings(bindings, exact_final_mode=True) + finally: + for opened_fd in reversed(descriptors): + os.close(opened_fd) + + +try: + repository_root = sys.argv[1] + action = sys.argv[2] + relative_path = sys.argv[3] + if action == "prepare": + prepare(repository_root, relative_path, sys.argv[4:]) + elif action == "validate-file": + validate_descriptor( + repository_root, + relative_path, + int(sys.argv[4]), + truncate=sys.argv[5] == "truncate", + ) + else: + fail("unknown private evidence operation") +except (OSError, ValueError) as error: + print(f"private evidence validation failed: {error}", file=sys.stderr) + raise SystemExit(1) from None +PY +} + +if ! private_evidence prepare "$log_dir" run.log zeus-api.log; then + fail "could not create a private evidence directory" +fi log_file="$log_dir/run.log" -exec > >(tee -a "$log_file") 2>&1 +run_log_fd=9 +exec 9<>"$log_file" +if ! private_evidence validate-file "$log_file" "$run_log_fd" truncate; then + exec 9>&- + fail "could not open the private evidence transcript" +fi +exec > >(tee -a "/dev/fd/$run_log_fd") 2>&1 api_pid="" @@ -25,9 +245,20 @@ run() { "$@" } -fail() { - echo "fresh VPS verification failed: $*" >&2 - exit 1 +run_stdout_to_fd() { + local output_fd="$1" + shift + printf '+' + printf ' %q' "$@" + printf '\n' + "$@" 1>&"$output_fd" +} + +run_sensitive() { + local label="$1" + shift + printf '+ %s\n' "$label" + "$@" } cleanup() { @@ -38,19 +269,6 @@ cleanup() { } trap cleanup EXIT INT TERM -safe_relative_dir() { - case "$1" in - "" | "/" | "." | ".." | /* | ../* | */../*) - fail "$2 must be a workspace-relative scratch directory" - ;; - esac -} - -sudo_prefix=() -if [ "${EUID:-$(id -u)}" -ne 0 ]; then - sudo_prefix=(sudo) -fi - section "Repository" test -f pyproject.toml || fail "run this script from a Zeus checkout" test -f zeus/cli.py || fail "missing zeus/cli.py" @@ -93,9 +311,52 @@ hash -r if command -v hermes >/dev/null 2>&1; then run command -v hermes elif [ "${ZEUS_VPS_INSTALL_HERMES:-0}" = "1" ]; then + installer_sha256="${ZEUS_VPS_HERMES_INSTALLER_SHA256:-}" + if [[ ! "$installer_sha256" =~ ^[0-9A-Fa-f]{64}$ ]]; then + fail "ZEUS_VPS_HERMES_INSTALLER_SHA256 must be exactly 64 hexadecimal characters" + fi + if ! private_evidence prepare "$log_dir" hermes-install.sh; then + fail "could not create a private Hermes installer evidence file" + fi installer="$log_dir/hermes-install.sh" - run curl -fsSL https://hermes-agent.nousresearch.com/install.sh -o "$installer" - run bash "$installer" + installer_fd=8 + exec 8<>"$installer" + if ! private_evidence validate-file "$installer" "$installer_fd" truncate; then + exec 8>&- + fail "could not open the private Hermes installer evidence file" + fi + run_stdout_to_fd "$installer_fd" curl -fsSL https://hermes-agent.nousresearch.com/install.sh + if ! private_evidence validate-file "$installer" "$installer_fd" preserve; then + exec 8>&- + fail "Hermes installer evidence changed during download" + fi + if ! python3 -I -B - "$installer_fd" "$installer_sha256" <<'PY' +import hashlib +import hmac +import os +import sys + +descriptor = int(sys.argv[1]) +expected = sys.argv[2].lower() +digest = hashlib.sha256() +offset = 0 +while True: + chunk = os.pread(descriptor, 1024 * 1024, offset) + if not chunk: + break + digest.update(chunk) + offset += len(chunk) +matches = hmac.compare_digest(digest.hexdigest(), expected) +if matches: + os.lseek(descriptor, 0, os.SEEK_SET) +raise SystemExit(0 if matches else 1) +PY + then + exec 8>&- + fail "Hermes installer SHA-256 mismatch; refusing to execute it" + fi + run bash "/dev/fd/$installer_fd" + exec 8>&- export PATH="$HOME/.local/bin:$HOME/.hermes/bin:$PATH" hash -r run command -v hermes @@ -114,7 +375,6 @@ run hermes doctor section "Real Hermes Compatibility" run env \ ZEUS_VERIFY_START_GATEWAY="${ZEUS_VPS_START_GATEWAY:-0}" \ - ZEUS_VERIFY_GATEWAY_SECONDS="${ZEUS_VPS_GATEWAY_SECONDS:-5}" \ sh scripts/verify_real_hermes.sh section "Multi Template Profile Check" @@ -133,7 +393,9 @@ done if [ -n "${ZEUS_VPS_ASYNC_PROMPT:-}" ]; then section "Optional Async Delegation Prompt" - run env HERMES_HOME="$repo_root/$multi_state/hermes" hermes -p vps-coder -z "$ZEUS_VPS_ASYNC_PROMPT" + run_sensitive "Hermes async prompt [redacted]" env \ + HERMES_HOME="$repo_root/$multi_state/hermes" \ + hermes -p vps-coder -z "$ZEUS_VPS_ASYNC_PROMPT" else echo "Skipping live async prompt. Set ZEUS_VPS_ASYNC_PROMPT to exercise delegate_task/background behavior with real credentials." fi @@ -143,9 +405,31 @@ api_state="${ZEUS_VPS_API_STATE_DIR:-.zeus-vps-api}" safe_relative_dir "$api_state" "ZEUS_VPS_API_STATE_DIR" rm -rf -- "$api_state" api_port="${ZEUS_VPS_API_PORT:-4311}" -api_key="${ZEUS_VPS_API_KEY:-vps-local-check}" +api_key="${ZEUS_VPS_API_KEY:-$(python -I -B -c 'import secrets; print(secrets.token_hex(32))')}" +if [[ "$api_key" == *$'\r'* || "$api_key" == *$'\n'* ]]; then + fail "ZEUS_VPS_API_KEY must not contain a line break" +fi + +write_api_curl_config() { + local include_json_header="$1" + local escaped_key + escaped_key=${api_key//\\/\\\\} + escaped_key=${escaped_key//\"/\\\"} + printf 'header = "x-zeus-api-key: %s"\n' "$escaped_key" + if [ "$include_json_header" = "1" ]; then + printf 'header = "content-type: application/json"\n' + fi +} + +api_log_file="$log_dir/zeus-api.log" +api_log_fd=7 +exec 7<>"$api_log_file" +if ! private_evidence validate-file "$api_log_file" "$api_log_fd" truncate; then + exec 7>&- + fail "could not open the private API evidence log" +fi ZEUS_STATE_DIR="$api_state" ZEUS_API_KEY="$api_key" \ - python -B -m zeus.api --host 127.0.0.1 --port "$api_port" >"$log_dir/zeus-api.log" 2>&1 & + python -B -m zeus.api --host 127.0.0.1 --port "$api_port" 1>&"$api_log_fd" 2>&1 & api_pid="$!" for _ in $(seq 1 40); do @@ -156,17 +440,18 @@ for _ in $(seq 1 40); do done run curl -fsS "http://127.0.0.1:$api_port/health" -run curl -fsS -H "x-zeus-api-key: $api_key" "http://127.0.0.1:$api_port/doctor" -run curl -fsS \ - -H "x-zeus-api-key: $api_key" \ - -H "content-type: application/json" \ +write_api_curl_config 0 | run_sensitive "authenticated API request [redacted]" \ + curl -fsS --config - "http://127.0.0.1:$api_port/doctor" +write_api_curl_config 1 | run_sensitive "authenticated API request [redacted]" \ + curl -fsS --config - \ --data '{"bot_id":"api-vps","template_id":"support-gateway"}' \ "http://127.0.0.1:$api_port/bots" cleanup api_pid="" +exec 7>&- section "Summary" echo "Fresh VPS verification completed." echo "Evidence log: $log_file" -echo "API log: $log_dir/zeus-api.log" +echo "API log: $api_log_file" diff --git a/scripts/repo_check.sh b/scripts/repo_check.sh index f910bad..2224d4a 100755 --- a/scripts/repo_check.sh +++ b/scripts/repo_check.sh @@ -46,6 +46,7 @@ systemd/zeus-reconcile.service systemd/zeus-reconcile.timer scripts/test.sh scripts/check_version_tag.py +scripts/check_verified_release_ref.py scripts/wheel_smoke.sh scripts/generate_checksums.sh scripts/verify_real_hermes.sh diff --git a/scripts/test.sh b/scripts/test.sh index 6a42de8..4656e27 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,13 +4,24 @@ set -eu tmp_dir=".tmp/test" +warning_log="$tmp_dir/unittest-stderr.log" cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT INT TERM -python3 -B -m compileall zeus tests -python3 -B -m unittest discover -s tests -v mkdir -p "$tmp_dir" +python3 -B -m compileall zeus tests +unittest_status=0 +python3 -B -W error::ResourceWarning -m unittest discover -s tests -v \ + 2>"$warning_log" || unittest_status=$? +cat "$warning_log" >&2 +if grep -F "ResourceWarning" "$warning_log" >/dev/null; then + echo "ResourceWarning detected in the test suite." >&2 + exit 1 +fi +if [ "$unittest_status" -ne 0 ]; then + exit "$unittest_status" +fi python3 -B -m zeus.cli doctor --json >"$tmp_dir/zeus-doctor.json" python3 -B -m zeus.cli template list >"$tmp_dir/zeus-templates.txt" diff --git a/scripts/verify_real_hermes.sh b/scripts/verify_real_hermes.sh index b89ae6d..8ae4e1e 100755 --- a/scripts/verify_real_hermes.sh +++ b/scripts/verify_real_hermes.sh @@ -6,18 +6,40 @@ set -eu bot_id="${ZEUS_VERIFY_BOT_ID:-real-hermes-check}" template_id="${ZEUS_VERIFY_TEMPLATE:-coding-bot}" state_dir="${ZEUS_VERIFY_STATE_DIR:-.zeus-real-hermes-check}" +evidence_dir="${ZEUS_VERIFY_EVIDENCE_DIR:-.tmp/real-hermes-evidence}" +expected_hermes_version="${ZEUS_VERIFY_EXPECTED_HERMES_VERSION:-}" api_server_host="${ZEUS_VERIFY_API_SERVER_HOST:-127.0.0.1}" api_server_port="${ZEUS_VERIFY_API_SERVER_PORT:-4312}" health_timeout_seconds="${ZEUS_VERIFY_HEALTH_TIMEOUT_SECONDS:-30}" health_interval_seconds="${ZEUS_VERIFY_HEALTH_INTERVAL_SECONDS:-0.5}" case "$state_dir" in - "" | "/" | "." | ".." | /* | ../* | */../*) + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) echo "unsafe ZEUS_VERIFY_STATE_DIR: use a workspace-relative scratch directory" >&2 exit 2 ;; esac +case "$evidence_dir" in + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) + echo "unsafe ZEUS_VERIFY_EVIDENCE_DIR: use a workspace-relative scratch directory" >&2 + exit 2 + ;; +esac + +case "$state_dir/" in + "$evidence_dir/"*) + echo "unsafe verification directories: state and evidence paths must not overlap" >&2 + exit 2 + ;; +esac +case "$evidence_dir/" in + "$state_dir/"*) + echo "unsafe verification directories: state and evidence paths must not overlap" >&2 + exit 2 + ;; +esac + case "$api_server_host" in 127.0.0.1 | localhost) ;; *) @@ -43,21 +65,55 @@ if ! command -v hermes >/dev/null 2>&1; then exit 2 fi +failure_stage="environment" cleanup() { + status=$? + trap - EXIT INT TERM if [ -d "$state_dir" ]; then ZEUS_STATE_DIR="$state_dir" \ ZEUS_HERMES_BIN="$(command -v hermes 2>/dev/null || printf '%s' hermes)" \ python3 -B -m zeus.cli bot stop "$bot_id" >/dev/null 2>&1 || true fi + rm -rf -- "$state_dir" + if [ "$status" -eq 0 ]; then + rm -rf -- "$evidence_dir" + else + rm -rf -- "$evidence_dir" + mkdir -p "$evidence_dir" + chmod 0700 "$evidence_dir" + ( + umask 077 + printf 'result=failed\nfailure_stage=%s\n' "$failure_stage" \ + >"$evidence_dir/summary.txt" + ) + fi + exit "$status" } -trap cleanup EXIT INT TERM - -rm -rf -- "$state_dir" +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +rm -rf -- "$state_dir" "$evidence_dir" + +failure_stage="hermes_version" +if [ -n "$expected_hermes_version" ]; then + actual_hermes_version="$( + python3 -c 'from importlib.metadata import version; print(version("hermes-agent"))' + )" + if [ "$actual_hermes_version" != "$expected_hermes_version" ]; then + echo "Hermes version mismatch: expected $expected_hermes_version, got $actual_hermes_version" >&2 + exit 1 + fi +fi +failure_stage="zeus_doctor" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" ZEUS_API_KEY="${ZEUS_VERIFY_API_KEY:-real-hermes-local-check}" python3 -B -m zeus.cli doctor --strict --json +failure_stage="profile_create" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" python3 -B -m zeus.cli bot create "$bot_id" --template "$template_id" +failure_stage="bot_doctor" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" python3 -B -m zeus.cli bot doctor "$bot_id" +failure_stage="rendered_profile" config_path="$state_dir/hermes/profiles/$bot_id/config.yaml" test -f "$config_path" grep -q "max_async_children" "$config_path" @@ -72,6 +128,7 @@ if [ "${ZEUS_VERIFY_START_GATEWAY:-0}" = "1" ]; then api_server_passthrough="$ZEUS_ENV_PASSTHROUGH,$api_server_passthrough" fi + failure_stage="gateway_start" API_SERVER_ENABLED=1 \ API_SERVER_HOST="$api_server_host" \ API_SERVER_PORT="$api_server_port" \ @@ -79,11 +136,12 @@ if [ "${ZEUS_VERIFY_START_GATEWAY:-0}" = "1" ]; then ZEUS_ENV_PASSTHROUGH="$api_server_passthrough" \ ZEUS_STATE_DIR="$state_dir" \ ZEUS_HERMES_BIN="$(command -v hermes)" \ - python3 -B -m zeus.cli bot start "$bot_id" - sleep "${ZEUS_VERIFY_GATEWAY_SECONDS:-3}" + python3 -B -m zeus.cli bot start "$bot_id" --wait --timeout "$health_timeout_seconds" + failure_stage="gateway_status" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot status "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "running"' + failure_stage="process_ownership" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot inspect "$bot_id" --json \ | python3 -c ' @@ -100,6 +158,7 @@ assert ownership["classification"] in { "legacy-marker-valid", }, ownership ' + failure_stage="loopback_health" python3 - "$api_server_host" "$api_server_port" "$health_timeout_seconds" "$health_interval_seconds" <<'PY' import json import sys @@ -136,9 +195,11 @@ while True: ) time.sleep(interval_seconds) PY + failure_stage="gateway_stop" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot stop "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "stopped"' + failure_stage="stopped_status" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot status "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "stopped"' @@ -146,4 +207,5 @@ else echo "Skipping gateway start. Set ZEUS_VERIFY_START_GATEWAY=1 to exercise hermes gateway run." fi +failure_stage="complete" echo "Real Hermes verification completed for $bot_id using $state_dir" diff --git a/scripts/wheel_smoke.sh b/scripts/wheel_smoke.sh index 26c05a5..08b4cad 100755 --- a/scripts/wheel_smoke.sh +++ b/scripts/wheel_smoke.sh @@ -3,14 +3,22 @@ # Maintained by BrainX: https://github.com/brainx set -eu -repo_root="$(pwd)" +repo_root="$(pwd -P)" tmp_dir="$repo_root/.tmp/wheel-smoke" build_artifacts="${ZEUS_WHEEL_SMOKE_BUILD:-1}" +venv_python="" +venv_zeus="" +venv_fake_hermes="" +state_dir="$tmp_dir/state" +demo_started=0 fail() { echo "wheel smoke failed: $*" >&2 exit 1 } cleanup() { + if [ "$demo_started" = "1" ] && [ -n "$venv_zeus" ] && [ -x "$venv_zeus" ]; then + ZEUS_STATE_DIR="$state_dir" "$venv_zeus" demo down --json >/dev/null 2>&1 || true + fi rm -rf "$tmp_dir" } trap cleanup EXIT INT TERM @@ -39,20 +47,63 @@ wheel_path="$1" "$python_cmd" -m venv "$tmp_dir/venv" venv_python="$tmp_dir/venv/bin/python" venv_zeus="$tmp_dir/venv/bin/zeus" -"$venv_python" -m pip install "$wheel_path" +venv_fake_hermes="$tmp_dir/venv/bin/zeus-fake-hermes" +PIP_NO_INDEX=1 "$venv_python" -m pip install --no-deps "$wheel_path" cd "$tmp_dir" +unset PYTHONPATH PYTHONHOME +export PYTHONNOUSERSITE=1 +export PATH="$tmp_dir/venv/bin:$PATH" +export ZEUS_STATE_DIR="$state_dir" + +installed_fake_hermes="$(command -v zeus-fake-hermes || true)" +[ "$installed_fake_hermes" = "$venv_fake_hermes" ] || + fail "PATH did not select the installed zeus-fake-hermes entry point" + +[ ! -e "$state_dir" ] || fail "state directory exists before stateless CLI checks" +"$venv_zeus" --help >zeus-help.txt +grep -F "usage: zeus" zeus-help.txt >/dev/null +metadata_version=$("$venv_python" -c \ + 'from importlib.metadata import version; print(version("zeus-hermes-orchestrator"))') +module_version=$("$venv_python" -c 'import zeus; print(zeus.__version__)') +cli_version=$("$venv_zeus" --version) +module_path=$("$venv_python" -c \ + 'from pathlib import Path; import zeus; print(Path(zeus.__file__).resolve())') +[ "$metadata_version" = "$module_version" ] || + fail "distribution and module versions differ: $metadata_version != $module_version" +[ "$cli_version" = "zeus $metadata_version" ] || + fail "CLI version does not match package metadata: $cli_version" +case "$module_path" in + "$tmp_dir"/venv/*) ;; + *) fail "zeus imported outside the isolated virtual environment: $module_path" ;; +esac +[ ! -e "$state_dir" ] || fail "help or version checks created runtime state" + +"$venv_python" -m pip check "$venv_zeus" template list >template-list.txt "$venv_zeus" doctor --json >doctor.json -"$venv_python" -c "import zeus; print(zeus.__version__)" -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo up --json >demo-up.json -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo status --json >demo-status.json -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo down --json >demo-down.json +for template_id in \ + coding-bot \ + deepseek-coding-bot \ + docs-writer-bot \ + gateway-operator \ + log-triage-bot \ + research-bot \ + support-gateway; do + grep "^${template_id}[[:space:]]" template-list.txt >/dev/null || + fail "installed wheel is missing bundled template: $template_id" +done -grep "coding-bot" template-list.txt >/dev/null -grep "deepseek-coding-bot" template-list.txt >/dev/null grep '"checks"' doctor.json >/dev/null +demo_started=1 +"$venv_zeus" demo up --json >demo-up.json +"$venv_zeus" demo status --json >demo-status.json +"$venv_zeus" demo down --json >demo-down.json +demo_started=0 + grep '"fake_hermes_bin"' demo-up.json >/dev/null +grep -F "\"fake_hermes_bin\": \"$venv_fake_hermes\"" demo-up.json >/dev/null +grep '"status": "running"' demo-up.json >/dev/null grep '"status": "running"' demo-status.json >/dev/null grep '"status": "stopped"' demo-down.json >/dev/null diff --git a/systemd/zeus-api.service b/systemd/zeus-api.service index 76c0a0a..0b96dde 100644 --- a/systemd/zeus-api.service +++ b/systemd/zeus-api.service @@ -15,6 +15,7 @@ EnvironmentFile=/etc/zeus/zeus.env Environment=ZEUS_HOST=127.0.0.1 Environment=ZEUS_PORT=4311 Environment=ZEUS_STATE_DIR=/var/lib/zeus +Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL ExecStart=/opt/zeus/.venv/bin/python -m zeus.api Restart=on-failure RestartSec=5s diff --git a/systemd/zeus-reconcile.service b/systemd/zeus-reconcile.service index 1ba1d6a..a9a8a53 100644 --- a/systemd/zeus-reconcile.service +++ b/systemd/zeus-reconcile.service @@ -9,6 +9,7 @@ User=zeus Group=zeus WorkingDirectory=/opt/zeus EnvironmentFile=/etc/zeus/zeus.env +Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL ExecStart=/opt/zeus/.venv/bin/zeus bot reconcile NoNewPrivileges=true PrivateTmp=true diff --git a/templates/coding-bot.toml b/templates/coding-bot.toml index a7777ca..f4cdc25 100644 --- a/templates/coding-bot.toml +++ b/templates/coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/deepseek-coding-bot.toml b/templates/deepseek-coding-bot.toml index c4fb3e7..6e66599 100644 --- a/templates/deepseek-coding-bot.toml +++ b/templates/deepseek-coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/docs-writer-bot.toml b/templates/docs-writer-bot.toml index d1a1e64..72199ff 100644 --- a/templates/docs-writer-bot.toml +++ b/templates/docs-writer-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/gateway-operator.toml b/templates/gateway-operator.toml index c9da28c..3256701 100644 --- a/templates/gateway-operator.toml +++ b/templates/gateway-operator.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/log-triage-bot.toml b/templates/log-triage-bot.toml index c8e5ab9..b97a2ee 100644 --- a/templates/log-triage-bot.toml +++ b/templates/log-triage-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/research-bot.toml b/templates/research-bot.toml index 0a41f2c..f5fd247 100644 --- a/templates/research-bot.toml +++ b/templates/research-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/support-gateway.toml b/templates/support-gateway.toml index 91b618f..52a682a 100644 --- a/templates/support-gateway.toml +++ b/templates/support-gateway.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 180 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/tests/test_api.py b/tests/test_api.py index 45132a1..a0b0a85 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import http.client +import io import json import os import socket @@ -12,17 +13,18 @@ import time import unittest from collections.abc import Iterator -from contextlib import closing, contextmanager +from contextlib import closing, contextmanager, redirect_stderr from datetime import UTC, datetime -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from http.server import ThreadingHTTPServer from pathlib import Path from typing import Any from unittest.mock import patch from zeus import api as api_module from zeus.api import make_handler, serve -from zeus.config import Settings -from zeus.models import BotRecord, BotStatus, BotStatusResponse +from zeus.config import Settings, SQLiteSynchronous +from zeus.errors import ZeusConflictError +from zeus.models import BotRecord, BotStatus, BotStatusResponse, TemplateError from zeus.process_lock import LockTimeoutError from zeus.rate_limit import TokenBucket from zeus.reconciliation import ( @@ -31,7 +33,7 @@ ReconcileOutcome, ReconcileRunSummary, ) -from zeus.state import StateStore +from zeus.state import SCHEMA_VERSION, StateReadinessError, StateStore JsonPayload = dict[str, Any] | list[Any] @@ -152,6 +154,28 @@ def raw_http_response( conn.close() +def raw_http_exchange( + port: int, + method: str, + path: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> tuple[int, dict[str, str], bytes]: + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + response_body = response.read() + return ( + response.status, + {name.lower(): value for name, value in response.getheaders()}, + response_body, + ) + finally: + conn.close() + + def json_request_body(payload: JsonPayload) -> bytes: return json.dumps(payload).encode("utf-8") @@ -235,6 +259,287 @@ def raw_post_without_content_length(port: int, body: bytes) -> tuple[int, dict[s class ApiBehaviorTests(unittest.TestCase): + def test_health_exact_bytes_are_public_and_never_probe_readiness(self) -> None: + with ( + patch.object( + StateStore, + "check_readiness", + side_effect=AssertionError("health must not probe state"), + ) as readiness, + api_server_with_state() as (port, state_dir), + ): + exchanges = [raw_http_exchange(port, "GET", path) for path in ("/health", "/v1/health")] + rows = wait_for_access_rows(state_dir, 2) + + for status, headers, body in exchanges: + self.assertEqual(200, status) + self.assertEqual(b'{"status": "ok"}', body) + self.assertEqual("application/json", headers["content-type"]) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + readiness.assert_not_called() + self.assertEqual(["/health", "/health"], [row["route"] for row in rows]) + self.assertEqual(["not_required", "not_required"], [row["auth_outcome"] for row in rows]) + + def test_readiness_auth_query_alias_and_override_contract(self) -> None: + with patch.object(StateStore, "check_readiness", return_value=SCHEMA_VERSION) as readiness: + with api_server() as port: + missing_config_status, _ = request_json(port, "GET", "/ready") + self.assertEqual(503, missing_config_status) + readiness.assert_not_called() + + configured = { + "ZEUS_API_KEY": "secret", + "ZEUS_API_AUTH_FAILURE_RATE_PER_MINUTE": "1", + "ZEUS_API_AUTH_FAILURE_BURST": "2", + } + with api_server(configured) as port: + missing_status, _ = request_json(port, "GET", "/ready") + wrong_status, _ = request_json( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "wrong"}, + ) + limited_status, limited_headers, limited_body = request_json_with_headers( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "wrong"}, + ) + query_status, query_body = request_json( + port, + "GET", + "/ready?debug=1", + headers={"x-zeus-api-key": "secret"}, + ) + ready_status, ready_headers, ready_body = raw_http_exchange( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "secret"}, + ) + alias_status, _alias_headers, alias_body = raw_http_exchange( + port, + "GET", + "/v1/ready", + headers={"x-zeus-api-key": "secret"}, + ) + + self.assertEqual((401, 401, 429), (missing_status, wrong_status, limited_status)) + self.assertEqual("auth_rate_limited", limited_body["error"]["code"]) + self.assertGreaterEqual(int(limited_headers["retry-after"]), 1) + self.assertEqual(400, query_status) + self.assertEqual("invalid_request", query_body["error"]["code"]) + expected = b'{"schema_version": 6, "status": "ready"}' + self.assertEqual((200, expected), (ready_status, ready_body)) + self.assertEqual((200, expected), (alias_status, alias_body)) + self.assertRegex(ready_headers["x-request-id"], r"^[0-9a-f]{32}$") + self.assertEqual(2, readiness.call_count) + + with api_server({"ZEUS_ALLOW_UNAUTH_READS": "1"}) as port: + override_status, override_body = request_json(port, "GET", "/ready") + self.assertEqual(200, override_status) + self.assertEqual({"schema_version": SCHEMA_VERSION, "status": "ready"}, override_body) + self.assertEqual(3, readiness.call_count) + + def test_readiness_expected_failure_is_sanitized_and_not_logged_as_exception(self) -> None: + with ( + patch.object( + StateStore, + "check_readiness", + side_effect=StateReadinessError("private sqlite detail"), + ), + api_server_with_state({"ZEUS_API_KEY": "secret"}) as (port, state_dir), + ): + status, headers, body = raw_http_exchange( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "secret"}, + ) + rows = wait_for_access_rows(state_dir, 1) + + self.assertEqual(503, status) + self.assertEqual( + b'{"error": {"code": "not_ready", ' + b'"message": "state store is not ready", "status": 503}}', + body, + ) + self.assertNotIn("retry-after", headers) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + self.assertEqual(["api.access"], [row["event"] for row in rows]) + self.assertEqual("/ready", rows[0]["route"]) + self.assertEqual("not_ready", rows[0]["error_code"]) + self.assertNotIn("private sqlite detail", json.dumps(rows)) + + def test_readiness_aliases_probe_the_real_current_store(self) -> None: + with api_server({"ZEUS_API_KEY": "secret"}) as port: + responses = [ + raw_http_exchange( + port, + "GET", + path, + headers={"x-zeus-api-key": "secret"}, + ) + for path in ("/ready", "/v1/ready") + ] + + for status, headers, body in responses: + self.assertEqual(200, status) + self.assertEqual(b'{"schema_version": 6, "status": "ready"}', body) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + + def test_exception_status_payload_and_header_contracts(self) -> None: + cases = ( + ( + "unknown_bot", + lambda: KeyError("unknown bot: coder"), + 404, + "unknown_bot", + "unknown bot: coder", + ), + ( + "unknown_template", + lambda: KeyError("unknown template: missing"), + 400, + "unknown_template", + "unknown template: missing", + ), + ( + "missing_field", + lambda: KeyError("bot_id"), + 400, + "invalid_request", + "missing required field: bot_id", + ), + ( + "invalid_bot_id", + lambda: TemplateError("bot_id must match ^[a-z][a-z0-9-]{1,62}$"), + 400, + "invalid_bot_id", + "bot_id must match ^[a-z][a-z0-9-]{1,62}$", + ), + ( + "template_error", + lambda: TemplateError("template contract failed"), + 400, + "invalid_request", + "template contract failed", + ), + ( + "reconcile_lock", + lambda: ReconcileLockTimeoutError(Path("/tmp/reconcile.lock"), 0.1), + 409, + "reconcile_locked", + "reconciliation is already in progress", + ), + ( + "bot_lock", + lambda: LockTimeoutError(Path("/tmp/coder.lock"), 0.1), + 409, + "bot_locked", + "bot lifecycle operation is already in progress", + ), + ( + "conflict", + lambda: ZeusConflictError("conflicting lifecycle state"), + 409, + "conflict", + "conflicting lifecycle state", + ), + ( + "value_error", + lambda: ValueError("invalid lifecycle request"), + 400, + "invalid_request", + "invalid lifecycle request", + ), + ( + "internal_error", + lambda: RuntimeError("private failure detail"), + 500, + "internal_error", + "internal server error", + ), + ) + + class RaisingSupervisor: + failure: Exception + + def __init__(self, *args: object, **kwargs: object) -> None: + return + + def status(self, *args: object, **kwargs: object) -> BotStatusResponse: + raise self.failure + + def start(self, *args: object, **kwargs: object) -> BotStatusResponse: + raise self.failure + + with ( + patch("zeus.api.Supervisor", RaisingSupervisor), + api_server({"ZEUS_API_KEY": "secret"}) as port, + ): + for name, make_error, expected_status, code, message in cases: + expected = { + "error": { + "code": code, + "message": message, + "status": expected_status, + } + } + expected_body = json.dumps(expected, sort_keys=True).encode("utf-8") + requests = ( + ("GET", "/bots/coder/status", {}), + ( + "POST", + "/bots/coder/start", + {"idempotency-key": f"characterization-{name}"}, + ), + ) + for method, path, extra_headers in requests: + with self.subTest(name=name, method=method): + RaisingSupervisor.failure = make_error() + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request( + method, + path, + headers={ + "x-zeus-api-key": "secret", + **extra_headers, + }, + ) + response = conn.getresponse() + body = response.read() + headers = {key.lower(): value for key, value in response.getheaders()} + finally: + conn.close() + + self.assertEqual(expected_status, response.status) + self.assertEqual(expected_body, body) + self.assertEqual( + { + "cache-control": "no-store", + "content-length": str(len(expected_body)), + "content-type": "application/json", + "cross-origin-resource-policy": "same-origin", + "referrer-policy": "no-referrer", + "x-content-type-options": "nosniff", + }, + { + key: headers[key] + for key in ( + "cache-control", + "content-length", + "content-type", + "cross-origin-resource-policy", + "referrer-policy", + "x-content-type-options", + ) + }, + ) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + def test_all_json_responses_include_generated_request_id(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: for method, path in (("GET", "/health"), ("GET", "/bots"), ("PUT", "/bots")): @@ -287,190 +592,6 @@ def test_api_resource_limits_are_configurable_and_bounded(self) -> None: with self.subTest(env=env), self.assertRaisesRegex(ValueError, message): Settings.from_env(env) - def test_http_server_rejects_requests_above_concurrency_limit(self) -> None: - first_entered = threading.Event() - release_first = threading.Event() - call_lock = threading.Lock() - call_count = 0 - - class LimitedHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 1 - api_request_timeout_seconds = 2.0 - - def do_GET(self) -> None: - nonlocal call_count - with call_lock: - call_count += 1 - current_call = call_count - if current_call == 1: - first_entered.set() - release_first.wait(timeout=3) - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - server = api_module.ThreadingHTTPServer(("127.0.0.1", 0), LimitedHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - first_result: list[tuple[int, dict[str, Any]]] = [] - first_thread = threading.Thread( - target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) - ) - server_thread.start() - first_thread.start() - try: - self.assertTrue(first_entered.wait(timeout=2)) - conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) - try: - conn.request("GET", "/health") - response = conn.getresponse() - raw_body = response.read() - finally: - conn.close() - - self.assertEqual(503, response.status) - self.assertEqual("1", response.getheader("retry-after")) - self.assertRegex( - response.getheader("x-request-id") or "", - r"^[0-9a-f]{32}$", - ) - body = json.loads(raw_body.decode("utf-8")) - self.assertEqual("server_busy", body["error"]["code"]) - finally: - release_first.set() - first_thread.join(timeout=3) - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - - self.assertFalse(first_thread.is_alive()) - self.assertEqual(200, first_result[0][0]) - - def test_http_server_drains_active_requests_and_rejects_new_work(self) -> None: - first_entered = threading.Event() - release_first = threading.Event() - - class DrainHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 2 - api_request_timeout_seconds = 2.0 - - def do_GET(self) -> None: - first_entered.set() - release_first.wait(timeout=3) - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - server = api_module.ThreadingHTTPServer(("127.0.0.1", 0), DrainHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - first_result: list[tuple[int, dict[str, Any]]] = [] - first_thread = threading.Thread( - target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) - ) - server_thread.start() - first_thread.start() - try: - self.assertTrue(first_entered.wait(timeout=2)) - request_graceful_shutdown = getattr(server, "request_graceful_shutdown", None) - wait_until_draining = getattr(server, "wait_until_draining", None) - wait_for_drain = getattr(server, "wait_for_drain", None) - self.assertTrue(callable(request_graceful_shutdown)) - self.assertTrue(callable(wait_until_draining)) - self.assertTrue(callable(wait_for_drain)) - request_graceful_shutdown(1.0) - self.assertTrue(wait_until_draining(1.0)) - - conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) - try: - conn.request("GET", "/health") - response = conn.getresponse() - raw_body = response.read() - finally: - conn.close() - - self.assertEqual(503, response.status) - self.assertEqual("1", response.getheader("retry-after")) - self.assertRegex( - response.getheader("x-request-id") or "", - r"^[0-9a-f]{32}$", - ) - body = json.loads(raw_body.decode("utf-8")) - self.assertEqual("server_draining", body["error"]["code"]) - self.assertFalse(wait_for_drain(0.05)) - - release_first.set() - first_thread.join(timeout=3) - self.assertTrue(wait_for_drain(1.0)) - server_thread.join(timeout=3) - finally: - release_first.set() - first_thread.join(timeout=3) - if server_thread.is_alive(): - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - - self.assertFalse(first_thread.is_alive()) - self.assertFalse(server_thread.is_alive()) - self.assertEqual(200, first_result[0][0]) - - def test_http_server_times_out_incomplete_requests_and_releases_capacity(self) -> None: - accepted = threading.Event() - - class TimeoutHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 1 - api_request_timeout_seconds = 0.1 - - def do_GET(self) -> None: - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - class TrackingServer(api_module.ThreadingHTTPServer): - def process_request(self, request: Any, client_address: Any) -> None: - accepted.set() - super().process_request(request, client_address) - - server = TrackingServer(("127.0.0.1", 0), TimeoutHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - server_thread.start() - slow_client = socket.create_connection(("127.0.0.1", server.server_port), timeout=2) - try: - slow_client.sendall(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n") - self.assertTrue(accepted.wait(timeout=2)) - slow_client.settimeout(1) - try: - closed_data = slow_client.recv(1) - except TimeoutError: - self.fail("incomplete API request was not closed after the configured timeout") - self.assertEqual(b"", closed_data) - - status, body = request_json(server.server_port, "GET", "/health") - self.assertEqual(200, status) - self.assertEqual({"status": "ok"}, body) - finally: - slow_client.close() - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - def test_health_is_public_but_missing_api_key_rejects_bots_list(self) -> None: with api_server() as port: status, body = request_json(port, "GET", "/health") @@ -615,6 +736,135 @@ def test_v1_prefix_routes_health_and_bots(self) -> None: self.assertEqual(200, status) self.assertEqual("coder", body[0]["bot_id"]) + def test_v1_alias_normalizes_roots_and_forwards_lifecycle_queries(self) -> None: + calls: list[tuple[object, ...]] = [] + + class CapturingSupervisor: + def __init__(self, *args: object, **kwargs: object) -> None: + return + + def start( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("start", bot_id, wait, timeout_seconds, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.starting, 123, "/profiles/coder") + + def stop( + self, + bot_id: str, + *, + kill_after_timeout: bool | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("stop", bot_id, kill_after_timeout, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.stopped, None, "/profiles/coder") + + def restart( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("restart", bot_id, wait, timeout_seconds, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.running, 456, "/profiles/coder") + + def reconcile_summary( + self, + bot_id: str | None = None, + *, + source: str = "cli", + request_id: str | None = None, + **kwargs: object, + ) -> object: + calls.append(("reconcile_summary", bot_id, source, request_id)) + + class Summary: + def to_dict(self) -> dict[str, object]: + return {"ok": True, "scope": "bot"} + + return Summary() + + headers = {"x-zeus-api-key": "secret"} + with ( + patch("zeus.api.Supervisor", CapturingSupervisor), + api_server({"ZEUS_API_KEY": "secret"}) as port, + ): + for root in ("/v1", "/v1/"): + with self.subTest(root=root): + status, body = request_json(port, "GET", root, headers=headers) + self.assertEqual(404, status) + self.assertEqual( + { + "error": { + "code": "invalid_request", + "message": "not found", + "status": 404, + } + }, + body, + ) + + requests = ( + ( + "/v1/bots/coder/start?wait=1&timeout=2.5", + { + "bot_id": "coder", + "message": "", + "pid": 123, + "profile_path": "/profiles/coder", + "status": "starting", + }, + ("start", "coder", True, 2.5, "api"), + ), + ( + "/v1/bots/coder/stop?kill_after_timeout=1", + { + "bot_id": "coder", + "message": "", + "pid": None, + "profile_path": "/profiles/coder", + "status": "stopped", + }, + ("stop", "coder", True, "api"), + ), + ( + "/v1/bots/coder/restart?wait=0&timeout=3.5", + { + "bot_id": "coder", + "message": "", + "pid": 456, + "profile_path": "/profiles/coder", + "status": "running", + }, + ("restart", "coder", False, 3.5, "api"), + ), + ( + "/v1/bots/coder/reconcile?summary=1", + {"ok": True, "scope": "bot"}, + ("reconcile_summary", "coder", "api"), + ), + ) + for path, expected_body, expected_call in requests: + with self.subTest(path=path): + status, body = request_json(port, "POST", path, headers=headers) + self.assertEqual(200, status) + self.assertEqual(expected_body, body) + call = calls.pop(0) + self.assertEqual(expected_call, call[:-1]) + self.assertRegex(str(call[-1]), r"^[0-9a-f]{32}$") + + self.assertEqual([], calls) + def test_bot_create_and_list_expose_desired_state_and_convergence(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: status, created = request_json( @@ -771,7 +1021,7 @@ def test_reconcile_summary_returns_canonical_fleet_and_bot_run_payloads(self) -> }, set(payload["results"][0]), ) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 2, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -833,7 +1083,7 @@ def test_reconcile_summary_query_is_strict_and_missing_bot_remains_404(self) -> ) self.assertEqual(404, status) self.assertEqual("unknown_bot", body["error"]["code"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 0, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -1010,7 +1260,7 @@ def claim_after_registry_growth( self.assertEqual(["coder"], [item["bot_id"] for item in first["results"]]) self.assertNotIn("idempotency-replayed", first_headers) self.assertEqual("true", replay_headers["idempotency-replayed"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 1, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -1136,7 +1386,7 @@ def reconcile_summary(self, *args: object, **kwargs: object) -> ReconcileRunSumm ) self.assertEqual(1, effects) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( (0, 1), ( @@ -1168,7 +1418,7 @@ def test_keyed_summary_over_replay_budget_is_rejected_before_run(self) -> None: self.assertEqual(422, status) self.assertEqual("idempotency_response_too_large", body["error"]["code"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( (0, 0), ( @@ -1486,6 +1736,44 @@ def test_bot_inspect_returns_diagnostics_and_redacts_logs(self) -> None: self.assertEqual(404, status) self.assertEqual("unknown_bot", body["error"]["code"]) + def test_bot_logs_and_inspect_hide_unsafe_gateway_log_path(self) -> None: + with api_server({"ZEUS_API_KEY": "secret"}) as port: + status, created = request_json( + port, + "POST", + "/bots", + body=json_request_body({"bot_id": "coder", "template_id": "coding-bot"}), + headers=auth_json_headers(), + ) + self.assertEqual(200, status) + profile_path = Path(created["profile_path"]) + logs_path = profile_path / "logs" + logs_path.rmdir() + external_logs = profile_path.parent / "external-api-logs" + external_logs.mkdir(mode=0o755) + target_log = external_logs / "zeus-gateway.log" + target_log.write_text("API-TARGET-SENTINEL\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + logs_path.symlink_to(external_logs, target_is_directory=True) + + for endpoint in ("logs", "inspect"): + with self.subTest(endpoint=endpoint): + status, body = request_json( + port, + "GET", + f"/bots/coder/{endpoint}", + headers={"x-zeus-api-key": "secret"}, + ) + self.assertEqual(500, status) + self.assertEqual("internal_error", body["error"]["code"]) + self.assertEqual("internal server error", body["error"]["message"]) + serialized = json.dumps(body) + self.assertNotIn("API-TARGET-SENTINEL", serialized) + self.assertNotIn(str(external_logs), serialized) + + self.assertEqual("API-TARGET-SENTINEL\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + def test_rejects_malformed_json(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: status, body = request_json( @@ -2063,6 +2351,7 @@ def test_second_api_process_cannot_overwrite_pid_marker(self) -> None: except subprocess.TimeoutExpired: first.kill() first.wait(timeout=5) + first.communicate() class ApiRateLimitTests(unittest.TestCase): @@ -2386,5 +2675,62 @@ def bucket_factory(rate_per_minute: int, burst: int) -> TokenBucket: self.assertEqual(404, refilled_status) +class ApiSQLiteDurabilityTests(unittest.TestCase): + def test_make_handler_forwards_full_to_state_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env( + { + "ZEUS_STATE_DIR": str(Path(tmp) / "state"), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL", + } + ) + with ( + patch.object(Settings, "ensure_dirs"), + patch("zeus.api.StateStore") as store_type, + patch("zeus.api.ApiLogWriter"), + ): + make_handler(settings) + + store_type.assert_called_once_with( + settings.database_path, + synchronous=SQLiteSynchronous.FULL, + ) + + def test_api_main_rejects_invalid_mode_before_runtime_setup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + stderr = io.StringIO() + with ( + patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL; VACUUM", + }, + clear=True, + ), + patch("zeus.config.load_dotenv", return_value={}), + patch("zeus.api.serve") as serve_entrypoint, + patch("zeus.api.StateStore") as store_type, + patch("zeus.api.make_handler") as handler_factory, + patch("zeus.api.ThreadingHTTPServer") as server_type, + patch.object(Settings, "ensure_dirs") as ensure_dirs, + redirect_stderr(stderr), + ): + result = api_module.main([]) + + self.assertEqual(1, result) + self.assertEqual( + "Invalid Zeus configuration: ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL\n", + stderr.getvalue(), + ) + serve_entrypoint.assert_not_called() + store_type.assert_not_called() + handler_factory.assert_not_called() + server_type.assert_not_called() + ensure_dirs.assert_not_called() + self.assertFalse(state_dir.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_api_errors.py b/tests/test_api_errors.py new file mode 100644 index 0000000..b118ef4 --- /dev/null +++ b/tests/test_api_errors.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import unittest +from http import HTTPStatus +from pathlib import Path + +from zeus.api_errors import ApiError, map_api_exception +from zeus.errors import ZeusConflictError +from zeus.models import TemplateError +from zeus.process_lock import LockTimeoutError +from zeus.reconciliation import ReconcileLockTimeoutError + + +class ApiErrorMappingTests(unittest.TestCase): + def test_maps_public_exception_contract(self) -> None: + cases = ( + ( + "unknown_bot", + KeyError("unknown bot: coder"), + ApiError(HTTPStatus.NOT_FOUND, "unknown_bot", "unknown bot: coder"), + ), + ( + "unknown_template", + KeyError("unknown template: missing"), + ApiError( + HTTPStatus.BAD_REQUEST, + "unknown_template", + "unknown template: missing", + ), + ), + ( + "missing_field", + KeyError("bot_id"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "missing required field: bot_id", + ), + ), + ( + "invalid_bot_id", + TemplateError("bot_id must match ^[a-z][a-z0-9-]{1,62}$"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_bot_id", + "bot_id must match ^[a-z][a-z0-9-]{1,62}$", + ), + ), + ( + "template_error", + TemplateError("template contract failed"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "template contract failed", + ), + ), + ( + "reconcile_lock", + ReconcileLockTimeoutError(Path("/tmp/reconcile.lock"), 0.1), + ApiError( + HTTPStatus.CONFLICT, + "reconcile_locked", + "reconciliation is already in progress", + ), + ), + ( + "bot_lock", + LockTimeoutError(Path("/tmp/coder.lock"), 0.1), + ApiError( + HTTPStatus.CONFLICT, + "bot_locked", + "bot lifecycle operation is already in progress", + ), + ), + ( + "conflict", + ZeusConflictError("conflicting lifecycle state"), + ApiError( + HTTPStatus.CONFLICT, + "conflict", + "conflicting lifecycle state", + ), + ), + ( + "value_error", + ValueError("invalid lifecycle request"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "invalid lifecycle request", + ), + ), + ( + "internal_error", + RuntimeError("private failure detail"), + ApiError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "internal_error", + "internal server error", + log_exception=True, + ), + ), + ) + + for name, exception, expected in cases: + with self.subTest(name=name): + self.assertEqual(expected, map_api_exception(exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_api_logging.py b/tests/test_api_logging.py index 23e407c..73072b5 100644 --- a/tests/test_api_logging.py +++ b/tests/test_api_logging.py @@ -3,9 +3,7 @@ import http.client import io import json -import os import socket -import stat import tempfile import threading import time @@ -15,11 +13,12 @@ from http.server import ThreadingHTTPServer from pathlib import Path from typing import Any -from unittest.mock import call, patch +from unittest.mock import patch from zeus.api import make_handler from zeus.api_logging import ApiLogWriter from zeus.config import Settings +from zeus.private_io import UnsafeFileError from zeus.request_context import ( IDEMPOTENCY_OUTCOMES, RequestContext, @@ -56,7 +55,7 @@ def raw_http_request(port: int, request: bytes) -> tuple[int, dict[str, str], di class ApiLoggingTests(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() - self.root = Path(self.temp_dir.name) + self.root = Path(self.temp_dir.name).resolve() def tearDown(self) -> None: self.temp_dir.cleanup() @@ -109,6 +108,8 @@ def test_idempotency_outcomes_are_a_closed_secret_free_vocabulary(self) -> None: def test_route_templates_hide_bot_ids_and_normalize_v1(self) -> None: self.assertEqual("/bots/{bot_id}/start", route_template("/v1/bots/secret-bot/start")) self.assertEqual("/bots/{bot_id}/history", route_template("/v1/bots/secret-bot/history")) + self.assertEqual("/ready", route_template("/ready")) + self.assertEqual("/ready", route_template("/v1/ready")) self.assertIsNone(route_template("/not-a-route")) def test_api_log_writer_writes_parseable_redacted_lines(self) -> None: @@ -250,6 +251,42 @@ def test_api_log_failure_is_fail_open(self) -> None: ApiLogWriter(disabled_path, enabled=False).access({"status": 200}) self.assertFalse(disabled_path.exists()) + def test_api_log_writer_delegates_exact_canonical_utf8_line(self) -> None: + path = self.root / "api.jsonl" + + with ( + patch("zeus.api_logging.datetime") as clock, + patch("zeus.api_logging.append_private_bytes") as append, + ): + clock.now.return_value.isoformat.return_value = "2026-07-21T12:34:56+00:00" + ApiLogWriter(path, enabled=True).access({"status": 200}) + + append.assert_called_once_with( + path, + b'{"event":"api.access","level":"info","schema_version":1,' + b'"status":200,"ts":"2026-07-21T12:34:56+00:00"}\n', + ) + + def test_api_log_writer_keeps_private_io_failures_fail_open(self) -> None: + path = self.root / "api.jsonl" + + with patch( + "zeus.api_logging.append_private_bytes", + side_effect=UnsafeFileError("unsafe path"), + ) as append: + ApiLogWriter(path, enabled=True).access({"status": 200}) + ApiLogWriter(path, enabled=True).error("a" * 32, RuntimeError("safe")) + + self.assertEqual(2, append.call_count) + + def test_disabled_api_log_writer_does_not_delegate(self) -> None: + with patch("zeus.api_logging.append_private_bytes") as append: + writer = ApiLogWriter(self.root / "api.jsonl", enabled=False) + writer.access({"status": 200}) + writer.error("a" * 32, RuntimeError("safe")) + + append.assert_not_called() + def test_api_access_payload_building_is_fail_open(self) -> None: class BrokenString: def __str__(self) -> str: @@ -272,93 +309,6 @@ def __str__(self) -> str: self.assertEqual("Exception", row["error_type"]) self.assertEqual("Unexpected API error", row["message"]) - def test_api_log_writer_creates_private_directory_and_file(self) -> None: - path = self.root / "logs" / "api.jsonl" - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(0o700, stat.S_IMODE(path.parent.stat().st_mode)) - self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) - - def test_api_log_writer_tightens_existing_directory_and_file_modes(self) -> None: - directory = self.root / "logs" - directory.mkdir(mode=0o755) - path = directory / "api.jsonl" - path.write_text("", encoding="utf-8") - directory.chmod(0o755) - path.chmod(0o644) - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(0o700, stat.S_IMODE(directory.stat().st_mode)) - self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) - self.assertEqual(200, json.loads(path.read_text(encoding="utf-8"))["status"]) - - @unittest.skipUnless(hasattr(os, "O_NOFOLLOW"), "O_NOFOLLOW unavailable") - def test_api_log_writer_does_not_follow_symlinked_log_directory(self) -> None: - outside = self.root / "outside" - outside.mkdir(mode=0o755) - outside.chmod(0o755) - outside_mode = stat.S_IMODE(outside.stat().st_mode) - logs = self.root / "state" / "logs" - logs.parent.mkdir() - logs.symlink_to(outside, target_is_directory=True) - - ApiLogWriter(logs / "api.jsonl", enabled=True).access({"status": 200}) - - self.assertFalse((outside / "api.jsonl").exists()) - self.assertEqual(outside_mode, stat.S_IMODE(outside.stat().st_mode)) - - def test_api_log_writer_closes_descriptor_when_fchmod_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.fchmod", side_effect=OSError("denied")), - patch("zeus.api_logging.os.close", wraps=os.close) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - close.assert_called_once() - self.assertFalse(path.exists()) - - def test_api_log_writer_closes_both_descriptors_when_file_fchmod_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.fchmod", side_effect=[None, OSError("denied")]), - patch("zeus.api_logging.os.close", wraps=os.close) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(2, close.call_count) - self.assertEqual("", path.read_text(encoding="utf-8")) - - def test_api_log_writer_attempts_directory_close_when_file_close_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.open", side_effect=[101, 202]), - patch("zeus.api_logging.os.fchmod", side_effect=[None, OSError("denied")]), - patch( - "zeus.api_logging.os.close", - side_effect=[OSError("file close failed"), None], - ) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual([call(202), call(101)], close.call_args_list) - - @unittest.skipUnless(hasattr(os, "O_NOFOLLOW"), "O_NOFOLLOW unavailable") - def test_api_log_writer_does_not_follow_symlinks(self) -> None: - target = self.root / "target.jsonl" - target.write_text("original\n", encoding="utf-8") - path = self.root / "api.jsonl" - path.symlink_to(target) - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual("original\n", target.read_text(encoding="utf-8")) - def test_api_error_omits_exception_messages_and_tracebacks(self) -> None: path = self.root / "api.jsonl" sensitive_message = ( diff --git a/tests/test_api_request.py b/tests/test_api_request.py new file mode 100644 index 0000000..defc8d1 --- /dev/null +++ b/tests/test_api_request.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import unittest + +from zeus.api_request import decode_json_object, normalize_api_path, parse_query + + +class ApiPathTests(unittest.TestCase): + def test_normalizes_only_the_v1_alias(self) -> None: + cases = ( + ("/v1", "/"), + ("/v1/", "/"), + ("/v1/bots", "/bots"), + ("/health", "/health"), + ("/v10/bots", "/v10/bots"), + ) + + for target, expected in cases: + with self.subTest(target=target): + self.assertEqual(expected, normalize_api_path(target)) + + def test_rejects_request_target_fragments(self) -> None: + with self.assertRaisesRegex( + ValueError, + "^request target must not include a fragment$", + ): + normalize_api_path("/health#internal") + + +class ApiQueryTests(unittest.TestCase): + def test_preserves_blank_values_and_field_order(self) -> None: + values = parse_query( + "/bots?second=2&first=", + frozenset({"first", "second"}), + ) + + self.assertEqual(["second", "first"], list(values)) + self.assertEqual({"second": ["2"], "first": [""]}, values) + + def test_accepts_sixteen_query_fields(self) -> None: + names = tuple(f"field{index}" for index in range(16)) + query = "&".join(f"{name}=1" for name in names) + + values = parse_query(f"/health?{query}", frozenset(names)) + + self.assertEqual(16, len(values)) + + def test_rejects_seventeen_query_fields(self) -> None: + names = tuple(f"field{index}" for index in range(17)) + query = "&".join(f"{name}=1" for name in names) + + with self.assertRaisesRegex(ValueError, "^too many query parameters$"): + parse_query(f"/health?{query}", frozenset(names)) + + def test_reports_unknown_field_before_duplicate_field(self) -> None: + with self.assertRaisesRegex(ValueError, "^unknown query parameter: debug$"): + parse_query( + "/health?known=1&known=2&debug=1", + frozenset({"known"}), + ) + + def test_rejects_duplicate_fields(self) -> None: + with self.assertRaisesRegex( + ValueError, + "^query parameter replace must be specified once$", + ): + parse_query( + "/bots?replace=0&replace=1", + frozenset({"replace"}), + ) + + +class ApiJsonTests(unittest.TestCase): + def test_decodes_a_json_object(self) -> None: + self.assertEqual( + {"bot_id": "coder", "env": {"MODE": "safe"}}, + decode_json_object(b'{"bot_id":"coder","env":{"MODE":"safe"}}'), + ) + + def test_rejects_duplicate_fields_at_every_object_depth(self) -> None: + cases = ( + (b'{"bot_id":"coder","bot_id":"other"}', "bot_id"), + (b'{"env":{"MODE":"safe","MODE":"unsafe"}}', "MODE"), + ) + + for data, field in cases: + with ( + self.subTest(field=field), + self.assertRaisesRegex( + ValueError, + f"^duplicate JSON field: {field}$", + ), + ): + decode_json_object(data) + + def test_rejects_nonstandard_json_constants(self) -> None: + for constant in ("NaN", "Infinity", "-Infinity"): + with ( + self.subTest(constant=constant), + self.assertRaisesRegex( + ValueError, + f"^invalid JSON constant: {constant}$", + ), + ): + decode_json_object(f'{{"value":{constant}}}'.encode()) + + def test_rejects_array_and_scalar_roots(self) -> None: + for data in (b"[]", b'"value"', b"1", b"null"): + with ( + self.subTest(data=data), + self.assertRaisesRegex( + ValueError, + "^request body must be a JSON object$", + ), + ): + decode_json_object(data) + + def test_accepts_and_rejects_exact_depth_boundary(self) -> None: + data = b'{"nested":[0]}' + + self.assertEqual({"nested": [0]}, decode_json_object(data, max_depth=3)) + with self.assertRaisesRegex(ValueError, "^request JSON nesting exceeds 2$"): + decode_json_object(data, max_depth=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_api_server.py b/tests/test_api_server.py new file mode 100644 index 0000000..bd2cc02 --- /dev/null +++ b/tests/test_api_server.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import http.client +import json +import socket +import threading +import unittest +from http.server import BaseHTTPRequestHandler +from typing import Any + +from zeus.api import ThreadingHTTPServer as public_server +from zeus.api_server import ThreadingHTTPServer as extracted_server + + +def request_json(port: int, method: str, path: str) -> tuple[int, dict[str, Any]]: + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request(method, path) + response = conn.getresponse() + body = json.loads(response.read().decode("utf-8")) + return response.status, body + finally: + conn.close() + + +class ApiServerTests(unittest.TestCase): + def test_public_server_is_extracted_server(self) -> None: + self.assertIs(public_server, extracted_server) + + def test_rejects_requests_above_concurrency_limit(self) -> None: + first_entered = threading.Event() + release_first = threading.Event() + call_lock = threading.Lock() + call_count = 0 + + class LimitedHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 1 + api_request_timeout_seconds = 2.0 + + def do_GET(self) -> None: + nonlocal call_count + with call_lock: + call_count += 1 + current_call = call_count + if current_call == 1: + first_entered.set() + release_first.wait(timeout=3) + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server = extracted_server(("127.0.0.1", 0), LimitedHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + first_result: list[tuple[int, dict[str, Any]]] = [] + first_thread = threading.Thread( + target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) + ) + server_thread.start() + first_thread.start() + try: + self.assertTrue(first_entered.wait(timeout=2)) + conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) + try: + conn.request("GET", "/health") + response = conn.getresponse() + raw_body = response.read() + finally: + conn.close() + + self.assertEqual(503, response.status) + self.assertEqual("1", response.getheader("retry-after")) + self.assertRegex( + response.getheader("x-request-id") or "", + r"^[0-9a-f]{32}$", + ) + body = json.loads(raw_body.decode("utf-8")) + self.assertEqual("server_busy", body["error"]["code"]) + finally: + release_first.set() + first_thread.join(timeout=3) + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + self.assertFalse(first_thread.is_alive()) + self.assertEqual(200, first_result[0][0]) + + def test_drains_active_requests_and_rejects_new_work(self) -> None: + first_entered = threading.Event() + release_first = threading.Event() + + class DrainHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 2 + api_request_timeout_seconds = 2.0 + + def do_GET(self) -> None: + first_entered.set() + release_first.wait(timeout=3) + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server = extracted_server(("127.0.0.1", 0), DrainHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + first_result: list[tuple[int, dict[str, Any]]] = [] + first_thread = threading.Thread( + target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) + ) + server_thread.start() + first_thread.start() + try: + self.assertTrue(first_entered.wait(timeout=2)) + request_graceful_shutdown = getattr(server, "request_graceful_shutdown", None) + wait_until_draining = getattr(server, "wait_until_draining", None) + wait_for_drain = getattr(server, "wait_for_drain", None) + self.assertTrue(callable(request_graceful_shutdown)) + self.assertTrue(callable(wait_until_draining)) + self.assertTrue(callable(wait_for_drain)) + request_graceful_shutdown(1.0) + self.assertTrue(wait_until_draining(1.0)) + + conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) + try: + conn.request("GET", "/health") + response = conn.getresponse() + raw_body = response.read() + finally: + conn.close() + + self.assertEqual(503, response.status) + self.assertEqual("1", response.getheader("retry-after")) + self.assertRegex( + response.getheader("x-request-id") or "", + r"^[0-9a-f]{32}$", + ) + body = json.loads(raw_body.decode("utf-8")) + self.assertEqual("server_draining", body["error"]["code"]) + self.assertFalse(wait_for_drain(0.05)) + + release_first.set() + first_thread.join(timeout=3) + self.assertTrue(wait_for_drain(1.0)) + server_thread.join(timeout=3) + finally: + release_first.set() + first_thread.join(timeout=3) + if server_thread.is_alive(): + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + self.assertFalse(first_thread.is_alive()) + self.assertFalse(server_thread.is_alive()) + self.assertEqual(200, first_result[0][0]) + + def test_times_out_incomplete_requests_and_releases_capacity(self) -> None: + accepted = threading.Event() + + class TimeoutHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 1 + api_request_timeout_seconds = 0.1 + + def do_GET(self) -> None: + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + class TrackingServer(extracted_server): + def process_request(self, request: Any, client_address: Any) -> None: + accepted.set() + super().process_request(request, client_address) + + server = TrackingServer(("127.0.0.1", 0), TimeoutHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + slow_client = socket.create_connection(("127.0.0.1", server.server_port), timeout=2) + try: + slow_client.sendall(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n") + self.assertTrue(accepted.wait(timeout=2)) + slow_client.settimeout(1) + try: + closed_data = slow_client.recv(1) + except TimeoutError: + self.fail("incomplete API request was not closed after the configured timeout") + self.assertEqual(b"", closed_data) + + status, body = request_json(server.server_port, "GET", "/health") + self.assertEqual(200, status) + self.assertEqual({"status": "ok"}, body) + finally: + slow_client.close() + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bot_lifecycle_store.py b/tests/test_bot_lifecycle_store.py new file mode 100644 index 0000000..7eebeca --- /dev/null +++ b/tests/test_bot_lifecycle_store.py @@ -0,0 +1,1271 @@ +from __future__ import annotations + +import ast +import inspect +import json +import sqlite3 +import tempfile +import unittest +from collections.abc import Callable +from contextlib import closing +from dataclasses import FrozenInstanceError +from datetime import UTC, datetime, timedelta +from operator import methodcaller +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +from zeus.bot_lifecycle_store import ( + LIFECYCLE_ERROR_CODE_RE, + LIFECYCLE_ID_RE, + LIFECYCLE_INTENT_ACTIONS, + LIFECYCLE_SOURCES, + BotLifecycleStore, +) +from zeus.config import SQLiteSynchronous +from zeus.idempotency_store import IdempotencyStore +from zeus.lifecycle import LifecycleEvent, LifecycleEventInput +from zeus.models import BotRecord, BotStatus +from zeus.reconcile_store import ReconcileStore +from zeus.sanitization import MAX_SANITIZED_JSON_BYTES +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import LIFECYCLE_ERROR_CODE_RE as STATE_LIFECYCLE_ERROR_CODE_RE +from zeus.state import LIFECYCLE_ID_RE as STATE_LIFECYCLE_ID_RE +from zeus.state import LIFECYCLE_INTENT_ACTIONS as STATE_LIFECYCLE_INTENT_ACTIONS +from zeus.state import LIFECYCLE_SOURCES as STATE_LIFECYCLE_SOURCES +from zeus.state import StateStore + +FIXED_NOW = datetime(2026, 7, 21, 12, 34, 56, tzinfo=UTC) +NEXT_RESTART = FIXED_NOW + timedelta(minutes=5) + +PUBLIC_METHODS = ( + "begin_lifecycle_intent", + "complete_lifecycle_intent", + "clear_stale_intent", + "audit_log_path", + "append_audit_event", + "upsert_bot", + "upsert_bot_with_event", + "get_bot", + "list_bots", + "update_status", + "update_lifecycle_state", + "update_lifecycle_with_event", + "update_restart_state", + "update_restart_with_event", + "delete_bot", + "delete_bot_with_event", + "list_lifecycle_events", + "history_payload", +) + + +class _TracingSQLiteDatabase(SQLiteDatabase): + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces: list[tuple[int, str]] = [] + self.connection_count = 0 + + def connect(self) -> sqlite3.Connection: + self.connection_count += 1 + conn = super().connect() + connection_serial = self.connection_count + conn.set_trace_callback(lambda sql: self.traces.append((connection_serial, sql))) + return conn + + +class _CountingSQLiteDatabase(SQLiteDatabase): + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.connection_count = 0 + + def connect(self) -> sqlite3.Connection: + self.connection_count += 1 + return super().connect() + + +def _record(root: Path, *, bot_id: str = "coder") -> BotRecord: + return BotRecord( + bot_id=bot_id, + template_id="coding-bot", + display_name=bot_id.title(), + profile_path=str(root / "profiles" / bot_id), + ) + + +def _event( + action: str, + *, + bot_id: str = "coder", + operation_character: str = "a", +) -> LifecycleEventInput: + return LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_character * 32, + source="cli", + action=action, + outcome="success", + reason="test transition", + details={"kind": action}, + ) + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _connection_id, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statements(trace: list[tuple[int, str]]) -> list[str]: + return [sql for _connection_id, sql in trace] + + +def _statement_index(statements: list[str], prefix: str, *, start: int = 0) -> int: + normalized_prefix = " ".join(prefix.split()).upper() + return next( + index + for index, statement in enumerate(statements[start:], start=start) + if " ".join(statement.split()).upper().startswith(normalized_prefix) + ) + + +def _table_bytes(database_path: Path, table: str, order_by: str) -> tuple[tuple[object, ...], ...]: + with closing(sqlite3.connect(database_path)) as conn: + columns = tuple(row[1] for row in conn.execute(f"PRAGMA table_info({table})")) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + rows = conn.execute(f"SELECT {fields} FROM {table} ORDER BY {order_by}").fetchall() + return tuple(rows) + + +def _database_snapshot( + database_path: Path, +) -> tuple[tuple[tuple[object, ...], ...], tuple[tuple[object, ...], ...]]: + return ( + _table_bytes(database_path, "bots", "bot_id"), + _table_bytes(database_path, "lifecycle_events", "event_id"), + ) + + +def _install_event_rejection(database_path: Path) -> None: + with closing(sqlite3.connect(database_path)) as conn: + conn.execute( + """ + CREATE TRIGGER reject_test_lifecycle_event + BEFORE INSERT ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'injected lifecycle event failure'); + END + """ + ) + conn.commit() + + +class BotLifecycleStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.root = Path(self.temporary_directory.name) + self.database_path = self.root / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = BotLifecycleStore(self.database) + + def _new_tracing_store( + self, + name: str, + ) -> tuple[_TracingSQLiteDatabase, BotLifecycleStore, Path]: + root = self.root / name + database = _TracingSQLiteDatabase(root / "zeus.db") + SchemaManager(database).init() + database.traces.clear() + return database, BotLifecycleStore(database), root + + def _prepare_atomic( + self, + store: BotLifecycleStore, + root: Path, + operation: str, + ) -> None: + if operation != "create": + store.upsert_bot(_record(root)) + if operation in {"complete_intent", "clear_intent"}: + store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="b" * 32, + source="cli", + ) + + def _invoke_atomic( + self, + store: BotLifecycleStore, + root: Path, + operation: str, + ) -> object: + if operation == "create": + return store.upsert_bot_with_event( + _record(root), + event=_event("bot.create"), + ) + if operation == "lifecycle": + return store.update_lifecycle_with_event( + "coder", + BotStatus.running, + 4321, + event=_event("bot.start"), + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + ) + if operation == "restart": + return store.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event("bot.restart.schedule"), + ) + if operation == "delete": + return store.delete_bot_with_event( + "coder", + event=_event("bot.delete"), + ) + if operation == "begin_intent": + return store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + ) + if operation == "complete_intent": + return store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="b" * 32, + desired_revision=1, + status=BotStatus.running, + pid=4321, + source="cli", + ) + if operation == "clear_intent": + return store.clear_stale_intent( + "coder", + action="start", + operation_id="b" * 32, + desired_revision=1, + source="recovery", + reason="stale intent", + ) + raise AssertionError(f"unknown atomic operation: {operation}") + + def test_ownership_construction_and_compatibility_aliases(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + BotLifecycleStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + import zeus.bot_lifecycle_store as bot_lifecycle_store_module + + source = inspect.getsource(bot_lifecycle_store_module) + tree = ast.parse(source) + imported_modules = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + } + self.assertTrue( + imported_modules.isdisjoint( + { + "zeus.reconcile_store", + "zeus.reconciliation", + "zeus.schema", + "zeus.state", + } + ) + ) + self.assertNotIn("CREATE TABLE", source.upper()) + self.assertNotIn("CREATE TRIGGER", source.upper()) + self.assertIs(LIFECYCLE_ID_RE, STATE_LIFECYCLE_ID_RE) + self.assertIs(LIFECYCLE_ERROR_CODE_RE, STATE_LIFECYCLE_ERROR_CODE_RE) + self.assertIs(LIFECYCLE_SOURCES, STATE_LIFECYCLE_SOURCES) + self.assertIs(LIFECYCLE_INTENT_ACTIONS, STATE_LIFECYCLE_INTENT_ACTIONS) + + def test_direct_store_round_trip_covers_every_public_family(self) -> None: + coder = _record(self.root) + self.store.upsert_bot(coder) + self.assertEqual(coder, self.store.get_bot("coder")) + self.assertEqual([coder], self.store.list_bots()) + + self.store.update_status("coder", BotStatus.starting, 1111, reset_restart=True) + self.store.update_lifecycle_state( + "coder", + BotStatus.running, + 2222, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + last_transition_reason="ready", + ) + self.store.update_restart_state( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + ) + loaded = self.store.get_bot("coder") + assert loaded is not None + self.assertEqual(BotStatus.failed, loaded.status) + self.assertEqual(2, loaded.restart_attempts) + self.assertEqual(NEXT_RESTART, loaded.next_restart_at) + + begun = self.store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + ) + self.assertEqual(1, begun.desired_revision) + completed = self.store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=3333, + source="cli", + ) + self.assertTrue(completed.converged) + restarted = self.store.begin_lifecycle_intent( + "coder", + action="restart", + operation_id="b" * 32, + source="cli", + ) + cleared = self.store.clear_stale_intent( + "coder", + action="restart", + operation_id="b" * 32, + desired_revision=restarted.desired_revision, + source="recovery", + reason="stale intent", + ) + self.assertIsNone(cleared.pending_operation_id) + + evented = _record(self.root, bot_id="evented") + created = self.store.upsert_bot_with_event( + evented, + event=_event("bot.create", bot_id="evented", operation_character="c"), + ) + transitioned = self.store.update_lifecycle_with_event( + "evented", + BotStatus.running, + 4444, + event=_event("bot.start", bot_id="evented", operation_character="d"), + ) + scheduled = self.store.update_restart_with_event( + "evented", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event( + "bot.restart.schedule", + bot_id="evented", + operation_character="e", + ), + ) + deleted = self.store.delete_bot_with_event( + "evented", + event=_event("bot.delete", bot_id="evented", operation_character="f"), + ) + self.assertEqual( + ["bot.delete", "bot.restart.schedule", "bot.start", "bot.create"], + [event.action for event in self.store.list_lifecycle_events("evented", 10, None)], + ) + self.assertIsInstance(created, LifecycleEvent) + self.assertIsInstance(transitioned, LifecycleEvent) + self.assertIsInstance(scheduled, LifecycleEvent) + self.assertIs(deleted, True) + self.assertEqual("evented", self.store.history_payload("evented", 2, None)["bot_id"]) + self.assertEqual(self.root / "logs" / "audit.jsonl", self.store.audit_log_path()) + + self.store.update_status("missing", BotStatus.running, 9999) + with self.assertRaises(KeyError): + self.store.update_lifecycle_state("missing", BotStatus.failed) + with self.assertRaises(KeyError): + self.store.update_restart_state( + "missing", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=None, + ) + self.assertIs(self.store.delete_bot("missing"), False) + with self.assertRaises(KeyError): + self.store.delete_bot_with_event( + "missing", + event=_event("bot.delete", bot_id="missing"), + ) + self.assertIs(self.store.delete_bot("coder"), True) + + def test_invalid_inputs_are_rejected_before_connecting(self) -> None: + database = _CountingSQLiteDatabase(self.root / "validation.db") + store = BotLifecycleStore(database) + record = _record(self.root) + + invalid_calls = ( + lambda: store.begin_lifecycle_intent( + "coder", + action="delete", + operation_id="a" * 32, + source="cli", + ), + lambda: store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="api", + ), + lambda: store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=0, + status=BotStatus.running, + pid=1, + source="cli", + ), + lambda: store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.stopped, + pid=None, + source="cli", + ), + lambda: store.upsert_bot_with_event( + record, + event=_event("bot.create", bot_id="other"), + ), + lambda: store.list_lifecycle_events("coder", 0, None), + lambda: store.history_payload("coder", 10, 0), + ) + for invalid_call in invalid_calls: + with self.subTest(call=invalid_call), self.assertRaises((TypeError, ValueError)): + invalid_call() + self.assertEqual(0, database.connection_count) + + def test_all_seven_atomic_paths_have_one_immediate_transaction_and_required_order(self) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + pre_event_prefixes = { + "create": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "INSERT INTO BOTS", + ), + "lifecycle": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + ), + "restart": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + ), + "delete": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "DELETE FROM BOTS", + ), + "begin_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET DESIRED_STATE", + ), + "complete_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + "UPDATE BOTS SET PENDING_OPERATION_ID", + ), + "clear_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET PENDING_OPERATION_ID", + ), + } + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"atomic-{operation}") + self._prepare_atomic(store, root, operation) + database.traces.clear() + database.connection_count = 0 + + self._invoke_atomic(store, root, operation) + trace = list(database.traces) + statements = _statements(trace) + + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(trace)) + begin = _statement_index(statements, "BEGIN IMMEDIATE") + ordered_indices: list[int] = [] + search_start = begin + 1 + for prefix in pre_event_prefixes[operation]: + index = _statement_index(statements, prefix, start=search_start) + ordered_indices.append(index) + search_start = index + 1 + event_insert = _statement_index( + statements, + "INSERT INTO LIFECYCLE_EVENTS", + start=search_start, + ) + event_select = _statement_index( + statements, + "SELECT * FROM LIFECYCLE_EVENTS WHERE EVENT_ID", + start=event_insert, + ) + commit = _statement_index(statements, "COMMIT") + self.assertEqual(sorted(ordered_indices), ordered_indices) + self.assertLess(ordered_indices[-1], event_insert) + self.assertLess(event_insert, event_select) + self.assertLess(event_select, commit) + last_event_updates = [ + index + for index, sql in enumerate(statements) + if sql.lstrip().upper().startswith("UPDATE BOTS SET LAST_EVENT_ID") + ] + if operation == "delete": + self.assertEqual([], last_event_updates) + else: + self.assertTrue(last_event_updates) + self.assertLess(event_insert, last_event_updates[0]) + self.assertLess(last_event_updates[0], event_select) + if operation.endswith("intent"): + projection_selects = [ + index + for index, sql in enumerate(statements) + if sql.lstrip().upper().startswith("SELECT * FROM BOTS WHERE BOT_ID") + ] + self.assertGreaterEqual(len(projection_selects), 2) + self.assertLess(event_select, projection_selects[-1]) + self.assertLess(projection_selects[-1], commit) + + def test_plain_writes_keep_implicit_transactions_and_reads_remain_transaction_free( + self, + ) -> None: + write_operations = ( + "upsert", + "status", + "lifecycle", + "restart", + "delete", + ) + for operation in write_operations: + with self.subTest(write=operation): + database, store, root = self._new_tracing_store(f"plain-{operation}") + if operation != "upsert": + store.upsert_bot(_record(root)) + database.traces.clear() + database.connection_count = 0 + + if operation == "upsert": + store.upsert_bot(_record(root)) + elif operation == "status": + store.update_status("coder", BotStatus.starting, 1001) + elif operation == "lifecycle": + store.update_lifecycle_state( + "coder", + BotStatus.running, + 1002, + started_at=FIXED_NOW, + ) + elif operation == "restart": + store.update_restart_state( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=3, + next_restart_at=NEXT_RESTART, + ) + else: + self.assertIs(store.delete_bot("coder"), True) + + trace = list(database.traces) + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual(["BEGIN", "COMMIT"], _control_statements(trace)) + self.assertNotIn("BEGIN IMMEDIATE", _control_statements(trace)) + + database, store, root = self._new_tracing_store("reads") + store.upsert_bot_with_event(_record(root), event=_event("bot.create")) + read_operations = ( + lambda: store.get_bot("coder"), + store.list_bots, + lambda: store.list_lifecycle_events("coder", 10, None), + lambda: store.history_payload("coder", 10, None), + ) + for read_operation in read_operations: + with self.subTest(read=read_operation): + database.traces.clear() + database.connection_count = 0 + read_operation() + trace = list(database.traces) + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual([], _control_statements(trace)) + self.assertTrue( + all(sql.lstrip().upper().startswith("SELECT") for _identity, sql in trace) + ) + database.traces.clear() + database.connection_count = 0 + self.assertEqual(root / "logs" / "audit.jsonl", store.audit_log_path()) + self.assertEqual([], database.traces) + self.assertEqual(0, database.connection_count) + + def test_sql_trigger_rolls_back_all_seven_atomic_paths_byte_for_byte(self) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"rollback-{operation}") + self._prepare_atomic(store, root, operation) + _install_event_rejection(database.database_path) + before = _database_snapshot(database.database_path) + database.traces.clear() + database.connection_count = 0 + + with self.assertRaisesRegex( + sqlite3.DatabaseError, + "injected lifecycle event failure", + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual( + ["BEGIN IMMEDIATE", "ROLLBACK"], + _control_statements(database.traces), + ) + self.assertEqual(1, database.connection_count) + self.assertEqual(before, _database_snapshot(database.database_path)) + + def test_late_materialization_failures_roll_back_event_and_projection(self) -> None: + for operation in ("create", "lifecycle", "restart", "delete"): + with self.subTest(event=operation): + database, store, root = self._new_tracing_store(f"late-event-{operation}") + self._prepare_atomic(store, root, operation) + before = _database_snapshot(database.database_path) + + with ( + patch.object( + store, + "_row_to_lifecycle_event", + side_effect=sqlite3.DatabaseError("event materialization failed"), + ), + self.assertRaisesRegex(sqlite3.DatabaseError, "event materialization failed"), + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual(before, _database_snapshot(database.database_path)) + + for operation in ("begin_intent", "complete_intent", "clear_intent"): + with self.subTest(record=operation): + database, store, root = self._new_tracing_store(f"late-record-{operation}") + self._prepare_atomic(store, root, operation) + before = _database_snapshot(database.database_path) + + with ( + patch.object( + store, + "_row_to_record", + side_effect=sqlite3.DatabaseError("record materialization failed"), + ), + self.assertRaisesRegex(sqlite3.DatabaseError, "record materialization failed"), + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual(before, _database_snapshot(database.database_path)) + + def test_audit_callback_observes_commit_and_failure_is_fail_open_for_every_atomic_path( + self, + ) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + expected_projection = { + "create": ("stopped", "stopped", None, None, 0), + "lifecycle": ("running", "stopped", None, None, 0), + "restart": ("failed", "stopped", None, None, 1), + "delete": None, + "begin_intent": ("stopped", "running", "a" * 32, "start", 0), + "complete_intent": ("running", "running", None, None, 0), + "clear_intent": ("stopped", "running", None, None, 0), + } + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"audit-order-{operation}") + self._prepare_atomic(store, root, operation) + before_events = len(_database_snapshot(database.database_path)[1]) + database.traces.clear() + database.connection_count = 0 + callback_observations: list[tuple[list[str], int, tuple[object, ...] | None]] = [] + + def observe_commit( + _event: LifecycleEvent, + *, + _database: _TracingSQLiteDatabase = database, + _observations: list[ + tuple[list[str], int, tuple[object, ...] | None] + ] = callback_observations, + ) -> None: + with closing(sqlite3.connect(_database.database_path)) as conn: + event_count = conn.execute( + "SELECT COUNT(*) FROM lifecycle_events" + ).fetchone()[0] + projection_row = conn.execute( + """ + SELECT status, + desired_state, + pending_operation_id, + pending_action, + restart_attempts + FROM bots + WHERE bot_id = 'coder' + """ + ).fetchone() + projection = tuple(projection_row) if projection_row is not None else None + _observations.append( + (_control_statements(_database.traces), event_count, projection) + ) + raise RuntimeError("audit callback failed") + + with patch.object(store, "_append_lifecycle_audit", side_effect=observe_commit): + result = self._invoke_atomic(store, root, operation) + + self.assertEqual( + [ + ( + ["BEGIN IMMEDIATE", "COMMIT"], + before_events + 1, + expected_projection[operation], + ) + ], + callback_observations, + ) + self.assertEqual(1, database.connection_count) + self.assertEqual( + before_events + 1, len(_database_snapshot(database.database_path)[1]) + ) + if operation == "delete": + self.assertIs(result, True) + + def test_history_cursor_deleted_bot_and_event_immutability(self) -> None: + self.store.upsert_bot_with_event( + _record(self.root), event=_event("bot.create", operation_character="a") + ) + self.store.update_lifecycle_with_event( + "coder", + BotStatus.running, + 4321, + event=_event("bot.start", operation_character="b"), + ) + self.store.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event("bot.restart.schedule", operation_character="c"), + ) + self.store.delete_bot_with_event( + "coder", + event=_event("bot.delete", operation_character="d"), + ) + + first_page = self.store.list_lifecycle_events("coder", 2, None) + second_page = self.store.list_lifecycle_events( + "coder", + 2, + first_page[-1].event_id, + ) + self.assertEqual(["bot.delete", "bot.restart.schedule"], [e.action for e in first_page]) + self.assertEqual(["bot.start", "bot.create"], [e.action for e in second_page]) + self.assertTrue( + {event.event_id for event in first_page}.isdisjoint( + event.event_id for event in second_page + ) + ) + payload = self.store.history_payload("coder", 2, None) + self.assertIsNotNone(payload["next_before"]) + self.assertEqual("coder", payload["bot_id"]) + self.assertEqual([], self.store.list_lifecycle_events("unknown", 10, None)) + with self.assertRaisesRegex(KeyError, "unknown bot"): + self.store.history_payload("unknown", 10, None) + + event = first_page[0] + with self.assertRaises(FrozenInstanceError): + event.action = "mutated" # type: ignore[misc] + with self.assertRaises(TypeError): + event.details["kind"] = "mutated" # type: ignore[index] + before = _database_snapshot(self.database_path) + for statement in ( + "UPDATE lifecycle_events SET action = 'mutated' WHERE event_id = ?", + "DELETE FROM lifecycle_events WHERE event_id = ?", + ): + with ( + self.subTest(statement=statement), + closing(sqlite3.connect(self.database_path)) as conn, + ): + with self.assertRaisesRegex(sqlite3.DatabaseError, "immutable"): + conn.execute(statement, (event.event_id,)) + conn.rollback() + self.assertEqual(before, _database_snapshot(self.database_path)) + + def test_audit_exact_bytes_mapping_bounds_hostile_input_and_symlink_fail_open(self) -> None: + exact_root = (self.root / "exact").resolve() + store = BotLifecycleStore(SQLiteDatabase(exact_root / "zeus.db")) + with ( + patch("zeus.bot_lifecycle_store.datetime") as clock, + patch("zeus.bot_lifecycle_store.append_private_bytes") as append, + ): + clock.now.return_value = FIXED_NOW + store.append_audit_event("bot.test", label="Δ", api_key="secret") + append.assert_called_once_with( + exact_root / "logs" / "audit.jsonl", + b'{"api_key": "[redacted]", "event": "bot.test", "label": "\\u0394", ' + b'"ts": "2026-07-21T12:34:56+00:00"}\n', + ) + + lifecycle_event = LifecycleEvent( + event_id=7, + bot_id="coder", + operation_id="a" * 32, + request_id=None, + occurred_at=FIXED_NOW, + source="cli", + action="bot.start", + outcome="success", + status_before="stopped", + status_after="running", + pid_before=None, + pid_after=4321, + reason="operator request", + error_code=None, + error_message=None, + details={"phase": "ready"}, + ) + with patch.object(store, "append_audit_event") as append_event: + store._append_lifecycle_audit(lifecycle_event) + append_event.assert_called_once_with( + "bot.start", + bot_id="coder", + operation_id="a" * 32, + request_id=None, + source="cli", + outcome="success", + status_before="stopped", + status_after="running", + pid_before=None, + pid_after=4321, + reason="operator request", + error_code=None, + error_message=None, + details=lifecycle_event.details, + ) + + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + bounded = BotLifecycleStore(SQLiteDatabase(self.root / "bounded" / "zeus.db")) + secret = "audit-sentinel" + bounded.append_audit_event( + "bot.hostile", + api_key=secret, + nonfinite=float("nan"), + hostile=Hostile(), + ) + bounded.append_audit_event(chr(0x1F512) * 2_048, message="ordinary") + lines = bounded.audit_log_path().read_bytes().splitlines() + self.assertEqual("[redacted]", json.loads(lines[0])["api_key"]) + self.assertIsNone(json.loads(lines[0])["nonfinite"]) + self.assertEqual("[unsupported]", json.loads(lines[0])["hostile"]) + self.assertNotIn(secret.encode(), b"\n".join(lines)) + self.assertNotIn(b"NaN", b"\n".join(lines)) + self.assertTrue(all(len(line) <= MAX_SANITIZED_JSON_BYTES for line in lines)) + self.assertTrue(json.loads(lines[1])["truncated"]) + + symlink_root = self.root / "symlink" + state_dir = symlink_root / "state" + state_dir.mkdir(parents=True) + external = symlink_root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("unchanged\n", encoding="utf-8") + mode_before = external.stat().st_mode + (state_dir / "logs").symlink_to(external, target_is_directory=True) + unsafe = BotLifecycleStore(SQLiteDatabase(state_dir / "zeus.db")) + + unsafe.append_audit_event("bot.test", message="fail open") + + self.assertEqual("unchanged\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(mode_before, external.stat().st_mode) + self.assertFalse((external / "audit.jsonl").exists()) + + def test_state_facade_forwards_all_eighteen_methods_with_exact_signatures(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + idempotency = MagicMock(spec=IdempotencyStore) + reconcile = MagicMock(spec=ReconcileStore) + delegate = MagicMock(spec=BotLifecycleStore) + record = _record(Path("profiles")) + event_input = _event("bot.test") + event = MagicMock(spec=LifecycleEvent) + history: dict[str, object] = { + "bot_id": "coder", + "events": [], + "next_before": None, + } + audit_path = Path("state") / "logs" / "audit.jsonl" + delegate.begin_lifecycle_intent.return_value = record + delegate.complete_lifecycle_intent.return_value = record + delegate.clear_stale_intent.return_value = record + delegate.audit_log_path.return_value = audit_path + void_sentinel = object() + delegate.append_audit_event.return_value = void_sentinel + delegate.upsert_bot.return_value = void_sentinel + delegate.update_status.return_value = void_sentinel + delegate.update_lifecycle_state.return_value = void_sentinel + delegate.update_restart_state.return_value = void_sentinel + delegate.upsert_bot_with_event.return_value = event + delegate.get_bot.return_value = record + delegate.list_bots.return_value = [record] + delegate.update_lifecycle_with_event.return_value = event + delegate.update_restart_with_event.return_value = event + delegate.delete_bot.return_value = True + delegate.delete_bot_with_event.return_value = True + delegate.list_lifecycle_events.return_value = [event] + delegate.history_payload.return_value = history + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=idempotency) as idempotency_type, + patch("zeus.state.ReconcileStore", return_value=reconcile) as reconcile_type, + patch("zeus.state.BotLifecycleStore", return_value=delegate) as lifecycle_type, + ): + facade = StateStore(database_path) + actual_begin = facade.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + request_id=None, + reason="begin", + ) + actual_complete = facade.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=1234, + source="cli", + outcome="success", + request_id=None, + reason="complete", + error_code=None, + error_message=None, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=None, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + actual_clear = facade.clear_stale_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + source="recovery", + reason="clear", + request_id=None, + ) + actual_audit_path = facade.audit_log_path() + actual_audit = methodcaller("append_audit_event", "bot.test", bot_id="coder")(facade) + actual_upsert = methodcaller("upsert_bot", record)(facade) + actual_created = facade.upsert_bot_with_event(record, event=event_input) + actual_record = facade.get_bot("coder") + actual_records = facade.list_bots() + actual_status = methodcaller( + "update_status", + "coder", + BotStatus.running, + 1234, + reset_restart=True, + )(facade) + actual_lifecycle_state = methodcaller( + "update_lifecycle_state", + "coder", + BotStatus.running, + 1234, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=0, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + )(facade) + actual_lifecycle = facade.update_lifecycle_with_event( + "coder", + BotStatus.running, + 1234, + event=event_input, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=0, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + actual_restart_state = methodcaller( + "update_restart_state", + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + )(facade) + actual_restart = facade.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + event=event_input, + ) + actual_delete = facade.delete_bot("coder") + actual_delete_event = facade.delete_bot_with_event("coder", event=event_input) + actual_events = facade.list_lifecycle_events("coder", 10, 20) + actual_history = facade.history_payload("coder", 10, 20) + + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], idempotency_type.call_args_list) + self.assertEqual([call(database)], reconcile_type.call_args_list) + self.assertEqual([call(database)], lifecycle_type.call_args_list) + database.connect.assert_not_called() + self.assertIs(database_path, facade.database_path) + self.assertIs(record, actual_begin) + self.assertIs(record, actual_complete) + self.assertIs(record, actual_clear) + self.assertIs(audit_path, actual_audit_path) + self.assertIsNone(actual_audit) + self.assertIsNone(actual_upsert) + self.assertIs(event, actual_created) + self.assertIs(record, actual_record) + self.assertIs(delegate.list_bots.return_value, actual_records) + self.assertIsNone(actual_status) + self.assertIsNone(actual_lifecycle_state) + self.assertIs(event, actual_lifecycle) + self.assertIsNone(actual_restart_state) + self.assertIs(event, actual_restart) + self.assertIs(actual_delete, True) + self.assertIs(actual_delete_event, True) + self.assertIs(delegate.list_lifecycle_events.return_value, actual_events) + self.assertIs(history, actual_history) + + delegate.begin_lifecycle_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + request_id=None, + reason="begin", + ) + delegate.complete_lifecycle_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=1234, + source="cli", + outcome="success", + request_id=None, + reason="complete", + error_code=None, + error_message=None, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=None, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + delegate.clear_stale_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + source="recovery", + reason="clear", + request_id=None, + ) + delegate.audit_log_path.assert_called_once_with() + delegate.append_audit_event.assert_called_once_with("bot.test", bot_id="coder") + delegate.upsert_bot.assert_called_once_with(record) + delegate.upsert_bot_with_event.assert_called_once_with(record, event=event_input) + delegate.get_bot.assert_called_once_with("coder") + delegate.list_bots.assert_called_once_with() + delegate.update_status.assert_called_once_with( + "coder", BotStatus.running, 1234, reset_restart=True + ) + lifecycle_kwargs = { + "started_at": FIXED_NOW, + "ready_at": FIXED_NOW, + "stopped_at": None, + "last_exit_code": 0, + "last_error": None, + "last_transition_reason": "ready", + "reset_restart": True, + "clear_ready_at": False, + "clear_stopped_at": True, + } + delegate.update_lifecycle_state.assert_called_once_with( + "coder", BotStatus.running, 1234, **lifecycle_kwargs + ) + delegate.update_lifecycle_with_event.assert_called_once_with( + "coder", + BotStatus.running, + 1234, + event=event_input, + **lifecycle_kwargs, + ) + delegate.update_restart_state.assert_called_once_with( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + ) + delegate.update_restart_with_event.assert_called_once_with( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + event=event_input, + ) + delegate.delete_bot.assert_called_once_with("coder") + delegate.delete_bot_with_event.assert_called_once_with("coder", event=event_input) + delegate.list_lifecycle_events.assert_called_once_with("coder", 10, 20) + delegate.history_payload.assert_called_once_with("coder", 10, 20) + for method_name in PUBLIC_METHODS: + self.assertEqual( + inspect.signature(getattr(BotLifecycleStore, method_name)), + inspect.signature(getattr(StateStore, method_name)), + ) + + def assert_exception_identity( + method_name: str, + invoke: Callable[[], object], + ) -> None: + error = RuntimeError(f"facade sentinel: {method_name}") + method = getattr(delegate, method_name) + method.side_effect = error + try: + with self.assertRaises(RuntimeError) as caught: + invoke() + self.assertIs(error, caught.exception) + finally: + method.side_effect = None + + exception_cases: tuple[tuple[str, Callable[[], object]], ...] = ( + ( + "begin_lifecycle_intent", + lambda: facade.begin_lifecycle_intent( + "other", + action="start", + operation_id="c" * 32, + source="cli", + ), + ), + ("audit_log_path", facade.audit_log_path), + ( + "append_audit_event", + lambda: facade.append_audit_event("bot.test", bot_id="other"), + ), + ( + "upsert_bot_with_event", + lambda: facade.upsert_bot_with_event(record, event=event_input), + ), + ("get_bot", lambda: facade.get_bot("other")), + ("list_bots", facade.list_bots), + ( + "update_status", + lambda: facade.update_status("other", BotStatus.failed), + ), + ("delete_bot", lambda: facade.delete_bot("other")), + ( + "list_lifecycle_events", + lambda: facade.list_lifecycle_events("other", 10, None), + ), + ( + "history_payload", + lambda: facade.history_payload("other", 10, None), + ), + ) + for method_name, invoke in exception_cases: + with self.subTest(exception=method_name): + assert_exception_identity(method_name, invoke) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index 6ed0137..fe4410d 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import errno import hashlib import json import os @@ -20,6 +21,7 @@ from unittest.mock import patch from zeus import models +from zeus.bot_lifecycle_store import BotLifecycleStore from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy from zeus.state import StateStore from zeus.supervisor import Supervisor @@ -699,6 +701,173 @@ def racing_open(path: object, flags: int, *args: object, **kwargs: object) -> in self.assertTrue(swapped) + def test_profile_chain_rejects_final_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_profile_chain + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / "hermes" / "profiles" / "coder" + profile.parent.mkdir(parents=True) + external = root / "external-profile" + external.mkdir() + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == profile.name and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + profile.symlink_to(external, target_is_directory=True) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaisesRegex(LaunchPayloadError, "appeared"), + ): + _open_profile_chain(profile) + + self.assertTrue(appeared) + self.assertTrue(profile.is_symlink()) + + def test_profile_chain_rejects_intermediate_symlink_appearing_during_missing_lookup( + self, + ) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_profile_chain + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes = root / "hermes" + hermes.mkdir() + profiles = hermes / "profiles" + profile = profiles / "coder" + external = root / "external-profiles" + (external / "coder").mkdir(parents=True) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == profiles.name and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + profiles.symlink_to(external, target_is_directory=True) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaisesRegex(LaunchPayloadError, "appeared"), + ): + _open_profile_chain(profile) + + self.assertTrue(appeared) + self.assertTrue(profiles.is_symlink()) + + def test_directory_open_closes_new_descriptor_when_post_open_fstat_fails(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_directory_at + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + child = root / "child" + child.mkdir() + parent_fd = os.open(root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + real_open = os.open + real_fstat = os.fstat + opened_fd: int | None = None + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal opened_fd + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + if path == child.name and dir_fd == parent_fd: + opened_fd = descriptor + return descriptor + + def failing_fstat(fd: int) -> os.stat_result: + if fd == opened_fd: + raise OSError(errno.EIO, "injected directory fstat failure") + return real_fstat(fd) + + try: + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.gateway_launcher.os.fstat", side_effect=failing_fstat), + self.assertRaises(LaunchPayloadError), + ): + _open_directory_at(parent_fd, child.name) + finally: + os.close(parent_fd) + + self.assertIsNotNone(opened_fd) + with self.assertRaises(OSError): + os.fstat(opened_fd) # type: ignore[arg-type] + + def test_marker_open_closes_new_descriptor_when_post_open_fstat_fails(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_regular_marker + + with tempfile.TemporaryDirectory() as tmp: + logs = Path(tmp) / "logs" + logs.mkdir() + marker = logs / "zeus-gateway.pid.json" + marker.write_text("{}", encoding="utf-8") + logs_fd = os.open(logs, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + real_open = os.open + real_fstat = os.fstat + opened_fd: int | None = None + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal opened_fd + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + if path == marker.name and dir_fd == logs_fd: + opened_fd = descriptor + return descriptor + + def failing_fstat(fd: int) -> os.stat_result: + if fd == opened_fd: + raise OSError(errno.EIO, "injected marker fstat failure") + return real_fstat(fd) + + try: + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.gateway_launcher.os.fstat", side_effect=failing_fstat), + self.assertRaises(LaunchPayloadError), + ): + _open_regular_marker(logs_fd) + finally: + os.close(logs_fd) + + self.assertIsNotNone(opened_fd) + with self.assertRaises(OSError): + os.fstat(opened_fd) # type: ignore[arg-type] + def test_ack_failure_removes_only_the_owned_marker(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -1308,7 +1477,9 @@ def test_intent_event_failure_rolls_back_projection(self) -> None: with ( patch.object( - store, "_insert_lifecycle_event", side_effect=sqlite3.DatabaseError("boom") + BotLifecycleStore, + "_insert_lifecycle_event", + side_effect=sqlite3.DatabaseError("boom"), ), self.assertRaisesRegex(sqlite3.DatabaseError, "boom"), ): @@ -1336,7 +1507,9 @@ def test_intent_completion_and_clear_failures_roll_back_projection(self) -> None with ( patch.object( - store, "_insert_lifecycle_event", side_effect=sqlite3.DatabaseError("boom") + BotLifecycleStore, + "_insert_lifecycle_event", + side_effect=sqlite3.DatabaseError("boom"), ), self.assertRaisesRegex(sqlite3.DatabaseError, "boom"), ): @@ -1475,6 +1648,120 @@ def _write_runtime_marker( marker_path.write_text(json.dumps(marker), encoding="utf-8") return marker_path + def test_strict_marker_final_profile_validation_enoent_is_untrusted(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + _store, supervisor = self._fixture(root) + record = _store.get_bot("coder") + assert record is not None + profile = Path(record.profile_path) + marker_path = self._write_runtime_marker( + supervisor, + record.profile_path, + self._runtime_marker(supervisor), + ) + displaced = root / "profile-displaced-during-validation" + profiles_stat = profile.parent.stat() + real_stat = os.stat + profile_observations = 0 + restored = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal profile_observations, restored + if path == profile.name and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == profiles_stat.st_dev + and parent.st_ino == profiles_stat.st_ino + ): + profile_observations += 1 + if profile_observations == 2: + profile.rename(displaced) + try: + return real_stat( + path, + dir_fd=dir_fd, + follow_symlinks=follow_symlinks, + ) + except FileNotFoundError: + displaced.rename(profile) + restored = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat): + observation = supervisor._read_strict_runtime_marker("coder", record.profile_path) + + self.assertEqual(2, profile_observations) + self.assertTrue(restored) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(profile.is_dir()) + self.assertTrue(marker_path.is_file()) + + def test_inspect_marker_final_profile_validation_enoent_is_unsafe(self) -> None: + from zeus.private_io import UnsafeFileError + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + _store, supervisor = self._fixture(root) + record = _store.get_bot("coder") + assert record is not None + profile = Path(record.profile_path) + marker_path = self._write_runtime_marker( + supervisor, + record.profile_path, + self._runtime_marker(supervisor), + ) + displaced = root / "profile-displaced-during-inspect" + profiles_stat = profile.parent.stat() + real_stat = os.stat + profile_observations = 0 + restored = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal profile_observations, restored + if path == profile.name and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == profiles_stat.st_dev + and parent.st_ino == profiles_stat.st_ino + ): + profile_observations += 1 + if profile_observations == 2: + profile.rename(displaced) + try: + return real_stat( + path, + dir_fd=dir_fd, + follow_symlinks=follow_symlinks, + ) + except FileNotFoundError: + displaced.rename(profile) + restored = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(record.profile_path) + + self.assertEqual(2, profile_observations) + self.assertTrue(restored) + self.assertTrue(profile.is_dir()) + self.assertTrue(marker_path.is_file()) + def _compat_marker( self, supervisor: Supervisor, @@ -1566,6 +1853,65 @@ def _begin_restart_over_running_gateway( ) return pending, marker_path + def test_start_and_stop_effects_observe_committed_lifecycle_intents(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store, supervisor = self._fixture(Path(tmp)) + observed: list[tuple[str | None, int, str]] = [] + + def spawn_after_intent(*args: object, **kwargs: object) -> _RecoveryPopen: + pending = store.get_bot("coder") + assert pending is not None + event = store.list_lifecycle_events("coder", limit=1, before=None)[0] + observed.append((pending.pending_action, pending.desired_revision, event.action)) + return _RecoveryPopen(*args, **kwargs) + + supervisor.popen_factory = spawn_after_intent + _RecoveryPopen.launches.clear() + + started = supervisor.start("coder") + + self.assertEqual(BotStatus.running, started.status) + self.assertEqual([("start", 1, "bot.start.intent")], observed) + + with tempfile.TemporaryDirectory() as tmp: + alive = True + store, supervisor = self._fixture( + Path(tmp), + desired=DesiredState.running, + pid_alive_fn=lambda pid: alive, + ) + record = store.get_bot("coder") + assert record is not None + running = replace( + record, + status=BotStatus.running, + pid=4321, + desired_revision=1, + ) + store.upsert_bot(running) + self._write_runtime_marker( + supervisor, + running.profile_path, + self._runtime_marker(supervisor, desired_revision=1), + ) + observed = [] + + def signal_after_intent(pid: int, sent_signal: object) -> None: + nonlocal alive + pending = store.get_bot("coder") + assert pending is not None + event = store.list_lifecycle_events("coder", limit=1, before=None)[0] + observed.append((pending.pending_action, pending.desired_revision, event.action)) + alive = False + + supervisor.kill_fn = signal_after_intent + supervisor.stop_grace_seconds = 0 + + stopped = supervisor.stop("coder") + + self.assertEqual(BotStatus.stopped, stopped.status) + self.assertEqual([("stop", 2, "bot.stop.intent")], observed) + def test_pending_compat_restart_fails_closed_without_side_effects(self) -> None: for schema in (2, None): for alive in (True, False): @@ -1752,7 +2098,8 @@ def test_status_never_spawns_to_enforce_desired_running(self) -> None: def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> None: with tempfile.TemporaryDirectory() as tmp: - store, supervisor = self._fixture(Path(tmp)) + root = Path(tmp) + store, _interrupted_supervisor = self._fixture(root) _RecoveryPopen.launches.clear() pending = store.begin_lifecycle_intent( "coder", @@ -1761,6 +2108,27 @@ def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> N source="cli", ) + hermes = root / "bin" / "hermes" + hermes_root = root / "hermes" + recovered_store = StateStore(root / "zeus.db") + recovered_store.init() + supervisor = Supervisor( + recovered_store, + str(hermes), + hermes_root, + popen_factory=_RecoveryPopen, + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [ + str(hermes.resolve()), + "-p", + "coder", + "gateway", + "run", + ], + proc_start_fingerprint_reader=lambda pid: "test-start:4321", + startup_grace_seconds=0, + ) + result = supervisor.reconcile("coder")[0] self.assertEqual(BotStatus.running, result.status) @@ -1768,7 +2136,7 @@ def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> N marker = _RecoveryPopen.launches[0]["marker"] self.assertEqual("a" * 32, marker["operation_id"]) self.assertEqual(pending.desired_revision, marker["desired_revision"]) - loaded = store.get_bot("coder") + loaded = recovered_store.get_bot("coder") assert loaded is not None self.assertIsNone(loaded.pending_operation_id) @@ -3065,7 +3433,14 @@ def wait(self, timeout: float) -> None: raise subprocess.TimeoutExpired("hermes", timeout) with tempfile.TemporaryDirectory() as tmp: - fingerprints = iter(["test-start:4321", "test-start:4321", "test-start:reused"]) + fingerprints = iter( + [ + "test-start:4321", + "test-start:4321", + "test-start:4321", + "test-start:reused", + ] + ) store, supervisor = self._fixture( Path(tmp), desired=DesiredState.running, diff --git a/tests/test_fresh_vps_script.py b/tests/test_fresh_vps_script.py new file mode 100644 index 0000000..47db09c --- /dev/null +++ b/tests/test_fresh_vps_script.py @@ -0,0 +1,437 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "fresh_vps_verify.sh" +INSTALLER_URL = "https://hermes-agent.nousresearch.com/install.sh" + + +class FreshVPSVerifierTests(unittest.TestCase): + def setUp(self) -> None: + self._temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self._temporary_directory.cleanup) + self.root = Path(self._temporary_directory.name) + self.checkout = self.root / "checkout" + self.checkout.mkdir() + (self.checkout / "scripts").mkdir() + (self.checkout / "zeus").mkdir() + shutil.copy2(SCRIPT, self.checkout / "scripts" / SCRIPT.name) + (self.checkout / "pyproject.toml").write_text("[project]\nname='test'\n") + (self.checkout / "zeus" / "cli.py").write_text("# verifier fixture\n") + + self.home = self.root / "home" + self.home.mkdir() + self.stub_bin = self.root / "bin" + self.stub_bin.mkdir() + self.curl_capture = self.root / "curl.jsonl" + self.apt_capture = self.root / "apt.log" + self.installer_executed = self.root / "installer-executed" + self.fake_installer = self.root / "fake-hermes-installer.sh" + self.fake_installer.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -eu", + 'mkdir -p "$HOME/.local/bin"', + "cat > \"$HOME/.local/bin/hermes\" <<'HERMES'", + "#!/usr/bin/env bash", + "exit 0", + "HERMES", + 'chmod 700 "$HOME/.local/bin/hermes"', + "printf 'executed\\n' > \"$INSTALLER_EXECUTED\"", + "", + ] + ), + encoding="utf-8", + ) + self.installer_digest = hashlib.sha256(self.fake_installer.read_bytes()).hexdigest() + self._write_stubs() + + def _write_executable(self, name: str, content: str) -> None: + path = self.stub_bin / name + path.write_text(content, encoding="utf-8") + path.chmod(0o700) + + def _write_stubs(self) -> None: + python_stub = textwrap.dedent( + f"""\ + #!/usr/bin/env bash + set -eu + real_python={str(Path(sys.executable))!r} + if [ "${{1:-}}" = "-I" ]; then + exec "$real_python" "$@" + fi + if [ "${{1:-}}" = "-B" ] && [ "${{2:-}}" = "-c" ]; then + exec "$real_python" "$@" + fi + if [ "${{1:-}}" = "-m" ] && [ "${{2:-}}" = "venv" ]; then + mkdir -p "$3/bin" + : > "$3/bin/activate" + exit 0 + fi + if [ "${{1:-}}" = "--version" ]; then + printf 'Python fixture\n' + fi + exit 0 + """ + ) + self._write_executable("python3", python_stub) + self._write_executable("python", python_stub) + self._write_executable( + "sh", + "#!/usr/bin/env bash\nset -eu\nexit 0\n", + ) + + curl_stub = textwrap.dedent( + f"""\ + #!{sys.executable} + import json + import os + from pathlib import Path + import shutil + import sys + + args = sys.argv[1:] + config_stdin = sys.stdin.read() if "--config" in args and "-" in args else "" + capture = Path(os.environ["CURL_CAPTURE"]) + with capture.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({{"argv": args, "stdin": config_stdin}}) + "\\n") + + if {INSTALLER_URL!r} in args: + source = Path(os.environ["FAKE_INSTALLER_SOURCE"]) + if "-o" in args: + output = Path(args[args.index("-o") + 1]) + shutil.copyfile(source, output) + else: + sys.stdout.buffer.write(source.read_bytes()) + else: + print('{{"status":"ok"}}') + """ + ) + self._write_executable("curl", curl_stub) + + def _base_environment(self) -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.stub_bin}:/usr/bin:/bin", + "HOME": str(self.home), + "CURL_CAPTURE": str(self.curl_capture), + "FAKE_INSTALLER_SOURCE": str(self.fake_installer), + "INSTALLER_EXECUTED": str(self.installer_executed), + "ZEUS_VPS_INSTALL_HERMES": "1", + "ZEUS_VPS_LOG_DIR": "evidence", + "ZEUS_VPS_VENV_DIR": ".fixture-venv", + "ZEUS_VPS_MULTI_STATE_DIR": ".fixture-multi", + "ZEUS_VPS_API_STATE_DIR": ".fixture-api", + } + ) + environment.pop("ZEUS_VPS_API_KEY", None) + environment.pop("ZEUS_VPS_HERMES_INSTALLER_SHA256", None) + return environment + + def _pythonless_bootstrap_path(self) -> str: + bootstrap_bin = self.root / "bootstrap-bin" + bootstrap_bin.mkdir() + for name in ("curl", "sh"): + (bootstrap_bin / name).symlink_to(self.stub_bin / name) + for name in ( + "bash", + "cat", + "chmod", + "date", + "dirname", + "env", + "ln", + "mkdir", + "rm", + "seq", + "sleep", + "tee", + "uname", + ): + executable = shutil.which(name) + if executable is None: + self.fail(f"required fixture executable is unavailable: {name}") + (bootstrap_bin / name).symlink_to(executable) + + sudo = bootstrap_bin / "sudo" + sudo.write_text('#!/bin/bash\nset -eu\nexec "$@"\n', encoding="utf-8") + sudo.chmod(0o700) + apt_get = bootstrap_bin / "apt-get" + apt_get.write_text( + "\n".join( + [ + "#!/bin/bash", + "set -eu", + 'printf \'%s\\n\' "$*" >> "$APT_CAPTURE"', + 'if [ "${1:-}" = "install" ]; then', + ' ln -sf "$PYTHON_STUB_SOURCE" "$BOOTSTRAP_BIN/python3"', + ' ln -sf "$PYTHON_STUB_SOURCE" "$BOOTSTRAP_BIN/python"', + "fi", + "", + ] + ), + encoding="utf-8", + ) + apt_get.chmod(0o700) + return str(bootstrap_bin) + + def _run( + self, + *, + timeout: float = 20, + **overrides: str, + ) -> subprocess.CompletedProcess[str]: + environment = self._base_environment() + environment.update(overrides) + arguments = ["/bin/bash", "scripts/fresh_vps_verify.sh"] + process = subprocess.Popen( + arguments, + cwd=self.checkout, + env=environment, + text=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate("", timeout=timeout) + except subprocess.TimeoutExpired as error: + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + error.cmd, + error.timeout, + output=stdout, + stderr=stderr, + ) from None + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + + def _curl_records(self) -> list[dict[str, object]]: + if not self.curl_capture.exists(): + return [] + return [ + json.loads(line) for line in self.curl_capture.read_text(encoding="utf-8").splitlines() + ] + + def test_missing_installer_digest_fails_before_download(self) -> None: + result = self._run() + + self.assertNotEqual(0, result.returncode) + self.assertIn("ZEUS_VPS_HERMES_INSTALLER_SHA256", result.stdout + result.stderr) + self.assertFalse(self.installer_executed.exists()) + self.assertFalse(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_pythonless_bootstrap_path_contains_only_explicit_stubs(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + + self.assertEqual([str(self.root / "bootstrap-bin")], bootstrap_path.split(os.pathsep)) + self.assertIsNone(shutil.which("python3", path=bootstrap_path)) + + def test_missing_python_is_bootstrapped_before_private_evidence(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + result = self._run( + PATH=bootstrap_path, + APT_CAPTURE=str(self.apt_capture), + BOOTSTRAP_BIN=bootstrap_path.removesuffix(":/bin"), + PYTHON_STUB_SOURCE=str(self.stub_bin / "python3"), + ZEUS_VPS_INSTALL_PACKAGES="1", + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="bootstrap-test-key", + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("Bootstrapping the Python 3 prerequisite", result.stdout) + apt_commands = self.apt_capture.read_text(encoding="utf-8").splitlines() + self.assertIn("install -y python3", apt_commands) + self.assertTrue(any("python3-venv" in command for command in apt_commands)) + self.assertEqual(0o700, stat.S_IMODE((self.checkout / "evidence").stat().st_mode)) + + def test_missing_python_without_package_bootstrap_fails_deterministically(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + result = self._run( + PATH=bootstrap_path, + APT_CAPTURE=str(self.apt_capture), + BOOTSTRAP_BIN=bootstrap_path.removesuffix(":/bin"), + PYTHON_STUB_SOURCE=str(self.stub_bin / "python3"), + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("Python 3 is required", result.stdout + result.stderr) + self.assertFalse((self.checkout / "evidence").exists()) + + def test_malformed_installer_digest_fails_before_download(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256="g" * 64) + + self.assertNotEqual(0, result.returncode) + self.assertIn("64", result.stdout + result.stderr) + self.assertFalse(self.installer_executed.exists()) + self.assertFalse(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_mismatched_installer_digest_never_executes(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256="0" * 64) + + self.assertNotEqual(0, result.returncode) + self.assertIn("mismatch", (result.stdout + result.stderr).lower()) + self.assertFalse(self.installer_executed.exists()) + self.assertTrue(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_matching_digest_executes_and_sensitive_curl_uses_stdin(self) -> None: + api_key = 'sentinel-api-key-"quote\\nslash' + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY=api_key, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertTrue(self.installer_executed.exists()) + transcript = result.stdout + result.stderr + run_log = (self.checkout / "evidence" / "run.log").read_text(encoding="utf-8") + self.assertNotIn(api_key, transcript) + self.assertNotIn(api_key, run_log) + self.assertIn("authenticated API request [redacted]", transcript) + + records = self._curl_records() + argv_text = "\n".join(" ".join(record["argv"]) for record in records) + self.assertNotIn(api_key, argv_text) + authenticated = [record for record in records if record["stdin"]] + self.assertEqual(2, len(authenticated)) + escaped = api_key.replace("\\", "\\\\").replace('"', '\\"') + for record in authenticated: + self.assertIn("--config", record["argv"]) + self.assertNotIn("-H", record["argv"]) + self.assertIn(f'header = "x-zeus-api-key: {escaped}"', record["stdin"]) + + evidence = self.checkout / "evidence" + self.assertEqual(0o700, stat.S_IMODE(evidence.stat().st_mode)) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + with self.subTest(name=name): + self.assertEqual(0o600, stat.S_IMODE((evidence / name).stat().st_mode)) + + def test_existing_evidence_permissions_are_repaired(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir() + evidence.chmod(0o777) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + path = evidence / name + path.write_text("old evidence\n", encoding="utf-8") + path.chmod(0o666) + + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="permission-test-key", + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertEqual(0o700, stat.S_IMODE(evidence.stat().st_mode)) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + with self.subTest(name=name): + self.assertEqual(0o600, stat.S_IMODE((evidence / name).stat().st_mode)) + + def test_async_prompt_uses_a_fixed_sensitive_transcript_label(self) -> None: + prompt = "SENTINEL_ASYNC_PROMPT_must_not_be_logged" + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="async-test-key", + ZEUS_VPS_ASYNC_PROMPT=prompt, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + transcript = result.stdout + result.stderr + run_log = (self.checkout / "evidence" / "run.log").read_text(encoding="utf-8") + self.assertIn("Hermes async prompt [redacted]", transcript) + self.assertNotIn(prompt, transcript) + self.assertNotIn(prompt, run_log) + + def test_default_api_key_is_ephemeral_and_generated_by_secrets(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + authenticated = [record for record in self._curl_records() if record["stdin"]] + self.assertEqual(2, len(authenticated)) + headers = [] + for record in authenticated: + for line in str(record["stdin"]).splitlines(): + if line.startswith('header = "x-zeus-api-key: '): + header = line.removeprefix('header = "x-zeus-api-key: ') + headers.append(header.removesuffix('"')) + self.assertEqual(2, len(headers)) + self.assertEqual(headers[0], headers[1]) + self.assertRegex(headers[0], r"^[0-9a-f]{64}$") + self.assertNotIn(headers[0], result.stdout + result.stderr) + + def test_api_key_with_line_breaks_is_rejected_without_disclosure(self) -> None: + api_key = "sentinel\r\ninjected-header" + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY=api_key, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("line break", (result.stdout + result.stderr).lower()) + self.assertNotIn(api_key, result.stdout + result.stderr) + self.assertFalse(any(record["stdin"] for record in self._curl_records())) + + def test_symlinked_evidence_directory_is_rejected_without_mutating_target(self) -> None: + external = self.root / "external" + external.mkdir(mode=0o755) + (self.checkout / "evidence").symlink_to(external, target_is_directory=True) + + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertNotEqual(0, result.returncode) + self.assertEqual([], list(external.iterdir())) + self.assertEqual(0o755, stat.S_IMODE(external.stat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + + def test_symlinked_evidence_file_is_rejected_without_mutating_target(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir(mode=0o700) + external = self.root / "external.log" + external.write_text("preserve me\n", encoding="utf-8") + external.chmod(0o644) + (evidence / "run.log").symlink_to(external) + + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("preserve me\n", external.read_text(encoding="utf-8")) + self.assertEqual(0o644, stat.S_IMODE(external.stat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + + def test_fifo_evidence_file_is_rejected_without_blocking(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir(mode=0o700) + fifo = evidence / "run.log" + os.mkfifo(fifo, mode=0o600) + + try: + result = self._run( + timeout=1.0, + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ) + except subprocess.TimeoutExpired: + self.fail("verifier blocked while opening a pre-existing evidence FIFO") + + self.assertNotEqual(0, result.returncode) + self.assertTrue(stat.S_ISFIFO(fifo.lstat().st_mode)) + self.assertEqual(0o600, stat.S_IMODE(fifo.lstat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_gateway_marker.py b/tests/test_gateway_marker.py new file mode 100644 index 0000000..f7de218 --- /dev/null +++ b/tests/test_gateway_marker.py @@ -0,0 +1,549 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import unittest +from dataclasses import FrozenInstanceError +from typing import Any + +from zeus.gateway_marker import ( + GatewayGeneration, + GatewayLaunchMarker, + GatewayRuntimeMarker, + MarkerValidationError, + command_fingerprint, + is_compat_runtime_marker, + is_owned_runtime_marker, + parse_launch_marker, + parse_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.readiness import ReadinessProbe + + +def _probe_payload() -> dict[str, object]: + return { + "url": "http://127.0.0.1:4312/health", + "expected_status": "ok", + "expected_platform": "hermes", + "timeout_seconds": 10, + "interval_seconds": 0.25, + } + + +def _launch_payload(*, probe: object = ...) -> dict[str, object]: + argv = ["/opt/hermes/bin/hermes", "-p", "coder", "gateway", "run"] + return { + "schema": 3, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "operation_id": "a" * 32, + "desired_revision": 7, + "argv": argv, + "resolved_hermes_bin": argv[0], + "command_fingerprint": command_fingerprint(argv), + "readiness_probe": _probe_payload() if probe is ... else probe, + } + + +def _runtime_payload(*, include_start_fingerprint: bool = True) -> dict[str, object]: + payload = _launch_payload() + payload.update({"pid": 4321, "started_at": 1_780_000_000}) + if include_start_fingerprint: + payload["proc_start_fingerprint"] = "linux:/proc-starttime:987654321" + return payload + + +class GatewayMarkerTests(unittest.TestCase): + def assert_marker_error(self, expected: str, payload: object, *, runtime: bool = False) -> None: + parser = parse_runtime_marker if runtime else parse_launch_marker + with self.assertRaisesRegex(MarkerValidationError, f"^{expected}$"): + parser(payload) + + def test_launch_and_runtime_round_trip_as_immutable_models(self) -> None: + launch_payload = _launch_payload() + expected_launch_payload = _launch_payload() + launch = parse_launch_marker(launch_payload) + + self.assertIsInstance(launch, GatewayLaunchMarker) + self.assertIs(type(launch.argv), tuple) + self.assertEqual(launch_payload, launch.to_payload()) + with self.assertRaises(FrozenInstanceError): + launch.bot_id = "other" # type: ignore[misc] + + source_argv = launch_payload["argv"] + source_probe = launch_payload["readiness_probe"] + assert isinstance(source_argv, list) + assert isinstance(source_probe, dict) + source_argv[0] = "/tmp/source-was-mutated" + source_probe["url"] = "http://localhost:1/source-was-mutated" + self.assertEqual(expected_launch_payload, launch.to_payload()) + + first = launch.to_payload() + second = launch.to_payload() + self.assertIsNot(first, second) + self.assertIsNot(first["argv"], second["argv"]) + self.assertIsNot(first["readiness_probe"], second["readiness_probe"]) + first_argv = first["argv"] + first_probe = first["readiness_probe"] + assert isinstance(first_argv, list) + assert isinstance(first_probe, dict) + first_argv[0] = "/tmp/replaced" + first_probe["url"] = "http://localhost:1/replaced" + self.assertEqual(expected_launch_payload, launch.to_payload()) + + runtime_payload = _runtime_payload() + runtime = parse_runtime_marker(runtime_payload) + self.assertIsInstance(runtime, GatewayRuntimeMarker) + self.assertEqual(runtime_payload, runtime.to_payload()) + self.assertEqual( + GatewayGeneration( + operation_id="a" * 32, + desired_revision=7, + pid=4321, + command_fingerprint=str(runtime_payload["command_fingerprint"]), + proc_start_fingerprint="linux:/proc-starttime:987654321", + ), + runtime.generation(), + ) + + without_start = _runtime_payload(include_start_fingerprint=False) + parsed_without_start = parse_runtime_marker(without_start) + self.assertEqual(without_start, parsed_without_start.to_payload()) + self.assertIsNone(parsed_without_start.generation().proc_start_fingerprint) + + def test_command_fingerprint_uses_canonical_utf8_json(self) -> None: + argv = ["/opt/hérmes", "-p", "coder", "gateway", "run"] + expected = hashlib.sha256( + json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + self.assertEqual(expected, command_fingerprint(argv)) + + def test_parsers_require_exact_launch_runtime_and_probe_key_sets(self) -> None: + self.assert_marker_error("marker must be an object", []) + with self.assertRaises(MarkerValidationError): + parse_runtime_marker([]) + + launch = _launch_payload() + for key in tuple(launch): + invalid = dict(launch) + del invalid[key] + with self.subTest(kind="launch-missing", key=key): + self.assert_marker_error("marker has invalid keys", invalid) + with_extra = dict(launch, extra=True) + self.assert_marker_error("marker has invalid keys", with_extra) + + runtime = _runtime_payload() + for key in tuple(runtime): + invalid = dict(runtime) + del invalid[key] + if key == "proc_start_fingerprint": + self.assertEqual( + _runtime_payload(include_start_fingerprint=False), + parse_runtime_marker(invalid).to_payload(), + ) + else: + with self.subTest(kind="runtime-missing", key=key): + self.assert_marker_error( + "runtime marker has invalid keys", invalid, runtime=True + ) + self.assert_marker_error( + "runtime marker has invalid keys", dict(runtime, extra=True), runtime=True + ) + + invalid_probe = _launch_payload() + probe = _probe_payload() + del probe["interval_seconds"] + invalid_probe["readiness_probe"] = probe + self.assert_marker_error("readiness_probe has invalid keys", invalid_probe) + + def test_launch_parser_rejects_invalid_intent_correlation_and_bool_integers(self) -> None: + cases: tuple[tuple[str, str, object], ...] = ( + ("schema-bool", "schema", True), + ("schema-old", "schema", 2), + ("component", "component", "worker"), + ("action", "action", "stop"), + ("operation-uppercase", "operation_id", "A" * 32), + ("operation-length", "operation_id", "a" * 31), + ("revision-bool", "desired_revision", True), + ("revision-zero", "desired_revision", 0), + ("revision-large", "desired_revision", 2**63), + ) + expected = { + "schema-bool": "marker schema is invalid", + "schema-old": "marker schema is invalid", + "component": "marker command intent is invalid", + "action": "marker command intent is invalid", + "operation-uppercase": "operation_id is invalid", + "operation-length": "operation_id is invalid", + "revision-bool": "desired_revision is invalid", + "revision-zero": "desired_revision is invalid", + "revision-large": "desired_revision is invalid", + } + for name, key, value in cases: + payload = _launch_payload() + payload[key] = value + with self.subTest(name=name): + self.assert_marker_error(expected[name], payload) + + def test_runtime_parser_rejects_invalid_pid_time_and_start_fingerprint(self) -> None: + cases: tuple[tuple[str, str, object, str], ...] = ( + ("pid-bool", "pid", True, "marker PID is invalid"), + ("pid-zero", "pid", 0, "marker PID is invalid"), + ("started-bool", "started_at", True, "marker started_at is invalid"), + ("started-zero", "started_at", 0, "marker started_at is invalid"), + ("started-nan", "started_at", math.nan, "marker started_at is invalid"), + ("started-inf", "started_at", math.inf, "marker started_at is invalid"), + ( + "start-null", + "proc_start_fingerprint", + None, + "process start fingerprint is invalid", + ), + ( + "start-non-string", + "proc_start_fingerprint", + 123, + "process start fingerprint is invalid", + ), + ( + "start-empty", + "proc_start_fingerprint", + "", + "process start fingerprint is invalid", + ), + ( + "start-long", + "proc_start_fingerprint", + "x" * 513, + "process start fingerprint is invalid", + ), + ) + for name, key, value, error in cases: + payload = _runtime_payload() + payload[key] = value + with self.subTest(name=name): + self.assert_marker_error(error, payload, runtime=True) + + def test_parser_rejects_invalid_argv_path_and_fingerprint(self) -> None: + cases: list[tuple[str, dict[str, object], str]] = [] + + not_a_list = _launch_payload() + not_a_list["argv"] = tuple(not_a_list["argv"]) # type: ignore[arg-type] + cases.append(("argv-type", not_a_list, "argv must be a bounded non-empty list")) + + empty_argv = _launch_payload() + empty_argv["argv"] = [] + cases.append(("argv-empty", empty_argv, "argv must be a bounded non-empty list")) + + too_many_parts = _launch_payload() + too_many_parts["argv"] = ["x"] * 65 + cases.append(("argv-too-many", too_many_parts, "argv must be a bounded non-empty list")) + + empty_part = _launch_payload() + empty_part["argv"] = ["/opt/hermes/bin/hermes", "", "coder", "gateway", "run"] + cases.append( + ("argv-empty-part", empty_part, "argv item must be a bounded non-empty string") + ) + + nul_part = _launch_payload() + nul_part["argv"] = ["/opt/hermes/bin/hermes\0", "-p", "coder", "gateway", "run"] + cases.append(("argv-nul-part", nul_part, "argv item must be a bounded non-empty string")) + + oversized_part = _launch_payload() + oversized_part["argv"] = ["x" * (16 * 1024 + 1)] + cases.append( + ( + "argv-oversized-part", + oversized_part, + "argv item must be a bounded non-empty string", + ) + ) + + oversized_argv = _launch_payload() + oversized_argv["argv"] = ["x" * 14_000] * 5 + cases.append(("argv-total-bytes", oversized_argv, "argv is too large")) + + wrong_command = _launch_payload() + wrong_command["argv"] = ["/opt/hermes/bin/hermes", "-p", "coder", "gateway", "stop"] + cases.append(("argv-command", wrong_command, "argv is not a Hermes gateway command")) + + wrong_profile = _launch_payload() + wrong_profile["argv"] = [ + "/opt/hermes/bin/hermes", + "-p", + "other", + "gateway", + "run", + ] + cases.append(("argv-profile", wrong_profile, "argv is not a Hermes gateway command")) + + non_absolute = _launch_payload() + non_absolute["resolved_hermes_bin"] = "hermes" + cases.append( + ( + "relative-path", + non_absolute, + "resolved_hermes_bin must be an absolute path without traversal", + ) + ) + + traversal = _launch_payload() + traversal["resolved_hermes_bin"] = "/opt/hermes/../bin/hermes" + cases.append( + ( + "traversal", + traversal, + "resolved_hermes_bin must be an absolute path without traversal", + ) + ) + + mismatched_path = _launch_payload() + mismatched_path["resolved_hermes_bin"] = "/usr/bin/hermes" + cases.append( + ( + "path-mismatch", + mismatched_path, + "exec argv does not use the resolved Hermes binary", + ) + ) + + invalid_fingerprint = _launch_payload() + invalid_fingerprint["command_fingerprint"] = "f" * 64 + cases.append( + ("fingerprint-mismatch", invalid_fingerprint, "command fingerprint is invalid") + ) + + for name, payload, error in cases: + with self.subTest(name=name): + self.assert_marker_error(error, payload) + + def test_resolved_path_comparison_preserves_baseline_lexical_normalization(self) -> None: + for resolved_path in ( + "/opt//hermes/bin/hermes", + "/opt/./hermes/bin/hermes", + "/opt/hermes/bin/hermes/", + ): + with self.subTest(kind="canonical-argv", resolved_path=resolved_path): + payload = _launch_payload() + payload["resolved_hermes_bin"] = resolved_path + parsed = parse_launch_marker(payload) + self.assertEqual(resolved_path, parsed.resolved_hermes_bin) + self.assertEqual(resolved_path, parsed.to_payload()["resolved_hermes_bin"]) + + with self.subTest(kind="noncanonical-argv", resolved_path=resolved_path): + payload = _launch_payload() + argv = payload["argv"] + assert isinstance(argv, list) + argv[0] = resolved_path + payload["resolved_hermes_bin"] = resolved_path + payload["command_fingerprint"] = command_fingerprint(argv) + self.assert_marker_error( + "exec argv does not use the resolved Hermes binary", payload + ) + + def test_readiness_conversion_round_trips_and_validates_the_contract(self) -> None: + payload = _probe_payload() + probe = readiness_probe_from_payload(payload) + + self.assertIsInstance(probe, ReadinessProbe) + self.assertEqual(payload, readiness_probe_to_payload(probe)) + self.assertIsNone(readiness_probe_from_payload(None)) + self.assertIsNone(readiness_probe_to_payload(None)) + + invalid_cases: tuple[tuple[str, object], ...] = ( + ("non-object", []), + ("https", dict(payload, url="https://127.0.0.1:4312/health")), + ("remote", dict(payload, url="http://example.com/health")), + ("credentials", dict(payload, url="http://user@localhost:4312/health")), + ("query", dict(payload, url="http://localhost:4312/health?secret=yes")), + ("fragment", dict(payload, url="http://localhost:4312/health#fragment")), + ("empty-status", dict(payload, expected_status="")), + ("empty-platform", dict(payload, expected_platform="")), + ("bool-timeout", dict(payload, timeout_seconds=True)), + ("zero-interval", dict(payload, interval_seconds=0)), + ("nan-timeout", dict(payload, timeout_seconds=math.nan)), + ("infinite-timeout", dict(payload, timeout_seconds=math.inf)), + ("large-timeout", dict(payload, timeout_seconds=3600.1)), + ) + for name, invalid in invalid_cases: + with self.subTest(name=name), self.assertRaises(MarkerValidationError): + readiness_probe_from_payload(invalid) + + strict_cases = ( + ("url-nul", "url", "http://localhost:4312/health\0", "readiness URL"), + ("url-long", "url", "http://localhost/" + "x" * 2049, "readiness URL"), + ("status-long", "expected_status", "x" * 129, "expected status"), + ("platform-nul", "expected_platform", "hermes\0agent", "expected platform"), + ) + for name, key, value, error_prefix in strict_cases: + marker_payload = _launch_payload() + marker_probe = marker_payload["readiness_probe"] + assert isinstance(marker_probe, dict) + marker_probe[key] = value + with ( + self.subTest(name=name), + self.assertRaisesRegex( + MarkerValidationError, f"^{error_prefix} must be a bounded non-empty string$" + ), + ): + parse_launch_marker(marker_payload) + + def test_strict_parsers_reject_compat_markers_while_recognizer_accepts_them(self) -> None: + schema2 = _runtime_payload() + schema2["schema"] = 2 + schemaless = _runtime_payload() + del schemaless["schema"] + explicit_null = _runtime_payload() + explicit_null["schema"] = None + + for name, payload in ( + ("schema2", schema2), + ("schemaless", schemaless), + ("explicit-null", explicit_null), + ): + with self.subTest(name=name): + with self.assertRaises(MarkerValidationError): + parse_runtime_marker(payload) + self.assertTrue(is_compat_runtime_marker(payload)) + + self.assertFalse(is_compat_runtime_marker(_runtime_payload())) + self.assertFalse(is_compat_runtime_marker([])) + self.assertFalse(is_compat_runtime_marker({"schema": True})) + + def test_owned_runtime_check_matches_full_schema_and_generation(self) -> None: + payload = _runtime_payload() + expected: dict[str, Any] = { + "bot_id": "coder", + "operation_id": "a" * 32, + "desired_revision": 7, + "pid": 4321, + "expected_fingerprint": payload["command_fingerprint"], + } + self.assertTrue(is_owned_runtime_marker(payload, **expected)) + + mismatches = { + "bot_id": "other", + "operation_id": "b" * 32, + "desired_revision": 8, + "pid": 4322, + "expected_fingerprint": "f" * 64, + } + for key, value in mismatches.items(): + call = dict(expected) + call[key] = value + with self.subTest(key=key): + self.assertFalse(is_owned_runtime_marker(payload, **call)) + + malformed = dict(payload, started_at=math.nan) + self.assertFalse(is_owned_runtime_marker(malformed, **expected)) + + def test_gateway_launcher_keeps_fingerprint_and_ownership_compatibility_exports(self) -> None: + from zeus.gateway_launcher import ( + _is_owned_runtime_marker as launcher_is_owned_runtime_marker, + ) + from zeus.gateway_launcher import command_fingerprint as launcher_command_fingerprint + + self.assertIs(command_fingerprint, launcher_command_fingerprint) + self.assertEqual( + inspect.signature(is_owned_runtime_marker), + inspect.signature(launcher_is_owned_runtime_marker), + ) + payload = _runtime_payload() + arguments: dict[str, Any] = { + "bot_id": "coder", + "operation_id": "a" * 32, + "desired_revision": 7, + "pid": 4321, + "expected_fingerprint": str(payload["command_fingerprint"]), + } + self.assertEqual( + is_owned_runtime_marker(payload, **arguments), + launcher_is_owned_runtime_marker(payload, **arguments), + ) + + def test_launcher_preserves_marker_bytes_and_validation_precedence(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _validate_payload + + marker = _launch_payload() + marker_argv = marker["argv"] + assert isinstance(marker_argv, list) + root_payload: dict[str, object] = { + "profile_path": "/opt/hermes/profiles/coder", + "marker_path": "/opt/hermes/profiles/coder/logs/zeus-gateway.pid.json", + "marker": marker, + "argv": list(marker_argv), + "env": {"HERMES_HOME": "/opt/hermes", "PATH": "/usr/bin"}, + } + _profile, validated_marker, _argv, _env = _validate_payload(root_payload) + self.assertEqual(marker, validated_marker) + runtime_marker = dict(validated_marker) + runtime_marker.update( + { + "pid": 4321, + "started_at": 1_780_000_000, + "proc_start_fingerprint": "linux:/proc-starttime:123", + } + ) + marker_bytes = ( + json.dumps(runtime_marker, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + self.assertEqual( + b'{"action":"run","argv":["/opt/hermes/bin/hermes","-p","coder",' + b'"gateway","run"],"bot_id":"coder","command_fingerprint":' + b'"7d203f8c4831e34eccfed055c8a8b82f9d68601ed15135a68dfde76c32a55321",' + b'"component":"gateway","desired_revision":7,"operation_id":' + b'"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","pid":4321,"proc_start_fingerprint":' + b'"linux:/proc-starttime:123","readiness_probe":{"expected_platform":' + b'"hermes","expected_status":"ok","interval_seconds":0.25,' + b'"timeout_seconds":10,"url":"http://127.0.0.1:4312/health"},' + b'"resolved_hermes_bin":"/opt/hermes/bin/hermes","schema":3,' + b'"started_at":1780000000}\n', + marker_bytes, + ) + self.assertNotIn(b"private", marker_bytes) + + boundary_first: Any = json.loads(json.dumps(root_payload)) + boundary_first["profile_path"] = "/opt/hermes/profiles/other" + boundary_first["marker_path"] = "/opt/hermes/profiles/other/logs/zeus-gateway.pid.json" + boundary_first["marker"]["schema"] = 2 + with self.assertRaisesRegex( + LaunchPayloadError, "^profile_path is outside the bot profile boundary$" + ): + _validate_payload(boundary_first) + + mismatch_first: Any = json.loads(json.dumps(root_payload)) + mismatch_first["marker"]["argv"][-1] = "stop" + with self.assertRaisesRegex(LaunchPayloadError, "^marker argv does not match exec argv$"): + _validate_payload(mismatch_first) + + outer_argv_first: Any = json.loads(json.dumps(root_payload)) + outer_argv_first["argv"] = "not-a-list" + outer_argv_first["marker"]["argv"][-1] = "stop" + with self.assertRaisesRegex(LaunchPayloadError, "^argv must be a bounded non-empty list$"): + _validate_payload(outer_argv_first) + + def test_supervisor_keeps_generation_compatibility_and_loose_readiness_adapter(self) -> None: + from zeus.supervisor import ( + Supervisor, + _GatewayGeneration, + _readiness_probe_from_marker, + _readiness_probe_marker_payload, + ) + + self.assertIs(GatewayGeneration, _GatewayGeneration) + self.assertTrue(Supervisor._is_compat_runtime_marker({"schema": 2})) + self.assertTrue(Supervisor._is_compat_runtime_marker({})) + compatible_probe = dict(_probe_payload(), legacy_extra="preserved compatibility") + parsed_probe = _readiness_probe_from_marker(compatible_probe) + self.assertIsInstance(parsed_probe, ReadinessProbe) + self.assertEqual(_probe_payload(), _readiness_probe_marker_payload(parsed_probe)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_gateway_runtime.py b/tests/test_gateway_runtime.py new file mode 100644 index 0000000..397fa15 --- /dev/null +++ b/tests/test_gateway_runtime.py @@ -0,0 +1,676 @@ +from __future__ import annotations + +import ast +import dataclasses +import inspect +import json +import os +import signal +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import ( + GatewayRuntime, + KillFn, + LaunchEffect, + MarkerObservation, + OwnershipCheck, + PopenFactory, + PopenLike, + SignalResult, + StopEffect, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.models import BotRecord, BotStatus, DesiredState +from zeus.profile_manager import ProfileManager +from zeus.state import StateStore +from zeus.supervisor import Supervisor + + +class _NeverExits: + def __init__(self, pid: int) -> None: + self.pid = pid + + def poll(self) -> int | None: + return None + + def wait(self, *, timeout: float) -> None: + del timeout + raise TimeoutError + + +class GatewayRuntimeTests(unittest.TestCase): + bot_id = "coder" + pid = 4321 + operation_id = "a" * 32 + + def _fake_hermes(self, root: Path) -> str: + hermes = root / "bin" / "hermes" + hermes.parent.mkdir(parents=True, exist_ok=True) + hermes.write_text("#!/bin/sh\n", encoding="utf-8") + hermes.chmod(0o755) + return str(hermes.resolve()) + + def _fixture( + self, + root: Path, + *, + fingerprints: list[str | None] | None = None, + signals: list[tuple[int, signal.Signals]] | None = None, + popen_factory=None, + startup_grace_seconds: float = 0.0, + stop_grace_seconds: float = 0.0, + kill_after_timeout: bool = False, + ) -> tuple[GatewayRuntime, BotRecord, Path, str]: + root = root.resolve() + hermes_root = root / "hermes" + profile = hermes_root / "profiles" / self.bot_id + profile.mkdir(parents=True) + (profile / ".env").write_text("", encoding="utf-8") + hermes = self._fake_hermes(root) + adapter = HermesAdapter(hermes, hermes_root) + manager = ProfileManager(hermes_root, root / "archive") + fingerprint_values = list(fingerprints or ["same"]) + + def read_fingerprint(pid: int) -> str | None: + self.assertEqual(self.pid, pid) + if len(fingerprint_values) > 1: + return fingerprint_values.pop(0) + return fingerprint_values[0] + + observed_signals = signals if signals is not None else [] + runtime = GatewayRuntime( + adapter, + manager, + hermes_root / "profiles", + popen_factory=popen_factory or (lambda *args, **kwargs: _NeverExits(self.pid)), + kill_fn=lambda pid, sig: observed_signals.append((pid, sig)), + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [hermes, "-p", self.bot_id, "gateway", "run"], + proc_start_fingerprint_reader=read_fingerprint, + startup_grace_seconds=startup_grace_seconds, + stop_grace_seconds=stop_grace_seconds, + kill_after_timeout=kill_after_timeout, + cleanup_process_group=False, + ) + record = BotRecord( + bot_id=self.bot_id, + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + status=BotStatus.running, + pid=self.pid, + desired_state=DesiredState.stopped, + desired_revision=2, + pending_operation_id="b" * 32, + pending_action="stop", + ) + return runtime, record, profile, hermes + + def _schema3_marker( + self, + runtime: GatewayRuntime, + profile: Path, + *, + operation_id: str | None = None, + desired_revision: int = 1, + pid: int | None = None, + process_start: str = "same", + ) -> tuple[dict[str, object], GatewayGeneration]: + payload = runtime.adapter.launcher_payload( + self.bot_id, + operation_id=operation_id or self.operation_id, + desired_revision=desired_revision, + readiness_probe=None, + ) + marker = dict(payload["marker"]) + marker.update( + { + "pid": pid or self.pid, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": process_start, + } + ) + marker_path = runtime.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(marker, sort_keys=True) + "\n", encoding="utf-8") + generation = GatewayGeneration( + operation_id=str(marker["operation_id"]), + desired_revision=int(marker["desired_revision"]), + pid=int(marker["pid"]), + command_fingerprint=str(marker["command_fingerprint"]), + proc_start_fingerprint=process_start, + ) + return marker, generation + + def test_effects_are_frozen_bounded_and_do_not_retain_secret_fields(self) -> None: + launch = LaunchEffect("failed", reason="x" * 2_000, readiness_message="z" * 2_000) + stop = StopEffect("pending", reason="y" * 2_000) + + self.assertLessEqual(len(launch.reason), 512) + self.assertLessEqual(len(launch.readiness_message or ""), 512) + self.assertLessEqual(len(stop.reason), 512) + self.assertNotIn("env", vars(launch)) + self.assertNotIn("payload", vars(launch)) + self.assertNotIn("argv", vars(launch)) + with self.assertRaises(dataclasses.FrozenInstanceError): + launch.reason = "changed" # type: ignore[misc] + with self.assertRaises(dataclasses.FrozenInstanceError): + stop.reason = "changed" # type: ignore[misc] + + def test_launch_secret_is_confined_to_private_descriptor_payload(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + captured: dict[str, object] = {} + secret = "descriptor-only-launch-secret" + + class CapturingPopen: + pid = self.pid + + def __init__(self, argv, env, stdout, stderr, **kwargs): + del stdout, stderr, kwargs + captured["argv"] = list(argv) + captured["env"] = dict(env) + payload_fd = os.dup(int(argv[-2])) + ack_fd = os.dup(int(argv[-1])) + + def publish() -> None: + try: + chunks: list[bytes] = [] + while chunk := os.read(payload_fd, 65_536): + chunks.append(chunk) + payload = json.loads(b"".join(chunks)) + captured["payload"] = payload + marker = dict(payload["marker"]) + marker.update( + { + "pid": self.pid, + "started_at": time.time(), + "proc_start_fingerprint": "same", + } + ) + marker_path = Path(payload["marker_path"]) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(marker), encoding="utf-8") + os.write(ack_fd, b"1") + finally: + os.close(payload_fd) + os.close(ack_fd) + + threading.Thread(target=publish, daemon=True).start() + + def poll(self) -> int | None: + return None + + runtime, record, profile, _hermes = self._fixture( + root, + popen_factory=CapturingPopen, + ) + (profile / ".env").write_text( + f"OPENROUTER_API_KEY={secret}\n", + encoding="utf-8", + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + + with patch.dict(os.environ, {"PATH": os.environ.get("PATH", "")}, clear=True): + effect = runtime.launch(launching, probe=None, wait=False) + + private_payload = captured["payload"] + assert isinstance(private_payload, dict) + marker_text = runtime.pid_marker_path(str(profile)).read_text(encoding="utf-8") + self.assertEqual(secret, private_payload["env"]["OPENROUTER_API_KEY"]) + self.assertNotIn(secret, "\0".join(captured["argv"])) + self.assertNotIn(secret, json.dumps(captured["env"], sort_keys=True)) + self.assertNotIn(secret, marker_text) + self.assertNotIn(secret, repr(effect)) + self.assertNotIn(secret, json.dumps(vars(effect), default=str, sort_keys=True)) + + def test_launch_reports_exit_when_acknowledged_marker_process_is_already_dead(self) -> None: + class ExitedAfterAcknowledgment: + pid = self.pid + + def poll(self) -> int: + return 42 + + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + popen_factory=lambda *args, **kwargs: ExitedAfterAcknowledgment(), + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + marker, generation = self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=lambda *args, **kwargs: MarkerObservation( + "dead", payload=marker, reason="marker PID is dead" + ), + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("startup_exited", effect.outcome) + self.assertEqual(self.pid, effect.pid) + self.assertEqual(generation, effect.generation) + self.assertEqual(42, effect.returncode) + self.assertFalse(runtime.pid_marker_path(str(profile)).exists()) + + def test_launch_reports_exit_when_acknowledged_process_identity_disappears(self) -> None: + class ExitedAfterAcknowledgment: + pid = self.pid + + def poll(self) -> int: + return 42 + + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + popen_factory=lambda *args, **kwargs: ExitedAfterAcknowledgment(), + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=lambda *args, **kwargs: MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ), + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("startup_exited", effect.outcome) + self.assertEqual(self.pid, effect.pid) + self.assertIsNotNone(effect.generation) + self.assertEqual(42, effect.returncode) + self.assertFalse(runtime.pid_marker_path(str(profile)).exists()) + + def test_launch_retries_transient_post_exec_identity_before_registration(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + startup_grace_seconds=0.1, + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + marker, generation = self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + observations = iter( + ( + MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ), + MarkerObservation("live", payload=marker), + ) + ) + calls = 0 + + def marker_matcher(*args, **kwargs): + nonlocal calls + calls += 1 + return next(observations) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=marker_matcher, + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("running", effect.outcome) + self.assertEqual(generation, effect.generation) + self.assertEqual(2, calls) + + def test_exact_generation_rejects_changed_correlation_and_process_identity(self) -> None: + mutations = { + "operation": lambda marker: marker.__setitem__("operation_id", "c" * 32), + "revision": lambda marker: marker.__setitem__("desired_revision", 9), + "pid": lambda marker: marker.__setitem__("pid", 9999), + "command": lambda marker: marker.__setitem__("command_fingerprint", "0" * 64), + "start": lambda marker: marker.__setitem__("proc_start_fingerprint", "reused"), + } + for name, mutate in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture(Path(tmp)) + marker, generation = self._schema3_marker(runtime, profile) + baseline = runtime.classify_exact_gateway_generation(record, generation) + self.assertEqual("live", baseline.kind) + mutate(marker) + runtime.pid_marker_path(str(profile)).write_text( + json.dumps(marker, sort_keys=True) + "\n", + encoding="utf-8", + ) + + changed = runtime.classify_exact_gateway_generation(record, generation) + + self.assertEqual("untrusted", changed.kind) + + def test_final_pre_sigterm_fingerprint_drift_sends_no_signal_or_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + fingerprints=["same", "same", "reused"], + signals=signals, + ) + _marker, _generation = self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("term_reauthorization_failed", effect.outcome) + self.assertEqual([], signals) + self.assertTrue(marker_path.exists()) + + def test_pre_sigkill_fingerprint_drift_sends_term_only(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + fingerprints=["same", "same", "same", "reused"], + signals=signals, + kill_after_timeout=True, + ) + self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("kill_reauthorization_failed", effect.outcome) + self.assertEqual([(self.pid, signal.SIGTERM)], signals) + self.assertTrue(marker_path.exists()) + + def test_legacy_and_schema2_markers_never_signal_start_or_remove(self) -> None: + payloads: list[dict[str, object]] = [ + {"pid": self.pid, "argv": ["hermes", "gateway", "run"]}, + { + "schema": 2, + "pid": self.pid, + "bot_id": self.bot_id, + "component": "gateway", + "action": "run", + "argv": ["hermes", "-p", self.bot_id, "gateway", "run"], + }, + ] + for payload in payloads: + with self.subTest(schema=payload.get("schema")), tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + launches: list[object] = [] + + def forbidden_popen(*args, _launches=launches, **kwargs): + _launches.append((args, kwargs)) + raise AssertionError("stop must not start a process") + + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + signals=signals, + popen_factory=forbidden_popen, + ) + marker_path = runtime.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(payload), encoding="utf-8") + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("compat_untrusted", effect.outcome) + self.assertEqual([], signals) + self.assertEqual([], launches) + self.assertTrue(marker_path.exists()) + + def test_exact_cleanup_preserves_hardlink_symlink_and_replacement_evidence(self) -> None: + for case in ("hardlink", "symlink", "replacement"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture(Path(tmp)) + marker, generation = self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + evidence = Path(tmp) / "evidence.json" + if case == "hardlink": + os.link(marker_path, evidence) + elif case == "symlink": + evidence.write_text(marker_path.read_text(encoding="utf-8"), encoding="utf-8") + marker_path.unlink() + marker_path.symlink_to(evidence) + else: + marker["operation_id"] = "c" * 32 + marker_path.write_text(json.dumps(marker), encoding="utf-8") + + removed = runtime.remove_gateway_generation_marker_locked(record, generation) + + if case == "hardlink": + self.assertTrue(removed) + self.assertFalse(marker_path.exists()) + else: + self.assertFalse(removed) + self.assertTrue(os.path.lexists(marker_path)) + if case != "replacement": + self.assertTrue(evidence.exists()) + + def test_gateway_runtime_has_no_state_store_or_sqlite_dependency(self) -> None: + import zeus.gateway_runtime as gateway_runtime + + source = Path(gateway_runtime.__file__).read_text(encoding="utf-8") + tree = ast.parse(source) + imports = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } | {node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom)} + self.assertNotIn("sqlite3", imports) + self.assertNotIn("zeus.state", imports) + self.assertNotIn("StateStore", source) + + def test_supervisor_reexports_boundary_types_and_preserves_signatures(self) -> None: + import zeus.supervisor as supervisor_module + + self.assertIs(PopenLike, supervisor_module.PopenLike) + self.assertIs(PopenFactory, supervisor_module.PopenFactory) + self.assertIs(KillFn, supervisor_module.KillFn) + self.assertIs(OwnershipCheck, supervisor_module.OwnershipCheck) + self.assertIs(MarkerObservation, supervisor_module._MarkerObservation) + self.assertIs(SignalResult, supervisor_module._SignalResult) + self.assertEqual( + [ + "self", + "store", + "hermes_bin", + "hermes_root", + "popen_factory", + "kill_fn", + "pid_alive_fn", + "cmdline_reader", + "startup_grace_seconds", + "stop_grace_seconds", + "kill_after_timeout", + "lock_timeout_seconds", + "readiness_timeout_seconds", + "readiness_interval_seconds", + "allow_legacy_pid_markers", + "restart_backoff_cap_seconds", + "proc_start_fingerprint_reader", + ], + list(inspect.signature(Supervisor.__init__).parameters), + ) + self.assertEqual( + ["self", "bot_id", "wait", "timeout_seconds", "source", "request_id"], + list(inspect.signature(Supervisor.start).parameters), + ) + self.assertEqual( + ["self", "bot_id", "kill_after_timeout", "source", "request_id"], + list(inspect.signature(Supervisor.stop).parameters), + ) + + def test_supervisor_runtime_proxies_remain_live_after_construction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / "hermes" + (hermes_root / "profiles").mkdir(parents=True) + store = StateStore(root / "zeus.db") + store.init() + supervisor = Supervisor(store, "hermes", hermes_root) + + def popen(*args, **kwargs): + return _NeverExits(77) + + def killer(pid, sig): + return None + + def alive(pid): + return True + + def cmdline(pid): + return ["hermes"] + + def fingerprint(pid): + return "changed" + + processes = {"coder": _NeverExits(77)} + + supervisor.popen_factory = popen + supervisor.kill_fn = killer + supervisor.pid_alive_fn = alive + supervisor.cmdline_reader = cmdline + supervisor.proc_start_fingerprint_reader = fingerprint + supervisor._processes = processes + supervisor.stop_grace_seconds = 9.5 + supervisor.kill_after_timeout = True + supervisor.lock_timeout_seconds = 4.25 + + self.assertIs(popen, supervisor._runtime.popen_factory) + self.assertIs(killer, supervisor._runtime.kill_fn) + self.assertIs(alive, supervisor._runtime.pid_alive_fn) + self.assertIs(cmdline, supervisor._runtime.cmdline_reader) + self.assertIs(fingerprint, supervisor._runtime.proc_start_fingerprint_reader) + self.assertIs(processes, supervisor._runtime._processes) + self.assertEqual(9.5, supervisor._runtime.stop_grace_seconds) + self.assertTrue(supervisor._runtime.kill_after_timeout) + self.assertEqual(4.25, supervisor._runtime.lock_timeout_seconds) + + def test_supervisor_normal_stop_uses_shared_final_reauthorization_primitive(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runtime, record, profile, hermes = self._fixture(root) + self._schema3_marker(runtime, profile) + store = StateStore(root / "zeus.db") + store.init() + fingerprints = iter(("same", "same", "reused")) + supervisor = Supervisor( + store, + hermes, + root / "hermes", + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [hermes, "-p", self.bot_id, "gateway", "run"], + proc_start_fingerprint_reader=lambda pid: next(fingerprints), + kill_fn=lambda pid, sig: None, + ) + supervisor._runtime.stop_grace_seconds = 0 + + with patch.object( + supervisor._runtime, + "reauthorize_and_signal", + wraps=supervisor._runtime.reauthorize_and_signal, + ) as reauthorize: + supervisor._stop_record_effect_locked( + record, + kill_after_timeout=False, + context=supervisor._lifecycle_context("system", None), + complete_stop=False, + ) + + self.assertGreaterEqual(reauthorize.call_count, 1) + + def test_public_runtime_result_types_have_expected_frozen_surface(self) -> None: + observation = MarkerObservation("missing", reason="gone") + self.assertEqual("missing", observation.kind) + self.assertEqual("sent", SignalResult.sent.value) + with self.assertRaises(dataclasses.FrozenInstanceError): + observation.kind = "present" # type: ignore[misc] + + def test_marker_observation_snapshots_payload_and_hides_hostile_values(self) -> None: + hostile = "hostile-marker-value-must-not-escape-repr" + source: dict[str, object] = {"nested": {"value": hostile}} + observation = MarkerObservation("present", payload=source) + source_nested = source["nested"] + assert isinstance(source_nested, dict) + source_nested["value"] = "source-mutated" + + first = observation.payload + assert first is not None + first_nested = first["nested"] + assert isinstance(first_nested, dict) + self.assertEqual(hostile, first_nested["value"]) + first_nested["value"] = "returned-copy-mutated" + + second = observation.payload + assert second is not None + second_nested = second["nested"] + assert isinstance(second_nested, dict) + self.assertEqual(hostile, second_nested["value"]) + self.assertNotIn(hostile, repr(observation)) + + rejected = MarkerObservation("present", payload={"hostile": object()}) + self.assertEqual("untrusted", rejected.kind) + self.assertIsNone(rejected.payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_idempotency_store.py b/tests/test_idempotency_store.py new file mode 100644 index 0000000..5e9905e --- /dev/null +++ b/tests/test_idempotency_store.py @@ -0,0 +1,490 @@ +from __future__ import annotations + +import inspect +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, call, patch + +from zeus import api as api_module +from zeus.config import SQLiteSynchronous +from zeus.idempotency import IdempotencyClaim +from zeus.idempotency_store import ( + IDEMPOTENCY_HASH_RE, + IDEMPOTENCY_OWNER_RE, + MAX_IDEMPOTENCY_RESPONSE_BYTES, + IdempotencyStore, +) +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import ( + IDEMPOTENCY_HASH_RE as STATE_IDEMPOTENCY_HASH_RE, +) +from zeus.state import ( + IDEMPOTENCY_OWNER_RE as STATE_IDEMPOTENCY_OWNER_RE, +) +from zeus.state import ( + MAX_IDEMPOTENCY_RESPONSE_BYTES as STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, +) +from zeus.state import StateStore + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +class _UnavailableSQLiteDatabase(SQLiteDatabase): + def connect(self) -> sqlite3.Connection: + raise sqlite3.OperationalError("database unavailable") + + +def _row_bytes(database_path: Path, key_hash: str) -> tuple[object, ...] | None: + columns = ( + "key_hash", + "request_hash", + "state", + "owner_instance_id", + "response_status", + "response_json", + "created_at", + "updated_at", + "expires_at", + ) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + with closing(sqlite3.connect(database_path)) as conn: + return conn.execute( + f"SELECT {fields} FROM idempotency_records WHERE key_hash = ?", + (key_hash,), + ).fetchone() + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _identity, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statement_index(statements: list[str], prefix: str) -> int: + return next( + index + for index, statement in enumerate(statements) + if statement.lstrip().upper().startswith(prefix) + ) + + +class IdempotencyStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.database_path = Path(self.temporary_directory.name) / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = IdempotencyStore(self.database) + self.key_hash = "a" * 64 + self.request_hash = "b" * 64 + self.future = datetime.now(UTC) + timedelta(hours=1) + + def claim( + self, + *, + key_hash: str | None = None, + request_hash: str | None = None, + owner_instance_id: str = "owner-a", + max_records: int = 10_000, + ) -> IdempotencyClaim: + return self.store.claim_idempotency( + key_hash=key_hash or self.key_hash, + request_hash=request_hash or self.request_hash, + owner_instance_id=owner_instance_id, + expires_at=self.future, + max_records=max_records, + ) + + def complete(self, *, key_hash: str | None = None) -> None: + self.store.complete_idempotency( + key_hash=key_hash or self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=202, + response_json='{"ok":true}', + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + + def test_constructs_without_opening_the_database(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + IdempotencyStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + def test_direct_store_preserves_every_claim_outcome(self) -> None: + self.assertEqual("claimed", self.claim().kind) + self.assertEqual("in_progress", self.claim().kind) + self.assertEqual("conflict", self.claim(request_hash="c" * 64).kind) + self.assertEqual( + "indeterminate", + self.claim(owner_instance_id="owner-b").kind, + ) + + self.complete() + replay = self.claim() + self.assertEqual("replay", replay.kind) + self.assertEqual(202, replay.response_status) + self.assertEqual('{"ok":true}', replay.response_json) + + unavailable = IdempotencyStore( + _UnavailableSQLiteDatabase(self.database_path.with_name("unavailable.db")) + ) + unavailable_claim = unavailable.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + unavailable_lookup = unavailable.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + self.assertEqual("unavailable", unavailable_claim.kind) + self.assertIsNotNone(unavailable_lookup) + self.assertEqual("unavailable", unavailable_lookup.kind if unavailable_lookup else None) + + def test_validation_precedes_connection_and_completion_connection_errors_propagate( + self, + ) -> None: + database = MagicMock(spec=SQLiteDatabase) + store = IdempotencyStore(database) + + with self.assertRaisesRegex(ValueError, "key hash is invalid"): + store.claim_idempotency( + key_hash="invalid", + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + database.connect.assert_not_called() + + unavailable = IdempotencyStore( + _UnavailableSQLiteDatabase(self.database_path.with_name("unavailable.db")) + ) + with self.assertRaisesRegex(sqlite3.OperationalError, "database unavailable"): + unavailable.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + + def test_claim_complete_and_lookup_keep_exact_transaction_boundaries(self) -> None: + tracing_database = _TracingSQLiteDatabase( + self.database_path.with_name("transaction-trace.db") + ) + SchemaManager(tracing_database).init() + tracing_database.traces.clear() + store = IdempotencyStore(tracing_database) + + claimed = store.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + claim_trace = list(tracing_database.traces) + tracing_database.traces.clear() + + store.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + complete_trace = list(tracing_database.traces) + tracing_database.traces.clear() + + replay = store.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + lookup_trace = list(tracing_database.traces) + + claim_sql = [sql for _identity, sql in claim_trace] + complete_sql = [sql for _identity, sql in complete_trace] + lookup_sql = [sql for _identity, sql in lookup_trace] + self.assertEqual("claimed", claimed.kind) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(claim_trace)) + self.assertEqual(1, len({identity for identity, _sql in claim_trace})) + self.assertLess( + _statement_index(claim_sql, "SELECT *"), + _statement_index(claim_sql, "DELETE FROM"), + ) + self.assertLess( + _statement_index(claim_sql, "DELETE FROM"), + _statement_index(claim_sql, "SELECT COUNT"), + ) + self.assertLess( + _statement_index(claim_sql, "SELECT COUNT"), + _statement_index(claim_sql, "INSERT INTO"), + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(complete_trace)) + self.assertEqual(1, len({identity for identity, _sql in complete_trace})) + self.assertLess( + _statement_index(complete_sql, "SELECT *"), + _statement_index(complete_sql, "UPDATE"), + ) + + self.assertIsNotNone(replay) + self.assertEqual("replay", replay.kind if replay else None) + self.assertEqual([], _control_statements(lookup_trace)) + self.assertEqual(1, len({identity for identity, _sql in lookup_trace})) + self.assertTrue(lookup_sql) + self.assertTrue(all(sql.lstrip().upper().startswith("SELECT") for sql in lookup_sql)) + + def test_capacity_exhaustion_is_an_explicit_normal_commit(self) -> None: + tracing_database = _TracingSQLiteDatabase(self.database_path.with_name("capacity.db")) + SchemaManager(tracing_database).init() + store = IdempotencyStore(tracing_database) + store.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=1, + ) + tracing_database.traces.clear() + + result = store.claim_idempotency( + key_hash="c" * 64, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=1, + ) + trace = list(tracing_database.traces) + + self.assertEqual("unavailable", result.kind) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(trace)) + self.assertFalse(any(sql.lstrip().upper().startswith("INSERT") for _identity, sql in trace)) + + def test_corrupt_claim_rolls_back_without_changing_any_stored_bytes(self) -> None: + self.assertEqual("claimed", self.claim().kind) + with closing(sqlite3.connect(self.database_path)) as conn: + conn.execute( + "UPDATE idempotency_records SET request_hash = 'corrupt' WHERE key_hash = ?", + (self.key_hash,), + ) + conn.commit() + before = _row_bytes(self.database_path, self.key_hash) + + result = self.claim(owner_instance_id="owner-b") + + self.assertEqual("unavailable", result.kind) + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + with closing(sqlite3.connect(self.database_path)) as conn: + count = conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] + self.assertEqual(1, count) + + def test_failed_completion_rolls_back_without_changing_any_stored_bytes(self) -> None: + self.assertEqual("claimed", self.claim().kind) + with closing(sqlite3.connect(self.database_path)) as conn: + conn.execute( + """ + CREATE TRIGGER reject_test_completion + BEFORE UPDATE ON idempotency_records + WHEN NEW.state = 'completed' + BEGIN + SELECT RAISE(ABORT, 'injected completion failure'); + END + """ + ) + conn.commit() + before = _row_bytes(self.database_path, self.key_hash) + + with self.assertRaisesRegex(sqlite3.DatabaseError, "injected completion failure"): + self.complete() + + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + + def test_lookup_leaves_an_expired_record_byte_for_byte_unchanged(self) -> None: + self.assertEqual("claimed", self.claim().kind) + completed_at = datetime.now(UTC) + self.store.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=completed_at, + expires_at=completed_at, + ) + before = _row_bytes(self.database_path, self.key_hash) + + result = self.store.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + + self.assertIsNone(result) + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + + def test_state_facade_delegates_exact_keywords_and_propagates_results_and_errors(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + delegate = MagicMock(spec=IdempotencyStore) + claim_result = IdempotencyClaim("claimed") + delegate.claim_idempotency.return_value = claim_result + delegate.lookup_idempotency.return_value = None + completion_error = RuntimeError("completion sentinel") + delegate.complete_idempotency.side_effect = completion_error + completed_at = datetime(2026, 7, 22, 12, tzinfo=UTC) + expires_at = completed_at + timedelta(hours=1) + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=delegate) as store_type, + ): + facade = StateStore(database_path) + actual_claim = facade.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=expires_at, + max_records=321, + ) + actual_lookup = facade.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + with self.assertRaises(RuntimeError) as caught: + facade.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=201, + response_json='{"created":true}', + completed_at=completed_at, + expires_at=expires_at, + ) + + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], store_type.call_args_list) + database.connect.assert_not_called() + self.assertIs(claim_result, actual_claim) + self.assertIsNone(actual_lookup) + self.assertIs(completion_error, caught.exception) + delegate.claim_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=expires_at, + max_records=321, + ) + delegate.lookup_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + delegate.complete_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=201, + response_json='{"created":true}', + completed_at=completed_at, + expires_at=expires_at, + ) + self.assertEqual( + inspect.signature(IdempotencyStore.claim_idempotency), + inspect.signature(StateStore.claim_idempotency), + ) + self.assertEqual( + inspect.signature(IdempotencyStore.lookup_idempotency), + inspect.signature(StateStore.lookup_idempotency), + ) + self.assertEqual( + inspect.signature(IdempotencyStore.complete_idempotency), + inspect.signature(StateStore.complete_idempotency), + ) + + def test_state_compatibility_aliases_and_class_monkeypatch_remain_supported(self) -> None: + self.assertIs(IDEMPOTENCY_HASH_RE, STATE_IDEMPOTENCY_HASH_RE) + self.assertIs(IDEMPOTENCY_OWNER_RE, STATE_IDEMPOTENCY_OWNER_RE) + self.assertIs( + MAX_IDEMPOTENCY_RESPONSE_BYTES, + STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, + ) + self.assertIs( + STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, + api_module.MAX_IDEMPOTENCY_RESPONSE_BYTES, + ) + facade = StateStore(self.database_path.with_name("monkeypatch.db")) + patched_result = IdempotencyClaim("unavailable") + + with patch.object( + StateStore, + "claim_idempotency", + autospec=True, + return_value=patched_result, + ) as patched_claim: + result = facade.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=55, + ) + + self.assertIs(patched_result, result) + patched_claim.assert_called_once_with( + facade, + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=55, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_intent_recovery.py b/tests/test_intent_recovery.py new file mode 100644 index 0000000..d9e80f2 --- /dev/null +++ b/tests/test_intent_recovery.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import ast +import inspect +import unittest +from dataclasses import dataclass, replace +from pathlib import Path + +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import MarkerObservation, StopEffect +from zeus.intent_recovery import PendingIntentRecovery +from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState +from zeus.process_identity import PidState +from zeus.readiness import ReadinessProbe, ReadinessResult + + +@dataclass(frozen=True) +class _Context: + operation_id: str + source: str = "reconcile" + request_id: str | None = "request" + + +def _pending_record( + action: str, + *, + pid: int | None = None, + revision: int = 1, +) -> BotRecord: + return BotRecord( + "coder", + "coding-bot", + "Coder", + "/profiles/coder", + status=BotStatus.starting, + pid=pid, + desired_state=(DesiredState.stopped if action == "stop" else DesiredState.running), + desired_revision=revision, + pending_operation_id="1" * 32, + pending_action=action, + ) + + +class _StrictRecoveryHost: + def __init__(self, recovery: PendingIntentRecovery) -> None: + self.recovery = recovery + self.probe: ReadinessProbe | None = None + self.fingerprint = "f" * 64 + self.strict_marker = MarkerObservation("missing") + self.matching_marker = MarkerObservation("missing") + self.old_marker: MarkerObservation | None = None + self.pid_state = PidState.dead + self.pid_owned = False + self.start_calls = 0 + self.stop_effect_calls = 0 + self.signal_calls = 0 + self.marker_removals = 0 + self.state_updates = 0 + self.started_completions: list[tuple[str, int]] = [] + self.stopped_completions: list[str] = [] + self.audit_events: list[str] = [] + self.generation = GatewayGeneration( + operation_id="0" * 32, + desired_revision=1, + pid=41, + command_fingerprint="a" * 64, + proc_start_fingerprint="start", + ) + + def _recovery_lifecycle_context( + self, + operation_id: str, + context: _Context, + ) -> _Context: + return _Context(operation_id, context.source, context.request_id) + + @staticmethod + def _pending_action_required(record: BotRecord, reason: str) -> BotStatusResponse: + return BotStatusResponse( + record.bot_id, + record.status, + record.pid, + record.profile_path, + f"action required: {reason}", + ) + + def _pending_launch_preflight( + self, + _record: BotRecord, + _operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: + return self.probe, self.fingerprint + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse: + return self.recovery.recover_pending_stop_intent_locked( + self, + record, + context=context, + allow_stop=allow_stop, + ) + + def _recover_pending_restart_predecessor( + self, + record: BotRecord, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse | None: + return self.recovery.recover_pending_restart_predecessor_locked( + self, + record, + context=context, + allow_stop=allow_stop, + ) + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _Context, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: + return self.recovery.recover_pending_launch_locked( + self, + record, + context=context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, + ) + + def _matching_runtime_marker( + self, + _record: BotRecord, + *, + expected_fingerprint: str, + require_live_command: bool, + ) -> MarkerObservation: + if expected_fingerprint != self.fingerprint or not require_live_command: + raise AssertionError("recovery weakened exact marker classification") + return self.matching_marker + + @staticmethod + def _probe_once(_probe: ReadinessProbe) -> ReadinessResult: + return ReadinessResult(False, "not ready") + + def _complete_started_intent( + self, + _record: BotRecord, + *, + context: _Context, + status: BotStatus, + pid: int, + ready_at: object, + reset_restart: bool, + reason: str, + ) -> None: + del status, ready_at, reset_restart, reason + self.started_completions.append((context.operation_id, pid)) + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + context: _Context, + probe: ReadinessProbe | None, + ) -> BotStatusResponse: + del reset_restart, message, context, probe + self.start_calls += 1 + return BotStatusResponse( + record.bot_id, + BotStatus.running, + 99, + record.profile_path, + "started", + ) + + def _read_strict_runtime_marker( + self, + _bot_id: str, + _profile_path: str, + ) -> MarkerObservation: + return self.strict_marker + + def _remove_owned_launch_marker_locked( + self, + _record: BotRecord, + *, + observed: MarkerObservation, + ) -> bool: + del observed + self.marker_removals += 1 + return True + + def _complete_stopped_intent( + self, + _record: BotRecord, + *, + context: _Context, + reason: str, + ) -> None: + del reason + self.stopped_completions.append(context.operation_id) + + def _pid_state(self, _pid: int) -> PidState: + return self.pid_state + + def _pid_owned(self, _profile_path: str, _pid: int, _bot_id: str) -> bool: + return self.pid_owned + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _Context, + complete_stop: bool, + ) -> BotStatusResponse: + del kill_after_timeout, context, complete_stop + self.stop_effect_calls += 1 + return self._pending_action_required( + record, + "gateway marker ownership could not be verified", + ) + + @staticmethod + def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: + return payload.get("schema") in {None, 2} + + def _pending_restart_old_marker( + self, + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: + if self.old_marker is not None: + return self.old_marker + return self.recovery.pending_restart_old_marker(self, record, observed) + + def _classify_schema3_runtime_marker( + self, + _record: BotRecord, + _payload: dict[str, object], + *, + expected_pid: int | None, + expected_revision: int, + require_live_command: bool, + ) -> MarkerObservation: + del expected_pid, expected_revision, require_live_command + return MarkerObservation("untrusted", reason="not configured") + + def _recover_pending_restart_old_gateway( + self, + record: BotRecord, + marker: MarkerObservation, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse: + return self.recovery.recover_pending_restart_old_gateway( + self, + record, + marker, + context=context, + allow_stop=allow_stop, + ) + + def _gateway_generation( + self, + _marker: MarkerObservation, + ) -> GatewayGeneration | None: + return self.generation + + def _remove_gateway_generation_marker_locked( + self, + _record: BotRecord, + _generation: GatewayGeneration, + ) -> bool: + self.marker_removals += 1 + return True + + def _update_lifecycle( + self, + _context: _Context, + _bot_id: str, + _status: BotStatus, + **_values: object, + ) -> None: + self.state_updates += 1 + + def _stop_pending_restart_old_gateway( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + context: _Context, + ) -> BotStatusResponse: + return self.recovery.stop_pending_restart_old_gateway( + self, + record, + generation, + context=context, + ) + + def _stop_gateway_generation_locked( + self, + _record: BotRecord, + _generation: GatewayGeneration, + ) -> StopEffect: + self.signal_calls += 1 + return StopEffect("stopped", pid=self.generation.pid, generation=self.generation) + + def _append_recovery_audit_event(self, action: str, **_values: object) -> None: + self.audit_events.append(action) + + +class PendingIntentRecoveryTests(unittest.TestCase): + def test_original_publication_is_adopted_once_without_duplicate_launch(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("start") + host.matching_marker = MarkerObservation("live", {"pid": 73}) + + response = recovery.recover( + host, + record, + context=_Context("unrelated"), + allow_launch=True, + ) + + self.assertEqual(response.status, BotStatus.running) + self.assertEqual(response.pid, 73) + self.assertEqual(host.started_completions, [(record.pending_operation_id, 73)]) + self.assertEqual(host.start_calls, 0) + + def test_exact_live_restart_marker_is_adopted_without_popen_factory(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("restart", revision=2) + host.strict_marker = MarkerObservation("missing") + host.matching_marker = MarkerObservation("live", {"pid": 81}) + + response = recovery.recover( + host, + record, + context=_Context("different"), + allow_launch=True, + ) + + self.assertEqual(response.pid, 81) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.started_completions, [(record.pending_operation_id, 81)]) + + def test_compat_restart_and_stop_have_no_signal_delete_start_or_update(self) -> None: + recovery = PendingIntentRecovery() + restart_host = _StrictRecoveryHost(recovery) + restart = _pending_record("restart", pid=41, revision=2) + restart_host.strict_marker = MarkerObservation("present", {"schema": 2, "pid": 41}) + + restart_response = recovery.recover( + restart_host, + restart, + context=_Context("different"), + allow_launch=True, + ) + + stop_host = _StrictRecoveryHost(recovery) + stop = _pending_record("stop", pid=41) + stop_host.pid_state = PidState.alive + stop_response = recovery.recover( + stop_host, + stop, + context=_Context("different"), + allow_launch=False, + ) + + self.assertTrue(restart_response.message.startswith("action required:")) + self.assertTrue(stop_response.message.startswith("action required:")) + for host in (restart_host, stop_host): + self.assertEqual(host.signal_calls, 0) + self.assertEqual(host.marker_removals, 0) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.state_updates, 0) + + def test_restart_stop_and_launch_use_separate_passes_with_one_launch(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + first = _pending_record("restart", pid=41, revision=2) + host.strict_marker = MarkerObservation("present", {"schema": 3}) + host.old_marker = MarkerObservation("live", {"pid": 41}) + + first_response = recovery.recover( + host, + first, + context=_Context("different"), + allow_launch=True, + ) + + self.assertEqual(first_response.status, BotStatus.starting) + self.assertIn("launch on next reconcile", first_response.message) + self.assertEqual(host.signal_calls, 1) + self.assertEqual(host.start_calls, 0) + + second = replace(first, status=BotStatus.stopped, pid=None) + host.strict_marker = MarkerObservation("missing") + host.old_marker = None + host.matching_marker = MarkerObservation("missing") + second_response = recovery.recover( + host, + second, + context=_Context("new-context"), + allow_launch=True, + ) + + self.assertEqual(second_response.status, BotStatus.running) + self.assertEqual(host.start_calls, 1) + self.assertEqual(host.started_completions, []) + + def test_status_with_pending_running_intent_never_launches(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("start") + host.matching_marker = MarkerObservation("missing") + + response = recovery.recover( + host, + record, + context=_Context("status-context", source="cli"), + allow_launch=False, + ) + + self.assertTrue(response.message.startswith("action required:")) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.signal_calls, 0) + + def test_host_methods_are_resolved_after_recovery_construction(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("stop") + observed: list[str] = [] + + def patched_reader(bot_id: str, _profile_path: str) -> MarkerObservation: + observed.append(bot_id) + return MarkerObservation("missing") + + host._read_strict_runtime_marker = patched_reader # type: ignore[method-assign] + + response = recovery.recover_pending_stop_intent_locked( + host, + record, + context=_Context(record.pending_operation_id or ""), + allow_stop=True, + ) + + self.assertEqual(response.status, BotStatus.stopped) + self.assertEqual(observed, [record.bot_id]) + self.assertEqual(host.stopped_completions, [record.pending_operation_id]) + + def test_module_has_structural_host_and_no_store_lock_or_id_dependency(self) -> None: + import zeus.intent_recovery as module + + source = Path(inspect.getsourcefile(module) or "").read_text(encoding="utf-8") + tree = ast.parse(source) + imports = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + from_imports = { + node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + } + + self.assertTrue(hasattr(module, "_RecoveryHost")) + self.assertFalse( + any(isinstance(node, (ast.With, ast.AsyncWith)) for node in ast.walk(tree)) + ) + self.assertNotIn("sqlite3", imports | from_imports) + self.assertNotIn("uuid", imports | from_imports) + self.assertNotIn("zeus.state", from_imports) + self.assertNotIn("zeus.supervisor", from_imports) + self.assertNotIn("StateStore", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index f03c0f0..da3feff 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -6,12 +6,15 @@ import unittest from contextlib import closing from dataclasses import FrozenInstanceError -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from unittest.mock import patch +from zeus.bot_lifecycle_store import BotLifecycleStore from zeus.lifecycle import LifecycleEvent, LifecycleEventInput, serialize_lifecycle_details -from zeus.models import BotRecord, BotStatus +from zeus.models import BotRecord, BotStatus, DesiredState +from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunStart +from zeus.sanitization import sanitize_details, sanitize_text from zeus.state import StateStore @@ -321,6 +324,73 @@ def test_history_rejects_unsafe_limits_and_cursors(self) -> None: before=invalid_before, # type: ignore[arg-type] ) + def test_pending_intent_projection_and_event_are_committed_as_one_fence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot(self._record(root)) + + pending = store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="api", + request_id="b" * 32, + reason="gateway start requested", + ) + + history = store.list_lifecycle_events("coder", limit=10, before=None) + self.assertEqual(1, len(history)) + intent = history[0] + with closing(sqlite3.connect(store.database_path)) as conn: + last_event_id = conn.execute( + "SELECT last_event_id FROM bots WHERE bot_id = 'coder'" + ).fetchone()[0] + self.assertEqual(intent.event_id, last_event_id) + self.assertEqual(DesiredState.running, pending.desired_state) + self.assertEqual(1, pending.desired_revision) + self.assertEqual("a" * 32, pending.pending_operation_id) + self.assertEqual("start", pending.pending_action) + self.assertEqual( + { + "bot_id": "coder", + "operation_id": "a" * 32, + "request_id": "b" * 32, + "source": "api", + "action": "bot.start.intent", + "outcome": "pending", + "status_before": "stopped", + "status_after": "stopped", + "pid_before": None, + "pid_after": None, + "reason": "gateway start requested", + "error_code": None, + "error_message": None, + "details": { + "action": "start", + "desired_revision": 1, + "desired_state": "running", + }, + }, + { + "bot_id": intent.bot_id, + "operation_id": intent.operation_id, + "request_id": intent.request_id, + "source": intent.source, + "action": intent.action, + "outcome": intent.outcome, + "status_before": intent.status_before, + "status_after": intent.status_after, + "pid_before": intent.pid_before, + "pid_after": intent.pid_after, + "reason": intent.reason, + "error_code": intent.error_code, + "error_message": intent.error_message, + "details": dict(intent.details), + }, + ) + def test_event_input_is_frozen_and_recursively_redacts_bounded_details(self) -> None: nested_items = [{"visible": "value"}] nested: dict[str, object] = { @@ -432,6 +502,331 @@ def test_event_details_normalize_forbidden_names_and_are_json_compatible(self) - ) json.dumps(stored.to_dict(), sort_keys=True) + def test_sanitizer_bounds_cycles_nonfinite_numbers_and_free_text_secrets(self) -> None: + secret = "lifecycle-sentinel-9a21" + cycle: list[object] = [] + cycle.append(cycle) + deep: object = "leaf" + for _ in range(12): + deep = {"next": deep} + + sanitized = sanitize_details( + { + "api_key": secret, + "messages": [ + f"API_KEY={secret}", + f"authorization=Bearer {secret}", + f"request failed with Bearer {secret}", + ], + "cycle": cycle, + "deep": deep, + "nan": float("nan"), + "positive_infinity": float("inf"), + "negative_infinity": float("-inf"), + } + ) + encoded = json.dumps( + sanitized, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + self.assertFalse(secret.encode("utf-8") in encoded) + self.assertLessEqual(len(encoded), 8192) + self.assertIsNone(sanitized["nan"]) # type: ignore[index] + self.assertIsNone(sanitized["positive_infinity"]) # type: ignore[index] + self.assertIsNone(sanitized["negative_infinity"]) # type: ignore[index] + self.assertIn(b"[cycle]", encoded) + self.assertIn(b"[truncated]", encoded) + + oversized = json.dumps( + sanitize_details({"message": "Δ" * 20_000}), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + self.assertLessEqual(len(oversized), 8192) + + bounded_collection = sanitize_details({"items": list(range(1_000))}) + self.assertLessEqual(len(bounded_collection["items"]), 64) # type: ignore[index] + + def test_sanitizer_redacts_values_for_oversized_sensitive_keys(self) -> None: + secret = "long-key-sentinel-cb30" + sanitized = sanitize_details({f"{'x' * 256}_api_key": secret}) + encoded = json.dumps(sanitized, allow_nan=False).encode("utf-8") + + self.assertFalse(secret.encode("utf-8") in encoded) + + def test_sanitizer_never_invokes_hostile_string_or_repr_methods(self) -> None: + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + class HostileString(str): + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + hostile_key = Hostile() + hostile_value = Hostile() + + sanitized = sanitize_details( + { + "value": hostile_value, + "text": HostileString("TOKEN=hostile-string-sentinel"), + hostile_key: "untrusted key value", + } + ) + + self.assertEqual("[unsupported]", sanitized["value"]) # type: ignore[index] + self.assertEqual("[unsupported]", sanitized["text"]) # type: ignore[index] + json.dumps(sanitized, allow_nan=False) + + def test_lifecycle_text_fields_are_redacted_before_sqlite_persistence(self) -> None: + secret = "event-sentinel-84c2" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="a" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=f"readiness failed with Bearer {secret}", + error_code="readiness_failed", + error_message=f"cleanup failed: API_KEY={secret}", + ) + + store.upsert_bot_with_event(self._record(root), event=event) + + with closing(sqlite3.connect(store.database_path)) as conn: + row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + assert row is not None + persisted = "\n".join(str(value) for value in row if value is not None) + self.assertFalse(secret in persisted) + + def test_authorization_credentials_are_redacted_from_all_persistence_sinks(self) -> None: + credentials = ( + "basic-credential-sentinel-47d1", + "digest-credential-sentinel-8a20", + "custom-credential-sentinel-53fe", + ) + messages = ( + f"Authorization: Basic {credentials[0]}", + f'authorization=Digest username="operator", response="{credentials[1]}"', + f"Authorization: CustomScheme {credentials[2]}", + ) + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="a" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=messages[0], + error_code="authorization_failed", + error_message=messages[1], + details={"message": messages[2]}, + ) + store.upsert_bot_with_event( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=messages[0], + last_transition_reason=messages[1], + ), + event=event, + ) + store.append_audit_event( + "bot.authorization_failed", + cleanup_errors=list(messages), + ) + run = ReconcileRunStart( + run_id="authorization-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message="; ".join(messages), + error_code="authorization_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + with closing(sqlite3.connect(store.database_path)) as conn: + lifecycle_row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert lifecycle_row is not None + assert bot_row is not None + assert reconcile_row is not None + observed = ( + *(sanitize_text(message) for message in messages), + *lifecycle_row, + *bot_row, + *reconcile_row, + result.message, + store.audit_log_path().read_text(encoding="utf-8"), + ) + for credential in credentials: + self.assertFalse( + any(credential in value for value in observed if isinstance(value, str)) + ) + + def test_authorization_label_variants_and_folded_credentials_are_redacted(self) -> None: + credentials = tuple(f"authorization-variant-sentinel-{index:02d}" for index in range(16)) + messages = ( + f"Authorization Header: Basic {credentials[0]}", + f'authorization_header=Digest response="{credentials[1]}"', + f"Authorization Headers: CustomScheme {credentials[2]}", + f"authorizationHeaders=Bearer {credentials[3]}", + f"authorization-header: Basic {credentials[4]}", + f'authorization.headers=Digest response="{credentials[5]}"', + f"authorizationHeader=CustomScheme {credentials[6]}", + f"authorization/header: Basic {credentials[7]}", + f"Authorization: Basic\r\n {credentials[8]}", + f'Authorization Header: Digest\n\tresponse="{credentials[9]}"', + f"authorization_headers=CustomScheme\r\n {credentials[10]}", + f"authorizationHeaders: Bearer\n\t{credentials[11]}", + f"Authorization:\r\n Basic {credentials[12]}", + f'Authorization Header:\n\tDigest response="{credentials[13]}"', + f"authorization_headers=\r\n CustomScheme {credentials[14]}", + f'"Authorization":\n "Basic {credentials[15]}"', + ) + combined = "\nnext authorization case\n".join(messages) + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="b" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=combined, + error_code="authorization_failed", + error_message=combined, + details={"message": combined}, + ) + store.upsert_bot_with_event( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=combined, + last_transition_reason=combined, + ), + event=event, + ) + store.append_audit_event( + "bot.authorization_failed", + cleanup_errors=list(messages), + ) + run = ReconcileRunStart( + run_id="authorization-variant-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message=combined, + error_code="authorization_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + with closing(sqlite3.connect(store.database_path)) as conn: + lifecycle_row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert lifecycle_row is not None + assert bot_row is not None + assert reconcile_row is not None + observed_by_sink = { + "direct": tuple(sanitize_text(message) for message in messages), + "lifecycle": lifecycle_row, + "bot_projection": bot_row, + "reconciliation": (*reconcile_row, result.message), + "audit": (store.audit_log_path().read_text(encoding="utf-8"),), + } + for case_index, credential in enumerate(credentials): + for sink_name, observed in observed_by_sink.items(): + with self.subTest(case=case_index, sink=sink_name): + self.assertFalse( + any(credential in value for value in observed if isinstance(value, str)) + ) + + def test_audit_non_bmp_event_fallback_is_strictly_byte_bounded(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + + store.append_audit_event(chr(0x1F512) * 2_048, message="ordinary") + + line = store.audit_log_path().read_bytes() + self.assertLessEqual(len(line), 8192) + self.assertEqual(1, len(line.splitlines())) + self.assertTrue(json.loads(line)["truncated"]) + def test_upsert_bot_with_event_rolls_back_projection_when_event_insert_fails(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -440,7 +835,7 @@ def test_upsert_bot_with_event_rolls_back_projection_when_event_insert_fails(sel with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -460,7 +855,7 @@ def test_update_lifecycle_with_event_rolls_back_when_event_insert_fails(self) -> with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -488,7 +883,7 @@ def test_update_restart_with_event_rolls_back_projection_when_event_insert_fails with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -519,7 +914,7 @@ def test_delete_bot_with_event_rolls_back_projection_when_event_insert_fails(sel with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -541,7 +936,7 @@ def test_atomic_mutations_roll_back_when_event_materialization_fails(self) -> No with ( patch.object( - store, + BotLifecycleStore, "_row_to_lifecycle_event", side_effect=sqlite3.Error("materialization failed"), ), @@ -576,7 +971,7 @@ def test_atomic_mutations_ignore_post_commit_audit_failures(self) -> None: store.upsert_bot(self._record(root)) with patch.object( - store, + BotLifecycleStore, "_append_lifecycle_audit", side_effect=RuntimeError("audit failed"), ): diff --git a/tests/test_lifecycle_regressions.py b/tests/test_lifecycle_regressions.py index 9091ede..194c87c 100644 --- a/tests/test_lifecycle_regressions.py +++ b/tests/test_lifecycle_regressions.py @@ -887,13 +887,13 @@ def test_status_uses_launch_readiness_provenance_in_a_new_process(self) -> None: patch.dict(os.environ, {"ZEUS_ENV_PASSTHROUGH": ""}, clear=False), patch( "zeus.supervisor.probe_once", - return_value=ReadinessResult(False, "connection refused"), + return_value=ReadinessResult(False, "readiness probe failed"), ) as probe_once, ): status = status_supervisor.status("coder") self.assertEqual(BotStatus.starting, status.status) - self.assertEqual("connection refused", status.message) + self.assertEqual("readiness probe failed", status.message) probe_once.assert_called_once_with( "http://127.0.0.1:4312/health", timeout_seconds=0.5, diff --git a/tests/test_private_io.py b/tests/test_private_io.py new file mode 100644 index 0000000..2148a98 --- /dev/null +++ b/tests/test_private_io.py @@ -0,0 +1,939 @@ +from __future__ import annotations + +import io +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import BinaryIO +from unittest.mock import patch + +from zeus import private_io +from zeus.private_io import ( + UnsafeFileError, + append_private_bytes, + ensure_private_directory, + open_private_append, + read_private_tail, + validate_private_directory, +) + + +class PrivateIOTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name).resolve() + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def assert_all_closed(self, descriptors: set[int]) -> None: + for descriptor in descriptors: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_append_and_tail_preserve_exact_bytes(self) -> None: + path = self.root / "logs" / "events.bin" + first = b"first\x00line\n" + second = b"\xffsecond\n" + + append_private_bytes(path, first) + append_private_bytes(path, second) + + expected = first + second + self.assertEqual(expected, path.read_bytes()) + self.assertEqual(expected, read_private_tail(path, len(expected) + 100)) + self.assertEqual(expected[-7:], read_private_tail(path, 7)) + self.assertEqual(b"", read_private_tail(path, 0)) + + def test_open_private_append_is_unbuffered_and_always_appends(self) -> None: + path = self.root / "logs" / "events.bin" + append_private_bytes(path, b"first") + + with open_private_append(path) as handle: + self.assertIsInstance(handle, io.RawIOBase) + self.assertNotIsInstance(handle, io.BufferedIOBase) + handle.seek(0) + self.assertEqual(6, handle.write(b"second")) + + self.assertEqual(b"firstsecond", path.read_bytes()) + + def test_read_missing_leaf_returns_empty_without_creating_it(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "missing.log" + + self.assertEqual(b"", read_private_tail(path, 128)) + + self.assertFalse(path.exists()) + + def test_read_missing_ancestor_returns_empty_without_creating_it(self) -> None: + path = self.root / "missing" / "nested" / "events.log" + + self.assertEqual(b"", read_private_tail(path, 128)) + + self.assertFalse(self.root.joinpath("missing").exists()) + + def test_missing_tail_rejects_rebound_open_ancestor_and_closes_descriptors(self) -> None: + ancestor = self.root / "safe" + ancestor.mkdir(mode=0o700) + displaced = self.root / "safe-displaced" + external = self.root / "external" + (external / "missing").mkdir(parents=True) + target = external / "missing" / "events.log" + target.write_bytes(b"external target") + target_mode = stat.S_IMODE(target.stat().st_mode) + path = ancestor / "missing" / "events.log" + ancestor_identity = ancestor.stat() + opened: set[int] = set() + real_lstat = os.lstat + real_open = os.open + swapped = False + + def racing_lstat( + name: str | bytes, + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + if not swapped and name == "missing" and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == ancestor_identity.st_dev + and parent.st_ino == ancestor_identity.st_ino + ): + ancestor.rename(displaced) + ancestor.symlink_to(external, target_is_directory=True) + swapped = True + return real_lstat(name, dir_fd=dir_fd) + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + patch.object(private_io.os, "open", side_effect=tracking_open), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(swapped) + self.assertEqual(b"external target", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + self.assert_all_closed(opened) + + def test_missing_tail_rejects_leaf_appearing_after_observation_and_closes_descriptors( + self, + ) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + opened: set[int] = set() + real_lstat = os.lstat + real_open = os.open + appeared = False + + def racing_lstat( + name: str | bytes, + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal appeared + if not appeared and name == path.name and dir_fd is not None: + try: + return real_lstat(name, dir_fd=dir_fd) + except FileNotFoundError: + path.write_bytes(b"appeared after missing observation") + path.chmod(0o644) + appeared = True + raise + return real_lstat(name, dir_fd=dir_fd) + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + patch.object(private_io.os, "open", side_effect=tracking_open), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(appeared) + self.assertEqual(b"appeared after missing observation", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + self.assert_all_closed(opened) + + def test_tail_size_validation_happens_before_filesystem_mutation(self) -> None: + path = self.root / "missing" / "events.log" + + for value in (True, -1, 1.5, "1", None): + with self.subTest(value=value), self.assertRaises(TypeError): + read_private_tail(path, value) # type: ignore[arg-type] + + self.assertFalse(path.parent.exists()) + + def test_unsupported_path_shapes_fail_before_mutation(self) -> None: + traversal_parent = self.root / "new" / ".." / "escape" + + for path in (Path("relative.log"), traversal_parent / "events.log", Path("/")): + with self.subTest(path=path), self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + + self.assertFalse((self.root / "new").exists()) + + def test_missing_security_flag_fails_before_creation(self) -> None: + path = self.root / "missing" / "events.log" + + with ( + patch.object(private_io.os, "O_NOFOLLOW", 0), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertFalse(path.parent.exists()) + + def test_missing_descriptor_relative_primitive_fails_before_creation(self) -> None: + path = self.root / "missing" / "events.log" + + with ( + patch.object(private_io.os, "supports_dir_fd", frozenset()), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertFalse(path.parent.exists()) + + def test_linux_stat_capability_advertises_descriptor_relative_lstat(self) -> None: + path = self.root / "private" + linux_capabilities = frozenset({os.open, os.mkdir, os.stat}) + + with patch.object(private_io.os, "supports_dir_fd", linux_capabilities): + ensure_private_directory(path) + + self.assertTrue(path.is_dir()) + self.assertEqual(0o700, stat.S_IMODE(path.stat().st_mode)) + + def test_validate_private_directory_requires_existing_directory(self) -> None: + path = self.root / "missing" + + with self.assertRaises(UnsafeFileError): + validate_private_directory(path) + + self.assertFalse(path.exists()) + + def test_new_and_final_private_directories_are_tightened_without_changing_ancestors( + self, + ) -> None: + ancestor = self.root / "ancestor" + ancestor.mkdir(mode=0o755) + ancestor.chmod(0o755) + private_dir = ancestor / "private" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + + validate_private_directory(private_dir) + append_private_bytes(private_dir / "nested" / "events.log", b"event") + + self.assertEqual(0o755, stat.S_IMODE(ancestor.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE((private_dir / "nested").stat().st_mode)) + + def test_directory_link_count_is_not_restricted_to_one(self) -> None: + private_dir = self.root / "private" + (private_dir / "child").mkdir(parents=True) + self.assertGreater(private_dir.stat().st_nlink, 1) + + validate_private_directory(private_dir) + + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_validate_directory_rejects_mode_drift_before_final_lstat(self) -> None: + private_dir = self.root / "mode-drift-dir" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + real_lstat = os.lstat + directory_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal directory_lstats + if name == private_dir.name and dir_fd is not None: + directory_lstats += 1 + if directory_lstats == 3: + private_dir.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(3, directory_lstats) + self.assertEqual(0o777, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_validate_directory_rejects_mode_drift_during_final_binding_check(self) -> None: + private_dir = self.root / "final-mode-drift-dir" + private_dir.mkdir(mode=0o700) + private_dir.chmod(0o700) + real_lstat = os.lstat + directory_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal directory_lstats + if name == private_dir.name and dir_fd is not None: + directory_lstats += 1 + if directory_lstats == 4: + private_dir.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(4, directory_lstats) + self.assertEqual(0o777, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_ensure_directory_rechecks_created_intermediate_mode_at_final_binding(self) -> None: + created_parent = self.root / "created-parent" + private_dir = created_parent / "private" + real_lstat = os.lstat + parent_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal parent_lstats + if name == created_parent.name and dir_fd is not None: + parent_lstats += 1 + if parent_lstats == 5: + created_parent.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + ensure_private_directory(private_dir) + + self.assertEqual(5, parent_lstats) + self.assertEqual(0o777, stat.S_IMODE(created_parent.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_pin_directory_closes_duplicate_when_exit_binding_validation_fails(self) -> None: + private_dir = self.root / "private" + private_dir.mkdir(mode=0o700) + displaced = self.root / "private-displaced" + parent_stat = private_dir.parent.stat() + opened: set[int] = set() + duplicated: set[int] = set() + real_close = os.close + real_dup = os.dup + real_fstat = os.fstat + real_lstat = os.lstat + real_open = os.open + raced = False + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def tracking_dup(fd: int) -> int: + descriptor = real_dup(fd) + opened.add(descriptor) + duplicated.add(descriptor) + return descriptor + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal raced + if duplicated and not raced and name == private_dir.name and dir_fd is not None: + parent = real_fstat(dir_fd) + if parent.st_dev == parent_stat.st_dev and parent.st_ino == parent_stat.st_ino: + private_dir.rename(displaced) + raced = True + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "dup", side_effect=tracking_dup), + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + private_io.pin_private_directory(private_dir), + ): + self.fail("pin context must not be entered after binding drift") + + leaked: list[int] = [] + for descriptor in opened: + try: + real_fstat(descriptor) + except OSError: + continue + leaked.append(descriptor) + for descriptor in leaked: + real_close(descriptor) + + self.assertTrue(raced) + self.assertEqual(1, len(duplicated)) + self.assertEqual([], leaked) + + def test_existing_file_and_parent_modes_are_tightened_before_append(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o755) + path = private_dir / "events.log" + path.write_bytes(b"existing") + private_dir.chmod(0o755) + path.chmod(0o644) + + append_private_bytes(path, b"event") + + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) + self.assertEqual(b"existingevent", path.read_bytes()) + + def test_read_tightens_existing_file_mode(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"event") + path.chmod(0o644) + + self.assertEqual(b"event", read_private_tail(path, 10)) + + self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) + + def test_append_rejects_mode_drift_before_final_lstat(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "mode-drift.log" + path.write_bytes(b"original") + path.chmod(0o644) + real_lstat = os.lstat + file_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal file_lstats + if name == path.name and dir_fd is not None: + file_lstats += 1 + if file_lstats == 3: + path.chmod(0o644) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(3, file_lstats) + self.assertEqual(b"original", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + + def test_leaf_symlink_is_rejected_without_touching_target(self) -> None: + target = self.root / "target.log" + target.write_bytes(b"original") + path = self.root / "events.log" + path.symlink_to(target) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + self.assertEqual(b"original", target.read_bytes()) + + def test_intermediate_directory_symlink_is_rejected(self) -> None: + outside = self.root / "outside" + outside.mkdir() + linked = self.root / "linked" + linked.symlink_to(outside, target_is_directory=True) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(linked / "logs" / "events.log", b"event") + + self.assertFalse((outside / "logs").exists()) + + def test_final_parent_symlink_is_rejected_without_tightening_target(self) -> None: + outside = self.root / "outside" + outside.mkdir(mode=0o755) + outside.chmod(0o755) + state = self.root / "state" + state.mkdir() + linked = state / "logs" + linked.symlink_to(outside, target_is_directory=True) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(linked / "events.log", b"event") + + self.assertFalse((outside / "events.log").exists()) + self.assertEqual(0o755, stat.S_IMODE(outside.stat().st_mode)) + + def test_hardlink_is_rejected_before_chmod_or_io(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + target = private_dir / "target.log" + target.write_bytes(b"original") + target.chmod(0o644) + path = private_dir / "events.log" + os.link(target, path) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + self.assertEqual(b"original", target.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(target.stat().st_mode)) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_fifo_is_rejected_in_bounded_subprocesses(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.fifo" + os.mkfifo(path, mode=0o600) + expressions = ( + "append_private_bytes(path, b'event')", + "read_private_tail(path, 1)", + ) + + for expression in expressions: + script = f""" +import sys +from pathlib import Path +from zeus.private_io import UnsafeFileError, append_private_bytes, read_private_tail +path = Path(sys.argv[1]) +try: + {expression} +except UnsafeFileError: + raise SystemExit(0) +raise SystemExit(1) +""" + with self.subTest(expression=expression): + result = subprocess.run( + [sys.executable, "-c", script, str(path)], + check=False, + capture_output=True, + timeout=2, + ) + self.assertEqual(0, result.returncode, result.stderr.decode("utf-8", "replace")) + + def test_directory_leaf_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.mkdir() + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + def test_final_directory_owner_mismatch_is_rejected_before_chmod(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + + with ( + patch.object(private_io.os, "geteuid", return_value=os.geteuid() + 1), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(0o755, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_file_owner_mismatch_is_rejected_before_chmod_or_io(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + path.chmod(0o644) + real_fstat = os.fstat + + def mismatched_file_owner(fd: int) -> os.stat_result: + result = real_fstat(fd) + if stat.S_ISREG(result.st_mode): + fields = list(result) + fields[4] = result.st_uid + 1 + return os.stat_result(fields) + return result + + with ( + patch.object(private_io.os, "fstat", side_effect=mismatched_file_owner), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(b"original", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + + def test_file_replacement_between_lstat_and_open_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + replacement = private_dir / "replacement.log" + replacement.write_bytes(b"replacement") + displaced = private_dir / "displaced.log" + real_open = os.open + swapped = False + + def racing_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if not swapped and name == path.name and dir_fd is not None: + path.rename(displaced) + replacement.rename(path) + swapped = True + return real_open(name, flags, mode, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=racing_open), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertTrue(swapped) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"replacement", path.read_bytes()) + + def test_file_replacement_before_post_lstat_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + replacement = private_dir / "replacement.log" + replacement.write_bytes(b"replacement") + displaced = private_dir / "displaced.log" + real_lstat = os.lstat + leaf_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal leaf_lstats + if name == path.name and dir_fd is not None: + leaf_lstats += 1 + if leaf_lstats == 2: + path.rename(displaced) + replacement.rename(path) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(2, leaf_lstats) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"replacement", path.read_bytes()) + + def test_tail_rejects_leaf_replacement_after_read(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + displaced = private_dir / "displaced.log" + target = self.root / "external-target.log" + target.write_bytes(b"external") + target.chmod(0o644) + target_mode = stat.S_IMODE(target.stat().st_mode) + real_read = os.read + swapped = False + + def racing_read(fd: int, size: int) -> bytes: + nonlocal swapped + result = real_read(fd, size) + if result and not swapped and stat.S_ISREG(os.fstat(fd).st_mode): + path.rename(displaced) + path.symlink_to(target) + swapped = True + return result + + with ( + patch.object(private_io.os, "read", side_effect=racing_read), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(swapped) + self.assertTrue(path.is_symlink()) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"external", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + + def test_append_rejects_leaf_replacement_after_write(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + displaced = private_dir / "displaced.log" + target = self.root / "external-target.log" + target.write_bytes(b"external") + target.chmod(0o644) + target_mode = stat.S_IMODE(target.stat().st_mode) + real_fdopen = os.fdopen + swapped = False + + class RacingHandle: + def __init__(self, handle: BinaryIO) -> None: + self.handle = handle + + def write(self, data: bytes) -> int: + nonlocal swapped + written = self.handle.write(data) + if not swapped: + path.rename(displaced) + path.symlink_to(target) + swapped = True + return written + + def fileno(self) -> int: + return self.handle.fileno() + + def close(self) -> None: + self.handle.close() + + def racing_fdopen(fd: int, mode: str, buffering: int = -1) -> RacingHandle: + return RacingHandle(real_fdopen(fd, mode, buffering=buffering)) + + with ( + patch.object(private_io.os, "fdopen", side_effect=racing_fdopen), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertTrue(swapped) + self.assertTrue(path.is_symlink()) + self.assertEqual(b"originalevent", displaced.read_bytes()) + self.assertEqual(b"external", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + + def test_directory_replacement_between_lstat_and_open_is_rejected(self) -> None: + private_dir = self.root / "private" + logs = private_dir / "logs" + logs.mkdir(parents=True) + replacement = private_dir / "replacement" + replacement.mkdir() + displaced = private_dir / "displaced" + real_open = os.open + swapped = False + + def racing_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if not swapped and name == logs.name and dir_fd is not None: + logs.rename(displaced) + replacement.rename(logs) + swapped = True + return real_open(name, flags, mode, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=racing_open), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(logs / "events.log", b"event") + + self.assertTrue(swapped) + self.assertFalse((logs / "events.log").exists()) + + def test_fstat_identity_mismatch_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + decoy = private_dir / "decoy.log" + decoy.write_bytes(b"decoy") + decoy_stat = decoy.stat() + real_fstat = os.fstat + + def mismatched_file_identity(fd: int) -> os.stat_result: + result = real_fstat(fd) + if stat.S_ISREG(result.st_mode): + return decoy_stat + return result + + with ( + patch.object(private_io.os, "fstat", side_effect=mismatched_file_identity), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(b"original", path.read_bytes()) + + def test_all_descriptors_close_when_file_fchmod_fails(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + real_fchmod = os.fchmod + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def failing_file_fchmod(fd: int, mode: int) -> None: + if stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("file fchmod failed") + real_fchmod(fd, mode) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fchmod", side_effect=failing_file_fchmod), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + def test_all_descriptors_close_when_fdopen_fails(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=OSError("fdopen failed")), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + def test_file_close_failure_does_not_prevent_parent_cleanup(self) -> None: + path = self.root / "logs" / "events.log" + real_fdopen = os.fdopen + parent_fd: int | None = None + real_open = os.open + + class FailingCloseHandle: + def __init__(self, handle: BinaryIO) -> None: + self.handle = handle + + def write(self, data: bytes) -> int: + return self.handle.write(data) + + def close(self) -> None: + self.handle.close() + raise OSError("file close failed") + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal parent_fd + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + if name == path.name: + parent_fd = dir_fd + return descriptor + + def failing_close_fdopen(fd: int, mode: str, buffering: int = -1) -> FailingCloseHandle: + return FailingCloseHandle(real_fdopen(fd, mode, buffering=buffering)) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=failing_close_fdopen), + patch.object(private_io.os, "close", wraps=os.close) as close, + self.assertRaisesRegex(UnsafeFileError, "close"), + ): + append_private_bytes(path, b"event") + + self.assertIsNotNone(parent_fd) + close.assert_any_call(parent_fd) + with self.assertRaises(OSError): + os.fstat(parent_fd) # type: ignore[arg-type] + + def test_cleanup_errors_do_not_hide_primary_unsafe_error(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + real_close = os.close + fdopen_failed = False + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def failing_fdopen(*_args: object, **_kwargs: object) -> BinaryIO: + nonlocal fdopen_failed + fdopen_failed = True + raise UnsafeFileError("primary unsafe error") + + def noisy_close(fd: int) -> None: + real_close(fd) + if fdopen_failed: + raise OSError("cleanup noise") + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=failing_fdopen), + patch.object(private_io.os, "close", side_effect=noisy_close), + self.assertRaisesRegex(UnsafeFileError, "primary unsafe error"), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_process_identity.py b/tests/test_process_identity.py new file mode 100644 index 0000000..31b381f --- /dev/null +++ b/tests/test_process_identity.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import ast +import errno +import os +import subprocess +import sys +import tempfile +import unittest +from dataclasses import FrozenInstanceError +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import zeus.process_identity as identity +import zeus.supervisor as supervisor_module + + +class ProcessReaderTests(unittest.TestCase): + def test_linux_cmdline_parses_proc_bytes_and_handles_missing_process(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + proc_root = Path(tmp) / "proc" + pid_dir = proc_root / "4321" + pid_dir.mkdir(parents=True) + (pid_dir / "cmdline").write_bytes(b"hermes\0-p\0coder\0gateway\0run\0invalid-\xff\0") + + argv = identity.read_linux_cmdline(4321, proc_root=proc_root) + + self.assertEqual( + ["hermes", "-p", "coder", "gateway", "run", "invalid-\udcff"], + argv, + ) + self.assertEqual([], identity.read_linux_cmdline(9999, proc_root=proc_root)) + + def test_linux_start_fingerprint_reads_proc_stat_field_twenty_two(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + proc_root = Path(tmp) / "proc" + pid_dir = proc_root / "4321" + pid_dir.mkdir(parents=True) + fields = ["S", *["0"] * 18, "987654321", "0"] + (pid_dir / "stat").write_text( + f"4321 (hermes gateway) {' '.join(fields)}\n", + encoding="utf-8", + ) + + fingerprint = identity.read_linux_process_start_fingerprint( + 4321, + proc_root=proc_root, + ) + + self.assertEqual("linux:/proc-starttime:987654321", fingerprint) + self.assertIsNone( + identity.read_linux_process_start_fingerprint(9999, proc_root=proc_root) + ) + + def test_darwin_cmdline_uses_fixed_ps_command_and_parses_quotes(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout='"/Applications/Hermes CLI/hermes" -p coder gateway run\n', + ) + with patch("zeus.process_identity.subprocess.run", return_value=completed) as run: + argv = identity.read_darwin_cmdline(4321) + + self.assertEqual( + ["/Applications/Hermes CLI/hermes", "-p", "coder", "gateway", "run"], + argv, + ) + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_darwin_start_fingerprint_normalizes_ps_lstart(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout=" Mon Jun 29 16:54:50 2026\n", + ) + with patch("zeus.process_identity.subprocess.run", return_value=completed) as run: + fingerprint = identity.read_darwin_process_start_fingerprint(4321) + + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "lstart="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_platform_dispatchers_select_linux_darwin_or_unsupported(self) -> None: + with ( + patch("zeus.process_identity.platform.system", return_value="Linux"), + patch( + "zeus.process_identity.read_linux_cmdline", + return_value=["linux"], + ) as linux_cmdline, + patch( + "zeus.process_identity.read_linux_process_start_fingerprint", + return_value="linux-start", + ) as linux_start, + ): + self.assertEqual(["linux"], identity.read_process_cmdline(11)) + self.assertEqual("linux-start", identity.read_process_start_fingerprint(11)) + linux_cmdline.assert_called_once_with(11) + linux_start.assert_called_once_with(11) + + with ( + patch("zeus.process_identity.platform.system", return_value="Darwin"), + patch( + "zeus.process_identity.read_darwin_cmdline", + return_value=["darwin"], + ) as darwin_cmdline, + patch( + "zeus.process_identity.read_darwin_process_start_fingerprint", + return_value="darwin-start", + ) as darwin_start, + ): + self.assertEqual(["darwin"], identity.read_process_cmdline(22)) + self.assertEqual("darwin-start", identity.read_process_start_fingerprint(22)) + darwin_cmdline.assert_called_once_with(22) + darwin_start.assert_called_once_with(22) + + with patch("zeus.process_identity.platform.system", return_value="FreeBSD"): + self.assertIsNone(identity.read_process_cmdline(33)) + self.assertIsNone(identity.read_process_start_fingerprint(33)) + + +class CommandIdentityTests(unittest.TestCase): + @staticmethod + def _write_executable(path: Path, text: str = "#!/bin/sh\n") -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + return str(path.resolve()) + + def test_gateway_command_classifier_preserves_complete_shape_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_bin = self._write_executable(root / "bin" / "hermes") + fake_hermes = self._write_executable(root / "bin" / "fake-hermes") + cases = ( + (["hermes", "-p", "coder", "gateway", "run"], True, "direct-hermes", "ok"), + ([hermes_bin, "-p", "coder", "gateway", "run"], True, "direct-hermes", "ok"), + ( + [sys.executable, hermes_bin, "-p", "coder", "gateway", "run"], + True, + "python-script-wrapper", + "ok", + ), + ( + [sys.executable, fake_hermes, "-p", "coder", "gateway", "run"], + False, + "python-script-wrapper", + "untrusted-executable", + ), + ( + [sys.executable, hermes_bin, "-p", "other", "gateway", "run"], + False, + "python-script-wrapper", + "wrong-bot-id", + ), + ( + [sys.executable, hermes_bin, "-p", "coder", "doctor"], + False, + "python-script-wrapper", + "wrong-command-intent", + ), + ( + ["python", "-c", "sleep", "-p", "coder", "gateway", "run"], + False, + "python-script-wrapper", + "wrong-command-intent", + ), + (["sleep", "60"], False, "direct-hermes", "wrong-command-intent"), + ( + [hermes_bin, "gateway", "run"], + False, + "direct-hermes", + "wrong-command-intent", + ), + ( + [hermes_bin, "-p", "coder", "-p", "other", "gateway", "run"], + False, + "direct-hermes", + "wrong-command-intent", + ), + ) + with patch.dict(os.environ, {"PATH": str(root / "bin")}): + for argv, verified, classification, reason in cases: + with self.subTest(argv=argv): + check = identity.verify_gateway_command( + argv, + "coder", + hermes_bin, + require_trusted_path=True, + ) + self.assertEqual(verified, check.verified) + self.assertEqual(classification, check.classification) + self.assertEqual(reason, check.reason) + + def test_command_shape_and_python_interpreter_classification_are_bounded(self) -> None: + for command in ("python", "python3", "PYTHON3.11", "/usr/bin/python3.12"): + with self.subTest(command=command): + self.assertTrue(identity.looks_like_python_interpreter(command)) + for command in ("pypy3", "python3.11-config", "python-script", ""): + with self.subTest(command=command): + self.assertFalse(identity.looks_like_python_interpreter(command)) + + self.assertEqual("empty", identity.safe_command_shape([])) + self.assertEqual( + "direct-hermes hermes -p gateway run", + identity.safe_command_shape(["hermes", "-p", "secret-bot", "gateway", "run"]), + ) + self.assertEqual( + "python-script-wrapper hermes -p gateway run", + identity.safe_command_shape( + [sys.executable, "/opt/hermes", "-p", "secret-bot", "gateway", "run"] + ), + ) + self.assertEqual( + "direct-hermes unrecognized", + identity.safe_command_shape(["hermes", "doctor"]), + ) + + def test_executable_launcher_and_trusted_hermes_resolution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_bin = self._write_executable(root / "real" / "hermes") + path_hermes = self._write_executable(root / "bin" / "path-hermes") + launcher = root / "bin" / "hermes-launcher" + launcher_path = self._write_executable( + launcher, + f'#!/bin/sh\n# delegated install\nexec "{hermes_bin}" "$@"\n', + ) + + self.assertIsNone(identity.resolve_executable("")) + self.assertEqual( + path_hermes, identity.resolve_executable("path-hermes", str(root / "bin")) + ) + self.assertEqual(launcher_path, identity.resolve_executable(launcher_path)) + self.assertEqual(hermes_bin, identity.resolve_launcher_exec_target(launcher_path)) + self.assertEqual( + {launcher_path, hermes_bin}, + identity.trusted_hermes_paths(launcher_path), + ) + self.assertEqual({hermes_bin}, identity.trusted_hermes_paths(hermes_bin)) + + def test_launcher_resolution_rejects_untrusted_or_malformed_scripts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + not_hermes = self._write_executable(root / "bin" / "not-hermes") + cases = ( + ("plain", f'exec "{not_hermes}" "$@"\n'), + ("no-exec", "#!/bin/sh\nprintf ready\n"), + ("path-lookup", '#!/bin/sh\nexec hermes "$@"\n'), + ("malformed", "#!/bin/sh\nexec 'unterminated\n"), + ("wrong-target", f'#!/bin/sh\nexec "{not_hermes}" "$@"\n'), + ) + for name, text in cases: + with self.subTest(name=name): + script = root / name + script.write_text(text, encoding="utf-8") + self.assertIsNone(identity.resolve_launcher_exec_target(str(script))) + + +class PidStateTests(unittest.TestCase): + @staticmethod + def _raises(exc: BaseException) -> identity.PidAliveFn: + def raise_error(_pid: int) -> bool: + raise exc + + return raise_error + + def test_pid_state_classifies_callbacks_and_os_errors_exactly(self) -> None: + cases = ( + ("alive", lambda _pid: True, identity.PidState.alive), + ("false", lambda _pid: False, identity.PidState.dead), + ( + "esrch", + self._raises(OSError(errno.ESRCH, "no such process")), + identity.PidState.dead, + ), + ( + "eperm", + self._raises(OSError(errno.EPERM, "operation not permitted")), + identity.PidState.unknown, + ), + ( + "process-lookup", + self._raises(ProcessLookupError(errno.ESRCH, "no such process")), + identity.PidState.dead, + ), + ( + "permission", + self._raises(PermissionError(errno.EPERM, "operation not permitted")), + identity.PidState.unknown, + ), + ) + for name, callback, expected in cases: + with self.subTest(name=name): + self.assertIs(expected, identity.pid_state(4321, pid_alive_fn=callback)) + + def test_pid_state_default_probes_with_signal_zero(self) -> None: + with patch("zeus.process_identity.os.kill") as kill: + self.assertIs(identity.PidState.alive, identity.pid_state(4321)) + kill.assert_called_once_with(4321, 0) + + +class ProcessStartIdentityTests(unittest.TestCase): + def test_process_start_requirement_is_explicit_and_platform_bounded(self) -> None: + self.assertTrue(identity.process_start_fingerprint_required("Linux")) + self.assertTrue(identity.process_start_fingerprint_required("Darwin")) + self.assertFalse(identity.process_start_fingerprint_required("FreeBSD")) + self.assertFalse(identity.process_start_fingerprint_required("linux")) + + def test_process_start_fingerprint_validation_is_exact(self) -> None: + self.assertTrue(identity.valid_process_start_fingerprint("x")) + self.assertTrue(identity.valid_process_start_fingerprint("x" * 512)) + for value in (None, "", "x" * 513, 1, True, b"start"): + with self.subTest(value=value): + self.assertFalse(identity.valid_process_start_fingerprint(value)) + + def test_process_start_comparison_preserves_required_and_optional_semantics(self) -> None: + cases = ( + ("same", "same", True, None), + (None, "same", True, "process start fingerprint is unavailable"), + ("same", None, True, "process start fingerprint is unavailable"), + ("old", "new", True, "process start fingerprint does not match"), + (None, None, False, None), + ("same", "same", False, None), + (None, "live", False, "process start fingerprint does not match"), + ("marker", None, False, "process start fingerprint is unavailable"), + ("old", "new", False, "process start fingerprint does not match"), + ) + for marker, live, required, expected in cases: + with self.subTest(marker=marker, live=live, required=required): + self.assertEqual( + expected, + identity.process_start_identity_error( + marker, + live, + fingerprint_required=required, + ), + ) + + +class FrozenBoundaryTests(unittest.TestCase): + def test_identity_types_are_frozen_and_finite(self) -> None: + self.assertEqual( + {"alive", "dead", "unknown"}, + {state.value for state in identity.PidState}, + ) + check = identity.CommandCheck(True, "ok", "direct-hermes") + with self.assertRaises(FrozenInstanceError): + check.reason = "changed" # type: ignore[misc] + + def test_process_identity_has_only_standard_library_dependencies(self) -> None: + module_path = Path(identity.__file__) + source = module_path.read_text(encoding="utf-8") + tree = ast.parse(source) + imported_roots: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imported_roots.add(node.module.split(".", 1)[0]) + + self.assertNotIn("zeus", imported_roots) + self.assertNotIn("sqlite3", imported_roots) + self.assertNotIn("signal", imported_roots) + self.assertNotIn("StateStore", source) + self.assertNotIn("BotRecord", source) + + +class SupervisorCompatibilityTests(unittest.TestCase): + def test_legacy_types_and_pure_helpers_remain_direct_aliases(self) -> None: + self.assertIs(identity.PidState, supervisor_module._PidState) + self.assertIs(identity.CommandCheck, supervisor_module._CommandCheck) + self.assertIs(identity.PidAliveFn, supervisor_module.PidAliveFn) + self.assertIs(identity.CmdlineReader, supervisor_module.CmdlineReader) + self.assertIs( + identity.ProcStartFingerprintReader, + supervisor_module.ProcStartFingerprintReader, + ) + aliases = { + "_read_linux_cmdline": identity.read_linux_cmdline, + "_read_linux_process_start_fingerprint": ( + identity.read_linux_process_start_fingerprint + ), + "_verify_gateway_command": identity.verify_gateway_command, + "_looks_like_python_interpreter": identity.looks_like_python_interpreter, + "_resolve_executable": identity.resolve_executable, + "_trusted_hermes_paths": identity.trusted_hermes_paths, + "_resolve_launcher_exec_target": identity.resolve_launcher_exec_target, + "_safe_command_shape": identity.safe_command_shape, + } + for old_name, new_value in aliases.items(): + with self.subTest(old_name=old_name): + self.assertIs(new_value, getattr(supervisor_module, old_name)) + + def test_legacy_cmdline_dispatch_uses_current_supervisor_globals(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout='"/patched/hermes" -p coder gateway run\n', + ) + with ( + patch("zeus.supervisor.platform.system", return_value="Darwin") as system, + patch("zeus.supervisor.subprocess.run", return_value=completed) as run, + ): + argv = supervisor_module._read_process_cmdline(4321) + + self.assertEqual(["/patched/hermes", "-p", "coder", "gateway", "run"], argv) + system.assert_called_once_with() + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_legacy_direct_darwin_readers_use_current_supervisor_subprocess(self) -> None: + command = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="/patched/hermes -p coder gateway run\n", + ) + started = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="Mon Jun 29 16:54:50 2026\n", + ) + with patch("zeus.supervisor.subprocess.run", side_effect=[command, started]) as run: + argv = supervisor_module._read_darwin_cmdline(4321) + fingerprint = supervisor_module._read_darwin_process_start_fingerprint(4321) + + self.assertEqual(["/patched/hermes", "-p", "coder", "gateway", "run"], argv) + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + self.assertEqual(2, run.call_count) + + def test_legacy_start_dispatch_uses_current_platform_and_subprocess(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="Mon Jun 29 16:54:50 2026\n", + ) + with ( + patch("zeus.supervisor.platform.system", return_value="Darwin") as system, + patch("zeus.supervisor.subprocess.run", return_value=completed) as run, + ): + fingerprint = supervisor_module._read_process_start_fingerprint(4321) + + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + system.assert_called_once_with() + run.assert_called_once() + + def test_supervisor_pid_state_shim_uses_live_callback_or_current_os_kill(self) -> None: + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.pid_alive_fn = None + with patch("zeus.supervisor.os.kill") as kill: + self.assertIs(identity.PidState.alive, supervisor._pid_state(4321)) + supervisor.pid_alive_fn = lambda _pid: False + self.assertIs(identity.PidState.dead, supervisor._pid_state(4321)) + kill.assert_called_once_with(4321, 0) + + supervisor.pid_alive_fn = self._permission_denied + self.assertIs(identity.PidState.unknown, supervisor._pid_state(4321)) + + @staticmethod + def _permission_denied(_pid: int) -> bool: + raise PermissionError(errno.EPERM, "operation not permitted") + + def test_supervisor_process_start_shims_read_live_callback_and_platform(self) -> None: + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.proc_start_fingerprint_reader = lambda _pid: "same" + with patch("zeus.supervisor.platform.system", return_value="Linux"): + self.assertTrue(supervisor._process_start_fingerprint_required()) + self.assertTrue(supervisor._valid_marker_start("same")) + self.assertIsNone( + supervisor._process_start_identity_error( + {"proc_start_fingerprint": "same"}, + 4321, + ) + ) + supervisor.proc_start_fingerprint_reader = lambda _pid: "reused" + self.assertEqual( + "process start fingerprint does not match", + supervisor._process_start_identity_error( + {"proc_start_fingerprint": "same"}, + 4321, + ), + ) + + with patch("zeus.supervisor.platform.system", return_value="FreeBSD"): + self.assertFalse(supervisor._process_start_fingerprint_required()) + + def test_supervisor_hermes_resolution_shims_use_current_adapter_value(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + hermes = Path(tmp) / "hermes" + hermes.write_text("#!/bin/sh\n", encoding="utf-8") + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.adapter = SimpleNamespace(hermes_bin=str(hermes)) + + self.assertEqual(str(hermes.resolve()), supervisor._resolved_hermes_bin()) + self.assertEqual({str(hermes.resolve())}, supervisor._trusted_hermes_bins()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_profile_manager.py b/tests/test_profile_manager.py new file mode 100644 index 0000000..38fb4cf --- /dev/null +++ b/tests/test_profile_manager.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import ast +import inspect +import json +import os +import tempfile +import unittest +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from unittest.mock import patch + +import zeus.profile_manager as profile_manager_module +from zeus.errors import BotArchiveError, BotDeleteError +from zeus.models import BotCreateRequest, BotRecord +from zeus.profile_manager import ProfileArchive, ProfileDeletion, ProfileManager +from zeus.state import StateStore +from zeus.supervisor import Supervisor +from zeus.templates import TemplateStore + + +class ProfileManagerTests(unittest.TestCase): + def _manager(self, root: Path) -> ProfileManager: + return ProfileManager(root / "hermes", root / "archive") + + def _profile(self, root: Path) -> Path: + return root / "hermes" / "profiles" / "coder" + + def _write_profile(self, root: Path) -> Path: + profile = self._profile(root) + profile.mkdir(parents=True) + (profile / "sentinel.txt").write_text("original\n", encoding="utf-8") + return profile + + def _supervisor_with_bot(self, root: Path) -> tuple[StateStore, Supervisor, Path]: + profile = self._write_profile(root) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + ) + ) + return store, Supervisor(store, "hermes", root / "hermes"), profile + + def test_preflight_delegates_without_mutating_the_filesystem(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = TemplateStore().get("coding-bot") + + rendered = manager.preflight(request, template) + + self.assertEqual(template.soul.rstrip() + "\n", rendered["SOUL.md"]) + self.assertFalse((root / "hermes").exists()) + + def test_install_transaction_removes_a_new_profile_on_caller_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._profile(root) + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = TemplateStore().get("coding-bot") + + with ( + self.assertRaisesRegex(RuntimeError, "database unavailable"), + manager.install_transaction(request, template) as record, + ): + self.assertEqual("coder", record.bot_id) + self.assertTrue(profile.is_dir()) + raise RuntimeError("database unavailable") + + self.assertFalse(os.path.lexists(profile)) + self.assertEqual([], list(profile.parent.iterdir())) + + def test_install_transaction_restores_an_exact_replaced_profile_on_caller_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + (profile / "SOUL.md").write_text("original profile\n", encoding="utf-8") + original_inode = profile.stat().st_ino + before = { + path.relative_to(profile): path.read_bytes() + for path in profile.rglob("*") + if path.is_file() + } + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = replace(TemplateStore().get("coding-bot"), soul="replacement profile") + + with ( + self.assertRaisesRegex(RuntimeError, "database unavailable"), + manager.install_transaction(request, template), + ): + self.assertEqual( + "replacement profile\n", + (profile / "SOUL.md").read_text(encoding="utf-8"), + ) + raise RuntimeError("database unavailable") + + after = { + path.relative_to(profile): path.read_bytes() + for path in profile.rglob("*") + if path.is_file() + } + self.assertEqual(original_inode, profile.stat().st_ino) + self.assertEqual(before, after) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + + def test_delete_stage_rollback_and_finish_use_a_pinned_token(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + + deletion = manager.stage_delete("coder", str(profile)) + + self.assertIsInstance(deletion, ProfileDeletion) + assert deletion is not None + self.assertEqual(profile.resolve(), deletion.profile_path) + self.assertTrue(deletion.tombstone_path.is_dir()) + self.assertFalse(os.path.lexists(profile)) + + manager.rollback_delete(deletion) + + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + self.assertFalse(os.path.lexists(deletion.tombstone_path)) + + second_deletion = manager.stage_delete("coder", str(profile)) + assert second_deletion is not None + self.assertIsNone(manager.finish_delete(second_deletion)) + self.assertFalse(os.path.lexists(profile)) + self.assertFalse(os.path.lexists(second_deletion.tombstone_path)) + + def test_finish_delete_reports_cleanup_failure_and_retains_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + cleanup_error = OSError("cleanup unavailable") + + with patch("zeus.profile_manager.shutil.rmtree", side_effect=cleanup_error): + reported = manager.finish_delete(deletion) + + self.assertIs(cleanup_error, reported) + self.assertTrue(deletion.tombstone_path.is_dir()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_stage_and_rollback_use_a_pinned_token(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + + archive = manager.stage_archive("coder", str(profile)) + + self.assertIsInstance(archive, ProfileArchive) + assert archive is not None + self.assertEqual(profile.resolve(), archive.profile_path) + self.assertTrue(archive.archive_path.is_dir()) + self.assertFalse(os.path.lexists(profile)) + + manager.rollback_archive(archive) + + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + self.assertFalse(os.path.lexists(archive.archive_path)) + + def test_validate_profile_path_accepts_only_the_exact_bot_profile(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + hermes_root = root / "hermes" + profiles_root = hermes_root / "profiles" + exact_profile = profiles_root / "coder" + exact_profile.mkdir(parents=True) + + self.assertEqual( + exact_profile.resolve(), + manager.validate_profile_path("coder", str(exact_profile)), + ) + + rejected = { + "Hermes root": hermes_root, + "profiles root": profiles_root, + "sibling profiles tree": hermes_root.parent / "other-profiles" / "coder", + "other bot": profiles_root / "other-bot", + "nested path": exact_profile / "nested", + "outside path": root / "outside", + } + for label, candidate in rejected.items(): + with self.subTest(label=label), self.assertRaises(BotDeleteError): + manager.validate_profile_path("coder", str(candidate)) + + def test_delete_rollback_refuses_an_occupied_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + profile.mkdir() + replacement = profile / "replacement.txt" + replacement.write_text("replacement\n", encoding="utf-8") + + with ( + patch.object(manager, "finish_delete") as finish_delete, + self.assertRaises(BotDeleteError), + ): + manager.rollback_delete(deletion) + + finish_delete.assert_not_called() + self.assertEqual("replacement\n", replacement.read_text(encoding="utf-8")) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_delete_rollback_refuses_a_dangling_symlink_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + target = root / "untouched-delete-target" + profile.symlink_to(target, target_is_directory=True) + + with ( + patch.object(manager, "finish_delete") as finish_delete, + self.assertRaises(BotDeleteError), + ): + manager.rollback_delete(deletion) + + finish_delete.assert_not_called() + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_rollback_refuses_an_occupied_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + archive = manager.stage_archive("coder", str(profile)) + assert archive is not None + profile.mkdir() + replacement = profile / "replacement.txt" + replacement.write_text("replacement\n", encoding="utf-8") + + with ( + patch("zeus.profile_manager.shutil.rmtree") as cleanup, + self.assertRaises(BotArchiveError), + ): + manager.rollback_archive(archive) + + cleanup.assert_not_called() + self.assertEqual("replacement\n", replacement.read_text(encoding="utf-8")) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_rollback_refuses_a_dangling_symlink_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + archive = manager.stage_archive("coder", str(profile)) + assert archive is not None + target = root / "untouched-archive-target" + profile.symlink_to(target, target_is_directory=True) + + with ( + patch("zeus.profile_manager.shutil.rmtree") as cleanup, + self.assertRaises(BotArchiveError), + ): + manager.rollback_archive(archive) + + cleanup.assert_not_called() + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_supervisor_delete_carries_the_exact_staged_token_into_finish(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + real_finish_delete = manager.finish_delete + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def capture_finish(deletion: ProfileDeletion) -> OSError | None: + self.assertIs(staged[0], deletion) + return real_finish_delete(deletion) + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete", side_effect=capture_finish) as finish, + ): + response = supervisor.delete_bot("coder", remove_profile=True) + + self.assertEqual("deleted", response.message) + self.assertEqual(1, finish.call_count) + self.assertIsNone(store.get_bot("coder")) + self.assertFalse(os.path.lexists(profile)) + + def test_supervisor_delete_cleanup_failure_stays_post_commit_and_is_audited(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + cleanup_error = PermissionError("cleanup unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete", return_value=cleanup_error), + ): + response = supervisor.delete_bot("coder", remove_profile=True) + + self.assertEqual("deleted; profile cleanup is pending", response.message) + self.assertIsNone(store.get_bot("coder")) + self.assertFalse(os.path.lexists(profile)) + self.assertEqual( + "original\n", + (staged[0].tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + events = [ + json.loads(line) + for line in store.audit_log_path().read_text(encoding="utf-8").splitlines() + ] + self.assertTrue( + any( + event.get("event") == "bot.delete_cleanup_pending" + and event.get("error") == "PermissionError" + for event in events + ) + ) + self.assertTrue( + any( + event.get("event") == "bot.delete" + and event.get("profile_removed") is True + and event.get("cleanup_pending") is True + for event in events + ) + ) + + def test_supervisor_delete_carries_the_exact_staged_token_into_rollback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + real_rollback_delete = manager.rollback_delete + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def capture_rollback(deletion: ProfileDeletion) -> None: + self.assertIs(staged[0], deletion) + real_rollback_delete(deletion) + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "rollback_delete", side_effect=capture_rollback) as rollback, + patch.object(manager, "finish_delete") as finish, + patch.object(store, "delete_bot_with_event", side_effect=database_error), + self.assertRaisesRegex(RuntimeError, "database unavailable") as raised, + ): + supervisor.delete_bot("coder", remove_profile=True) + + self.assertIs(database_error, raised.exception) + self.assertEqual(1, rollback.call_count) + finish.assert_not_called() + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + + def test_supervisor_archive_carries_the_exact_staged_token_into_rollback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileArchive] = [] + real_stage_archive = manager.stage_archive + real_rollback_archive = manager.rollback_archive + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileArchive | None: + archive = real_stage_archive(bot_id, profile_path) + assert archive is not None + staged.append(archive) + return archive + + def capture_rollback(archive: ProfileArchive) -> None: + self.assertIs(staged[0], archive) + real_rollback_archive(archive) + + with ( + patch.object(manager, "stage_archive", side_effect=capture_stage), + patch.object( + manager, + "rollback_archive", + side_effect=capture_rollback, + ) as rollback, + patch.object(store, "delete_bot_with_event", side_effect=database_error), + self.assertRaisesRegex(RuntimeError, "database unavailable") as raised, + ): + supervisor.archive_bot("coder") + + self.assertIs(database_error, raised.exception) + self.assertEqual(1, rollback.call_count) + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + + def test_supervisor_delete_chains_refused_compensation_from_database_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def occupy_destination(*args: object, **kwargs: object) -> bool: + profile.mkdir() + (profile / "replacement.txt").write_text("replacement\n", encoding="utf-8") + raise database_error + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete") as finish, + patch.object(store, "delete_bot_with_event", side_effect=occupy_destination), + self.assertRaises(BotDeleteError) as raised, + ): + supervisor.delete_bot("coder", remove_profile=True) + + self.assertIs(database_error, raised.exception.__cause__) + finish.assert_not_called() + self.assertEqual( + "replacement\n", + (profile / "replacement.txt").read_text(encoding="utf-8"), + ) + self.assertEqual( + "original\n", + (staged[0].tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_supervisor_archive_chains_refused_compensation_from_database_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileArchive] = [] + real_stage_archive = manager.stage_archive + database_error = RuntimeError("database unavailable") + target = root / "untouched-supervisor-archive-target" + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileArchive | None: + archive = real_stage_archive(bot_id, profile_path) + assert archive is not None + staged.append(archive) + return archive + + def occupy_destination(*args: object, **kwargs: object) -> bool: + profile.symlink_to(target, target_is_directory=True) + raise database_error + + with ( + patch.object(manager, "stage_archive", side_effect=capture_stage), + patch.object(store, "delete_bot_with_event", side_effect=occupy_destination), + self.assertRaises(BotArchiveError) as raised, + ): + supervisor.archive_bot("coder") + + self.assertIs(database_error, raised.exception.__cause__) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (staged[0].archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_restore_wrappers_reject_unvalidated_destination_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + supervisor = Supervisor(store, "hermes", root / "hermes") + outside = root / "outside" / "coder" + tombstone = root / "delete-evidence" + archive_path = root / "archive-evidence" + for evidence in (tombstone, archive_path): + evidence.mkdir() + (evidence / "sentinel.txt").write_text("retain\n", encoding="utf-8") + + with self.assertRaises(BotDeleteError): + supervisor._restore_tombstoned_profile("coder", str(outside), tombstone) + with self.assertRaises(BotDeleteError): + supervisor._restore_archived_profile("coder", str(outside), archive_path) + + self.assertFalse(os.path.lexists(outside)) + self.assertEqual( + "retain\n", + (tombstone / "sentinel.txt").read_text(encoding="utf-8"), + ) + self.assertEqual( + "retain\n", + (archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_delete_restore_treats_an_exact_dangling_symlink_as_occupied(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + deletion = supervisor._profile_manager.stage_delete("coder", str(profile)) + assert deletion is not None + target = root / "untouched-legacy-delete-target" + profile.symlink_to(target, target_is_directory=True) + + with self.assertRaisesRegex(BotDeleteError, "original path is occupied"): + supervisor._restore_tombstoned_profile( + "coder", + str(profile), + deletion.tombstone_path, + ) + + self.assertIsNotNone(store.get_bot("coder")) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_archive_restore_treats_an_exact_dangling_symlink_as_occupied(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + archive = supervisor._profile_manager.stage_archive("coder", str(profile)) + assert archive is not None + target = root / "untouched-legacy-archive-target" + profile.symlink_to(target, target_is_directory=True) + + with self.assertRaisesRegex(BotArchiveError, "original path is occupied"): + supervisor._restore_archived_profile( + "coder", + str(profile), + archive.archive_path, + ) + + self.assertIsNotNone(store.get_bot("coder")) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_transaction_tokens_are_frozen(self) -> None: + deletion = ProfileDeletion(Path("profile"), Path("tombstone")) + archive = ProfileArchive(Path("profile"), Path("archive")) + + with self.assertRaises(FrozenInstanceError): + deletion.profile_path = Path("replacement") + with self.assertRaises(FrozenInstanceError): + archive.profile_path = Path("replacement") + + def test_module_has_no_state_runtime_or_process_dependency(self) -> None: + source = inspect.getsource(profile_manager_module) + tree = ast.parse(source) + imported_roots: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imported_roots.add(node.module.split(".", 1)[0]) + + self.assertTrue({"sqlite3", "signal", "subprocess"}.isdisjoint(imported_roots)) + self.assertNotIn("StateStore", source) + self.assertNotIn("GatewayRuntime", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 6896b79..240c244 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -1,12 +1,21 @@ from __future__ import annotations +import contextlib import json +import os +import socket +import socketserver import threading import unittest +from email.message import Message +from http.client import HTTPException from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from unittest.mock import patch +import zeus.readiness as readiness_module from zeus.readiness import ( + MAX_READINESS_RESPONSE_BYTES, ReadinessProbe, probe_once, readiness_probe_from_env, @@ -29,7 +38,39 @@ def log_message(self, format: str, *args: Any) -> None: return +class _IPv6ThreadingHTTPServer(ThreadingHTTPServer): + address_family = socket.AF_INET6 + + class ReadinessTests(unittest.TestCase): + def _run_server( + self, + handler: type[BaseHTTPRequestHandler], + *, + ipv6: bool = False, + ) -> tuple[ThreadingHTTPServer, threading.Thread]: + server_class = _IPv6ThreadingHTTPServer if ipv6 else ThreadingHTTPServer + host = "::1" if ipv6 else "127.0.0.1" + server = server_class((host, 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server, thread + + def _run_raw_server( + self, + handler: type[socketserver.BaseRequestHandler], + ) -> tuple[socketserver.ThreadingTCPServer, threading.Thread]: + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server, thread + def test_readiness_probe_from_env_builds_loopback_health_url(self) -> None: probe = readiness_probe_from_env( { @@ -74,6 +115,338 @@ def test_probe_once_accepts_expected_health_payload(self) -> None: self.assertTrue(result.ready) self.assertEqual("ready", result.message) + def test_probe_once_accepts_expected_health_payload_over_ipv6(self) -> None: + try: + server, _thread = self._run_server(_HealthHandler, ipv6=True) + except OSError as exc: + self.skipTest(f"IPv6 loopback is unavailable: {exc}") + + result = probe_once(f"http://[::1]:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_accepts_health_payload_without_content_length(self) -> None: + class NoContentLengthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + data = json.dumps(_HealthHandler.payload).encode("utf-8") + self.send_response(200) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(NoContentLengthHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_accepts_fixed_length_http_11_response_without_eof(self) -> None: + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + release_connection = threading.Event() + + class KeepAliveFixedLengthHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + self.request.recv(4096) + self.request.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: application/json\r\n" + + f"Content-Length: {len(health_payload)}\r\n".encode("ascii") + + b"Connection: keep-alive\r\n\r\n" + + health_payload + ) + release_connection.wait(1) + + server, _thread = self._run_raw_server(KeepAliveFixedLengthHandler) + + try: + result = probe_once( + f"http://127.0.0.1:{server.server_address[1]}/health", + timeout_seconds=0.2, + ) + finally: + release_connection.set() + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_accepts_chunked_health_payload(self) -> None: + class ChunkedHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + data = json.dumps(_HealthHandler.payload).encode("utf-8") + self.send_response(200) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(data):x}\r\n".encode("ascii")) + self.wfile.write(data + b"\r\n0\r\n\r\n") + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(ChunkedHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_does_not_follow_redirects_or_leak_location(self) -> None: + class RedirectTargetHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + target, _target_thread = self._run_server(RedirectTargetHandler) + secret = "API_KEY=sentinel-secret" + + class RedirectHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(302) + self.send_header( + "location", + f"http://127.0.0.1:{target.server_port}/health?{secret}", + ) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + return + + redirect, _redirect_thread = self._run_server(RedirectHandler) + + result = probe_once(f"http://127.0.0.1:{redirect.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("readiness endpoint returned an HTTP error", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + self.assertEqual(0, RedirectTargetHandler.requests) + + def test_probe_once_ignores_proxy_environment(self) -> None: + secret = "API_KEY=sentinel-secret" + + class TargetHandler(BaseHTTPRequestHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + data = json.dumps({"status": "not-ready", "detail": secret}).encode("utf-8") + self.send_response(200) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + class ProxyHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + target, _target_thread = self._run_server(TargetHandler) + proxy, _proxy_thread = self._run_server(ProxyHandler) + proxy_url = f"http://127.0.0.1:{proxy.server_port}" + proxy_env = { + "HTTP_PROXY": proxy_url, + "http_proxy": proxy_url, + "NO_PROXY": "", + "no_proxy": "", + } + + with ( + patch.dict(os.environ, proxy_env, clear=True), + patch.object( + readiness_module, + "build_opener", + wraps=readiness_module.build_opener, + ) as build_opener, + ): + result = probe_once(f"http://127.0.0.1:{target.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("unexpected readiness health payload", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + self.assertEqual(1, TargetHandler.requests) + self.assertEqual(0, ProxyHandler.requests) + build_opener.assert_called_once() + + def test_probe_once_rejects_oversized_responses_with_bounded_failures(self) -> None: + max_response_bytes = 64 * 1024 + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + oversized_body = health_payload + b" " * max_response_bytes + + def oversized_handler(include_content_length: bool) -> type[BaseHTTPRequestHandler]: + class OversizedHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + if include_content_length: + self.send_header("content-length", str(len(oversized_body))) + self.end_headers() + with contextlib.suppress(BrokenPipeError): + self.wfile.write(oversized_body) + + def log_message(self, format: str, *args: Any) -> None: + return + + return OversizedHandler + + for include_content_length in (True, False): + with self.subTest(include_content_length=include_content_length): + server, _thread = self._run_server(oversized_handler(include_content_length)) + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("readiness response exceeds size limit", result.message) + self.assertIsNone(result.payload) + + def test_probe_once_does_not_read_past_fixed_content_length(self) -> None: + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + connection_data = health_payload + b" " * MAX_READINESS_RESPONSE_BYTES + + class FixedLengthWithTrailingConnectionDataHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("content-length", str(len(health_payload))) + self.end_headers() + with contextlib.suppress(BrokenPipeError, ConnectionResetError): + self.wfile.write(connection_data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(FixedLengthWithTrailingConnectionDataHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_requests_exactly_one_max_plus_one_bounded_read(self) -> None: + class TrackingResponse: + def __init__(self) -> None: + self.headers = Message() + self.length: int | None = None + self.read_amounts: list[int] = [] + + def __enter__(self) -> TrackingResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def read(self, amount: int) -> bytes: + self.read_amounts.append(amount) + return b"x" * amount + + class TrackingOpener: + def __init__(self, response: TrackingResponse) -> None: + self.response = response + self.requests: list[object] = [] + + def open(self, *args: object, **kwargs: object) -> TrackingResponse: + self.requests.append(args[0]) + return self.response + + response = TrackingResponse() + opener = TrackingOpener(response) + + with patch.object(readiness_module, "_build_readiness_opener", return_value=opener): + result = probe_once("http://127.0.0.1:4312/health") + + self.assertEqual([MAX_READINESS_RESPONSE_BYTES + 1], response.read_amounts) + request = opener.requests[0] + get_header = getattr(request, "get_header", lambda _name: None) + self.assertEqual("close", get_header("Connection")) + self.assertFalse(result.ready) + self.assertEqual("readiness response exceeds size limit", result.message) + self.assertIsNone(result.payload) + + def test_probe_once_maps_malformed_http_to_safe_failure(self) -> None: + secret = "API_KEY=sentinel-secret" + + class MalformedProtocolHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + self.request.recv(4096) + self.request.sendall(f"NOT-HTTP {secret}\r\n\r\n".encode("ascii")) + + server, _thread = self._run_raw_server(MalformedProtocolHandler) + + try: + result = probe_once(f"http://127.0.0.1:{server.server_address[1]}/health") + except HTTPException: + result = None + + self.assertIsNotNone(result) + assert result is not None + self.assertFalse(result.ready) + self.assertEqual("readiness endpoint returned a malformed HTTP response", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + + def test_probe_once_rejects_url_data_before_network_io(self) -> None: + class TrackingHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + server, _thread = self._run_server(TrackingHandler) + port = server.server_port + urls = ( + f"http://user:password@127.0.0.1:{port}/health", + f"http://127.0.0.1:{port}/health?API_KEY=sentinel-secret", + f"http://127.0.0.1:{port}/health#API_KEY=sentinel-secret", + ) + + for url in urls: + with self.subTest(url=url): + result = probe_once(url) + self.assertFalse(result.ready) + self.assertEqual("readiness URL must be loopback HTTP", result.message) + self.assertIsNone(result.payload) + self.assertNotIn("sentinel-secret", repr(result)) + + self.assertEqual(0, TrackingHandler.requests) + + def test_probe_once_never_returns_unexpected_payload_data(self) -> None: + secret = "API_KEY=sentinel-secret" + + class SecretHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + data = json.dumps({"status": "not-ready", "detail": secret}).encode("utf-8") + self.send_response(200) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(SecretHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("unexpected readiness health payload", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + def test_wait_until_ready_returns_timeout_result(self) -> None: result = wait_until_ready( ReadinessProbe( diff --git a/tests/test_reconcile_store.py b/tests/test_reconcile_store.py new file mode 100644 index 0000000..b235c68 --- /dev/null +++ b/tests/test_reconcile_store.py @@ -0,0 +1,542 @@ +from __future__ import annotations + +import ast +import inspect +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, call, patch + +from zeus.config import SQLiteSynchronous +from zeus.idempotency_store import IdempotencyStore +from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS, ReconcileStore +from zeus.reconciliation import ( + BotReconcileResult, + PersistedReconcileRun, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, + summarize_results, +) +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import RECONCILE_COUNTER_COLUMNS as STATE_RECONCILE_COUNTER_COLUMNS +from zeus.state import StateStore + +RUN_STARTED_AT = datetime(2026, 7, 12, 11, 59, tzinfo=UTC) +RESULT_STARTED_AT = datetime(2026, 7, 12, 12, 0, tzinfo=UTC) +RESULT_FINISHED_AT = RESULT_STARTED_AT + timedelta(seconds=1) +RUN_FINISHED_AT = datetime(2026, 7, 12, 12, 1, tzinfo=UTC) + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +def _run( + run_id: str = "run-1", + *, + scope: str = "fleet", + requested_bot_id: str | None = None, +) -> ReconcileRunStart: + return ReconcileRunStart( + run_id=run_id, + scope=scope, + requested_bot_id=requested_bot_id, + source="cli", + force=False, + reset_restart=False, + started_at=RUN_STARTED_AT, + ) + + +def _result( + bot_id: str = "coder", + outcome: ReconcileOutcome = ReconcileOutcome.healthy, + *, + event_id: int | None = None, +) -> BotReconcileResult: + return BotReconcileResult( + bot_id=bot_id, + outcome=outcome, + desired_state="running", + observed_status="running", + pid=1234, + action="none" if outcome is ReconcileOutcome.healthy else "start", + message="reconciled", + error_code=None, + event_id=event_id, + started_at=RESULT_STARTED_AT, + finished_at=RESULT_FINISHED_AT, + ) + + +def _summary( + run: ReconcileRunStart, + results: list[BotReconcileResult], +) -> ReconcileRunSummary: + return summarize_results( + run.run_id, + run.scope, + results, + started_at=run.started_at, + finished_at=RUN_FINISHED_AT, + ) + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _connection_id, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statements(trace: list[tuple[int, str]]) -> list[str]: + return [sql for _connection_id, sql in trace] + + +def _statement_index(statements: list[str], prefix: str, *, start: int = 0) -> int: + return next( + index + for index, statement in enumerate(statements[start:], start=start) + if statement.lstrip().upper().startswith(prefix) + ) + + +def _run_bytes(database_path: Path, run_id: str) -> tuple[object, ...] | None: + columns = ( + "run_id", + "scope", + "requested_bot_id", + "source", + "force", + "reset_restart", + "started_at", + "finished_at", + "outcome", + "total", + "healthy_count", + "changed_count", + "pending_count", + "action_required_count", + "error_count", + "skipped_count", + ) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + with closing(sqlite3.connect(database_path)) as conn: + return conn.execute( + f"SELECT {fields} FROM reconcile_runs WHERE run_id = ?", + (run_id,), + ).fetchone() + + +class ReconcileStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.database_path = Path(self.temporary_directory.name) / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = ReconcileStore(self.database) + + def test_constructs_without_opening_database_and_owns_no_schema_or_lifecycle_store( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + ReconcileStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + import zeus.reconcile_store as reconcile_store_module + + source = inspect.getsource(reconcile_store_module) + tree = ast.parse(source) + imported_modules = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + } + self.assertTrue( + imported_modules.isdisjoint( + { + "zeus.bot_lifecycle_store", + "zeus.lifecycle", + "zeus.schema", + "zeus.state", + } + ) + ) + self.assertNotIn("CREATE TABLE", source.upper()) + self.assertNotIn("CREATE TRIGGER", source.upper()) + + def test_direct_store_round_trip_and_interruption(self) -> None: + run = _run() + result = _result(outcome=ReconcileOutcome.changed) + + self.store.begin_reconcile_run(run) + self.store.append_reconcile_result(run.run_id, result) + finished = self.store.finish_reconcile_run(_summary(run, [result])) + loaded = self.store.get_reconcile_run(run.run_id) + + self.assertIsInstance(finished, PersistedReconcileRun) + self.assertEqual(finished, loaded) + self.assertEqual("succeeded", finished.outcome) + self.assertEqual((result,), finished.results) + self.assertEqual(1, finished.counts[ReconcileOutcome.changed.value]) + + stale = _run("run-stale") + self.store.begin_reconcile_run(stale) + interrupted_at = RUN_FINISHED_AT + timedelta(minutes=1) + self.assertEqual( + 1, + self.store.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at), + ) + interrupted = self.store.get_reconcile_run(stale.run_id) + self.assertIsNotNone(interrupted) + self.assertEqual("interrupted", interrupted.outcome if interrupted else None) + self.assertEqual(interrupted_at, interrupted.finished_at if interrupted else None) + + def test_all_five_operations_keep_exact_transaction_boundaries(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("trace.db")) + SchemaManager(database).init() + database.traces.clear() + store = ReconcileStore(database) + run = _run() + result = _result() + + store.begin_reconcile_run(run) + begin_trace = list(database.traces) + database.traces.clear() + + store.append_reconcile_result(run.run_id, result) + append_trace = list(database.traces) + database.traces.clear() + + store.finish_reconcile_run(_summary(run, [result])) + finish_trace = list(database.traces) + database.traces.clear() + + loaded = store.get_reconcile_run(run.run_id) + get_trace = list(database.traces) + + store.begin_reconcile_run(_run("run-stale-a")) + store.begin_reconcile_run(_run("run-stale-b")) + database.traces.clear() + interrupted = store.interrupt_stale_reconcile_runs(interrupted_at=RUN_FINISHED_AT) + interrupt_trace = list(database.traces) + + self.assertIsNotNone(loaded) + self.assertEqual(2, interrupted) + for trace in (begin_trace, append_trace, finish_trace, get_trace, interrupt_trace): + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(begin_trace)) + begin_sql = _statements(begin_trace) + self.assertLess( + _statement_index(begin_sql, "BEGIN IMMEDIATE"), + _statement_index(begin_sql, "INSERT INTO RECONCILE_RUNS"), + ) + self.assertLess( + _statement_index(begin_sql, "INSERT INTO RECONCILE_RUNS"), + _statement_index(begin_sql, "COMMIT"), + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(append_trace)) + append_sql = _statements(append_trace) + result_insert = _statement_index(append_sql, "INSERT INTO RECONCILE_RESULTS") + counter_update = _statement_index(append_sql, "UPDATE RECONCILE_RUNS") + self.assertLess(result_insert, counter_update) + self.assertLess(counter_update, _statement_index(append_sql, "COMMIT")) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(finish_trace)) + finish_sql = _statements(finish_trace) + finish_update = _statement_index(finish_sql, "UPDATE RECONCILE_RUNS") + run_selects = [ + index + for index, sql in enumerate(finish_sql) + if sql.lstrip().upper().startswith("SELECT * FROM RECONCILE_RUNS") + ] + result_selects = [ + index + for index, sql in enumerate(finish_sql) + if sql.lstrip().upper().startswith("SELECT * FROM RECONCILE_RESULTS") + ] + self.assertEqual(2, len(run_selects)) + self.assertEqual(2, len(result_selects)) + self.assertLess(run_selects[0], result_selects[0]) + self.assertLess(result_selects[0], finish_update) + self.assertLess(finish_update, run_selects[1]) + self.assertLess(run_selects[1], result_selects[1]) + self.assertLess(result_selects[1], _statement_index(finish_sql, "COMMIT")) + + self.assertEqual(["BEGIN", "COMMIT"], _control_statements(get_trace)) + get_sql = _statements(get_trace) + self.assertEqual("BEGIN", get_sql[0].strip().upper()) + self.assertTrue( + all(sql.lstrip().upper().startswith(("BEGIN", "SELECT", "COMMIT")) for sql in get_sql) + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(interrupt_trace)) + interrupt_sql = _statements(interrupt_trace) + interrupt_updates = [ + index + for index, sql in enumerate(interrupt_sql) + if sql.lstrip().upper().startswith("UPDATE RECONCILE_RUNS") + ] + self.assertEqual(2, len(interrupt_updates)) + self.assertLess(interrupt_updates[-1], _statement_index(interrupt_sql, "COMMIT")) + + def test_result_insert_and_counter_update_roll_back_together(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("append-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + run = _run() + store.begin_reconcile_run(run) + with closing(database.connect()) as conn: + conn.execute( + """ + CREATE TRIGGER reject_reconcile_counter_update + BEFORE UPDATE ON reconcile_runs + WHEN NEW.total = OLD.total + 1 + BEGIN + SELECT RAISE(ABORT, 'injected counter failure'); + END + """ + ) + conn.commit() + database.traces.clear() + + with self.assertRaisesRegex(sqlite3.DatabaseError, "injected counter failure"): + store.append_reconcile_result(run.run_id, _result()) + append_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(append_trace)) + append_sql = _statements(append_trace) + self.assertLess( + _statement_index(append_sql, "INSERT INTO RECONCILE_RESULTS"), + _statement_index(append_sql, "UPDATE RECONCILE_RUNS"), + ) + with closing(sqlite3.connect(database.database_path)) as conn: + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + run_row = conn.execute( + """ + SELECT outcome, total, healthy_count, changed_count, pending_count, + action_required_count, error_count, skipped_count + FROM reconcile_runs + WHERE run_id = ? + """, + (run.run_id,), + ).fetchone() + self.assertEqual(0, result_count) + self.assertEqual(("running", 0, 0, 0, 0, 0, 0, 0), run_row) + + def test_finish_validation_rolls_back_and_leaves_run_open_byte_for_byte(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("finish-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + run = _run() + result = _result() + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + before = _run_bytes(database.database_path, run.run_id) + database.traces.clear() + + with self.assertRaisesRegex(RuntimeError, "persisted reconciliation results"): + store.finish_reconcile_run(_summary(run, [])) + finish_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(finish_trace)) + self.assertEqual(before, _run_bytes(database.database_path, run.run_id)) + with closing(sqlite3.connect(database.database_path)) as conn: + outcome, finished_at = conn.execute( + "SELECT outcome, finished_at FROM reconcile_runs WHERE run_id = ?", + (run.run_id,), + ).fetchone() + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + self.assertEqual(("running", None), (outcome, finished_at)) + self.assertEqual(1, result_count) + + def test_interrupt_rolls_back_every_running_run_when_one_has_a_late_result(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("interrupt-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + first = _run("run-a-empty") + late = _run("run-z-late") + store.begin_reconcile_run(first) + store.begin_reconcile_run(late) + store.append_reconcile_result(late.run_id, _result()) + first_before = _run_bytes(database.database_path, first.run_id) + late_before = _run_bytes(database.database_path, late.run_id) + interrupted_at = RESULT_STARTED_AT + timedelta(microseconds=500_000) + database.traces.clear() + + with self.assertRaisesRegex(ValueError, "finishes after"): + store.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at) + interrupt_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(interrupt_trace)) + interrupt_sql = _statements(interrupt_trace) + self.assertEqual( + 1, + sum(sql.lstrip().upper().startswith("UPDATE RECONCILE_RUNS") for sql in interrupt_sql), + ) + self.assertEqual(first_before, _run_bytes(database.database_path, first.run_id)) + self.assertEqual(late_before, _run_bytes(database.database_path, late.run_id)) + + def test_event_link_rejects_missing_and_wrong_bot_without_partial_results(self) -> None: + run = _run() + self.store.begin_reconcile_run(run) + + with self.assertRaisesRegex(ValueError, "lifecycle event"): + self.store.append_reconcile_result( + run.run_id, + _result(outcome=ReconcileOutcome.changed, event_id=999), + ) + + with closing(sqlite3.connect(self.database_path)) as conn: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, operation_id, occurred_at, source, action, outcome + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + "other", + "a" * 32, + RUN_STARTED_AT.isoformat(), + "cli", + "bot.create", + "success", + ), + ) + wrong_bot_event_id = int(cursor.lastrowid) + conn.commit() + + with self.assertRaisesRegex(ValueError, "lifecycle event"): + self.store.append_reconcile_result( + run.run_id, + _result( + outcome=ReconcileOutcome.changed, + event_id=wrong_bot_event_id, + ), + ) + + with closing(sqlite3.connect(self.database_path)) as conn: + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + total = conn.execute( + "SELECT total FROM reconcile_runs WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + self.assertEqual(0, result_count) + self.assertEqual(0, total) + + def test_state_facade_uses_shared_database_and_delegates_verbatim_signatures(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + idempotency = MagicMock(spec=IdempotencyStore) + delegate = MagicMock(spec=ReconcileStore) + run = _run() + result = _result() + summary = _summary(run, [result]) + finished = MagicMock(spec=PersistedReconcileRun) + loaded = MagicMock(spec=PersistedReconcileRun) + delegate.finish_reconcile_run.return_value = finished + delegate.interrupt_stale_reconcile_runs.return_value = 3 + delegate.get_reconcile_run.return_value = loaded + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=idempotency) as idempotency_type, + patch("zeus.state.ReconcileStore", return_value=delegate) as reconcile_type, + ): + facade = StateStore(database_path) + facade.begin_reconcile_run(run) + facade.append_reconcile_result(run.run_id, result) + actual_finished = facade.finish_reconcile_run(summary) + actual_interrupted = facade.interrupt_stale_reconcile_runs( + interrupted_at=RUN_FINISHED_AT + ) + actual_loaded = facade.get_reconcile_run(run.run_id) + + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], idempotency_type.call_args_list) + self.assertEqual([call(database)], reconcile_type.call_args_list) + database.connect.assert_not_called() + delegate.begin_reconcile_run.assert_called_once_with(run) + delegate.append_reconcile_result.assert_called_once_with(run.run_id, result) + delegate.finish_reconcile_run.assert_called_once_with(summary) + delegate.interrupt_stale_reconcile_runs.assert_called_once_with( + interrupted_at=RUN_FINISHED_AT + ) + delegate.get_reconcile_run.assert_called_once_with(run.run_id) + self.assertIs(finished, actual_finished) + self.assertEqual(3, actual_interrupted) + self.assertIs(loaded, actual_loaded) + for method_name in ( + "begin_reconcile_run", + "append_reconcile_result", + "finish_reconcile_run", + "interrupt_stale_reconcile_runs", + "get_reconcile_run", + ): + self.assertEqual( + inspect.signature(getattr(ReconcileStore, method_name)), + inspect.signature(getattr(StateStore, method_name)), + ) + + def test_reconcile_counter_columns_remain_a_state_compatibility_alias(self) -> None: + self.assertIs(RECONCILE_COUNTER_COLUMNS, STATE_RECONCILE_COUNTER_COLUMNS) + self.assertEqual( + { + ReconcileOutcome.healthy: "healthy_count", + ReconcileOutcome.changed: "changed_count", + ReconcileOutcome.pending: "pending_count", + ReconcileOutcome.action_required: "action_required_count", + ReconcileOutcome.error: "error_count", + ReconcileOutcome.skipped: "skipped_count", + }, + RECONCILE_COUNTER_COLUMNS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_renderer_state.py b/tests/test_renderer_state.py index fe5937c..b52b875 100644 --- a/tests/test_renderer_state.py +++ b/tests/test_renderer_state.py @@ -2,6 +2,7 @@ import os import sqlite3 +import stat import tempfile import unittest from concurrent.futures import ThreadPoolExecutor @@ -11,6 +12,7 @@ from pathlib import Path from unittest.mock import patch +from zeus import private_io from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEventInput from zeus.models import BotCreateRequest, BotRecord, BotStatus, HermesTemplate, TemplateError @@ -716,12 +718,169 @@ def test_renderer_atomically_replaces_profile_and_preserves_logs(self) -> None: request = BotCreateRequest(bot_id="coder", template_id="coding-bot") renderer.render(request, template) profile = hermes_root / "profiles" / "coder" - (profile / "logs" / "gateway.log").write_text("existing log\n", encoding="utf-8") + logs = profile / "logs" + (logs / "gateway.log").write_text("existing log\n", encoding="utf-8") + logs.chmod(0o755) renderer.render(request, replace(template, soul="replacement soul")) self.assertEqual("replacement soul\n", (profile / "SOUL.md").read_text()) - self.assertEqual("existing log\n", (profile / "logs" / "gateway.log").read_text()) + self.assertEqual("existing log\n", (logs / "gateway.log").read_text()) + self.assertEqual(os.geteuid(), logs.stat().st_uid) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + + def test_renderer_logs_tightening_does_not_follow_staging_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_log = profile / "logs" / "gateway.log" + live_log.write_text("live log\n", encoding="utf-8") + external_logs = root / "external-race-target" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + swapped = False + + def swap_staging_logs() -> None: + nonlocal swapped + if swapped: + return + candidates = list(profile.parent.glob(".coder.staging-*/logs")) + self.assertEqual(1, len(candidates)) + staging_logs = candidates[0] + staging_logs.rename(staging_logs.with_name("logs-displaced")) + staging_logs.symlink_to(external_logs, target_is_directory=True) + swapped = True + + real_chmod = Path.chmod + + def racing_chmod( + path: Path, + mode: int, + *, + follow_symlinks: bool = True, + ) -> None: + if path.name == "logs" and path.parent.name.startswith(".coder.staging-"): + swap_staging_logs() + real_chmod(path, mode, follow_symlinks=follow_symlinks) + + def racing_validate_private_directory(path: Path) -> None: + swap_staging_logs() + private_io.validate_private_directory(path) + + with ( + patch.object(Path, "chmod", new=racing_chmod), + patch( + "zeus.renderer.validate_private_directory", + side_effect=racing_validate_private_directory, + ), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be installed")) + + self.assertTrue(swapped) + self.assertEqual("external target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) + self.assertEqual("live log\n", live_log.read_text(encoding="utf-8")) + + def test_renderer_rejects_logs_mode_drift_during_final_binding_check(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + real_lstat = os.lstat + logs_lstats = 0 + drifted_mode: int | None = None + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal drifted_mode, logs_lstats + if path == "logs" and dir_fd is not None: + candidates = list(profile.parent.glob(".coder.staging-*/logs")) + if len(candidates) == 1: + staging_logs = candidates[0] + parent = staging_logs.parent.stat() + opened_parent = os.fstat(dir_fd) + if ( + parent.st_dev == opened_parent.st_dev + and parent.st_ino == opened_parent.st_ino + ): + logs_lstats += 1 + if logs_lstats == 4: + staging_logs.chmod(0o777) + drifted_mode = staging_logs.stat().st_mode & 0o777 + return real_lstat(path, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be committed")) + + self.assertEqual(4, logs_lstats) + self.assertEqual(0o777, drifted_mode) + self.assertEqual(live_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + self.assertEqual(0o700, (profile / "logs").stat().st_mode & 0o777) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + + def test_renderer_rejects_logs_replacement_immediately_before_install(self) -> None: + from zeus.renderer import _install_staged_profile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + live_log = profile / "logs" / "gateway.log" + live_log.write_text("live log\n", encoding="utf-8") + external_logs = root / "external-race-target" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external target\n", encoding="utf-8") + external_mode = stat.S_IMODE(external_logs.stat().st_mode) + swapped = False + + def racing_install(staging: Path, destination: Path) -> Path | None: + nonlocal swapped + staging_logs = staging / "logs" + staging_logs.rename(staging / "logs-displaced") + staging_logs.symlink_to(external_logs, target_is_directory=True) + swapped = True + return _install_staged_profile(staging, destination) + + with ( + patch("zeus.renderer._install_staged_profile", side_effect=racing_install), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be committed")) + + self.assertTrue(swapped) + self.assertEqual(live_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + self.assertFalse((profile / "logs").is_symlink()) + self.assertEqual("live log\n", live_log.read_text(encoding="utf-8")) + self.assertEqual("external target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, stat.S_IMODE(external_logs.stat().st_mode)) self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> None: @@ -764,7 +923,7 @@ def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> (profile / "runtime" / "nested" / "state.json").read_text(), ) - def test_renderer_replacement_preserves_symlinks_without_following_managed_links(self) -> None: + def test_renderer_replacement_handles_managed_links_without_following_targets(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) hermes_root = root / ".zeus" / "hermes" @@ -785,7 +944,9 @@ def test_renderer_replacement_preserves_symlinks_without_following_managed_links (external_skills / "SKILL.md").write_text("linked skill\n", encoding="utf-8") external_logs = root / "external-logs" external_logs.mkdir() + external_logs.chmod(0o755) (external_logs / "gateway.log").write_text("linked log\n", encoding="utf-8") + external_logs_mode = external_logs.stat().st_mode & 0o777 try: (profile / "operator-link").symlink_to(external) @@ -816,8 +977,89 @@ def test_renderer_replacement_preserves_symlinks_without_following_managed_links ) self.assertTrue((profile / "skills").is_symlink()) self.assertEqual("linked skill\n", (profile / "skills" / "SKILL.md").read_text()) - self.assertTrue((profile / "logs").is_symlink()) - self.assertEqual("linked log\n", (profile / "logs" / "gateway.log").read_text()) + self.assertFalse((profile / "logs").is_symlink()) + self.assertTrue((profile / "logs").is_dir()) + self.assertEqual(0o700, (profile / "logs").stat().st_mode & 0o777) + self.assertEqual([], list((profile / "logs").iterdir())) + self.assertEqual("linked log\n", (external_logs / "gateway.log").read_text()) + self.assertEqual(external_logs_mode, external_logs.stat().st_mode & 0o777) + + def test_renderer_replacement_replaces_dangling_logs_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + missing_target = root / "missing-external-logs" + logs.symlink_to(missing_target, target_is_directory=True) + + renderer.render(request, replace(template, soul="replacement soul")) + + self.assertFalse(logs.is_symlink()) + self.assertTrue(logs.is_dir()) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) + self.assertFalse(missing_target.exists()) + + def test_renderer_replacement_rejects_regular_logs_file_without_deleting_it(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + logs.write_text("operator-owned logs entry\n", encoding="utf-8") + original_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + + with self.assertRaisesRegex(TemplateError, "profile path must be a directory: logs"): + renderer.render(request, replace(template, soul="must not be installed")) + + self.assertTrue(logs.is_file()) + self.assertEqual("operator-owned logs entry\n", logs.read_text(encoding="utf-8")) + self.assertEqual(original_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + + def test_renderer_transaction_rollback_restores_logs_symlink_exactly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + external_logs = root / "external-rollback-logs" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "gateway.log" + sentinel.write_text("rollback target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + logs.symlink_to(external_logs, target_is_directory=True) + original_link = os.readlink(logs) + + with ( + self.assertRaisesRegex(RuntimeError, "injected database failure"), + renderer.transaction(request, replace(template, soul="replacement soul")), + ): + self.assertFalse(logs.is_symlink()) + self.assertTrue(logs.is_dir()) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) + self.assertEqual("rollback target\n", sentinel.read_text(encoding="utf-8")) + raise RuntimeError("injected database failure") + + self.assertTrue(logs.is_symlink()) + self.assertEqual(original_link, os.readlink(logs)) + self.assertEqual("rollback target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) def test_renderer_transaction_commits_replacement_and_cleans_backup(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 882a3cc..6cf57fc 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -1,12 +1,141 @@ from __future__ import annotations import json +import os import re +import shutil +import subprocess +import tempfile +import textwrap +import tomllib import unittest from pathlib import Path +from zeus.state import SCHEMA_VERSION + + +def _workflow_job_bodies(workflow: str) -> dict[str, str]: + jobs_header = re.search(r"(?m)^jobs:\s*$", workflow) + if jobs_header is None: + raise ValueError("workflow has no jobs mapping") + + jobs_text = workflow[jobs_header.end() :] + next_top_level = re.search(r"(?m)^[A-Za-z_][A-Za-z0-9_-]*:\s*", jobs_text) + if next_top_level is not None: + jobs_text = jobs_text[: next_top_level.start()] + + headers = list(re.finditer(r"(?m)^ (?P[A-Za-z0-9_-]+):\s*$", jobs_text)) + if not headers: + raise ValueError("workflow jobs mapping is empty") + + jobs: dict[str, str] = {} + for index, header in enumerate(headers): + name = header.group("name") + if name in jobs: + raise ValueError(f"duplicate workflow job: {name}") + end = headers[index + 1].start() if index + 1 < len(headers) else len(jobs_text) + jobs[name] = jobs_text[header.end() : end] + return jobs + + +def _job_level_scalar(job_body: str, key: str) -> str: + values = re.findall( + rf"(?m)^ {re.escape(key)}:\s*(?P[^\s#][^\n#]*?)\s*(?:#.*)?$", + job_body, + ) + if len(values) != 1: + raise ValueError(f"expected one job-level {key}, found {len(values)}") + return values[0] + + +def _job_python_versions(job_body: str) -> tuple[str, ...]: + matrix = re.search(r"(?m)^ python-version:\s*(?P\[[^\n]+\])\s*$", job_body) + if matrix is not None: + versions = json.loads(matrix.group("versions")) + if not isinstance(versions, list) or not all(isinstance(item, str) for item in versions): + raise ValueError("python-version matrix must be a list of strings") + return tuple(versions) + + setup_versions = re.findall( + r'(?m)^ python-version:\s*"(?P3\.\d+)"\s*$', job_body + ) + if len(setup_versions) != 1: + raise ValueError(f"expected one setup-python version, found {len(setup_versions)}") + return tuple(setup_versions) + + +def _job_run_commands(job_body: str) -> tuple[str, ...]: + lines = job_body.splitlines() + commands: list[str] = [] + index = 0 + while index < len(lines): + match = re.match(r"^ run:\s*(?P.*)$", lines[index]) + if match is None: + index += 1 + continue + + value = match.group("value").strip() + if value not in {"|", "|-"}: + commands.append(value) + index += 1 + continue + + index += 1 + block: list[str] = [] + while index < len(lines): + line = lines[index] + if line.strip() and len(line) - len(line.lstrip(" ")) <= 8: + break + block.append(line) + index += 1 + commands.append(textwrap.dedent("\n".join(block)).strip()) + return tuple(commands) + + +def _markdown_table_rows(markdown: str) -> dict[str, tuple[str, ...]]: + rows: dict[str, tuple[str, ...]] = {} + for line in markdown.splitlines(): + if not line.startswith("|"): + continue + cells = tuple(cell.strip() for cell in line.strip().strip("|").split("|")) + if not cells or cells[0] == "Gate" or set(cells[0]) <= {"-", ":"}: + continue + if cells[0] in rows: + raise ValueError(f"duplicate Markdown table row: {cells[0]}") + rows[cells[0]] = cells[1:] + return rows + class RepoContractTests(unittest.TestCase): + def test_builtin_template_copies_are_identical_and_images_are_digest_pinned( + self, + ) -> None: + source_root = Path("templates") + bundled_root = Path("zeus/bundled_templates") + expected_image = ( + "nikolaik/python-nodejs:python3.11-nodejs20@sha256:" + "8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" + ) + source_names = sorted(path.name for path in source_root.glob("*.toml")) + bundled_names = sorted(path.name for path in bundled_root.glob("*.toml")) + + self.assertEqual(source_names, bundled_names) + self.assertGreaterEqual(len(source_names), 7) + for name in source_names: + with self.subTest(template=name): + source_text = (source_root / name).read_text(encoding="utf-8") + bundled_text = (bundled_root / name).read_text(encoding="utf-8") + self.assertEqual(source_text, bundled_text) + + data = tomllib.loads(source_text) + docker_image = data["hermes"]["terminal"]["docker_image"] + self.assertIsInstance(docker_image, str) + self.assertEqual(expected_image, docker_image) + self.assertRegex( + docker_image, + r"\A[a-z0-9./_-]+:[a-zA-Z0-9._-]+@sha256:[0-9a-f]{64}\Z", + ) + def test_publishable_repository_files_exist(self) -> None: required = [ "README.md", @@ -25,11 +154,13 @@ def test_publishable_repository_files_exist(self) -> None: "docs/OPERATIONS.md", "docs/RECONCILE.md", "docs/RELEASE.md", + "docs/COMPATIBILITY.md", "docs/openapi.json", "docs/ROADMAP.md", "docs/assets/demo.cast", "docs/assets/zeus-hero.png", ".coveragerc", + "requirements-hermes-ci.txt", ".github/workflows/release.yml", ".github/ISSUE_TEMPLATE/bug_report.yml", ".github/ISSUE_TEMPLATE/feature_request.yml", @@ -39,6 +170,7 @@ def test_publishable_repository_files_exist(self) -> None: "systemd/zeus-reconcile.service", "systemd/zeus-reconcile.timer", "scripts/repo_check.sh", + "scripts/check_verified_release_ref.py", "scripts/wheel_smoke.sh", "scripts/fresh_vps_verify.sh", "zeus/bundled_templates/__init__.py", @@ -64,41 +196,313 @@ def test_publishable_repository_files_exist(self) -> None: def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + jobs = _workflow_job_bodies(workflow) + + expected_runners = { + "test": "ubuntu-24.04", + "python-3-14": "ubuntu-24.04", + "lifecycle-subprocess": "ubuntu-24.04", + "macos-process-lifecycle": "macos-26", + "real-hermes": "ubuntu-24.04", + "package": "ubuntu-24.04", + } + expected_python_versions = { + "test": ("3.11", "3.12", "3.13"), + "python-3-14": ("3.14",), + "lifecycle-subprocess": ("3.11",), + "macos-process-lifecycle": ("3.13",), + "real-hermes": ("3.11",), + "package": ("3.11",), + } + expected_setup_python = { + "test": "${{ matrix.python-version }}", + "python-3-14": '"3.14"', + "lifecycle-subprocess": '"3.11"', + "macos-process-lifecycle": '"3.13"', + "real-hermes": '"3.11"', + "package": '"3.11"', + } + expected_commands = { + "test": ( + "sudo apt-get update\nsudo apt-get install -y shellcheck", + 'python -m pip install -e ".[dev]"', + "ruff format --check .", + "ruff check .", + "mypy zeus", + "bandit -r zeus", + "shellcheck scripts/*.sh", + "sh scripts/test.sh", + "sh scripts/repo_check.sh", + "coverage erase\ncoverage run -m unittest discover -s tests\ncoverage report", + ), + "python-3-14": ( + 'python -m pip install -e ".[dev]"', + "sh scripts/test.sh", + ), + "lifecycle-subprocess": ( + 'python -m pip install -e ".[dev]"', + "python -m unittest tests.test_subprocess_lifecycle", + ), + "macos-process-lifecycle": ( + 'python -m pip install -e ".[dev]"', + "python -m unittest -v \\\n" + " tests.test_subprocess_lifecycle \\\n" + " tests.test_fake_hermes_integration \\\n" + " tests.test_crash_recovery.GatewayLauncherTests", + ), + "real-hermes": ( + "mkdir -p .tmp/real-hermes-evidence\n" + "printf '%s\\n' 'result=failed' 'failure_stage=ci_setup' > " + ".tmp/real-hermes-evidence/summary.txt", + "python -m pip install --require-hashes --only-binary=:all: " + "-r requirements-hermes-ci.txt", + "python -m pip install -e .", + "python -m pip check", + "ZEUS_VERIFY_START_GATEWAY=1 \\\n" + "ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 \\\n" + "ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence \\\n" + "sh scripts/verify_real_hermes.sh", + ), + "package": ( + 'python -m pip install -e ".[dev]"', + "python -m pip check", + "rm -rf dist\npython -m build", + "ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", + "twine check dist/*", + ), + } + self.assertEqual(set(expected_runners), set(jobs)) self.assertIn("workflow_dispatch:", workflow) - self.assertIn("3.11", workflow) - self.assertIn("3.12", workflow) - self.assertIn("3.13", workflow) - self.assertIn('pip install -e ".[dev]"', workflow) - self.assertIn("ruff format --check .", workflow) - self.assertIn("ruff check .", workflow) - self.assertIn("mypy zeus", workflow) - self.assertIn("bandit -r zeus", workflow) - self.assertIn("shellcheck scripts/*.sh", workflow) - self.assertIn("sh scripts/test.sh", workflow) - self.assertIn("coverage erase", workflow) - self.assertIn("coverage run -m unittest discover -s tests", workflow) - self.assertIn("coverage report", workflow) - self.assertNotIn("coverage report --fail-under", workflow) - self.assertIn("python -m build", workflow) - self.assertIn("twine check dist/*", workflow) - self.assertIn("sh scripts/wheel_smoke.sh", workflow) - - def test_test_script_runs_compile_unittest_and_doctor(self) -> None: - script = Path("scripts/test.sh").read_text(encoding="utf-8") - - self.assertIn("compileall zeus tests", script) - self.assertIn("unittest discover -s tests -v", script) - self.assertIn("trap cleanup EXIT INT TERM", script) - self.assertIn('mkdir -p "$tmp_dir"', script) - self.assertIn("zeus.cli doctor --json", script) + for job_name, job_body in jobs.items(): + with self.subTest(job=job_name): + self.assertEqual(expected_runners[job_name], _job_level_scalar(job_body, "runs-on")) + self.assertEqual(expected_python_versions[job_name], _job_python_versions(job_body)) + self.assertEqual( + [expected_setup_python[job_name]], + re.findall(r"(?m)^ python-version:\s*(.+?)\s*$", job_body), + ) + self.assertEqual(expected_commands[job_name], _job_run_commands(job_body)) + + job_level_continue_on_error: dict[str, str] = {} + for job_name, job_body in jobs.items(): + values = re.findall(r"(?m)^ continue-on-error:\s*([^\s#]+)\s*$", job_body) + self.assertLessEqual(len(values), 1, msg=f"duplicate setting in {job_name}") + if values: + job_level_continue_on_error[job_name] = values[0] + self.assertEqual({"python-3-14": "true"}, job_level_continue_on_error) + + real_hermes = jobs["real-hermes"] + self.assertEqual("15", _job_level_scalar(real_hermes, "timeout-minutes")) + self.assertNotIn("secrets.", real_hermes) + self.assertNotRegex(real_hermes, r"(?i)(openai|anthropic|openrouter).*api[_-]?key") + self.assertRegex(real_hermes, r"(?m)^ if: failure\(\)\s*$") + self.assertIn(".tmp/real-hermes-evidence/summary.txt", real_hermes) + self.assertIn("if-no-files-found: error", real_hermes) + + python_314 = jobs["python-3-14"].lower() + self.assertNotIn("hermes", python_314) + self.assertNotIn("verify_real_hermes", python_314) + + macos_selectors = re.findall( + r"(? None: + script_source = Path("scripts/test.sh").read_text(encoding="utf-8") + fake_python_source = textwrap.dedent( + """\ + #!/bin/sh + set -eu + : "${FAKE_PYTHON_LOG:?}" + : "${FAKE_UNITTEST_MODE:?}" + printf '%s\\n' "$*" >> "$FAKE_PYTHON_LOG" + case "$*" in + *"-m compileall zeus tests") + exit 0 + ;; + *"-m unittest discover -s tests -v") + case "$FAKE_UNITTEST_MODE" in + warning) + printf '%s\\n' "Exception ignored in: <_io.FileIO name='fixture'>" >&2 + printf '%s\\n' "ResourceWarning: unclosed file " >&2 + exit 0 + ;; + exit7) + printf '%s\\n' "synthetic unittest failure" >&2 + exit 7 + ;; + clean) + exit 0 + ;; + *) + exit 98 + ;; + esac + ;; + *"-m zeus.cli doctor --json") + printf '%s\\n' '{"ok": true}' + ;; + *"-m zeus.cli template list") + printf '%s\\n' 'coding-bot' + ;; + *) + printf '%s\\n' "unexpected python invocation: $*" >&2 + exit 97 + ;; + esac + """ + ) + + def run_case( + root: Path, mode: str + ) -> tuple[subprocess.CompletedProcess[str], tuple[str, ...]]: + case_root = root / mode + scripts_dir = case_root / "scripts" + bin_dir = case_root / "bin" + scripts_dir.mkdir(parents=True) + bin_dir.mkdir() + script = scripts_dir / "test.sh" + fake_python = bin_dir / "python3" + log = case_root / "python-calls.log" + script.write_text(script_source, encoding="utf-8") + fake_python.write_text(fake_python_source, encoding="utf-8") + fake_python.chmod(0o755) + + environment = os.environ.copy() + environment.update( + { + "FAKE_PYTHON_LOG": str(log), + "FAKE_UNITTEST_MODE": mode, + "PATH": str(bin_dir) + os.pathsep + environment.get("PATH", ""), + } + ) + shell = shutil.which("sh", path=environment["PATH"]) + if shell is None: + self.fail("POSIX sh is required for the shell contract") + result = subprocess.run( + [shell, str(script)], + cwd=case_root, + env=environment, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + calls = tuple(log.read_text(encoding="utf-8").splitlines()) + return result, calls + + pre_gate_calls = ( + "-B -m compileall zeus tests", + "-B -W error::ResourceWarning -m unittest discover -s tests -v", + ) + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + + warning_result, warning_calls = run_case(root, "warning") + self.assertNotEqual(0, warning_result.returncode) + self.assertIn("Exception ignored in:", warning_result.stderr) + self.assertIn("ResourceWarning", warning_result.stderr) + self.assertEqual(pre_gate_calls, warning_calls) + + failed_result, failed_calls = run_case(root, "exit7") + self.assertEqual(7, failed_result.returncode) + self.assertNotIn("ResourceWarning", failed_result.stderr) + self.assertEqual(pre_gate_calls, failed_calls) + + clean_result, clean_calls = run_case(root, "clean") + self.assertEqual(0, clean_result.returncode) + self.assertEqual( + ( + *pre_gate_calls, + "-B -m zeus.cli doctor --json", + "-B -m zeus.cli template list", + ), + clean_calls, + ) def test_wheel_smoke_exercises_installed_demo_entrypoint(self) -> None: script = Path("scripts/wheel_smoke.sh").read_text(encoding="utf-8") - self.assertIn('"$venv_zeus" demo up --json', script) - self.assertIn('"$venv_zeus" demo status --json', script) - self.assertIn('"$venv_zeus" demo down --json', script) + canonical_root = 'repo_root="$(pwd -P)"' + tmp_dir = 'tmp_dir="$repo_root/.tmp/wheel-smoke"' + self.assertIn(canonical_root, script) + self.assertIn(tmp_dir, script) + self.assertLess(script.index(canonical_root), script.index(tmp_dir)) + self.assertNotIn('repo_root="$(pwd)"', script) + + trap_index = script.index("trap cleanup EXIT INT TERM") + for initialization in ('venv_zeus=""', 'state_dir="$tmp_dir/state"', "demo_started=0"): + self.assertIn(initialization, script) + self.assertLess(script.index(initialization), trap_index) + + cleanup = script[script.index("cleanup() {") : trap_index] + self.assertIn('[ "$demo_started" = "1" ]', cleanup) + self.assertIn('ZEUS_STATE_DIR="$state_dir" "$venv_zeus" demo down --json', cleanup) + self.assertLess(cleanup.index("demo down --json"), cleanup.index('rm -rf "$tmp_dir"')) + + help_command = '"$venv_zeus" --help' + pip_check_command = '"$venv_python" -m pip check' + self.assertIn(help_command, script) + self.assertIn(pip_check_command, script) + cd_index = script.index('cd "$tmp_dir"') + help_index = script.index(help_command) + pip_check_index = script.index(pip_check_command) + self.assertLess(cd_index, help_index) + self.assertLess(cd_index, pip_check_index) + self.assertIn("unset PYTHONPATH PYTHONHOME", script) + self.assertIn("export PYTHONNOUSERSITE=1", script) + self.assertIn('export PATH="$tmp_dir/venv/bin:$PATH"', script) + self.assertIn("command -v zeus-fake-hermes", script) + self.assertIn('"$venv_zeus" --help >zeus-help.txt', script) + self.assertIn('grep -F "usage: zeus" zeus-help.txt', script) + + self.assertIn('version("zeus-hermes-orchestrator")', script) + self.assertIn("import zeus; print(zeus.__version__)", script) + self.assertIn('cli_version=$("$venv_zeus" --version)', script) + self.assertIn('[ "$metadata_version" = "$module_version" ]', script) + self.assertIn('[ "$cli_version" = "zeus $metadata_version" ]', script) + self.assertIn("zeus.__file__", script) + self.assertGreaterEqual(script.count('[ ! -e "$state_dir" ]'), 2) + + for template_id in ( + "coding-bot", + "deepseek-coding-bot", + "docs-writer-bot", + "gateway-operator", + "log-triage-bot", + "research-bot", + "support-gateway", + ): + self.assertIn(template_id, script) + + self.assertIn('export ZEUS_STATE_DIR="$state_dir"', script) + self.assertIn('"$venv_zeus" doctor --json', script) + up_index = script.index('"$venv_zeus" demo up --json') + status_index = script.index('"$venv_zeus" demo status --json') + down_index = script.index('"$venv_zeus" demo down --json', up_index) + self.assertLess(up_index, status_index) + self.assertLess(status_index, down_index) + self.assertNotEqual(-1, script.rfind("demo_started=1", 0, up_index)) + self.assertNotEqual(-1, script.find("demo_started=0", down_index)) self.assertIn('"fake_hermes_bin"', script) self.assertIn('"status": "running"', script) self.assertIn('"status": "stopped"', script) @@ -115,7 +519,8 @@ def test_coverage_gate_measures_production_source_and_branches(self) -> None: self.assertIn("branch = True", config) self.assertRegex(config, r"(?m)^source =\s*$") self.assertRegex(config, r"(?m)^[ \t]+zeus[ \t]*$") - self.assertIn("fail_under = 70", config) + self.assertIn("fail_under = 79", config) + self.assertNotIn("fail_under = 70", config) self.assertIn("precision = 2", config) def test_workflow_actions_are_pinned_to_immutable_commits(self) -> None: @@ -186,6 +591,187 @@ def test_readme_has_informative_github_landing_sections(self) -> None: self.assertIn("Do not expose the API", readme) self.assertIn("## Known Limitations", readme) + def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> None: + readme = Path("README.md").read_text(encoding="utf-8") + contributing = Path("CONTRIBUTING.md").read_text(encoding="utf-8") + roadmap = Path("docs/ROADMAP.md").read_text(encoding="utf-8") + roadmap_text = " ".join(roadmap.split()) + ci_workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + release_workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + pyproject = Path("pyproject.toml").read_text(encoding="utf-8") + compatibility_path = Path("docs/COMPATIBILITY.md") + + self.assertTrue(compatibility_path.is_file()) + compatibility = compatibility_path.read_text(encoding="utf-8") + compatibility_text = " ".join(compatibility.split()) + + offline_heading = "### 1. Credential-free offline demo" + hermes_heading = "### 2. Real Hermes setup" + self.assertLess(readme.index(offline_heading), readme.index(hermes_heading)) + offline_path = readme.split(offline_heading, 1)[1].split(hermes_heading, 1)[0] + hermes_path = readme.split(hermes_heading, 1)[1].split("\n## ", 1)[0] + for command in ( + "python3 -m venv .venv", + "python -m pip install -e .", + "zeus demo up", + "zeus demo status", + "zeus demo down", + ): + self.assertIn(command, offline_path) + for command in ( + "hermes version", + "cp .env.example .env", + "chmod 0600 .env", + "zeus doctor", + "--env-from OPENROUTER_API_KEY", + "zeus bot doctor coder", + ): + self.assertIn(command, hermes_path) + self.assertIn("real, non-empty provider key", hermes_path) + self.assertIn("docs/COMPATIBILITY.md", readme) + self.assertIn("no required third-party Python runtime dependencies", readme) + + self.assertIn('python -m pip install -e ".[dev]"', contributing) + self.assertIn("make check", contributing) + self.assertIn("docs/COMPATIBILITY.md", contributing) + self.assertIn("sh scripts/verify_real_hermes.sh", contributing) + + for heading in ( + "## Current status", + "## Shipped", + "## Near term", + "## Under evaluation", + "## Out of scope", + ): + self.assertIn(heading, roadmap) + for statement in ( + "v0.3.0", + "alpha", + "host-local", + "cross-host placement", + "distributed approvals", + "fleet rollout policy", + "control-plane ownership", + "Olymp", + ): + self.assertIn(statement, roadmap_text) + + ci_jobs = _workflow_job_bodies(ci_workflow) + release_jobs = _workflow_job_bodies(release_workflow) + automated_matrix = compatibility.split("## Automated matrix", 1)[1].split("\n## ", 1)[0] + compatibility_rows = _markdown_table_rows(automated_matrix) + compatibility_contracts = { + "test": ( + ci_jobs["test"], + "Main CI matrix", + "ubuntu-24.04", + ("3.11", "3.12", "3.13"), + ( + "Linux `ubuntu-24.04`", + "Python 3.11, 3.12, and 3.13", + "Unit and integration tests, repository contracts, source-and-branch " + "coverage, formatting, lint, typing, Bandit, and ShellCheck", + ), + ), + "python-3-14": ( + ci_jobs["python-3-14"], + "Provisional Python compatibility", + "ubuntu-24.04", + ("3.14",), + ( + "Linux `ubuntu-24.04`", + "Python 3.14", + "Full Zeus test suite; non-required and Zeus-only because the pinned " + "Hermes baseline requires Python below 3.14", + ), + ), + "lifecycle-subprocess": ( + ci_jobs["lifecycle-subprocess"], + "Subprocess lifecycle", + "ubuntu-24.04", + ("3.11",), + ( + "Linux `ubuntu-24.04`", + "Python 3.11", + "Focused multi-process lifecycle and locking behavior", + ), + ), + "macos-process-lifecycle": ( + ci_jobs["macos-process-lifecycle"], + "macOS process lifecycle", + "macos-26", + ("3.13",), + ( + "macOS `macos-26`", + "Python 3.13", + "Focused process, fake-Hermes integration, and gateway-launcher recovery tests", + ), + ), + "real-hermes": ( + ci_jobs["real-hermes"], + "Real Hermes compatibility", + "ubuntu-24.04", + ("3.11",), + ( + "Linux `ubuntu-24.04`", + "Python 3.11", + "Hash-locked Hermes Agent 0.19.0 profile rendering, strict diagnostics, " + "loopback gateway readiness, process ownership, and clean shutdown " + "without a model-provider credential", + ), + ), + "package": ( + ci_jobs["package"], + "Package build", + "ubuntu-24.04", + ("3.11",), + ( + "Linux `ubuntu-24.04`", + "Python 3.11", + "Wheel and source build, installed-wheel smoke test, dependency " + "consistency, and metadata checks", + ), + ), + "release:build": ( + release_jobs["build"], + "Tagged release build", + "ubuntu-latest", + ("3.11",), + ( + "Linux `ubuntu-latest`", + "Python 3.11", + "Full release gate, artifact checksums, and GitHub release artifacts", + ), + ), + } + requires_python = re.search(r'(?m)^requires-python = "([^"]+)"$', pyproject) + + self.assertIsNotNone(requires_python) + self.assertEqual( + {contract[1] for contract in compatibility_contracts.values()}, + set(compatibility_rows), + ) + for job_name, contract in compatibility_contracts.items(): + job_body, row_name, runner, python_versions, expected_row = contract + with self.subTest(compatibility_job=job_name): + self.assertEqual(runner, _job_level_scalar(job_body, "runs-on")) + self.assertEqual(python_versions, _job_python_versions(job_body)) + self.assertEqual(expected_row, compatibility_rows[row_name]) + self.assertIn( + f'`requires-python = "{requires_python.group(1)}"`', + compatibility_text, + ) + self.assertIn("lifecycle and package jobs use Python 3.11", compatibility_text) + self.assertIn("Debian and Ubuntu", compatibility_text) + self.assertIn("Hermes Agent 0.19.0", compatibility_text) + self.assertIn("complete 60-package wheel closure", compatibility_text) + self.assertIn("no model-provider credential", compatibility_text) + self.assertIn("whichever `hermes` executable is installed", compatibility_text) + self.assertIn("Python 3.14", compatibility_text) + self.assertIn("provisional Zeus-only", compatibility_text) + self.assertIn("focused process", compatibility_text.lower()) + self.assertIn("Windows is not currently automated", compatibility_text) + def test_env_example_lists_deepseek_and_api_auth(self) -> None: env = Path(".env.example").read_text(encoding="utf-8") api_docs = Path("docs/API.md").read_text(encoding="utf-8") @@ -238,6 +824,17 @@ def test_observability_and_lifecycle_history_are_documented(self) -> None: for field in access_fields: self.assertIn(f"`{field}`", api_docs) + def test_architecture_terminal_schema_compatibility_matches_runtime(self) -> None: + architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8") + compatibility_statements = re.findall( + r"Databases newer than\s+schema v(?P\d+) " + r"are rejected rather than downgraded\.", + architecture, + ) + + self.assertEqual(1, len(compatibility_statements)) + self.assertEqual(str(SCHEMA_VERSION), compatibility_statements[0]) + def test_every_openapi_response_documents_request_id_header(self) -> None: document = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) request_id_header = document["components"]["headers"]["XRequestID"] @@ -282,6 +879,30 @@ def test_real_hermes_verifier_is_isolated_and_checks_async_cap(self) -> None: self.assertIn("/health", script) self.assertIn("time.monotonic", script) self.assertIn("Hermes /health did not become ready", script) + self.assertIn("ZEUS_VERIFY_EXPECTED_HERMES_VERSION", script) + self.assertIn('bot start "$bot_id" --wait', script) + self.assertIn("ZEUS_VERIFY_EVIDENCE_DIR", script) + self.assertIn("failure_stage=%s", script) + self.assertIn('rm -rf -- "$state_dir"', script) + + def test_real_hermes_ci_lock_is_complete_and_hash_pinned(self) -> None: + requirements = Path("requirements-hermes-ci.txt").read_text(encoding="utf-8") + entries = re.findall( + r"(?m)^([a-z0-9][a-z0-9._-]*)==([^\\\s]+) \\$", + requirements, + ) + hashes = re.findall(r"(?m)^ --hash=sha256:([0-9a-f]{64})(?: \\)?$", requirements) + + self.assertEqual(60, len(entries)) + self.assertEqual(60, len({name for name, _version in entries})) + self.assertEqual(74, len(hashes)) + self.assertIn(("hermes-agent", "0.19.0"), entries) + self.assertIn( + "bd0bac012aee38a60894781f4597dc29ee7bedb3448540249921f10d3bef327f", + hashes, + ) + self.assertNotIn("--index-url", requirements) + self.assertNotIn("git+", requirements) def test_fresh_vps_verifier_bootstraps_and_captures_evidence(self) -> None: script = Path("scripts/fresh_vps_verify.sh").read_text(encoding="utf-8") @@ -308,6 +929,7 @@ def test_pyproject_has_no_placeholder_repository_urls(self) -> None: self.assertIn("mypy>=1.11.0", pyproject) self.assertIn("bandit>=1.7.9", pyproject) self.assertIn("coverage>=7.0.0", pyproject) + self.assertNotIn('"pytest', pyproject) self.assertIn('dynamic = ["version"]', pyproject) self.assertIn('version = {attr = "zeus.__version__"}', pyproject) self.assertIn('Repository = "https://github.com/brainx/zeus"', pyproject) @@ -446,6 +1068,9 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: self.assertIn("attestations: write", workflow) self.assertIn("git cat-file -t", workflow) self.assertIn("Release tags must be annotated tags.", workflow) + self.assertIn("Require GitHub-verified release ref", workflow) + self.assertIn("python scripts/check_verified_release_ref.py", workflow) + self.assertIn("GITHUB_TOKEN: ${{ github.token }}", workflow) self.assertIn("actions/attest-build-provenance@", workflow) self.assertIn("dist/*.tar.gz", workflow) self.assertIn("dist/*.whl", workflow) @@ -454,7 +1079,8 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: self.assertIn("actions/download-artifact@", workflow) self.assertIn("cd dist && sha256sum -c SHA256SUMS.txt", workflow) self.assertIn("annotated version tags", release_docs) - self.assertIn("does not cryptographically verify signer identity", release_docs) + self.assertIn("GitHub-verified annotated tag", release_docs) + self.assertIn("v0.3.0 release predates", release_docs) self.assertIn("gh attestation verify", release_docs) self.assertIn('build_artifacts="${ZEUS_WHEEL_SMOKE_BUILD:-1}"', wheel_smoke) self.assertIn('fail "ZEUS_WHEEL_SMOKE_BUILD must be 0 or 1"', wheel_smoke) @@ -462,6 +1088,9 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: def test_makefile_has_release_check_target(self) -> None: makefile = Path("Makefile").read_text(encoding="utf-8") + check_recipe = re.search(r"(?m)^check:\n(?P(?:\t[^\n]*\n)+)", makefile) + build_recipe = re.search(r"(?m)^build:\n(?P(?:\t[^\n]*\n)+)", makefile) + release_recipe = re.search(r"(?m)^release-check:\n(?P(?:\t[^\n]*\n)+)", makefile) self.assertIn("release-check:", makefile) self.assertIn("coverage:", makefile) @@ -471,6 +1100,12 @@ def test_makefile_has_release_check_target(self) -> None: self.assertIn("coverage report", makefile) self.assertNotIn("coverage report --fail-under", makefile) self.assertIn("shellcheck scripts/*.sh", makefile) + self.assertIsNotNone(check_recipe) + self.assertIsNotNone(build_recipe) + self.assertIsNotNone(release_recipe) + self.assertIn("shellcheck scripts/*.sh", check_recipe.group("body")) + self.assertIn("python -m pip check", build_recipe.group("body")) + self.assertIn("python -m pip check", release_recipe.group("body")) self.assertIn("rm -rf dist", makefile) self.assertIn("python -m build", makefile) self.assertIn("ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", makefile) @@ -522,6 +1157,43 @@ def test_openapi_contract_loads_and_documents_required_paths(self) -> None: history["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], ) + def test_readiness_openapi_and_operator_documentation_contract(self) -> None: + spec = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) + api_docs = Path("docs/API.md").read_text(encoding="utf-8") + operations = Path("docs/OPERATIONS.md").read_text(encoding="utf-8") + + self.assertIn("/ready", spec["paths"]) + self.assertNotIn("/v1/ready", spec["paths"]) + self.assertEqual([], spec["paths"]["/health"]["get"]["security"]) + readiness = spec["paths"]["/ready"]["get"] + self.assertEqual([{"ZeusApiKey": []}], readiness["security"]) + self.assertEqual({"200", "400", "401", "429", "503"}, set(readiness["responses"])) + self.assertEqual( + "#/components/schemas/ReadinessResponse", + readiness["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], + ) + retry_after = readiness["responses"]["429"]["headers"]["Retry-After"] + self.assertNotIn("$ref", retry_after) + self.assertEqual({"type": "integer", "minimum": 1}, retry_after["schema"]) + for response in readiness["responses"].values(): + self.assertEqual( + {"$ref": "#/components/headers/XRequestID"}, + response["headers"]["X-Request-ID"], + ) + + schema = spec["components"]["schemas"]["ReadinessResponse"] + self.assertEqual(["schema_version", "status"], schema["required"]) + self.assertEqual(SCHEMA_VERSION, schema["properties"]["schema_version"]["const"]) + self.assertEqual("ready", schema["properties"]["status"]["const"]) + error_codes = spec["components"]["schemas"]["Error"]["properties"]["error"]["properties"][ + "code" + ]["enum"] + self.assertIn("not_ready", error_codes) + for text in (api_docs, operations): + self.assertIn("/ready", text) + self.assertIn("/health", text) + self.assertIn("ZEUS_ALLOW_UNAUTH_READS", text) + def test_wave_two_replay_and_recovery_contracts_are_documented(self) -> None: spec = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) api_docs = Path("docs/API.md").read_text(encoding="utf-8") @@ -705,6 +1377,40 @@ def test_brainx_maintainer_is_credited(self) -> None: self.assertIn("https://github.com/brainx", architecture) self.assertIn('Maintainer = "https://github.com/brainx"', pyproject) + def test_sqlite_durability_policy_is_consistent_across_runtime_and_docs(self) -> None: + env_example = Path(".env.example").read_text(encoding="utf-8") + api_service = Path("systemd/zeus-api.service").read_text(encoding="utf-8") + reconcile_service = Path("systemd/zeus-reconcile.service").read_text(encoding="utf-8") + timer = Path("systemd/zeus-reconcile.timer").read_text(encoding="utf-8") + systemd_docs = Path("docs/SYSTEMD.md").read_text(encoding="utf-8") + compatibility = Path("docs/COMPATIBILITY.md").read_text(encoding="utf-8").lower() + architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8").lower() + operations = Path("docs/OPERATIONS.md").read_text(encoding="utf-8").lower() + + self.assertEqual(1, env_example.count("ZEUS_SQLITE_SYNCHRONOUS=NORMAL")) + self.assertNotIn("ZEUS_SQLITE_SYNCHRONOUS=FULL", env_example) + for service in (api_service, reconcile_service): + with self.subTest(service=service[:32]): + self.assertEqual( + 1, + service.count("Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL"), + ) + self.assertIn("EnvironmentFile=/etc/zeus/zeus.env", service) + self.assertNotIn("ZEUS_SQLITE_SYNCHRONOUS", timer) + self.assertIn("ZEUS_SQLITE_SYNCHRONOUS=FULL", systemd_docs) + self.assertIn("normal", compatibility) + self.assertIn("unset", compatibility) + self.assertIn("empty", compatibility) + self.assertIn("schema v6", compatibility) + for text in (architecture, operations): + self.assertIn("process crash", text) + self.assertIn("power loss", text) + self.assertIn("every process that writes the same database", operations) + self.assertIn("sqlite only", operations) + self.assertIn("rendered profile files", operations) + self.assertIn("audit jsonl", operations) + self.assertIn("backup", operations) + def test_root_scripts_credit_repo_maintainers(self) -> None: script_paths = [Path("Makefile"), *sorted(Path("scripts").glob("*.sh"))] diff --git a/tests/test_sqlite_schema.py b/tests/test_sqlite_schema.py new file mode 100644 index 0000000..af9616b --- /dev/null +++ b/tests/test_sqlite_schema.py @@ -0,0 +1,1108 @@ +from __future__ import annotations + +import hashlib +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from pathlib import Path +from typing import Any, ClassVar, cast +from unittest.mock import MagicMock, call, patch + +from zeus.config import SQLiteSynchronous +from zeus.schema import SCHEMA_VERSION, SchemaManager, _preflight_schema_compatibility +from zeus.sqlite_db import SQLiteDatabase, StateReadinessError +from zeus.state import StateReadinessError as FacadeStateReadinessError +from zeus.state import StateStore + +_ORIGINAL_SQLITE_CONNECT = sqlite3.connect + +DDL_OBJECT_NAMES = ( + "bots", + "schema_version", + "lifecycle_events", + "idx_lifecycle_events_bot", + "idx_lifecycle_events_operation", + "lifecycle_events_reject_update", + "lifecycle_events_reject_delete", + "idempotency_records", + "idx_idempotency_records_expires", + "bots_desired_intent_reject_partial_insert", + "bots_desired_intent_reject_partial_update", + "reconcile_runs", + "reconcile_results", +) + +BASE_BOTS_SCHEMA = """ +CREATE TABLE bots ( + bot_id TEXT PRIMARY KEY, + template_id TEXT NOT NULL, + display_name TEXT NOT NULL, + profile_path TEXT NOT NULL, + status TEXT NOT NULL, + pid INTEGER, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +) +""" + +# Captured from the pre-extraction implementation. Every value is the exact +# sqlite_master SQL BLOB rendered as uppercase hexadecimal, not normalized SQL. +_FROZEN_COMMON_DDL_ROWS = ( + ( + "index", + "idx_idempotency_records_expires", + "idempotency_records", + ( + "43524541544520494E444558206964785F6964656D706F74656E63795F7265636F7264735F65" + "7870697265730A2020202020202020202020204F4E206964656D706F74656E63795F7265636F" + "7264732028657870697265735F6174290A202020202020202020202020" + ), + ), + ( + "index", + "idx_lifecycle_events_bot", + "lifecycle_events", + ( + "43524541544520494E444558206964785F6C6966656379636C655F6576656E74735F626F740A" + "2020202020202020202020204F4E206C6966656379636C655F6576656E74732028626F745F69" + "642C206576656E745F69642044455343290A202020202020202020202020" + ), + ), + ( + "index", + "idx_lifecycle_events_operation", + "lifecycle_events", + ( + "43524541544520494E444558206964785F6C6966656379636C655F6576656E74735F6F706572" + "6174696F6E0A2020202020202020202020204F4E206C6966656379636C655F6576656E747320" + "286F7065726174696F6E5F69642C206576656E745F6964290A202020202020202020202020" + ), + ), + ( + "table", + "idempotency_records", + "idempotency_records", + ( + "435245415445205441424C45206964656D706F74656E63795F7265636F72647320280A202020" + "202020202020202020202020206B65795F686173682054455854205052494D415259204B4559" + "2C0A20202020202020202020202020202020726571756573745F686173682054455854204E4F" + "54204E554C4C2C0A2020202020202020202020202020202073746174652054455854204E4F54" + "204E554C4C20434845434B2028737461746520494E202827696E5F70726F6772657373272C20" + "27636F6D706C657465642729292C0A202020202020202020202020202020206F776E65725F69" + "6E7374616E63655F69642054455854204E4F54204E554C4C2C0A202020202020202020202020" + "20202020726573706F6E73655F73746174757320494E5445474552204E554C4C2C0A20202020" + "202020202020202020202020726573706F6E73655F6A736F6E2054455854204E554C4C2C0A20" + "202020202020202020202020202020637265617465645F61742054455854204E4F54204E554C" + "4C2C0A20202020202020202020202020202020757064617465645F61742054455854204E4F54" + "204E554C4C2C0A20202020202020202020202020202020657870697265735F61742054455854" + "204E4F54204E554C4C0A20202020202020202020202029" + ), + ), + ( + "table", + "lifecycle_events", + "lifecycle_events", + ( + "435245415445205441424C45206C6966656379636C655F6576656E747320280A202020202020" + "202020202020202020206576656E745F696420494E5445474552205052494D415259204B4559" + "204155544F494E4352454D454E542C0A20202020202020202020202020202020626F745F6964" + "2054455854204E4F54204E554C4C2C0A202020202020202020202020202020206F7065726174" + "696F6E5F69642054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "726571756573745F696420544558542C0A202020202020202020202020202020206F63637572" + "7265645F61742054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "736F757263652054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "616374696F6E2054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "6F7574636F6D652054455854204E4F54204E554C4C2C0A202020202020202020202020202020" + "207374617475735F6265666F726520544558542C0A2020202020202020202020202020202073" + "74617475735F616674657220544558542C0A202020202020202020202020202020207069645F" + "6265666F726520494E54454745522C0A202020202020202020202020202020207069645F6166" + "74657220494E54454745522C0A20202020202020202020202020202020726561736F6E205445" + "5854204E4F54204E554C4C2044454641554C542027272C0A2020202020202020202020202020" + "20206572726F725F636F646520544558542C0A20202020202020202020202020202020657272" + "6F725F6D65737361676520544558542C0A202020202020202020202020202020206465746169" + "6C735F6A736F6E2054455854204E4F54204E554C4C2044454641554C5420277B7D270A202020" + "20202020202020202029" + ), + ), + ( + "table", + "reconcile_results", + "reconcile_results", + ( + "435245415445205441424C45207265636F6E63696C655F726573756C747320280A2020202020" + "202020202020202020202072756E5F69642054455854204E4F54204E554C4C2C0A2020202020" + "2020202020202020202020626F745F69642054455854204E4F54204E554C4C2C0A2020202020" + "20202020202020202020206F7264696E616C20494E5445474552204E4F54204E554C4C204348" + "45434B20286F7264696E616C203E3D2030292C0A202020202020202020202020202020206F75" + "74636F6D652054455854204E4F54204E554C4C20434845434B20280A20202020202020202020" + "202020202020202020206F7574636F6D6520494E20280A202020202020202020202020202020" + "202020202020202020276865616C746879272C20276368616E676564272C202770656E64696E" + "67272C0A20202020202020202020202020202020202020202020202027616374696F6E5F7265" + "717569726564272C20276572726F72272C2027736B6970706564270A20202020202020202020" + "20202020202020202020290A20202020202020202020202020202020292C0A20202020202020" + "202020202020202020646573697265645F7374617465205445585420434845434B20280A2020" + "202020202020202020202020202020202020646573697265645F7374617465204953204E554C" + "4C204F5220646573697265645F737461746520494E20282772756E6E696E67272C202773746F" + "7070656427290A20202020202020202020202020202020292C0A202020202020202020202020" + "202020206F627365727665645F737461747573205445585420434845434B20280A2020202020" + "2020202020202020202020202020206F627365727665645F737461747573204953204E554C4C" + "0A20202020202020202020202020202020202020204F52206F627365727665645F7374617475" + "7320494E20282773746F70706564272C20277374617274696E67272C202772756E6E696E6727" + "2C20276661696C6564272C2027756E6B6E6F776E27290A202020202020202020202020202020" + "20292C0A2020202020202020202020202020202070696420494E544547455220434845434B20" + "28706964204953204E554C4C204F5220706964203E2030292C0A202020202020202020202020" + "20202020616374696F6E2054455854204E4F54204E554C4C20434845434B20286C656E677468" + "28616374696F6E29203C3D2032303438292C0A202020202020202020202020202020206D6573" + "736167652054455854204E4F54204E554C4C20434845434B20286C656E677468286D65737361" + "676529203C3D2032303438292C0A202020202020202020202020202020206572726F725F636F" + "6465205445585420434845434B20286572726F725F636F6465204953204E554C4C204F52206C" + "656E677468286572726F725F636F646529203C3D2032303438292C0A20202020202020202020" + "2020202020206576656E745F696420494E544547455220434845434B20286576656E745F6964" + "204953204E554C4C204F52206576656E745F6964203E2030292C0A2020202020202020202020" + "2020202020737461727465645F61742054455854204E4F54204E554C4C2C0A20202020202020" + "20202020202020202066696E69736865645F61742054455854204E4F54204E554C4C2C0A2020" + "20202020202020202020202020205052494D415259204B4559202872756E5F69642C20626F74" + "5F6964292C0A20202020202020202020202020202020554E49515545202872756E5F69642C20" + "6F7264696E616C292C0A20202020202020202020202020202020434845434B202866696E6973" + "6865645F6174203E3D20737461727465645F6174292C0A202020202020202020202020202020" + "20464F524549474E204B4559202872756E5F696429205245464552454E434553207265636F6E" + "63696C655F72756E732872756E5F696429204F4E2044454C45544520434153434144450A2020" + "2020202020202020202029" + ), + ), + ( + "table", + "reconcile_runs", + "reconcile_runs", + ( + "435245415445205441424C45207265636F6E63696C655F72756E7320280A2020202020202020" + "202020202020202072756E5F69642054455854205052494D415259204B45592C0A2020202020" + "202020202020202020202073636F70652054455854204E4F54204E554C4C20434845434B2028" + "73636F706520494E202827666C656574272C2027626F742729292C0A20202020202020202020" + "2020202020207265717565737465645F626F745F696420544558542C0A202020202020202020" + "20202020202020736F757263652054455854204E4F54204E554C4C2C0A202020202020202020" + "20202020202020666F72636520494E5445474552204E4F54204E554C4C20434845434B202866" + "6F72636520494E2028302C203129292C0A202020202020202020202020202020207265736574" + "5F7265737461727420494E5445474552204E4F54204E554C4C20434845434B20287265736574" + "5F7265737461727420494E2028302C203129292C0A2020202020202020202020202020202073" + "7461727465645F61742054455854204E4F54204E554C4C2C0A20202020202020202020202020" + "20202066696E69736865645F617420544558542C0A202020202020202020202020202020206F" + "7574636F6D652054455854204E4F54204E554C4C20434845434B20280A202020202020202020" + "20202020202020202020206F7574636F6D6520494E20282772756E6E696E67272C2027737563" + "636565646564272C2027636F6D706C657465645F776974685F6572726F7273272C2027696E74" + "657272757074656427290A20202020202020202020202020202020292C0A2020202020202020" + "2020202020202020746F74616C20494E5445474552204E4F54204E554C4C2044454641554C54" + "203020434845434B2028746F74616C203E3D2030292C0A202020202020202020202020202020" + "206865616C7468795F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C" + "54203020434845434B20286865616C7468795F636F756E74203E3D2030292C0A202020202020" + "202020202020202020206368616E6765645F636F756E7420494E5445474552204E4F54204E55" + "4C4C2044454641554C54203020434845434B20286368616E6765645F636F756E74203E3D2030" + "292C0A2020202020202020202020202020202070656E64696E675F636F756E7420494E544547" + "4552204E4F54204E554C4C2044454641554C54203020434845434B202870656E64696E675F63" + "6F756E74203E3D2030292C0A20202020202020202020202020202020616374696F6E5F726571" + "75697265645F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C542030" + "0A2020202020202020202020202020202020202020434845434B2028616374696F6E5F726571" + "75697265645F636F756E74203E3D2030292C0A20202020202020202020202020202020657272" + "6F725F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C542030204348" + "45434B20286572726F725F636F756E74203E3D2030292C0A2020202020202020202020202020" + "2020736B69707065645F636F756E7420494E5445474552204E4F54204E554C4C204445464155" + "4C54203020434845434B2028736B69707065645F636F756E74203E3D2030292C0A2020202020" + "2020202020202020202020434845434B20280A20202020202020202020202020202020202020" + "20746F74616C203D206865616C7468795F636F756E74202B206368616E6765645F636F756E74" + "202B2070656E64696E675F636F756E740A202020202020202020202020202020202020202020" + "20202020202B20616374696F6E5F72657175697265645F636F756E74202B206572726F725F63" + "6F756E74202B20736B69707065645F636F756E740A2020202020202020202020202020202029" + "2C0A20202020202020202020202020202020434845434B20280A202020202020202020202020" + "20202020202020202873636F7065203D2027666C6565742720414E4420726571756573746564" + "5F626F745F6964204953204E554C4C290A20202020202020202020202020202020202020204F" + "52202873636F7065203D2027626F742720414E44207265717565737465645F626F745F696420" + "4953204E4F54204E554C4C290A20202020202020202020202020202020292C0A202020202020" + "20202020202020202020434845434B20280A2020202020202020202020202020202020202020" + "286F7574636F6D65203D202772756E6E696E672720414E442066696E69736865645F61742049" + "53204E554C4C290A20202020202020202020202020202020202020204F5220286F7574636F6D" + "6520213D202772756E6E696E672720414E442066696E69736865645F6174204953204E4F5420" + "4E554C4C290A20202020202020202020202020202020292C0A20202020202020202020202020" + "202020434845434B202866696E69736865645F6174204953204E554C4C204F522066696E6973" + "6865645F6174203E3D20737461727465645F6174290A20202020202020202020202029" + ), + ), + ( + "trigger", + "bots_desired_intent_reject_partial_insert", + "bots", + ( + "435245415445205452494747455220626F74735F646573697265645F696E74656E745F72656A" + "6563745F7061727469616C5F696E736572740A2020202020202020202020204245464F524520" + "494E53455254204F4E20626F74730A2020202020202020202020205748454E204E4F5420280A" + "20202020202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E" + "5F6964204953204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64696E675F" + "73696E6365204953204E554C4C290A202020202020202020202020202020204F520A20202020" + "202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E5F696420" + "4953204E4F54204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E4F54204E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64" + "696E675F73696E6365204953204E4F54204E554C4C290A202020202020202020202020290A20" + "2020202020202020202020424547494E0A2020202020202020202020202020202053454C4543" + "542052414953452841424F52542C202770656E64696E67206C6966656379636C6520696E7465" + "6E74206D75737420626520616C6C206E756C6C206F7220616C6C20706F70756C617465642729" + "3B0A202020202020202020202020454E44" + ), + ), + ( + "trigger", + "bots_desired_intent_reject_partial_update", + "bots", + ( + "435245415445205452494747455220626F74735F646573697265645F696E74656E745F72656A" + "6563745F7061727469616C5F7570646174650A2020202020202020202020204245464F524520" + "555044415445204F4E20626F74730A2020202020202020202020205748454E204E4F5420280A" + "20202020202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E" + "5F6964204953204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64696E675F" + "73696E6365204953204E554C4C290A202020202020202020202020202020204F520A20202020" + "202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E5F696420" + "4953204E4F54204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E4F54204E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64" + "696E675F73696E6365204953204E4F54204E554C4C290A202020202020202020202020290A20" + "2020202020202020202020424547494E0A2020202020202020202020202020202053454C4543" + "542052414953452841424F52542C202770656E64696E67206C6966656379636C6520696E7465" + "6E74206D75737420626520616C6C206E756C6C206F7220616C6C20706F70756C617465642729" + "3B0A202020202020202020202020454E44" + ), + ), + ( + "trigger", + "lifecycle_events_reject_delete", + "lifecycle_events", + ( + "4352454154452054524947474552206C6966656379636C655F6576656E74735F72656A656374" + "5F64656C6574650A2020202020202020202020204245464F52452044454C455445204F4E206C" + "6966656379636C655F6576656E74730A202020202020202020202020424547494E0A20202020" + "20202020202020202020202053454C4543542052414953452841424F52542C20276C69666563" + "79636C65206576656E74732061726520696D6D757461626C6527293B0A202020202020202020" + "202020454E44" + ), + ), + ( + "trigger", + "lifecycle_events_reject_update", + "lifecycle_events", + ( + "4352454154452054524947474552206C6966656379636C655F6576656E74735F72656A656374" + "5F7570646174650A2020202020202020202020204245464F524520555044415445204F4E206C" + "6966656379636C655F6576656E74730A202020202020202020202020424547494E0A20202020" + "20202020202020202020202053454C4543542052414953452841424F52542C20276C69666563" + "79636C65206576656E74732061726520696D6D757461626C6527293B0A202020202020202020" + "202020454E44" + ), + ), +) +_FROZEN_FRESH_BOTS_SQL_HEX = ( + "435245415445205441424C4520626F747320280A20202020202020202020202020202020626F" + "745F69642054455854205052494D415259204B45592C0A202020202020202020202020202020" + "2074656D706C6174655F69642054455854204E4F54204E554C4C2C0A20202020202020202020" + "202020202020646973706C61795F6E616D652054455854204E4F54204E554C4C2C0A20202020" + "20202020202020202020202070726F66696C655F706174682054455854204E4F54204E554C4C" + "2C0A202020202020202020202020202020207374617475732054455854204E4F54204E554C4C" + "2C0A2020202020202020202020202020202070696420494E54454745522C0A20202020202020" + "202020202020202020726573746172745F706F6C6963792054455854204E4F54204E554C4C20" + "44454641554C5420276D616E75616C272C0A2020202020202020202020202020202072657374" + "6172745F6261636B6F66665F7365636F6E6473205245414C204E4F54204E554C4C2044454641" + "554C5420352E302C0A20202020202020202020202020202020726573746172745F6D61785F61" + "7474656D70747320494E5445474552204E4F54204E554C4C2044454641554C5420352C0A2020" + "2020202020202020202020202020726573746172745F617474656D70747320494E5445474552" + "204E4F54204E554C4C2044454641554C5420302C0A202020202020202020202020202020206E" + "6578745F726573746172745F617420544558542C0A2020202020202020202020202020202073" + "7461727465645F617420544558542C0A2020202020202020202020202020202072656164795F" + "617420544558542C0A2020202020202020202020202020202073746F707065645F6174205445" + "58542C0A202020202020202020202020202020206C6173745F657869745F636F646520494E54" + "454745522C0A202020202020202020202020202020206C6173745F6572726F7220544558542C" + "0A202020202020202020202020202020206C6173745F7472616E736974696F6E5F726561736F" + "6E20544558542C0A202020202020202020202020202020206C6173745F6576656E745F696420" + "494E54454745522C0A20202020202020202020202020202020646573697265645F7374617465" + "2054455854204E4F54204E554C4C2044454641554C54202773746F70706564270A2020202020" + "202020202020202020202020202020434845434B2028646573697265645F737461746520494E" + "20282772756E6E696E67272C202773746F707065642729292C0A202020202020202020202020" + "20202020646573697265645F7265766973696F6E20494E5445474552204E4F54204E554C4C20" + "44454641554C54203020434845434B2028646573697265645F7265766973696F6E203E3D2030" + "292C0A20202020202020202020202020202020646573697265645F757064617465645F617420" + "544558542C0A2020202020202020202020202020202070656E64696E675F6F7065726174696F" + "6E5F696420544558542C0A2020202020202020202020202020202070656E64696E675F616374" + "696F6E205445585420434845434B20280A202020202020202020202020202020202020202070" + "656E64696E675F616374696F6E204953204E554C4C204F522070656E64696E675F616374696F" + "6E20494E2028277374617274272C202773746F70272C20277265737461727427290A20202020" + "202020202020202020202020292C0A2020202020202020202020202020202070656E64696E67" + "5F73696E636520544558542C0A20202020202020202020202020202020637265617465645F61" + "742054455854204E4F54204E554C4C2C0A202020202020202020202020202020207570646174" + "65645F61742054455854204E4F54204E554C4C2C0A2020202020202020202020202020202043" + "4845434B20280A20202020202020202020202020202020202020202870656E64696E675F6F70" + "65726174696F6E5F6964204953204E554C4C0A20202020202020202020202020202020202020" + "2020414E442070656E64696E675F616374696F6E204953204E554C4C0A202020202020202020" + "202020202020202020202020414E442070656E64696E675F73696E6365204953204E554C4C29" + "0A20202020202020202020202020202020202020204F520A2020202020202020202020202020" + "2020202020202870656E64696E675F6F7065726174696F6E5F6964204953204E4F54204E554C" + "4C20414E442070656E64696E675F616374696F6E204953204E4F54204E554C4C0A2020202020" + "20202020202020202020202020202020414E442070656E64696E675F73696E6365204953204E" + "4F54204E554C4C290A20202020202020202020202020202020290A2020202020202020202020" + "2029" +) +_FROZEN_LEGACY_BOTS_SQL_HEX = ( + "435245415445205441424C4520626F747320280A20202020626F745F69642054455854205052" + "494D415259204B45592C0A2020202074656D706C6174655F69642054455854204E4F54204E55" + "4C4C2C0A20202020646973706C61795F6E616D652054455854204E4F54204E554C4C2C0A2020" + "202070726F66696C655F706174682054455854204E4F54204E554C4C2C0A2020202073746174" + "75732054455854204E4F54204E554C4C2C0A2020202070696420494E54454745522C0A202020" + "20637265617465645F61742054455854204E4F54204E554C4C2C0A2020202075706461746564" + "5F61742054455854204E4F54204E554C4C0A2C20726573746172745F706F6C69637920544558" + "54204E4F54204E554C4C2044454641554C5420276D616E75616C272C20726573746172745F62" + "61636B6F66665F7365636F6E6473205245414C204E4F54204E554C4C2044454641554C542035" + "2E302C20726573746172745F6D61785F617474656D70747320494E5445474552204E4F54204E" + "554C4C2044454641554C5420352C20726573746172745F617474656D70747320494E54454745" + "52204E4F54204E554C4C2044454641554C5420302C206E6578745F726573746172745F617420" + "544558542C20737461727465645F617420544558542C2072656164795F617420544558542C20" + "73746F707065645F617420544558542C206C6173745F657869745F636F646520494E54454745" + "522C206C6173745F6572726F7220544558542C206C6173745F7472616E736974696F6E5F7265" + "61736F6E20544558542C206C6173745F6576656E745F696420494E54454745522C2064657369" + "7265645F73746174652054455854204E4F54204E554C4C2044454641554C54202773746F7070" + "65642720434845434B2028646573697265645F737461746520494E20282772756E6E696E6727" + "2C202773746F707065642729292C20646573697265645F7265766973696F6E20494E54454745" + "52204E4F54204E554C4C2044454641554C54203020434845434B2028646573697265645F7265" + "766973696F6E203E3D2030292C20646573697265645F757064617465645F617420544558542C" + "2070656E64696E675F6F7065726174696F6E5F696420544558542C2070656E64696E675F6163" + "74696F6E205445585420434845434B202870656E64696E675F616374696F6E204953204E554C" + "4C204F522070656E64696E675F616374696F6E20494E2028277374617274272C202773746F70" + "272C2027726573746172742729292C2070656E64696E675F73696E6365205445585429" +) +_FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX = ( + "435245415445205441424C4520736368656D615F76657273696F6E20280A2020202020202020" + "202020202020202076657273696F6E20494E5445474552204E4F54204E554C4C0A2020202020" + "2020202020202029" +) +_FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX = ( + "435245415445205441424C4520736368656D615F76657273696F6E20280A2020202020202020" + "20202020202020202020202076657273696F6E20494E5445474552204E4F54204E554C4C0A20" + "20202020202020202020202020202029" +) +EXPECTED_DDL_CATALOGS: dict[str, tuple[tuple[str, str, str, str], ...]] = { + "fresh": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_FRESH_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v0": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v1": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v2": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v3": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v4": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v5": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), +} + + +def _ddl_catalog(database_path: Path) -> tuple[tuple[str, str, str, str], ...]: + placeholders = ", ".join("?" for _name in DDL_OBJECT_NAMES) + with closing(sqlite3.connect(database_path)) as conn: + rows = conn.execute( + f""" + SELECT type, name, tbl_name, hex(CAST(sql AS BLOB)) + FROM sqlite_master + WHERE name IN ({placeholders}) + ORDER BY type, name + """, + DDL_OBJECT_NAMES, + ).fetchall() + return tuple((str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows) + + +def _seed_legacy_database(database_path: Path, version: int, *, with_bot: bool = False) -> None: + if not 0 <= version <= 5: + raise ValueError("legacy schema version must be between 0 and 5") + manager = SchemaManager(SQLiteDatabase(database_path)) + with closing(sqlite3.connect(database_path)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("BEGIN IMMEDIATE") + conn.execute(BASE_BOTS_SCHEMA) + if version >= 1: + manager._ensure_restart_schema(conn) + if version >= 2: + manager._ensure_lifecycle_schema(conn) + if with_bot: + conn.execute( + """ + INSERT INTO bots ( + bot_id, template_id, display_name, profile_path, status, pid, + created_at, updated_at + ) VALUES ( + 'coder', 'coding-bot', 'Coder', '/profiles/coder', 'running', 4321, + '2026-01-01T00:00:00+00:00', '2026-01-01T00:00:00+00:00' + ) + """ + ) + if version >= 3: + manager._migrate_v2_to_v3(conn) + if version >= 4: + manager._migrate_v3_to_v4(conn) + if version >= 5: + manager._migrate_v4_to_v5(conn) + if version > 0: + conn.execute( + """ + CREATE TABLE schema_version ( + version INTEGER NOT NULL + ) + """ + ) + conn.execute("INSERT INTO schema_version (version) VALUES (?)", (version,)) + conn.commit() + + +def _database_snapshot( + database_path: Path, +) -> tuple[ + str, + tuple[str, ...], + str, + tuple[tuple[str, bool], ...], +]: + digest = hashlib.sha256(database_path.read_bytes()).hexdigest() + uri = f"{database_path.resolve().as_uri()}?mode=ro" + with closing(_ORIGINAL_SQLITE_CONNECT(uri, uri=True)) as conn: + dump = tuple(conn.iterdump()) + journal_mode = str(conn.execute("PRAGMA journal_mode").fetchone()[0]) + sidecars = tuple( + (suffix, database_path.with_name(database_path.name + suffix).exists()) + for suffix in ("-wal", "-shm", "-journal") + ) + return digest, dump, journal_mode, sidecars + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +class SQLiteSchemaTests(unittest.TestCase): + def test_readiness_uses_one_read_only_connection_and_exact_current_schema(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + database = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + expected_uri = f"{database_path.resolve().as_uri()}?mode=ro" + calls: list[tuple[object, bool]] = [] + traces: list[str] = [] + real_connect = sqlite3.connect + + def connect_spy(*args: object, **kwargs: object) -> sqlite3.Connection: + calls.append((args[0], bool(kwargs.get("uri", False)))) + conn = real_connect(*args, **kwargs) + conn.set_trace_callback(traces.append) + return conn + + with patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy): + version = database.check_readiness() + + self.assertEqual(SCHEMA_VERSION, version) + self.assertEqual([(expected_uri, True)], calls) + self.assertEqual( + [ + statement + for statement in traces + if "sqlite_master" in statement + or "schema_version" in statement + or statement == "SELECT 1" + ], + traces, + ) + self.assertEqual("SELECT 1", traces[-1]) + self.assertFalse( + any(statement.lstrip().upper().startswith("PRAGMA") for statement in traces) + ) + + def test_readiness_rejects_missing_malformed_old_and_new_state_without_writes(self) -> None: + cases = ( + "missing", + "missing_table", + "empty", + "malformed_text", + "malformed_real", + "old", + "new", + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for case in cases: + with self.subTest(case=case): + database_path = root / case / "zeus.db" + if case != "missing": + database_path.parent.mkdir(parents=True) + with closing(sqlite3.connect(database_path)) as conn: + if case != "missing_table": + conn.execute("CREATE TABLE schema_version (version)") + if case == "malformed_text": + conn.execute("INSERT INTO schema_version VALUES ('six')") + elif case == "malformed_real": + conn.execute("INSERT INTO schema_version VALUES (6.5)") + elif case == "old": + conn.execute( + "INSERT INTO schema_version VALUES (?)", (SCHEMA_VERSION - 1,) + ) + elif case == "new": + conn.execute( + "INSERT INTO schema_version VALUES (?)", (SCHEMA_VERSION + 1,) + ) + conn.commit() + parent_existed = database_path.parent.exists() + before = _database_snapshot(database_path) if database_path.exists() else None + + with self.assertRaisesRegex(StateReadinessError, "state store is not ready"): + SQLiteDatabase(database_path).check_readiness() + + self.assertEqual(parent_existed, database_path.parent.exists()) + self.assertEqual( + before, + _database_snapshot(database_path) if database_path.exists() else None, + ) + if case == "missing": + self.assertFalse(database_path.exists()) + + def test_readiness_normalizes_operational_error_closes_connection_and_chains_cause( + self, + ) -> None: + connection = MagicMock(spec=sqlite3.Connection) + connection.row_factory = None + connection.execute.side_effect = [ + MagicMock(fetchone=MagicMock(return_value=(1,))), + MagicMock(fetchone=MagicMock(return_value={"version": SCHEMA_VERSION})), + sqlite3.OperationalError("injected readiness failure"), + ] + with ( + patch("zeus.sqlite_db.sqlite3.connect", return_value=connection), + self.assertRaises(StateReadinessError) as raised, + ): + SQLiteDatabase("state/zeus.db").check_readiness() + + self.assertIsInstance(raised.exception.__cause__, sqlite3.OperationalError) + self.assertEqual("state store is not ready", str(raised.exception)) + connection.close.assert_called_once_with() + + def test_state_store_readiness_wrapper_and_error_reexport(self) -> None: + database = MagicMock(spec=SQLiteDatabase) + database.check_readiness.return_value = SCHEMA_VERSION + with patch("zeus.state.SQLiteDatabase", return_value=database): + store = StateStore("state/zeus.db") + result = store.check_readiness() + + self.assertIs(StateReadinessError, FacadeStateReadinessError) + self.assertEqual(SCHEMA_VERSION, result) + database.check_readiness.assert_called_once_with() + + def test_new_schema_components_construct_without_opening_the_database(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "nested" / "zeus.db" + + database = SQLiteDatabase(database_path) + SchemaManager(database) + + self.assertEqual(database_path, database.database_path) + self.assertFalse(database_path.parent.exists()) + + def test_connection_preserves_row_factory_and_all_pragmas(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase(Path(tmp) / "nested" / "zeus.db") + + with closing(database.connect()) as conn: + values = { + "foreign_keys": conn.execute("PRAGMA foreign_keys").fetchone()[0], + "journal_mode": conn.execute("PRAGMA journal_mode").fetchone()[0], + "synchronous": conn.execute("PRAGMA synchronous").fetchone()[0], + "busy_timeout": conn.execute("PRAGMA busy_timeout").fetchone()[0], + } + row = conn.execute("SELECT 1 AS value").fetchone() + + self.assertIs(sqlite3.Row, conn.row_factory) + self.assertIsInstance(row, sqlite3.Row) + self.assertEqual( + { + "foreign_keys": 1, + "journal_mode": "wal", + "synchronous": 1, + "busy_timeout": 5000, + }, + values, + ) + + def test_immediate_uses_one_connection_and_commits_or_rolls_back(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = _TracingSQLiteDatabase(Path(tmp) / "zeus.db") + with database.immediate() as conn: + conn.execute("CREATE TABLE committed (value TEXT NOT NULL)") + conn.execute("INSERT INTO committed VALUES ('kept')") + + successful_trace = list(database.traces) + database.traces.clear() + with ( + self.assertRaisesRegex(RuntimeError, "injected rollback"), + database.immediate() as conn, + ): + conn.execute("INSERT INTO committed VALUES ('discarded')") + raise RuntimeError("injected rollback") + rollback_trace = list(database.traces) + + with closing(database.connect()) as conn: + values = [row[0] for row in conn.execute("SELECT value FROM committed")] + + successful_sql = [sql for _identity, sql in successful_trace] + rollback_sql = [sql for _identity, sql in rollback_trace] + self.assertEqual(1, len({identity for identity, _sql in successful_trace})) + self.assertEqual("BEGIN IMMEDIATE", successful_sql[0]) + self.assertEqual("COMMIT", successful_sql[-1]) + self.assertEqual("BEGIN IMMEDIATE", rollback_sql[0]) + self.assertEqual("ROLLBACK", rollback_sql[-1]) + self.assertEqual(["kept"], values) + + def test_state_store_delegates_without_opening_during_construction(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + connection = MagicMock(spec=sqlite3.Connection) + database.connect.return_value = connection + schema = MagicMock(spec=SchemaManager) + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + ): + store = StateStore(database_path) + database.connect.assert_not_called() + schema.init.assert_not_called() + schema.migrate.assert_not_called() + + self.assertEqual(database_path, store.database_path) + self.assertIs(connection, store.connect()) + store.init() + store.migrate() + + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) + self.assertEqual([call(database)], schema_type.call_args_list) + database.connect.assert_called_once_with() + schema.init.assert_called_once_with() + schema.migrate.assert_called_once_with() + + def test_fresh_and_every_legacy_branch_match_frozen_ddl_catalogs(self) -> None: + for case in ("fresh", "v0", "v1", "v2", "v3", "v4", "v5"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + store = StateStore(database_path) + if case == "fresh": + store.init() + else: + _seed_legacy_database(database_path, int(case[1:])) + store.migrate() + + self.assertEqual(EXPECTED_DDL_CATALOGS[case], _ddl_catalog(database_path)) + with closing(store.connect()) as conn: + self.assertEqual( + SCHEMA_VERSION, + conn.execute("SELECT version FROM schema_version").fetchone()[0], + ) + + def test_schema_init_and_migrate_each_use_one_immediate_transaction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + init_database = _TracingSQLiteDatabase(Path(tmp) / "fresh.db") + SchemaManager(init_database).init() + init_sql = [sql for _identity, sql in init_database.traces] + + migrate_path = Path(tmp) / "legacy.db" + _seed_legacy_database(migrate_path, 5) + migrate_database = _TracingSQLiteDatabase(migrate_path) + SchemaManager(migrate_database).migrate() + migrate_sql = [sql for _identity, sql in migrate_database.traces] + + self.assertEqual(1, len({identity for identity, _sql in init_database.traces})) + self.assertEqual("BEGIN IMMEDIATE", init_sql[0]) + self.assertEqual("COMMIT", init_sql[-1]) + self.assertLess( + next(index for index, sql in enumerate(init_sql) if "bots (" in sql), + next(index for index, sql in enumerate(init_sql) if "UPDATE schema_version" in sql), + ) + self.assertEqual(1, len({identity for identity, _sql in migrate_database.traces})) + self.assertEqual("BEGIN IMMEDIATE", migrate_sql[0]) + self.assertIn("CREATE TABLE reconcile_runs", "\n".join(migrate_sql)) + self.assertIn("UPDATE schema_version SET version = 6", migrate_sql) + self.assertEqual("COMMIT", migrate_sql[-1]) + + def test_migration_failure_rolls_back_ddl_backfill_and_version_together(self) -> None: + class FailingSchemaManager(SchemaManager): + def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: + super()._migrate_v5_to_v6(conn) + raise sqlite3.OperationalError("injected migration failure") + + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + _seed_legacy_database(database_path, 2, with_bot=True) + before_dump = _database_snapshot(database_path)[1] + database = _TracingSQLiteDatabase(database_path) + + with self.assertRaisesRegex(sqlite3.OperationalError, "injected migration failure"): + FailingSchemaManager(database).migrate() + + after_dump = _database_snapshot(database_path)[1] + with closing(sqlite3.connect(database_path)) as conn: + version = conn.execute("SELECT version FROM schema_version").fetchone()[0] + last_event_column = [ + row[1] + for row in conn.execute("PRAGMA table_info(bots)") + if row[1] == "last_event_id" + ] + lifecycle_table = conn.execute( + "SELECT 1 FROM sqlite_master WHERE name = 'lifecycle_events'" + ).fetchone() + bot = conn.execute( + "SELECT status, pid, updated_at FROM bots WHERE bot_id = 'coder'" + ).fetchone() + + sql = [statement for _identity, statement in database.traces] + self.assertEqual(before_dump, after_dump) + self.assertEqual(2, version) + self.assertEqual([], last_event_column) + self.assertIsNone(lifecycle_table) + self.assertEqual(("running", 4321, "2026-01-01T00:00:00+00:00"), bot) + self.assertEqual("BEGIN IMMEDIATE", sql[0]) + self.assertIn("INSERT INTO lifecycle_events", "\n".join(sql)) + self.assertIn("UPDATE schema_version SET version = 5", sql) + self.assertEqual("ROLLBACK", sql[-1]) + + def test_v7_rejection_is_read_only_for_connect_init_and_migrate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + with closing(sqlite3.connect(database_path)) as conn: + conn.execute("UPDATE schema_version SET version = 7") + conn.commit() + before = _database_snapshot(database_path) + expected_uri = f"{database_path.resolve().as_uri()}?mode=ro" + real_connect = sqlite3.connect + + for entry_point in ("connect", "init", "migrate"): + calls: list[tuple[object, bool]] = [] + + def connect_spy( + *args: object, + _calls: list[tuple[object, bool]] = calls, + **kwargs: object, + ) -> sqlite3.Connection: + _calls.append((args[0], bool(kwargs.get("uri", False)))) + return real_connect(*args, **kwargs) + + with ( + self.subTest(entry_point=entry_point), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy), + self.assertRaisesRegex(RuntimeError, "newer than supported"), + ): + getattr(StateStore(database_path), entry_point)() + + self.assertEqual([(expected_uri, True)], calls) + self.assertEqual(before, _database_snapshot(database_path)) + + def test_toctou_second_guard_rejects_before_pragmas_or_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + real_connect = sqlite3.connect + traces: list[str] = [] + post_swap_snapshot: list[object] = [] + calls: list[tuple[object, bool]] = [] + + def swap_after_preflight(path: Path) -> None: + _preflight_schema_compatibility(path) + with closing(real_connect(path)) as conn: + conn.execute("UPDATE schema_version SET version = 7") + conn.commit() + post_swap_snapshot.append(_database_snapshot(path)) + + def connect_spy(*args: object, **kwargs: object) -> sqlite3.Connection: + calls.append((args[0], bool(kwargs.get("uri", False)))) + conn = real_connect(*args, **kwargs) + if not kwargs.get("uri", False): + conn.set_trace_callback(traces.append) + return conn + + with ( + patch( + "zeus.sqlite_db._preflight_schema_compatibility", + side_effect=swap_after_preflight, + ), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy), + self.assertRaisesRegex(RuntimeError, "newer than supported"), + ): + SQLiteDatabase(database_path).connect() + + self.assertEqual(1, len(post_swap_snapshot)) + self.assertEqual(post_swap_snapshot[0], _database_snapshot(database_path)) + self.assertEqual(2, len(calls)) + self.assertTrue(calls[0][1]) + self.assertFalse(calls[1][1]) + self.assertTrue(any("schema_version" in sql for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("PRAGMA") for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("BEGIN") for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("UPDATE") for sql in traces)) + + +class SQLiteDurabilityTests(unittest.TestCase): + def test_database_default_is_normal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase(Path(tmp) / "zeus.db") + self.assertIs(SQLiteSynchronous.NORMAL, database.synchronous) + SchemaManager(database).init() + with closing(database.connect()) as conn: + synchronous = conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, synchronous) + + def test_database_full_is_applied_to_every_open(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase( + Path(tmp) / "zeus.db", + synchronous="FULL", + ) + self.assertIs(SQLiteSynchronous.FULL, database.synchronous) + SchemaManager(database).init() + with ( + closing(database.connect()) as first, + closing(database.connect()) as second, + database.immediate() as immediate, + ): + observed = [ + conn.execute("PRAGMA synchronous").fetchone()[0] + for conn in (first, second, immediate) + ] + + self.assertEqual([2, 2, 2], observed) + + def test_operational_connection_installs_full_in_guarded_order(self) -> None: + events: list[str] = [] + connection = MagicMock(spec=sqlite3.Connection) + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "nested" / "zeus.db" + database = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + + def preflight(path: Path) -> None: + self.assertTrue(path.parent.is_dir()) + events.append("preflight") + + def open_connection(path: Path) -> sqlite3.Connection: + self.assertEqual(database_path, path) + events.append("open") + return connection + + def guard(conn: sqlite3.Connection) -> None: + self.assertIs(connection, conn) + events.append("guard") + + def execute(sql: str) -> MagicMock: + events.append(sql) + return MagicMock() + + connection.execute.side_effect = execute + with ( + patch("zeus.sqlite_db._preflight_schema_compatibility", side_effect=preflight), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=open_connection), + patch("zeus.sqlite_db._assert_schema_compatible", side_effect=guard), + ): + actual = database.connect() + + self.assertIs(connection, actual) + self.assertIs(sqlite3.Row, connection.row_factory) + self.assertEqual( + [ + "preflight", + "open", + "guard", + "PRAGMA foreign_keys=ON", + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=FULL", + "PRAGMA busy_timeout=5000", + ], + events, + ) + + def test_connections_keep_independent_synchronous_modes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + normal = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.NORMAL) + full = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + SchemaManager(normal).init() + with closing(normal.connect()) as normal_conn, closing(full.connect()) as full_conn: + normal_value = normal_conn.execute("PRAGMA synchronous").fetchone()[0] + full_value = full_conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, normal_value) + self.assertEqual(2, full_value) + + def test_state_store_default_remains_normal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + store.init() + with closing(store.connect()) as conn: + synchronous = conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, synchronous) + + def test_state_store_rejects_invalid_mode_before_parent_creation(self) -> None: + invalid_values: tuple[object, ...] = ( + "OFF", + "EXTRA", + "full", + "FULL ", + "1", + "FULL; VACUUM", + 1, + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for index, value in enumerate(invalid_values): + with self.subTest(value=value): + database_path = root / str(index) / "zeus.db" + with self.assertRaises(ValueError): + StateStore(database_path, synchronous=cast(Any, value)) + self.assertFalse(database_path.parent.exists()) + + def test_state_store_passes_one_configured_database_to_every_delegate(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager") as schema_type, + patch("zeus.state.IdempotencyStore") as idempotency_type, + patch("zeus.state.ReconcileStore") as reconcile_type, + patch("zeus.state.BotLifecycleStore") as lifecycle_type, + ): + StateStore(database_path, synchronous=SQLiteSynchronous.FULL) + + database_type.assert_called_once_with( + database_path, + synchronous=SQLiteSynchronous.FULL, + ) + for child_type in ( + schema_type, + idempotency_type, + reconcile_type, + lifecycle_type, + ): + child_type.assert_called_once_with(database) + + def test_durability_configuration_does_not_change_schema_version(self) -> None: + self.assertEqual(6, SCHEMA_VERSION) + for mode in (SQLiteSynchronous.NORMAL, SQLiteSynchronous.FULL): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path, synchronous=mode).init() + self.assertEqual(EXPECTED_DDL_CATALOGS["fresh"], _ddl_catalog(database_path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index bc199cc..98221a1 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -1,10 +1,13 @@ from __future__ import annotations +import hashlib +import hmac import http.client import io import json import os import sqlite3 +import stat import subprocess import sys import tempfile @@ -12,25 +15,33 @@ import time import unittest import uuid -from contextlib import redirect_stderr, redirect_stdout +from contextlib import closing, redirect_stderr, redirect_stdout from datetime import UTC, datetime, timedelta from pathlib import Path -from unittest.mock import patch +from unittest.mock import call, patch +from zeus import __version__ from zeus.api import main as api_main from zeus.api import make_handler -from zeus.cli import _services, build_parser +from zeus.cli import _demo_services, _parse_env, _services, build_parser from zeus.cli import main as cli_main -from zeus.config import Settings -from zeus.doctor import _check_runtime_paths, run_doctor -from zeus.errors import BotArchiveError, BotDeleteError +from zeus.config import Settings, SQLiteSynchronous +from zeus.doctor import _check_bots, _check_runtime_paths, run_doctor +from zeus.envfile import parse_env_text +from zeus.errors import BotArchiveError, BotDeleteError, BotExistsError from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEventInput from zeus.logging_utils import redact_secrets from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState, RestartPolicy +from zeus.private_io import UnsafeFileError from zeus.process_lock import LockTimeoutError from zeus.readiness import ReadinessResult -from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunSummary +from zeus.reconciliation import ( + BotReconcileResult, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, +) from zeus.state import StateStore from zeus.supervisor import ( Supervisor, @@ -119,6 +130,19 @@ def _run_cli_failure(self, argv: list[str]) -> str: self.assertEqual(1, cli_main(argv)) return stdout.getvalue() + def _run_cli_result(self, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = cli_main(argv) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def _copy_cli_template(self, root: Path, template_id: str = "coding-bot") -> None: + templates = root / "templates" + templates.mkdir() + source = Path(__file__).resolve().parents[1] / "templates" / f"{template_id}.toml" + (templates / source.name).write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + def _fake_hermes_path(self, root: Path) -> str: hermes = root / "bin" / "hermes" hermes.parent.mkdir(parents=True, exist_ok=True) @@ -372,6 +396,113 @@ def test_adapter_builds_secret_safe_launcher_command_and_private_payload(self) - payload["marker_path"], ) + def test_strict_runtime_contract_accepts_schema3_and_rejects_compat_markers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / "hermes" + profile = hermes_root / "profiles" / "coder" + profile.mkdir(parents=True) + hermes_bin = self._fake_hermes_path(root) + store = StateStore(root / "zeus.db") + store.init() + running = BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + status=BotStatus.running, + pid=4321, + desired_state=DesiredState.running, + desired_revision=1, + ) + store.upsert_bot(running) + signals: list[tuple[int, object]] = [] + supervisor = Supervisor( + store, + hermes_bin, + hermes_root, + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: self._gateway_argv(hermes_bin), + proc_start_fingerprint_reader=lambda pid: "test-process-start:4321", + kill_fn=lambda pid, sent_signal: signals.append((pid, sent_signal)), + ) + argv = self._gateway_argv(hermes_bin) + command_fingerprint = hashlib.sha256( + json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + schema3 = { + "schema": 3, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "operation_id": "a" * 32, + "desired_revision": 1, + "argv": argv, + "resolved_hermes_bin": hermes_bin, + "command_fingerprint": command_fingerprint, + "readiness_probe": None, + "pid": 4321, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + schema2 = { + "schema": 2, + "pid": 4321, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "argv": self._gateway_argv(hermes_bin), + "resolved_hermes_bin": hermes_bin, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + legacy = { + "pid": 4321, + "argv": self._gateway_argv(hermes_bin), + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + marker_path = supervisor.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True) + + self.assertEqual(3, schema3["schema"]) + marker_path.write_text(json.dumps(schema3, sort_keys=True), encoding="utf-8") + inspected = supervisor.inspect("coder") + + self.assertEqual(3, inspected["pid_marker"]["schema"]) + self.assertIs(True, inspected["live_cmdline_verified"]) + self.assertEqual( + { + "verified": True, + "reason": "ok", + "classification": "direct-hermes", + "expected": { + "bot_id": "coder", + "component": "gateway", + "action": "run", + }, + }, + inspected["ownership"], + ) + + for name, marker in (("schema2", schema2), ("legacy", legacy)): + with self.subTest(name=name): + store.upsert_bot(running) + marker_path.write_text( + json.dumps(marker, sort_keys=True), + encoding="utf-8", + ) + result = supervisor.stop("coder") + + self.assertEqual(BotStatus.failed, result.status) + self.assertEqual( + "action required: schema-v2 or legacy gateway stop " + "requires manual process resolution", + result.message, + ) + self.assertEqual(marker, json.loads(marker_path.read_text(encoding="utf-8"))) + self.assertEqual([], signals) + def test_linux_cmdline_reader_uses_proc_cmdline(self) -> None: with tempfile.TemporaryDirectory() as tmp: proc_root = Path(tmp) / "proc" @@ -1955,6 +2086,472 @@ def test_cli_creates_bot(self) -> None: finally: os.chdir(old_cwd) + def test_cli_version_is_single_sourced_and_has_no_runtime_side_effects(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + old_cwd = Path.cwd() + stdout = io.StringIO() + stderr = io.StringIO() + try: + os.chdir(root) + with ( + patch("zeus.cli.Settings.from_env") as from_env, + patch("zeus.cli._services") as services, + patch("zeus.cli._demo_services") as demo_services, + redirect_stdout(stdout), + redirect_stderr(stderr), + self.assertRaises(SystemExit) as raised, + ): + cli_main(["--version"]) + + self.assertEqual(0, raised.exception.code) + self.assertEqual(f"zeus {__version__}\n", stdout.getvalue()) + self.assertEqual("", stderr.getvalue()) + from_env.assert_not_called() + services.assert_not_called() + demo_services.assert_not_called() + self.assertFalse((root / ".zeus").exists()) + finally: + os.chdir(old_cwd) + + def test_cli_help_is_descriptive_and_has_no_runtime_side_effects(self) -> None: + help_cases = ( + ( + ["--help"], + ( + "Manage local Hermes bot gateways and the Zeus API.", + "Run the local Zeus HTTP API server.", + "Check Zeus configuration, runtime paths, and dependencies.", + "Inspect available Hermes bot templates.", + "Run the bundled fake-Hermes lifecycle demo.", + "Create, inspect, and manage Hermes bot gateways.", + ), + ), + ( + ["serve", "--help"], + ("Run the local Zeus HTTP API server.", "bind host", "listen port", "4311"), + ), + ( + ["doctor", "--help"], + ( + "Check Zeus configuration, runtime paths, and dependencies.", + "machine-readable JSON", + "treat warnings as failures", + ), + ), + ( + ["template", "--help"], + ("Inspect available Hermes bot templates.", "List available bot templates."), + ), + ( + ["demo", "--help"], + ( + "Run the bundled fake-Hermes lifecycle demo.", + "Create and start the demo bot.", + "Show the demo bot status.", + "Stop the demo bot.", + ), + ), + ( + ["bot", "--help"], + ( + "Create, inspect, and manage Hermes bot gateways.", + "Create a bot profile from a template.", + "List registered bots.", + "Delete a bot registration.", + "Archive a bot profile.", + "Reconcile desired and observed bot state.", + "Show bot diagnostics.", + "Show immutable lifecycle history.", + "Start a bot gateway.", + "Stop a bot gateway gracefully.", + "Restart a bot gateway.", + "Show current bot status.", + "Show redacted bot logs.", + "Run Hermes doctor for a bot profile.", + ), + ), + ( + ["bot", "create", "--help"], + ( + "Create a bot profile from a template.", + "--replace", + "replace an existing bot", + "stop a running bot before replacement", + ), + ), + ( + ["bot", "start", "--help"], + ("Start a bot gateway.", "--wait", "wait for readiness", "--no-wait"), + ), + ( + ["bot", "reconcile", "--help"], + ( + "Reconcile desired and observed bot state.", + "--force", + "eligible restart now", + "--reset-restart", + "reset restart backoff", + ), + ), + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + old_cwd = Path.cwd() + try: + os.chdir(root) + with ( + patch("zeus.cli.Settings.from_env") as from_env, + patch("zeus.cli._services") as services, + patch("zeus.cli._demo_services") as demo_services, + ): + for argv, expected_fragments in help_cases: + with self.subTest(argv=argv): + stdout = io.StringIO() + stderr = io.StringIO() + with ( + redirect_stdout(stdout), + redirect_stderr(stderr), + self.assertRaises(SystemExit) as raised, + ): + cli_main(argv) + + self.assertEqual(0, raised.exception.code) + self.assertEqual("", stderr.getvalue()) + help_text = " ".join(stdout.getvalue().split()) + for fragment in expected_fragments: + self.assertIn(fragment, help_text) + + from_env.assert_not_called() + services.assert_not_called() + demo_services.assert_not_called() + self.assertFalse((root / ".zeus").exists()) + finally: + os.chdir(old_cwd) + + def test_bug_report_version_command_and_changelog_are_current(self) -> None: + bug_template = Path(".github/ISSUE_TEMPLATE/bug_report.yml").read_text(encoding="utf-8") + changelog = Path("CHANGELOG.md").read_text(encoding="utf-8") + unreleased = changelog.split("## 0.3.0", 1)[0] + + self.assertIn( + "description: Run `zeus --version` and paste the complete output.", + bug_template, + ) + self.assertNotIn("or the installed package version", bug_template) + self.assertIn("descriptive CLI help", unreleased) + self.assertIn("`zeus --version`", unreleased) + + def test_cli_create_help_recommends_env_from_and_warns_legacy_env_is_unsafe(self) -> None: + stdout = io.StringIO() + with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: + build_parser().parse_args(["bot", "create", "--help"]) + + self.assertEqual(0, raised.exception.code) + help_text = stdout.getvalue() + self.assertIn("--env-from", help_text) + self.assertIn("process environment", help_text) + self.assertIn("unsafe for secrets", help_text) + + def test_cli_env_from_process_value_wins_without_entering_parser_namespace_or_output( + self, + ) -> None: + process_secret = "process-precedence-sentinel" + dotenv_secret = "dotenv-precedence-sentinel" + parsed = build_parser().parse_args( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + self.assertEqual(["OPENROUTER_API_KEY"], parsed.env_from) + self.assertFalse(process_secret in repr(vars(parsed))) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + (root / ".env").write_text(f"OPENROUTER_API_KEY={dotenv_secret}\n", encoding="utf-8") + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENROUTER_API_KEY": process_secret, + }, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], process_secret) + ) + self.assertFalse(dotenv_secret in profile_env) + self.assertFalse(process_secret in stdout + stderr) + self.assertFalse(dotenv_secret in stdout + stderr) + audit_text = (root / ".zeus" / "logs" / "audit.jsonl").read_text(encoding="utf-8") + self.assertFalse(process_secret in audit_text) + self.assertFalse(dotenv_secret in audit_text) + database_bytes = (root / ".zeus" / "zeus.db").read_bytes() + self.assertFalse(process_secret.encode() in database_bytes) + self.assertFalse(dotenv_secret.encode() in database_bytes) + + def test_cli_env_from_uses_trusted_dotenv_fallback_without_echoing_value(self) -> None: + dotenv_secret = "trusted-dotenv-fallback-sentinel" + unrequested_secret = "unrequested-dotenv-sentinel" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + (root / ".env").write_text( + f"OPENROUTER_API_KEY={dotenv_secret}\nOPENAI_API_KEY={unrequested_secret}\n", + encoding="utf-8", + ) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + {"ZEUS_STATE_DIR": str(root / ".zeus")}, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], dotenv_secret) + ) + self.assertFalse(dotenv_secret in stdout + stderr) + self.assertFalse(unrequested_secret in profile_env) + self.assertFalse(unrequested_secret in stdout + stderr) + profile_env_path = root / ".zeus" / "hermes" / "profiles" / "coder" / ".env" + self.assertEqual(0o600, stat.S_IMODE(profile_env_path.stat().st_mode)) + + def test_cli_env_from_missing_and_empty_values_fail_before_service_creation(self) -> None: + fallback_secret = "must-not-fall-back-sentinel" + for value_state in ("missing", "empty"): + for as_json in (False, True): + with ( + self.subTest(value_state=value_state, as_json=as_json), + tempfile.TemporaryDirectory() as tmp, + ): + root = Path(tmp) + self._copy_cli_template(root) + if value_state == "empty": + (root / ".env").write_text( + f"OPENROUTER_API_KEY={fallback_secret}\n", encoding="utf-8" + ) + env = {"ZEUS_STATE_DIR": str(root / ".zeus")} + if value_state == "empty": + env["OPENROUTER_API_KEY"] = "" + argv = [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + if as_json: + argv.append("--json") + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict(os.environ, env, clear=True): + exit_code, stdout, stderr = self._run_cli_result(argv) + finally: + os.chdir(old_cwd) + + self.assertEqual(1, exit_code) + output = stdout + stderr + self.assertIn("OPENROUTER_API_KEY", output) + self.assertIn("missing or empty", output) + self.assertFalse(fallback_secret in output) + self.assertFalse((root / ".zeus").exists()) + if as_json: + payload = json.loads(stdout) + self.assertEqual("invalid_request", payload["error"]["code"]) + self.assertEqual("", stderr) + else: + self.assertEqual("", stdout) + + def test_cli_env_from_rejects_duplicate_and_invalid_names_without_values(self) -> None: + imported_secret = "imported-duplicate-sentinel" + legacy_secret = "legacy-duplicate-sentinel" + invalid_name = "invalid-name\ninjected-control-text" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENROUTER_API_KEY": imported_secret, + }, + clear=True, + ): + duplicate = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env", + f"OPENROUTER_API_KEY={legacy_secret}", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + invalid_import = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + invalid_name, + ] + ) + finally: + os.chdir(old_cwd) + + for exit_code, stdout, stderr in (duplicate, invalid_import): + self.assertEqual(1, exit_code) + self.assertFalse(imported_secret in stdout + stderr) + self.assertFalse(legacy_secret in stdout + stderr) + self.assertIn("provided by both --env and --env-from", duplicate[2]) + self.assertIn("valid environment variable name", invalid_import[2]) + self.assertFalse(invalid_name in invalid_import[2]) + self.assertNotIn("injected-control-text", invalid_import[2]) + self.assertFalse((root / ".zeus").exists()) + + def test_cli_env_from_value_is_not_disclosed_when_template_rejects_key(self) -> None: + secret = "controlled-failure-sentinel" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENAI_API_KEY": secret, + }, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENAI_API_KEY", + "--json", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(1, exit_code) + payload = json.loads(stdout) + self.assertEqual("invalid_request", payload["error"]["code"]) + self.assertIn("OPENAI_API_KEY", payload["error"]["message"]) + self.assertFalse(secret in stdout + stderr) + self.assertFalse((root / ".zeus" / "hermes" / "profiles" / "coder").exists()) + + def test_cli_legacy_env_syntax_remains_compatible(self) -> None: + legacy_value = "legacy-compatible-value" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + {"ZEUS_STATE_DIR": str(root / ".zeus")}, + clear=True, + ): + exit_code, _stdout, _stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env", + f"OPENROUTER_API_KEY={legacy_value}", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], legacy_value) + ) + + def test_cli_legacy_env_parser_keeps_historical_permissive_and_error_contract(self) -> None: + self.assertEqual({"legacy-key": "value"}, _parse_env(["legacy-key=value"])) + + malformed = "legacy-key" + with self.assertRaises(SystemExit) as raised: + _parse_env([malformed]) + self.assertEqual(f"--env must be NAME=VALUE, got {malformed!r}", str(raised.exception)) + def test_cli_lifecycle_failures_return_nonzero(self) -> None: with tempfile.TemporaryDirectory() as tmp: state_dir = Path(tmp) / ".zeus" @@ -1970,6 +2567,167 @@ def test_cli_lifecycle_failures_return_nonzero(self) -> None: self.assertEqual(BotStatus.failed, started["status"]) self.assertEqual(BotStatus.failed, status["status"]) + def test_cli_create_start_stop_and_reconcile_json_exit_contracts(self) -> None: + timestamp = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + created = BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path="/profiles/coder", + created_at=timestamp, + updated_at=timestamp, + ) + started = BotStatusResponse( + bot_id="start-failed", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/start-failed", + message="failed to start gateway: missing hermes", + ) + stopped = BotStatusResponse( + bot_id="coder", + status=BotStatus.stopped, + pid=None, + profile_path="/profiles/coder", + message="gateway shutdown completed", + ) + pending = BotStatusResponse( + bot_id="pending-bot", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/pending-bot", + message="restart scheduled: attempt 1/5 in 5s", + ) + terminal = BotStatusResponse( + bot_id="broken-bot", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/broken-bot", + message="manual policy: not restarting", + ) + + class CliContractSupervisor: + def create_bot(self, request, template, **kwargs): + if request.bot_id == "existing": + raise BotExistsError("bot already exists: existing") + return created + + def start(self, bot_id: str, **kwargs) -> BotStatusResponse: + return started + + def stop(self, bot_id: str, **kwargs) -> BotStatusResponse: + return stopped + + def reconcile(self, bot_id: str | None, **kwargs) -> list[BotStatusResponse]: + return [pending if bot_id == "pending-bot" else terminal] + + def run(argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = cli_main(argv) + return exit_code, stdout.getvalue(), stderr.getvalue() + + created_json = { + "bot_id": "coder", + "template_id": "coding-bot", + "display_name": "Coder", + "profile_path": "/profiles/coder", + "status": "stopped", + "pid": None, + "restart_policy": "manual", + "restart_backoff_seconds": 5.0, + "restart_max_attempts": 5, + "restart_attempts": 0, + "next_restart_at": None, + "started_at": None, + "ready_at": None, + "stopped_at": None, + "last_exit_code": None, + "last_error": None, + "last_transition_reason": None, + "desired_state": "stopped", + "converged": True, + "created_at": "2026-07-21T12:00:00+00:00", + "updated_at": "2026-07-21T12:00:00+00:00", + } + start_json = { + "bot_id": "start-failed", + "status": "failed", + "pid": None, + "profile_path": "/profiles/start-failed", + "message": "failed to start gateway: missing hermes", + } + stop_json = { + "bot_id": "coder", + "status": "stopped", + "pid": None, + "profile_path": "/profiles/coder", + "message": "gateway shutdown completed", + } + pending_json = { + "bot_id": "pending-bot", + "status": "failed", + "pid": None, + "profile_path": "/profiles/pending-bot", + "message": "restart scheduled: attempt 1/5 in 5s", + } + terminal_json = { + "bot_id": "broken-bot", + "status": "failed", + "pid": None, + "profile_path": "/profiles/broken-bot", + "message": "manual policy: not restarting", + } + + cases = ( + ( + "create_success", + ["bot", "create", "coder", "--template", "coding-bot", "--json"], + 0, + created_json, + ), + ( + "create_conflict", + ["bot", "create", "existing", "--template", "coding-bot", "--json"], + 1, + { + "error": { + "code": "bot_exists", + "message": "bot already exists: existing", + } + }, + ), + ("start_failure", ["bot", "start", "start-failed"], 1, start_json), + ("stop_success", ["bot", "stop", "coder"], 0, stop_json), + ( + "reconcile_pending", + ["bot", "reconcile", "pending-bot", "--json"], + 0, + [pending_json], + ), + ( + "reconcile_terminal", + ["bot", "reconcile", "broken-bot", "--json"], + 1, + [terminal_json], + ), + ) + with ( + tempfile.TemporaryDirectory() as tmp, + patch.dict(os.environ, {"ZEUS_STATE_DIR": str(Path(tmp) / ".zeus")}), + patch("zeus.cli._services", return_value=(object(), CliContractSupervisor())), + ): + for name, argv, expected_exit, expected_payload in cases: + with self.subTest(name=name): + exit_code, output, error_output = run(argv) + self.assertEqual(expected_exit, exit_code) + self.assertEqual( + json.dumps(expected_payload, sort_keys=True) + "\n", + output, + ) + self.assertEqual("", error_output) + def test_cli_wait_timeout_returns_nonzero_but_async_starting_is_successful(self) -> None: response = BotStatusResponse( bot_id="coder", @@ -2090,7 +2848,7 @@ def test_cli_reconcile_summary_json_runs_once_and_human_output_is_deterministic( }, payload["counts"], ) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 1, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -2709,6 +3467,123 @@ def test_audit_log_records_lifecycle_and_redacts_secret_like_fields(self) -> Non self.assertNotIn("plain-secret", store.audit_log_path().read_text(encoding="utf-8")) self.assertIn("[redacted]", store.audit_log_path().read_text(encoding="utf-8")) + def test_audit_sanitization_is_bounded_recursive_and_best_effort(self) -> None: + secret = "audit-sentinel-71b4" + + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + store.append_audit_event( + "bot.start_registration_failed", + cleanup_errors=[ + f"cleanup failed: API_KEY={secret}", + {"message": f"authorization=Bearer {secret}"}, + ], + readiness_message=f"probe returned Bearer {secret}", + nonfinite=float("nan"), + hostile=Hostile(), + ) + store.append_audit_event("bot.oversized", oversized="Δ" * 20_000) + store.append_audit_event( + "e" * 2_048, + first="x" * 1_800, + second="x" * 1_800, + third="x" * 1_800, + fourth="x" * 1_800, + ) + + raw = store.audit_log_path().read_bytes() + lines = raw.splitlines() + payload = json.loads(lines[0]) + self.assertFalse(secret.encode("utf-8") in raw) + self.assertNotIn(b"NaN", raw) + self.assertTrue(all(len(line) <= 8192 for line in lines)) + self.assertEqual("bot.start_registration_failed", payload["event"]) + self.assertEqual("[unsupported]", payload["hostile"]) + self.assertIsNone(payload["nonfinite"]) + self.assertTrue(json.loads(lines[1])["truncated"]) + self.assertTrue(json.loads(lines[2])["truncated"]) + + def test_projection_and_reconcile_messages_are_sanitized_before_persistence(self) -> None: + secret = "projection-sentinel-2f96" + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=f"readiness API_KEY={secret}", + last_transition_reason=f"cleanup Bearer {secret}", + ) + ) + initial_record = store.get_bot("coder") + assert initial_record is not None + self.assertFalse(secret in (initial_record.last_error or "")) + self.assertFalse(secret in (initial_record.last_transition_reason or "")) + store.update_lifecycle_state( + "coder", + BotStatus.failed, + last_error=f"updated readiness API_KEY={secret}", + last_transition_reason=f"updated cleanup Bearer {secret}", + ) + run = ReconcileRunStart( + run_id="security-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message=f"readiness API_KEY={secret}; cleanup Bearer {secret}", + error_code="readiness_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + loaded_record = store.get_bot("coder") + loaded_run = store.get_reconcile_run(run.run_id) + assert loaded_record is not None + assert loaded_run is not None + self.assertFalse(secret in (loaded_record.last_error or "")) + self.assertFalse(secret in (loaded_record.last_transition_reason or "")) + self.assertFalse(secret in result.message) + self.assertFalse(secret in loaded_run.results[0].message) + self.assertEqual(result, loaded_run.results[0]) + with closing(sqlite3.connect(store.database_path)) as conn: + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert bot_row is not None + assert reconcile_row is not None + persisted = "\n".join((*bot_row, *reconcile_row)) + self.assertFalse(secret in persisted) + def test_audit_write_failure_does_not_break_lifecycle_action(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -2746,23 +3621,649 @@ def test_audit_write_failure_does_not_break_lifecycle_action(self) -> None: self.assertEqual(2, len(events)) self.assertEqual(["running", "stopped"], [event.status_after for event in events]) - def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: + def test_audit_append_delegates_exact_utf8_json_line(self) -> None: with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - hermes_bin = self._fake_hermes_path(root) - profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" - (profile_path / "cron").mkdir(parents=True) - (profile_path / "logs").mkdir() - (profile_path / "config.yaml").write_text("model: test\n", encoding="utf-8") - (profile_path / "SOUL.md").write_text("soul\n", encoding="utf-8") - (profile_path / ".env").write_text( - "OPENROUTER_API_KEY=plain-secret\n", encoding="utf-8" - ) - (profile_path / "mcp.json").write_text("{}", encoding="utf-8") - (profile_path / "cron" / "jobs.json").write_text("[]", encoding="utf-8") - (profile_path / "logs" / "zeus-gateway.log").write_text( - "OPENAI_API_KEY=plain-log-secret\nready\n", - encoding="utf-8", + root = Path(tmp).resolve() + store = StateStore(root / "state" / "zeus.db") + fixed_now = datetime(2026, 7, 21, 12, 34, 56, tzinfo=UTC) + + with ( + patch("zeus.bot_lifecycle_store.datetime") as clock, + patch("zeus.bot_lifecycle_store.append_private_bytes") as append, + ): + clock.now.return_value = fixed_now + store.append_audit_event( + "bot.test", + label="Δ", + api_key="secret", + ) + + append.assert_called_once_with( + root / "state" / "logs" / "audit.jsonl", + b'{"api_key": "[redacted]", "event": "bot.test", "label": "\\u0394", ' + b'"ts": "2026-07-21T12:34:56+00:00"}\n', + ) + + def test_audit_parent_symlink_is_fail_open_without_mutating_target(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + store = StateStore(state_dir / "zeus.db") + store.init() + external_logs = root / "external-audit-target" + external_logs.mkdir(mode=0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external audit target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + (state_dir / "logs").symlink_to(external_logs, target_is_directory=True) + + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + hermes_bin = self._fake_hermes_path(root) + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor( + store, + hermes_bin, + root / ".zeus" / "hermes", + popen_factory=FakePopen, + pid_alive_fn=lambda pid: bool(FakePopen.launch_count), + cmdline_reader=lambda pid: self._gateway_argv(hermes_bin), + proc_start_fingerprint_reader=lambda pid: f"test-process-start:{pid}", + ) + + status = supervisor.start("coder") + + self.assertEqual(BotStatus.running, status.status) + self.assertEqual("external audit target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_logs.iterdir()) + ) + + def test_unsafe_gateway_log_prevents_pipe_and_process_creation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + logs_path = profile_path / "logs" + logs_path.mkdir(parents=True, mode=0o700) + target_log = root / "external-gateway-target.log" + target_log.write_text("external gateway target\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + (logs_path / "zeus-gateway.log").symlink_to(target_log) + hermes_bin = self._fake_hermes_path(root) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + + def forbidden_popen(*args: object, **kwargs: object) -> FakePopen: + self.fail("process factory must not be called for an unsafe gateway log") + + supervisor = Supervisor( + store, + hermes_bin, + root / ".zeus" / "hermes", + popen_factory=forbidden_popen, + startup_grace_seconds=0, + ) + + with patch( + "zeus.supervisor.os.pipe", + side_effect=AssertionError("pipe allocated before gateway log validation"), + ) as pipe: + status = supervisor.start("coder") + + pipe.assert_not_called() + self.assertEqual(BotStatus.failed, status.status) + self.assertIsNone(status.pid) + self.assertEqual("external gateway target\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + + def test_direct_logs_and_inspect_reject_gateway_log_parent_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-log-target" + external_logs.mkdir(mode=0o755) + target_log = external_logs / "zeus-gateway.log" + target_log.write_text("TARGET-SENTINEL\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") + + for operation in (supervisor.logs, supervisor.inspect): + with self.subTest(operation=operation.__name__): + with self.assertRaises(UnsafeFileError) as raised: + operation("coder") + self.assertNotIn("TARGET-SENTINEL", str(raised.exception)) + + self.assertEqual("TARGET-SENTINEL\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_direct_inspect_rejects_linked_pid_marker_fifo_without_blocking(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-logs" + external_logs.mkdir(mode=0o755) + fifo = external_logs / "zeus-gateway.pid.json" + os.mkfifo(fifo, mode=0o600) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external marker target\n", encoding="utf-8") + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + script = """ +import sys +from pathlib import Path +from zeus.private_io import UnsafeFileError +from zeus.state import StateStore +from zeus.supervisor import Supervisor + +root = Path(sys.argv[1]) +store = StateStore(root / "state" / "zeus.db") +supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") +try: + supervisor.inspect("coder") +except UnsafeFileError: + raise SystemExit(0) +raise SystemExit(2) +""" + + try: + completed = subprocess.run( + [sys.executable, "-c", script, str(root)], + check=False, + capture_output=True, + timeout=2, + ) + except subprocess.TimeoutExpired: + self.fail("direct inspect blocked on a linked PID-marker FIFO") + + self.assertEqual(0, completed.returncode, completed.stderr.decode("utf-8", "replace")) + self.assertEqual("external marker target\n", sentinel.read_text(encoding="utf-8")) + self.assertTrue(stat.S_ISFIFO(fifo.stat().st_mode)) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_api_inspect_returns_generic_500_for_linked_pid_marker_fifo_without_blocking( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-logs" + external_logs.mkdir(mode=0o755) + fifo = external_logs / "zeus-gateway.pid.json" + os.mkfifo(fifo, mode=0o600) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external API marker target\n", encoding="utf-8") + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(state_dir / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + script = """ +import http.client +import json +import sys +import threading +from http.server import ThreadingHTTPServer +from pathlib import Path +from zeus.api import make_handler +from zeus.config import Settings + +root = Path(sys.argv[1]) +settings = Settings.from_env({ + "ZEUS_STATE_DIR": str(root / "state"), + "ZEUS_API_KEY": "inspect-test-key", + "ZEUS_HOST": "127.0.0.1", + "ZEUS_PORT": "0", +}) +server = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(settings)) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +try: + connection = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=1) + connection.request( + "GET", + "/bots/coder/inspect", + headers={"x-zeus-api-key": "inspect-test-key"}, + ) + response = connection.getresponse() + body = json.loads(response.read()) + expected = { + "error": { + "code": "internal_error", + "message": "internal server error", + "status": 500, + } + } + if response.status != 500 or body != expected: + raise SystemExit(2) +finally: + server.shutdown() + server.server_close() +raise SystemExit(0) +""" + + try: + completed = subprocess.run( + [sys.executable, "-c", script, str(root)], + check=False, + capture_output=True, + timeout=4, + ) + except subprocess.TimeoutExpired: + self.fail("API inspect blocked on a linked PID-marker FIFO") + + self.assertEqual(0, completed.returncode, completed.stderr.decode("utf-8", "replace")) + self.assertEqual("external API marker target\n", sentinel.read_text(encoding="utf-8")) + self.assertTrue(stat.S_ISFIFO(fifo.stat().st_mode)) + + def test_pid_marker_read_rejects_profile_ancestry_replacement_during_read(self) -> None: + from zeus.supervisor import _read_bounded_file + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + external_logs = external / "logs" + external_logs.mkdir(parents=True) + (external_logs / marker.name).write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + opened: set[int] = set() + swapped = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def racing_read(fd: int, limit: int = 64 * 1024) -> bytes: + nonlocal swapped + result = _read_bounded_file(fd, limit) + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + return result + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor._read_bounded_file", side_effect=racing_read), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(swapped) + self.assertTrue(profile.is_symlink()) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_strict_marker_read_rejects_profile_ancestry_replacement_during_read(self) -> None: + from zeus.supervisor import _read_bounded_file + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + external_logs = external / "logs" + external_logs.mkdir(parents=True) + (external_logs / marker.name).write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + opened: set[int] = set() + swapped = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def racing_read(fd: int, limit: int = 64 * 1024) -> bytes: + nonlocal swapped + result = _read_bounded_file(fd, limit) + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + return result + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor._read_bounded_file", side_effect=racing_read), + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(swapped) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(profile.is_symlink()) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_pid_marker_read_revalidates_ancestry_after_read_failure(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + (external / "logs").mkdir(parents=True) + _store, supervisor = self._supervisor_for_profile_path(root, profile) + swapped = False + + def failing_read(_fd: int, _limit: int = 64 * 1024) -> bytes: + nonlocal swapped + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + raise LaunchPayloadError("injected bounded read failure") + + with ( + patch("zeus.supervisor._read_bounded_file", side_effect=failing_read), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(swapped) + self.assertTrue(profile.is_symlink()) + + def test_strict_marker_read_closes_all_descriptors_when_one_close_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + real_close = os.close + opened: set[int] = set() + close_failed = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def noisy_close(fd: int) -> None: + nonlocal close_failed + real_close(fd) + if not close_failed: + close_failed = True + raise OSError("injected close failure") + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor.os.close", side_effect=noisy_close), + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(close_failed) + self.assertEqual("present", observation.kind) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_strict_marker_read_rejects_current_owner_mismatch(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + marker.write_text('{"pid":4321}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + marker_stats = 0 + + def mismatched_owner_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal marker_stats + result = real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + if path == MARKER_NAME and dir_fd is not None: + marker_stats += 1 + if marker_stats == 2: + fields = list(result) + fields[4] = result.st_uid + 1 + return os.stat_result(fields) + return result + + with patch("zeus.gateway_launcher.os.stat", side_effect=mismatched_owner_stat): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertGreaterEqual(marker_stats, 2) + self.assertEqual("untrusted", observation.kind) + + def test_strict_marker_read_rechecks_leaf_after_final_directory_validation(self) -> None: + from zeus.gateway_launcher import _validate_open_directory_binding + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = logs / "displaced-marker.json" + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + logs_validations = 0 + swapped = False + + def racing_directory_validation( + parent_fd: int, + name: str, + directory_fd: int, + description: str, + ) -> os.stat_result: + nonlocal logs_validations, swapped + result = _validate_open_directory_binding( + parent_fd, + name, + directory_fd, + description, + ) + if name == "logs": + logs_validations += 1 + if logs_validations == 2: + marker.rename(displaced) + marker.symlink_to(target) + swapped = True + return result + + with patch( + "zeus.gateway_launcher._validate_open_directory_binding", + side_effect=racing_directory_validation, + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(swapped) + self.assertEqual(2, logs_validations) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(marker.is_symlink()) + + def test_pid_marker_read_rejects_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == MARKER_NAME and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + marker.symlink_to(target) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(appeared) + self.assertTrue(marker.is_symlink()) + self.assertEqual('{"pid":9876}', target.read_text(encoding="utf-8")) + + def test_strict_marker_read_rejects_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == MARKER_NAME and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + marker.symlink_to(target) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(appeared) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(marker.is_symlink()) + self.assertEqual('{"pid":9876}', target.read_text(encoding="utf-8")) + + def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_bin = self._fake_hermes_path(root) + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + (profile_path / "cron").mkdir(parents=True) + (profile_path / "logs").mkdir() + (profile_path / "config.yaml").write_text("model: test\n", encoding="utf-8") + (profile_path / "SOUL.md").write_text("soul\n", encoding="utf-8") + (profile_path / ".env").write_text( + "OPENROUTER_API_KEY=plain-secret\n", encoding="utf-8" + ) + (profile_path / "mcp.json").write_text("{}", encoding="utf-8") + (profile_path / "cron" / "jobs.json").write_text("[]", encoding="utf-8") + (profile_path / "logs" / "zeus-gateway.log").write_text( + "OPENAI_API_KEY=plain-log-secret\nready\n", + encoding="utf-8", ) store = StateStore(root / "zeus.db") store.init() @@ -2813,6 +4314,35 @@ def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: self.assertNotIn("OPENROUTER_API_KEY", serialized) self.assertNotIn(hermes_bin, serialized) + def test_supervisor_inspect_preserves_invalid_pid_marker_payload(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + logs_path = profile_path / "logs" + logs_path.mkdir(parents=True, mode=0o700) + marker = logs_path / "zeus-gateway.pid.json" + marker.write_text("{invalid json\n", encoding="utf-8") + marker.chmod(0o640) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") + + inspected = supervisor.inspect("coder") + + self.assertEqual(True, inspected["pid_marker"]["exists"]) + self.assertEqual(False, inspected["pid_marker"]["valid"]) + self.assertEqual("0640", inspected["pid_marker"]["mode"]) + self.assertIn("error", inspected["pid_marker"]) + self.assertEqual("", inspected["recent_logs"]) + def test_settings_parses_stop_kill_after_timeout_flag(self) -> None: with tempfile.TemporaryDirectory() as tmp: disabled = Settings.from_env( @@ -3039,11 +4569,293 @@ def test_doctor_rejects_world_accessible_state_directory(self) -> None: try: os.chdir(root) check = _check_runtime_paths(settings) + mode_after = state_dir.stat().st_mode & 0o777 finally: os.chdir(old_cwd) self.assertEqual("fail", check.status) self.assertIn("must not be accessible to other users", check.message) + self.assertEqual(0o755, mode_after) + + def test_doctor_rejects_real_and_broken_logs_symlinks_without_target_mutation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + state_dir.chmod(0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + external_logs = root / "external-doctor-target" + external_logs.mkdir(mode=0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("doctor target\n", encoding="utf-8") + target_mode = external_logs.stat().st_mode & 0o777 + logs_link = state_dir / "logs" + + logs_link.symlink_to(external_logs, target_is_directory=True) + real_check = _check_runtime_paths(settings) + logs_link.unlink() + logs_link.symlink_to(root / "missing-doctor-target", target_is_directory=True) + broken_check = _check_runtime_paths(settings) + + self.assertEqual("fail", real_check.status) + self.assertEqual("fail", broken_check.status) + self.assertIn("logs", real_check.message.lower()) + self.assertIn("logs", broken_check.message.lower()) + self.assertNotIn(str(external_logs), real_check.message) + self.assertEqual("doctor target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(target_mode, external_logs.stat().st_mode & 0o777) + + def test_doctor_rejects_permissive_logs_directory_without_repair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + logs_dir = state_dir / "logs" + logs_dir.mkdir(parents=True, mode=0o755) + state_dir.chmod(0o700) + logs_dir.chmod(0o755) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertIn("logs", check.message.lower()) + self.assertEqual(0o755, logs_dir.stat().st_mode & 0o777) + + def test_doctor_rejects_runtime_directory_owned_by_another_user(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + + def foreign_owner(path: Path) -> os.stat_result: + metadata = real_lstat(path) + fields = list(metadata) + fields[4] = os.geteuid() + 1 + return os.stat_result(fields) + + with patch("zeus.doctor.os.lstat", side_effect=foreign_owner): + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertIn("owned by the current user", check.message) + self.assertEqual(0o700, state_dir.stat().st_mode & 0o777) + + def test_doctor_rejects_state_symlink_without_target_mutation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + external_state = root / "external-state-target" + external_state.mkdir(mode=0o700) + external_state.chmod(0o700) + sentinel = external_state / "sentinel.txt" + sentinel.write_text("state target\n", encoding="utf-8") + target_mode = external_state.stat().st_mode & 0o777 + state_dir.symlink_to(external_state, target_is_directory=True) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + + self.assertEqual(state_dir, settings.state_dir) + self.assertEqual("fail", check.status) + self.assertNotIn(str(external_state), check.message) + self.assertEqual("state target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(target_mode, external_state.stat().st_mode & 0o777) + + def test_doctor_does_not_repair_state_replaced_after_preflight(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + displaced = root / "state-displaced" + replacement = root / "state-replacement" + replacement.mkdir(mode=0o777) + replacement.chmod(0o777) + sentinel = replacement / "sentinel.txt" + sentinel.write_text("replacement state\n", encoding="utf-8") + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + swapped = False + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + metadata = real_lstat(path, dir_fd=dir_fd) + if not swapped and dir_fd is None and Path(path) == state_dir: + state_dir.rename(displaced) + replacement.rename(state_dir) + swapped = True + return metadata + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertTrue(swapped) + self.assertEqual("fail", check.status) + self.assertEqual( + "replacement state\n", + (state_dir / sentinel.name).read_text(encoding="utf-8"), + ) + self.assertEqual(0o777, state_dir.stat().st_mode & 0o777) + + def test_doctor_does_not_repair_logs_replaced_after_preflight(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + logs_dir = state_dir / "logs" + logs_dir.mkdir(parents=True, mode=0o700) + state_dir.chmod(0o700) + logs_dir.chmod(0o700) + displaced = state_dir / "logs-displaced" + replacement = state_dir / "logs-replacement" + replacement.mkdir(mode=0o777) + replacement.chmod(0o777) + sentinel = replacement / "sentinel.txt" + sentinel.write_text("replacement logs\n", encoding="utf-8") + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + swapped = False + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + metadata = real_lstat(path, dir_fd=dir_fd) + if not swapped and dir_fd is None and Path(path) == logs_dir: + logs_dir.rename(displaced) + replacement.rename(logs_dir) + swapped = True + return metadata + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertTrue(swapped) + self.assertEqual("fail", check.status) + self.assertIn("logs", check.message.lower()) + self.assertEqual( + "replacement logs\n", + (logs_dir / sentinel.name).read_text(encoding="utf-8"), + ) + self.assertEqual(0o777, logs_dir.stat().st_mode & 0o777) + + def test_doctor_rejects_same_inode_mode_drift_at_final_binding(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + state_dir.chmod(0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + state_lstats = 0 + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal state_lstats + if path == state_dir.name and dir_fd is not None: + state_lstats += 1 + if state_lstats == 3: + state_dir.chmod(0o777) + return real_lstat(path, dir_fd=dir_fd) + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(3, state_lstats) + self.assertEqual(0o777, state_dir.stat().st_mode & 0o777) + + def test_settings_ensure_dirs_creates_missing_private_runtime_tree(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env({"ZEUS_STATE_DIR": str(Path(tmp) / "state")}) + + settings.ensure_dirs() + + for path in ( + settings.state_dir, + settings.hermes_root, + settings.state_dir / "logs", + settings.state_dir / "locks", + settings.state_dir / "locks" / "bots", + ): + with self.subTest(path=path): + self.assertTrue(path.is_dir()) + self.assertEqual(0o700, path.stat().st_mode & 0o777) + + def test_runtime_entrypoints_reject_real_and_dangling_state_links(self) -> None: + for dangling in (False, True): + with self.subTest(dangling=dangling), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + external_state = root / "external-state" + if not dangling: + external_state.mkdir(mode=0o755) + external_state.chmod(0o755) + (external_state / "sentinel.txt").write_text( + "external state\n", encoding="utf-8" + ) + state_dir.symlink_to(external_state, target_is_directory=True) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + with self.assertRaises(UnsafeFileError): + settings.ensure_dirs() + StateStore(settings.database_path).append_audit_event("link.proof") + with self.assertRaises(UnsafeFileError): + make_handler(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(state_dir, settings.state_dir) + if dangling: + self.assertFalse(external_state.exists()) + else: + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_state.iterdir()) + ) + self.assertEqual(0o755, external_state.stat().st_mode & 0o777) + + def test_runtime_entrypoints_reject_real_and_dangling_state_ancestor_links(self) -> None: + for dangling in (False, True): + with self.subTest(dangling=dangling), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + linked_parent = root / "linked-parent" + external_parent = root / "external-parent" + external_state = external_parent / "state" + if not dangling: + external_state.mkdir(parents=True, mode=0o700) + external_state.chmod(0o700) + (external_state / "sentinel.txt").write_text( + "external ancestor state\n", encoding="utf-8" + ) + linked_parent.symlink_to(external_parent, target_is_directory=True) + state_dir = linked_parent / "state" + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + with self.assertRaises(UnsafeFileError): + settings.ensure_dirs() + StateStore(settings.database_path).append_audit_event("ancestor-link.proof") + with self.assertRaises(UnsafeFileError): + make_handler(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(state_dir, settings.state_dir) + if dangling: + self.assertFalse(external_parent.exists()) + else: + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_state.iterdir()) + ) + self.assertEqual(0o700, external_state.stat().st_mode & 0o777) def test_api_health_and_auth(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -3308,5 +5120,99 @@ def test_api_allow_unauth_reads_keeps_mutations_locked(self) -> None: server.server_close() +class SQLiteDurabilityConfigurationTests(unittest.TestCase): + def test_settings_accept_exact_modes_and_default_missing_or_empty_to_normal(self) -> None: + cases = ( + (None, SQLiteSynchronous.NORMAL), + ("", SQLiteSynchronous.NORMAL), + ("NORMAL", SQLiteSynchronous.NORMAL), + ("FULL", SQLiteSynchronous.FULL), + ) + for raw_value, expected in cases: + with self.subTest(raw_value=raw_value): + env: dict[str, str] = {} + if raw_value is not None: + env["ZEUS_SQLITE_SYNCHRONOUS"] = raw_value + with patch("zeus.config.load_dotenv", return_value={}): + settings = Settings.from_env(env) + self.assertIs(expected, settings.sqlite_synchronous) + + def test_invalid_modes_fail_exactly_before_creating_state(self) -> None: + invalid_values = ("OFF", "EXTRA", "full", "FULL ", "1", "FULL; VACUUM") + for value in invalid_values: + with self.subTest(value=value), tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + with ( + patch("zeus.config.load_dotenv", return_value={}), + self.assertRaisesRegex( + ValueError, + r"^ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL$", + ), + ): + Settings.from_env( + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": value, + } + ) + self.assertFalse(state_dir.exists()) + + def test_cli_demo_and_doctor_forward_full_by_name(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env( + { + "ZEUS_STATE_DIR": str(Path(tmp) / "state"), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL", + } + ) + with ( + patch.object(Settings, "ensure_dirs"), + patch("zeus.cli.StateStore") as cli_store_type, + patch("zeus.cli._demo_hermes_bin", return_value="fake-hermes"), + patch("zeus.cli._demo_cmdline_reader", return_value=lambda _pid: []), + ): + _services(settings) + _demo_services(settings, "coder") + + with patch("zeus.doctor.StateStore") as doctor_store_type: + _check_bots(settings) + + expected = call( + settings.database_path, + synchronous=SQLiteSynchronous.FULL, + ) + self.assertEqual([expected, expected], cli_store_type.call_args_list) + self.assertEqual([expected], doctor_store_type.call_args_list) + + def test_cli_main_rejects_invalid_mode_before_runtime_state_or_services(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + stderr = io.StringIO() + with ( + patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": "OFF", + }, + clear=True, + ), + patch("zeus.config.load_dotenv", return_value={}), + patch("zeus.cli.StateStore") as store_type, + patch.object(Settings, "ensure_dirs") as ensure_dirs, + redirect_stderr(stderr), + ): + result = cli_main(["bot", "list"]) + + self.assertEqual(1, result) + self.assertEqual( + "Invalid Zeus configuration: ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL\n", + stderr.getvalue(), + ) + store_type.assert_not_called() + ensure_dirs.assert_not_called() + self.assertFalse(state_dir.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_verified_release_ref.py b/tests/test_verified_release_ref.py new file mode 100644 index 0000000..6283f40 --- /dev/null +++ b/tests/test_verified_release_ref.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import io +import runpy +import unittest +from pathlib import Path +from typing import Any +from urllib.error import URLError + +SCRIPT = runpy.run_path(str(Path("scripts/check_verified_release_ref.py"))) +ReleaseVerificationError = SCRIPT["ReleaseVerificationError"] +decode_json = SCRIPT["_decode_json"] +fetch_github_json = SCRIPT["_fetch_github_json"] +main = SCRIPT["main"] +verify_release_ref = SCRIPT["verify_release_ref"] + +TAG_OBJECT_SHA = "a" * 40 +COMMIT_SHA = "b" * 40 +TOKEN = "github-token-secret-sentinel" + + +def _environment() -> dict[str, str]: + return { + "GITHUB_TOKEN": TOKEN, + "GITHUB_REPOSITORY": "brainx/zeus", + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/tags/v0.4.0", + "GITHUB_REF_NAME": "v0.4.0", + "GITHUB_REF_TYPE": "tag", + "GITHUB_SHA": COMMIT_SHA, + } + + +def _responses() -> dict[str, dict[str, Any]]: + verified = { + "verified": True, + "reason": "valid", + "verified_at": "2026-07-22T01:23:45Z", + } + return { + "/repos/brainx/zeus/git/ref/tags/v0.4.0": { + "ref": "refs/tags/v0.4.0", + "object": {"type": "tag", "sha": TAG_OBJECT_SHA}, + }, + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}": { + "tag": "v0.4.0", + "sha": TAG_OBJECT_SHA, + "object": {"type": "commit", "sha": COMMIT_SHA}, + "verification": verified, + }, + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}": { + "sha": COMMIT_SHA, + "verification": verified, + }, + } + + +def _fetch_from(responses: dict[str, Any]) -> Any: + def fetch(path: str, _token: str) -> Any: + return responses[path] + + return fetch + + +class VerifiedReleaseRefTests(unittest.TestCase): + def test_accepts_verified_annotated_tag_bound_to_verified_event_commit(self) -> None: + responses = _responses() + calls: list[tuple[str, str]] = [] + + def fetch(path: str, token: str) -> dict[str, Any]: + calls.append((path, token)) + return responses[path] + + self.assertEqual( + ("v0.4.0", COMMIT_SHA), + verify_release_ref(_environment(), fetch_json=fetch), + ) + self.assertEqual(list(responses), [path for path, _token in calls]) + self.assertTrue(all(token == TOKEN for _path, token in calls)) + + def test_rejects_missing_or_malformed_workflow_environment_before_network(self) -> None: + cases = { + "missing token": ("GITHUB_TOKEN", ""), + "non-ascii token": ("GITHUB_TOKEN", "token-\N{SNOWMAN}"), + "wrong event": ("GITHUB_EVENT_NAME", "workflow_dispatch"), + "wrong ref type": ("GITHUB_REF_TYPE", "branch"), + "mismatched ref": ("GITHUB_REF", "refs/tags/v9.9.9"), + "unsafe repository": ("GITHUB_REPOSITORY", "brainx/zeus/extra"), + "unsafe tag": ("GITHUB_REF_NAME", "release/candidate"), + "malformed sha": ("GITHUB_SHA", "not-a-sha"), + } + for label, (name, value) in cases.items(): + with self.subTest(case=label): + environ = _environment() + environ[name] = value + + def unexpected_fetch(_path: str, _token: str) -> dict[str, Any]: + raise AssertionError("network fetch must not run") + + with self.assertRaises(ReleaseVerificationError): + verify_release_ref(environ, fetch_json=unexpected_fetch) + + def test_rejects_lightweight_or_mismatched_tag_ref(self) -> None: + for label, replacement in { + "lightweight": {"type": "commit", "sha": COMMIT_SHA}, + "malformed tag object": {"type": "tag", "sha": "not-a-sha"}, + }.items(): + with self.subTest(case=label): + responses = _responses() + responses["/repos/brainx/zeus/git/ref/tags/v0.4.0"]["object"] = replacement + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_unverified_tag_or_commit(self) -> None: + for endpoint in ( + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}", + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}", + ): + with self.subTest(endpoint=endpoint): + responses = _responses() + responses[endpoint]["verification"] = { + "verified": False, + "reason": "unsigned", + "verified_at": None, + } + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_tag_that_does_not_reference_event_commit(self) -> None: + responses = _responses() + responses[f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}"]["object"] = { + "type": "commit", + "sha": "d" * 40, + } + + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_malformed_api_shapes(self) -> None: + endpoints_and_payloads: tuple[tuple[str, Any], ...] = ( + ("/repos/brainx/zeus/git/ref/tags/v0.4.0", []), + ("/repos/brainx/zeus/git/ref/tags/v0.4.0", {"ref": "refs/tags/v0.4.0"}), + ( + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}", + {"tag": "v0.4.0", "sha": TAG_OBJECT_SHA}, + ), + ( + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}", + {"sha": COMMIT_SHA, "verification": {"verified": "true"}}, + ), + ) + for endpoint, payload in endpoints_and_payloads: + with self.subTest(endpoint=endpoint, payload=payload): + responses = _responses() + responses[endpoint] = payload + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_json_decoder_rejects_duplicates_constants_and_raw_body_disclosure(self) -> None: + secret_body = b'{"token":"raw-response-secret-sentinel",' + for body in ( + secret_body, + b'{"verification":NaN}', + b'{"sha":"one","sha":"two"}', + b"[]", + ): + with self.subTest(body=body): + with self.assertRaises(ReleaseVerificationError) as raised: + decode_json(body) + self.assertNotIn("raw-response-secret-sentinel", str(raised.exception)) + self.assertNotIn(body.decode("utf-8", errors="ignore"), str(raised.exception)) + + def test_network_failure_is_redacted_and_token_is_header_only(self) -> None: + class FailingOpener: + request: Any = None + assert_timeout: int | None = None + + def open(self, request: Any, *, timeout: int) -> Any: + self.request = request + self.assert_timeout = timeout + raise URLError(f"network failure containing {TOKEN}") + + opener = FailingOpener() + with self.assertRaises(ReleaseVerificationError) as raised: + fetch_github_json("/repos/brainx/zeus/git/ref/tags/v0.4.0", TOKEN, opener=opener) + + self.assertNotIn(TOKEN, str(raised.exception)) + self.assertNotIn(TOKEN, opener.request.full_url) + self.assertEqual(f"Bearer {TOKEN}", opener.request.get_header("Authorization")) + self.assertEqual(15, opener.assert_timeout) + + def test_main_emits_only_fixed_success_or_redacted_failure_messages(self) -> None: + responses = _responses() + stdout = io.StringIO() + stderr = io.StringIO() + status = main( + environ=_environment(), + fetch_json=lambda path, _token: responses[path], + stdout=stdout, + stderr=stderr, + ) + self.assertEqual(0, status) + self.assertEqual("", stderr.getvalue()) + self.assertIn("v0.4.0", stdout.getvalue()) + self.assertNotIn(TOKEN, stdout.getvalue()) + + stdout = io.StringIO() + stderr = io.StringIO() + + def fail_fetch(_path: str, _token: str) -> dict[str, Any]: + raise ReleaseVerificationError("GitHub API request failed") + + status = main( + environ=_environment(), + fetch_json=fail_fetch, + stdout=stdout, + stderr=stderr, + ) + self.assertEqual(1, status) + self.assertEqual("", stdout.getvalue()) + self.assertEqual( + "release ref verification failed: GitHub API request failed\n", + stderr.getvalue(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/api.py b/zeus/api.py index e3b4846..6839c56 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -5,7 +5,6 @@ import hmac import json import os -import signal import sys import threading import time @@ -14,33 +13,37 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus from http.server import BaseHTTPRequestHandler -from http.server import ThreadingHTTPServer as _ThreadingHTTPServer -from pathlib import Path from typing import Any, NoReturn -from urllib.parse import parse_qs, urlparse +from urllib.parse import urlparse +from zeus import api_request +from zeus.api_errors import map_api_exception from zeus.api_logging import ApiLogWriter -from zeus.config import Settings, validate_api_exposure +from zeus.api_request import ( + decode_json_object, + normalize_api_path, + parse_query, +) +from zeus.api_server import ThreadingHTTPServer +from zeus.api_server import serve as _serve_server +from zeus.config import Settings from zeus.doctor import run_doctor -from zeus.errors import ZeusConflictError from zeus.idempotency import IdempotencyClaim, canonical_request_hash, hash_key from zeus.models import ( BotCreateRequest, BotStatus, HermesTemplate, RestartPolicy, - TemplateError, validate_id, ) -from zeus.process_lock import BotProcessLock, LockTimeoutError +from zeus.process_lock import LockTimeoutError from zeus.rate_limit import TokenBucket from zeus.reconciliation import ( MAX_RECONCILE_TEXT_LENGTH, - ReconcileLockTimeoutError, ReconcileOutcome, ) from zeus.request_context import RequestContext, new_request_id, route_template -from zeus.state import MAX_IDEMPOTENCY_RESPONSE_BYTES, StateStore +from zeus.state import MAX_IDEMPOTENCY_RESPONSE_BYTES, StateReadinessError, StateStore from zeus.supervisor import Supervisor from zeus.templates import TemplateStore @@ -55,8 +58,8 @@ "restart_max_attempts", } ) -MAX_JSON_DEPTH = 64 -MAX_QUERY_FIELDS = 16 +MAX_JSON_DEPTH = api_request.MAX_JSON_DEPTH +MAX_QUERY_FIELDS = api_request.MAX_QUERY_FIELDS MAX_IDEMPOTENCY_MESSAGE_JSON_BYTES = 4_096 _IDEMPOTENCY_MESSAGE_REPLACEMENT = "response message omitted because it exceeded the replay budget" _IDEMPOTENCY_OWNER_LOCK = threading.Lock() @@ -218,173 +221,16 @@ def _bound_idempotent_messages(value: object) -> tuple[object, bool]: return value, False -def _json_object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate JSON field: {key}") - result[key] = value - return result - - -def _reject_json_constant(value: str) -> Any: - raise ValueError(f"invalid JSON constant: {value}") - - -def _validate_json_depth(value: Any) -> None: - stack = [(value, 1)] - while stack: - current, depth = stack.pop() - if depth > MAX_JSON_DEPTH: - raise ValueError(f"request JSON nesting exceeds {MAX_JSON_DEPTH}") - if isinstance(current, dict): - stack.extend((child, depth + 1) for child in current.values()) - elif isinstance(current, list): - stack.extend((child, depth + 1) for child in current) - - -class ThreadingHTTPServer(_ThreadingHTTPServer): - daemon_threads = True - - def __init__( - self, - server_address: Any, - RequestHandlerClass: type[BaseHTTPRequestHandler], - bind_and_activate: bool = True, - ) -> None: - self.api_max_concurrent_requests = int( - getattr(RequestHandlerClass, "api_max_concurrent_requests", 32) - ) - self.api_request_timeout_seconds = float( - getattr(RequestHandlerClass, "api_request_timeout_seconds", 10.0) - ) - self._request_slots = threading.BoundedSemaphore(self.api_max_concurrent_requests) - self._request_state = threading.Condition() - self._active_requests: set[int] = set() - self._draining = False - self._drain_started = threading.Event() - self._graceful_shutdown_requested = False - self._graceful_shutdown_timeout_seconds = 0.0 - self._graceful_shutdown_thread: threading.Thread | None = None - self._graceful_shutdown_result: bool | None = None - super().__init__(server_address, RequestHandlerClass, bind_and_activate) - - def process_request(self, request: Any, client_address: Any) -> None: - with self._request_state: - if self._draining: - rejection = ("server_draining", "API server is draining") - elif not self._request_slots.acquire(blocking=False): - rejection = ("server_busy", "API request capacity is exhausted") - else: - self._active_requests.add(id(request)) - rejection = None - - if rejection is not None: - self._reject_unavailable_request(request, *rejection) - self.shutdown_request(request) - return - try: - super().process_request(request, client_address) - except Exception: - self._finish_request(request) - raise - - def process_request_thread(self, request: Any, client_address: Any) -> None: - try: - request.settimeout(self.api_request_timeout_seconds) - super().process_request_thread(request, client_address) - finally: - self._finish_request(request) - - def begin_draining(self) -> None: - with self._request_state: - self._draining = True - self._drain_started.set() - - def request_graceful_shutdown(self, timeout_seconds: float) -> None: - self._graceful_shutdown_timeout_seconds = timeout_seconds - self._graceful_shutdown_requested = True - - def wait_until_draining(self, timeout_seconds: float) -> bool: - return self._drain_started.wait(timeout_seconds) - - def service_actions(self) -> None: - super().service_actions() - if not self._graceful_shutdown_requested or self._graceful_shutdown_thread is not None: - return - self.begin_draining() - coordinator = threading.Thread(target=self._complete_graceful_shutdown, daemon=True) - coordinator.start() - self._graceful_shutdown_thread = coordinator - - def finish_graceful_shutdown(self) -> bool | None: - coordinator = self._graceful_shutdown_thread - if coordinator is None: - return None - coordinator.join() - return self._graceful_shutdown_result - - def wait_for_drain(self, timeout_seconds: float) -> bool: - deadline = time.monotonic() + timeout_seconds - with self._request_state: - while self._active_requests: - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._request_state.wait(remaining) - return True - - def _finish_request(self, request: Any) -> None: - with self._request_state: - request_id = id(request) - if request_id not in self._active_requests: - return - self._active_requests.remove(request_id) - self._request_slots.release() - if not self._active_requests: - self._request_state.notify_all() - - def _complete_graceful_shutdown(self) -> None: - try: - self._graceful_shutdown_result = self.wait_for_drain( - self._graceful_shutdown_timeout_seconds - ) - finally: - self.shutdown() - - def _reject_unavailable_request(self, request: Any, code: str, message: str) -> None: - request_id = new_request_id() - body = json.dumps( - { - "error": { - "code": code, - "message": message, - "status": HTTPStatus.SERVICE_UNAVAILABLE.value, - } - }, - sort_keys=True, - ).encode("utf-8") - response = ( - b"HTTP/1.1 503 Service Unavailable\r\n" - b"connection: close\r\n" - b"content-type: application/json\r\n" - b"cache-control: no-store\r\n" - + f"x-request-id: {request_id}\r\n".encode("ascii") - + b"retry-after: 1\r\n" - + f"content-length: {len(body)}\r\n\r\n".encode("ascii") - + body - ) - with contextlib.suppress(OSError): - request.sendall(response) - - def make_handler(settings: Settings) -> type[BaseHTTPRequestHandler]: settings.ensure_dirs() api_log_writer = ApiLogWriter( settings.state_dir / "logs" / "api.jsonl", enabled=settings.api_log_enabled, ) - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() supervisor = Supervisor( store, @@ -413,6 +259,7 @@ class ZeusHandler(BaseHTTPRequestHandler): _request_context: RequestContext _response_status: int _response_error_code: str | None + _validated_query_values: dict[str, list[str]] def send_error( self, @@ -442,7 +289,21 @@ def _dispatch_get(self) -> None: else: self._validate_query_parameters(set()) self._require_key(read=not self._get_requires_strict_auth(path)) - if path == "/doctor": + if path == "/ready": + try: + schema_version = store.check_readiness() + except StateReadinessError: + self._json_error_response( + HTTPStatus.SERVICE_UNAVAILABLE, + "not_ready", + "state store is not ready", + ) + return + self._json( + HTTPStatus.OK, + {"schema_version": schema_version, "status": "ready"}, + ) + elif path == "/doctor": self._json(HTTPStatus.OK, run_doctor(settings).to_dict()) elif path == "/templates": self._json(HTTPStatus.OK, [template_to_dict(t) for t in TemplateStore().list()]) @@ -820,18 +681,7 @@ def _read_json(self) -> dict[str, Any]: if length > 1_000_000: raise ValueError("request body too large") data = self.rfile.read(length) if length else b"{}" - try: - parsed = json.loads( - data.decode("utf-8"), - object_pairs_hook=_json_object_without_duplicates, - parse_constant=_reject_json_constant, - ) - except RecursionError as exc: - raise ValueError(f"request JSON nesting exceeds {MAX_JSON_DEPTH}") from exc - _validate_json_depth(parsed) - if not isinstance(parsed, dict): - raise ValueError("request body must be a JSON object") - return parsed + return decode_json_object(data) def _require_json_content_type(self) -> None: content_encoding = self.headers.get("content-encoding", "").strip().lower() @@ -882,34 +732,13 @@ def _bot_id_from_path(self, path: str, action: str) -> str: return validate_id(parts[1], "bot_id") def _normalized_path(self) -> str: - parsed = urlparse(self.path) - if parsed.fragment: - raise ValueError("request target must not include a fragment") - path = parsed.path - if path == "/v1": - return "/" - if path.startswith("/v1/"): - return path[3:] - return path + return normalize_api_path(self.path) def _query_values(self) -> dict[str, list[str]]: - try: - return parse_qs( - urlparse(self.path).query, - keep_blank_values=True, - max_num_fields=MAX_QUERY_FIELDS, - ) - except ValueError as exc: - raise ValueError("too many query parameters") from exc + return self._validated_query_values def _validate_query_parameters(self, allowed: set[str]) -> None: - values = self._query_values() - unknown = sorted(set(values) - allowed) - if unknown: - raise ValueError(f"unknown query parameter: {unknown[0]}") - duplicates = sorted(name for name, entries in values.items() if len(entries) > 1) - if duplicates: - raise ValueError(f"query parameter {duplicates[0]} must be specified once") + self._validated_query_values = parse_query(self.path, frozenset(allowed)) def _post_query_parameters(self, path: str) -> set[str] | None: if path == "/bots": @@ -1057,39 +886,10 @@ def _consume_mutation_capacity(self) -> None: def _json_error(self, exc: Exception) -> None: if isinstance(exc, _ResponseSent): return - if isinstance(exc, KeyError): - status, code, message = _key_error_response(exc) - self._json_error_response(status, code, message) - elif isinstance(exc, TemplateError): - code = ( - "invalid_bot_id" - if str(exc).startswith("bot_id must match") - else "invalid_request" - ) - self._json_error_response(HTTPStatus.BAD_REQUEST, code, str(exc)) - elif isinstance(exc, ReconcileLockTimeoutError): - self._json_error_response( - HTTPStatus.CONFLICT, - "reconcile_locked", - "reconciliation is already in progress", - ) - elif isinstance(exc, LockTimeoutError): - self._json_error_response( - HTTPStatus.CONFLICT, - "bot_locked", - "bot lifecycle operation is already in progress", - ) - elif isinstance(exc, ZeusConflictError): - self._json_error_response(HTTPStatus.CONFLICT, exc.code, str(exc)) - elif isinstance(exc, ValueError): - self._json_error_response(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) - else: + error = map_api_exception(exc) + if error.log_exception: api_log_writer.error(self._request_context.request_id, exc) - self._json_error_response( - HTTPStatus.INTERNAL_SERVER_ERROR, - "internal_error", - "internal server error", - ) + self._json_error_response(error.status, error.code, error.message) def _handle_idempotency_outcome(self, claim: IdempotencyClaim) -> bool: if claim.kind == "claimed": @@ -1150,41 +950,19 @@ def _handle_idempotency_outcome(self, claim: IdempotencyClaim) -> bool: raise RuntimeError("invalid idempotency claim outcome") def _buffer_error(self, exc: Exception) -> BufferedJsonResponse: - if isinstance(exc, KeyError): - status, code, message = _key_error_response(exc) - elif isinstance(exc, TemplateError): - status = HTTPStatus.BAD_REQUEST - code = ( - "invalid_bot_id" - if str(exc).startswith("bot_id must match") - else "invalid_request" - ) - message = str(exc) - elif isinstance(exc, ReconcileLockTimeoutError): - status = HTTPStatus.CONFLICT - code = "reconcile_locked" - message = "reconciliation is already in progress" - elif isinstance(exc, LockTimeoutError): - status = HTTPStatus.CONFLICT - code = "bot_locked" - message = "bot lifecycle operation is already in progress" - elif isinstance(exc, ZeusConflictError): - status = HTTPStatus.CONFLICT - code = exc.code - message = str(exc) - elif isinstance(exc, ValueError): - status = HTTPStatus.BAD_REQUEST - code = "invalid_request" - message = str(exc) - else: - status = HTTPStatus.INTERNAL_SERVER_ERROR - code = "internal_error" - message = "internal server error" + error = map_api_exception(exc) + if error.log_exception: api_log_writer.error(self._request_context.request_id, exc) - self._response_error_code = code + self._response_error_code = error.code return BufferedJsonResponse( - status, - {"error": {"code": code, "message": message, "status": status.value}}, + error.status, + { + "error": { + "code": error.code, + "message": error.message, + "status": error.status.value, + } + }, ) def _buffer_internal_error(self, exc: Exception) -> BufferedJsonResponse: @@ -1317,6 +1095,7 @@ def _handle_request(self, dispatch: Callable[[], None]) -> None: self._request_context.route = _safe_route_template(self.path) self._response_status = HTTPStatus.INTERNAL_SERVER_ERROR.value self._response_error_code = None + self._validated_query_values = {} try: dispatch() except Exception as exc: @@ -1353,16 +1132,6 @@ def _safe_route_template(target: str) -> str | None: return None -def _key_error_response(exc: KeyError) -> tuple[HTTPStatus, str, str]: - detail = exc.args[0] if exc.args else "missing required field" - message = str(detail) - if message.startswith("unknown bot:"): - return HTTPStatus.NOT_FOUND, "unknown_bot", message - if message.startswith("unknown template:"): - return HTTPStatus.BAD_REQUEST, "unknown_template", message - return HTTPStatus.BAD_REQUEST, "invalid_request", f"missing required field: {message}" - - def template_to_dict(template: HermesTemplate) -> dict[str, Any]: return { "id": template.id, @@ -1376,89 +1145,13 @@ def template_to_dict(template: HermesTemplate) -> dict[str, Any]: def serve(host: str, port: int, settings: Settings | None = None) -> None: base_settings = settings or Settings.from_env() - runtime_settings = replace(base_settings, host=host, port=port) - if not 0 <= port <= 65535: - raise ValueError("port must be between 0 and 65535") - validate_api_exposure( - runtime_settings.host, - runtime_settings.api_key, - runtime_settings.allow_unauth_reads, - ) - runtime_settings.ensure_dirs() - pid = os.getpid() - pid_path = runtime_settings.state_dir / "zeus.pid" - api_lock = BotProcessLock( - runtime_settings.state_dir / "locks" / "api.lock", - timeout_seconds=min(runtime_settings.lock_timeout_seconds, 1.0), + _serve_server( + host, + port, + base_settings, + handler_factory=make_handler, + server_factory=ThreadingHTTPServer, ) - with api_lock: - handler = make_handler(runtime_settings) - server = ThreadingHTTPServer((host, port), handler) - previous_signal_handlers: dict[signal.Signals, Any] = {} - try: - if threading.current_thread() is threading.main_thread(): - for signum in (signal.SIGTERM, signal.SIGINT): - previous_signal_handlers[signum] = signal.getsignal(signum) - signal.signal( - signum, - lambda _signum, _frame: server.request_graceful_shutdown( - runtime_settings.api_shutdown_drain_seconds - ), - ) - with contextlib.suppress(KeyboardInterrupt): - _write_api_pid(pid_path, pid) - server.serve_forever() - finally: - for signum, previous_handler in previous_signal_handlers.items(): - signal.signal(signum, previous_handler) - finish_graceful_shutdown = getattr(server, "finish_graceful_shutdown", None) - drain_result = ( - finish_graceful_shutdown() if callable(finish_graceful_shutdown) else None - ) - if drain_result is None: - begin_draining = getattr(server, "begin_draining", None) - if callable(begin_draining): - begin_draining() - server.server_close() - wait_for_drain = getattr(server, "wait_for_drain", None) - drain_result = ( - wait_for_drain(runtime_settings.api_shutdown_drain_seconds) - if callable(wait_for_drain) - else True - ) - else: - server.server_close() - if not drain_result: - print("warning: API shutdown drain deadline expired", file=sys.stderr) - _remove_api_pid_if_owned(pid_path, pid) - - -def _write_api_pid(path: Path, pid: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - tmp = path.with_name(f".{path.name}.{pid}.{time.time_ns()}.tmp") - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(f"{pid}\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp, path) - with contextlib.suppress(OSError): - path.chmod(0o600) - finally: - with contextlib.suppress(FileNotFoundError): - tmp.unlink() - - -def _remove_api_pid_if_owned(path: Path, pid: int) -> None: - try: - recorded_pid = int(path.read_text(encoding="utf-8").strip()) - except (FileNotFoundError, OSError, ValueError): - return - if recorded_pid != pid: - return - with contextlib.suppress(FileNotFoundError): - path.unlink() def main(argv: list[str] | None = None) -> int: diff --git a/zeus/api_errors.py b/zeus/api_errors.py new file mode 100644 index 0000000..5ac2802 --- /dev/null +++ b/zeus/api_errors.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from dataclasses import dataclass +from http import HTTPStatus + +from zeus.errors import ZeusConflictError +from zeus.models import TemplateError +from zeus.process_lock import LockTimeoutError +from zeus.reconciliation import ReconcileLockTimeoutError + + +@dataclass(frozen=True) +class ApiError: + status: HTTPStatus + code: str + message: str + log_exception: bool = False + + +def map_api_exception(exc: Exception) -> ApiError: + if isinstance(exc, KeyError): + return _map_key_error(exc) + if isinstance(exc, TemplateError): + code = "invalid_bot_id" if str(exc).startswith("bot_id must match") else "invalid_request" + return ApiError(HTTPStatus.BAD_REQUEST, code, str(exc)) + if isinstance(exc, ReconcileLockTimeoutError): + return ApiError( + HTTPStatus.CONFLICT, + "reconcile_locked", + "reconciliation is already in progress", + ) + if isinstance(exc, LockTimeoutError): + return ApiError( + HTTPStatus.CONFLICT, + "bot_locked", + "bot lifecycle operation is already in progress", + ) + if isinstance(exc, ZeusConflictError): + return ApiError(HTTPStatus.CONFLICT, exc.code, str(exc)) + if isinstance(exc, ValueError): + return ApiError(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) + return ApiError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "internal_error", + "internal server error", + log_exception=True, + ) + + +def _map_key_error(exc: KeyError) -> ApiError: + detail = exc.args[0] if exc.args else "missing required field" + message = str(detail) + if message.startswith("unknown bot:"): + return ApiError(HTTPStatus.NOT_FOUND, "unknown_bot", message) + if message.startswith("unknown template:"): + return ApiError(HTTPStatus.BAD_REQUEST, "unknown_template", message) + return ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + f"missing required field: {message}", + ) diff --git a/zeus/api_logging.py b/zeus/api_logging.py index 7fbf831..41fa9e7 100644 --- a/zeus/api_logging.py +++ b/zeus/api_logging.py @@ -2,13 +2,13 @@ import json import math -import os import re import threading from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path +from zeus.private_io import append_private_bytes from zeus.request_context import AUTH_OUTCOMES, IDEMPOTENCY_OUTCOMES, route_template _HTTP_METHODS = frozenset({"GET", "POST", "UNSUPPORTED"}) @@ -140,37 +140,10 @@ def _write(self, payload: Mapping[str, object]) -> None: if not self.enabled: return try: - line = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + line = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) with self._lock: - self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - directory_flags = os.O_RDONLY - directory_flags |= getattr(os, "O_DIRECTORY", 0) - directory_flags |= getattr(os, "O_CLOEXEC", 0) - directory_flags |= getattr(os, "O_NOFOLLOW", 0) - file_flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT - file_flags |= getattr(os, "O_CLOEXEC", 0) - file_flags |= getattr(os, "O_NOFOLLOW", 0) - directory_fd: int | None = None - file_fd: int | None = None - try: - directory_fd = os.open(self.path.parent, directory_flags) - os.fchmod(directory_fd, 0o700) - file_fd = os.open( - self.path.name, - file_flags, - 0o600, - dir_fd=directory_fd, - ) - os.fchmod(file_fd, 0o600) - with os.fdopen(file_fd, "a", encoding="utf-8") as handle: - file_fd = None - handle.write(line) - finally: - try: - if file_fd is not None: - os.close(file_fd) - finally: - if directory_fd is not None: - os.close(directory_fd) + append_private_bytes(self.path, line) except (OSError, TypeError, ValueError): return diff --git a/zeus/api_request.py b/zeus/api_request.py new file mode 100644 index 0000000..480f07c --- /dev/null +++ b/zeus/api_request.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import json +from typing import Any, cast +from urllib.parse import parse_qs, urlparse + +MAX_JSON_DEPTH = 64 +MAX_QUERY_FIELDS = 16 + + +def normalize_api_path(target: str) -> str: + parsed = urlparse(target) + if parsed.fragment: + raise ValueError("request target must not include a fragment") + path = parsed.path + if path == "/v1": + return "/" + if path.startswith("/v1/"): + return path[3:] + return path + + +def parse_query(target: str, allowed: frozenset[str]) -> dict[str, list[str]]: + try: + values = parse_qs( + urlparse(target).query, + keep_blank_values=True, + max_num_fields=MAX_QUERY_FIELDS, + ) + except ValueError as exc: + raise ValueError("too many query parameters") from exc + unknown = sorted(set(values) - allowed) + if unknown: + raise ValueError(f"unknown query parameter: {unknown[0]}") + duplicates = sorted(name for name, entries in values.items() if len(entries) > 1) + if duplicates: + raise ValueError(f"query parameter {duplicates[0]} must be specified once") + return values + + +def decode_json_object(data: bytes, *, max_depth: int = MAX_JSON_DEPTH) -> dict[str, Any]: + try: + parsed: object = json.loads( + data.decode("utf-8"), + object_pairs_hook=_json_object_without_duplicates, + parse_constant=_reject_json_constant, + ) + except RecursionError as exc: + raise ValueError(f"request JSON nesting exceeds {max_depth}") from exc + _validate_json_depth(parsed, max_depth=max_depth) + if not isinstance(parsed, dict): + raise ValueError("request body must be a JSON object") + return cast(dict[str, Any], parsed) + + +def _json_object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON field: {key}") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"invalid JSON constant: {value}") + + +def _validate_json_depth(value: object, *, max_depth: int) -> None: + stack = [(value, 1)] + while stack: + current, depth = stack.pop() + if depth > max_depth: + raise ValueError(f"request JSON nesting exceeds {max_depth}") + if isinstance(current, dict): + stack.extend((child, depth + 1) for child in current.values()) + elif isinstance(current, list): + stack.extend((child, depth + 1) for child in current) diff --git a/zeus/api_server.py b/zeus/api_server.py new file mode 100644 index 0000000..d93e430 --- /dev/null +++ b/zeus/api_server.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import contextlib +import json +import os +import signal +import sys +import threading +import time +from collections.abc import Callable +from dataclasses import replace +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler +from http.server import ThreadingHTTPServer as _ThreadingHTTPServer +from pathlib import Path +from typing import Any + +from zeus.config import Settings, validate_api_exposure +from zeus.process_lock import BotProcessLock +from zeus.request_context import new_request_id + +HandlerFactory = Callable[[Settings], type[BaseHTTPRequestHandler]] +ServerFactory = Callable[[tuple[str, int], type[BaseHTTPRequestHandler]], Any] + + +class ThreadingHTTPServer(_ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + server_address: Any, + RequestHandlerClass: type[BaseHTTPRequestHandler], + bind_and_activate: bool = True, + ) -> None: + self.api_max_concurrent_requests = int( + getattr(RequestHandlerClass, "api_max_concurrent_requests", 32) + ) + self.api_request_timeout_seconds = float( + getattr(RequestHandlerClass, "api_request_timeout_seconds", 10.0) + ) + self._request_slots = threading.BoundedSemaphore(self.api_max_concurrent_requests) + self._request_state = threading.Condition() + self._active_requests: set[int] = set() + self._draining = False + self._drain_started = threading.Event() + self._graceful_shutdown_requested = False + self._graceful_shutdown_timeout_seconds = 0.0 + self._graceful_shutdown_thread: threading.Thread | None = None + self._graceful_shutdown_result: bool | None = None + super().__init__(server_address, RequestHandlerClass, bind_and_activate) + + def process_request(self, request: Any, client_address: Any) -> None: + with self._request_state: + if self._draining: + rejection = ("server_draining", "API server is draining") + elif not self._request_slots.acquire(blocking=False): + rejection = ("server_busy", "API request capacity is exhausted") + else: + self._active_requests.add(id(request)) + rejection = None + + if rejection is not None: + self._reject_unavailable_request(request, *rejection) + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._finish_request(request) + raise + + def process_request_thread(self, request: Any, client_address: Any) -> None: + try: + request.settimeout(self.api_request_timeout_seconds) + super().process_request_thread(request, client_address) + finally: + self._finish_request(request) + + def begin_draining(self) -> None: + with self._request_state: + self._draining = True + self._drain_started.set() + + def request_graceful_shutdown(self, timeout_seconds: float) -> None: + self._graceful_shutdown_timeout_seconds = timeout_seconds + self._graceful_shutdown_requested = True + + def wait_until_draining(self, timeout_seconds: float) -> bool: + return self._drain_started.wait(timeout_seconds) + + def service_actions(self) -> None: + super().service_actions() + if not self._graceful_shutdown_requested or self._graceful_shutdown_thread is not None: + return + self.begin_draining() + coordinator = threading.Thread(target=self._complete_graceful_shutdown, daemon=True) + coordinator.start() + self._graceful_shutdown_thread = coordinator + + def finish_graceful_shutdown(self) -> bool | None: + coordinator = self._graceful_shutdown_thread + if coordinator is None: + return None + coordinator.join() + return self._graceful_shutdown_result + + def wait_for_drain(self, timeout_seconds: float) -> bool: + deadline = time.monotonic() + timeout_seconds + with self._request_state: + while self._active_requests: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._request_state.wait(remaining) + return True + + def _finish_request(self, request: Any) -> None: + with self._request_state: + request_id = id(request) + if request_id not in self._active_requests: + return + self._active_requests.remove(request_id) + self._request_slots.release() + if not self._active_requests: + self._request_state.notify_all() + + def _complete_graceful_shutdown(self) -> None: + try: + self._graceful_shutdown_result = self.wait_for_drain( + self._graceful_shutdown_timeout_seconds + ) + finally: + self.shutdown() + + def _reject_unavailable_request(self, request: Any, code: str, message: str) -> None: + request_id = new_request_id() + body = json.dumps( + { + "error": { + "code": code, + "message": message, + "status": HTTPStatus.SERVICE_UNAVAILABLE.value, + } + }, + sort_keys=True, + ).encode("utf-8") + response = ( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"connection: close\r\n" + b"content-type: application/json\r\n" + b"cache-control: no-store\r\n" + + f"x-request-id: {request_id}\r\n".encode("ascii") + + b"retry-after: 1\r\n" + + f"content-length: {len(body)}\r\n\r\n".encode("ascii") + + body + ) + with contextlib.suppress(OSError): + request.sendall(response) + + +def serve( + host: str, + port: int, + settings: Settings, + *, + handler_factory: HandlerFactory, + server_factory: ServerFactory, +) -> None: + runtime_settings = replace(settings, host=host, port=port) + if not 0 <= port <= 65535: + raise ValueError("port must be between 0 and 65535") + validate_api_exposure( + runtime_settings.host, + runtime_settings.api_key, + runtime_settings.allow_unauth_reads, + ) + runtime_settings.ensure_dirs() + pid = os.getpid() + pid_path = runtime_settings.state_dir / "zeus.pid" + api_lock = BotProcessLock( + runtime_settings.state_dir / "locks" / "api.lock", + timeout_seconds=min(runtime_settings.lock_timeout_seconds, 1.0), + ) + with api_lock: + handler = handler_factory(runtime_settings) + server = server_factory((host, port), handler) + previous_signal_handlers: dict[signal.Signals, Any] = {} + try: + if threading.current_thread() is threading.main_thread(): + for signum in (signal.SIGTERM, signal.SIGINT): + previous_signal_handlers[signum] = signal.getsignal(signum) + signal.signal( + signum, + lambda _signum, _frame: server.request_graceful_shutdown( + runtime_settings.api_shutdown_drain_seconds + ), + ) + with contextlib.suppress(KeyboardInterrupt): + _write_api_pid(pid_path, pid) + server.serve_forever() + finally: + for signum, previous_handler in previous_signal_handlers.items(): + signal.signal(signum, previous_handler) + finish_graceful_shutdown = getattr(server, "finish_graceful_shutdown", None) + drain_result = ( + finish_graceful_shutdown() if callable(finish_graceful_shutdown) else None + ) + if drain_result is None: + begin_draining = getattr(server, "begin_draining", None) + if callable(begin_draining): + begin_draining() + server.server_close() + wait_for_drain = getattr(server, "wait_for_drain", None) + drain_result = ( + wait_for_drain(runtime_settings.api_shutdown_drain_seconds) + if callable(wait_for_drain) + else True + ) + else: + server.server_close() + if not drain_result: + print("warning: API shutdown drain deadline expired", file=sys.stderr) + _remove_api_pid_if_owned(pid_path, pid) + + +def _write_api_pid(path: Path, pid: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + tmp = path.with_name(f".{path.name}.{pid}.{time.time_ns()}.tmp") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(f"{pid}\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + with contextlib.suppress(OSError): + path.chmod(0o600) + finally: + with contextlib.suppress(FileNotFoundError): + tmp.unlink() + + +def _remove_api_pid_if_owned(path: Path, pid: int) -> None: + try: + recorded_pid = int(path.read_text(encoding="utf-8").strip()) + except (FileNotFoundError, OSError, ValueError): + return + if recorded_pid != pid: + return + with contextlib.suppress(FileNotFoundError): + path.unlink() diff --git a/zeus/bot_lifecycle_store.py b/zeus/bot_lifecycle_store.py new file mode 100644 index 0000000..ef77106 --- /dev/null +++ b/zeus/bot_lifecycle_store.py @@ -0,0 +1,1174 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal + +from zeus.lifecycle import ( + LifecycleEvent, + LifecycleEventInput, + deserialize_lifecycle_details, + serialize_lifecycle_details, +) +from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy +from zeus.private_io import append_private_bytes, nofollow_absolute_path +from zeus.sanitization import MAX_SANITIZED_JSON_BYTES, sanitize_details, sanitize_text +from zeus.sqlite_db import SQLiteDatabase + +LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") +LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) +LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) + + +def _sanitize_optional_persisted_text(value: str | None) -> str | None: + if value is None: + return None + return sanitize_text(value) + + +class BotLifecycleStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def begin_lifecycle_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + source: str, + request_id: str | None = None, + reason: str = "", + ) -> BotRecord: + self._validate_intent_action(action) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent", + outcome="pending", + request_id=request_id, + reason=reason, + details={"action": action, "desired_state": desired_state.value}, + ) + self._validate_event_target(bot_id, event) + now = datetime.now(UTC) + now_text = now.isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + if prior_row["pending_operation_id"] is not None: + raise RuntimeError("bot already has a pending lifecycle intent") + desired_revision = int(prior_row["desired_revision"]) + 1 + cursor = conn.execute( + """ + UPDATE bots + SET desired_state = ?, + desired_revision = ?, + desired_updated_at = ?, + pending_operation_id = ?, + pending_action = ?, + pending_since = ?, + updated_at = ? + WHERE bot_id = ? AND pending_operation_id IS NULL + """, + ( + desired_state.value, + desired_revision, + now_text, + operation_id, + action, + now_text, + now_text, + bot_id, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("lifecycle intent could not be started") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent", + outcome="pending", + request_id=request_id, + reason=reason, + details={ + "action": action, + "desired_state": desired_state.value, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=str(prior_row["status"]), + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def complete_lifecycle_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + desired_revision: int, + status: BotStatus, + pid: int | None, + source: str, + outcome: Literal["success", "failure"] = "success", + request_id: str | None = None, + reason: str = "", + error_code: str | None = None, + error_message: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> BotRecord: + self._validate_intent_action(action) + self._validate_intent_revision(desired_revision) + self._validate_intent_correlation(bot_id, operation_id, source, request_id) + self._validate_intent_completion( + action, + outcome=outcome, + status=status, + pid=pid, + error_code=error_code, + error_message=error_message, + ) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._require_matching_intent( + prior_row, + action, + operation_id, + desired_revision, + ) + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + cursor = conn.execute( + """ + UPDATE bots + SET pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL + WHERE bot_id = ? + AND pending_operation_id = ? + AND pending_action = ? + AND desired_revision = ? + AND desired_state = ? + """, + (bot_id, operation_id, action, desired_revision, desired_state.value), + ) + if cursor.rowcount != 1: + raise RuntimeError("pending lifecycle intent does not match") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.complete", + outcome=outcome, + request_id=request_id, + reason=reason, + error_code=error_code, + error_message=error_message, + details={ + "action": action, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def clear_stale_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + desired_revision: int, + source: str, + reason: str, + request_id: str | None = None, + ) -> BotRecord: + self._validate_intent_action(action) + self._validate_intent_revision(desired_revision) + self._validate_intent_correlation(bot_id, operation_id, source, request_id) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._require_matching_intent( + prior_row, + action, + operation_id, + desired_revision, + ) + cursor = conn.execute( + """ + UPDATE bots + SET pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL, + updated_at = ? + WHERE bot_id = ? + AND pending_operation_id = ? + AND pending_action = ? + AND desired_revision = ? + AND desired_state = ? + """, + ( + now, + bot_id, + operation_id, + action, + desired_revision, + desired_state.value, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("pending lifecycle intent does not match") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent.clear", + outcome="cleared", + request_id=request_id, + reason=reason, + details={ + "action": action, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=str(prior_row["status"]), + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def _validate_intent_action(self, action: str) -> None: + if type(action) is not str or action not in LIFECYCLE_INTENT_ACTIONS: + raise ValueError("invalid lifecycle intent action") + + def _validate_intent_completion( + self, + action: str, + *, + outcome: Literal["success", "failure"], + status: BotStatus, + pid: int | None, + error_code: str | None, + error_message: str | None, + ) -> None: + if outcome not in {"success", "failure"}: + raise ValueError("invalid lifecycle intent outcome") + if not isinstance(status, BotStatus): + raise TypeError("lifecycle intent status must be a BotStatus") + if pid is not None and (type(pid) is not int or pid <= 0): + raise ValueError("lifecycle intent terminal state has an invalid PID") + if error_code is not None and ( + type(error_code) is not str or LIFECYCLE_ERROR_CODE_RE.fullmatch(error_code) is None + ): + raise ValueError("invalid lifecycle intent error code") + if error_message is not None and type(error_message) is not str: + raise TypeError("lifecycle intent error message must be a string") + if outcome == "success": + if error_code is not None or error_message is not None: + raise ValueError("successful lifecycle intent cannot include error metadata") + if action == "stop": + compatible = status is BotStatus.stopped and pid is None + else: + compatible = status in {BotStatus.starting, BotStatus.running} and pid is not None + else: + compatible = status in {BotStatus.failed, BotStatus.unknown, BotStatus.stopped} + if status in {BotStatus.failed, BotStatus.stopped} and pid is not None: + compatible = False + if not compatible: + raise ValueError("lifecycle intent terminal state is incompatible with action outcome") + + def _validate_intent_revision(self, desired_revision: int) -> None: + if type(desired_revision) is not int or desired_revision < 1: + raise ValueError("desired revision must be a positive integer") + + def _validate_intent_correlation( + self, + bot_id: str, + operation_id: str, + source: str, + request_id: str | None, + ) -> None: + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action="lifecycle.intent.validate", + outcome="validation", + request_id=request_id, + ) + self._validate_event_target(bot_id, event) + + def _require_matching_intent( + self, + row: sqlite3.Row, + action: str, + operation_id: str, + desired_revision: int, + ) -> None: + desired_state = ( + DesiredState.stopped.value if action == "stop" else DesiredState.running.value + ) + if ( + row["pending_operation_id"] != operation_id + or int(row["desired_revision"]) != desired_revision + or row["pending_action"] != action + or row["desired_state"] != desired_state + ): + raise RuntimeError("pending lifecycle intent does not match") + + def _record_in_transaction(self, conn: sqlite3.Connection, bot_id: str) -> BotRecord: + row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() + if row is None: + raise sqlite3.IntegrityError("updated bot projection is missing") + return self._row_to_record(row) + + def audit_log_path(self) -> Path: + return self._database.database_path.parent / "logs" / "audit.jsonl" + + def append_audit_event(self, event: str, **fields: object) -> None: + try: + safe_fields = sanitize_details(fields) + if type(safe_fields) is not dict: + return + timestamp = datetime.now(UTC).isoformat() + safe_event = sanitize_text(event) + payload = { + "ts": timestamp, + "event": safe_event, + **safe_fields, + } + line = (json.dumps(payload, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + line = ( + json.dumps( + { + "ts": timestamp, + "event": "audit.truncated", + "truncated": True, + }, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + return + append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) + except Exception: + return + + def upsert_bot(self, record: BotRecord) -> None: + with closing(self._database.connect()) as conn: + self._upsert_bot_row(conn, record) + conn.commit() + + def _upsert_bot_row(self, conn: sqlite3.Connection, record: BotRecord) -> None: + conn.execute( + """ + INSERT INTO bots ( + bot_id, + template_id, + display_name, + profile_path, + status, + pid, + restart_policy, + restart_backoff_seconds, + restart_max_attempts, + restart_attempts, + next_restart_at, + started_at, + ready_at, + stopped_at, + last_exit_code, + last_error, + last_transition_reason, + desired_state, + desired_revision, + desired_updated_at, + pending_operation_id, + pending_action, + pending_since, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(bot_id) DO UPDATE SET + template_id = excluded.template_id, + display_name = excluded.display_name, + profile_path = excluded.profile_path, + status = excluded.status, + pid = excluded.pid, + restart_policy = excluded.restart_policy, + restart_backoff_seconds = excluded.restart_backoff_seconds, + restart_max_attempts = excluded.restart_max_attempts, + restart_attempts = excluded.restart_attempts, + next_restart_at = excluded.next_restart_at, + started_at = excluded.started_at, + ready_at = excluded.ready_at, + stopped_at = excluded.stopped_at, + last_exit_code = excluded.last_exit_code, + last_error = excluded.last_error, + last_transition_reason = excluded.last_transition_reason, + desired_state = excluded.desired_state, + desired_revision = excluded.desired_revision, + desired_updated_at = excluded.desired_updated_at, + pending_operation_id = excluded.pending_operation_id, + pending_action = excluded.pending_action, + pending_since = excluded.pending_since, + updated_at = excluded.updated_at + """, + ( + record.bot_id, + record.template_id, + record.display_name, + record.profile_path, + record.status.value, + record.pid, + record.restart_policy.value, + record.restart_backoff_seconds, + record.restart_max_attempts, + record.restart_attempts, + record.next_restart_at.isoformat() if record.next_restart_at else None, + record.started_at.isoformat() if record.started_at else None, + record.ready_at.isoformat() if record.ready_at else None, + record.stopped_at.isoformat() if record.stopped_at else None, + record.last_exit_code, + _sanitize_optional_persisted_text(record.last_error), + _sanitize_optional_persisted_text(record.last_transition_reason), + record.desired_state.value, + record.desired_revision, + record.desired_updated_at.isoformat() if record.desired_updated_at else None, + record.pending_operation_id, + record.pending_action, + record.pending_since.isoformat() if record.pending_since else None, + record.created_at.isoformat(), + record.updated_at.isoformat(), + ), + ) + + def upsert_bot_with_event( + self, + record: BotRecord, + *, + event: LifecycleEventInput, + ) -> LifecycleEvent: + self._validate_event_target(record.bot_id, event) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (record.bot_id,) + ).fetchone() + self._upsert_bot_row(conn, record) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]) if prior_row else None, + status_after=record.status.value, + pid_before=int(prior_row["pid"]) if prior_row and prior_row["pid"] else None, + pid_after=record.pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, record.bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def get_bot(self, bot_id: str) -> BotRecord | None: + with closing(self._database.connect()) as conn: + row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() + return self._row_to_record(row) if row else None + + def list_bots(self) -> list[BotRecord]: + with closing(self._database.connect()) as conn: + rows = conn.execute("SELECT * FROM bots ORDER BY bot_id").fetchall() + return [self._row_to_record(row) for row in rows] + + def update_status( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + reset_restart: bool = False, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + if reset_restart: + conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + restart_attempts = 0, + next_restart_at = NULL, + updated_at = ? + WHERE bot_id = ? + """, + (status.value, pid, now, bot_id), + ) + else: + conn.execute( + "UPDATE bots SET status = ?, pid = ?, updated_at = ? WHERE bot_id = ?", + (status.value, pid, now, bot_id), + ) + conn.commit() + + def update_lifecycle_state( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + conn.commit() + + def _update_lifecycle_row( + self, + conn: sqlite3.Connection, + bot_id: str, + status: BotStatus, + pid: int | None, + *, + started_at: datetime | None, + ready_at: datetime | None, + stopped_at: datetime | None, + last_exit_code: int | None, + last_error: str | None, + last_transition_reason: str | None, + reset_restart: bool, + clear_ready_at: bool, + clear_stopped_at: bool, + now: str, + ) -> None: + cursor = conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + started_at = COALESCE(?, started_at), + ready_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, ready_at) END, + stopped_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, stopped_at) END, + last_exit_code = ?, + last_error = ?, + last_transition_reason = COALESCE(?, last_transition_reason), + updated_at = ?, + restart_attempts = CASE WHEN ? THEN 0 ELSE restart_attempts END, + next_restart_at = CASE WHEN ? THEN NULL ELSE next_restart_at END + WHERE bot_id = ? + """, + ( + status.value, + pid, + started_at.isoformat() if started_at else None, + int(clear_ready_at), + ready_at.isoformat() if ready_at else None, + int(clear_stopped_at), + stopped_at.isoformat() if stopped_at else None, + last_exit_code, + _sanitize_optional_persisted_text(last_error), + _sanitize_optional_persisted_text(last_transition_reason), + now, + int(reset_restart), + int(reset_restart), + bot_id, + ), + ) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + + def update_lifecycle_with_event( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + event: LifecycleEventInput, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> LifecycleEvent: + self._validate_event_target(bot_id, event) + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def update_restart_state( + self, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + self._update_restart_row( + conn, + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + now=now, + ) + conn.commit() + + def _update_restart_row( + self, + conn: sqlite3.Connection, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + now: str, + ) -> None: + cursor = conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + restart_attempts = ?, + next_restart_at = ?, + updated_at = ? + WHERE bot_id = ? + """, + ( + status.value, + pid, + restart_attempts, + next_restart_at.isoformat() if next_restart_at else None, + now, + bot_id, + ), + ) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + + def update_restart_with_event( + self, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + event: LifecycleEventInput, + ) -> LifecycleEvent: + self._validate_event_target(bot_id, event) + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._update_restart_row( + conn, + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + now=now, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def delete_bot(self, bot_id: str) -> bool: + with closing(self._database.connect()) as conn: + cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) + conn.commit() + return cursor.rowcount > 0 + + def delete_bot_with_event( + self, + bot_id: str, + *, + event: LifecycleEventInput, + ) -> bool: + self._validate_event_target(bot_id, event) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=None, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=None, + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return True + + def _validate_event_target(self, bot_id: str, event: LifecycleEventInput) -> None: + if event.bot_id != bot_id: + raise ValueError("lifecycle event bot_id must match the projection target") + if LIFECYCLE_ID_RE.fullmatch(event.operation_id) is None: + raise ValueError("lifecycle operation_id must be generated UUID hex") + if event.source not in LIFECYCLE_SOURCES: + raise ValueError("invalid lifecycle event source") + if event.source == "api": + if event.request_id is None or LIFECYCLE_ID_RE.fullmatch(event.request_id) is None: + raise ValueError("API lifecycle events require a generated request ID") + elif event.request_id is not None: + raise ValueError("only API lifecycle events may carry a request ID") + + def _insert_lifecycle_event( + self, + conn: sqlite3.Connection, + event: LifecycleEventInput, + *, + status_before: str | None, + status_after: str | None, + pid_before: int | None, + pid_after: int | None, + ) -> int: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, + operation_id, + request_id, + occurred_at, + source, + action, + outcome, + status_before, + status_after, + pid_before, + pid_after, + reason, + error_code, + error_message, + details_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + event.bot_id, + event.operation_id, + event.request_id, + datetime.now(UTC).isoformat(), + event.source, + event.action, + event.outcome, + status_before, + status_after, + pid_before, + pid_after, + event.reason, + event.error_code, + event.error_message, + serialize_lifecycle_details(event.details), + ), + ) + if cursor.lastrowid is None: + raise sqlite3.IntegrityError("lifecycle event insert returned no event id") + return int(cursor.lastrowid) + + def _materialize_lifecycle_event( + self, + conn: sqlite3.Connection, + event_id: int, + ) -> LifecycleEvent: + row = conn.execute( + "SELECT * FROM lifecycle_events WHERE event_id = ?", (event_id,) + ).fetchone() + if row is None: + raise sqlite3.IntegrityError("inserted lifecycle event is missing") + return self._row_to_lifecycle_event(row) + + def _append_lifecycle_audit_fail_open(self, event: LifecycleEvent) -> None: + try: + self._append_lifecycle_audit(event) + except Exception: + return + + def _append_lifecycle_audit(self, event: LifecycleEvent) -> None: + self.append_audit_event( + event.action, + bot_id=event.bot_id, + operation_id=event.operation_id, + request_id=event.request_id, + source=event.source, + outcome=event.outcome, + status_before=event.status_before, + status_after=event.status_after, + pid_before=event.pid_before, + pid_after=event.pid_after, + reason=event.reason, + error_code=event.error_code, + error_message=event.error_message, + details=event.details, + ) + + def list_lifecycle_events( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> list[LifecycleEvent]: + self._validate_history_page(bot_id, limit, before) + with closing(self._database.connect()) as conn: + rows = self._list_lifecycle_event_rows(conn, bot_id, limit, before) + return [self._row_to_lifecycle_event(row) for row in rows] + + def history_payload( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> dict[str, object]: + self._validate_history_page(bot_id, limit, before) + with closing(self._database.connect()) as conn: + rows = self._list_lifecycle_event_rows(conn, bot_id, limit + 1, before) + if not rows: + bot_is_known = ( + conn.execute( + """ + SELECT 1 FROM bots WHERE bot_id = ? + UNION ALL + SELECT 1 FROM lifecycle_events WHERE bot_id = ? + LIMIT 1 + """, + (bot_id, bot_id), + ).fetchone() + is not None + ) + if not bot_is_known: + raise KeyError(f"unknown bot: {bot_id}") + has_more = len(rows) > limit + events = [self._row_to_lifecycle_event(row) for row in rows[:limit]] + return { + "bot_id": bot_id, + "events": [event.to_dict() for event in events], + "next_before": events[-1].event_id if has_more else None, + } + + def _validate_history_page( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> None: + if not isinstance(bot_id, str): + raise TypeError("bot_id must be a string") + if isinstance(limit, bool) or not isinstance(limit, int): + raise TypeError("limit must be an integer") + if limit < 1 or limit > 1000: + raise ValueError("limit must be between 1 and 1000") + if before is not None: + if isinstance(before, bool) or not isinstance(before, int): + raise TypeError("before must be an integer or null") + if before < 1: + raise ValueError("before must be positive") + + def _list_lifecycle_event_rows( + self, + conn: sqlite3.Connection, + bot_id: str, + limit: int, + before: int | None, + ) -> list[sqlite3.Row]: + if before is None: + return conn.execute( + """ + SELECT * FROM lifecycle_events + WHERE bot_id = ? + ORDER BY event_id DESC + LIMIT ? + """, + (bot_id, limit), + ).fetchall() + return conn.execute( + """ + SELECT * FROM lifecycle_events + WHERE bot_id = ? AND event_id < ? + ORDER BY event_id DESC + LIMIT ? + """, + (bot_id, before, limit), + ).fetchall() + + def _row_to_lifecycle_event(self, row: sqlite3.Row) -> LifecycleEvent: + return LifecycleEvent( + event_id=int(row["event_id"]), + bot_id=str(row["bot_id"]), + operation_id=str(row["operation_id"]), + request_id=str(row["request_id"]) if row["request_id"] is not None else None, + occurred_at=datetime.fromisoformat(str(row["occurred_at"])), + source=str(row["source"]), + action=str(row["action"]), + outcome=str(row["outcome"]), + status_before=(str(row["status_before"]) if row["status_before"] is not None else None), + status_after=(str(row["status_after"]) if row["status_after"] is not None else None), + pid_before=int(row["pid_before"]) if row["pid_before"] is not None else None, + pid_after=int(row["pid_after"]) if row["pid_after"] is not None else None, + reason=str(row["reason"]), + error_code=str(row["error_code"]) if row["error_code"] is not None else None, + error_message=(str(row["error_message"]) if row["error_message"] is not None else None), + details=deserialize_lifecycle_details(str(row["details_json"])), + ) + + def _row_to_record(self, row: sqlite3.Row) -> BotRecord: + next_restart_at = row["next_restart_at"] + started_at = row["started_at"] + ready_at = row["ready_at"] + stopped_at = row["stopped_at"] + desired_updated_at = row["desired_updated_at"] + pending_since = row["pending_since"] + return BotRecord( + bot_id=row["bot_id"], + template_id=row["template_id"], + display_name=row["display_name"], + profile_path=row["profile_path"], + status=BotStatus(row["status"]), + pid=row["pid"], + restart_policy=RestartPolicy(row["restart_policy"]), + restart_backoff_seconds=float(row["restart_backoff_seconds"]), + restart_max_attempts=int(row["restart_max_attempts"]), + restart_attempts=int(row["restart_attempts"]), + next_restart_at=datetime.fromisoformat(next_restart_at) if next_restart_at else None, + started_at=datetime.fromisoformat(started_at) if started_at else None, + ready_at=datetime.fromisoformat(ready_at) if ready_at else None, + stopped_at=datetime.fromisoformat(stopped_at) if stopped_at else None, + last_exit_code=row["last_exit_code"], + last_error=row["last_error"], + last_transition_reason=row["last_transition_reason"], + desired_state=DesiredState(row["desired_state"]), + desired_revision=int(row["desired_revision"]), + desired_updated_at=( + datetime.fromisoformat(desired_updated_at) if desired_updated_at else None + ), + pending_operation_id=row["pending_operation_id"], + pending_action=row["pending_action"], + pending_since=datetime.fromisoformat(pending_since) if pending_since else None, + created_at=datetime.fromisoformat(row["created_at"]), + updated_at=datetime.fromisoformat(row["updated_at"]), + ) diff --git a/zeus/bundled_templates/coding-bot.toml b/zeus/bundled_templates/coding-bot.toml index a7777ca..f4cdc25 100644 --- a/zeus/bundled_templates/coding-bot.toml +++ b/zeus/bundled_templates/coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/deepseek-coding-bot.toml b/zeus/bundled_templates/deepseek-coding-bot.toml index c4fb3e7..6e66599 100644 --- a/zeus/bundled_templates/deepseek-coding-bot.toml +++ b/zeus/bundled_templates/deepseek-coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/docs-writer-bot.toml b/zeus/bundled_templates/docs-writer-bot.toml index d1a1e64..72199ff 100644 --- a/zeus/bundled_templates/docs-writer-bot.toml +++ b/zeus/bundled_templates/docs-writer-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/gateway-operator.toml b/zeus/bundled_templates/gateway-operator.toml index c9da28c..3256701 100644 --- a/zeus/bundled_templates/gateway-operator.toml +++ b/zeus/bundled_templates/gateway-operator.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/log-triage-bot.toml b/zeus/bundled_templates/log-triage-bot.toml index c8e5ab9..b97a2ee 100644 --- a/zeus/bundled_templates/log-triage-bot.toml +++ b/zeus/bundled_templates/log-triage-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/research-bot.toml b/zeus/bundled_templates/research-bot.toml index 0a41f2c..f5fd247 100644 --- a/zeus/bundled_templates/research-bot.toml +++ b/zeus/bundled_templates/research-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/support-gateway.toml b/zeus/bundled_templates/support-gateway.toml index 91b618f..52a682a 100644 --- a/zeus/bundled_templates/support-gateway.toml +++ b/zeus/bundled_templates/support-gateway.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 180 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/cli.py b/zeus/cli.py index 100312a..829b2b3 100644 --- a/zeus/cli.py +++ b/zeus/cli.py @@ -2,16 +2,19 @@ import argparse import json +import os import shutil import sys import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import cast +from zeus import __version__ from zeus.api import serve, template_to_dict -from zeus.config import Settings +from zeus.config import Settings, load_dotenv from zeus.doctor import report_to_json, report_to_text, run_doctor +from zeus.envfile import ENV_KEY_RE from zeus.errors import ZeusConflictError from zeus.models import BotCreateRequest, BotStatus, BotStatusResponse, RestartPolicy, TemplateError from zeus.process_lock import LockTimeoutError @@ -26,95 +29,337 @@ def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="zeus") + parser = argparse.ArgumentParser( + prog="zeus", + description="Manage local Hermes bot gateways and the Zeus API.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {__version__}", + ) sub = parser.add_subparsers(dest="resource", required=True) - serve_cmd = sub.add_parser("serve") - serve_cmd.add_argument("--host") - serve_cmd.add_argument("--port", type=int) - - doctor = sub.add_parser("doctor") - doctor.add_argument("--json", action="store_true", dest="as_json") - doctor.add_argument("--strict", action="store_true") + serve_description = "Run the local Zeus HTTP API server." + serve_cmd = sub.add_parser( + "serve", + help=serve_description, + description=serve_description, + ) + serve_cmd.add_argument("--host", help="bind host (default: ZEUS_HOST or 127.0.0.1)") + serve_cmd.add_argument("--port", type=int, help="listen port (default: ZEUS_PORT or 4311)") + + doctor_description = "Check Zeus configuration, runtime paths, and dependencies." + doctor = sub.add_parser( + "doctor", + help=doctor_description, + description=doctor_description, + ) + doctor.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + doctor.add_argument( + "--strict", + action="store_true", + help="treat warnings as failures", + ) - template = sub.add_parser("template") + template_description = "Inspect available Hermes bot templates." + template = sub.add_parser( + "template", + help=template_description, + description=template_description, + ) template_sub = template.add_subparsers(dest="action", required=True) - template_list = template_sub.add_parser("list") - template_list.add_argument("--json", action="store_true", dest="as_json") + template_list_description = "List available bot templates." + template_list = template_sub.add_parser( + "list", + help=template_list_description, + description=template_list_description, + ) + template_list.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) - demo = sub.add_parser("demo") + demo_description = "Run the bundled fake-Hermes lifecycle demo." + demo = sub.add_parser( + "demo", + help=demo_description, + description=demo_description, + ) demo_sub = demo.add_subparsers(dest="action", required=True) - for action in ["up", "status", "down"]: - command = demo_sub.add_parser(action) - command.add_argument("--bot-id", default=DEMO_BOT_ID) - command.add_argument("--json", action="store_true", dest="as_json") + demo_actions = { + "up": "Create and start the demo bot.", + "status": "Show the demo bot status.", + "down": "Stop the demo bot.", + } + for action, description in demo_actions.items(): + command = demo_sub.add_parser(action, help=description, description=description) + command.add_argument( + "--bot-id", + default=DEMO_BOT_ID, + help=f"demo bot ID (default: {DEMO_BOT_ID})", + ) + command.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) - bot = sub.add_parser("bot") + bot_description = "Create, inspect, and manage Hermes bot gateways." + bot = sub.add_parser( + "bot", + help=bot_description, + description=bot_description, + ) bot_sub = bot.add_subparsers(dest="action", required=True) - create = bot_sub.add_parser("create") - create.add_argument("bot_id") - create.add_argument("--template", required=True, dest="template_id") - create.add_argument("--name", dest="display_name") + create_description = "Create a bot profile from a template." + create = bot_sub.add_parser( + "create", + help=create_description, + description=create_description, + ) + create.add_argument("bot_id", help="unique bot ID") + create.add_argument( + "--template", + required=True, + dest="template_id", + help="template ID used to render the bot profile", + ) + create.add_argument("--name", dest="display_name", help="display name (defaults to bot ID)") create.add_argument( "--env", action="append", default=[], - help="NAME=VALUE for env keys declared by the selected template", - ) - create.add_argument("--restart-policy", choices=["manual", "on-failure"], default="manual") - create.add_argument("--restart-backoff-seconds", type=float, default=5.0) - create.add_argument("--restart-max-attempts", type=int, default=5) - create.add_argument("--replace", action="store_true", dest="replace_existing") - create.add_argument("--stop", action="store_true", dest="stop_if_running") - create.add_argument("--json", action="store_true", dest="as_json") - - bot_list = bot_sub.add_parser("list") - bot_list.add_argument("--json", action="store_true", dest="as_json") - delete = bot_sub.add_parser("delete") - delete.add_argument("bot_id") - delete.add_argument("--stop", action="store_true", dest="stop_if_running") - delete.add_argument("--remove-profile", action="store_true") - delete.add_argument("--json", action="store_true", dest="as_json") - archive = bot_sub.add_parser("archive") - archive.add_argument("bot_id") - archive.add_argument("--stop", action="store_true", dest="stop_if_running") - archive.add_argument("--json", action="store_true", dest="as_json") - reconcile = bot_sub.add_parser("reconcile") - reconcile.add_argument("bot_id", nargs="?") - reconcile.add_argument("--json", action="store_true", dest="as_json") - reconcile.add_argument("--summary", action="store_true") - reconcile.add_argument("--force", action="store_true") - reconcile.add_argument("--reset-restart", action="store_true") - inspect = bot_sub.add_parser("inspect") - inspect.add_argument("bot_id") - inspect.add_argument("--json", action="store_true", dest="as_json") - history = bot_sub.add_parser("history") - history.add_argument("bot_id") - history.add_argument("--limit", type=_history_limit, default=50) - history.add_argument("--before", type=_positive_event_id) - history.add_argument("--json", action="store_true", dest="as_json") - for action in ["start", "stop", "restart", "status", "logs", "doctor"]: - command = bot_sub.add_parser(action) - command.add_argument("bot_id") + help=( + "NAME=VALUE for env keys declared by the selected template; " + "unsafe for secrets because values enter argv" + ), + ) + create.add_argument( + "--env-from", + action="append", + default=[], + metavar="NAME", + help="import NAME from the process environment, then trusted ./.env, without argv values", + ) + create.add_argument( + "--restart-policy", + choices=["manual", "on-failure"], + default="manual", + help="automatic restart policy (default: manual)", + ) + create.add_argument( + "--restart-backoff-seconds", + type=float, + default=5.0, + help="initial on-failure restart backoff in seconds (default: 5)", + ) + create.add_argument( + "--restart-max-attempts", + type=int, + default=5, + help="maximum consecutive restart attempts (default: 5)", + ) + create.add_argument( + "--replace", + action="store_true", + dest="replace_existing", + help="replace an existing bot (requires --stop if it is running)", + ) + create.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop a running bot before replacement", + ) + create.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + + list_description = "List registered bots." + bot_list = bot_sub.add_parser( + "list", + help=list_description, + description=list_description, + ) + bot_list.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + delete_description = "Delete a bot registration." + delete = bot_sub.add_parser( + "delete", + help=delete_description, + description=delete_description, + ) + delete.add_argument("bot_id", help="bot ID") + delete.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop the bot before deletion", + ) + delete.add_argument( + "--remove-profile", + action="store_true", + help="also remove the managed profile directory", + ) + delete.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + archive_description = "Archive a bot profile." + archive = bot_sub.add_parser( + "archive", + help=archive_description, + description=archive_description, + ) + archive.add_argument("bot_id", help="bot ID") + archive.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop the bot before archiving", + ) + archive.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + reconcile_description = "Reconcile desired and observed bot state." + reconcile = bot_sub.add_parser( + "reconcile", + help=reconcile_description, + description=reconcile_description, + ) + reconcile.add_argument("bot_id", nargs="?", help="bot ID; omit to reconcile all bots") + reconcile.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + reconcile.add_argument( + "--summary", + action="store_true", + help="emit persisted run metadata and per-bot results", + ) + reconcile.add_argument( + "--force", + action="store_true", + help="run an eligible restart now instead of waiting for backoff", + ) + reconcile.add_argument( + "--reset-restart", + action="store_true", + help="reset restart backoff and retry budget before reconciling", + ) + inspect_description = "Show bot diagnostics." + inspect = bot_sub.add_parser( + "inspect", + help=inspect_description, + description=inspect_description, + ) + inspect.add_argument("bot_id", help="bot ID") + inspect.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + history_description = "Show immutable lifecycle history." + history = bot_sub.add_parser( + "history", + help=history_description, + description=history_description, + ) + history.add_argument("bot_id", help="bot ID") + history.add_argument( + "--limit", + type=_history_limit, + default=50, + help="maximum events to return, 1-1000 (default: 50)", + ) + history.add_argument( + "--before", + type=_positive_event_id, + help="return events before this exclusive event ID", + ) + history.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + lifecycle_descriptions = { + "start": "Start a bot gateway.", + "stop": "Stop a bot gateway gracefully.", + "restart": "Restart a bot gateway.", + "status": "Show current bot status.", + "logs": "Show redacted bot logs.", + "doctor": "Run Hermes doctor for a bot profile.", + } + for action, description in lifecycle_descriptions.items(): + command = bot_sub.add_parser(action, help=description, description=description) + command.add_argument("bot_id", help="bot ID") if action in {"start", "restart"}: - command.add_argument("--wait", dest="wait", action="store_true", default=False) - command.add_argument("--no-wait", dest="wait", action="store_false") - command.add_argument("--timeout", type=float, dest="timeout_seconds") + command.add_argument( + "--wait", + dest="wait", + action="store_true", + default=False, + help="wait for readiness before returning", + ) + command.add_argument( + "--no-wait", + dest="wait", + action="store_false", + help="return after launch without waiting for readiness (default)", + ) + command.add_argument( + "--timeout", + type=float, + dest="timeout_seconds", + help="readiness timeout in seconds", + ) if action == "stop": command.add_argument( "--kill-after-timeout", dest="kill_after_timeout", action="store_true", default=None, + help="send SIGKILL if graceful shutdown times out", ) command.add_argument( "--no-kill-after-timeout", dest="kill_after_timeout", action="store_false", + help="never escalate a timed-out graceful shutdown", ) if action == "logs": - command.add_argument("--json", action="store_true", dest="as_json") + command.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) return parser @@ -141,7 +386,10 @@ def _positive_event_id(value: str) -> int: def _services(settings: Settings) -> tuple[StateStore, Supervisor]: settings.ensure_dirs() - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() return store, Supervisor( store, @@ -158,7 +406,10 @@ def _services(settings: Settings) -> tuple[StateStore, Supervisor]: def _demo_services(settings: Settings, bot_id: str) -> tuple[StateStore, Supervisor, str]: settings.ensure_dirs() fake_hermes = _demo_hermes_bin(settings) - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() return ( store, @@ -234,8 +485,44 @@ def _parse_env(pairs: list[str]) -> dict[str, str]: return values +def _resolve_create_env( + pairs: list[str], + imported_names: list[str], + *, + process_env: Mapping[str, str] | None = None, +) -> dict[str, str]: + values = _parse_env(pairs) + for name in imported_names: + if not ENV_KEY_RE.fullmatch(name): + raise ValueError("--env-from requires a valid environment variable name") + + duplicate_names = sorted(set(values).intersection(imported_names)) + if duplicate_names: + raise ValueError( + "environment variable provided by both --env and --env-from: " + + ", ".join(duplicate_names) + ) + + source_env = os.environ if process_env is None else process_env + needs_dotenv = any(name not in source_env for name in imported_names) + dotenv = load_dotenv(Path(".env")) if needs_dotenv else {} + for name in imported_names: + value = source_env[name] if name in source_env else dotenv.get(name) + if value is None or value == "": + raise ValueError(f"environment variable {name} is missing or empty") + values[name] = value + return values + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + create_env: dict[str, str] = {} + if args.resource == "bot" and args.action == "create": + try: + create_env = _resolve_create_env(args.env, args.env_from) + except ValueError as exc: + return _print_cli_error("invalid_request", str(exc), as_json=args.as_json) + try: settings = Settings.from_env() except ValueError as exc: @@ -305,7 +592,7 @@ def main(argv: list[str] | None = None) -> int: bot_id=args.bot_id, template_id=args.template_id, display_name=args.display_name, - env=_parse_env(args.env), + env=create_env, restart_policy=RestartPolicy(args.restart_policy), restart_backoff_seconds=args.restart_backoff_seconds, restart_max_attempts=args.restart_max_attempts, diff --git a/zeus/config.py b/zeus/config.py index d5c4307..a40c31b 100644 --- a/zeus/config.py +++ b/zeus/config.py @@ -3,10 +3,17 @@ import os from collections.abc import Mapping from dataclasses import dataclass +from enum import StrEnum from ipaddress import ip_address from pathlib import Path from zeus.envfile import parse_env_text +from zeus.private_io import ensure_private_directory, nofollow_absolute_path + + +class SQLiteSynchronous(StrEnum): + NORMAL = "NORMAL" + FULL = "FULL" def load_dotenv(path: Path = Path(".env")) -> dict[str, str]: @@ -40,12 +47,14 @@ class Settings: api_idempotency_retention_seconds: int api_idempotency_max_records: int api_log_enabled: bool = True + sqlite_synchronous: SQLiteSynchronous = SQLiteSynchronous.NORMAL @classmethod def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: merged: dict[str, str] = load_dotenv() merged.update(dict(os.environ if env is None else env)) - state_dir = Path(merged.get("ZEUS_STATE_DIR") or ".zeus").resolve() + sqlite_synchronous = _sqlite_synchronous_env(merged) + state_dir = nofollow_absolute_path(Path(merged.get("ZEUS_STATE_DIR") or ".zeus")) port = _port_env(merged, "ZEUS_PORT", default=4311) return cls( state_dir=state_dir, @@ -139,13 +148,17 @@ def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: maximum=1_000_000, ), api_log_enabled=merged.get("ZEUS_API_LOG_ENABLED", "1") == "1", + sqlite_synchronous=sqlite_synchronous, ) def ensure_dirs(self) -> None: - self.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) - self.hermes_root.mkdir(parents=True, exist_ok=True, mode=0o700) - (self.state_dir / "logs").mkdir(parents=True, exist_ok=True, mode=0o700) - (self.state_dir / "locks" / "bots").mkdir(parents=True, exist_ok=True, mode=0o700) + for path in ( + self.state_dir, + self.hermes_root, + self.state_dir / "logs", + self.state_dir / "locks" / "bots", + ): + ensure_private_directory(path) def is_loopback_host(host: str) -> bool: @@ -169,6 +182,15 @@ def validate_api_exposure(host: str, api_key: str | None, allow_unauth_reads: bo raise ValueError("non-loopback API bind requires ZEUS_API_KEY with at least 16 characters") +def _sqlite_synchronous_env(env: Mapping[str, str]) -> SQLiteSynchronous: + raw_value = env.get("ZEUS_SQLITE_SYNCHRONOUS") + value = SQLiteSynchronous.NORMAL.value if raw_value is None or raw_value == "" else raw_value + try: + return SQLiteSynchronous(value) + except ValueError as exc: + raise ValueError("ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL") from exc + + def _port_env(env: Mapping[str, str], name: str, *, default: int) -> int: raw_value = env.get(name) if raw_value is None or raw_value == "": diff --git a/zeus/doctor.py b/zeus/doctor.py index de2cd71..ad3d3ba 100644 --- a/zeus/doctor.py +++ b/zeus/doctor.py @@ -1,8 +1,10 @@ from __future__ import annotations import json +import os import shutil import socket +import stat import subprocess # nosec B404 import sys from dataclasses import dataclass @@ -10,6 +12,7 @@ from typing import Any from zeus.config import Settings, is_loopback_host +from zeus.private_io import inspect_private_directory, nofollow_absolute_path from zeus.state import StateStore from zeus.templates import TemplateStore @@ -100,20 +103,79 @@ def _check_templates() -> DoctorCheck: ) -def _check_runtime_paths(settings: Settings) -> DoctorCheck: - state_dir = settings.state_dir.resolve() - if state_dir.exists() and not state_dir.is_dir(): - return DoctorCheck( +def _existing_runtime_directory_check( + path: Path, + *, + label: str, +) -> tuple[bool, DoctorCheck | None]: + try: + metadata = os.lstat(path) + except FileNotFoundError: + try: + exists = inspect_private_directory(path, missing_ok=True) + except OSError: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} could not be validated safely", + ) + if exists: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} changed while it was inspected", + ) + return False, None + except OSError: + return True, DoctorCheck( "runtime_paths", "fail", - f"State path {state_dir} must be a directory", + f"{label} {path} could not be inspected safely", ) - if state_dir.exists() and state_dir.stat().st_mode & 0o007: - return DoctorCheck( + if not stat.S_ISDIR(metadata.st_mode): + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} path {path} must be a directory and not a symlink", + ) + if metadata.st_uid != os.geteuid(): + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} must be owned by the current user", + ) + if stat.S_IMODE(metadata.st_mode) & 0o077: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} must not be accessible to other users or groups", + ) + try: + inspect_private_directory(path) + except OSError: + return True, DoctorCheck( "runtime_paths", "fail", - f"State directory {state_dir} must not be accessible to other users", + f"{label} {path} could not be validated safely", ) + return True, None + + +def _check_runtime_paths(settings: Settings) -> DoctorCheck: + state_dir = nofollow_absolute_path(settings.state_dir) + state_exists, failure = _existing_runtime_directory_check( + state_dir, + label="State directory", + ) + if failure is not None: + return failure + if state_exists: + _, failure = _existing_runtime_directory_check( + state_dir / "logs", + label="State logs directory", + ) + if failure is not None: + return failure workspace = Path.cwd().resolve() gitignore = workspace / ".gitignore" if not gitignore.exists(): @@ -253,7 +315,10 @@ def _check_api_auth(settings: Settings) -> DoctorCheck: def _check_bots(settings: Settings) -> list[DoctorCheck]: - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) if not settings.database_path.exists(): return [DoctorCheck("bots", "pass", "No bot registry exists yet")] try: diff --git a/zeus/gateway_launcher.py b/zeus/gateway_launcher.py index d96feb2..6fe7c7c 100644 --- a/zeus/gateway_launcher.py +++ b/zeus/gateway_launcher.py @@ -1,22 +1,34 @@ from __future__ import annotations import contextlib -import hashlib import json import math import os import platform -import re import stat import subprocess # nosec B404 import sys import time +from dataclasses import dataclass from pathlib import Path from types import TracebackType from typing import NoReturn -from urllib.parse import urlparse -from zeus.models import ID_RE +from zeus.gateway_marker import ( + MarkerValidationError, + _finish_launch_marker, + _parse_launch_marker_argv, + _parse_launch_marker_correlation, + _parse_launch_marker_identity, +) +from zeus.gateway_marker import ( + command_fingerprint as _command_fingerprint, +) +from zeus.gateway_marker import ( + is_owned_runtime_marker as _marker_is_owned_runtime_marker, +) + +command_fingerprint = _command_fingerprint if os.name == "posix": import fcntl @@ -30,9 +42,6 @@ MARKER_PUBLICATION_LOCK_NAME = ".zeus-gateway-marker.lock" MARKER_PUBLICATION_LOCK_TIMEOUT_SECONDS = 30.0 -_OPERATION_ID_RE = re.compile(r"^[0-9a-f]{32}$") -_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{64}$") - class _UnspecifiedProcessStart: pass @@ -40,34 +49,14 @@ class _UnspecifiedProcessStart: _UNSPECIFIED_PROCESS_START = _UnspecifiedProcessStart() _ROOT_KEYS = frozenset({"profile_path", "marker_path", "marker", "argv", "env"}) -_MARKER_KEYS = frozenset( - { - "schema", - "bot_id", - "component", - "action", - "operation_id", - "desired_revision", - "argv", - "resolved_hermes_bin", - "command_fingerprint", - "readiness_probe", - } -) -_RUNTIME_MARKER_KEYS = _MARKER_KEYS | frozenset({"pid", "started_at"}) -_RUNTIME_MARKER_FINGERPRINT_KEYS = _RUNTIME_MARKER_KEYS | frozenset({"proc_start_fingerprint"}) -_PROBE_KEYS = frozenset( - {"url", "expected_status", "expected_platform", "timeout_seconds", "interval_seconds"} -) class LaunchPayloadError(ValueError): pass -def command_fingerprint(argv: list[str]) -> str: - encoded = json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() +class _ConfirmedMissing(LaunchPayloadError): + """A missing path component whose absence was rechecked on retained descriptors.""" def _exact_dict(value: object, keys: frozenset[str], name: str) -> dict[str, object]: @@ -126,77 +115,44 @@ def _validate_path(value: object, name: str) -> Path: return path -def _valid_probe_number(value: object) -> bool: - if isinstance(value, bool) or not isinstance(value, int | float): - return False - return math.isfinite(float(value)) and 0 < float(value) <= 3600 - - -def _validate_readiness_probe(value: object) -> object: - if value is None: - return None - probe = _exact_dict(value, _PROBE_KEYS, "readiness_probe") - url = _exact_string(probe["url"], "readiness URL", max_length=2048) - parsed = urlparse(url) - if ( - parsed.scheme != "http" - or parsed.hostname not in {"127.0.0.1", "localhost", "::1"} - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - ): - raise LaunchPayloadError("readiness URL must be loopback HTTP") - _exact_string(probe["expected_status"], "expected status", max_length=128) - _exact_string(probe["expected_platform"], "expected platform", max_length=128) - if not _valid_probe_number(probe["timeout_seconds"]) or not _valid_probe_number( - probe["interval_seconds"] - ): - raise LaunchPayloadError("readiness timing is invalid") - return probe - - def _validate_payload(value: object) -> tuple[Path, dict[str, object], list[str], dict[str, str]]: payload = _exact_dict(value, _ROOT_KEYS, "payload") profile_path = _validate_path(payload["profile_path"], "profile_path") marker_path = _validate_path(payload["marker_path"], "marker_path") - marker = _exact_dict(payload["marker"], _MARKER_KEYS, "marker") - bot_id = _exact_string(marker["bot_id"], "bot_id", max_length=63) - if ID_RE.fullmatch(bot_id) is None: - raise LaunchPayloadError("bot_id is invalid") + try: + marker, bot_id = _parse_launch_marker_identity(payload["marker"]) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc if profile_path.name != bot_id or profile_path.parent.name != "profiles": raise LaunchPayloadError("profile_path is outside the bot profile boundary") if marker_path != profile_path / "logs" / MARKER_NAME: raise LaunchPayloadError("marker_path is outside the bot profile boundary") - if marker["schema"] != 3 or type(marker["schema"]) is not int: - raise LaunchPayloadError("marker schema is invalid") - if marker["component"] != "gateway" or marker["action"] != "run": - raise LaunchPayloadError("marker command intent is invalid") - operation_id = _exact_string(marker["operation_id"], "operation_id", max_length=32) - if _OPERATION_ID_RE.fullmatch(operation_id) is None: - raise LaunchPayloadError("operation_id is invalid") - revision = marker["desired_revision"] - if type(revision) is not int or not 1 <= revision <= 2**63 - 1: - raise LaunchPayloadError("desired_revision is invalid") - + try: + operation_id, revision = _parse_launch_marker_correlation(marker) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc argv = _validate_argv(payload["argv"]) - marker_argv = _validate_argv(marker["argv"]) - if argv != marker_argv: + try: + marker_argv = _parse_launch_marker_argv(marker) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc + if argv != list(marker_argv): raise LaunchPayloadError("marker argv does not match exec argv") - if len(argv) != 5 or argv[1:] != ["-p", bot_id, "gateway", "run"]: - raise LaunchPayloadError("argv is not a Hermes gateway command") - resolved_hermes = _validate_path(marker["resolved_hermes_bin"], "resolved_hermes_bin") - if argv[0] != str(resolved_hermes): - raise LaunchPayloadError("exec argv does not use the resolved Hermes binary") - fingerprint = _exact_string(marker["command_fingerprint"], "command_fingerprint", max_length=64) - if _FINGERPRINT_RE.fullmatch(fingerprint) is None or fingerprint != command_fingerprint(argv): - raise LaunchPayloadError("command fingerprint is invalid") - _validate_readiness_probe(marker["readiness_probe"]) + try: + parsed_marker = _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=marker_argv, + ) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc env = _validate_env(payload["env"]) if env.get("HERMES_HOME") != str(profile_path.parent.parent): raise LaunchPayloadError("HERMES_HOME does not match the bot profile root") - return profile_path, marker, argv, env + return profile_path, parsed_marker.to_payload(), argv, env def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: @@ -239,6 +195,108 @@ def _same_file(before: os.stat_result, after: os.stat_result) -> bool: return before.st_dev == after.st_dev and before.st_ino == after.st_ino +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + +def _validate_open_directory_binding( + parent_fd: int, + name: str, + directory_fd: int, + description: str, +) -> os.stat_result: + try: + opened = os.fstat(directory_fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError(f"{description} changed while it was used") from exc + if ( + not stat.S_ISDIR(opened.st_mode) + or not stat.S_ISDIR(current.st_mode) + or not _same_file(opened, current) + ): + raise LaunchPayloadError(f"{description} changed while it was used") + return opened + + +@dataclass +class _OpenedProfile: + descriptors: list[int] + names: tuple[str, ...] + + @property + def fd(self) -> int: + if not self.descriptors: + raise LaunchPayloadError("profile descriptor chain is closed") + return self.descriptors[-1] + + def validate_bindings(self, *, require_profile_owner: bool = True) -> None: + if len(self.descriptors) != len(self.names) + 1: + raise LaunchPayloadError("profile descriptor chain is invalid") + try: + opened_root = os.fstat(self.descriptors[0]) + current_root = os.stat("/", follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError("filesystem root changed while it was used") from exc + if ( + not stat.S_ISDIR(opened_root.st_mode) + or not stat.S_ISDIR(current_root.st_mode) + or not _same_file(opened_root, current_root) + ): + raise LaunchPayloadError("filesystem root changed while it was used") + for parent_fd, name, directory_fd in zip( + self.descriptors[:-1], + self.names, + self.descriptors[1:], + strict=True, + ): + _validate_open_directory_binding( + parent_fd, + name, + directory_fd, + "profile path component", + ) + try: + profile_stat = os.fstat(self.fd) + except OSError as exc: + raise LaunchPayloadError("profile directory changed while it was used") from exc + if require_profile_owner and hasattr(os, "geteuid") and profile_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("profile directory has an unexpected owner") + + def confirm_missing(self, name: str) -> None: + for _attempt in range(2): + self.validate_bindings(require_profile_owner=False) + try: + os.stat(name, dir_fd=self.fd, follow_symlinks=False) + except FileNotFoundError: + continue + except OSError as exc: + raise LaunchPayloadError("missing profile entry cannot be confirmed") from exc + raise LaunchPayloadError("profile entry appeared while absence was confirmed") + self.validate_bindings(require_profile_owner=False) + + def detach_fd(self) -> int: + result = self.fd + ancestors = self.descriptors[:-1] + self.descriptors = [] + for descriptor in reversed(ancestors): + with contextlib.suppress(OSError): + os.close(descriptor) + return result + + def close(self) -> None: + descriptors = self.descriptors + self.descriptors = [] + for descriptor in reversed(descriptors): + with contextlib.suppress(OSError): + os.close(descriptor) + + def _open_directory_at(parent_fd: int, name: str) -> int: try: before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) @@ -250,34 +308,57 @@ def _open_directory_at(parent_fd: int, name: str) -> int: opened_fd = os.open(name, _directory_flags(), dir_fd=parent_fd) except OSError as exc: raise LaunchPayloadError("profile path component cannot be opened safely") from exc - after = os.fstat(opened_fd) - if not stat.S_ISDIR(after.st_mode) or not _same_file(before, after): - os.close(opened_fd) - raise LaunchPayloadError("profile path changed while it was opened") - return opened_fd + try: + after = os.fstat(opened_fd) + if not stat.S_ISDIR(after.st_mode) or not _same_file(before, after): + raise LaunchPayloadError("profile path changed while it was opened") + return opened_fd + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise LaunchPayloadError("profile path could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise -def _open_profile(profile_path: Path) -> int: +def _open_profile_chain(profile_path: Path) -> _OpenedProfile: if not profile_path.is_absolute() or not profile_path.parts or profile_path.parts[0] != "/": raise LaunchPayloadError("profile path must be absolute") try: - current_fd = os.open("/", _directory_flags()) + root_fd = os.open("/", _directory_flags()) except OSError as exc: raise LaunchPayloadError("filesystem root cannot be opened safely") from exc + descriptors = [root_fd] + names: list[str] = [] try: for component in profile_path.parts[1:]: - next_fd = _open_directory_at(current_fd, component) - os.close(current_fd) - current_fd = next_fd - profile_stat = os.fstat(current_fd) - if hasattr(os, "geteuid") and profile_stat.st_uid != os.geteuid(): - raise LaunchPayloadError("profile directory has an unexpected owner") - result = current_fd - current_fd = -1 - return result - finally: - if current_fd >= 0: - os.close(current_fd) + try: + next_fd = _open_directory_at(descriptors[-1], component) + except LaunchPayloadError as exc: + if _caused_by_missing_path(exc): + _OpenedProfile(descriptors, tuple(names)).confirm_missing(component) + raise _ConfirmedMissing("profile path component is missing") from exc + raise + descriptors.append(next_fd) + names.append(component) + opened = _OpenedProfile(descriptors, tuple(names)) + opened.validate_bindings() + return opened + except BaseException: + for descriptor in reversed(descriptors): + with contextlib.suppress(OSError): + os.close(descriptor) + raise + + +def _open_profile(profile_path: Path) -> int: + return _open_profile_chain(profile_path).detach_fd() class _MarkerPublicationLock: @@ -400,14 +481,25 @@ def _open_logs(profile_fd: int, *, create: bool) -> int: logs_fd = _open_directory_at(profile_fd, "logs") except OSError as exc: raise LaunchPayloadError("marker directory cannot be opened safely") from exc - logs_stat = os.fstat(logs_fd) - if not stat.S_ISDIR(logs_stat.st_mode): - os.close(logs_fd) - raise LaunchPayloadError("marker directory is not a directory") - if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): - os.close(logs_fd) - raise LaunchPayloadError("marker directory has an unexpected owner") - return logs_fd + try: + logs_stat = os.fstat(logs_fd) + if not stat.S_ISDIR(logs_stat.st_mode): + raise LaunchPayloadError("marker directory is not a directory") + if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("marker directory has an unexpected owner") + return logs_fd + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise LaunchPayloadError("marker directory could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise def _write_all(fd: int, data: bytes) -> None: @@ -481,50 +573,13 @@ def _is_owned_runtime_marker( pid: int, expected_fingerprint: str, ) -> bool: - if type(value) is not dict or frozenset(value) not in { - _RUNTIME_MARKER_KEYS, - _RUNTIME_MARKER_FINGERPRINT_KEYS, - }: - return False - marker = value - if ( - type(marker["schema"]) is not int - or marker["schema"] != 3 - or marker["bot_id"] != bot_id - or marker["component"] != "gateway" - or marker["action"] != "run" - or marker["operation_id"] != operation_id - or type(marker["desired_revision"]) is not int - or marker["desired_revision"] != desired_revision - or type(marker["pid"]) is not int - or marker["pid"] != pid - or marker["command_fingerprint"] != expected_fingerprint - ): - return False - started_at = marker["started_at"] - if ( - isinstance(started_at, bool) - or not isinstance(started_at, int | float) - or not math.isfinite(float(started_at)) - or float(started_at) <= 0 - ): - return False - if "proc_start_fingerprint" in marker: - fingerprint = marker["proc_start_fingerprint"] - if type(fingerprint) is not str or not fingerprint or len(fingerprint) > 512: - return False - try: - argv = _validate_argv(marker["argv"]) - resolved_hermes = _validate_path(marker["resolved_hermes_bin"], "resolved_hermes_bin") - _validate_readiness_probe(marker["readiness_probe"]) - except LaunchPayloadError: - return False - return ( - len(argv) == 5 - and argv[1:] == ["-p", bot_id, "gateway", "run"] - and argv[0] == str(resolved_hermes) - and _FINGERPRINT_RE.fullmatch(expected_fingerprint) is not None - and command_fingerprint(argv) == expected_fingerprint + return _marker_is_owned_runtime_marker( + value, + bot_id=bot_id, + operation_id=operation_id, + desired_revision=desired_revision, + pid=pid, + expected_fingerprint=expected_fingerprint, ) @@ -535,16 +590,104 @@ def _open_regular_marker(logs_fd: int) -> tuple[int, os.stat_result]: raise LaunchPayloadError("marker is unavailable") from exc if not stat.S_ISREG(before.st_mode): raise LaunchPayloadError("marker is not a regular file") + nonblocking = getattr(os, "O_NONBLOCK", None) + if type(nonblocking) is not int or nonblocking == 0: + raise LaunchPayloadError("bounded marker reads are unavailable") flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= nonblocking try: marker_fd = os.open(MARKER_NAME, flags, dir_fd=logs_fd) except OSError as exc: raise LaunchPayloadError("marker cannot be opened safely") from exc - after = os.fstat(marker_fd) - if not stat.S_ISREG(after.st_mode) or not _same_file(before, after): - os.close(marker_fd) - raise LaunchPayloadError("marker changed while it was opened") - return marker_fd, after + try: + after = os.fstat(marker_fd) + if not stat.S_ISREG(after.st_mode) or not _same_file(before, after): + raise LaunchPayloadError("marker changed while it was opened") + return marker_fd, after + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise LaunchPayloadError("marker could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise + + +def _validate_open_marker_binding( + logs_fd: int, + marker_fd: int, + marker_stat: os.stat_result, +) -> os.stat_result: + try: + opened_marker = os.fstat(marker_fd) + current_marker = os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError("marker changed while it was read") from exc + snapshots = (marker_stat, opened_marker, current_marker) + if not all(stat.S_ISREG(snapshot.st_mode) for snapshot in snapshots) or not all( + _same_file(marker_stat, snapshot) for snapshot in snapshots[1:] + ): + raise LaunchPayloadError("marker changed while it was read") + if hasattr(os, "geteuid") and any(snapshot.st_uid != os.geteuid() for snapshot in snapshots): + raise LaunchPayloadError("marker has an unexpected owner") + if any(snapshot.st_nlink != 1 for snapshot in snapshots): + raise LaunchPayloadError("marker has unexpected links") + return current_marker + + +def _validate_marker_bindings( + profile: _OpenedProfile, + logs_fd: int, + marker_fd: int, + marker_stat: os.stat_result, +) -> os.stat_result: + current_marker = marker_stat + for _attempt in range(2): + profile.validate_bindings() + logs_stat = _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) + if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("marker directory has an unexpected owner") + current_marker = _validate_open_marker_binding( + logs_fd, + marker_fd, + marker_stat, + ) + return current_marker + + +def _confirm_marker_missing(profile: _OpenedProfile, logs_fd: int) -> None: + for _attempt in range(2): + profile.validate_bindings() + _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) + try: + os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) + except FileNotFoundError: + continue + except OSError as exc: + raise LaunchPayloadError("missing marker cannot be confirmed") from exc + raise LaunchPayloadError("marker appeared while absence was confirmed") + profile.validate_bindings() + _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) def _remove_marker_if_owned_locked( diff --git a/zeus/gateway_marker.py b/zeus/gateway_marker.py new file mode 100644 index 0000000..b1f61d3 --- /dev/null +++ b/zeus/gateway_marker.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TypeGuard +from urllib.parse import urlparse + +from zeus.models import ID_RE +from zeus.readiness import ReadinessProbe + +__all__ = [ + "GatewayGeneration", + "GatewayLaunchMarker", + "GatewayRuntimeMarker", + "MarkerValidationError", + "command_fingerprint", + "is_compat_runtime_marker", + "is_owned_runtime_marker", + "parse_launch_marker", + "parse_runtime_marker", + "readiness_probe_from_payload", + "readiness_probe_to_payload", +] + +_MAX_ARGV_PARTS = 64 +_MAX_ARG_BYTES = 64 * 1024 +_LAUNCH_MARKER_KEYS = frozenset( + { + "schema", + "bot_id", + "component", + "action", + "operation_id", + "desired_revision", + "argv", + "resolved_hermes_bin", + "command_fingerprint", + "readiness_probe", + } +) +_RUNTIME_MARKER_KEYS = _LAUNCH_MARKER_KEYS | frozenset({"pid", "started_at"}) +_RUNTIME_MARKER_FINGERPRINT_KEYS = _RUNTIME_MARKER_KEYS | frozenset({"proc_start_fingerprint"}) +_READINESS_PROBE_KEYS = frozenset( + {"url", "expected_status", "expected_platform", "timeout_seconds", "interval_seconds"} +) +_OPERATION_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{64}$") + + +class MarkerValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class GatewayGeneration: + operation_id: str + desired_revision: int + pid: int + command_fingerprint: str + proc_start_fingerprint: str | None + + +@dataclass(frozen=True) +class GatewayLaunchMarker: + bot_id: str + operation_id: str + desired_revision: int + argv: tuple[str, ...] + resolved_hermes_bin: str + command_fingerprint: str + readiness_probe: ReadinessProbe | None + + def to_payload(self) -> dict[str, object]: + return { + "schema": 3, + "bot_id": self.bot_id, + "component": "gateway", + "action": "run", + "operation_id": self.operation_id, + "desired_revision": self.desired_revision, + "argv": list(self.argv), + "resolved_hermes_bin": self.resolved_hermes_bin, + "command_fingerprint": self.command_fingerprint, + "readiness_probe": readiness_probe_to_payload(self.readiness_probe), + } + + +@dataclass(frozen=True) +class GatewayRuntimeMarker(GatewayLaunchMarker): + pid: int + started_at: int | float + proc_start_fingerprint: str | None = None + + def to_payload(self) -> dict[str, object]: + payload = super().to_payload() + payload["pid"] = self.pid + payload["started_at"] = self.started_at + if self.proc_start_fingerprint is not None: + payload["proc_start_fingerprint"] = self.proc_start_fingerprint + return payload + + def generation(self) -> GatewayGeneration: + return GatewayGeneration( + operation_id=self.operation_id, + desired_revision=self.desired_revision, + pid=self.pid, + command_fingerprint=self.command_fingerprint, + proc_start_fingerprint=self.proc_start_fingerprint, + ) + + +def command_fingerprint(argv: list[str]) -> str: + encoded = json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def parse_launch_marker(value: object) -> GatewayLaunchMarker: + marker, bot_id = _parse_launch_marker_identity(value) + operation_id, revision = _parse_launch_marker_correlation(marker) + argv = _parse_launch_marker_argv(marker) + return _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=argv, + ) + + +def parse_runtime_marker(value: object) -> GatewayRuntimeMarker: + if type(value) is not dict: + raise MarkerValidationError("runtime marker must be an object") + marker = value + if frozenset(marker) not in { + _RUNTIME_MARKER_KEYS, + _RUNTIME_MARKER_FINGERPRINT_KEYS, + } or not all(type(key) is str for key in marker): + raise MarkerValidationError("runtime marker has invalid keys") + + bot_id = _parse_bot_id(marker) + operation_id, revision = _parse_launch_marker_correlation(marker) + argv = _parse_launch_marker_argv(marker) + launch = _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=argv, + ) + pid = marker["pid"] + if type(pid) is not int or pid <= 0: + raise MarkerValidationError("marker PID is invalid") + started_at = marker["started_at"] + if not _is_finite_positive_number(started_at): + raise MarkerValidationError("marker started_at is invalid") + process_start: str | None = None + if "proc_start_fingerprint" in marker: + raw_process_start = marker["proc_start_fingerprint"] + if ( + type(raw_process_start) is not str + or not raw_process_start + or len(raw_process_start) > 512 + ): + raise MarkerValidationError("process start fingerprint is invalid") + process_start = raw_process_start + + return GatewayRuntimeMarker( + bot_id=launch.bot_id, + operation_id=launch.operation_id, + desired_revision=launch.desired_revision, + argv=launch.argv, + resolved_hermes_bin=launch.resolved_hermes_bin, + command_fingerprint=launch.command_fingerprint, + readiness_probe=launch.readiness_probe, + pid=pid, + started_at=started_at, + proc_start_fingerprint=process_start, + ) + + +def is_owned_runtime_marker( + value: object, + *, + bot_id: str, + operation_id: str, + desired_revision: int, + pid: int, + expected_fingerprint: str, +) -> bool: + try: + marker = parse_runtime_marker(value) + except MarkerValidationError: + return False + return ( + marker.bot_id == bot_id + and marker.operation_id == operation_id + and marker.desired_revision == desired_revision + and marker.pid == pid + and marker.command_fingerprint == expected_fingerprint + ) + + +def is_compat_runtime_marker(value: object) -> bool: + if type(value) is not dict: + return False + schema = value.get("schema") + return (type(schema) is int and schema == 2) or schema is None + + +def readiness_probe_to_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: + if probe is None: + return None + return { + "url": probe.url, + "expected_status": probe.expected_status, + "expected_platform": probe.expected_platform, + "timeout_seconds": probe.timeout_seconds, + "interval_seconds": probe.interval_seconds, + } + + +def readiness_probe_from_payload(value: object) -> ReadinessProbe | None: + if value is None: + return None + if not isinstance(value, dict): + raise MarkerValidationError("readiness_probe must be an object or null") + url = value.get("url") + expected_status = value.get("expected_status") + expected_platform = value.get("expected_platform") + timeout_seconds = value.get("timeout_seconds") + interval_seconds = value.get("interval_seconds") + if not isinstance(url, str): + raise MarkerValidationError("url must be a string") + try: + parsed = urlparse(url) + valid_url = ( + parsed.scheme == "http" + and parsed.hostname in {"127.0.0.1", "localhost", "::1"} + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + ) + except ValueError: + valid_url = False + if not valid_url: + raise MarkerValidationError("url must be loopback HTTP without credentials or query data") + if not isinstance(expected_status, str) or not expected_status: + raise MarkerValidationError("expected_status must be a non-empty string") + if not isinstance(expected_platform, str) or not expected_platform: + raise MarkerValidationError("expected_platform must be a non-empty string") + if not _is_valid_probe_number(timeout_seconds) or not _is_valid_probe_number(interval_seconds): + raise MarkerValidationError("probe timing values must be finite positive numbers") + return ReadinessProbe( + url=url, + expected_status=expected_status, + expected_platform=expected_platform, + timeout_seconds=float(timeout_seconds), + interval_seconds=float(interval_seconds), + ) + + +def _parse_launch_marker_identity(value: object) -> tuple[dict[str, object], str]: + marker = _exact_dict(value, _LAUNCH_MARKER_KEYS, "marker") + return marker, _parse_bot_id(marker) + + +def _parse_bot_id(marker: dict[str, object]) -> str: + bot_id = _exact_string(marker["bot_id"], "bot_id", max_length=63) + if ID_RE.fullmatch(bot_id) is None: + raise MarkerValidationError("bot_id is invalid") + return bot_id + + +def _parse_launch_marker_correlation(marker: dict[str, object]) -> tuple[str, int]: + if marker["schema"] != 3 or type(marker["schema"]) is not int: + raise MarkerValidationError("marker schema is invalid") + if marker["component"] != "gateway" or marker["action"] != "run": + raise MarkerValidationError("marker command intent is invalid") + operation_id = _exact_string(marker["operation_id"], "operation_id", max_length=32) + if _OPERATION_ID_RE.fullmatch(operation_id) is None: + raise MarkerValidationError("operation_id is invalid") + revision = marker["desired_revision"] + if type(revision) is not int or not 1 <= revision <= 2**63 - 1: + raise MarkerValidationError("desired_revision is invalid") + return operation_id, revision + + +def _parse_launch_marker_argv(marker: dict[str, object]) -> tuple[str, ...]: + return _parse_argv(marker["argv"]) + + +def _finish_launch_marker( + marker: dict[str, object], + *, + bot_id: str, + operation_id: str, + revision: int, + argv: tuple[str, ...], +) -> GatewayLaunchMarker: + if len(argv) != 5 or list(argv[1:]) != ["-p", bot_id, "gateway", "run"]: + raise MarkerValidationError("argv is not a Hermes gateway command") + resolved_hermes, canonical_hermes = _parse_absolute_path( + marker["resolved_hermes_bin"], "resolved_hermes_bin" + ) + if argv[0] != canonical_hermes: + raise MarkerValidationError("exec argv does not use the resolved Hermes binary") + fingerprint = _exact_string(marker["command_fingerprint"], "command_fingerprint", max_length=64) + if _FINGERPRINT_RE.fullmatch(fingerprint) is None or fingerprint != command_fingerprint( + list(argv) + ): + raise MarkerValidationError("command fingerprint is invalid") + readiness = _strict_readiness_probe_from_payload(marker["readiness_probe"]) + return GatewayLaunchMarker( + bot_id=bot_id, + operation_id=operation_id, + desired_revision=revision, + argv=argv, + resolved_hermes_bin=resolved_hermes, + command_fingerprint=fingerprint, + readiness_probe=readiness, + ) + + +def _strict_readiness_probe_from_payload(value: object) -> ReadinessProbe | None: + if value is None: + return None + probe = _exact_dict(value, _READINESS_PROBE_KEYS, "readiness_probe") + url = _exact_string(probe["url"], "readiness URL", max_length=2048) + try: + parsed = urlparse(url) + valid_url = ( + parsed.scheme == "http" + and parsed.hostname in {"127.0.0.1", "localhost", "::1"} + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + ) + except ValueError: + valid_url = False + if not valid_url: + raise MarkerValidationError("readiness URL must be loopback HTTP") + expected_status = _exact_string(probe["expected_status"], "expected status", max_length=128) + expected_platform = _exact_string( + probe["expected_platform"], "expected platform", max_length=128 + ) + timeout_seconds = probe["timeout_seconds"] + interval_seconds = probe["interval_seconds"] + if not _is_valid_probe_number(timeout_seconds) or not _is_valid_probe_number(interval_seconds): + raise MarkerValidationError("readiness timing is invalid") + return ReadinessProbe( + url=url, + expected_status=expected_status, + expected_platform=expected_platform, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + + +def _exact_dict(value: object, keys: frozenset[str], name: str) -> dict[str, object]: + if type(value) is not dict: + raise MarkerValidationError(f"{name} must be an object") + result = value + if set(result) != keys or not all(type(key) is str for key in result): + raise MarkerValidationError(f"{name} has invalid keys") + return result + + +def _exact_string(value: object, name: str, *, max_length: int) -> str: + if type(value) is not str or not value or len(value) > max_length or "\0" in value: + raise MarkerValidationError(f"{name} must be a bounded non-empty string") + return value + + +def _parse_argv(value: object) -> tuple[str, ...]: + if type(value) is not list or not value or len(value) > _MAX_ARGV_PARTS: + raise MarkerValidationError("argv must be a bounded non-empty list") + argv: list[str] = [] + total = 0 + for item in value: + part = _exact_string(item, "argv item", max_length=16 * 1024) + total += len(part.encode("utf-8")) + if total > _MAX_ARG_BYTES: + raise MarkerValidationError("argv is too large") + argv.append(part) + return tuple(argv) + + +def _parse_absolute_path(value: object, name: str) -> tuple[str, str]: + raw = _exact_string(value, name, max_length=16 * 1024) + path = Path(raw) + if not path.is_absolute() or any(part in {".", ".."} for part in path.parts): + raise MarkerValidationError(f"{name} must be an absolute path without traversal") + return raw, str(path) + + +def _is_valid_probe_number(value: object) -> TypeGuard[int | float]: + if isinstance(value, bool) or not isinstance(value, int | float): + return False + try: + number = float(value) + except OverflowError: + return False + return math.isfinite(number) and 0 < number <= 3600 + + +def _is_finite_positive_number(value: object) -> bool: + if isinstance(value, bool) or not isinstance(value, int | float): + return False + try: + number = float(value) + except OverflowError: + return False + return math.isfinite(number) and number > 0 diff --git a/zeus/gateway_runtime.py b/zeus/gateway_runtime.py new file mode 100644 index 0000000..f828d27 --- /dev/null +++ b/zeus/gateway_runtime.py @@ -0,0 +1,1676 @@ +from __future__ import annotations + +import contextlib +import errno +import json +import math +import os +import platform +import select +import signal +import stat +import subprocess # nosec B404 +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Protocol + +from zeus import process_identity +from zeus.errors import BotRunningError +from zeus.fs_utils import atomic_write_json +from zeus.gateway_launcher import ( + MAX_PAYLOAD_BYTES, + LaunchPayloadError, + _confirm_marker_missing, + _ConfirmedMissing, + _open_logs, + _open_profile_chain, + _open_regular_marker, + _read_bounded_file, + _reject_duplicate_keys, + _remove_marker_if_owned_locked, + _validate_marker_bindings, + marker_publication_lock, + remove_marker_if_owned, +) +from zeus.gateway_marker import ( + GatewayGeneration, + is_compat_runtime_marker, + is_owned_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.models import BotRecord, TemplateError +from zeus.private_io import UnsafeFileError, nofollow_absolute_path, open_private_append +from zeus.profile_manager import ProfileManager +from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env + + +class PopenLike(Protocol): + pid: int + + def poll(self) -> int | None: ... + + +PopenFactory = Callable[..., PopenLike] +KillFn = Callable[[int, signal.Signals], None] +PidAliveFn = process_identity.PidAliveFn +CmdlineReader = process_identity.CmdlineReader +ProcStartFingerprintReader = process_identity.ProcStartFingerprintReader + + +class SignalResult(Enum): + sent = "sent" + missing = "missing" + denied = "denied" + + +_MAX_EFFECT_TEXT = 512 +_TRANSIENT_POST_EXEC_MARKER_REASONS = frozenset( + { + "live gateway command is unavailable", + "process start fingerprint is unavailable", + } +) + + +def _bounded_text(value: str) -> str: + return value[:_MAX_EFFECT_TEXT] + + +@dataclass(frozen=True) +class OwnershipCheck: + verified: bool + reason: str + classification: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + if self.classification is not None: + object.__setattr__( + self, + "classification", + _bounded_text(self.classification), + ) + + +@dataclass(frozen=True, init=False) +class MarkerObservation: + kind: str + reason: str + _payload_json: bytes | None = field(repr=False) + + def __init__( + self, + kind: str, + payload: dict[str, object] | None = None, + reason: str = "", + ) -> None: + snapshot: bytes | None = None + if payload is not None: + try: + snapshot = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError, UnicodeEncodeError): + kind = "untrusted" + reason = "marker payload could not be snapshotted safely" + else: + if len(snapshot) > MAX_PAYLOAD_BYTES: + kind = "untrusted" + reason = "marker payload snapshot is too large" + snapshot = None + object.__setattr__(self, "kind", kind) + object.__setattr__(self, "reason", _bounded_text(reason)) + object.__setattr__(self, "_payload_json", snapshot) + + @property + def payload(self) -> dict[str, object] | None: + if self._payload_json is None: + return None + value = json.loads(self._payload_json) + if type(value) is not dict: + return None + return value + + +@dataclass(frozen=True) +class LaunchEffect: + outcome: str + pid: int | None = None + generation: GatewayGeneration | None = None + reason: str = "" + returncode: int | None = None + error_type: str | None = None + readiness_message: str | None = None + cleanup_complete: bool | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + if self.error_type is not None: + object.__setattr__(self, "error_type", _bounded_text(self.error_type)) + if self.readiness_message is not None: + object.__setattr__( + self, + "readiness_message", + _bounded_text(self.readiness_message), + ) + + +@dataclass(frozen=True) +class StopEffect: + outcome: str + pid: int | None = None + generation: GatewayGeneration | None = None + reason: str = "" + term_result: SignalResult | None = None + kill_result: SignalResult | None = None + marker_removed: bool = False + kill_attempted: bool = False + kill_succeeded: bool | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + + +PipeFn = Callable[[], tuple[int, int]] +CloseFn = Callable[[int], None] +ReadBoundedFileFn = Callable[[int], bytes] +RemoveMarkerLockedFn = Callable[..., bool] +ProbeOnceFn = Callable[..., ReadinessResult] + + +@dataclass(frozen=True) +class RuntimeHooks: + pipe: PipeFn + close: CloseFn + read_bounded_file: ReadBoundedFileFn + remove_marker_if_owned_locked: RemoveMarkerLockedFn + probe_once: ProbeOnceFn + + +def default_runtime_hooks() -> RuntimeHooks: + return RuntimeHooks( + pipe=os.pipe, + close=os.close, + read_bounded_file=_read_bounded_file, + remove_marker_if_owned_locked=_remove_marker_if_owned_locked, + probe_once=probe_once, + ) + + +def gateway_process_launch_kwargs() -> dict[str, object]: + if os.name == "posix": + return {"start_new_session": True} + creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + return {"creationflags": creationflags} if creationflags else {} + + +def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + +class GatewayRuntime: + def __init__( + self, + adapter: HermesAdapter, + profile_manager: ProfileManager, + marker_profiles_root: Path, + *, + popen_factory: PopenFactory = subprocess.Popen, + kill_fn: KillFn = os.kill, + pid_alive_fn: PidAliveFn | None = None, + cmdline_reader: CmdlineReader, + proc_start_fingerprint_reader: ProcStartFingerprintReader, + startup_grace_seconds: float = 0.25, + stop_grace_seconds: float = 15.0, + kill_after_timeout: bool = False, + lock_timeout_seconds: float = 30.0, + readiness_timeout_seconds: float = 30.0, + readiness_interval_seconds: float = 0.5, + allow_legacy_pid_markers: bool = True, + cleanup_process_group: bool = False, + hooks_provider: Callable[[], RuntimeHooks] = default_runtime_hooks, + ) -> None: + self.adapter = adapter + self.profile_manager = profile_manager + self.marker_profiles_root = marker_profiles_root + self.popen_factory = popen_factory + self.kill_fn = kill_fn + self.pid_alive_fn = pid_alive_fn + self.cmdline_reader = cmdline_reader + self.proc_start_fingerprint_reader = proc_start_fingerprint_reader + self.startup_grace_seconds = startup_grace_seconds + self.stop_grace_seconds = stop_grace_seconds + self.kill_after_timeout = kill_after_timeout + self.lock_timeout_seconds = lock_timeout_seconds + self.readiness_timeout_seconds = readiness_timeout_seconds + self.readiness_interval_seconds = readiness_interval_seconds + self.allow_legacy_pid_markers = allow_legacy_pid_markers + self.cleanup_process_group = cleanup_process_group + self._hooks_provider = hooks_provider + self._processes: dict[str, PopenLike] = {} + + def _hooks(self) -> RuntimeHooks: + return self._hooks_provider() + + def safe_profile_path(self, bot_id: str, profile_path: str) -> Path: + return self.profile_manager.validate_profile_path(bot_id, profile_path) + + def marker_publication_lock( + self, + record: BotRecord, + ) -> contextlib.AbstractContextManager[object]: + profile_path = self.safe_profile_path(record.bot_id, record.profile_path) + if not os.path.lexists(profile_path): + return contextlib.nullcontext() + return marker_publication_lock( + profile_path, + timeout_seconds=self.lock_timeout_seconds, + ) + + def log_path(self, profile_path: str) -> Path: + return nofollow_absolute_path(Path(profile_path) / "logs" / "zeus-gateway.log") + + def pid_marker_path(self, profile_path: str) -> Path: + return Path(profile_path) / "logs" / "zeus-gateway.pid.json" + + def preflight_start( + self, + record: BotRecord, + *, + timeout_seconds: float | None, + ) -> ReadinessProbe | None: + expected_profile = (Path(self.adapter.hermes_root) / "profiles" / record.bot_id).resolve() + if Path(record.profile_path).resolve() != expected_profile: + raise TemplateError("registered bot profile does not match the Hermes profile path") + safe_profile = self.safe_profile_path(record.bot_id, record.profile_path) + if not safe_profile.is_dir() or safe_profile.is_symlink(): + raise TemplateError("registered bot profile is not a safe directory") + _argv, env = self.adapter.command(record.bot_id, "gateway", "run") + readiness = self.readiness_probe(env, timeout_seconds=timeout_seconds) + self.adapter.launcher_payload( + record.bot_id, + operation_id="0" * 32, + desired_revision=max(1, record.desired_revision + 1), + readiness_probe=readiness, + ) + return readiness + + @staticmethod + def write_pipe_payload(fd: int, payload: bytes) -> None: + offset = 0 + while offset < len(payload): + written = os.write(fd, payload[offset:]) + if written <= 0: + raise OSError("short launcher payload write") + offset += written + + @staticmethod + def read_launcher_ack(fd: int) -> bytes: + deadline = time.monotonic() + 5.0 + acknowledgment = bytearray() + while len(acknowledgment) <= 1: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("gateway launcher acknowledgment timed out") + readable, _writable, _exceptional = select.select([fd], [], [], remaining) + if not readable: + raise TimeoutError("gateway launcher acknowledgment timed out") + chunk = os.read(fd, 2 - len(acknowledgment)) + if not chunk: + return bytes(acknowledgment) + acknowledgment.extend(chunk) + return bytes(acknowledgment) + + def launch( + self, + record: BotRecord, + *, + probe: ReadinessProbe | None, + wait: bool, + marker_lock: Callable[[BotRecord], contextlib.AbstractContextManager[object]] | None = None, + marker_matcher: Callable[..., MarkerObservation] | None = None, + ack_reader: Callable[[int], bytes] | None = None, + pipe_writer: Callable[[int, bytes], None] | None = None, + ) -> LaunchEffect: + operation_id = record.pending_operation_id + if record.pending_action not in {"start", "restart"} or operation_id is None: + raise RuntimeError("gateway launch requires a pending start or restart intent") + payload = self.adapter.launcher_payload( + record.bot_id, + operation_id=operation_id, + desired_revision=record.desired_revision, + readiness_probe=probe, + ) + marker_data = payload["marker"] + if type(marker_data) is not dict: + raise RuntimeError("launcher produced an invalid marker payload") + expected_fingerprint = str(marker_data["command_fingerprint"]) + process: PopenLike | None = None + generation: GatewayGeneration | None = None + marker_acknowledged = False + payload_read = payload_write = ack_read = ack_write = -1 + hooks = self._hooks() + marker_lock = marker_lock or self.marker_publication_lock + marker_matcher = marker_matcher or self.matching_runtime_marker + ack_reader = ack_reader or self.read_launcher_ack + pipe_writer = pipe_writer or self.write_pipe_payload + try: + encoded_payload = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + if not encoded_payload or len(encoded_payload) > MAX_PAYLOAD_BYTES: + raise ValueError("launcher payload is too large") + with open_private_append(self.log_path(record.profile_path)) as log_file: + payload_read, payload_write = hooks.pipe() + ack_read, ack_write = hooks.pipe() + launcher_argv = self.adapter.launcher_command(payload_read, ack_write) + process = self.popen_factory( + launcher_argv, + env=dict(os.environ), + stdout=log_file, + stderr=log_file, + pass_fds=(payload_read, ack_write), + close_fds=True, + **gateway_process_launch_kwargs(), + ) + hooks.close(payload_read) + payload_read = -1 + hooks.close(ack_write) + ack_write = -1 + pipe_writer(payload_write, encoded_payload) + hooks.close(payload_write) + payload_write = -1 + acknowledgment = ack_reader(ack_read) + hooks.close(ack_read) + ack_read = -1 + if acknowledgment != b"1": + raise RuntimeError("gateway launcher did not acknowledge marker publication") + marker_acknowledged = True + with marker_lock(record): + registration_deadline = time.monotonic() + max(self.startup_grace_seconds, 0) + while True: + marker = marker_matcher( + record, + expected_fingerprint=expected_fingerprint, + expected_pid=process.pid, + require_live_command=True, + ) + generation = self.gateway_generation(marker) + if marker.kind == "live" and generation is not None: + break + if process.poll() is not None: + raise RuntimeError("gateway process exited before marker registration") + remaining = registration_deadline - time.monotonic() + if marker.reason not in _TRANSIENT_POST_EXEC_MARKER_REASONS or remaining <= 0: + raise RuntimeError( + "gateway launcher ownership marker could not be verified: " + + marker.reason + ) + time.sleep(min(0.01, remaining)) + self._processes[record.bot_id] = process + except BaseException as exc: + for fd in (payload_read, payload_write, ack_read, ack_write): + if fd >= 0: + with contextlib.suppress(OSError): + hooks.close(fd) + if process is None: + if not isinstance(exc, (OSError, ValueError)): + raise + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "launch_failed", + reason=str(exc), + error_type=type(exc).__name__, + ) + returncode = process.poll() + cleanup_complete = self.cleanup_interrupted_launch( + record, + process, + expected_fingerprint=expected_fingerprint, + ) + if not isinstance(exc, Exception): + raise + if marker_acknowledged and returncode is not None and cleanup_complete: + if generation is None: + generation = GatewayGeneration( + operation_id=operation_id, + desired_revision=record.desired_revision, + pid=process.pid, + command_fingerprint=expected_fingerprint, + proc_start_fingerprint=None, + ) + return LaunchEffect( + "startup_exited", + pid=process.pid, + generation=generation, + reason="gateway exited during startup grace period", + returncode=returncode, + cleanup_complete=True, + ) + return LaunchEffect( + "registration_failed_clean" if cleanup_complete else "registration_failed_unknown", + pid=None if cleanup_complete else process.pid, + reason=str(exc), + error_type=type(exc).__name__, + cleanup_complete=cleanup_complete, + ) + if process is None or generation is None: + raise RuntimeError("gateway process factory returned no process") + returncode = self.poll_startup(process) + if returncode is not None: + self.remove_gateway_generation_marker(record, generation) + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "startup_exited", + generation=generation, + reason="gateway exited during startup grace period", + returncode=returncode, + ) + if probe is not None: + if wait: + readiness = self.wait_for_readiness(process, probe) + if process.poll() is not None: + returncode = process.poll() + self.remove_gateway_generation_marker(record, generation) + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "readiness_exited", + generation=generation, + reason="gateway process exited during readiness check", + returncode=returncode, + ) + if readiness.ready: + return LaunchEffect("ready", process.pid, generation) + return LaunchEffect( + "readiness_timeout", + process.pid, + generation, + reason="readiness probe timed out", + readiness_message=readiness.message, + ) + return LaunchEffect("readiness_pending", process.pid, generation) + return LaunchEffect("running", process.pid, generation) + + def cleanup_interrupted_launch( + self, + record: BotRecord, + process: PopenLike, + *, + expected_fingerprint: str, + ) -> bool: + cleanup_errors: list[str] = [] + if not self.terminate_spawned_process(process, cleanup_errors): + return False + self._processes.pop(record.bot_id, None) + operation_id = record.pending_operation_id + if operation_id is None: + return False + remove_marker_if_owned( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=operation_id, + desired_revision=record.desired_revision, + pid=process.pid, + command_fingerprint=expected_fingerprint, + ) + return self.read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + + def cleanup_registered_launch( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + process = self._processes.get(record.bot_id) + if process is None or process.pid != generation.pid: + return False + cleanup_errors: list[str] = [] + if not self.terminate_spawned_process(process, cleanup_errors): + return False + self._processes.pop(record.bot_id, None) + self.remove_gateway_generation_marker(record, generation) + return self.read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + + def read_strict_runtime_marker( + self, + bot_id: str, + registered_profile_path: str, + ) -> MarkerObservation: + profile_path = nofollow_absolute_path(Path(registered_profile_path)) + expected_profile = self.marker_profiles_root / bot_id + if not profile_path.is_absolute() or profile_path != expected_profile: + return MarkerObservation( + "untrusted", + reason="registered profile path does not match the trusted Hermes profile", + ) + profile = None + logs_fd = marker_fd = -1 + hooks = self._hooks() + try: + profile = _open_profile_chain(profile_path) + except _ConfirmedMissing: + return MarkerObservation("missing", reason="marker is missing") + except (OSError, ValueError) as exc: + return MarkerObservation( + "untrusted", reason=f"registered profile cannot be opened safely: {exc}" + ) + try: + try: + logs_fd = _open_logs(profile.fd, create=False) + except ValueError as exc: + if isinstance(exc.__cause__, FileNotFoundError): + try: + profile.confirm_missing("logs") + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", + reason=f"marker directory absence is untrusted: {confirm_error}", + ) + return MarkerObservation("missing", reason="marker is missing") + return MarkerObservation( + "untrusted", reason=f"marker directory cannot be opened safely: {exc}" + ) + try: + marker_fd, marker_stat = _open_regular_marker(logs_fd) + raw = hooks.read_bounded_file(marker_fd) + marker_stat = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except FileNotFoundError: + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", reason=f"marker absence is untrusted: {confirm_error}" + ) + return MarkerObservation("missing", reason="marker is missing") + except ValueError as exc: + if isinstance(exc.__cause__, FileNotFoundError): + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", reason=f"marker absence is untrusted: {confirm_error}" + ) + return MarkerObservation("missing", reason="marker is missing") + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + except FileNotFoundError as exc: + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + finally: + for fd in (marker_fd, logs_fd): + if fd >= 0: + with contextlib.suppress(OSError): + hooks.close(fd) + if profile is not None: + profile.close() + if type(value) is not dict: + return MarkerObservation("untrusted", reason="marker is not an object") + if marker_stat.st_nlink != 1: + return MarkerObservation("untrusted", reason="marker has unexpected hard links") + return MarkerObservation("present", payload=value) + + def matching_runtime_marker( + self, + record: BotRecord, + *, + expected_fingerprint: str, + expected_pid: int | None = None, + require_live_command: bool, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return observed + operation_id = record.pending_operation_id + if operation_id is None: + return MarkerObservation("untrusted", reason="pending operation is missing") + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=expected_pid, + expected_operation_id=operation_id, + expected_revision=record.desired_revision, + expected_fingerprint=expected_fingerprint, + require_live_command=require_live_command, + ) + + def classify_schema3_runtime_marker( + self, + record: BotRecord, + payload: dict[str, object], + *, + expected_pid: int | None = None, + expected_operation_id: str | None = None, + expected_revision: int | None = None, + expected_fingerprint: str | None = None, + require_live_command: bool, + ) -> MarkerObservation: + pid_value = payload.get("pid") + if type(pid_value) is not int or pid_value <= 0: + return MarkerObservation("untrusted", reason="marker PID is invalid") + pid = pid_value + if expected_pid is not None and pid != expected_pid: + return MarkerObservation("untrusted", reason="marker PID does not match") + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + fingerprint = payload.get("command_fingerprint") + if ( + type(operation_id) is not str + or type(revision) is not int + or type(fingerprint) is not str + ): + return MarkerObservation("untrusted", reason="marker correlation is invalid") + if expected_operation_id is not None and operation_id != expected_operation_id: + return MarkerObservation("untrusted", reason="marker operation does not match") + if expected_revision is not None and revision != expected_revision: + return MarkerObservation("untrusted", reason="marker revision does not match") + if expected_fingerprint is not None and fingerprint != expected_fingerprint: + return MarkerObservation("untrusted", reason="marker command does not match") + if not is_owned_runtime_marker( + payload, + bot_id=record.bot_id, + operation_id=operation_id, + desired_revision=revision, + pid=pid, + expected_fingerprint=fingerprint, + ): + return MarkerObservation("untrusted", reason="marker schema or command does not match") + expected_hermes = self.resolved_hermes_bin() + if expected_hermes is None or payload.get("resolved_hermes_bin") != expected_hermes: + return MarkerObservation("untrusted", reason="marker executable is not trusted") + if not require_live_command: + start_identity_error = self.process_start_identity_error(payload, pid) + if start_identity_error is not None: + return MarkerObservation("untrusted", reason=start_identity_error) + return MarkerObservation("live", payload=payload) + pid_state = self.pid_state(pid) + if pid_state is process_identity.PidState.unknown: + return MarkerObservation("untrusted", reason="marker PID liveness is unknown") + if pid_state is process_identity.PidState.dead: + if self.process_start_fingerprint_required() and not self.valid_marker_start( + payload.get("proc_start_fingerprint") + ): + return MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ) + return MarkerObservation("dead", payload=payload, reason="marker PID is dead") + start_identity_error = self.process_start_identity_error(payload, pid) + if start_identity_error is not None: + return MarkerObservation("untrusted", reason=start_identity_error) + live_argv = self.cmdline_reader(pid) + if not live_argv: + return MarkerObservation("untrusted", reason="live gateway command is unavailable") + command_check = process_identity.verify_gateway_command( + live_argv, + record.bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + if not command_check.verified: + return MarkerObservation("untrusted", reason="live gateway command does not match") + return MarkerObservation("live", payload=payload) + + def process_start_identity_error(self, payload: dict[str, object], pid: int) -> str | None: + return process_identity.process_start_identity_error( + payload.get("proc_start_fingerprint"), + self.proc_start_fingerprint_reader(pid), + fingerprint_required=self.process_start_fingerprint_required(), + ) + + @staticmethod + def valid_marker_start(value: object) -> bool: + return process_identity.valid_process_start_fingerprint(value) + + @staticmethod + def process_start_fingerprint_required() -> bool: + return process_identity.process_start_fingerprint_required(platform.system()) + + def classify_existing_runtime_marker( + self, + record: BotRecord, + *, + expected_pid: int | None = None, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return observed + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=expected_pid, + require_live_command=True, + ) + + @staticmethod + def gateway_generation(marker: MarkerObservation) -> GatewayGeneration | None: + payload = marker.payload + if marker.kind not in {"live", "dead"} or payload is None: + return None + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + pid = payload.get("pid") + fingerprint = payload.get("command_fingerprint") + process_start = payload.get("proc_start_fingerprint") + if ( + type(operation_id) is not str + or type(revision) is not int + or type(pid) is not int + or type(fingerprint) is not str + or (process_start is not None and type(process_start) is not str) + ): + return None + return GatewayGeneration( + operation_id=operation_id, + desired_revision=revision, + pid=pid, + command_fingerprint=fingerprint, + proc_start_fingerprint=process_start, + ) + + def classify_exact_gateway_generation( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return MarkerObservation("untrusted", reason="previous gateway marker changed") + if observed.payload.get("proc_start_fingerprint") != generation.proc_start_fingerprint: + return MarkerObservation( + "untrusted", reason="previous gateway process identity changed" + ) + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=generation.pid, + expected_operation_id=generation.operation_id, + expected_revision=generation.desired_revision, + expected_fingerprint=generation.command_fingerprint, + require_live_command=True, + ) + + def remove_exact_schema3_marker( + self, + record: BotRecord, + marker: MarkerObservation, + ) -> bool: + generation = self.gateway_generation(marker) + return bool( + marker.kind == "dead" + and generation is not None + and self.remove_gateway_generation_marker(record, generation) + ) + + def remove_gateway_generation_marker( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + return remove_marker_if_owned( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=generation.operation_id, + desired_revision=generation.desired_revision, + pid=generation.pid, + command_fingerprint=generation.command_fingerprint, + expected_proc_start_fingerprint=generation.proc_start_fingerprint, + lock_timeout_seconds=self.lock_timeout_seconds, + ) + + def remove_gateway_generation_marker_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + return self._hooks().remove_marker_if_owned_locked( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=generation.operation_id, + desired_revision=generation.desired_revision, + pid=generation.pid, + command_fingerprint=generation.command_fingerprint, + expected_proc_start_fingerprint=generation.proc_start_fingerprint, + ) + + def remove_owned_launch_marker_locked( + self, + record: BotRecord, + *, + observed: MarkerObservation | None = None, + ) -> bool: + if observed is None: + observed = self.read_strict_runtime_marker(record.bot_id, record.profile_path) + if observed.kind == "missing": + return True + if observed.kind != "present" or observed.payload is None or record.pid is None: + return False + if record.pending_action not in {"stop", "restart"}: + return False + marker = self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=record.pid, + expected_revision=record.desired_revision - 1, + require_live_command=True, + ) + generation = self.gateway_generation(marker) + return bool( + marker.kind == "dead" + and generation is not None + and self.remove_gateway_generation_marker_locked(record, generation) + ) + + def reauthorize_and_signal( + self, + record: BotRecord, + generation: GatewayGeneration, + sig: signal.Signals, + *, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + ) -> tuple[MarkerObservation, SignalResult | None]: + classify_exact = classify_exact or self.classify_exact_gateway_generation + current = classify_exact(record, generation) + if current.kind != "live": + return current, None + return current, self.send_signal(generation.pid, sig) + + def stop_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + classify_existing: Callable[..., MarkerObservation] | None = None, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + remove_owned: Callable[..., bool] | None = None, + remove_generation: Callable[[BotRecord, GatewayGeneration], bool] | None = None, + ) -> StopEffect: + read_marker = read_marker or self.read_strict_runtime_marker + classify_existing = classify_existing or self.classify_existing_runtime_marker + classify_exact = classify_exact or self.classify_exact_gateway_generation + remove_owned = remove_owned or self.remove_owned_launch_marker_locked + remove_generation = remove_generation or self.remove_gateway_generation_marker_locked + observed = read_marker(record.bot_id, record.profile_path) + if ( + observed.kind == "present" + and observed.payload is not None + and is_compat_runtime_marker(observed.payload) + ): + return StopEffect( + "compat_untrusted", + record.pid, + reason="schema-v2 or legacy gateway stop requires manual process resolution", + ) + if not record.pid: + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + marker = classify_existing(record, expected_pid=record.pid) + generation = self.gateway_generation(marker) + if marker.kind == "live" and generation is not None: + return self.stop_generation_locked( + record, + generation, + kill_after_timeout=kill_after_timeout, + classify_exact=classify_exact, + remove_generation=remove_generation, + ) + if marker.kind == "dead": + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + pid_state = self.pid_state(record.pid) + if pid_state is process_identity.PidState.unknown: + return StopEffect("pid_unknown", record.pid, reason="gateway PID liveness is unknown") + if pid_state is process_identity.PidState.dead: + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + if marker.kind != "live" or generation is None: + return StopEffect( + "ownership_unverified", + record.pid, + reason="refusing to stop process because PID ownership could not be verified", + ) + raise AssertionError("unreachable gateway marker state") + + def stop_generation_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + kill_after_timeout: bool | None, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + remove_generation: Callable[[BotRecord, GatewayGeneration], bool] | None = None, + ) -> StopEffect: + classify_exact = classify_exact or self.classify_exact_gateway_generation + remove_generation = remove_generation or self.remove_gateway_generation_marker_locked + current = classify_exact(record, generation) + term_result: SignalResult | None = None + kill_result: SignalResult | None = None + if current.kind == "dead": + stopped = True + elif current.kind == "live": + current, term_result = self.reauthorize_and_signal( + record, + generation, + signal.SIGTERM, + classify_exact=classify_exact, + ) + if current.kind != "live" or term_result is None: + return StopEffect( + "term_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGTERM", + ) + if term_result is SignalResult.denied: + return StopEffect( + "term_denied", + generation.pid, + generation, + "could not send SIGTERM to the gateway", + term_result=term_result, + ) + stopped = term_result is SignalResult.missing + if not stopped: + stopped = self.wait_for_exit(record.bot_id, generation.pid) + else: + return StopEffect( + "term_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGTERM", + ) + should_kill = self.kill_after_timeout if kill_after_timeout is None else kill_after_timeout + kill_attempted = False + kill_succeeded: bool | None = None + if not stopped and should_kill: + kill_attempted = True + current, kill_result = self.reauthorize_and_signal( + record, + generation, + signal.SIGKILL, + classify_exact=classify_exact, + ) + if current.kind == "dead": + stopped = True + kill_succeeded = True + elif current.kind != "live" or kill_result is None: + return StopEffect( + "kill_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGKILL", + term_result=term_result, + kill_attempted=True, + ) + elif kill_result is SignalResult.denied: + return StopEffect( + "kill_denied", + generation.pid, + generation, + "could not send SIGKILL to the gateway", + term_result=term_result, + kill_result=kill_result, + kill_attempted=True, + kill_succeeded=False, + ) + else: + stopped = kill_result is SignalResult.missing + if not stopped: + stopped = self.wait_for_exit(record.bot_id, generation.pid) + kill_succeeded = stopped + if not stopped: + return StopEffect( + "grace_expired", + generation.pid, + generation, + "gateway did not stop before grace period expired", + term_result=term_result, + kill_result=kill_result, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + if not remove_generation(record, generation): + return StopEffect( + "cleanup_unverified", + generation.pid, + generation, + "stopped gateway marker cleanup could not be verified", + term_result=term_result, + kill_result=kill_result, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + self._processes.pop(record.bot_id, None) + return StopEffect( + "stopped", + generation.pid, + generation, + term_result=term_result, + kill_result=kill_result, + marker_removed=True, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + + def readiness_probe_for_bot( + self, + bot_id: str, + *, + timeout_seconds: float | None = None, + ) -> ReadinessProbe | None: + _argv, env = self.adapter.command(bot_id, "gateway", "run") + return self.readiness_probe(env, timeout_seconds=timeout_seconds) + + def readiness_probe_for_live_record( + self, + record: BotRecord, + ) -> tuple[ReadinessProbe | None, str | None]: + try: + payload = json.loads( + self.pid_marker_path(record.profile_path).read_text(encoding="utf-8") + ) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return None, "readiness provenance is unavailable from the PID marker" + if not isinstance(payload, dict): + return None, "readiness provenance in the PID marker is invalid" + if payload.get("schema") not in {2, 3} or "readiness_probe" not in payload: + return self.readiness_probe_for_bot(record.bot_id), None + try: + return readiness_probe_from_payload(payload["readiness_probe"]), None + except ValueError as exc: + return None, f"readiness provenance in the PID marker is invalid: {exc}" + + def readiness_probe( + self, + env: dict[str, str], + *, + timeout_seconds: float | None = None, + ) -> ReadinessProbe | None: + resolved_timeout = ( + self.readiness_timeout_seconds if timeout_seconds is None else timeout_seconds + ) + if ( + isinstance(resolved_timeout, bool) + or not isinstance(resolved_timeout, (int, float)) + or not math.isfinite(float(resolved_timeout)) + or not 0.1 <= float(resolved_timeout) <= 300 + ): + raise TemplateError("readiness timeout must be a finite number between 0.1 and 300") + return readiness_probe_from_env( + env, + timeout_seconds=float(resolved_timeout), + interval_seconds=self.readiness_interval_seconds, + ) + + def wait_for_readiness( + self, + process: PopenLike, + readiness: ReadinessProbe, + ) -> ReadinessResult: + deadline = time.monotonic() + readiness.timeout_seconds + last = ReadinessResult(False, "not probed yet") + probe = self._hooks().probe_once + while time.monotonic() < deadline: + if process.poll() is not None: + return ReadinessResult(False, "gateway process exited during readiness check") + last = probe( + readiness.url, + timeout_seconds=min(5.0, max(0.2, readiness.interval_seconds)), + expected_status=readiness.expected_status, + expected_platform=readiness.expected_platform, + ) + if last.ready: + return last + time.sleep(readiness.interval_seconds) + return ReadinessResult(False, f"readiness timeout: {last.message}", last.payload) + + def pid_state(self, pid: int) -> process_identity.PidState: + if self.pid_alive_fn is not None: + return process_identity.pid_state(pid, pid_alive_fn=self.pid_alive_fn) + + def probe_with_current_kill(probe_pid: int) -> bool: + os.kill(probe_pid, 0) + return True + + return process_identity.pid_state(pid, pid_alive_fn=probe_with_current_kill) + + def send_signal(self, pid: int, sig: signal.Signals) -> SignalResult: + try: + self.kill_fn(pid, sig) + except ProcessLookupError: + return SignalResult.missing + except PermissionError: + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + return SignalResult.denied + raise + return SignalResult.sent + + def assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) -> None: + marker_path = self.pid_marker_path(str(profile_path)) + if not marker_path.exists(): + return + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BotRunningError( + "unregistered bot profile has an unreadable PID marker; refusing replacement" + ) from exc + pid = payload.get("pid") if isinstance(payload, dict) else None + if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: + raise BotRunningError( + "unregistered bot profile has an invalid PID marker; refusing replacement" + ) + if self.pid_state(pid) is not process_identity.PidState.dead: + raise BotRunningError( + f"unregistered bot profile may still own gateway PID {pid}; refusing replacement" + ) + + def write_pid_marker( + self, + profile_path: str, + pid: int, + bot_id: str, + argv: list[str], + *, + readiness_probe: ReadinessProbe | None, + include_readiness_probe: bool, + ) -> None: + path = self.pid_marker_path(profile_path) + path.parent.mkdir(parents=True, exist_ok=True) + resolved_hermes_bin = self.resolved_hermes_bin() + marker_argv = list(argv) + if resolved_hermes_bin: + marker_argv[0] = resolved_hermes_bin + fingerprint = self.proc_start_fingerprint_reader(pid) + payload: dict[str, object] = { + "schema": 2, + "pid": pid, + "bot_id": bot_id, + "component": "gateway", + "action": "run", + "argv": marker_argv, + "resolved_hermes_bin": resolved_hermes_bin, + "started_at": time.time(), + } + if include_readiness_probe: + payload["readiness_probe"] = readiness_probe_to_payload(readiness_probe) + if fingerprint: + payload["proc_start_fingerprint"] = fingerprint + atomic_write_json(path, payload, mode=0o600) + + def remove_pid_marker(self, profile_path: str) -> None: + try: + self.pid_marker_path(profile_path).unlink() + except FileNotFoundError: + return + + def read_pid_marker(self, profile_path: str) -> dict[str, object]: + safe_profile_path = nofollow_absolute_path(Path(profile_path)) + profile = None + logs_fd = marker_fd = -1 + hooks = self._hooks() + try: + try: + profile = _open_profile_chain(safe_profile_path) + logs_fd = _open_logs(profile.fd, create=False) + marker_fd, marker_stat = _open_regular_marker(logs_fd) + except _ConfirmedMissing: + return {"exists": False} + except (LaunchPayloadError, OSError, ValueError) as exc: + if _caused_by_missing_path(exc): + try: + if profile is not None and logs_fd >= 0: + _confirm_marker_missing(profile, logs_fd) + elif profile is not None: + profile.confirm_missing("logs") + else: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from exc + except (LaunchPayloadError, OSError, ValueError) as confirm_error: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from confirm_error + return {"exists": False} + raise UnsafeFileError("PID marker cannot be opened safely") from exc + if marker_stat.st_uid != os.geteuid() or marker_stat.st_nlink != 1: + raise UnsafeFileError("PID marker is not a private regular file") + marker_mode = f"{stat.S_IMODE(marker_stat.st_mode):04o}" + try: + raw = hooks.read_bounded_file(marker_fd) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + try: + _validate_marker_bindings(profile, logs_fd, marker_fd, marker_stat) + except (LaunchPayloadError, OSError, TypeError, ValueError) as binding_error: + raise UnsafeFileError( + "PID marker changed while it was inspected" + ) from binding_error + return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} + try: + current_marker = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + raise UnsafeFileError("PID marker changed while it was inspected") from exc + if ( + not stat.S_ISREG(current_marker.st_mode) + or current_marker.st_uid != os.geteuid() + or current_marker.st_nlink != 1 + or not _same_identity(marker_stat, current_marker) + ): + raise UnsafeFileError("PID marker changed while it was inspected") + finally: + for descriptor in (marker_fd, logs_fd): + if descriptor >= 0: + with contextlib.suppress(OSError): + hooks.close(descriptor) + if profile is not None: + profile.close() + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} + if not isinstance(payload, dict): + return { + "exists": True, + "valid": False, + "mode": marker_mode, + "error": "pid marker must be a JSON object", + } + deprecated = payload.get("schema") is None + safe_payload: dict[str, object] = { + "exists": True, + "valid": True, + "mode": marker_mode, + "deprecated": deprecated, + } + for key in ( + "schema", + "pid", + "bot_id", + "component", + "action", + "started_at", + "proc_start_fingerprint", + ): + if key in payload: + safe_payload[key] = payload[key] + if "readiness_probe" in payload: + try: + readiness = readiness_probe_from_payload(payload["readiness_probe"]) + except ValueError: + safe_payload["readiness_probe"] = "invalid" + else: + safe_payload["readiness_probe"] = readiness_probe_to_payload(readiness) + argv_value = payload.get("argv") + if isinstance(argv_value, list) and all(isinstance(part, str) for part in argv_value): + safe_payload["argv_shape"] = process_identity.safe_command_shape(argv_value) + return safe_payload + + def verify_gateway_pid_ownership( + self, + profile_path: str, + pid: int, + bot_id: str, + *, + expected_record: BotRecord | None, + ) -> OwnershipCheck: + if expected_record is not None and expected_record.profile_path != profile_path: + return OwnershipCheck(False, "marker-mismatch") + observed = self.read_strict_runtime_marker(bot_id, profile_path) + if observed.kind == "missing": + return OwnershipCheck(False, "marker-missing") + if observed.kind != "present" or observed.payload is None: + return OwnershipCheck(False, "marker-mismatch") + payload = observed.payload + if payload.get("schema") == 3: + if expected_record is None: + return OwnershipCheck(False, "marker-mismatch") + marker = self.classify_schema3_runtime_marker( + expected_record, + payload, + expected_pid=pid, + require_live_command=True, + ) + if marker.kind != "live": + return OwnershipCheck(False, marker.reason or "marker-mismatch") + live_argv = self.cmdline_reader(pid) + if not live_argv: + return OwnershipCheck(False, "live-cmdline-missing") + live_check = process_identity.verify_gateway_command( + live_argv, + bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + return OwnershipCheck( + live_check.verified, + live_check.reason, + live_check.classification, + ) + if payload.get("pid") != pid: + return OwnershipCheck(False, "marker-mismatch") + argv_value = payload.get("argv") + if not isinstance(argv_value, list) or not all( + isinstance(part, str) for part in argv_value + ): + return OwnershipCheck(False, "marker-mismatch") + trusted_hermes = self.resolved_hermes_bin() + if trusted_hermes is None: + return OwnershipCheck(False, "untrusted-executable") + marker_check = self.verify_marker_payload(payload, list(argv_value), bot_id) + if not marker_check.verified: + return OwnershipCheck(False, marker_check.reason, marker_check.classification) + live_argv = self.cmdline_reader(pid) + if not live_argv: + return OwnershipCheck(False, "live-cmdline-missing") + live_check = process_identity.verify_gateway_command( + live_argv, + bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + if not live_check.verified: + return OwnershipCheck(False, live_check.reason, live_check.classification) + marker_schema = payload.get("schema") + fingerprint = payload.get("proc_start_fingerprint") + if marker_schema == 2: + live_fingerprint = self.proc_start_fingerprint_reader(pid) + if live_fingerprint and not (isinstance(fingerprint, str) and fingerprint): + return OwnershipCheck(False, "pid-start-time-missing", live_check.classification) + if isinstance(fingerprint, str) and fingerprint and live_fingerprint != fingerprint: + return OwnershipCheck(False, "pid-start-time-mismatch", live_check.classification) + elif isinstance(fingerprint, str) and fingerprint: + live_fingerprint = self.proc_start_fingerprint_reader(pid) + if live_fingerprint != fingerprint: + return OwnershipCheck(False, "pid-start-time-mismatch", live_check.classification) + classification = ( + "legacy-marker-valid" + if marker_check.classification == "legacy-marker-valid" + else live_check.classification + ) + return OwnershipCheck(True, "ok", classification) + + def verify_marker_payload( + self, + payload: dict[str, object], + argv: list[str], + bot_id: str, + ) -> OwnershipCheck: + schema = payload.get("schema") + if schema == 3: + pid = payload.get("pid") + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + fingerprint = payload.get("command_fingerprint") + if ( + type(pid) is not int + or pid <= 0 + or type(operation_id) is not str + or len(operation_id) != 32 + or any(character not in "0123456789abcdef" for character in operation_id) + or type(revision) is not int + or revision <= 0 + or type(fingerprint) is not str + ): + return OwnershipCheck(False, "marker-mismatch") + if not is_owned_runtime_marker( + payload, + bot_id=bot_id, + operation_id=operation_id, + desired_revision=revision, + pid=pid, + expected_fingerprint=fingerprint, + ): + return OwnershipCheck(False, "marker-mismatch") + resolved_hermes_bin = self.resolved_hermes_bin() + marker_hermes = payload.get("resolved_hermes_bin") + if ( + resolved_hermes_bin is None + or type(marker_hermes) is not str + or process_identity.resolve_executable(marker_hermes) != resolved_hermes_bin + ): + return OwnershipCheck(False, "untrusted-executable") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + resolved_hermes_bin, + require_trusted_path=True, + ) + return OwnershipCheck( + marker_check.verified, + marker_check.reason, + marker_check.classification, + ) + if schema == 2: + if payload.get("bot_id") != bot_id: + return OwnershipCheck(False, "wrong-bot-id") + if payload.get("component") != "gateway" or payload.get("action") != "run": + return OwnershipCheck(False, "wrong-command-intent") + resolved_hermes_bin = self.resolved_hermes_bin() + if not isinstance(payload.get("resolved_hermes_bin"), str): + return OwnershipCheck(False, "untrusted-executable") + marker_hermes = process_identity.resolve_executable(str(payload["resolved_hermes_bin"])) + if marker_hermes != resolved_hermes_bin: + return OwnershipCheck(False, "untrusted-executable") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + resolved_hermes_bin, + require_trusted_path=True, + ) + return OwnershipCheck( + marker_check.verified, + marker_check.reason, + marker_check.classification, + ) + if schema is not None: + return OwnershipCheck(False, "marker-mismatch") + if not self.allow_legacy_pid_markers: + return OwnershipCheck(False, "legacy-marker-disabled") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + None, + require_trusted_path=False, + ) + if not marker_check.verified: + return OwnershipCheck(False, marker_check.reason, marker_check.classification) + return OwnershipCheck(True, "ok", "legacy-marker-valid") + + def resolved_hermes_bin(self) -> str | None: + return process_identity.resolve_executable(self.adapter.hermes_bin) + + def trusted_hermes_bins(self) -> set[str]: + return process_identity.trusted_hermes_paths(self.adapter.hermes_bin) + + def terminate_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + ) -> bool: + if process.poll() is not None: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + if self.spawned_tree_stopped(process, timeout=0): + return True + term_result = self.signal_spawned_process(process, signal.SIGTERM, cleanup_errors) + if term_result is SignalResult.missing: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + return self.spawned_tree_stopped(process, timeout=0) + if term_result is SignalResult.denied: + return False + self.reap_spawned_process(process, cleanup_errors, timeout=self.stop_grace_seconds) + if self.spawned_tree_stopped(process, timeout=0): + return True + kill_result = self.signal_spawned_process(process, signal.SIGKILL, cleanup_errors) + if kill_result is SignalResult.missing: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + return self.spawned_tree_stopped(process, timeout=0) + if kill_result is SignalResult.denied: + return False + self.reap_spawned_process(process, cleanup_errors, timeout=self.stop_grace_seconds) + return self.spawned_tree_stopped(process, timeout=self.stop_grace_seconds) + + def signal_spawned_process( + self, + process: PopenLike, + sig: signal.Signals, + cleanup_errors: list[str], + ) -> SignalResult: + if self.cleanup_process_group: + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + return SignalResult.missing + except PermissionError as exc: + cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") + return SignalResult.denied + raise + return SignalResult.sent + method_name = "terminate" if sig == signal.SIGTERM else "kill" + method = getattr(process, method_name, None) + if not callable(method): + return self.send_signal(process.pid, sig) + try: + method() + except ProcessLookupError: + return SignalResult.missing + except PermissionError as exc: + cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") + return SignalResult.denied + raise + return SignalResult.sent + + def reap_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + *, + timeout: float, + ) -> bool: + wait = getattr(process, "wait", None) + if callable(wait): + try: + wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + except Exception as exc: + cleanup_errors.append(f"wait: {type(exc).__name__}: {exc}") + return ( + process.poll() is not None + or self.pid_state(process.pid) is process_identity.PidState.dead + ) + + def spawned_tree_stopped(self, process: PopenLike, *, timeout: float) -> bool: + if not self.cleanup_process_group: + return ( + process.poll() is not None + or self.pid_state(process.pid) is process_identity.PidState.dead + ) + deadline = time.monotonic() + timeout + while True: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return True + except PermissionError: + return False + except OSError as exc: + return exc.errno == errno.ESRCH + if time.monotonic() >= deadline: + return False + time.sleep(0.05) + + def wait_for_exit(self, bot_id: str, pid: int) -> bool: + process = self._processes.get(bot_id) + if process is not None and hasattr(process, "wait"): + try: + process.wait(timeout=self.stop_grace_seconds) + return True + except subprocess.TimeoutExpired: + return False + except Exception: + return False + deadline = time.monotonic() + self.stop_grace_seconds + while ( + self.pid_state(pid) is not process_identity.PidState.dead + and time.monotonic() < deadline + ): + time.sleep(0.1) + return self.pid_state(pid) is process_identity.PidState.dead + + def poll_startup(self, process: PopenLike) -> int | None: + returncode = process.poll() + if returncode is not None or self.startup_grace_seconds <= 0: + return returncode + deadline = time.monotonic() + self.startup_grace_seconds + while time.monotonic() < deadline: + time.sleep(min(0.01, max(deadline - time.monotonic(), 0))) + returncode = process.poll() + if returncode is not None: + return returncode + return process.poll() diff --git a/zeus/hermes_adapter.py b/zeus/hermes_adapter.py index a074b8d..61cbcc3 100644 --- a/zeus/hermes_adapter.py +++ b/zeus/hermes_adapter.py @@ -7,7 +7,7 @@ from pathlib import Path from zeus.envfile import parse_env_text -from zeus.gateway_launcher import command_fingerprint +from zeus.gateway_marker import command_fingerprint, readiness_probe_to_payload from zeus.models import ID_RE from zeus.readiness import ReadinessProbe @@ -88,15 +88,6 @@ def launcher_payload( exec_argv = [resolved_hermes, *argv[1:]] profile_path = self.hermes_root / "profiles" / bot_id marker_path = profile_path / "logs" / "zeus-gateway.pid.json" - probe_payload: dict[str, object] | None = None - if readiness_probe is not None: - probe_payload = { - "url": readiness_probe.url, - "expected_status": readiness_probe.expected_status, - "expected_platform": readiness_probe.expected_platform, - "timeout_seconds": readiness_probe.timeout_seconds, - "interval_seconds": readiness_probe.interval_seconds, - } marker: dict[str, object] = { "schema": 3, "bot_id": bot_id, @@ -107,7 +98,7 @@ def launcher_payload( "argv": exec_argv, "resolved_hermes_bin": resolved_hermes, "command_fingerprint": command_fingerprint(exec_argv), - "readiness_probe": probe_payload, + "readiness_probe": readiness_probe_to_payload(readiness_probe), } return { "profile_path": str(profile_path), diff --git a/zeus/idempotency_store.py b/zeus/idempotency_store.py new file mode 100644 index 0000000..09396dc --- /dev/null +++ b/zeus/idempotency_store.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from dataclasses import dataclass +from datetime import UTC, datetime + +from zeus.idempotency import IdempotencyClaim +from zeus.sqlite_db import SQLiteDatabase + +IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") +IDEMPOTENCY_OWNER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$", re.ASCII) +MAX_IDEMPOTENCY_RESPONSE_BYTES = 1_000_000 + + +def _validate_idempotency_hash(value: str, label: str) -> str: + if not isinstance(value, str): + raise TypeError(f"idempotency {label} must be a string") + if IDEMPOTENCY_HASH_RE.fullmatch(value) is None: + raise ValueError(f"idempotency {label} is invalid") + return value + + +def _validate_idempotency_owner(value: str) -> str: + if not isinstance(value, str): + raise TypeError("idempotency owner must be a string") + if IDEMPOTENCY_OWNER_RE.fullmatch(value) is None: + raise ValueError("idempotency owner is invalid") + return value + + +def _validate_idempotency_timestamp(value: datetime, label: str) -> str: + if not isinstance(value, datetime): + raise TypeError(f"idempotency {label} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"idempotency {label} is invalid") + return value.astimezone(UTC).isoformat() + + +def _validate_idempotency_status(value: int) -> int: + if type(value) is not int or not 200 <= value <= 599: + raise ValueError("idempotency response status is invalid") + return value + + +def _reject_non_finite_json(_value: str) -> object: + raise ValueError("idempotency response JSON is invalid") + + +def _validate_idempotency_response_json(value: str) -> str: + if not isinstance(value, str): + raise TypeError("idempotency response JSON must be a string") + try: + encoded = value.encode("utf-8") + except UnicodeError: + raise ValueError("idempotency response JSON is invalid") from None + if len(encoded) > MAX_IDEMPOTENCY_RESPONSE_BYTES: + raise ValueError("idempotency response JSON is too large") + try: + json.loads(value, parse_constant=_reject_non_finite_json) + except (json.JSONDecodeError, TypeError, ValueError, RecursionError): + raise ValueError("idempotency response JSON is invalid") from None + return value + + +def _parse_idempotency_timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str): + raise TypeError(f"stored idempotency {label} must be a string") + try: + parsed = datetime.fromisoformat(value) + except ValueError: + raise ValueError(f"stored idempotency {label} is invalid") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError(f"stored idempotency {label} is invalid") + normalized = parsed.astimezone(UTC) + if normalized.isoformat() != value: + raise ValueError(f"stored idempotency {label} is invalid") + return normalized + + +@dataclass(frozen=True) +class _ValidatedIdempotencyRow: + request_hash: str + state: str + owner_instance_id: str + response_status: int | None + response_json: str | None + created_at: datetime + updated_at: datetime + expires_at: datetime + + +def _validate_idempotency_row(row: sqlite3.Row) -> _ValidatedIdempotencyRow: + _validate_idempotency_hash(row["key_hash"], "stored key hash") + request_hash = _validate_idempotency_hash(row["request_hash"], "stored request hash") + owner = _validate_idempotency_owner(row["owner_instance_id"]) + state = row["state"] + if state not in {"in_progress", "completed"}: + raise ValueError("stored idempotency state is invalid") + created_at = _parse_idempotency_timestamp(row["created_at"], "creation timestamp") + updated_at = _parse_idempotency_timestamp(row["updated_at"], "update timestamp") + expires_at = _parse_idempotency_timestamp(row["expires_at"], "expiry timestamp") + response_status = row["response_status"] + response_json = row["response_json"] + if state == "in_progress": + if created_at > updated_at or updated_at > expires_at: + raise ValueError("stored in-progress idempotency timestamps are invalid") + if response_status is not None or response_json is not None: + raise ValueError("stored in-progress idempotency response is invalid") + safe_status = None + safe_json = None + else: + if created_at > updated_at or updated_at > expires_at: + raise ValueError("stored completed idempotency timestamps are invalid") + if response_status is None or response_json is None: + raise ValueError("stored completed idempotency response is incomplete") + safe_status = _validate_idempotency_status(response_status) + safe_json = _validate_idempotency_response_json(response_json) + return _ValidatedIdempotencyRow( + request_hash=request_hash, + state=state, + owner_instance_id=owner, + response_status=safe_status, + response_json=safe_json, + created_at=created_at, + updated_at=updated_at, + expires_at=expires_at, + ) + + +def _idempotency_claim_from_row( + row: sqlite3.Row, + *, + request_hash: str, + owner_instance_id: str, +) -> IdempotencyClaim: + validated = _validate_idempotency_row(row) + if validated.request_hash != request_hash: + return IdempotencyClaim("conflict") + if validated.state == "completed": + if validated.response_status is None or validated.response_json is None: + raise AssertionError("validated completed idempotency response is incomplete") + return IdempotencyClaim( + "replay", + validated.response_status, + validated.response_json, + ) + if validated.owner_instance_id == owner_instance_id: + return IdempotencyClaim("in_progress") + return IdempotencyClaim("indeterminate") + + +class IdempotencyStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def claim_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + expires_at: datetime, + max_records: int = 10_000, + ) -> IdempotencyClaim: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") + if type(max_records) is not int or not 1 <= max_records <= 1_000_000: + raise ValueError("idempotency capacity is invalid") + current_time = datetime.now(UTC) + if datetime.fromisoformat(safe_expiry) < current_time: + raise ValueError("idempotency expiry timestamp is invalid") + now = current_time.isoformat() + + try: + conn = self._database.connect() + except (OSError, sqlite3.Error): + return IdempotencyClaim("unavailable") + with closing(conn): + try: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is not None: + validated = _validate_idempotency_row(row) + reclaimable = validated.expires_at <= current_time and ( + validated.state == "completed" or validated.owner_instance_id != safe_owner + ) + if reclaimable: + conn.execute( + "DELETE FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ) + row = None + conn.execute( + """ + DELETE FROM idempotency_records + WHERE expires_at <= ? + AND (state = 'completed' OR owner_instance_id != ?) + AND key_hash != ? + """, + (now, safe_owner, safe_key_hash), + ) + if row is None: + count = int( + conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] + ) + if count >= max_records: + conn.commit() + return IdempotencyClaim("unavailable") + conn.execute( + """ + INSERT INTO idempotency_records ( + key_hash, + request_hash, + state, + owner_instance_id, + response_status, + response_json, + created_at, + updated_at, + expires_at + ) VALUES (?, ?, 'in_progress', ?, NULL, NULL, ?, ?, ?) + """, + ( + safe_key_hash, + safe_request_hash, + safe_owner, + now, + now, + safe_expiry, + ), + ) + conn.commit() + return IdempotencyClaim("claimed") + + result = _idempotency_claim_from_row( + row, + request_hash=safe_request_hash, + owner_instance_id=safe_owner, + ) + conn.commit() + return result + except (OSError, sqlite3.Error, ValueError, TypeError): + conn.rollback() + return IdempotencyClaim("unavailable") + + def lookup_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + ) -> IdempotencyClaim | None: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + current_time = datetime.now(UTC) + + try: + conn = self._database.connect() + except (OSError, sqlite3.Error): + return IdempotencyClaim("unavailable") + with closing(conn): + try: + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is None: + return None + validated = _validate_idempotency_row(row) + if validated.expires_at <= current_time and ( + validated.state == "completed" or validated.owner_instance_id != safe_owner + ): + return None + return _idempotency_claim_from_row( + row, + request_hash=safe_request_hash, + owner_instance_id=safe_owner, + ) + except (OSError, sqlite3.Error, ValueError, TypeError): + return IdempotencyClaim("unavailable") + + def complete_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + response_status: int, + response_json: str, + completed_at: datetime, + expires_at: datetime, + ) -> None: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + safe_status = _validate_idempotency_status(response_status) + safe_json = _validate_idempotency_response_json(response_json) + safe_completed = _validate_idempotency_timestamp(completed_at, "completion timestamp") + safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") + if safe_expiry < safe_completed: + raise ValueError("idempotency expiry timestamp is invalid") + completed_time = datetime.fromisoformat(safe_completed) + + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is None: + raise RuntimeError("idempotency completion failed") + validated = _validate_idempotency_row(row) + if ( + validated.request_hash != safe_request_hash + or validated.owner_instance_id != safe_owner + or validated.state != "in_progress" + ): + raise RuntimeError("idempotency completion failed") + if completed_time < validated.created_at: + raise ValueError("idempotency completion timestamp is invalid") + cursor = conn.execute( + """ + UPDATE idempotency_records + SET state = 'completed', + response_status = ?, + response_json = ?, + updated_at = ?, + expires_at = ? + WHERE key_hash = ? + AND request_hash = ? + AND owner_instance_id = ? + AND state = 'in_progress' + """, + ( + safe_status, + safe_json, + safe_completed, + safe_expiry, + safe_key_hash, + safe_request_hash, + safe_owner, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("idempotency completion failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() diff --git a/zeus/intent_recovery.py b/zeus/intent_recovery.py new file mode 100644 index 0000000..9792c05 --- /dev/null +++ b/zeus/intent_recovery.py @@ -0,0 +1,630 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Protocol, TypeVar + +from zeus.errors import BotDeleteError +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import MarkerObservation, StopEffect +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, + TemplateError, +) +from zeus.process_identity import PidState +from zeus.readiness import ReadinessProbe, ReadinessResult + +_ContextT = TypeVar("_ContextT") + + +class _RecoveryHost(Protocol[_ContextT]): + def _recovery_lifecycle_context( + self, + operation_id: str, + context: _ContextT, + ) -> _ContextT: ... + + def _pending_action_required( + self, + record: BotRecord, + reason: str, + ) -> BotStatusResponse: ... + + def _pending_launch_preflight( + self, + record: BotRecord, + operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: ... + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: ... + + def _recover_pending_restart_predecessor( + self, + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse | None: ... + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _ContextT, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: ... + + def _matching_runtime_marker( + self, + record: BotRecord, + *, + expected_fingerprint: str, + require_live_command: bool, + ) -> MarkerObservation: ... + + def _probe_once(self, probe: ReadinessProbe) -> ReadinessResult: ... + + def _complete_started_intent( + self, + record: BotRecord, + *, + context: _ContextT, + status: BotStatus, + pid: int, + reason: str, + ready_at: datetime | None = ..., + last_error: str | None = ..., + reset_restart: bool = ..., + ) -> BotRecord: ... + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + context: _ContextT, + probe: ReadinessProbe | None, + ) -> BotStatusResponse: ... + + def _pid_state(self, pid: int) -> PidState: ... + + def _read_strict_runtime_marker( + self, + bot_id: str, + profile_path: str, + ) -> MarkerObservation: ... + + def _remove_owned_launch_marker_locked( + self, + record: BotRecord, + *, + observed: MarkerObservation, + ) -> bool: ... + + def _complete_stopped_intent( + self, + record: BotRecord, + *, + context: _ContextT, + reason: str, + ) -> BotRecord: ... + + def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: ... + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _ContextT, + complete_stop: bool, + ) -> BotStatusResponse: ... + + def _is_compat_runtime_marker(self, payload: dict[str, object]) -> bool: ... + + def _pending_restart_old_marker( + self, + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: ... + + def _classify_schema3_runtime_marker( + self, + record: BotRecord, + payload: dict[str, object], + *, + expected_pid: int | None, + expected_revision: int, + require_live_command: bool, + ) -> MarkerObservation: ... + + def _recover_pending_restart_old_gateway( + self, + record: BotRecord, + marker: MarkerObservation, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: ... + + def _gateway_generation( + self, + marker: MarkerObservation, + ) -> GatewayGeneration | None: ... + + def _stop_pending_restart_old_gateway( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + context: _ContextT, + ) -> BotStatusResponse: ... + + def _remove_gateway_generation_marker_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: ... + + def _update_lifecycle( + self, + context: _ContextT, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def _stop_gateway_generation_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> StopEffect: ... + + def _append_recovery_audit_event(self, action: str, **values: object) -> None: ... + + +class PendingIntentRecovery: + def recover( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_launch: bool, + ) -> BotStatusResponse: + action = record.pending_action + operation_id = record.pending_operation_id + if action is None or operation_id is None: + return host._pending_action_required( + record, + "pending lifecycle correlation is invalid", + ) + recovery_context = host._recovery_lifecycle_context(operation_id, context) + if action == "stop": + return host._recover_pending_stop_intent( + record, + context=recovery_context, + allow_stop=allow_launch, + ) + if action not in {"start", "restart"}: + return host._pending_action_required( + record, + "pending lifecycle action is invalid", + ) + try: + probe, fingerprint = host._pending_launch_preflight(record, operation_id) + except (OSError, ValueError, TemplateError, BotDeleteError) as exc: + return host._pending_action_required(record, f"launch preflight failed: {exc}") + if action == "restart": + predecessor_result = host._recover_pending_restart_predecessor( + record, + context=recovery_context, + allow_stop=allow_launch, + ) + if predecessor_result is not None: + return predecessor_result + pending_result = host._recover_pending_launch( + record, + context=recovery_context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, + ) + if pending_result is not None: + return pending_result + return host._start_record( + record, + reset_restart=action == "restart", + message="recovered interrupted gateway launch", + context=recovery_context, + probe=probe, + ) + + def recover_pending_launch_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: + marker = host._matching_runtime_marker( + record, + expected_fingerprint=fingerprint, + require_live_command=True, + ) + if marker.kind == "live" and marker.payload is not None: + marker_pid = marker.payload["pid"] + if isinstance(marker_pid, bool) or not isinstance(marker_pid, int): + return host._pending_action_required(record, "live marker PID is invalid") + status = BotStatus.starting if probe is not None else BotStatus.running + ready_at = None if probe is not None else datetime.now(UTC) + if probe is not None: + readiness = host._probe_once(probe) + if readiness.ready: + status = BotStatus.running + ready_at = datetime.now(UTC) + try: + host._complete_started_intent( + record, + context=context, + status=status, + pid=marker_pid, + ready_at=ready_at, + reset_restart=True, + reason="recovery adopted registered gateway", + ) + except Exception: + return host._pending_action_required( + record, + "gateway adoption could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + status, + marker_pid, + record.profile_path, + "recovered registered gateway", + ) + if marker.kind == "untrusted": + return host._pending_action_required(record, marker.reason) + if not allow_launch: + return host._pending_action_required( + record, + "desired running gateway is missing; reconcile is required", + ) + if marker.kind == "dead": + generation = host._gateway_generation(marker) + if generation is None or not host._remove_gateway_generation_marker_locked( + record, + generation, + ): + return host._pending_action_required( + record, + "dead gateway marker cleanup failed", + ) + if action == "restart": + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: removed dead gateway marker; launch on next reconcile", + ) + return None + + def recover_pending_stop_intent_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: + pid_state = host._pid_state(record.pid) if record.pid else PidState.dead + if pid_state is PidState.unknown: + return host._pending_action_required(record, "gateway PID liveness is unknown") + if not record.pid or pid_state is PidState.dead: + observed = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if not host._remove_owned_launch_marker_locked(record, observed=observed): + return host._pending_action_required( + record, + "stale gateway marker ownership could not be verified", + ) + try: + host._complete_stopped_intent( + record, + context=context, + reason="recovery confirmed gateway stopped", + ) + except Exception: + return host._pending_action_required( + record, + "stopped state could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.stopped, + None, + record.profile_path, + "recovered interrupted stop", + ) + if not allow_stop: + if not host._pid_owned(record.profile_path, record.pid, record.bot_id): + return host._pending_action_required( + record, + "gateway ownership could not be verified", + ) + return host._pending_action_required( + record, + "stop intent is pending reconciliation", + ) + return host._stop_record_effect_locked( + record, + kill_after_timeout=None, + context=context, + complete_stop=True, + ) + + def pending_restart_old_marker( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: + if observed is None: + observed = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return None + payload = observed.payload + marker_operation = payload.get("operation_id") + marker_revision = payload.get("desired_revision") + if ( + type(marker_operation) is not str + or marker_operation == record.pending_operation_id + or type(marker_revision) is not int + or marker_revision != record.desired_revision - 1 + ): + return None + return host._classify_schema3_runtime_marker( + record, + payload, + expected_pid=record.pid, + expected_revision=record.desired_revision - 1, + require_live_command=True, + ) + + def recover_pending_restart_predecessor_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse | None: + predecessor = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if ( + record.pid is not None + and predecessor.kind == "present" + and predecessor.payload is not None + and host._is_compat_runtime_marker(predecessor.payload) + ): + return host._pending_action_required( + record, + "schema-v2 or legacy gateway restart requires manual process resolution", + ) + old_marker = host._pending_restart_old_marker(record, predecessor) + if old_marker is not None: + return host._recover_pending_restart_old_gateway( + record, + old_marker, + context=context, + allow_stop=allow_stop, + ) + if record.pid is None or predecessor.kind != "missing": + return None + pid_state = host._pid_state(record.pid) + if pid_state is PidState.unknown: + return host._pending_action_required( + record, + "previous gateway PID liveness is unknown", + ) + if pid_state is PidState.alive: + return host._pending_action_required(record, "previous gateway marker is missing") + if not allow_stop: + return host._pending_action_required( + record, + "restart intent is pending reconciliation", + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_recovered", + stopped_at=datetime.now(UTC), + last_transition_reason="recovery confirmed the previous gateway stopped", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: recovered stopped gateway; launch on next reconcile", + ) + + def recover_pending_restart_old_gateway( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + marker: MarkerObservation, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: + if marker.kind == "untrusted": + return host._pending_action_required( + record, + marker.reason or "previous gateway ownership could not be verified", + ) + if not allow_stop: + return host._pending_action_required( + record, + "restart intent is pending reconciliation", + ) + generation = host._gateway_generation(marker) + if generation is None: + return host._pending_action_required( + record, + "previous gateway marker correlation is invalid", + ) + if marker.kind == "live": + if record.pid is None: + return host._pending_action_required( + record, + "previous gateway PID is not recorded", + ) + stopped = host._stop_pending_restart_old_gateway( + record, + generation, + context=context, + ) + if stopped.status is not BotStatus.stopped: + return stopped + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: previous gateway stopped; launch on next reconcile", + ) + if marker.kind != "dead" or marker.payload is None: + return host._pending_action_required( + record, + "previous gateway marker ownership could not be verified", + ) + if not host._remove_gateway_generation_marker_locked(record, generation): + return host._pending_action_required( + record, + "previous gateway marker cleanup could not be verified", + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_recovered", + stopped_at=datetime.now(UTC), + last_transition_reason="recovery confirmed the previous gateway stopped", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: recovered stopped gateway; launch on next reconcile", + ) + + def stop_pending_restart_old_gateway( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + generation: GatewayGeneration, + *, + context: _ContextT, + ) -> BotStatusResponse: + effect = host._stop_gateway_generation_locked(record, generation) + if effect.outcome != "stopped": + if effect.kill_result is not None: + host._append_recovery_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=generation.pid, + succeeded=bool(effect.kill_succeeded), + ) + reasons = { + "term_denied": "could not send SIGTERM to the previous gateway", + "kill_denied": "could not send SIGKILL to the previous gateway", + "grace_expired": "previous gateway did not stop before the grace period expired", + "cleanup_unverified": ("previous gateway marker cleanup could not be verified"), + } + return host._pending_action_required( + record, + reasons.get(effect.outcome, effect.reason), + ) + if effect.kill_result is not None: + host._append_recovery_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=generation.pid, + succeeded=bool(effect.kill_succeeded), + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_stopped", + stopped_at=datetime.now(UTC), + last_transition_reason="restart stopped the previous gateway", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + host._append_recovery_audit_event( + "bot.stop", + bot_id=record.bot_id, + pid=generation.pid, + ) + return BotStatusResponse( + record.bot_id, + BotStatus.stopped, + None, + record.profile_path, + "gateway shutdown completed", + ) diff --git a/zeus/lifecycle.py b/zeus/lifecycle.py index 87a3503..f54c18b 100644 --- a/zeus/lifecycle.py +++ b/zeus/lifecycle.py @@ -1,114 +1,33 @@ from __future__ import annotations import json -import math -import re from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime from types import MappingProxyType -from zeus.logging_utils import redact_secrets +from zeus.sanitization import JSONValue, sanitize_details, sanitize_text -MAX_DETAILS_JSON_LENGTH = 8192 -MAX_DETAIL_DEPTH = 4 -MAX_DETAIL_ITEMS = 32 -MAX_DETAIL_STRING_LENGTH = 2048 MAX_EVENT_TEXT_LENGTH = 2048 -_CAMEL_ACRONYM_BOUNDARY_RE = re.compile(r"([A-Z]+)([A-Z][a-z])") -_CAMEL_WORD_BOUNDARY_RE = re.compile(r"([a-z0-9])([A-Z])") -_NON_NAME_CHARACTER_RE = re.compile(r"[^a-z0-9]+") -_SECRET_NAME_PARTS = frozenset({"apikey", "key", "token", "secret", "password"}) -_FORBIDDEN_DETAIL_NAMES = frozenset( - { - "authorization", - "authorization_header", - "authorization_headers", - "header", - "headers", - "request_body", - "response_body", - "body", - "raw_query", - "query_string", - "query", - "forwarded_for", - "x_forwarded_for", - "client_address", - "client_ip", - "client_port", - "remote_addr", - "traceback", - "exception_trace", - "idempotency_key", - } -) - def _safe_text(value: str, *, maximum: int = MAX_EVENT_TEXT_LENGTH) -> str: - return redact_secrets(value)[:maximum] - - -def _normalize_detail_name(key: str) -> str: - separated = _CAMEL_ACRONYM_BOUNDARY_RE.sub(r"\1_\2", key) - separated = _CAMEL_WORD_BOUNDARY_RE.sub(r"\1_\2", separated).lower() - return _NON_NAME_CHARACTER_RE.sub("_", separated).strip("_") - - -def _contains_name(normalized: str, candidate: str) -> bool: - return f"_{candidate}_" in f"_{normalized}_" - - -def _is_forbidden_detail_name(key: str) -> bool: - normalized = _normalize_detail_name(key) - if any(_contains_name(normalized, name) for name in _FORBIDDEN_DETAIL_NAMES): - return True - return any(_contains_name(normalized, part) for part in _SECRET_NAME_PARTS) - - -def _safe_detail_value(key: str, value: object, *, depth: int) -> object: - if _is_forbidden_detail_name(key): - return "[redacted]" - if depth >= MAX_DETAIL_DEPTH: - return "[truncated]" - if value is None or isinstance(value, bool | int): - return value - if isinstance(value, float): - return value if math.isfinite(value) else None - if isinstance(value, str): - return _safe_text(value, maximum=MAX_DETAIL_STRING_LENGTH) - if isinstance(value, Mapping): - result: dict[str, object] = {} - for raw_child_key, child_value in list(value.items())[:MAX_DETAIL_ITEMS]: - child_key = _safe_text(str(raw_child_key), maximum=128) - result[child_key] = _safe_detail_value( - child_key, - child_value, - depth=depth + 1, - ) - return result - if isinstance(value, list | tuple): - return [_safe_detail_value(key, item, depth=depth + 1) for item in value[:MAX_DETAIL_ITEMS]] - return "[unsupported]" - - -def safe_lifecycle_details(details: Mapping[str, object]) -> dict[str, object]: - result: dict[str, object] = {} - for raw_key, value in list(details.items())[:MAX_DETAIL_ITEMS]: - key = _safe_text(str(raw_key), maximum=128) - result[key] = _safe_detail_value(key, value, depth=0) - if len(json.dumps(result, sort_keys=True, separators=(",", ":"))) > MAX_DETAILS_JSON_LENGTH: - return {"truncated": True} + return sanitize_text(value, max_length=maximum) + + +def safe_lifecycle_details(details: Mapping[str, object]) -> dict[str, JSONValue]: + result = sanitize_details(details) + if type(result) is not dict: + raise TypeError("lifecycle event details must be a mapping") return result -def _freeze_detail_value(value: object) -> object: - if isinstance(value, Mapping): +def _freeze_detail_value(value: JSONValue) -> object: + if type(value) is dict: return MappingProxyType( - {str(key): _freeze_detail_value(child_value) for key, child_value in value.items()} + {key: _freeze_detail_value(child_value) for key, child_value in value.items()} ) - if isinstance(value, list | tuple): + if type(value) is list: return tuple(_freeze_detail_value(child_value) for child_value in value) return value @@ -120,27 +39,20 @@ def freeze_lifecycle_details(details: Mapping[str, object]) -> Mapping[str, obje return frozen -def _thaw_detail_value(value: object) -> object: - if isinstance(value, Mapping): - return {str(key): _thaw_detail_value(child_value) for key, child_value in value.items()} - if isinstance(value, list | tuple): - return [_thaw_detail_value(child_value) for child_value in value] - return value - - -def thaw_lifecycle_details(details: Mapping[str, object]) -> dict[str, object]: - return {str(key): _thaw_detail_value(value) for key, value in details.items()} +def thaw_lifecycle_details(details: Mapping[str, object]) -> dict[str, JSONValue]: + return safe_lifecycle_details(details) def serialize_lifecycle_details(details: Mapping[str, object]) -> str: return json.dumps( - thaw_lifecycle_details(safe_lifecycle_details(details)), + safe_lifecycle_details(details), sort_keys=True, separators=(",", ":"), + allow_nan=False, ) -def deserialize_lifecycle_details(value: str) -> dict[str, object]: +def deserialize_lifecycle_details(value: str) -> dict[str, JSONValue]: parsed = json.loads(value) if not isinstance(parsed, dict): raise ValueError("lifecycle event details must be a JSON object") diff --git a/zeus/logging_utils.py b/zeus/logging_utils.py index 63ca60a..6ef7502 100644 --- a/zeus/logging_utils.py +++ b/zeus/logging_utils.py @@ -1,40 +1,11 @@ from __future__ import annotations -import re from pathlib import Path -SECRET_KV_RE = re.compile( - r"""(?ix) - (?P["']?) - (?P[A-Z0-9_.-]*(?:API[_-]?KEY|KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_.-]*) - (?P=prefix) - (?P\s*[:=]\s*) - (?P - "(?:[^"\\]|\\.)*" | - '(?:[^'\\]|\\.)*' | - [^\s,}]+ - ) - """ -) -BEARER_RE = re.compile(r"(?i)(Authorization\s*:\s*Bearer\s+)([A-Za-z0-9._~+/=-]+)") - - -def redact_secrets(text: str) -> str: - text = BEARER_RE.sub(r"\1[redacted]", text) - return SECRET_KV_RE.sub( - lambda match: ( - f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" - f"{match.group('sep')}[redacted]" - ), - text, - ) +from zeus.private_io import read_private_tail +from zeus.sanitization import redact_secrets def tail_file(path: Path, max_bytes: int = 20_000) -> str: - if not path.exists(): - return "" - with path.open("rb") as handle: - handle.seek(0, 2) - size = handle.tell() - handle.seek(max(0, size - max_bytes)) - return redact_secrets(handle.read().decode("utf-8", errors="replace")) + data = read_private_tail(path, max_bytes) + return redact_secrets(data.decode("utf-8", errors="replace")) diff --git a/zeus/private_io.py b/zeus/private_io.py new file mode 100644 index 0000000..86629fb --- /dev/null +++ b/zeus/private_io.py @@ -0,0 +1,779 @@ +from __future__ import annotations + +import os +import stat +import sys +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, suppress +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import BinaryIO, cast + +_DIRECTORY_MODE = 0o700 +_FILE_MODE = 0o600 +_SECURITY_FLAGS = ("O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC", "O_NONBLOCK") +_REQUIRED_FUNCTIONS = ( + "close", + "dup", + "fchmod", + "fdopen", + "fstat", + "geteuid", + "lseek", + "lstat", + "mkdir", + "open", + "read", + "write", +) +_OPEN_DIR_FD_PROBE = os.open +_MKDIR_DIR_FD_PROBE = os.mkdir +# CPython advertises descriptor-relative lstat through os.stat on Linux even +# though os.lstat itself accepts dir_fd. Actual inspections still use lstat. +_LSTAT_DIR_FD_PROBE = os.stat + + +class UnsafeFileError(OSError): + pass + + +class _PrivatePathMissing(Exception): + pass + + +class _DirectoryRequirement(Enum): + identity = "identity" + exact_private = "exact_private" + inspect_private = "inspect_private" + + +@dataclass(frozen=True) +class _Platform: + euid: int + directory_flags: int + append_flags: int + read_flags: int + create_exclusive_flags: int + + +@dataclass(frozen=True) +class _OpenedDirectoryPath: + descriptors: tuple[int, ...] + names: tuple[str, ...] + requirements: tuple[_DirectoryRequirement, ...] + euid: int + + @property + def fd(self) -> int: + return self.descriptors[-1] + + def validate_bindings(self) -> None: + _validate_directory_bindings( + self.descriptors, + self.names, + self.requirements, + self.euid, + ) + + def confirm_missing(self, name: str) -> None: + self.validate_bindings() + try: + os.lstat(name, dir_fd=self.fd) + except FileNotFoundError: + self.validate_bindings() + return + except OSError as exc: + raise UnsafeFileError("missing private path could not be confirmed") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("missing private path cannot be inspected safely") from exc + raise UnsafeFileError("private path appeared while absence was confirmed") + + +@dataclass(frozen=True) +class _OpenedPrivateFile: + fd: int + parent_fd: int + name: str + identity: os.stat_result + platform: _Platform + + def validate_binding(self) -> None: + try: + opened = os.fstat(self.fd) + current = os.lstat(self.name, dir_fd=self.parent_fd) + snapshots = (self.identity, opened, current) + for snapshot in snapshots: + _validate_file_snapshot(snapshot, self.platform) + if not _same_files(snapshots): + raise UnsafeFileError("private file changed while it was used") + if any(stat.S_IMODE(snapshot.st_mode) != _FILE_MODE for snapshot in (opened, current)): + raise UnsafeFileError("private file does not have private permissions") + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("private file binding changed while it was used") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file binding cannot be inspected safely") from exc + + +@dataclass(frozen=True) +class _PinnedPrivateDirectory: + fd: int + identity: os.stat_result + platform: _Platform + + def validate_at(self, path: Path) -> None: + parts = _validate_path(path, file_path=False) + with _open_directory_path( + parts, + create=False, + tighten=False, + platform=self.platform, + ) as current: + try: + pinned = os.fstat(self.fd) + installed = os.fstat(current.fd) + snapshots = (self.identity, pinned, installed) + _validate_directory_snapshots(snapshots, "pinned private directory") + _validate_directory_requirement( + snapshots, + _DirectoryRequirement.exact_private, + self.platform.euid, + "pinned private directory", + ) + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("pinned private directory is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError( + "pinned private directory cannot be inspected safely" + ) from exc + + +def _required_flag(name: str, *, allow_zero: bool = False) -> int: + value = getattr(os, name, None) + if type(value) is not int or (not allow_zero and value == 0): + raise UnsafeFileError(f"required POSIX flag {name} is unavailable") + return value + + +def _require_platform() -> _Platform: + if os.name != "posix": + raise UnsafeFileError("descriptor-safe private file access requires POSIX") + for name in _REQUIRED_FUNCTIONS: + if not callable(getattr(os, name, None)): + raise UnsafeFileError(f"required POSIX primitive os.{name} is unavailable") + supported = getattr(os, "supports_dir_fd", ()) + for function, name in ( + (_OPEN_DIR_FD_PROBE, "open"), + (_MKDIR_DIR_FD_PROBE, "mkdir"), + (_LSTAT_DIR_FD_PROBE, "lstat"), + ): + if function not in supported: + raise UnsafeFileError(f"descriptor-relative os.{name} is unavailable") + security_flags = {name: _required_flag(name) for name in _SECURITY_FLAGS} + o_rdonly = _required_flag("O_RDONLY", allow_zero=True) + o_wronly = _required_flag("O_WRONLY") + o_append = _required_flag("O_APPEND") + o_creat = _required_flag("O_CREAT") + o_excl = _required_flag("O_EXCL") + euid = os.geteuid() + if isinstance(euid, bool) or not isinstance(euid, int) or euid < 0: + raise UnsafeFileError("effective UID is unavailable") + common_leaf = ( + security_flags["O_NOFOLLOW"] | security_flags["O_CLOEXEC"] | security_flags["O_NONBLOCK"] + ) + return _Platform( + euid=euid, + directory_flags=( + o_rdonly + | security_flags["O_DIRECTORY"] + | security_flags["O_NOFOLLOW"] + | security_flags["O_CLOEXEC"] + ), + append_flags=o_wronly | o_append | common_leaf, + read_flags=o_rdonly | common_leaf, + create_exclusive_flags=o_creat | o_excl, + ) + + +def nofollow_absolute_path(path: Path) -> Path: + """Return an absolute lexical path without resolving supplied components.""" + absolute = Path(os.path.abspath(path.expanduser())) + if ( + sys.platform == "darwin" + and len(absolute.parts) > 1 + and absolute.parts[1] in {"etc", "tmp", "var"} + ): + return Path("/private", *absolute.parts[1:]) + return absolute + + +def _validate_path(path: Path, *, file_path: bool) -> tuple[str, ...]: + if not isinstance(path, Path): + raise UnsafeFileError("private path must be a pathlib.Path") + if not path.is_absolute() or path.anchor != "/": + raise UnsafeFileError("private path must be an absolute POSIX path") + parts = path.parts[1:] + if not parts or (file_path and len(parts) < 2): + raise UnsafeFileError("private path cannot use the filesystem root") + if any(part in {"", ".", ".."} or "\0" in part for part in parts): + raise UnsafeFileError("private path contains an unsupported component") + return parts + + +def _same_file(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _same_files(results: tuple[os.stat_result, ...]) -> bool: + return all(_same_file(results[0], result) for result in results[1:]) + + +def _close_suppressing_error(fd: int) -> None: + with suppress(OSError): + os.close(fd) + + +def _close_descriptor(fd: int, description: str) -> None: + try: + os.close(fd) + except OSError as exc: + raise UnsafeFileError(f"{description} descriptor could not be closed") from exc + + +def _lstat_at(parent_fd: int, name: str, description: str) -> os.stat_result: + try: + return os.lstat(name, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be inspected safely") from exc + + +def _validate_directory_snapshots( + snapshots: tuple[os.stat_result, ...], + description: str, +) -> None: + if not all(stat.S_ISDIR(snapshot.st_mode) for snapshot in snapshots): + raise UnsafeFileError(f"{description} is not a directory") + if not _same_files(snapshots): + raise UnsafeFileError(f"{description} changed while it was opened") + + +def _validate_directory_bindings( + descriptors: tuple[int, ...], + names: tuple[str, ...], + requirements: tuple[_DirectoryRequirement, ...], + euid: int, +) -> None: + if len(descriptors) != len(names) + 1 or len(requirements) != len(descriptors): + raise UnsafeFileError("private directory descriptor chain is invalid") + try: + root_snapshots = (os.fstat(descriptors[0]), os.lstat("/")) + _validate_directory_snapshots(root_snapshots, "filesystem root") + _validate_directory_requirement( + root_snapshots, + requirements[0], + euid, + "filesystem root", + ) + for parent_fd, name, directory_fd, requirement in zip( + descriptors[:-1], + names, + descriptors[1:], + requirements[1:], + strict=True, + ): + description = f"private path component {name!r}" + snapshots = (os.fstat(directory_fd), os.lstat(name, dir_fd=parent_fd)) + _validate_directory_snapshots(snapshots, description) + _validate_directory_requirement(snapshots, requirement, euid, description) + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("private directory path binding changed") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private directory path cannot be inspected safely") from exc + + +def _validate_directory_requirement( + snapshots: tuple[os.stat_result, ...], + requirement: _DirectoryRequirement, + euid: int, + description: str, +) -> None: + if requirement is _DirectoryRequirement.identity: + return + if not all(snapshot.st_uid == euid for snapshot in snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + modes = tuple(stat.S_IMODE(snapshot.st_mode) for snapshot in snapshots) + if requirement is _DirectoryRequirement.exact_private: + unsafe = any(mode != _DIRECTORY_MODE for mode in modes) + else: + unsafe = any(mode & 0o077 for mode in modes) + if unsafe: + raise UnsafeFileError(f"{description} does not have private permissions") + + +def _tighten_directory( + parent_fd: int, + name: str, + directory_fd: int, + snapshots: tuple[os.stat_result, ...], + platform: _Platform, + description: str, +) -> None: + if not all(snapshot.st_uid == platform.euid for snapshot in snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + try: + os.fchmod(directory_fd, _DIRECTORY_MODE) + tightened = os.fstat(directory_fd) + current = os.lstat(name, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} permissions could not be tightened") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be validated safely") from exc + final_snapshots = (*snapshots, tightened, current) + _validate_directory_snapshots(final_snapshots, description) + if not all(snapshot.st_uid == platform.euid for snapshot in final_snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + if any(stat.S_IMODE(snapshot.st_mode) != _DIRECTORY_MODE for snapshot in (tightened, current)): + raise UnsafeFileError(f"{description} does not have private permissions") + + +def _open_root(platform: _Platform) -> int: + try: + before = os.lstat("/") + root_fd = os.open("/", platform.directory_flags) + except OSError as exc: + raise UnsafeFileError("filesystem root cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("filesystem root cannot be inspected safely") from exc + try: + opened = os.fstat(root_fd) + after = os.lstat("/") + _validate_directory_snapshots((before, opened, after), "filesystem root") + return root_fd + except UnsafeFileError: + _close_suppressing_error(root_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(root_fd) + raise UnsafeFileError("filesystem root changed while it was opened") from exc + except BaseException: + _close_suppressing_error(root_fd) + raise + + +def _open_directory_at( + parent_fd: int, + name: str, + *, + create: bool, + missing_ok: bool, + private: bool, + tighten: bool, + platform: _Platform, +) -> tuple[int, _DirectoryRequirement]: + description = f"private path component {name!r}" + created = False + try: + before = os.lstat(name, dir_fd=parent_fd) + except FileNotFoundError as exc: + if not create: + if missing_ok: + raise _PrivatePathMissing from exc + raise UnsafeFileError(f"{description} is unavailable") from exc + try: + os.mkdir(name, mode=_DIRECTORY_MODE, dir_fd=parent_fd) + created = True + except FileExistsError as race: + raise UnsafeFileError(f"{description} appeared while it was created") from race + except OSError as mkdir_error: + raise UnsafeFileError(f"{description} could not be created safely") from mkdir_error + except (TypeError, ValueError) as mkdir_error: + raise UnsafeFileError(f"{description} cannot be created safely") from mkdir_error + before = _lstat_at(parent_fd, name, description) + except OSError as exc: + raise UnsafeFileError(f"{description} is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be inspected safely") from exc + + if not stat.S_ISDIR(before.st_mode): + raise UnsafeFileError(f"{description} is not a directory") + try: + directory_fd = os.open(name, platform.directory_flags, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be opened safely") from exc + try: + opened = os.fstat(directory_fd) + after = os.lstat(name, dir_fd=parent_fd) + snapshots = (before, opened, after) + _validate_directory_snapshots(snapshots, description) + if created or (private and tighten): + _tighten_directory( + parent_fd, + name, + directory_fd, + snapshots, + platform, + description, + ) + requirement = _DirectoryRequirement.exact_private + elif private: + requirement = _DirectoryRequirement.inspect_private + _validate_directory_requirement( + snapshots, + requirement, + platform.euid, + description, + ) + else: + requirement = _DirectoryRequirement.identity + return directory_fd, requirement + except UnsafeFileError: + _close_suppressing_error(directory_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(directory_fd) + raise UnsafeFileError(f"{description} changed while it was opened") from exc + except BaseException: + _close_suppressing_error(directory_fd) + raise + + +@contextmanager +def _open_directory_path( + parts: tuple[str, ...], + *, + create: bool, + missing_ok: bool = False, + tighten: bool = True, + platform: _Platform, +) -> Iterator[_OpenedDirectoryPath]: + descriptors = [_open_root(platform)] + names: list[str] = [] + requirements = [_DirectoryRequirement.identity] + try: + for index, component in enumerate(parts): + try: + next_fd, requirement = _open_directory_at( + descriptors[-1], + component, + create=create, + missing_ok=missing_ok, + private=index == len(parts) - 1, + tighten=tighten, + platform=platform, + ) + except _PrivatePathMissing: + _OpenedDirectoryPath( + tuple(descriptors), + tuple(names), + tuple(requirements), + platform.euid, + ).confirm_missing(component) + raise + descriptors.append(next_fd) + names.append(component) + requirements.append(requirement) + except BaseException: + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) + raise + + opened = _OpenedDirectoryPath( + tuple(descriptors), + tuple(names), + tuple(requirements), + platform.euid, + ) + try: + yield opened + except BaseException: + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) + raise + else: + try: + opened.validate_bindings() + except BaseException: + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) + raise + close_error: UnsafeFileError | None = None + for descriptor in reversed(descriptors): + try: + _close_descriptor(descriptor, "private directory") + except UnsafeFileError as exc: + if close_error is None: + close_error = exc + if close_error is not None: + raise close_error + + +def _validate_file_snapshot(snapshot: os.stat_result, platform: _Platform) -> None: + if not stat.S_ISREG(snapshot.st_mode): + raise UnsafeFileError("private file is not a regular file") + if snapshot.st_uid != platform.euid: + raise UnsafeFileError("private file has an unexpected owner") + if snapshot.st_nlink != 1: + raise UnsafeFileError("private file has unexpected links") + + +def _open_private_file_at( + parent_fd: int, + name: str, + *, + append: bool, + create: bool, + platform: _Platform, +) -> _OpenedPrivateFile | None: + before: os.stat_result | None + try: + before = os.lstat(name, dir_fd=parent_fd) + except FileNotFoundError: + before = None + except OSError as exc: + raise UnsafeFileError("private file is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file cannot be inspected safely") from exc + if before is None and not create: + return None + if before is not None: + _validate_file_snapshot(before, platform) + + flags = platform.append_flags if append else platform.read_flags + if before is None: + flags |= platform.create_exclusive_flags + try: + file_fd = os.open(name, flags, _FILE_MODE, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError("private file cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file cannot be opened safely") from exc + try: + opened = os.fstat(file_fd) + current = os.lstat(name, dir_fd=parent_fd) + initial_snapshots = (opened, current) if before is None else (before, opened, current) + for snapshot in initial_snapshots: + _validate_file_snapshot(snapshot, platform) + if not _same_files(initial_snapshots): + raise UnsafeFileError("private file changed while it was opened") + + os.fchmod(file_fd, _FILE_MODE) + tightened = os.fstat(file_fd) + final = os.lstat(name, dir_fd=parent_fd) + final_snapshots = (*initial_snapshots, tightened, final) + for snapshot in final_snapshots: + _validate_file_snapshot(snapshot, platform) + if not _same_files(final_snapshots): + raise UnsafeFileError("private file changed while it was opened") + if any(stat.S_IMODE(snapshot.st_mode) != _FILE_MODE for snapshot in (tightened, final)): + raise UnsafeFileError("private file does not have private permissions") + return _OpenedPrivateFile( + fd=file_fd, + parent_fd=parent_fd, + name=name, + identity=tightened, + platform=platform, + ) + except UnsafeFileError: + _close_suppressing_error(file_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(file_fd) + raise UnsafeFileError("private file could not be validated safely") from exc + except BaseException: + _close_suppressing_error(file_fd) + raise + + +@contextmanager +def _private_append_context( + parent_parts: tuple[str, ...], + name: str, + platform: _Platform, +) -> Iterator[BinaryIO]: + with _open_directory_path(parent_parts, create=True, platform=platform) as parent: + opened_file = _open_private_file_at( + parent.fd, + name, + append=True, + create=True, + platform=platform, + ) + if opened_file is None: + raise UnsafeFileError("private file was not created") + file_fd = opened_file.fd + try: + try: + handle = cast(BinaryIO, os.fdopen(file_fd, "ab", buffering=0)) + except UnsafeFileError: + _close_suppressing_error(file_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(file_fd) + raise UnsafeFileError("private file descriptor could not be wrapped") from exc + file_fd = -1 + try: + yield handle + except BaseException: + with suppress(OSError): + handle.close() + raise + else: + try: + opened_file.validate_binding() + except BaseException: + with suppress(OSError): + handle.close() + raise + try: + handle.close() + except OSError as exc: + raise UnsafeFileError("private file descriptor could not be closed") from exc + finally: + if file_fd >= 0: + _close_suppressing_error(file_fd) + + +def append_private_bytes(path: Path, data: bytes) -> None: + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + with open_private_append(path) as handle: + offset = 0 + while offset < len(data): + try: + written = handle.write(data[offset:]) + except OSError as exc: + raise UnsafeFileError("private file write failed") from exc + if written is None or written <= 0 or written > len(data) - offset: + raise UnsafeFileError("private file write was incomplete") + offset += written + + +def open_private_append(path: Path) -> AbstractContextManager[BinaryIO]: + parts = _validate_path(path, file_path=True) + platform = _require_platform() + return _private_append_context(parts[:-1], parts[-1], platform) + + +def read_private_tail(path: Path, max_bytes: int) -> bytes: + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int): + raise TypeError("max_bytes must be an integer") + if max_bytes < 0: + raise TypeError("max_bytes must be non-negative") + parts = _validate_path(path, file_path=True) + platform = _require_platform() + try: + with _open_directory_path( + parts[:-1], create=False, missing_ok=True, platform=platform + ) as parent: + opened_file = _open_private_file_at( + parent.fd, + parts[-1], + append=False, + create=False, + platform=platform, + ) + if opened_file is None: + parent.confirm_missing(parts[-1]) + return b"" + file_fd = opened_file.fd + try: + try: + end = os.lseek(file_fd, 0, os.SEEK_END) + start = max(0, end - max_bytes) + os.lseek(file_fd, start, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = min(max_bytes, end - start) + while remaining: + chunk = os.read(file_fd, min(65536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + result = b"".join(chunks) + except OSError as exc: + raise UnsafeFileError("private file tail could not be read") from exc + opened_file.validate_binding() + except BaseException: + _close_suppressing_error(file_fd) + raise + else: + _close_descriptor(file_fd, "private file") + return result + except _PrivatePathMissing: + return b"" + + +def validate_private_directory(path: Path) -> None: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + with _open_directory_path(parts, create=False, platform=platform): + pass + + +@contextmanager +def pin_private_directory(path: Path) -> Iterator[_PinnedPrivateDirectory]: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + pinned_fd = -1 + try: + with _open_directory_path(parts, create=False, platform=platform) as opened: + try: + pinned_fd = os.dup(opened.fd) + identity = os.fstat(pinned_fd) + _validate_directory_snapshots((identity,), "pinned private directory") + _validate_directory_requirement( + (identity,), + _DirectoryRequirement.exact_private, + platform.euid, + "pinned private directory", + ) + except UnsafeFileError: + raise + except (OSError, TypeError, ValueError) as exc: + raise UnsafeFileError("private directory could not be pinned safely") from exc + except BaseException: + if pinned_fd >= 0: + _close_suppressing_error(pinned_fd) + raise + pinned = _PinnedPrivateDirectory(pinned_fd, identity, platform) + try: + yield pinned + except BaseException: + _close_suppressing_error(pinned_fd) + raise + else: + _close_descriptor(pinned_fd, "pinned private directory") + + +def ensure_private_directory(path: Path) -> None: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + with _open_directory_path(parts, create=True, platform=platform): + pass + + +def inspect_private_directory(path: Path, *, missing_ok: bool = False) -> bool: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + try: + with _open_directory_path( + parts, + create=False, + missing_ok=missing_ok, + tighten=False, + platform=platform, + ): + pass + except _PrivatePathMissing: + return False + return True diff --git a/zeus/process_identity.py b/zeus/process_identity.py new file mode 100644 index 0000000..6bd9753 --- /dev/null +++ b/zeus/process_identity.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import errno +import os +import platform +import re +import shlex +import shutil +import subprocess # nosec B404 +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import cast + +PidAliveFn = Callable[[int], bool] +CmdlineReader = Callable[[int], list[str] | None] +ProcStartFingerprintReader = Callable[[int], str | None] + +_SubprocessRun = Callable[..., subprocess.CompletedProcess[str]] +_PYTHON_INTERPRETER_RE = re.compile(r"^python(?:\d+(?:\.\d+)?)?$") + + +@dataclass(frozen=True) +class CommandCheck: + verified: bool + reason: str + classification: str | None = None + + +class PidState(Enum): + alive = "alive" + dead = "dead" + unknown = "unknown" + + +def read_process_cmdline( + pid: int, + *, + system: str | None = None, + run_process: _SubprocessRun | None = None, +) -> list[str] | None: + current_system = platform.system() if system is None else system + if current_system == "Linux": + return read_linux_cmdline(pid) + if current_system == "Darwin": + if run_process is None: + return read_darwin_cmdline(pid) + return read_darwin_cmdline(pid, run_process=run_process) + return None + + +def read_linux_cmdline(pid: int, proc_root: Path = Path("/proc")) -> list[str]: + try: + raw = (proc_root / str(pid) / "cmdline").read_bytes() + except FileNotFoundError: + return [] + except OSError: + return [] + return [part.decode("utf-8", errors="surrogateescape") for part in raw.split(b"\0") if part] + + +def read_darwin_cmdline( + pid: int, + *, + run_process: _SubprocessRun | None = None, +) -> list[str] | None: + runner = _subprocess_runner(run_process) + try: + completed = runner( + ["/bin/ps", "-p", str(pid), "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return [] + command = completed.stdout.strip() + if not command: + return [] + try: + return shlex.split(command) + except ValueError: + return None + + +def read_process_start_fingerprint( + pid: int, + *, + system: str | None = None, + run_process: _SubprocessRun | None = None, +) -> str | None: + current_system = platform.system() if system is None else system + if current_system == "Darwin": + if run_process is None: + return read_darwin_process_start_fingerprint(pid) + return read_darwin_process_start_fingerprint(pid, run_process=run_process) + if current_system != "Linux": + return None + return read_linux_process_start_fingerprint(pid) + + +def read_darwin_process_start_fingerprint( + pid: int, + *, + run_process: _SubprocessRun | None = None, +) -> str | None: + runner = _subprocess_runner(run_process) + try: + completed = runner( + ["/bin/ps", "-p", str(pid), "-o", "lstart="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return None + started = " ".join(completed.stdout.split()) + return f"darwin:ps-lstart:{started}" if started else None + + +def read_linux_process_start_fingerprint( + pid: int, + proc_root: Path = Path("/proc"), +) -> str | None: + try: + stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8") + except (FileNotFoundError, OSError, UnicodeDecodeError): + return None + try: + fields = stat.rsplit(") ", 1)[1].split() + except IndexError: + return None + if len(fields) < 20: + return None + return f"linux:/proc-starttime:{fields[19]}" + + +def pid_state(pid: int, *, pid_alive_fn: PidAliveFn | None = None) -> PidState: + try: + if pid_alive_fn is not None: + return PidState.alive if pid_alive_fn(pid) else PidState.dead + os.kill(pid, 0) + except ProcessLookupError: + return PidState.dead + except PermissionError: + return PidState.unknown + except OSError as exc: + return PidState.dead if exc.errno == errno.ESRCH else PidState.unknown + return PidState.alive + + +def valid_process_start_fingerprint(value: object) -> bool: + return type(value) is str and bool(value) and len(value) <= 512 + + +def process_start_fingerprint_required(system: str) -> bool: + return system in {"Darwin", "Linux"} + + +def process_start_identity_error( + marker_fingerprint: object, + live_fingerprint: str | None, + *, + fingerprint_required: bool, +) -> str | None: + if fingerprint_required: + marker_is_valid = valid_process_start_fingerprint(marker_fingerprint) + live_is_valid = valid_process_start_fingerprint(live_fingerprint) + if not marker_is_valid or not live_is_valid: + return "process start fingerprint is unavailable" + if marker_fingerprint != live_fingerprint: + return "process start fingerprint does not match" + elif live_fingerprint and marker_fingerprint != live_fingerprint: + return "process start fingerprint does not match" + elif marker_fingerprint and live_fingerprint != marker_fingerprint: + return "process start fingerprint is unavailable" + return None + + +def verify_gateway_command( + argv: list[str], + bot_id: str, + trusted_hermes_bin: str | set[str] | None, + *, + require_trusted_path: bool, +) -> CommandCheck: + if not argv: + return CommandCheck(False, "live-cmdline-missing") + classification = "direct-hermes" + hermes_command = argv[0] + args = argv[1:] + if len(argv) >= 2 and looks_like_python_interpreter(argv[0]): + classification = "python-script-wrapper" + hermes_command = argv[1] + args = argv[2:] + if len(args) != 4 or args.count("-p") != 1 or args[0] != "-p": + return CommandCheck(False, "wrong-command-intent", classification) + if args[1] != bot_id: + return CommandCheck(False, "wrong-bot-id", classification) + if args[2:] != ["gateway", "run"]: + return CommandCheck(False, "wrong-command-intent", classification) + if require_trusted_path: + resolved_command = resolve_executable(hermes_command) + if isinstance(trusted_hermes_bin, str): + trusted_hermes_bins = {trusted_hermes_bin} + else: + trusted_hermes_bins = trusted_hermes_bin or set() + if not trusted_hermes_bins or resolved_command not in trusted_hermes_bins: + return CommandCheck(False, "untrusted-executable", classification) + return CommandCheck(True, "ok", classification) + + +def looks_like_python_interpreter(command: str) -> bool: + return bool(_PYTHON_INTERPRETER_RE.fullmatch(Path(command).name.lower())) + + +def resolve_executable(command: str, path: str | None = None) -> str | None: + if not command: + return None + candidate = command if "/" in command else shutil.which(command, path=path) + if candidate is None: + return None + try: + return str(Path(candidate).expanduser().resolve()) + except (OSError, RuntimeError): + return str(Path(candidate).expanduser().absolute()) + + +def trusted_hermes_paths(command: str) -> set[str]: + resolved = resolve_executable(command) + if resolved is None: + return set() + paths = {resolved} + delegated = resolve_launcher_exec_target(resolved) + if delegated is not None: + paths.add(delegated) + return paths + + +def resolve_launcher_exec_target(command: str) -> str | None: + path = Path(command) + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except (OSError, UnicodeDecodeError): + return None + if not text.startswith("#!"): + return None + for line in text.splitlines()[1:20]: + stripped = line.strip() + if not stripped.startswith("exec "): + continue + try: + parts = shlex.split(stripped) + except ValueError: + continue + if len(parts) < 2 or parts[0] != "exec": + continue + target = parts[1] + if "/" not in target: + continue + resolved = resolve_executable(target) + if resolved and Path(resolved).name == "hermes": + return resolved + return None + + +def safe_command_shape(argv: list[str]) -> str: + if not argv: + return "empty" + classification = "direct-hermes" + args = argv[1:] + if len(argv) >= 2 and looks_like_python_interpreter(argv[0]): + classification = "python-script-wrapper" + args = argv[2:] + if len(args) == 4 and args[0] == "-p" and args[2:] == ["gateway", "run"]: + return f"{classification} hermes -p gateway run" + return f"{classification} unrecognized" + + +def _subprocess_runner(run_process: _SubprocessRun | None) -> _SubprocessRun: + if run_process is not None: + return run_process + return cast(_SubprocessRun, subprocess.run) diff --git a/zeus/profile_manager.py b/zeus/profile_manager.py new file mode 100644 index 0000000..d2b5ac4 --- /dev/null +++ b/zeus/profile_manager.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import os +import shutil +import time +from contextlib import AbstractContextManager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from zeus.errors import BotArchiveError, BotDeleteError +from zeus.models import BotCreateRequest, BotRecord, HermesTemplate, validate_id +from zeus.renderer import ProfileRenderer + + +@dataclass(frozen=True) +class ProfileDeletion: + profile_path: Path + tombstone_path: Path + + +@dataclass(frozen=True) +class ProfileArchive: + profile_path: Path + archive_path: Path + + +class ProfileManager: + def __init__(self, hermes_root: Path | str, archive_root: Path | str) -> None: + self.hermes_root = Path(hermes_root) + self.archive_root = Path(archive_root) + self._renderer = ProfileRenderer(self.hermes_root) + + def preflight( + self, + request: BotCreateRequest, + template: HermesTemplate, + ) -> dict[str, str]: + return self._renderer.preflight(request, template) + + def install_transaction( + self, + request: BotCreateRequest, + template: HermesTemplate, + ) -> AbstractContextManager[BotRecord]: + return self._renderer.transaction(request, template) + + def validate_profile_path(self, bot_id: str, profile_path: Path | str) -> Path: + safe_bot_id = validate_id(bot_id, "bot_id") + profile = Path(profile_path).resolve() + profiles_root = (self.hermes_root / "profiles").resolve() + return self._validate_exact_profile_path(safe_bot_id, profile, profiles_root) + + def _pin_profile_path(self, bot_id: str, profile_path: Path | str) -> Path: + safe_bot_id = validate_id(bot_id, "bot_id") + absolute_profile = Path(os.path.abspath(profile_path)) + profile = absolute_profile.parent.resolve() / absolute_profile.name + profiles_root = (self.hermes_root / "profiles").resolve() + return self._validate_exact_profile_path(safe_bot_id, profile, profiles_root) + + @staticmethod + def _validate_exact_profile_path( + safe_bot_id: str, + profile: Path, + profiles_root: Path, + ) -> Path: + try: + relative = profile.relative_to(profiles_root) + except ValueError as exc: + raise BotDeleteError("bot profile path is outside the Hermes profiles root") from exc + if len(relative.parts) != 1 or relative.parts[0] != safe_bot_id: + raise BotDeleteError("bot profile path does not match bot id") + return profile + + def stage_delete( + self, + bot_id: str, + profile_path: Path | str, + ) -> ProfileDeletion | None: + profile = self.validate_profile_path(bot_id, profile_path) + if not profile.exists(): + return None + tombstone = profile.with_name(f".{profile.name}.deleting-{os.getpid()}-{time.time_ns()}") + try: + os.replace(profile, tombstone) + except OSError as exc: + raise BotDeleteError("could not stage the bot profile for deletion") from exc + return ProfileDeletion(profile_path=profile, tombstone_path=tombstone) + + def rollback_delete(self, deletion: ProfileDeletion) -> None: + if os.path.lexists(deletion.profile_path): + raise BotDeleteError( + "bot state deletion failed and the profile could not be restored because " + "its original path is occupied" + ) + try: + os.replace(deletion.tombstone_path, deletion.profile_path) + except OSError as exc: + raise BotDeleteError( + "bot state deletion failed and the profile could not be restored" + ) from exc + + def finish_delete(self, deletion: ProfileDeletion) -> OSError | None: + try: + shutil.rmtree(deletion.tombstone_path) + except OSError as exc: + return exc + return None + + def stage_archive( + self, + bot_id: str, + profile_path: Path | str, + ) -> ProfileArchive | None: + profile = self.validate_profile_path(bot_id, profile_path) + if not profile.exists(): + return None + self.archive_root.mkdir(parents=True, exist_ok=True) + archive_path = self.archive_root / ( + f"{validate_id(bot_id, 'bot_id')}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" + ) + shutil.move(str(profile), str(archive_path)) + return ProfileArchive(profile_path=profile, archive_path=archive_path) + + def rollback_archive(self, archive: ProfileArchive) -> None: + if os.path.lexists(archive.profile_path): + raise BotArchiveError( + "bot state deletion failed and the archived profile could not be restored " + "because its original path is occupied" + ) + try: + shutil.move(str(archive.archive_path), str(archive.profile_path)) + except OSError as exc: + raise BotArchiveError( + "bot state deletion failed and the archived profile could not be restored" + ) from exc diff --git a/zeus/readiness.py b/zeus/readiness.py index f5e3195..e87999b 100644 --- a/zeus/readiness.py +++ b/zeus/readiness.py @@ -4,9 +4,27 @@ import time from collections.abc import Mapping from dataclasses import dataclass +from http.client import HTTPException from urllib.error import HTTPError, URLError from urllib.parse import urlparse -from urllib.request import urlopen +from urllib.request import ( + HTTPRedirectHandler, + OpenerDirector, + ProxyHandler, + Request, + build_opener, +) + +MAX_READINESS_RESPONSE_BYTES = 64 * 1024 + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, *args: object, **kwargs: object) -> None: + return None + + +def _build_readiness_opener() -> OpenerDirector: + return build_opener(ProxyHandler({}), _RejectRedirects()) @dataclass(frozen=True) @@ -54,12 +72,41 @@ def probe_once( expected_status: str = "ok", expected_platform: str = "hermes-agent", ) -> ReadinessResult: - parsed = urlparse(url) - if parsed.scheme != "http" or parsed.hostname not in {"127.0.0.1", "localhost", "::1"}: + try: + parsed = urlparse(url) + hostname = parsed.hostname + _port = parsed.port + except ValueError: + return ReadinessResult(False, "readiness URL must be loopback HTTP") + if ( + parsed.scheme != "http" + or hostname not in {"127.0.0.1", "localhost", "::1"} + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): return ReadinessResult(False, "readiness URL must be loopback HTTP") try: - with urlopen(url, timeout=timeout_seconds) as response: # nosec B310 - payload = json.loads(response.read().decode("utf-8")) + request = Request(url, headers={"Connection": "close"}) # nosec B310 + with _build_readiness_opener().open(request, timeout=timeout_seconds) as response: + content_lengths = response.headers.get_all("content-length", []) + try: + declared_lengths = {int(value) for value in content_lengths} + except ValueError: + return ReadinessResult(False, "readiness response has invalid content length") + if any(length < 0 for length in declared_lengths) or len(declared_lengths) > 1: + return ReadinessResult(False, "readiness response has invalid content length") + if any(length > MAX_READINESS_RESPONSE_BYTES for length in declared_lengths): + return ReadinessResult(False, "readiness response exceeds size limit") + # HTTPResponse.read(amt) clamps fixed-length responses to the declared + # Content-Length, so valid HTTP/1.1 keep-alive responses return without + # waiting for EOF. Responses without a fixed length remain bounded by amt; + # chunk framing is handled by HTTPResponse. + body = response.read(MAX_READINESS_RESPONSE_BYTES + 1) + if len(body) > MAX_READINESS_RESPONSE_BYTES: + return ReadinessResult(False, "readiness response exceeds size limit") + payload = json.loads(body.decode("utf-8")) if not isinstance(payload, dict): return ReadinessResult(False, "health payload must be a JSON object") if ( @@ -67,9 +114,16 @@ def probe_once( and payload.get("platform") == expected_platform ): return ReadinessResult(True, "ready", payload) - return ReadinessResult(False, f"unexpected health payload: {payload!r}", payload) - except (HTTPError, URLError, OSError, json.JSONDecodeError, ValueError) as exc: - return ReadinessResult(False, f"{type(exc).__name__}: {exc}") + return ReadinessResult(False, "unexpected readiness health payload") + except HTTPError as exc: + exc.close() + return ReadinessResult(False, "readiness endpoint returned an HTTP error") + except (URLError, OSError): + return ReadinessResult(False, "readiness probe failed") + except HTTPException: + return ReadinessResult(False, "readiness endpoint returned a malformed HTTP response") + except (UnicodeDecodeError, json.JSONDecodeError, ValueError): + return ReadinessResult(False, "readiness response is not valid JSON") def wait_until_ready(probe: ReadinessProbe) -> ReadinessResult: diff --git a/zeus/reconcile_store.py b/zeus/reconcile_store.py new file mode 100644 index 0000000..4e2f16e --- /dev/null +++ b/zeus/reconcile_store.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime + +from zeus.reconciliation import ( + BotReconcileResult, + PersistedReconcileRun, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, +) +from zeus.sqlite_db import SQLiteDatabase + +RECONCILE_COUNTER_COLUMNS = { + ReconcileOutcome.healthy: "healthy_count", + ReconcileOutcome.changed: "changed_count", + ReconcileOutcome.pending: "pending_count", + ReconcileOutcome.action_required: "action_required_count", + ReconcileOutcome.error: "error_count", + ReconcileOutcome.skipped: "skipped_count", +} + + +def _validate_reconcile_run_id(value: str) -> str: + if not isinstance(value, str): + raise TypeError("reconciliation run_id must be a string") + if not value or len(value) > 128: + raise ValueError("reconciliation run_id must be between 1 and 128 characters") + if any(ord(character) < 0x20 or 0x7F <= ord(character) <= 0x9F for character in value): + raise ValueError("reconciliation run_id must not contain control characters") + return value + + +def _serialize_reconcile_timestamp(value: datetime, label: str) -> str: + if not isinstance(value, datetime): + raise TypeError(f"reconciliation {label} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"reconciliation {label} must be timezone-aware") + return value.astimezone(UTC).isoformat() + + +def _parse_reconcile_timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str): + raise TypeError(f"stored reconciliation {label} must be a string") + try: + parsed = datetime.fromisoformat(value) + except ValueError: + raise ValueError(f"stored reconciliation {label} is invalid") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError(f"stored reconciliation {label} is invalid") + normalized = parsed.astimezone(UTC) + if normalized.isoformat() != value: + raise ValueError(f"stored reconciliation {label} is not normalized") + return normalized + + +def _validate_stored_nonnegative_integer(value: object, label: str) -> int: + if type(value) is not int or value < 0: + raise ValueError(f"stored reconciliation {label} must be a non-negative integer") + return value + + +def _validate_stored_optional_positive_integer(value: object, label: str) -> int | None: + if value is None: + return None + if type(value) is not int or value <= 0: + raise ValueError(f"stored reconciliation {label} must be a positive integer or null") + return value + + +def _validate_stored_boolean_flag(value: object, label: str) -> bool: + if type(value) is not int or value not in {0, 1}: + raise ValueError(f"stored reconciliation {label} must be exactly 0 or 1") + return value == 1 + + +class ReconcileStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def begin_reconcile_run(self, run: ReconcileRunStart) -> None: + if not isinstance(run, ReconcileRunStart): + raise TypeError("run must be a ReconcileRunStart") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + """ + INSERT INTO reconcile_runs ( + run_id, + scope, + requested_bot_id, + source, + force, + reset_restart, + started_at, + finished_at, + outcome + ) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'running') + """, + ( + run.run_id, + run.scope, + run.requested_bot_id, + run.source, + int(run.force), + int(run.reset_restart), + _serialize_reconcile_timestamp(run.started_at, "start timestamp"), + ), + ) + except Exception: + conn.rollback() + raise + else: + conn.commit() + + def append_reconcile_result( + self, + run_id: str, + result: BotReconcileResult, + ) -> None: + safe_run_id = _validate_reconcile_run_id(run_id) + if not isinstance(result, BotReconcileResult): + raise TypeError("result must be a BotReconcileResult") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + run_row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (safe_run_id,), + ).fetchone() + if run_row is None: + raise KeyError(f"unknown reconciliation run: {safe_run_id}") + if run_row["outcome"] != "running": + raise RuntimeError("reconciliation run is not running") + current_run = self._materialize_reconcile_run(conn, safe_run_id) + if current_run is None: + raise sqlite3.IntegrityError("reconciliation run disappeared") + if current_run.scope == "bot" and result.bot_id != current_run.requested_bot_id: + raise ValueError("reconciliation result must match the requested bot") + if result.started_at < current_run.started_at: + raise ValueError("reconciliation result starts before its run") + if result.event_id is not None: + event_row = conn.execute( + "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", + (result.event_id,), + ).fetchone() + if event_row is None or event_row["bot_id"] != result.bot_id: + raise ValueError( + "reconciliation lifecycle event must exist and match the result bot" + ) + conn.execute( + """ + INSERT INTO reconcile_results ( + run_id, + bot_id, + ordinal, + outcome, + desired_state, + observed_status, + pid, + action, + message, + error_code, + event_id, + started_at, + finished_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + safe_run_id, + result.bot_id, + current_run.total, + result.outcome.value, + result.desired_state, + result.observed_status, + result.pid, + result.action, + result.message, + result.error_code, + result.event_id, + _serialize_reconcile_timestamp( + result.started_at, + "result start timestamp", + ), + _serialize_reconcile_timestamp( + result.finished_at, + "result finish timestamp", + ), + ), + ) + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET total = total + 1, + healthy_count = healthy_count + ?, + changed_count = changed_count + ?, + pending_count = pending_count + ?, + action_required_count = action_required_count + ?, + error_count = error_count + ?, + skipped_count = skipped_count + ? + WHERE run_id = ? AND outcome = 'running' + """, + ( + int(result.outcome is ReconcileOutcome.healthy), + int(result.outcome is ReconcileOutcome.changed), + int(result.outcome is ReconcileOutcome.pending), + int(result.outcome is ReconcileOutcome.action_required), + int(result.outcome is ReconcileOutcome.error), + int(result.outcome is ReconcileOutcome.skipped), + safe_run_id, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation result counter update failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() + + def finish_reconcile_run( + self, + summary: ReconcileRunSummary, + ) -> PersistedReconcileRun: + if not isinstance(summary, ReconcileRunSummary): + raise TypeError("summary must be a ReconcileRunSummary") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + run_row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (summary.run_id,), + ).fetchone() + if run_row is None: + raise KeyError(f"unknown reconciliation run: {summary.run_id}") + if run_row["outcome"] != "running": + raise RuntimeError("reconciliation run is not running") + if run_row["scope"] != summary.scope: + raise RuntimeError("reconciliation summary scope does not match run") + stored_started_at = _parse_reconcile_timestamp( + run_row["started_at"], + "run start timestamp", + ) + if stored_started_at != summary.started_at: + raise RuntimeError("reconciliation summary start timestamp does not match run") + persisted_results = self._reconcile_results_in_transaction(conn, summary.run_id) + if tuple(summary.results) != persisted_results: + raise RuntimeError("persisted reconciliation results do not match summary") + observed_counts = self._reconcile_counts_from_results(persisted_results) + stored_counts = self._reconcile_counts_from_run_row(run_row) + stored_total = _validate_stored_nonnegative_integer( + run_row["total"], + "total", + ) + if stored_counts != observed_counts or stored_total != len(persisted_results): + raise RuntimeError("persisted reconciliation counters do not match results") + if dict(summary.counts) != observed_counts or summary.total != len( + persisted_results + ): + raise RuntimeError("reconciliation summary counters do not match results") + if any(result.finished_at > summary.finished_at for result in persisted_results): + raise RuntimeError("reconciliation summary finishes before a persisted result") + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET finished_at = ?, + outcome = ?, + total = ?, + healthy_count = ?, + changed_count = ?, + pending_count = ?, + action_required_count = ?, + error_count = ?, + skipped_count = ? + WHERE run_id = ? AND outcome = 'running' + """, + ( + _serialize_reconcile_timestamp( + summary.finished_at, + "finish timestamp", + ), + summary.outcome, + summary.total, + summary.counts[ReconcileOutcome.healthy.value], + summary.counts[ReconcileOutcome.changed.value], + summary.counts[ReconcileOutcome.pending.value], + summary.counts[ReconcileOutcome.action_required.value], + summary.counts[ReconcileOutcome.error.value], + summary.counts[ReconcileOutcome.skipped.value], + summary.run_id, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation run finalization failed") + finished = self._materialize_reconcile_run(conn, summary.run_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + if finished is None: + raise sqlite3.IntegrityError("finished reconciliation run is missing") + return finished + + def interrupt_stale_reconcile_runs(self, *, interrupted_at: datetime) -> int: + safe_interrupted_at = _serialize_reconcile_timestamp( + interrupted_at, + "interruption timestamp", + ) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + "SELECT * FROM reconcile_runs WHERE outcome = 'running' ORDER BY run_id" + ).fetchall() + for row in rows: + run_id = str(row["run_id"]) + current_run = self._materialize_reconcile_run(conn, run_id) + if current_run is None: + raise sqlite3.IntegrityError("reconciliation run disappeared") + if interrupted_at < current_run.started_at: + raise ValueError("interruption timestamp precedes a running reconciliation") + if any(result.finished_at > interrupted_at for result in current_run.results): + raise ValueError( + "reconciliation result finishes after the interruption timestamp" + ) + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET outcome = 'interrupted', finished_at = ? + WHERE run_id = ? AND outcome = 'running' + """, + (safe_interrupted_at, run_id), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation interruption update failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() + return len(rows) + + def get_reconcile_run(self, run_id: str) -> PersistedReconcileRun | None: + safe_run_id = _validate_reconcile_run_id(run_id) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN") + run = self._materialize_reconcile_run(conn, safe_run_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + return run + + def _materialize_reconcile_run( + self, + conn: sqlite3.Connection, + run_id: str, + ) -> PersistedReconcileRun | None: + row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (run_id,), + ).fetchone() + if row is None: + return None + results = self._reconcile_results_in_transaction(conn, run_id) + finished_at = row["finished_at"] + return PersistedReconcileRun( + run_id=str(row["run_id"]), + scope=str(row["scope"]), + requested_bot_id=( + str(row["requested_bot_id"]) if row["requested_bot_id"] is not None else None + ), + source=str(row["source"]), + force=_validate_stored_boolean_flag(row["force"], "force flag"), + reset_restart=_validate_stored_boolean_flag( + row["reset_restart"], + "reset-restart flag", + ), + started_at=_parse_reconcile_timestamp(row["started_at"], "run start timestamp"), + finished_at=( + _parse_reconcile_timestamp(finished_at, "run finish timestamp") + if finished_at is not None + else None + ), + outcome=str(row["outcome"]), + total=_validate_stored_nonnegative_integer(row["total"], "total"), + counts=self._reconcile_counts_from_run_row(row), + results=results, + ) + + def _reconcile_results_in_transaction( + self, + conn: sqlite3.Connection, + run_id: str, + ) -> tuple[BotReconcileResult, ...]: + rows = conn.execute( + "SELECT * FROM reconcile_results WHERE run_id = ? ORDER BY ordinal", + (run_id,), + ).fetchall() + results: list[BotReconcileResult] = [] + for expected_ordinal, row in enumerate(rows): + ordinal = _validate_stored_nonnegative_integer(row["ordinal"], "result ordinal") + if ordinal != expected_ordinal: + raise ValueError("stored reconciliation ordinals are not contiguous") + bot_id = str(row["bot_id"]) + event_id = _validate_stored_optional_positive_integer( + row["event_id"], + "result event id", + ) + if event_id is not None: + event_row = conn.execute( + "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", + (event_id,), + ).fetchone() + if event_row is None or event_row["bot_id"] != bot_id: + raise ValueError( + "stored reconciliation lifecycle event does not match the result bot" + ) + results.append( + BotReconcileResult( + bot_id=bot_id, + outcome=ReconcileOutcome(str(row["outcome"])), + desired_state=( + str(row["desired_state"]) if row["desired_state"] is not None else None + ), + observed_status=( + str(row["observed_status"]) if row["observed_status"] is not None else None + ), + pid=_validate_stored_optional_positive_integer(row["pid"], "result pid"), + action=str(row["action"]), + message=str(row["message"]), + error_code=(str(row["error_code"]) if row["error_code"] is not None else None), + event_id=event_id, + started_at=_parse_reconcile_timestamp( + row["started_at"], + "result start timestamp", + ), + finished_at=_parse_reconcile_timestamp( + row["finished_at"], + "result finish timestamp", + ), + ) + ) + return tuple(results) + + def _reconcile_counts_from_run_row(self, row: sqlite3.Row) -> dict[str, int]: + return { + outcome.value: _validate_stored_nonnegative_integer( + row[column], + f"{outcome.value} counter", + ) + for outcome, column in RECONCILE_COUNTER_COLUMNS.items() + } + + def _reconcile_counts_from_results( + self, + results: tuple[BotReconcileResult, ...], + ) -> dict[str, int]: + counts = {outcome.value: 0 for outcome in ReconcileOutcome} + for result in results: + counts[result.outcome.value] += 1 + return counts diff --git a/zeus/reconciliation.py b/zeus/reconciliation.py index fea3b1b..606bcd3 100644 --- a/zeus/reconciliation.py +++ b/zeus/reconciliation.py @@ -11,6 +11,7 @@ from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState from zeus.process_lock import BotProcessLock, LockTimeoutError +from zeus.sanitization import sanitize_text MAX_RECONCILE_ID_LENGTH = 128 MAX_RECONCILE_TEXT_LENGTH = 2048 @@ -48,7 +49,7 @@ def _validate_required_text(value: str, name: str, *, maximum: int) -> str: def _bound_text(value: str, name: str) -> str: - if not isinstance(value, str): + if type(value) is not str: raise ValueError(f"{name} must be a string") return value[:MAX_RECONCILE_TEXT_LENGTH] @@ -56,7 +57,7 @@ def _bound_text(value: str, name: str) -> str: def _bound_optional_text(value: str | None, name: str) -> str | None: if value is None: return None - if not isinstance(value, str) or not value: + if type(value) is not str or not value: raise ValueError(f"{name} must be a non-empty string when provided") return value[:MAX_RECONCILE_TEXT_LENGTH] @@ -144,7 +145,14 @@ def __post_init__(self) -> None: object.__setattr__(self, "observed_status", observed_status) object.__setattr__(self, "pid", _validate_positive_optional_int(self.pid, "pid")) object.__setattr__(self, "action", _bound_text(self.action, "action")) - object.__setattr__(self, "message", _bound_text(self.message, "message")) + object.__setattr__( + self, + "message", + sanitize_text( + _bound_text(self.message, "message"), + max_length=MAX_RECONCILE_TEXT_LENGTH, + ), + ) object.__setattr__( self, "error_code", @@ -432,7 +440,8 @@ def summarize_results( class _ReconciliationStore(Protocol): - database_path: Path + @property + def database_path(self) -> Path: ... def list_bots(self) -> list[BotRecord]: ... diff --git a/zeus/renderer.py b/zeus/renderer.py index 716a72e..864a1e1 100644 --- a/zeus/renderer.py +++ b/zeus/renderer.py @@ -4,6 +4,7 @@ import logging import os import shutil +import stat import tempfile from collections.abc import Iterator from contextlib import contextmanager @@ -13,6 +14,12 @@ from zeus.envfile import dump_env from zeus.models import BotCreateRequest, BotRecord, HermesTemplate, TemplateError +from zeus.private_io import ( + UnsafeFileError, + nofollow_absolute_path, + pin_private_directory, + validate_private_directory, +) _LOG = logging.getLogger(__name__) @@ -95,7 +102,19 @@ def transaction( staging = Path(tempfile.mkdtemp(prefix=f".{request.bot_id}.staging-", dir=profiles_root)) try: _write_staged_profile(staging, profile, rendered_files) - backup = _install_staged_profile(staging, profile) + backup: Path | None = None + installed = False + try: + with pin_private_directory(nofollow_absolute_path(staging / "logs")) as pinned: + backup = _install_staged_profile(staging, profile) + installed = True + pinned.validate_at(nofollow_absolute_path(profile / "logs")) + except UnsafeFileError as exc: + if installed: + _rollback_installed_profile(profile, backup) + raise TemplateError( + "installed profile logs directory could not be verified safely" + ) from exc finally: try: _remove_path(staging) @@ -133,8 +152,8 @@ def _write_staged_profile( ) _ensure_staged_directory(staging / "cron") - for relative_path in ("logs", "skills"): - _ensure_preserved_directory(staging / relative_path) + _ensure_private_logs_directory(staging / "logs") + _ensure_preserved_directory(staging / "skills") for relative_path, content in rendered_files.items(): _write_staged_file(staging / relative_path, content) @@ -157,6 +176,27 @@ def _ensure_preserved_directory(path: Path) -> None: raise TemplateError(f"profile path must be a directory: {path.name}") +def _ensure_private_logs_directory(path: Path) -> None: + try: + if _path_exists(path): + metadata = os.lstat(path) + if stat.S_ISLNK(metadata.st_mode): + path.unlink() + elif not stat.S_ISDIR(metadata.st_mode): + raise TemplateError(f"profile path must be a directory: {path.name}") + if not _path_exists(path): + path.mkdir(mode=0o700) + metadata = os.lstat(path) + except OSError as exc: + raise TemplateError("profile logs directory could not be prepared safely") from exc + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid(): + raise TemplateError("profile logs directory must be owned by the current user") + try: + validate_private_directory(nofollow_absolute_path(path)) + except OSError as exc: + raise TemplateError("profile logs directory could not be made private") from exc + + def _write_staged_file(path: Path, content: str) -> None: if _path_exists(path): if path.is_dir() and not path.is_symlink(): diff --git a/zeus/request_context.py b/zeus/request_context.py index 3ee9363..0e25607 100644 --- a/zeus/request_context.py +++ b/zeus/request_context.py @@ -49,6 +49,7 @@ _STATIC_ROUTE_TEMPLATES = { "/health": "/health", + "/ready": "/ready", "/doctor": "/doctor", "/templates": "/templates", "/bots": "/bots", diff --git a/zeus/sanitization.py b/zeus/sanitization.py new file mode 100644 index 0000000..b455e8c --- /dev/null +++ b/zeus/sanitization.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import json +import math +import re +from types import MappingProxyType +from typing import TypeAlias + +JSONScalar: TypeAlias = None | bool | int | float | str +JSONValue: TypeAlias = JSONScalar | list["JSONValue"] | dict[str, "JSONValue"] + +MAX_SANITIZED_JSON_BYTES = 8192 +MAX_DETAIL_KEY_LENGTH = 128 +MAX_EVENT_TEXT_LENGTH = 2048 + +REDACTED_VALUE = "[redacted]" +TRUNCATED_VALUE = "[truncated]" +CYCLE_VALUE = "[cycle]" +UNSUPPORTED_VALUE = "[unsupported]" + +_CAMEL_ACRONYM_BOUNDARY_RE = re.compile(r"([A-Z]+)([A-Z][a-z])") +_CAMEL_WORD_BOUNDARY_RE = re.compile(r"([a-z0-9])([A-Z])") +_NON_NAME_CHARACTER_RE = re.compile(r"[^a-z0-9]+") +_SECRET_NAME_PARTS = frozenset({"apikey", "key", "token", "secret", "password"}) +_FORBIDDEN_DETAIL_NAMES = frozenset( + { + "authorization", + "authorization_header", + "authorization_headers", + "header", + "headers", + "request_body", + "response_body", + "body", + "raw_query", + "query_string", + "query", + "forwarded_for", + "x_forwarded_for", + "client_address", + "client_ip", + "client_port", + "remote_addr", + "traceback", + "exception_trace", + "idempotency_key", + } +) + +_SECRET_KV_RE = re.compile( + r"""(?ix) + (?P["']?) + (?P + [A-Z0-9_.-]*(?:API[_-]?KEY|KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_.-]* + ) + (?P=prefix) + (?P\s*[:=]\s*) + (?P + "(?:[^"\\]|\\.)*" | + '(?:[^'\\]|\\.)*' | + [^\s,}]+ + ) + """ +) +_AUTHORIZATION_RE = re.compile( + r"""(?ix) + (?P["']?) + (?PAUTHORIZATION(?:[^A-Z0-9\r\n:=]*HEADERS?)?) + (?P=prefix) + (?P[ \t]*[:=][ \t]*) + (?P(?:(?:\r\n|[\r\n])[ \t]+)?) + (?P + "(?:[^"\\\r\n]|\\.)*" | + '(?:[^'\\\r\n]|\\.)*' | + [^\r\n]+ + ) + (?P(?:(?:\r\n|[\r\n])[ \t]+[^\r\n]*)*) + """ +) +_BEARER_RE = re.compile(r"(?i)(\bBearer\s+)([A-Za-z0-9._~+/=-]+)") + + +def redact_secrets(text: str) -> str: + """Redact common secret assignments and bearer credentials from free text.""" + + redacted = _AUTHORIZATION_RE.sub( + lambda match: ( + f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" + f"{match.group('sep')}[redacted]" + ), + text, + ) + redacted = _BEARER_RE.sub(r"\1[redacted]", redacted) + return _SECRET_KV_RE.sub( + lambda match: ( + f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" + f"{match.group('sep')}[redacted]" + ), + redacted, + ) + + +def sanitize_text(value: str, *, max_length: int = MAX_EVENT_TEXT_LENGTH) -> str: + """Return bounded, redacted text without coercing non-string objects.""" + + if type(value) is not str: + return UNSUPPORTED_VALUE + if type(max_length) is not int or max_length < 0: + raise ValueError("max_length must be a non-negative integer") + return redact_secrets(value[:max_length])[:max_length] + + +def _normalize_detail_name(key: str) -> str: + separated = _CAMEL_ACRONYM_BOUNDARY_RE.sub(r"\1_\2", key) + separated = _CAMEL_WORD_BOUNDARY_RE.sub(r"\1_\2", separated).lower() + return _NON_NAME_CHARACTER_RE.sub("_", separated).strip("_") + + +def _contains_name(normalized: str, candidate: str) -> bool: + return f"_{candidate}_" in f"_{normalized}_" + + +def _is_forbidden_detail_name(key: str) -> bool: + normalized = _normalize_detail_name(key) + if any(_contains_name(normalized, name) for name in _FORBIDDEN_DETAIL_NAMES): + return True + return any(_contains_name(normalized, part) for part in _SECRET_NAME_PARTS) + + +def _sanitize_key(value: object, *, max_length: int) -> str | None: + if type(value) is str: + return sanitize_text(value, max_length=max_length) + if type(value) is int: + try: + return str(value)[:max_length] + except ValueError: + return None + if type(value) is float and math.isfinite(value): + return str(value)[:max_length] + if value is None: + return "null" + if type(value) is bool: + return "true" if value else "false" + return None + + +class _SanitizationState: + def __init__( + self, + *, + max_depth: int, + max_items: int, + max_string_length: int, + ) -> None: + self.max_depth = max_depth + self.remaining_items = max_items + self.max_string_length = max_string_length + self.active_container_ids: set[int] = set() + + def consume_item(self) -> bool: + if self.remaining_items <= 0: + return False + self.remaining_items -= 1 + return True + + +def _sanitize_value( + value: object, + *, + state: _SanitizationState, + depth: int, + key: str | None, +) -> JSONValue: + if key is not None and _is_forbidden_detail_name(key): + return REDACTED_VALUE + if value is None: + return None + if type(value) is bool: + return value + if type(value) is int: + return value + if type(value) is float: + return value if math.isfinite(value) else None + if type(value) is str: + return sanitize_text(value, max_length=state.max_string_length) + + if type(value) is dict or type(value) is MappingProxyType: + if depth >= state.max_depth: + return TRUNCATED_VALUE + container_id = id(value) + if container_id in state.active_container_ids: + return CYCLE_VALUE + state.active_container_ids.add(container_id) + try: + result: dict[str, JSONValue] = {} + for raw_key, child_value in value.items(): + if not state.consume_item(): + break + child_key = _sanitize_key(raw_key, max_length=MAX_DETAIL_KEY_LENGTH) + if child_key is None: + continue + if type(raw_key) is str and len(raw_key) > MAX_DETAIL_KEY_LENGTH: + result[child_key] = REDACTED_VALUE + else: + result[child_key] = _sanitize_value( + child_value, + state=state, + depth=depth + 1, + key=child_key, + ) + return result + finally: + state.active_container_ids.remove(container_id) + + if type(value) is list or type(value) is tuple: + if depth >= state.max_depth: + return TRUNCATED_VALUE + container_id = id(value) + if container_id in state.active_container_ids: + return CYCLE_VALUE + state.active_container_ids.add(container_id) + try: + result_list: list[JSONValue] = [] + for child_value in value: + if not state.consume_item(): + break + result_list.append( + _sanitize_value( + child_value, + state=state, + depth=depth + 1, + key=key, + ) + ) + return result_list + finally: + state.active_container_ids.remove(container_id) + + return UNSUPPORTED_VALUE + + +def _validate_limit(value: int, name: str) -> int: + if type(value) is not int or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + return value + + +def _canonical_json_bytes(value: JSONValue) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def sanitize_details( + value: object, + *, + max_depth: int = 6, + max_items: int = 64, + max_string_length: int = 2048, +) -> JSONValue: + """Produce a bounded JSON value without invoking arbitrary conversions.""" + + state = _SanitizationState( + max_depth=_validate_limit(max_depth, "max_depth"), + max_items=_validate_limit(max_items, "max_items"), + max_string_length=_validate_limit(max_string_length, "max_string_length"), + ) + sanitized = _sanitize_value(value, state=state, depth=0, key=None) + try: + if len(_canonical_json_bytes(sanitized)) <= MAX_SANITIZED_JSON_BYTES: + return sanitized + except (TypeError, ValueError, RecursionError, UnicodeError): + pass + if type(sanitized) is dict: + return {"truncated": True} + return TRUNCATED_VALUE diff --git a/zeus/schema.py b/zeus/schema.py new file mode 100644 index 0000000..96b1926 --- /dev/null +++ b/zeus/schema.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import AbstractContextManager, closing +from pathlib import Path +from typing import Protocol + +from zeus.lifecycle import serialize_lifecycle_details + +SCHEMA_VERSION = 6 + + +class _SchemaDatabase(Protocol): + def connect(self) -> sqlite3.Connection: ... + + def immediate(self) -> AbstractContextManager[sqlite3.Connection]: ... + + +def _read_schema_version(conn: sqlite3.Connection) -> int | None: + table = conn.execute( + """ + SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'schema_version' + """ + ).fetchone() + if table is None: + return None + row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() + if row is None: + return None + version = row["version"] + if type(version) is not int: + raise TypeError("database schema version must be an integer") + return version + + +def _assert_schema_compatible(conn: sqlite3.Connection) -> None: + version = _read_schema_version(conn) + if version is not None and version > SCHEMA_VERSION: + raise RuntimeError( + f"database schema version {version} is newer than supported version {SCHEMA_VERSION}" + ) + + +def _assert_schema_current(conn: sqlite3.Connection) -> int: + version = _read_schema_version(conn) + if version != SCHEMA_VERSION: + raise RuntimeError("database schema is not current") + return version + + +def _preflight_schema_compatibility(database_path: Path) -> None: + if not database_path.exists(): + return + uri = f"{database_path.resolve().as_uri()}?mode=ro" + with closing(sqlite3.connect(uri, uri=True)) as conn: + conn.row_factory = sqlite3.Row + _assert_schema_compatible(conn) + + +class SchemaManager: + def __init__(self, database: _SchemaDatabase) -> None: + self._database = database + + def init(self) -> None: + with self._database.immediate() as conn: + self._create_bots_table(conn) + self._migrate(conn) + + def migrate(self) -> None: + with self._database.immediate() as conn: + self._migrate(conn) + + def _create_bots_table(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS bots ( + bot_id TEXT PRIMARY KEY, + template_id TEXT NOT NULL, + display_name TEXT NOT NULL, + profile_path TEXT NOT NULL, + status TEXT NOT NULL, + pid INTEGER, + restart_policy TEXT NOT NULL DEFAULT 'manual', + restart_backoff_seconds REAL NOT NULL DEFAULT 5.0, + restart_max_attempts INTEGER NOT NULL DEFAULT 5, + restart_attempts INTEGER NOT NULL DEFAULT 0, + next_restart_at TEXT, + started_at TEXT, + ready_at TEXT, + stopped_at TEXT, + last_exit_code INTEGER, + last_error TEXT, + last_transition_reason TEXT, + last_event_id INTEGER, + desired_state TEXT NOT NULL DEFAULT 'stopped' + CHECK (desired_state IN ('running', 'stopped')), + desired_revision INTEGER NOT NULL DEFAULT 0 CHECK (desired_revision >= 0), + desired_updated_at TEXT, + pending_operation_id TEXT, + pending_action TEXT CHECK ( + pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart') + ), + pending_since TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ( + (pending_operation_id IS NULL + AND pending_action IS NULL + AND pending_since IS NULL) + OR + (pending_operation_id IS NOT NULL AND pending_action IS NOT NULL + AND pending_since IS NOT NULL) + ) + ) + """ + ) + + def _migrate(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER NOT NULL + ) + """ + ) + stored_version = _read_schema_version(conn) + current_version = stored_version or 0 + if stored_version is None: + conn.execute("INSERT INTO schema_version (version) VALUES (0)") + if current_version > SCHEMA_VERSION: + raise RuntimeError( + f"database schema version {current_version} is newer than supported " + f"version {SCHEMA_VERSION}" + ) + if current_version < 1: + self._ensure_restart_schema(conn) + conn.execute("UPDATE schema_version SET version = ?", (1,)) + current_version = 1 + if current_version < 2: + self._ensure_lifecycle_schema(conn) + conn.execute("UPDATE schema_version SET version = ?", (2,)) + current_version = 2 + if current_version < 3: + self._migrate_v2_to_v3(conn) + conn.execute("UPDATE schema_version SET version = ?", (3,)) + current_version = 3 + if current_version < 4: + self._migrate_v3_to_v4(conn) + conn.execute("UPDATE schema_version SET version = ?", (4,)) + current_version = 4 + if current_version < 5: + self._migrate_v4_to_v5(conn) + conn.execute("UPDATE schema_version SET version = ?", (5,)) + current_version = 5 + if current_version < 6: + self._migrate_v5_to_v6(conn) + conn.execute("UPDATE schema_version SET version = ?", (6,)) + + def _ensure_restart_schema(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "restart_policy": ( + "ALTER TABLE bots ADD COLUMN restart_policy TEXT NOT NULL DEFAULT 'manual'" + ), + "restart_backoff_seconds": ( + "ALTER TABLE bots ADD COLUMN restart_backoff_seconds REAL NOT NULL DEFAULT 5.0" + ), + "restart_max_attempts": ( + "ALTER TABLE bots ADD COLUMN restart_max_attempts INTEGER NOT NULL DEFAULT 5" + ), + "restart_attempts": ( + "ALTER TABLE bots ADD COLUMN restart_attempts INTEGER NOT NULL DEFAULT 0" + ), + "next_restart_at": "ALTER TABLE bots ADD COLUMN next_restart_at TEXT", + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + def _ensure_lifecycle_schema(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "started_at": "ALTER TABLE bots ADD COLUMN started_at TEXT", + "ready_at": "ALTER TABLE bots ADD COLUMN ready_at TEXT", + "stopped_at": "ALTER TABLE bots ADD COLUMN stopped_at TEXT", + "last_exit_code": "ALTER TABLE bots ADD COLUMN last_exit_code INTEGER", + "last_error": "ALTER TABLE bots ADD COLUMN last_error TEXT", + "last_transition_reason": ("ALTER TABLE bots ADD COLUMN last_transition_reason TEXT"), + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + def _migrate_v2_to_v3(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + if "last_event_id" not in columns: + conn.execute("ALTER TABLE bots ADD COLUMN last_event_id INTEGER") + conn.execute( + """ + CREATE TABLE lifecycle_events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + bot_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + request_id TEXT, + occurred_at TEXT NOT NULL, + source TEXT NOT NULL, + action TEXT NOT NULL, + outcome TEXT NOT NULL, + status_before TEXT, + status_after TEXT, + pid_before INTEGER, + pid_after INTEGER, + reason TEXT NOT NULL DEFAULT '', + error_code TEXT, + error_message TEXT, + details_json TEXT NOT NULL DEFAULT '{}' + ) + """ + ) + conn.execute( + """ + CREATE INDEX idx_lifecycle_events_bot + ON lifecycle_events (bot_id, event_id DESC) + """ + ) + conn.execute( + """ + CREATE INDEX idx_lifecycle_events_operation + ON lifecycle_events (operation_id, event_id) + """ + ) + conn.execute( + """ + CREATE TRIGGER lifecycle_events_reject_update + BEFORE UPDATE ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'lifecycle events are immutable'); + END + """ + ) + conn.execute( + """ + CREATE TRIGGER lifecycle_events_reject_delete + BEFORE DELETE ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'lifecycle events are immutable'); + END + """ + ) + + rows = conn.execute( + "SELECT bot_id, status, pid, updated_at FROM bots ORDER BY bot_id" + ).fetchall() + for row in rows: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, + operation_id, + occurred_at, + source, + action, + outcome, + status_before, + status_after, + pid_before, + pid_after, + reason, + details_json + ) VALUES (?, 'migration-v3', ?, 'migration', 'migration.snapshot', + 'success', ?, ?, ?, ?, 'schema v3 snapshot', '{}') + """, + ( + row["bot_id"], + row["updated_at"], + row["status"], + row["status"], + row["pid"], + row["pid"], + ), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (cursor.lastrowid, row["bot_id"]), + ) + + invalid_projection = conn.execute( + """ + SELECT bots.bot_id + FROM bots + LEFT JOIN lifecycle_events + ON lifecycle_events.event_id = bots.last_event_id + AND lifecycle_events.bot_id = bots.bot_id + WHERE lifecycle_events.event_id IS NULL + LIMIT 1 + """ + ).fetchone() + if invalid_projection is not None: + raise sqlite3.IntegrityError("lifecycle snapshot invariant failed") + + def _migrate_v3_to_v4(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE idempotency_records ( + key_hash TEXT PRIMARY KEY, + request_hash TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('in_progress', 'completed')), + owner_instance_id TEXT NOT NULL, + response_status INTEGER NULL, + response_json TEXT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + expires_at TEXT NOT NULL + ) + """ + ) + conn.execute( + """ + CREATE INDEX idx_idempotency_records_expires + ON idempotency_records (expires_at) + """ + ) + + def _migrate_v4_to_v5(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "desired_state": ( + "ALTER TABLE bots ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'stopped' " + "CHECK (desired_state IN ('running', 'stopped'))" + ), + "desired_revision": ( + "ALTER TABLE bots ADD COLUMN desired_revision INTEGER NOT NULL DEFAULT 0 " + "CHECK (desired_revision >= 0)" + ), + "desired_updated_at": "ALTER TABLE bots ADD COLUMN desired_updated_at TEXT", + "pending_operation_id": "ALTER TABLE bots ADD COLUMN pending_operation_id TEXT", + "pending_action": ( + "ALTER TABLE bots ADD COLUMN pending_action TEXT CHECK " + "(pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart'))" + ), + "pending_since": "ALTER TABLE bots ADD COLUMN pending_since TEXT", + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + conn.execute( + """ + CREATE TRIGGER bots_desired_intent_reject_partial_insert + BEFORE INSERT ON bots + WHEN NOT ( + (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL + AND NEW.pending_since IS NULL) + OR + (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL + AND NEW.pending_since IS NOT NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); + END + """ + ) + conn.execute( + """ + CREATE TRIGGER bots_desired_intent_reject_partial_update + BEFORE UPDATE ON bots + WHEN NOT ( + (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL + AND NEW.pending_since IS NULL) + OR + (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL + AND NEW.pending_since IS NOT NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); + END + """ + ) + + conn.execute( + """ + UPDATE bots + SET desired_state = CASE + WHEN status IN ('running', 'starting') THEN 'running' + WHEN status IN ('failed', 'unknown') AND restart_policy = 'on-failure' + THEN 'running' + ELSE 'stopped' + END, + desired_revision = 0, + desired_updated_at = updated_at, + pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL + """ + ) + + rows = conn.execute( + """ + SELECT bot_id, status, pid, desired_state, desired_revision, desired_updated_at + FROM bots ORDER BY bot_id + """ + ).fetchall() + for row in rows: + details = serialize_lifecycle_details( + { + "desired_state": str(row["desired_state"]), + "desired_revision": int(row["desired_revision"]), + } + ) + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, operation_id, occurred_at, source, action, outcome, + status_before, status_after, pid_before, pid_after, reason, details_json + ) VALUES (?, 'migration-v5', ?, 'migration', + 'migration.desired_state_snapshot', 'success', ?, ?, ?, ?, + 'schema v5 desired-state snapshot', ?) + """, + ( + row["bot_id"], + row["desired_updated_at"], + row["status"], + row["status"], + row["pid"], + row["pid"], + details, + ), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (cursor.lastrowid, row["bot_id"]), + ) + + invalid_projection = conn.execute( + """ + SELECT bots.bot_id + FROM bots + LEFT JOIN lifecycle_events + ON lifecycle_events.event_id = bots.last_event_id + AND lifecycle_events.bot_id = bots.bot_id + WHERE lifecycle_events.event_id IS NULL + LIMIT 1 + """ + ).fetchone() + if invalid_projection is not None: + raise sqlite3.IntegrityError("desired-state snapshot invariant failed") + + def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE reconcile_runs ( + run_id TEXT PRIMARY KEY, + scope TEXT NOT NULL CHECK (scope IN ('fleet', 'bot')), + requested_bot_id TEXT, + source TEXT NOT NULL, + force INTEGER NOT NULL CHECK (force IN (0, 1)), + reset_restart INTEGER NOT NULL CHECK (reset_restart IN (0, 1)), + started_at TEXT NOT NULL, + finished_at TEXT, + outcome TEXT NOT NULL CHECK ( + outcome IN ('running', 'succeeded', 'completed_with_errors', 'interrupted') + ), + total INTEGER NOT NULL DEFAULT 0 CHECK (total >= 0), + healthy_count INTEGER NOT NULL DEFAULT 0 CHECK (healthy_count >= 0), + changed_count INTEGER NOT NULL DEFAULT 0 CHECK (changed_count >= 0), + pending_count INTEGER NOT NULL DEFAULT 0 CHECK (pending_count >= 0), + action_required_count INTEGER NOT NULL DEFAULT 0 + CHECK (action_required_count >= 0), + error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), + skipped_count INTEGER NOT NULL DEFAULT 0 CHECK (skipped_count >= 0), + CHECK ( + total = healthy_count + changed_count + pending_count + + action_required_count + error_count + skipped_count + ), + CHECK ( + (scope = 'fleet' AND requested_bot_id IS NULL) + OR (scope = 'bot' AND requested_bot_id IS NOT NULL) + ), + CHECK ( + (outcome = 'running' AND finished_at IS NULL) + OR (outcome != 'running' AND finished_at IS NOT NULL) + ), + CHECK (finished_at IS NULL OR finished_at >= started_at) + ) + """ + ) + conn.execute( + """ + CREATE TABLE reconcile_results ( + run_id TEXT NOT NULL, + bot_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + outcome TEXT NOT NULL CHECK ( + outcome IN ( + 'healthy', 'changed', 'pending', + 'action_required', 'error', 'skipped' + ) + ), + desired_state TEXT CHECK ( + desired_state IS NULL OR desired_state IN ('running', 'stopped') + ), + observed_status TEXT CHECK ( + observed_status IS NULL + OR observed_status IN ('stopped', 'starting', 'running', 'failed', 'unknown') + ), + pid INTEGER CHECK (pid IS NULL OR pid > 0), + action TEXT NOT NULL CHECK (length(action) <= 2048), + message TEXT NOT NULL CHECK (length(message) <= 2048), + error_code TEXT CHECK (error_code IS NULL OR length(error_code) <= 2048), + event_id INTEGER CHECK (event_id IS NULL OR event_id > 0), + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + PRIMARY KEY (run_id, bot_id), + UNIQUE (run_id, ordinal), + CHECK (finished_at >= started_at), + FOREIGN KEY (run_id) REFERENCES reconcile_runs(run_id) ON DELETE CASCADE + ) + """ + ) diff --git a/zeus/sqlite_db.py b/zeus/sqlite_db.py new file mode 100644 index 0000000..212c29b --- /dev/null +++ b/zeus/sqlite_db.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterator +from contextlib import closing, contextmanager +from pathlib import Path + +from zeus.config import SQLiteSynchronous +from zeus.schema import ( + _assert_schema_compatible, + _assert_schema_current, + _preflight_schema_compatibility, +) + +_SYNCHRONOUS_PRAGMA = { + SQLiteSynchronous.NORMAL: "PRAGMA synchronous=NORMAL", + SQLiteSynchronous.FULL: "PRAGMA synchronous=FULL", +} + + +class StateReadinessError(RuntimeError): + pass + + +class SQLiteDatabase: + def __init__( + self, + database_path: Path | str, + *, + synchronous: SQLiteSynchronous | str = SQLiteSynchronous.NORMAL, + ) -> None: + self.database_path = Path(database_path) + self.synchronous = SQLiteSynchronous(synchronous) + + def connect(self) -> sqlite3.Connection: + self.database_path.parent.mkdir(parents=True, exist_ok=True) + _preflight_schema_compatibility(self.database_path) + conn = sqlite3.connect(self.database_path) + conn.row_factory = sqlite3.Row + try: + _assert_schema_compatible(conn) + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA journal_mode=WAL") + conn.execute(_SYNCHRONOUS_PRAGMA[self.synchronous]) + conn.execute("PRAGMA busy_timeout=5000") + return conn + except Exception: + conn.close() + raise + + def check_readiness(self) -> int: + uri = f"{self.database_path.resolve().as_uri()}?mode=ro" + try: + with closing(sqlite3.connect(uri, uri=True)) as conn: + conn.row_factory = sqlite3.Row + version = _assert_schema_current(conn) + row = conn.execute("SELECT 1").fetchone() + if row is None or type(row[0]) is not int or row[0] != 1: + raise RuntimeError("state store probe returned an invalid result") + return version + except (sqlite3.Error, OSError, RuntimeError, TypeError, ValueError) as exc: + raise StateReadinessError("state store is not ready") from exc + + @contextmanager + def immediate(self) -> Iterator[sqlite3.Connection]: + with closing(self.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + yield conn + except Exception: + conn.rollback() + raise + else: + conn.commit() diff --git a/zeus/state.py b/zeus/state.py index 4562a70..1dba788 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -1,1158 +1,68 @@ from __future__ import annotations -import json -import re import sqlite3 -from collections.abc import Mapping -from contextlib import closing -from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import datetime from pathlib import Path from typing import Literal +from zeus.bot_lifecycle_store import LIFECYCLE_ERROR_CODE_RE as LIFECYCLE_ERROR_CODE_RE +from zeus.bot_lifecycle_store import LIFECYCLE_ID_RE as LIFECYCLE_ID_RE +from zeus.bot_lifecycle_store import LIFECYCLE_INTENT_ACTIONS as LIFECYCLE_INTENT_ACTIONS +from zeus.bot_lifecycle_store import LIFECYCLE_SOURCES as LIFECYCLE_SOURCES +from zeus.bot_lifecycle_store import BotLifecycleStore +from zeus.config import SQLiteSynchronous from zeus.idempotency import IdempotencyClaim +from zeus.idempotency_store import IDEMPOTENCY_HASH_RE as IDEMPOTENCY_HASH_RE +from zeus.idempotency_store import IDEMPOTENCY_OWNER_RE as IDEMPOTENCY_OWNER_RE +from zeus.idempotency_store import MAX_IDEMPOTENCY_RESPONSE_BYTES as MAX_IDEMPOTENCY_RESPONSE_BYTES +from zeus.idempotency_store import IdempotencyStore from zeus.lifecycle import ( LifecycleEvent, LifecycleEventInput, - deserialize_lifecycle_details, - serialize_lifecycle_details, ) -from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy +from zeus.models import BotRecord, BotStatus +from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS as RECONCILE_COUNTER_COLUMNS +from zeus.reconcile_store import ReconcileStore from zeus.reconciliation import ( BotReconcileResult, PersistedReconcileRun, - ReconcileOutcome, ReconcileRunStart, ReconcileRunSummary, ) - -SCHEMA_VERSION = 6 -AUDIT_SECRET_RE = re.compile(r"(KEY|TOKEN|SECRET|PASSWORD)$") -LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") -LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") -IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") -IDEMPOTENCY_OWNER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$", re.ASCII) -MAX_IDEMPOTENCY_RESPONSE_BYTES = 1_000_000 -LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) -LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) -RECONCILE_COUNTER_COLUMNS = { - ReconcileOutcome.healthy: "healthy_count", - ReconcileOutcome.changed: "changed_count", - ReconcileOutcome.pending: "pending_count", - ReconcileOutcome.action_required: "action_required_count", - ReconcileOutcome.error: "error_count", - ReconcileOutcome.skipped: "skipped_count", -} - - -def _safe_audit_value(key: str, value: object) -> object: - if AUDIT_SECRET_RE.search(key.upper()): - return "[redacted]" - if value is None or isinstance(value, str | int | float | bool): - return value - if isinstance(value, Path): - return str(value) - if isinstance(value, Mapping): - return { - str(child_key): _safe_audit_value(str(child_key), child_value) - for child_key, child_value in value.items() - } - if isinstance(value, list): - return [_safe_audit_value(key, child_value) for child_value in value] - return str(value) - - -def _validate_idempotency_hash(value: str, label: str) -> str: - if not isinstance(value, str): - raise TypeError(f"idempotency {label} must be a string") - if IDEMPOTENCY_HASH_RE.fullmatch(value) is None: - raise ValueError(f"idempotency {label} is invalid") - return value - - -def _validate_idempotency_owner(value: str) -> str: - if not isinstance(value, str): - raise TypeError("idempotency owner must be a string") - if IDEMPOTENCY_OWNER_RE.fullmatch(value) is None: - raise ValueError("idempotency owner is invalid") - return value - - -def _validate_idempotency_timestamp(value: datetime, label: str) -> str: - if not isinstance(value, datetime): - raise TypeError(f"idempotency {label} must be a datetime") - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"idempotency {label} is invalid") - return value.astimezone(UTC).isoformat() - - -def _validate_idempotency_status(value: int) -> int: - if type(value) is not int or not 200 <= value <= 599: - raise ValueError("idempotency response status is invalid") - return value - - -def _reject_non_finite_json(_value: str) -> object: - raise ValueError("idempotency response JSON is invalid") - - -def _validate_idempotency_response_json(value: str) -> str: - if not isinstance(value, str): - raise TypeError("idempotency response JSON must be a string") - try: - encoded = value.encode("utf-8") - except UnicodeError: - raise ValueError("idempotency response JSON is invalid") from None - if len(encoded) > MAX_IDEMPOTENCY_RESPONSE_BYTES: - raise ValueError("idempotency response JSON is too large") - try: - json.loads(value, parse_constant=_reject_non_finite_json) - except (json.JSONDecodeError, TypeError, ValueError, RecursionError): - raise ValueError("idempotency response JSON is invalid") from None - return value - - -def _parse_idempotency_timestamp(value: object, label: str) -> datetime: - if not isinstance(value, str): - raise TypeError(f"stored idempotency {label} must be a string") - try: - parsed = datetime.fromisoformat(value) - except ValueError: - raise ValueError(f"stored idempotency {label} is invalid") from None - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise ValueError(f"stored idempotency {label} is invalid") - normalized = parsed.astimezone(UTC) - if normalized.isoformat() != value: - raise ValueError(f"stored idempotency {label} is invalid") - return normalized - - -def _validate_reconcile_run_id(value: str) -> str: - if not isinstance(value, str): - raise TypeError("reconciliation run_id must be a string") - if not value or len(value) > 128: - raise ValueError("reconciliation run_id must be between 1 and 128 characters") - if any(ord(character) < 0x20 or 0x7F <= ord(character) <= 0x9F for character in value): - raise ValueError("reconciliation run_id must not contain control characters") - return value - - -def _serialize_reconcile_timestamp(value: datetime, label: str) -> str: - if not isinstance(value, datetime): - raise TypeError(f"reconciliation {label} must be a datetime") - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"reconciliation {label} must be timezone-aware") - return value.astimezone(UTC).isoformat() - - -def _parse_reconcile_timestamp(value: object, label: str) -> datetime: - if not isinstance(value, str): - raise TypeError(f"stored reconciliation {label} must be a string") - try: - parsed = datetime.fromisoformat(value) - except ValueError: - raise ValueError(f"stored reconciliation {label} is invalid") from None - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise ValueError(f"stored reconciliation {label} is invalid") - normalized = parsed.astimezone(UTC) - if normalized.isoformat() != value: - raise ValueError(f"stored reconciliation {label} is not normalized") - return normalized - - -def _validate_stored_nonnegative_integer(value: object, label: str) -> int: - if type(value) is not int or value < 0: - raise ValueError(f"stored reconciliation {label} must be a non-negative integer") - return value - - -def _validate_stored_optional_positive_integer(value: object, label: str) -> int | None: - if value is None: - return None - if type(value) is not int or value <= 0: - raise ValueError(f"stored reconciliation {label} must be a positive integer or null") - return value - - -def _validate_stored_boolean_flag(value: object, label: str) -> bool: - if type(value) is not int or value not in {0, 1}: - raise ValueError(f"stored reconciliation {label} must be exactly 0 or 1") - return value == 1 - - -@dataclass(frozen=True) -class _ValidatedIdempotencyRow: - request_hash: str - state: str - owner_instance_id: str - response_status: int | None - response_json: str | None - created_at: datetime - updated_at: datetime - expires_at: datetime - - -def _validate_idempotency_row(row: sqlite3.Row) -> _ValidatedIdempotencyRow: - _validate_idempotency_hash(row["key_hash"], "stored key hash") - request_hash = _validate_idempotency_hash(row["request_hash"], "stored request hash") - owner = _validate_idempotency_owner(row["owner_instance_id"]) - state = row["state"] - if state not in {"in_progress", "completed"}: - raise ValueError("stored idempotency state is invalid") - created_at = _parse_idempotency_timestamp(row["created_at"], "creation timestamp") - updated_at = _parse_idempotency_timestamp(row["updated_at"], "update timestamp") - expires_at = _parse_idempotency_timestamp(row["expires_at"], "expiry timestamp") - response_status = row["response_status"] - response_json = row["response_json"] - if state == "in_progress": - if created_at > updated_at or updated_at > expires_at: - raise ValueError("stored in-progress idempotency timestamps are invalid") - if response_status is not None or response_json is not None: - raise ValueError("stored in-progress idempotency response is invalid") - safe_status = None - safe_json = None - else: - if created_at > updated_at or updated_at > expires_at: - raise ValueError("stored completed idempotency timestamps are invalid") - if response_status is None or response_json is None: - raise ValueError("stored completed idempotency response is incomplete") - safe_status = _validate_idempotency_status(response_status) - safe_json = _validate_idempotency_response_json(response_json) - return _ValidatedIdempotencyRow( - request_hash=request_hash, - state=state, - owner_instance_id=owner, - response_status=safe_status, - response_json=safe_json, - created_at=created_at, - updated_at=updated_at, - expires_at=expires_at, - ) - - -def _idempotency_claim_from_row( - row: sqlite3.Row, - *, - request_hash: str, - owner_instance_id: str, -) -> IdempotencyClaim: - validated = _validate_idempotency_row(row) - if validated.request_hash != request_hash: - return IdempotencyClaim("conflict") - if validated.state == "completed": - if validated.response_status is None or validated.response_json is None: - raise AssertionError("validated completed idempotency response is incomplete") - return IdempotencyClaim( - "replay", - validated.response_status, - validated.response_json, - ) - if validated.owner_instance_id == owner_instance_id: - return IdempotencyClaim("in_progress") - return IdempotencyClaim("indeterminate") +from zeus.schema import SCHEMA_VERSION as SCHEMA_VERSION +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.sqlite_db import StateReadinessError as StateReadinessError class StateStore: - def __init__(self, database_path: Path | str) -> None: - self.database_path = Path(database_path) - - def connect(self) -> sqlite3.Connection: - self.database_path.parent.mkdir(parents=True, exist_ok=True) - self._preflight_schema_compatibility() - conn = sqlite3.connect(self.database_path) - conn.row_factory = sqlite3.Row - try: - self._assert_schema_compatible(conn) - conn.execute("PRAGMA foreign_keys=ON") - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") - conn.execute("PRAGMA busy_timeout=5000") - return conn - except Exception: - conn.close() - raise - - def _preflight_schema_compatibility(self) -> None: - if not self.database_path.exists(): - return - uri = f"{self.database_path.resolve().as_uri()}?mode=ro" - with closing(sqlite3.connect(uri, uri=True)) as conn: - conn.row_factory = sqlite3.Row - self._assert_schema_compatible(conn) - - def _assert_schema_compatible(self, conn: sqlite3.Connection) -> None: - table = conn.execute( - """ - SELECT 1 FROM sqlite_master - WHERE type = 'table' AND name = 'schema_version' - """ - ).fetchone() - if table is None: - return - row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - if row is not None and int(row["version"]) > SCHEMA_VERSION: - raise RuntimeError( - f"database schema version {int(row['version'])} is newer than supported " - f"version {SCHEMA_VERSION}" - ) - - def init(self) -> None: - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - self._create_bots_table(conn) - self._migrate(conn) - except Exception: - conn.rollback() - raise - else: - conn.commit() - - def migrate(self) -> None: - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - self._migrate(conn) - except Exception: - conn.rollback() - raise - else: - conn.commit() - - def _create_bots_table(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS bots ( - bot_id TEXT PRIMARY KEY, - template_id TEXT NOT NULL, - display_name TEXT NOT NULL, - profile_path TEXT NOT NULL, - status TEXT NOT NULL, - pid INTEGER, - restart_policy TEXT NOT NULL DEFAULT 'manual', - restart_backoff_seconds REAL NOT NULL DEFAULT 5.0, - restart_max_attempts INTEGER NOT NULL DEFAULT 5, - restart_attempts INTEGER NOT NULL DEFAULT 0, - next_restart_at TEXT, - started_at TEXT, - ready_at TEXT, - stopped_at TEXT, - last_exit_code INTEGER, - last_error TEXT, - last_transition_reason TEXT, - last_event_id INTEGER, - desired_state TEXT NOT NULL DEFAULT 'stopped' - CHECK (desired_state IN ('running', 'stopped')), - desired_revision INTEGER NOT NULL DEFAULT 0 CHECK (desired_revision >= 0), - desired_updated_at TEXT, - pending_operation_id TEXT, - pending_action TEXT CHECK ( - pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart') - ), - pending_since TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - CHECK ( - (pending_operation_id IS NULL - AND pending_action IS NULL - AND pending_since IS NULL) - OR - (pending_operation_id IS NOT NULL AND pending_action IS NOT NULL - AND pending_since IS NOT NULL) - ) - ) - """ - ) - - def _migrate(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER NOT NULL - ) - """ - ) - row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - current_version = int(row["version"]) if row else 0 - if row is None: - conn.execute("INSERT INTO schema_version (version) VALUES (0)") - if current_version > SCHEMA_VERSION: - raise RuntimeError( - f"database schema version {current_version} is newer than supported " - f"version {SCHEMA_VERSION}" - ) - if current_version < 1: - self._ensure_restart_schema(conn) - conn.execute("UPDATE schema_version SET version = ?", (1,)) - current_version = 1 - if current_version < 2: - self._ensure_lifecycle_schema(conn) - conn.execute("UPDATE schema_version SET version = ?", (2,)) - current_version = 2 - if current_version < 3: - self._migrate_v2_to_v3(conn) - conn.execute("UPDATE schema_version SET version = ?", (3,)) - current_version = 3 - if current_version < 4: - self._migrate_v3_to_v4(conn) - conn.execute("UPDATE schema_version SET version = ?", (4,)) - current_version = 4 - if current_version < 5: - self._migrate_v4_to_v5(conn) - conn.execute("UPDATE schema_version SET version = ?", (5,)) - current_version = 5 - if current_version < 6: - self._migrate_v5_to_v6(conn) - conn.execute("UPDATE schema_version SET version = ?", (6,)) - - def _ensure_restart_schema(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "restart_policy": ( - "ALTER TABLE bots ADD COLUMN restart_policy TEXT NOT NULL DEFAULT 'manual'" - ), - "restart_backoff_seconds": ( - "ALTER TABLE bots ADD COLUMN restart_backoff_seconds REAL NOT NULL DEFAULT 5.0" - ), - "restart_max_attempts": ( - "ALTER TABLE bots ADD COLUMN restart_max_attempts INTEGER NOT NULL DEFAULT 5" - ), - "restart_attempts": ( - "ALTER TABLE bots ADD COLUMN restart_attempts INTEGER NOT NULL DEFAULT 0" - ), - "next_restart_at": "ALTER TABLE bots ADD COLUMN next_restart_at TEXT", - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - def _ensure_lifecycle_schema(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "started_at": "ALTER TABLE bots ADD COLUMN started_at TEXT", - "ready_at": "ALTER TABLE bots ADD COLUMN ready_at TEXT", - "stopped_at": "ALTER TABLE bots ADD COLUMN stopped_at TEXT", - "last_exit_code": "ALTER TABLE bots ADD COLUMN last_exit_code INTEGER", - "last_error": "ALTER TABLE bots ADD COLUMN last_error TEXT", - "last_transition_reason": ("ALTER TABLE bots ADD COLUMN last_transition_reason TEXT"), - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - def _migrate_v2_to_v3(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - if "last_event_id" not in columns: - conn.execute("ALTER TABLE bots ADD COLUMN last_event_id INTEGER") - conn.execute( - """ - CREATE TABLE lifecycle_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - bot_id TEXT NOT NULL, - operation_id TEXT NOT NULL, - request_id TEXT, - occurred_at TEXT NOT NULL, - source TEXT NOT NULL, - action TEXT NOT NULL, - outcome TEXT NOT NULL, - status_before TEXT, - status_after TEXT, - pid_before INTEGER, - pid_after INTEGER, - reason TEXT NOT NULL DEFAULT '', - error_code TEXT, - error_message TEXT, - details_json TEXT NOT NULL DEFAULT '{}' - ) - """ - ) - conn.execute( - """ - CREATE INDEX idx_lifecycle_events_bot - ON lifecycle_events (bot_id, event_id DESC) - """ - ) - conn.execute( - """ - CREATE INDEX idx_lifecycle_events_operation - ON lifecycle_events (operation_id, event_id) - """ - ) - conn.execute( - """ - CREATE TRIGGER lifecycle_events_reject_update - BEFORE UPDATE ON lifecycle_events - BEGIN - SELECT RAISE(ABORT, 'lifecycle events are immutable'); - END - """ - ) - conn.execute( - """ - CREATE TRIGGER lifecycle_events_reject_delete - BEFORE DELETE ON lifecycle_events - BEGIN - SELECT RAISE(ABORT, 'lifecycle events are immutable'); - END - """ - ) - - rows = conn.execute( - "SELECT bot_id, status, pid, updated_at FROM bots ORDER BY bot_id" - ).fetchall() - for row in rows: - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, - operation_id, - occurred_at, - source, - action, - outcome, - status_before, - status_after, - pid_before, - pid_after, - reason, - details_json - ) VALUES (?, 'migration-v3', ?, 'migration', 'migration.snapshot', - 'success', ?, ?, ?, ?, 'schema v3 snapshot', '{}') - """, - ( - row["bot_id"], - row["updated_at"], - row["status"], - row["status"], - row["pid"], - row["pid"], - ), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (cursor.lastrowid, row["bot_id"]), - ) - - invalid_projection = conn.execute( - """ - SELECT bots.bot_id - FROM bots - LEFT JOIN lifecycle_events - ON lifecycle_events.event_id = bots.last_event_id - AND lifecycle_events.bot_id = bots.bot_id - WHERE lifecycle_events.event_id IS NULL - LIMIT 1 - """ - ).fetchone() - if invalid_projection is not None: - raise sqlite3.IntegrityError("lifecycle snapshot invariant failed") - - def _migrate_v3_to_v4(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE idempotency_records ( - key_hash TEXT PRIMARY KEY, - request_hash TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('in_progress', 'completed')), - owner_instance_id TEXT NOT NULL, - response_status INTEGER NULL, - response_json TEXT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - expires_at TEXT NOT NULL - ) - """ - ) - conn.execute( - """ - CREATE INDEX idx_idempotency_records_expires - ON idempotency_records (expires_at) - """ - ) - - def _migrate_v4_to_v5(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "desired_state": ( - "ALTER TABLE bots ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'stopped' " - "CHECK (desired_state IN ('running', 'stopped'))" - ), - "desired_revision": ( - "ALTER TABLE bots ADD COLUMN desired_revision INTEGER NOT NULL DEFAULT 0 " - "CHECK (desired_revision >= 0)" - ), - "desired_updated_at": "ALTER TABLE bots ADD COLUMN desired_updated_at TEXT", - "pending_operation_id": "ALTER TABLE bots ADD COLUMN pending_operation_id TEXT", - "pending_action": ( - "ALTER TABLE bots ADD COLUMN pending_action TEXT CHECK " - "(pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart'))" - ), - "pending_since": "ALTER TABLE bots ADD COLUMN pending_since TEXT", - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - conn.execute( - """ - CREATE TRIGGER bots_desired_intent_reject_partial_insert - BEFORE INSERT ON bots - WHEN NOT ( - (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL - AND NEW.pending_since IS NULL) - OR - (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL - AND NEW.pending_since IS NOT NULL) - ) - BEGIN - SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); - END - """ - ) - conn.execute( - """ - CREATE TRIGGER bots_desired_intent_reject_partial_update - BEFORE UPDATE ON bots - WHEN NOT ( - (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL - AND NEW.pending_since IS NULL) - OR - (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL - AND NEW.pending_since IS NOT NULL) - ) - BEGIN - SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); - END - """ - ) - - conn.execute( - """ - UPDATE bots - SET desired_state = CASE - WHEN status IN ('running', 'starting') THEN 'running' - WHEN status IN ('failed', 'unknown') AND restart_policy = 'on-failure' - THEN 'running' - ELSE 'stopped' - END, - desired_revision = 0, - desired_updated_at = updated_at, - pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL - """ - ) - - rows = conn.execute( - """ - SELECT bot_id, status, pid, desired_state, desired_revision, desired_updated_at - FROM bots ORDER BY bot_id - """ - ).fetchall() - for row in rows: - details = serialize_lifecycle_details( - { - "desired_state": str(row["desired_state"]), - "desired_revision": int(row["desired_revision"]), - } - ) - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, operation_id, occurred_at, source, action, outcome, - status_before, status_after, pid_before, pid_after, reason, details_json - ) VALUES (?, 'migration-v5', ?, 'migration', - 'migration.desired_state_snapshot', 'success', ?, ?, ?, ?, - 'schema v5 desired-state snapshot', ?) - """, - ( - row["bot_id"], - row["desired_updated_at"], - row["status"], - row["status"], - row["pid"], - row["pid"], - details, - ), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (cursor.lastrowid, row["bot_id"]), - ) - - invalid_projection = conn.execute( - """ - SELECT bots.bot_id - FROM bots - LEFT JOIN lifecycle_events - ON lifecycle_events.event_id = bots.last_event_id - AND lifecycle_events.bot_id = bots.bot_id - WHERE lifecycle_events.event_id IS NULL - LIMIT 1 - """ - ).fetchone() - if invalid_projection is not None: - raise sqlite3.IntegrityError("desired-state snapshot invariant failed") - - def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE reconcile_runs ( - run_id TEXT PRIMARY KEY, - scope TEXT NOT NULL CHECK (scope IN ('fleet', 'bot')), - requested_bot_id TEXT, - source TEXT NOT NULL, - force INTEGER NOT NULL CHECK (force IN (0, 1)), - reset_restart INTEGER NOT NULL CHECK (reset_restart IN (0, 1)), - started_at TEXT NOT NULL, - finished_at TEXT, - outcome TEXT NOT NULL CHECK ( - outcome IN ('running', 'succeeded', 'completed_with_errors', 'interrupted') - ), - total INTEGER NOT NULL DEFAULT 0 CHECK (total >= 0), - healthy_count INTEGER NOT NULL DEFAULT 0 CHECK (healthy_count >= 0), - changed_count INTEGER NOT NULL DEFAULT 0 CHECK (changed_count >= 0), - pending_count INTEGER NOT NULL DEFAULT 0 CHECK (pending_count >= 0), - action_required_count INTEGER NOT NULL DEFAULT 0 - CHECK (action_required_count >= 0), - error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), - skipped_count INTEGER NOT NULL DEFAULT 0 CHECK (skipped_count >= 0), - CHECK ( - total = healthy_count + changed_count + pending_count - + action_required_count + error_count + skipped_count - ), - CHECK ( - (scope = 'fleet' AND requested_bot_id IS NULL) - OR (scope = 'bot' AND requested_bot_id IS NOT NULL) - ), - CHECK ( - (outcome = 'running' AND finished_at IS NULL) - OR (outcome != 'running' AND finished_at IS NOT NULL) - ), - CHECK (finished_at IS NULL OR finished_at >= started_at) - ) - """ - ) - conn.execute( - """ - CREATE TABLE reconcile_results ( - run_id TEXT NOT NULL, - bot_id TEXT NOT NULL, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - outcome TEXT NOT NULL CHECK ( - outcome IN ( - 'healthy', 'changed', 'pending', - 'action_required', 'error', 'skipped' - ) - ), - desired_state TEXT CHECK ( - desired_state IS NULL OR desired_state IN ('running', 'stopped') - ), - observed_status TEXT CHECK ( - observed_status IS NULL - OR observed_status IN ('stopped', 'starting', 'running', 'failed', 'unknown') - ), - pid INTEGER CHECK (pid IS NULL OR pid > 0), - action TEXT NOT NULL CHECK (length(action) <= 2048), - message TEXT NOT NULL CHECK (length(message) <= 2048), - error_code TEXT CHECK (error_code IS NULL OR length(error_code) <= 2048), - event_id INTEGER CHECK (event_id IS NULL OR event_id > 0), - started_at TEXT NOT NULL, - finished_at TEXT NOT NULL, - PRIMARY KEY (run_id, bot_id), - UNIQUE (run_id, ordinal), - CHECK (finished_at >= started_at), - FOREIGN KEY (run_id) REFERENCES reconcile_runs(run_id) ON DELETE CASCADE - ) - """ - ) - - def begin_reconcile_run(self, run: ReconcileRunStart) -> None: - if not isinstance(run, ReconcileRunStart): - raise TypeError("run must be a ReconcileRunStart") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - conn.execute( - """ - INSERT INTO reconcile_runs ( - run_id, - scope, - requested_bot_id, - source, - force, - reset_restart, - started_at, - finished_at, - outcome - ) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'running') - """, - ( - run.run_id, - run.scope, - run.requested_bot_id, - run.source, - int(run.force), - int(run.reset_restart), - _serialize_reconcile_timestamp(run.started_at, "start timestamp"), - ), - ) - except Exception: - conn.rollback() - raise - else: - conn.commit() - - def append_reconcile_result( + def __init__( self, - run_id: str, - result: BotReconcileResult, + database_path: Path | str, + *, + synchronous: SQLiteSynchronous | str = SQLiteSynchronous.NORMAL, ) -> None: - safe_run_id = _validate_reconcile_run_id(run_id) - if not isinstance(result, BotReconcileResult): - raise TypeError("result must be a BotReconcileResult") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - run_row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (safe_run_id,), - ).fetchone() - if run_row is None: - raise KeyError(f"unknown reconciliation run: {safe_run_id}") - if run_row["outcome"] != "running": - raise RuntimeError("reconciliation run is not running") - current_run = self._materialize_reconcile_run(conn, safe_run_id) - if current_run is None: - raise sqlite3.IntegrityError("reconciliation run disappeared") - if current_run.scope == "bot" and result.bot_id != current_run.requested_bot_id: - raise ValueError("reconciliation result must match the requested bot") - if result.started_at < current_run.started_at: - raise ValueError("reconciliation result starts before its run") - if result.event_id is not None: - event_row = conn.execute( - "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", - (result.event_id,), - ).fetchone() - if event_row is None or event_row["bot_id"] != result.bot_id: - raise ValueError( - "reconciliation lifecycle event must exist and match the result bot" - ) - conn.execute( - """ - INSERT INTO reconcile_results ( - run_id, - bot_id, - ordinal, - outcome, - desired_state, - observed_status, - pid, - action, - message, - error_code, - event_id, - started_at, - finished_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - safe_run_id, - result.bot_id, - current_run.total, - result.outcome.value, - result.desired_state, - result.observed_status, - result.pid, - result.action, - result.message, - result.error_code, - result.event_id, - _serialize_reconcile_timestamp( - result.started_at, - "result start timestamp", - ), - _serialize_reconcile_timestamp( - result.finished_at, - "result finish timestamp", - ), - ), - ) - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET total = total + 1, - healthy_count = healthy_count + ?, - changed_count = changed_count + ?, - pending_count = pending_count + ?, - action_required_count = action_required_count + ?, - error_count = error_count + ?, - skipped_count = skipped_count + ? - WHERE run_id = ? AND outcome = 'running' - """, - ( - int(result.outcome is ReconcileOutcome.healthy), - int(result.outcome is ReconcileOutcome.changed), - int(result.outcome is ReconcileOutcome.pending), - int(result.outcome is ReconcileOutcome.action_required), - int(result.outcome is ReconcileOutcome.error), - int(result.outcome is ReconcileOutcome.skipped), - safe_run_id, - ), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation result counter update failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() + self._database = SQLiteDatabase(database_path, synchronous=synchronous) + self._schema = SchemaManager(self._database) + self._idempotency = IdempotencyStore(self._database) + self._reconcile = ReconcileStore(self._database) + self._bot_lifecycle = BotLifecycleStore(self._database) - def finish_reconcile_run( - self, - summary: ReconcileRunSummary, - ) -> PersistedReconcileRun: - if not isinstance(summary, ReconcileRunSummary): - raise TypeError("summary must be a ReconcileRunSummary") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - run_row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (summary.run_id,), - ).fetchone() - if run_row is None: - raise KeyError(f"unknown reconciliation run: {summary.run_id}") - if run_row["outcome"] != "running": - raise RuntimeError("reconciliation run is not running") - if run_row["scope"] != summary.scope: - raise RuntimeError("reconciliation summary scope does not match run") - stored_started_at = _parse_reconcile_timestamp( - run_row["started_at"], - "run start timestamp", - ) - if stored_started_at != summary.started_at: - raise RuntimeError("reconciliation summary start timestamp does not match run") - persisted_results = self._reconcile_results_in_transaction(conn, summary.run_id) - if tuple(summary.results) != persisted_results: - raise RuntimeError("persisted reconciliation results do not match summary") - observed_counts = self._reconcile_counts_from_results(persisted_results) - stored_counts = self._reconcile_counts_from_run_row(run_row) - stored_total = _validate_stored_nonnegative_integer( - run_row["total"], - "total", - ) - if stored_counts != observed_counts or stored_total != len(persisted_results): - raise RuntimeError("persisted reconciliation counters do not match results") - if dict(summary.counts) != observed_counts or summary.total != len( - persisted_results - ): - raise RuntimeError("reconciliation summary counters do not match results") - if any(result.finished_at > summary.finished_at for result in persisted_results): - raise RuntimeError("reconciliation summary finishes before a persisted result") - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET finished_at = ?, - outcome = ?, - total = ?, - healthy_count = ?, - changed_count = ?, - pending_count = ?, - action_required_count = ?, - error_count = ?, - skipped_count = ? - WHERE run_id = ? AND outcome = 'running' - """, - ( - _serialize_reconcile_timestamp( - summary.finished_at, - "finish timestamp", - ), - summary.outcome, - summary.total, - summary.counts[ReconcileOutcome.healthy.value], - summary.counts[ReconcileOutcome.changed.value], - summary.counts[ReconcileOutcome.pending.value], - summary.counts[ReconcileOutcome.action_required.value], - summary.counts[ReconcileOutcome.error.value], - summary.counts[ReconcileOutcome.skipped.value], - summary.run_id, - ), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation run finalization failed") - finished = self._materialize_reconcile_run(conn, summary.run_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - if finished is None: - raise sqlite3.IntegrityError("finished reconciliation run is missing") - return finished + @property + def database_path(self) -> Path: + return self._database.database_path - def interrupt_stale_reconcile_runs(self, *, interrupted_at: datetime) -> int: - safe_interrupted_at = _serialize_reconcile_timestamp( - interrupted_at, - "interruption timestamp", - ) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - rows = conn.execute( - "SELECT * FROM reconcile_runs WHERE outcome = 'running' ORDER BY run_id" - ).fetchall() - for row in rows: - run_id = str(row["run_id"]) - current_run = self._materialize_reconcile_run(conn, run_id) - if current_run is None: - raise sqlite3.IntegrityError("reconciliation run disappeared") - if interrupted_at < current_run.started_at: - raise ValueError("interruption timestamp precedes a running reconciliation") - if any(result.finished_at > interrupted_at for result in current_run.results): - raise ValueError( - "reconciliation result finishes after the interruption timestamp" - ) - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET outcome = 'interrupted', finished_at = ? - WHERE run_id = ? AND outcome = 'running' - """, - (safe_interrupted_at, run_id), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation interruption update failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() - return len(rows) - - def get_reconcile_run(self, run_id: str) -> PersistedReconcileRun | None: - safe_run_id = _validate_reconcile_run_id(run_id) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN") - run = self._materialize_reconcile_run(conn, safe_run_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - return run - - def _materialize_reconcile_run( - self, - conn: sqlite3.Connection, - run_id: str, - ) -> PersistedReconcileRun | None: - row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (run_id,), - ).fetchone() - if row is None: - return None - results = self._reconcile_results_in_transaction(conn, run_id) - finished_at = row["finished_at"] - return PersistedReconcileRun( - run_id=str(row["run_id"]), - scope=str(row["scope"]), - requested_bot_id=( - str(row["requested_bot_id"]) if row["requested_bot_id"] is not None else None - ), - source=str(row["source"]), - force=_validate_stored_boolean_flag(row["force"], "force flag"), - reset_restart=_validate_stored_boolean_flag( - row["reset_restart"], - "reset-restart flag", - ), - started_at=_parse_reconcile_timestamp(row["started_at"], "run start timestamp"), - finished_at=( - _parse_reconcile_timestamp(finished_at, "run finish timestamp") - if finished_at is not None - else None - ), - outcome=str(row["outcome"]), - total=_validate_stored_nonnegative_integer(row["total"], "total"), - counts=self._reconcile_counts_from_run_row(row), - results=results, - ) + def connect(self) -> sqlite3.Connection: + return self._database.connect() - def _reconcile_results_in_transaction( - self, - conn: sqlite3.Connection, - run_id: str, - ) -> tuple[BotReconcileResult, ...]: - rows = conn.execute( - "SELECT * FROM reconcile_results WHERE run_id = ? ORDER BY ordinal", - (run_id,), - ).fetchall() - results: list[BotReconcileResult] = [] - for expected_ordinal, row in enumerate(rows): - ordinal = _validate_stored_nonnegative_integer(row["ordinal"], "result ordinal") - if ordinal != expected_ordinal: - raise ValueError("stored reconciliation ordinals are not contiguous") - bot_id = str(row["bot_id"]) - event_id = _validate_stored_optional_positive_integer( - row["event_id"], - "result event id", - ) - if event_id is not None: - event_row = conn.execute( - "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", - (event_id,), - ).fetchone() - if event_row is None or event_row["bot_id"] != bot_id: - raise ValueError( - "stored reconciliation lifecycle event does not match the result bot" - ) - results.append( - BotReconcileResult( - bot_id=bot_id, - outcome=ReconcileOutcome(str(row["outcome"])), - desired_state=( - str(row["desired_state"]) if row["desired_state"] is not None else None - ), - observed_status=( - str(row["observed_status"]) if row["observed_status"] is not None else None - ), - pid=_validate_stored_optional_positive_integer(row["pid"], "result pid"), - action=str(row["action"]), - message=str(row["message"]), - error_code=(str(row["error_code"]) if row["error_code"] is not None else None), - event_id=event_id, - started_at=_parse_reconcile_timestamp( - row["started_at"], - "result start timestamp", - ), - finished_at=_parse_reconcile_timestamp( - row["finished_at"], - "result finish timestamp", - ), - ) - ) - return tuple(results) + def check_readiness(self) -> int: + return self._database.check_readiness() - def _reconcile_counts_from_run_row(self, row: sqlite3.Row) -> dict[str, int]: - return { - outcome.value: _validate_stored_nonnegative_integer( - row[column], - f"{outcome.value} counter", - ) - for outcome, column in RECONCILE_COUNTER_COLUMNS.items() - } + def init(self) -> None: + self._schema.init() - def _reconcile_counts_from_results( - self, - results: tuple[BotReconcileResult, ...], - ) -> dict[str, int]: - counts = {outcome.value: 0 for outcome in ReconcileOutcome} - for result in results: - counts[result.outcome.value] += 1 - return counts + def migrate(self) -> None: + self._schema.migrate() def claim_idempotency( self, @@ -1163,91 +73,13 @@ def claim_idempotency( expires_at: datetime, max_records: int = 10_000, ) -> IdempotencyClaim: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") - if type(max_records) is not int or not 1 <= max_records <= 1_000_000: - raise ValueError("idempotency capacity is invalid") - current_time = datetime.now(UTC) - if datetime.fromisoformat(safe_expiry) < current_time: - raise ValueError("idempotency expiry timestamp is invalid") - now = current_time.isoformat() - - try: - conn = self.connect() - except (OSError, sqlite3.Error): - return IdempotencyClaim("unavailable") - with closing(conn): - try: - conn.execute("BEGIN IMMEDIATE") - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is not None: - validated = _validate_idempotency_row(row) - reclaimable = validated.expires_at <= current_time and ( - validated.state == "completed" or validated.owner_instance_id != safe_owner - ) - if reclaimable: - conn.execute( - "DELETE FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ) - row = None - conn.execute( - """ - DELETE FROM idempotency_records - WHERE expires_at <= ? - AND (state = 'completed' OR owner_instance_id != ?) - AND key_hash != ? - """, - (now, safe_owner, safe_key_hash), - ) - if row is None: - count = int( - conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] - ) - if count >= max_records: - conn.commit() - return IdempotencyClaim("unavailable") - conn.execute( - """ - INSERT INTO idempotency_records ( - key_hash, - request_hash, - state, - owner_instance_id, - response_status, - response_json, - created_at, - updated_at, - expires_at - ) VALUES (?, ?, 'in_progress', ?, NULL, NULL, ?, ?, ?) - """, - ( - safe_key_hash, - safe_request_hash, - safe_owner, - now, - now, - safe_expiry, - ), - ) - conn.commit() - return IdempotencyClaim("claimed") - - result = _idempotency_claim_from_row( - row, - request_hash=safe_request_hash, - owner_instance_id=safe_owner, - ) - conn.commit() - return result - except (OSError, sqlite3.Error, ValueError, TypeError): - conn.rollback() - return IdempotencyClaim("unavailable") + return self._idempotency.claim_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + expires_at=expires_at, + max_records=max_records, + ) def lookup_idempotency( self, @@ -1256,35 +88,11 @@ def lookup_idempotency( request_hash: str, owner_instance_id: str, ) -> IdempotencyClaim | None: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - current_time = datetime.now(UTC) - - try: - conn = self.connect() - except (OSError, sqlite3.Error): - return IdempotencyClaim("unavailable") - with closing(conn): - try: - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is None: - return None - validated = _validate_idempotency_row(row) - if validated.expires_at <= current_time and ( - validated.state == "completed" or validated.owner_instance_id != safe_owner - ): - return None - return _idempotency_claim_from_row( - row, - request_hash=safe_request_hash, - owner_instance_id=safe_owner, - ) - except (OSError, sqlite3.Error, ValueError, TypeError): - return IdempotencyClaim("unavailable") + return self._idempotency.lookup_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + ) def complete_idempotency( self, @@ -1297,65 +105,37 @@ def complete_idempotency( completed_at: datetime, expires_at: datetime, ) -> None: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - safe_status = _validate_idempotency_status(response_status) - safe_json = _validate_idempotency_response_json(response_json) - safe_completed = _validate_idempotency_timestamp(completed_at, "completion timestamp") - safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") - if safe_expiry < safe_completed: - raise ValueError("idempotency expiry timestamp is invalid") - completed_time = datetime.fromisoformat(safe_completed) + self._idempotency.complete_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + response_status=response_status, + response_json=response_json, + completed_at=completed_at, + expires_at=expires_at, + ) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is None: - raise RuntimeError("idempotency completion failed") - validated = _validate_idempotency_row(row) - if ( - validated.request_hash != safe_request_hash - or validated.owner_instance_id != safe_owner - or validated.state != "in_progress" - ): - raise RuntimeError("idempotency completion failed") - if completed_time < validated.created_at: - raise ValueError("idempotency completion timestamp is invalid") - cursor = conn.execute( - """ - UPDATE idempotency_records - SET state = 'completed', - response_status = ?, - response_json = ?, - updated_at = ?, - expires_at = ? - WHERE key_hash = ? - AND request_hash = ? - AND owner_instance_id = ? - AND state = 'in_progress' - """, - ( - safe_status, - safe_json, - safe_completed, - safe_expiry, - safe_key_hash, - safe_request_hash, - safe_owner, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("idempotency completion failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() + def begin_reconcile_run(self, run: ReconcileRunStart) -> None: + self._reconcile.begin_reconcile_run(run) + + def append_reconcile_result( + self, + run_id: str, + result: BotReconcileResult, + ) -> None: + self._reconcile.append_reconcile_result(run_id, result) + + def finish_reconcile_run( + self, + summary: ReconcileRunSummary, + ) -> PersistedReconcileRun: + return self._reconcile.finish_reconcile_run(summary) + + def interrupt_stale_reconcile_runs(self, *, interrupted_at: datetime) -> int: + return self._reconcile.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at) + + def get_reconcile_run(self, run_id: str) -> PersistedReconcileRun | None: + return self._reconcile.get_reconcile_run(run_id) def begin_lifecycle_intent( self, @@ -1367,92 +147,14 @@ def begin_lifecycle_intent( request_id: str | None = None, reason: str = "", ) -> BotRecord: - self._validate_intent_action(action) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - event = LifecycleEventInput( - bot_id=bot_id, + return self._bot_lifecycle.begin_lifecycle_intent( + bot_id, + action=action, operation_id=operation_id, source=source, - action=f"bot.{action}.intent", - outcome="pending", request_id=request_id, reason=reason, - details={"action": action, "desired_state": desired_state.value}, ) - self._validate_event_target(bot_id, event) - now = datetime.now(UTC) - now_text = now.isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - if prior_row["pending_operation_id"] is not None: - raise RuntimeError("bot already has a pending lifecycle intent") - desired_revision = int(prior_row["desired_revision"]) + 1 - cursor = conn.execute( - """ - UPDATE bots - SET desired_state = ?, - desired_revision = ?, - desired_updated_at = ?, - pending_operation_id = ?, - pending_action = ?, - pending_since = ?, - updated_at = ? - WHERE bot_id = ? AND pending_operation_id IS NULL - """, - ( - desired_state.value, - desired_revision, - now_text, - operation_id, - action, - now_text, - now_text, - bot_id, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("lifecycle intent could not be started") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.intent", - outcome="pending", - request_id=request_id, - reason=reason, - details={ - "action": action, - "desired_state": desired_state.value, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=str(prior_row["status"]), - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record def complete_lifecycle_intent( self, @@ -1479,101 +181,29 @@ def complete_lifecycle_intent( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> BotRecord: - self._validate_intent_action(action) - self._validate_intent_revision(desired_revision) - self._validate_intent_correlation(bot_id, operation_id, source, request_id) - self._validate_intent_completion( - action, - outcome=outcome, + return self._bot_lifecycle.complete_lifecycle_intent( + bot_id, + action=action, + operation_id=operation_id, + desired_revision=desired_revision, status=status, pid=pid, + source=source, + outcome=outcome, + request_id=request_id, + reason=reason, error_code=error_code, error_message=error_message, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, ) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._require_matching_intent( - prior_row, - action, - operation_id, - desired_revision, - ) - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - cursor = conn.execute( - """ - UPDATE bots - SET pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL - WHERE bot_id = ? - AND pending_operation_id = ? - AND pending_action = ? - AND desired_revision = ? - AND desired_state = ? - """, - (bot_id, operation_id, action, desired_revision, desired_state.value), - ) - if cursor.rowcount != 1: - raise RuntimeError("pending lifecycle intent does not match") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.complete", - outcome=outcome, - request_id=request_id, - reason=reason, - error_code=error_code, - error_message=error_message, - details={ - "action": action, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record def clear_stale_intent( self, @@ -1586,276 +216,24 @@ def clear_stale_intent( reason: str, request_id: str | None = None, ) -> BotRecord: - self._validate_intent_action(action) - self._validate_intent_revision(desired_revision) - self._validate_intent_correlation(bot_id, operation_id, source, request_id) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._require_matching_intent( - prior_row, - action, - operation_id, - desired_revision, - ) - cursor = conn.execute( - """ - UPDATE bots - SET pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL, - updated_at = ? - WHERE bot_id = ? - AND pending_operation_id = ? - AND pending_action = ? - AND desired_revision = ? - AND desired_state = ? - """, - ( - now, - bot_id, - operation_id, - action, - desired_revision, - desired_state.value, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("pending lifecycle intent does not match") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.intent.clear", - outcome="cleared", - request_id=request_id, - reason=reason, - details={ - "action": action, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=str(prior_row["status"]), - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record - - def _validate_intent_action(self, action: str) -> None: - if type(action) is not str or action not in LIFECYCLE_INTENT_ACTIONS: - raise ValueError("invalid lifecycle intent action") - - def _validate_intent_completion( - self, - action: str, - *, - outcome: Literal["success", "failure"], - status: BotStatus, - pid: int | None, - error_code: str | None, - error_message: str | None, - ) -> None: - if outcome not in {"success", "failure"}: - raise ValueError("invalid lifecycle intent outcome") - if not isinstance(status, BotStatus): - raise TypeError("lifecycle intent status must be a BotStatus") - if pid is not None and (type(pid) is not int or pid <= 0): - raise ValueError("lifecycle intent terminal state has an invalid PID") - if error_code is not None and ( - type(error_code) is not str or LIFECYCLE_ERROR_CODE_RE.fullmatch(error_code) is None - ): - raise ValueError("invalid lifecycle intent error code") - if error_message is not None and type(error_message) is not str: - raise TypeError("lifecycle intent error message must be a string") - if outcome == "success": - if error_code is not None or error_message is not None: - raise ValueError("successful lifecycle intent cannot include error metadata") - if action == "stop": - compatible = status is BotStatus.stopped and pid is None - else: - compatible = status in {BotStatus.starting, BotStatus.running} and pid is not None - else: - compatible = status in {BotStatus.failed, BotStatus.unknown, BotStatus.stopped} - if status in {BotStatus.failed, BotStatus.stopped} and pid is not None: - compatible = False - if not compatible: - raise ValueError("lifecycle intent terminal state is incompatible with action outcome") - - def _validate_intent_revision(self, desired_revision: int) -> None: - if type(desired_revision) is not int or desired_revision < 1: - raise ValueError("desired revision must be a positive integer") - - def _validate_intent_correlation( - self, - bot_id: str, - operation_id: str, - source: str, - request_id: str | None, - ) -> None: - event = LifecycleEventInput( - bot_id=bot_id, + return self._bot_lifecycle.clear_stale_intent( + bot_id, + action=action, operation_id=operation_id, + desired_revision=desired_revision, source=source, - action="lifecycle.intent.validate", - outcome="validation", + reason=reason, request_id=request_id, ) - self._validate_event_target(bot_id, event) - - def _require_matching_intent( - self, - row: sqlite3.Row, - action: str, - operation_id: str, - desired_revision: int, - ) -> None: - desired_state = ( - DesiredState.stopped.value if action == "stop" else DesiredState.running.value - ) - if ( - row["pending_operation_id"] != operation_id - or int(row["desired_revision"]) != desired_revision - or row["pending_action"] != action - or row["desired_state"] != desired_state - ): - raise RuntimeError("pending lifecycle intent does not match") - - def _record_in_transaction(self, conn: sqlite3.Connection, bot_id: str) -> BotRecord: - row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() - if row is None: - raise sqlite3.IntegrityError("updated bot projection is missing") - return self._row_to_record(row) def audit_log_path(self) -> Path: - return self.database_path.parent / "logs" / "audit.jsonl" + return self._bot_lifecycle.audit_log_path() def append_audit_event(self, event: str, **fields: object) -> None: - payload: dict[str, object] = { - "ts": datetime.now(UTC).isoformat(), - "event": event, - } - for key, value in fields.items(): - payload[key] = _safe_audit_value(key, value) - try: - path = self.audit_log_path() - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, sort_keys=True) + "\n") - except (OSError, TypeError, ValueError): - return + self._bot_lifecycle.append_audit_event(event, **fields) def upsert_bot(self, record: BotRecord) -> None: - with closing(self.connect()) as conn: - self._upsert_bot_row(conn, record) - conn.commit() - - def _upsert_bot_row(self, conn: sqlite3.Connection, record: BotRecord) -> None: - conn.execute( - """ - INSERT INTO bots ( - bot_id, - template_id, - display_name, - profile_path, - status, - pid, - restart_policy, - restart_backoff_seconds, - restart_max_attempts, - restart_attempts, - next_restart_at, - started_at, - ready_at, - stopped_at, - last_exit_code, - last_error, - last_transition_reason, - desired_state, - desired_revision, - desired_updated_at, - pending_operation_id, - pending_action, - pending_since, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(bot_id) DO UPDATE SET - template_id = excluded.template_id, - display_name = excluded.display_name, - profile_path = excluded.profile_path, - status = excluded.status, - pid = excluded.pid, - restart_policy = excluded.restart_policy, - restart_backoff_seconds = excluded.restart_backoff_seconds, - restart_max_attempts = excluded.restart_max_attempts, - restart_attempts = excluded.restart_attempts, - next_restart_at = excluded.next_restart_at, - started_at = excluded.started_at, - ready_at = excluded.ready_at, - stopped_at = excluded.stopped_at, - last_exit_code = excluded.last_exit_code, - last_error = excluded.last_error, - last_transition_reason = excluded.last_transition_reason, - desired_state = excluded.desired_state, - desired_revision = excluded.desired_revision, - desired_updated_at = excluded.desired_updated_at, - pending_operation_id = excluded.pending_operation_id, - pending_action = excluded.pending_action, - pending_since = excluded.pending_since, - updated_at = excluded.updated_at - """, - ( - record.bot_id, - record.template_id, - record.display_name, - record.profile_path, - record.status.value, - record.pid, - record.restart_policy.value, - record.restart_backoff_seconds, - record.restart_max_attempts, - record.restart_attempts, - record.next_restart_at.isoformat() if record.next_restart_at else None, - record.started_at.isoformat() if record.started_at else None, - record.ready_at.isoformat() if record.ready_at else None, - record.stopped_at.isoformat() if record.stopped_at else None, - record.last_exit_code, - record.last_error, - record.last_transition_reason, - record.desired_state.value, - record.desired_revision, - record.desired_updated_at.isoformat() if record.desired_updated_at else None, - record.pending_operation_id, - record.pending_action, - record.pending_since.isoformat() if record.pending_since else None, - record.created_at.isoformat(), - record.updated_at.isoformat(), - ), - ) + self._bot_lifecycle.upsert_bot(record) def upsert_bot_with_event( self, @@ -1863,44 +241,13 @@ def upsert_bot_with_event( *, event: LifecycleEventInput, ) -> LifecycleEvent: - self._validate_event_target(record.bot_id, event) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (record.bot_id,) - ).fetchone() - self._upsert_bot_row(conn, record) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]) if prior_row else None, - status_after=record.status.value, - pid_before=int(prior_row["pid"]) if prior_row and prior_row["pid"] else None, - pid_after=record.pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, record.bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.upsert_bot_with_event(record, event=event) def get_bot(self, bot_id: str) -> BotRecord | None: - with closing(self.connect()) as conn: - row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() - return self._row_to_record(row) if row else None + return self._bot_lifecycle.get_bot(bot_id) def list_bots(self) -> list[BotRecord]: - with closing(self.connect()) as conn: - rows = conn.execute("SELECT * FROM bots ORDER BY bot_id").fetchall() - return [self._row_to_record(row) for row in rows] + return self._bot_lifecycle.list_bots() def update_status( self, @@ -1910,27 +257,12 @@ def update_status( *, reset_restart: bool = False, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - if reset_restart: - conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - restart_attempts = 0, - next_restart_at = NULL, - updated_at = ? - WHERE bot_id = ? - """, - (status.value, pid, now, bot_id), - ) - else: - conn.execute( - "UPDATE bots SET status = ?, pid = ?, updated_at = ? WHERE bot_id = ?", - (status.value, pid, now, bot_id), - ) - conn.commit() + self._bot_lifecycle.update_status( + bot_id, + status, + pid, + reset_restart=reset_restart, + ) def update_lifecycle_state( self, @@ -1948,79 +280,20 @@ def update_lifecycle_state( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - conn.commit() - - def _update_lifecycle_row( - self, - conn: sqlite3.Connection, - bot_id: str, - status: BotStatus, - pid: int | None, - *, - started_at: datetime | None, - ready_at: datetime | None, - stopped_at: datetime | None, - last_exit_code: int | None, - last_error: str | None, - last_transition_reason: str | None, - reset_restart: bool, - clear_ready_at: bool, - clear_stopped_at: bool, - now: str, - ) -> None: - cursor = conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - started_at = COALESCE(?, started_at), - ready_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, ready_at) END, - stopped_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, stopped_at) END, - last_exit_code = ?, - last_error = ?, - last_transition_reason = COALESCE(?, last_transition_reason), - updated_at = ?, - restart_attempts = CASE WHEN ? THEN 0 ELSE restart_attempts END, - next_restart_at = CASE WHEN ? THEN NULL ELSE next_restart_at END - WHERE bot_id = ? - """, - ( - status.value, - pid, - started_at.isoformat() if started_at else None, - int(clear_ready_at), - ready_at.isoformat() if ready_at else None, - int(clear_stopped_at), - stopped_at.isoformat() if stopped_at else None, - last_exit_code, - last_error, - last_transition_reason, - now, - int(reset_restart), - int(reset_restart), - bot_id, - ), + self._bot_lifecycle.update_lifecycle_state( + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, ) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") def update_lifecycle_with_event( self, @@ -2039,52 +312,21 @@ def update_lifecycle_with_event( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> LifecycleEvent: - self._validate_event_target(bot_id, event) - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.update_lifecycle_with_event( + bot_id, + status, + pid, + event=event, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + ) def update_restart_state( self, @@ -2095,51 +337,13 @@ def update_restart_state( restart_attempts: int, next_restart_at: datetime | None, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - self._update_restart_row( - conn, - bot_id, - status=status, - pid=pid, - restart_attempts=restart_attempts, - next_restart_at=next_restart_at, - now=now, - ) - conn.commit() - - def _update_restart_row( - self, - conn: sqlite3.Connection, - bot_id: str, - *, - status: BotStatus, - pid: int | None, - restart_attempts: int, - next_restart_at: datetime | None, - now: str, - ) -> None: - cursor = conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - restart_attempts = ?, - next_restart_at = ?, - updated_at = ? - WHERE bot_id = ? - """, - ( - status.value, - pid, - restart_attempts, - next_restart_at.isoformat() if next_restart_at else None, - now, - bot_id, - ), + self._bot_lifecycle.update_restart_state( + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, ) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") def update_restart_with_event( self, @@ -2151,51 +355,17 @@ def update_restart_with_event( next_restart_at: datetime | None, event: LifecycleEventInput, ) -> LifecycleEvent: - self._validate_event_target(bot_id, event) - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._update_restart_row( - conn, - bot_id, - status=status, - pid=pid, - restart_attempts=restart_attempts, - next_restart_at=next_restart_at, - now=now, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.update_restart_with_event( + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + event=event, + ) def delete_bot(self, bot_id: str) -> bool: - with closing(self.connect()) as conn: - cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) - conn.commit() - return cursor.rowcount > 0 + return self._bot_lifecycle.delete_bot(bot_id) def delete_bot_with_event( self, @@ -2203,135 +373,7 @@ def delete_bot_with_event( *, event: LifecycleEventInput, ) -> bool: - self._validate_event_target(bot_id, event) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=None, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=None, - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return True - - def _validate_event_target(self, bot_id: str, event: LifecycleEventInput) -> None: - if event.bot_id != bot_id: - raise ValueError("lifecycle event bot_id must match the projection target") - if LIFECYCLE_ID_RE.fullmatch(event.operation_id) is None: - raise ValueError("lifecycle operation_id must be generated UUID hex") - if event.source not in LIFECYCLE_SOURCES: - raise ValueError("invalid lifecycle event source") - if event.source == "api": - if event.request_id is None or LIFECYCLE_ID_RE.fullmatch(event.request_id) is None: - raise ValueError("API lifecycle events require a generated request ID") - elif event.request_id is not None: - raise ValueError("only API lifecycle events may carry a request ID") - - def _insert_lifecycle_event( - self, - conn: sqlite3.Connection, - event: LifecycleEventInput, - *, - status_before: str | None, - status_after: str | None, - pid_before: int | None, - pid_after: int | None, - ) -> int: - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, - operation_id, - request_id, - occurred_at, - source, - action, - outcome, - status_before, - status_after, - pid_before, - pid_after, - reason, - error_code, - error_message, - details_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - event.bot_id, - event.operation_id, - event.request_id, - datetime.now(UTC).isoformat(), - event.source, - event.action, - event.outcome, - status_before, - status_after, - pid_before, - pid_after, - event.reason, - event.error_code, - event.error_message, - serialize_lifecycle_details(event.details), - ), - ) - if cursor.lastrowid is None: - raise sqlite3.IntegrityError("lifecycle event insert returned no event id") - return int(cursor.lastrowid) - - def _materialize_lifecycle_event( - self, - conn: sqlite3.Connection, - event_id: int, - ) -> LifecycleEvent: - row = conn.execute( - "SELECT * FROM lifecycle_events WHERE event_id = ?", (event_id,) - ).fetchone() - if row is None: - raise sqlite3.IntegrityError("inserted lifecycle event is missing") - return self._row_to_lifecycle_event(row) - - def _append_lifecycle_audit_fail_open(self, event: LifecycleEvent) -> None: - try: - self._append_lifecycle_audit(event) - except Exception: - return - - def _append_lifecycle_audit(self, event: LifecycleEvent) -> None: - self.append_audit_event( - event.action, - bot_id=event.bot_id, - operation_id=event.operation_id, - request_id=event.request_id, - source=event.source, - outcome=event.outcome, - status_before=event.status_before, - status_after=event.status_after, - pid_before=event.pid_before, - pid_after=event.pid_after, - reason=event.reason, - error_code=event.error_code, - error_message=event.error_message, - details=event.details, - ) + return self._bot_lifecycle.delete_bot_with_event(bot_id, event=event) def list_lifecycle_events( self, @@ -2339,10 +381,7 @@ def list_lifecycle_events( limit: int, before: int | None, ) -> list[LifecycleEvent]: - self._validate_history_page(bot_id, limit, before) - with closing(self.connect()) as conn: - rows = self._list_lifecycle_event_rows(conn, bot_id, limit, before) - return [self._row_to_lifecycle_event(row) for row in rows] + return self._bot_lifecycle.list_lifecycle_events(bot_id, limit, before) def history_payload( self, @@ -2350,130 +389,4 @@ def history_payload( limit: int, before: int | None, ) -> dict[str, object]: - self._validate_history_page(bot_id, limit, before) - with closing(self.connect()) as conn: - rows = self._list_lifecycle_event_rows(conn, bot_id, limit + 1, before) - if not rows: - bot_is_known = ( - conn.execute( - """ - SELECT 1 FROM bots WHERE bot_id = ? - UNION ALL - SELECT 1 FROM lifecycle_events WHERE bot_id = ? - LIMIT 1 - """, - (bot_id, bot_id), - ).fetchone() - is not None - ) - if not bot_is_known: - raise KeyError(f"unknown bot: {bot_id}") - has_more = len(rows) > limit - events = [self._row_to_lifecycle_event(row) for row in rows[:limit]] - return { - "bot_id": bot_id, - "events": [event.to_dict() for event in events], - "next_before": events[-1].event_id if has_more else None, - } - - def _validate_history_page( - self, - bot_id: str, - limit: int, - before: int | None, - ) -> None: - if not isinstance(bot_id, str): - raise TypeError("bot_id must be a string") - if isinstance(limit, bool) or not isinstance(limit, int): - raise TypeError("limit must be an integer") - if limit < 1 or limit > 1000: - raise ValueError("limit must be between 1 and 1000") - if before is not None: - if isinstance(before, bool) or not isinstance(before, int): - raise TypeError("before must be an integer or null") - if before < 1: - raise ValueError("before must be positive") - - def _list_lifecycle_event_rows( - self, - conn: sqlite3.Connection, - bot_id: str, - limit: int, - before: int | None, - ) -> list[sqlite3.Row]: - if before is None: - return conn.execute( - """ - SELECT * FROM lifecycle_events - WHERE bot_id = ? - ORDER BY event_id DESC - LIMIT ? - """, - (bot_id, limit), - ).fetchall() - return conn.execute( - """ - SELECT * FROM lifecycle_events - WHERE bot_id = ? AND event_id < ? - ORDER BY event_id DESC - LIMIT ? - """, - (bot_id, before, limit), - ).fetchall() - - def _row_to_lifecycle_event(self, row: sqlite3.Row) -> LifecycleEvent: - return LifecycleEvent( - event_id=int(row["event_id"]), - bot_id=str(row["bot_id"]), - operation_id=str(row["operation_id"]), - request_id=str(row["request_id"]) if row["request_id"] is not None else None, - occurred_at=datetime.fromisoformat(str(row["occurred_at"])), - source=str(row["source"]), - action=str(row["action"]), - outcome=str(row["outcome"]), - status_before=(str(row["status_before"]) if row["status_before"] is not None else None), - status_after=(str(row["status_after"]) if row["status_after"] is not None else None), - pid_before=int(row["pid_before"]) if row["pid_before"] is not None else None, - pid_after=int(row["pid_after"]) if row["pid_after"] is not None else None, - reason=str(row["reason"]), - error_code=str(row["error_code"]) if row["error_code"] is not None else None, - error_message=(str(row["error_message"]) if row["error_message"] is not None else None), - details=deserialize_lifecycle_details(str(row["details_json"])), - ) - - def _row_to_record(self, row: sqlite3.Row) -> BotRecord: - next_restart_at = row["next_restart_at"] - started_at = row["started_at"] - ready_at = row["ready_at"] - stopped_at = row["stopped_at"] - desired_updated_at = row["desired_updated_at"] - pending_since = row["pending_since"] - return BotRecord( - bot_id=row["bot_id"], - template_id=row["template_id"], - display_name=row["display_name"], - profile_path=row["profile_path"], - status=BotStatus(row["status"]), - pid=row["pid"], - restart_policy=RestartPolicy(row["restart_policy"]), - restart_backoff_seconds=float(row["restart_backoff_seconds"]), - restart_max_attempts=int(row["restart_max_attempts"]), - restart_attempts=int(row["restart_attempts"]), - next_restart_at=datetime.fromisoformat(next_restart_at) if next_restart_at else None, - started_at=datetime.fromisoformat(started_at) if started_at else None, - ready_at=datetime.fromisoformat(ready_at) if ready_at else None, - stopped_at=datetime.fromisoformat(stopped_at) if stopped_at else None, - last_exit_code=row["last_exit_code"], - last_error=row["last_error"], - last_transition_reason=row["last_transition_reason"], - desired_state=DesiredState(row["desired_state"]), - desired_revision=int(row["desired_revision"]), - desired_updated_at=( - datetime.fromisoformat(desired_updated_at) if desired_updated_at else None - ), - pending_operation_id=row["pending_operation_id"], - pending_action=row["pending_action"], - pending_since=datetime.fromisoformat(pending_since) if pending_since else None, - created_at=datetime.fromisoformat(row["created_at"]), - updated_at=datetime.fromisoformat(row["updated_at"]), - ) + return self._bot_lifecycle.history_payload(bot_id, limit, before) diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 49e0e1c..54038a9 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -1,30 +1,19 @@ from __future__ import annotations import contextlib -import errno -import json -import math import os import platform import re -import select -import shlex -import shutil import signal -import stat import subprocess # nosec B404 -import sys import threading -import time import uuid -from collections.abc import Callable, Sequence +from collections.abc import Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta -from enum import Enum from pathlib import Path -from typing import Protocol, TypeGuard -from urllib.parse import urlparse +from zeus import process_identity as _process_identity from zeus.errors import ( BotArchiveError, BotDeleteError, @@ -32,21 +21,31 @@ BotReplaceError, BotRunningError, ) -from zeus.fs_utils import atomic_write_json from zeus.gateway_launcher import ( - MAX_PAYLOAD_BYTES, LaunchPayloadError, - _is_owned_runtime_marker, - _open_logs, - _open_profile, - _open_regular_marker, _read_bounded_file, - _reject_duplicate_keys, _remove_marker_if_owned_locked, - marker_publication_lock, - remove_marker_if_owned, +) +from zeus.gateway_marker import ( + GatewayGeneration, + is_compat_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.gateway_runtime import ( + GatewayRuntime, + KillFn, + MarkerObservation, + OwnershipCheck, + PopenFactory, + PopenLike, + RuntimeHooks, + SignalResult, + StopEffect, + gateway_process_launch_kwargs, ) from zeus.hermes_adapter import HermesAdapter +from zeus.intent_recovery import PendingIntentRecovery from zeus.lifecycle import LifecycleEvent, LifecycleEventInput from zeus.logging_utils import tail_file from zeus.models import ( @@ -57,11 +56,12 @@ DesiredState, HermesTemplate, RestartPolicy, - TemplateError, validate_id, ) +from zeus.private_io import nofollow_absolute_path from zeus.process_lock import BotProcessLock, LockTimeoutError -from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env +from zeus.profile_manager import ProfileArchive, ProfileDeletion, ProfileManager +from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once from zeus.reconciliation import ( BotReconcileResult, FleetReconciler, @@ -71,54 +71,31 @@ ReconcileRunSummary, ReconcileSnapshotDriftError, ) -from zeus.renderer import ProfileRenderer from zeus.state import StateStore +PidAliveFn = _process_identity.PidAliveFn +CmdlineReader = _process_identity.CmdlineReader +ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader -class PopenLike(Protocol): - pid: int - - def poll(self) -> int | None: ... - - -PopenFactory = Callable[..., PopenLike] -KillFn = Callable[[int, signal.Signals], None] -PidAliveFn = Callable[[int], bool] -CmdlineReader = Callable[[int], list[str] | None] -ProcStartFingerprintReader = Callable[[int], str | None] - - -@dataclass(frozen=True) -class OwnershipCheck: - verified: bool - reason: str - classification: str | None = None - - -@dataclass(frozen=True) -class _CommandCheck: - verified: bool - reason: str - classification: str | None = None - +_CommandCheck = _process_identity.CommandCheck +_PidState = _process_identity.PidState +_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter +_read_linux_cmdline = _process_identity.read_linux_cmdline +_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint +_resolve_executable = _process_identity.resolve_executable +_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target +_safe_command_shape = _process_identity.safe_command_shape +_trusted_hermes_paths = _process_identity.trusted_hermes_paths +_verify_gateway_command = _process_identity.verify_gateway_command -class _PidState(Enum): - alive = "alive" - dead = "dead" - unknown = "unknown" - -class _SignalResult(Enum): - sent = "sent" - missing = "missing" - denied = "denied" +_SignalResult = SignalResult class _ReadinessProbeUnset: pass -_PYTHON_INTERPRETER_RE = re.compile(r"^python(?:\d+(?:\.\d+)?)?$") _REQUEST_ID_RE = re.compile(r"^[0-9a-f]{32}$") _LIFECYCLE_SOURCES = frozenset({"api", "cli", "reconcile", "recovery", "system"}) _READINESS_PROBE_UNSET = _ReadinessProbeUnset() @@ -131,20 +108,10 @@ class _LifecycleContext: request_id: str | None -@dataclass(frozen=True) -class _MarkerObservation: - kind: str - payload: dict[str, object] | None = None - reason: str = "" +_MarkerObservation = MarkerObservation -@dataclass(frozen=True) -class _GatewayGeneration: - operation_id: str - desired_revision: int - pid: int - command_fingerprint: str - proc_start_fingerprint: str | None +_GatewayGeneration = GatewayGeneration @dataclass(frozen=True) @@ -156,26 +123,24 @@ class _ReconcileLaunch: def _gateway_process_launch_kwargs() -> dict[str, object]: - if os.name == "posix": - return {"start_new_session": True} - creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) - return {"creationflags": creationflags} if creationflags else {} + return gateway_process_launch_kwargs() def _nofollow_absolute_path(path: Path) -> Path: - absolute = Path(os.path.abspath(path.expanduser())) - if ( - sys.platform == "darwin" - and len(absolute.parts) > 1 - and absolute.parts[1] - in { - "etc", - "tmp", - "var", - } - ): - return Path("/private", *absolute.parts[1:]) - return absolute + return nofollow_absolute_path(path) + + +def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False class Supervisor: @@ -204,28 +169,194 @@ def __init__( hermes_bin=hermes_bin, hermes_root=configured_hermes_root.resolve(), ) + self._profile_manager = ProfileManager( + self.adapter.hermes_root, + self.store.database_path.parent / "archive", + ) self._marker_profiles_root = configured_hermes_root / "profiles" - self.popen_factory = popen_factory - self.kill_fn = kill_fn - self.pid_alive_fn = pid_alive_fn - self.cmdline_reader = cmdline_reader or _read_process_cmdline self.startup_grace_seconds = startup_grace_seconds - self.stop_grace_seconds = stop_grace_seconds - self.kill_after_timeout = kill_after_timeout self.lock_dir = self.store.database_path.parent / "locks" / "bots" - self.lock_timeout_seconds = lock_timeout_seconds self.readiness_timeout_seconds = readiness_timeout_seconds self.readiness_interval_seconds = readiness_interval_seconds self.allow_legacy_pid_markers = allow_legacy_pid_markers self.restart_backoff_cap_seconds = restart_backoff_cap_seconds - self.proc_start_fingerprint_reader = ( - proc_start_fingerprint_reader or _read_process_start_fingerprint - ) self._cleanup_process_group = os.name == "posix" and popen_factory is subprocess.Popen - self._processes: dict[str, PopenLike] = {} + self._runtime = GatewayRuntime( + self.adapter, + self._profile_manager, + self._marker_profiles_root, + popen_factory=popen_factory, + kill_fn=kill_fn, + pid_alive_fn=pid_alive_fn, + cmdline_reader=cmdline_reader or _read_process_cmdline, + proc_start_fingerprint_reader=( + proc_start_fingerprint_reader or _read_process_start_fingerprint + ), + startup_grace_seconds=startup_grace_seconds, + stop_grace_seconds=stop_grace_seconds, + kill_after_timeout=kill_after_timeout, + lock_timeout_seconds=lock_timeout_seconds, + readiness_timeout_seconds=readiness_timeout_seconds, + readiness_interval_seconds=readiness_interval_seconds, + allow_legacy_pid_markers=allow_legacy_pid_markers, + cleanup_process_group=self._cleanup_process_group, + hooks_provider=self._runtime_hooks, + ) + self._intent_recovery = PendingIntentRecovery() self._locks_guard = threading.Lock() self._bot_locks: dict[str, threading.RLock] = {} + def _runtime_hooks(self) -> RuntimeHooks: + return RuntimeHooks( + pipe=os.pipe, + close=os.close, + read_bounded_file=_read_bounded_file, + remove_marker_if_owned_locked=_remove_marker_if_owned_locked, + probe_once=probe_once, + ) + + def _get_runtime_proxy(self, name: str) -> object: + runtime = self.__dict__.get("_runtime") + if runtime is not None: + return getattr(runtime, name) + return self.__dict__.get(f"_runtime_proxy_{name}") + + def _set_runtime_proxy(self, name: str, value: object) -> None: + runtime = self.__dict__.get("_runtime") + history = self.__dict__.setdefault(f"_runtime_proxy_history_{name}", []) + if isinstance(history, list): + if len(history) >= 32: + del history[0] + history.append( + getattr(runtime, name) + if runtime is not None + else self.__dict__.get(f"_runtime_proxy_{name}") + ) + if runtime is not None: + setattr(runtime, name, value) + else: + self.__dict__[f"_runtime_proxy_{name}"] = value + + def _delete_runtime_proxy(self, name: str) -> None: + history = self.__dict__.get(f"_runtime_proxy_history_{name}") + if not isinstance(history, list) or not history: + self.__dict__.pop(f"_runtime_proxy_{name}", None) + return + previous = history.pop() + runtime = self.__dict__.get("_runtime") + if runtime is not None: + setattr(runtime, name, previous) + else: + self.__dict__[f"_runtime_proxy_{name}"] = previous + + @property + def popen_factory(self) -> PopenFactory: + return self._get_runtime_proxy("popen_factory") # type: ignore[return-value] + + @popen_factory.setter + def popen_factory(self, value: PopenFactory) -> None: + self._set_runtime_proxy("popen_factory", value) + + @popen_factory.deleter + def popen_factory(self) -> None: + self._delete_runtime_proxy("popen_factory") + + @property + def kill_fn(self) -> KillFn: + return self._get_runtime_proxy("kill_fn") # type: ignore[return-value] + + @kill_fn.setter + def kill_fn(self, value: KillFn) -> None: + self._set_runtime_proxy("kill_fn", value) + + @kill_fn.deleter + def kill_fn(self) -> None: + self._delete_runtime_proxy("kill_fn") + + @property + def pid_alive_fn(self) -> PidAliveFn | None: + return self._get_runtime_proxy("pid_alive_fn") # type: ignore[return-value] + + @pid_alive_fn.setter + def pid_alive_fn(self, value: PidAliveFn | None) -> None: + self._set_runtime_proxy("pid_alive_fn", value) + + @pid_alive_fn.deleter + def pid_alive_fn(self) -> None: + self._delete_runtime_proxy("pid_alive_fn") + + @property + def cmdline_reader(self) -> CmdlineReader: + return self._get_runtime_proxy("cmdline_reader") # type: ignore[return-value] + + @cmdline_reader.setter + def cmdline_reader(self, value: CmdlineReader) -> None: + self._set_runtime_proxy("cmdline_reader", value) + + @cmdline_reader.deleter + def cmdline_reader(self) -> None: + self._delete_runtime_proxy("cmdline_reader") + + @property + def proc_start_fingerprint_reader(self) -> ProcStartFingerprintReader: + return self._get_runtime_proxy("proc_start_fingerprint_reader") # type: ignore[return-value] + + @proc_start_fingerprint_reader.setter + def proc_start_fingerprint_reader(self, value: ProcStartFingerprintReader) -> None: + self._set_runtime_proxy("proc_start_fingerprint_reader", value) + + @proc_start_fingerprint_reader.deleter + def proc_start_fingerprint_reader(self) -> None: + self._delete_runtime_proxy("proc_start_fingerprint_reader") + + @property + def _processes(self) -> dict[str, PopenLike]: + return self._get_runtime_proxy("_processes") # type: ignore[return-value] + + @_processes.setter + def _processes(self, value: dict[str, PopenLike]) -> None: + self._set_runtime_proxy("_processes", value) + + @_processes.deleter + def _processes(self) -> None: + self._delete_runtime_proxy("_processes") + + @property + def stop_grace_seconds(self) -> float: + return self._get_runtime_proxy("stop_grace_seconds") # type: ignore[return-value] + + @stop_grace_seconds.setter + def stop_grace_seconds(self, value: float) -> None: + self._set_runtime_proxy("stop_grace_seconds", value) + + @stop_grace_seconds.deleter + def stop_grace_seconds(self) -> None: + self._delete_runtime_proxy("stop_grace_seconds") + + @property + def kill_after_timeout(self) -> bool: + return self._get_runtime_proxy("kill_after_timeout") # type: ignore[return-value] + + @kill_after_timeout.setter + def kill_after_timeout(self, value: bool) -> None: + self._set_runtime_proxy("kill_after_timeout", value) + + @kill_after_timeout.deleter + def kill_after_timeout(self) -> None: + self._delete_runtime_proxy("kill_after_timeout") + + @property + def lock_timeout_seconds(self) -> float: + return self._get_runtime_proxy("lock_timeout_seconds") # type: ignore[return-value] + + @lock_timeout_seconds.setter + def lock_timeout_seconds(self, value: float) -> None: + self._set_runtime_proxy("lock_timeout_seconds", value) + + @lock_timeout_seconds.deleter + def lock_timeout_seconds(self) -> None: + self._delete_runtime_proxy("lock_timeout_seconds") + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: if source not in _LIFECYCLE_SOURCES: raise ValueError("invalid lifecycle event source") @@ -362,15 +493,7 @@ def _marker_publication_lock( self, record: BotRecord, ) -> contextlib.AbstractContextManager[object]: - profile_path = self._safe_profile_path(record.bot_id, record.profile_path) - if not os.path.lexists(profile_path): - # A schema-v3 launcher cannot publish beneath an absent profile, so - # marker absence is already stable without creating profile state. - return contextlib.nullcontext() - return marker_publication_lock( - profile_path, - timeout_seconds=self.lock_timeout_seconds, - ) + return self._runtime.marker_publication_lock(record) def create_bot( self, @@ -409,8 +532,7 @@ def create_bot( ) self._assert_unregistered_profile_inactive(bot_id, profile_path) - renderer = ProfileRenderer(self.adapter.hermes_root) - renderer.preflight(request, template) + self._profile_manager.preflight(request, template) stopped_record: BotRecord | None = None if existing is not None: active = self._record_may_be_active(existing) @@ -421,7 +543,7 @@ def create_bot( stopped_record = existing try: - with renderer.transaction(request, template) as record: + with self._profile_manager.install_transaction(request, template) as record: self._remove_pid_marker(record.profile_path) self.store.upsert_bot_with_event( record, @@ -473,10 +595,10 @@ def delete_bot( stopped = self._stop_locked(safe_bot_id, context=context) if stopped.status != BotStatus.stopped: raise BotDeleteError(f"could not stop bot before delete: {stopped.message}") - profile_tombstone: Path | None = None + profile_deletion: ProfileDeletion | None = None try: if remove_profile: - profile_tombstone = self._stage_profile_deletion( + profile_deletion = self._profile_manager.stage_delete( safe_bot_id, record.profile_path ) else: @@ -493,11 +615,12 @@ def delete_bot( ) if not deleted: raise KeyError(f"unknown bot: {safe_bot_id}") - except BaseException: - if profile_tombstone is not None: - self._restore_tombstoned_profile( - safe_bot_id, record.profile_path, profile_tombstone - ) + except BaseException as operation_error: + if profile_deletion is not None: + try: + self._profile_manager.rollback_delete(profile_deletion) + except BotDeleteError as rollback_error: + raise rollback_error from operation_error if was_active: try: self._recover_previously_active_bot(record, "deletion", context=context) @@ -507,15 +630,14 @@ def delete_bot( ) from recovery_error raise cleanup_pending = False - if profile_tombstone is not None: - try: - shutil.rmtree(profile_tombstone) - except OSError as exc: + if profile_deletion is not None: + cleanup_error = self._profile_manager.finish_delete(profile_deletion) + if cleanup_error is not None: cleanup_pending = True self.store.append_audit_event( "bot.delete_cleanup_pending", bot_id=safe_bot_id, - error=type(exc).__name__, + error=type(cleanup_error).__name__, ) self.store.append_audit_event( "bot.delete", @@ -555,16 +677,9 @@ def archive_bot( if stopped.status != BotStatus.stopped: raise BotArchiveError(f"could not stop bot before archive: {stopped.message}") - archive_path: Path | None = None + profile_archive: ProfileArchive | None = None try: - if profile_path.exists(): - archive_root = self.store.database_path.parent / "archive" - archive_root.mkdir(parents=True, exist_ok=True) - candidate = archive_root / ( - f"{safe_bot_id}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" - ) - shutil.move(str(profile_path), str(candidate)) - archive_path = candidate + profile_archive = self._profile_manager.stage_archive(safe_bot_id, profile_path) deleted = self.store.delete_bot_with_event( safe_bot_id, event=self._event( @@ -576,9 +691,12 @@ def archive_bot( ) if not deleted: raise KeyError(f"unknown bot: {safe_bot_id}") - except BaseException: - if archive_path is not None: - self._restore_archived_profile(safe_bot_id, record.profile_path, archive_path) + except BaseException as operation_error: + if profile_archive is not None: + try: + self._profile_manager.rollback_archive(profile_archive) + except BotArchiveError as rollback_error: + raise rollback_error from operation_error if was_active: try: self._recover_previously_active_bot(record, "archive", context=context) @@ -587,6 +705,7 @@ def archive_bot( "bot archive failed and the previous bot could not be restarted" ) from recovery_error raise + archive_path = profile_archive.archive_path if profile_archive is not None else None self.store.append_audit_event( "bot.archive", bot_id=safe_bot_id, @@ -742,140 +861,71 @@ def _start_record( record.profile_path, f"restart aborted: launch preflight failed: {exc}", ) - payload = self.adapter.launcher_payload( - bot_id, - operation_id=operation_id, - desired_revision=record.desired_revision, - readiness_probe=probe, + effect = self._runtime.launch( + record, + probe=probe, + wait=wait, + marker_lock=self._marker_publication_lock, + marker_matcher=self._matching_runtime_marker, + ack_reader=self._read_launcher_ack, + pipe_writer=self._write_pipe_payload, ) - marker_data = payload["marker"] - if type(marker_data) is not dict: - raise RuntimeError("launcher produced an invalid marker payload") - expected_fingerprint = str(marker_data["command_fingerprint"]) - log_path = self.log_path(record.profile_path) - log_path.parent.mkdir(parents=True, exist_ok=True) - process: PopenLike | None = None - payload_read = payload_write = ack_read = ack_write = -1 - try: - encoded_payload = json.dumps( - payload, - ensure_ascii=False, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - if not encoded_payload or len(encoded_payload) > MAX_PAYLOAD_BYTES: - raise ValueError("launcher payload is too large") - payload_read, payload_write = os.pipe() - ack_read, ack_write = os.pipe() - launcher_argv = self.adapter.launcher_command(payload_read, ack_write) - with log_path.open("ab") as log_file: - process = self.popen_factory( - launcher_argv, - env=dict(os.environ), - stdout=log_file, - stderr=log_file, - pass_fds=(payload_read, ack_write), - close_fds=True, - **_gateway_process_launch_kwargs(), - ) - os.close(payload_read) - payload_read = -1 - os.close(ack_write) - ack_write = -1 - self._write_pipe_payload(payload_write, encoded_payload) - os.close(payload_write) - payload_write = -1 - acknowledgment = self._read_launcher_ack(ack_read) - os.close(ack_read) - ack_read = -1 - if acknowledgment != b"1": - raise RuntimeError("gateway launcher did not acknowledge marker publication") - with self._marker_publication_lock(record): - marker = self._matching_runtime_marker( + if effect.outcome == "launch_failed": + failure_message = f"failed to start gateway: {effect.reason}" + try: + self._complete_failed_intent( record, - expected_fingerprint=expected_fingerprint, - expected_pid=process.pid, - require_live_command=True, - ) - if marker.kind != "live": - raise RuntimeError( - "gateway launcher ownership marker could not be verified: " + marker.reason - ) - self._processes[bot_id] = process - except BaseException as exc: - for fd in (payload_read, payload_write, ack_read, ack_write): - if fd >= 0: - with contextlib.suppress(OSError): - os.close(fd) - if process is None: - if not isinstance(exc, (OSError, ValueError)): - raise - self._processes.pop(bot_id, None) - failure_message = f"failed to start gateway: {exc}" - try: - self._complete_failed_intent( - record, - context=context, - pid=None, - message=failure_message, - reason="gateway process launch failed", - ) - except Exception: - return self._pending_action_required( - record, "launch failure could not be persisted" - ) - self.store.append_audit_event( - "bot.start_failed", - bot_id=bot_id, - error=type(exc).__name__, - message=str(exc), - ) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, + context=context, pid=None, - profile_path=record.profile_path, message=failure_message, + reason="gateway process launch failed", ) - cleanup_complete = self._cleanup_interrupted_intent_launch( - record, - process, - expected_fingerprint=expected_fingerprint, - ) - if not isinstance(exc, Exception): - raise - if cleanup_complete: - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, - pid=None, - profile_path=record.profile_path, - message="gateway start registration failed; spawned process was stopped", + except Exception: + return self._pending_action_required( + record, "launch failure could not be persisted" ) + self.store.append_audit_event( + "bot.start_failed", + bot_id=bot_id, + error=effect.error_type, + message=effect.reason, + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message=failure_message, + ) + if effect.outcome == "registration_failed_clean": + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message="gateway start registration failed; spawned process was stopped", + ) + if effect.outcome == "registration_failed_unknown": return BotStatusResponse( bot_id=bot_id, status=BotStatus.unknown, - pid=process.pid, + pid=effect.pid, profile_path=record.profile_path, message=( "gateway start registration failed and spawned process cleanup " "could not be confirmed" ), ) - if process is None: # Defensive guard for custom process factories. - raise RuntimeError("gateway process factory returned no process") - returncode = self._poll_startup(process) - if returncode is not None: - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - self._processes.pop(bot_id, None) - message = f"gateway exited during startup grace period with return code {returncode}" - terminal = replace(record, pid=process.pid) + generation = effect.generation + if generation is None: + raise RuntimeError("gateway runtime returned no launch generation") + pid = effect.pid if effect.pid is not None else generation.pid + if effect.outcome == "startup_exited": + returncode = effect.returncode + failure_message = ( + f"gateway exited during startup grace period with return code {returncode}" + ) + terminal = replace(record, pid=pid) try: self._complete_failed_intent( terminal, @@ -883,17 +933,15 @@ def _start_record( pid=None, stopped_at=datetime.now(UTC), last_exit_code=returncode, - message=message, + message=failure_message, reason="gateway exited during startup grace period", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) + return self._launch_completion_failure_response(record, generation) self.store.append_audit_event( "bot.start_failed", bot_id=bot_id, - pid=process.pid, + pid=pid, returncode=returncode, ) return BotStatusResponse( @@ -901,143 +949,125 @@ def _start_record( status=BotStatus.failed, pid=None, profile_path=record.profile_path, - message=message, - ) - if probe is not None: - if wait: - readiness = self._wait_for_readiness(process, probe) - if process.poll() is not None: - returncode = process.poll() - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - self._processes.pop(bot_id, None) - try: - self._complete_failed_intent( - record, - context=context, - pid=None, - stopped_at=datetime.now(UTC), - last_exit_code=returncode, - message="gateway process exited during readiness check", - reason="readiness process exited", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event( - "bot.start_failed", - bot_id=bot_id, - pid=process.pid, - returncode=returncode, - reason="readiness_process_exited", - ) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, - pid=None, - profile_path=record.profile_path, - message="gateway process exited during readiness check", - ) - if readiness.ready: - try: - self._complete_started_intent( - record, - context=context, - status=BotStatus.running, - pid=process.pid, - ready_at=datetime.now(UTC), - reset_restart=reset_restart, - reason="gateway readiness probe passed", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=process.pid) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.running, - pid=process.pid, - profile_path=record.profile_path, - message="gateway ready", - ) - try: - self._complete_started_intent( - record, - context=context, - status=BotStatus.starting, - pid=process.pid, - last_error=readiness.message, - reason="readiness probe timed out", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event( - "bot.start_readiness_pending", - bot_id=bot_id, - pid=process.pid, - url=probe.url, - message=readiness.message, + message=failure_message, + ) + if effect.outcome == "readiness_exited": + try: + self._complete_failed_intent( + record, + context=context, + pid=None, + stopped_at=datetime.now(UTC), + last_exit_code=effect.returncode, + message="gateway process exited during readiness check", + reason="readiness process exited", ) - return BotStatusResponse( - bot_id=bot_id, + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event( + "bot.start_failed", + bot_id=bot_id, + pid=pid, + returncode=effect.returncode, + reason="readiness_process_exited", + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message="gateway process exited during readiness check", + ) + if effect.outcome == "ready": + try: + self._complete_started_intent( + record, + context=context, + status=BotStatus.running, + pid=pid, + ready_at=datetime.now(UTC), + reset_restart=reset_restart, + reason="gateway readiness probe passed", + ) + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.running, + pid=pid, + profile_path=record.profile_path, + message="gateway ready", + ) + if effect.outcome == "readiness_timeout": + try: + self._complete_started_intent( + record, + context=context, status=BotStatus.starting, - pid=process.pid, - profile_path=record.profile_path, - message="readiness timeout; gateway process still alive", + pid=pid, + last_error=effect.readiness_message, + reason="readiness probe timed out", ) + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event( + "bot.start_readiness_pending", + bot_id=bot_id, + pid=pid, + url=probe.url if probe is not None else None, + message=effect.readiness_message, + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.starting, + pid=pid, + profile_path=record.profile_path, + message="readiness timeout; gateway process still alive", + ) + if effect.outcome == "readiness_pending": try: self._complete_started_intent( record, context=context, status=BotStatus.starting, - pid=process.pid, + pid=pid, reason="gateway process started; readiness probe pending", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) + return self._launch_completion_failure_response(record, generation) self.store.append_audit_event( "bot.start_readiness_pending", bot_id=bot_id, - pid=process.pid, - url=probe.url, + pid=pid, + url=probe.url if probe is not None else None, ) return BotStatusResponse( bot_id=bot_id, status=BotStatus.starting, - pid=process.pid, + pid=pid, profile_path=record.profile_path, message="started; readiness probe pending", ) + if effect.outcome != "running": + raise RuntimeError(f"unknown gateway launch outcome: {effect.outcome}") try: self._complete_started_intent( record, context=context, status=BotStatus.running, - pid=process.pid, + pid=pid, ready_at=datetime.now(UTC), reset_restart=reset_restart, reason="gateway process started without readiness probe", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=process.pid) + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) return BotStatusResponse( bot_id=bot_id, status=BotStatus.running, - pid=process.pid, + pid=pid, profile_path=record.profile_path, message=message, ) @@ -1045,45 +1075,13 @@ def _start_record( def _preflight_start( self, record: BotRecord, *, timeout_seconds: float | None ) -> ReadinessProbe | None: - expected_profile = (Path(self.adapter.hermes_root) / "profiles" / record.bot_id).resolve() - if Path(record.profile_path).resolve() != expected_profile: - raise TemplateError("registered bot profile does not match the Hermes profile path") - safe_profile = self._safe_profile_path(record.bot_id, record.profile_path) - if not safe_profile.is_dir() or safe_profile.is_symlink(): - raise TemplateError("registered bot profile is not a safe directory") - _argv, env = self.adapter.command(record.bot_id, "gateway", "run") - probe = self._readiness_probe(env, timeout_seconds=timeout_seconds) - self.adapter.launcher_payload( - record.bot_id, - operation_id="0" * 32, - desired_revision=max(1, record.desired_revision + 1), - readiness_probe=probe, - ) - return probe + return self._runtime.preflight_start(record, timeout_seconds=timeout_seconds) def _write_pipe_payload(self, fd: int, payload: bytes) -> None: - offset = 0 - while offset < len(payload): - written = os.write(fd, payload[offset:]) - if written <= 0: - raise OSError("short launcher payload write") - offset += written + self._runtime.write_pipe_payload(fd, payload) def _read_launcher_ack(self, fd: int) -> bytes: - deadline = time.monotonic() + 5.0 - acknowledgment = bytearray() - while len(acknowledgment) <= 1: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("gateway launcher acknowledgment timed out") - readable, _writable, _exceptional = select.select([fd], [], [], remaining) - if not readable: - raise TimeoutError("gateway launcher acknowledgment timed out") - chunk = os.read(fd, 2 - len(acknowledgment)) - if not chunk: - return bytes(acknowledgment) - acknowledgment.extend(chunk) - return bytes(acknowledgment) + return self._runtime.read_launcher_ack(fd) def _complete_started_intent( self, @@ -1162,37 +1160,18 @@ def _cleanup_interrupted_intent_launch( *, expected_fingerprint: str, ) -> bool: - cleanup_errors: list[str] = [] - stopped = self._terminate_spawned_process(process, cleanup_errors) - if not stopped: - return False - self._processes.pop(record.bot_id, None) - operation_id = record.pending_operation_id - if operation_id is None: - return False - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - return ( - self._read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + return self._runtime.cleanup_interrupted_launch( + record, + process, + expected_fingerprint=expected_fingerprint, ) def _launch_completion_failure_response( self, record: BotRecord, - process: PopenLike, - *, - expected_fingerprint: str, + generation: _GatewayGeneration, ) -> BotStatusResponse: - cleaned = self._cleanup_interrupted_intent_launch( - record, - process, - expected_fingerprint=expected_fingerprint, - ) + cleaned = self._runtime.cleanup_registered_launch(record, generation) if cleaned: return BotStatusResponse( record.bot_id, @@ -1204,7 +1183,7 @@ def _launch_completion_failure_response( return BotStatusResponse( record.bot_id, BotStatus.unknown, - process.pid, + generation.pid, record.profile_path, "gateway start completion is unknown and cleanup could not be confirmed", ) @@ -1212,54 +1191,7 @@ def _launch_completion_failure_response( def _read_strict_runtime_marker( self, bot_id: str, registered_profile_path: str ) -> _MarkerObservation: - profile_path = _nofollow_absolute_path(Path(registered_profile_path)) - expected_profile = self._marker_profiles_root / bot_id - if not profile_path.is_absolute() or profile_path != expected_profile: - return _MarkerObservation( - "untrusted", - reason="registered profile path does not match the trusted Hermes profile", - ) - profile_fd = logs_fd = marker_fd = -1 - try: - profile_fd = _open_profile(profile_path) - except (OSError, ValueError) as exc: - if isinstance(exc.__cause__, FileNotFoundError): - return _MarkerObservation("missing", reason="marker is missing") - return _MarkerObservation( - "untrusted", reason=f"registered profile cannot be opened safely: {exc}" - ) - try: - try: - logs_fd = _open_logs(profile_fd, create=False) - except ValueError as exc: - if isinstance(exc.__cause__, FileNotFoundError): - return _MarkerObservation("missing", reason="marker is missing") - return _MarkerObservation( - "untrusted", reason=f"marker directory cannot be opened safely: {exc}" - ) - try: - marker_fd, marker_stat = _open_regular_marker(logs_fd) - raw = _read_bounded_file(marker_fd) - value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) - except FileNotFoundError: - return _MarkerObservation("missing", reason="marker is missing") - except ValueError as exc: - if isinstance(exc.__cause__, FileNotFoundError): - return _MarkerObservation("missing", reason="marker is missing") - return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - except FileNotFoundError: - return _MarkerObservation("missing", reason="marker is missing") - finally: - for fd in (marker_fd, logs_fd, profile_fd): - if fd >= 0: - os.close(fd) - if type(value) is not dict: - return _MarkerObservation("untrusted", reason="marker is not an object") - if marker_stat.st_nlink != 1: - return _MarkerObservation("untrusted", reason="marker has unexpected hard links") - return _MarkerObservation("present", payload=value) + return self._runtime.read_strict_runtime_marker(bot_id, registered_profile_path) def _matching_runtime_marker( self, @@ -1269,20 +1201,12 @@ def _matching_runtime_marker( expected_pid: int | None = None, require_live_command: bool, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return observed - operation_id = record.pending_operation_id - if operation_id is None: - return _MarkerObservation("untrusted", reason="pending operation is missing") - return self._classify_schema3_runtime_marker( + return self._runtime.matching_runtime_marker( record, - observed.payload, - expected_pid=expected_pid, - expected_operation_id=operation_id, - expected_revision=record.desired_revision, expected_fingerprint=expected_fingerprint, + expected_pid=expected_pid, require_live_command=require_live_command, + read_marker=self._read_strict_runtime_marker, ) def _classify_schema3_runtime_marker( @@ -1296,94 +1220,32 @@ def _classify_schema3_runtime_marker( expected_fingerprint: str | None = None, require_live_command: bool, ) -> _MarkerObservation: - pid_value = payload.get("pid") - if type(pid_value) is not int or pid_value <= 0: - return _MarkerObservation("untrusted", reason="marker PID is invalid") - pid = pid_value - if expected_pid is not None and pid != expected_pid: - return _MarkerObservation("untrusted", reason="marker PID does not match") - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - fingerprint = payload.get("command_fingerprint") - if ( - type(operation_id) is not str - or type(revision) is not int - or type(fingerprint) is not str - ): - return _MarkerObservation("untrusted", reason="marker correlation is invalid") - if expected_operation_id is not None and operation_id != expected_operation_id: - return _MarkerObservation("untrusted", reason="marker operation does not match") - if expected_revision is not None and revision != expected_revision: - return _MarkerObservation("untrusted", reason="marker revision does not match") - if expected_fingerprint is not None and fingerprint != expected_fingerprint: - return _MarkerObservation("untrusted", reason="marker command does not match") - if not _is_owned_runtime_marker( + return self._runtime.classify_schema3_runtime_marker( + record, payload, - bot_id=record.bot_id, - operation_id=operation_id, - desired_revision=revision, - pid=pid, - expected_fingerprint=fingerprint, - ): - return _MarkerObservation("untrusted", reason="marker schema or command does not match") - expected_hermes = self._resolved_hermes_bin() - if expected_hermes is None or payload.get("resolved_hermes_bin") != expected_hermes: - return _MarkerObservation("untrusted", reason="marker executable is not trusted") - if not require_live_command: - start_identity_error = self._process_start_identity_error(payload, pid) - if start_identity_error is not None: - return _MarkerObservation("untrusted", reason=start_identity_error) - return _MarkerObservation("live", payload=payload) - pid_state = self._pid_state(pid) - if pid_state is _PidState.unknown: - return _MarkerObservation("untrusted", reason="marker PID liveness is unknown") - if pid_state is _PidState.dead: - if self._process_start_fingerprint_required() and not self._valid_marker_start( - payload.get("proc_start_fingerprint") - ): - return _MarkerObservation( - "untrusted", reason="process start fingerprint is unavailable" - ) - return _MarkerObservation("dead", payload=payload, reason="marker PID is dead") - start_identity_error = self._process_start_identity_error(payload, pid) - if start_identity_error is not None: - return _MarkerObservation("untrusted", reason=start_identity_error) - live_argv = self.cmdline_reader(pid) - if not live_argv: - return _MarkerObservation("untrusted", reason="live gateway command is unavailable") - command_check = _verify_gateway_command( - live_argv, - record.bot_id, - self._trusted_hermes_bins(), - require_trusted_path=True, + expected_pid=expected_pid, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_fingerprint=expected_fingerprint, + require_live_command=require_live_command, ) - if not command_check.verified: - return _MarkerObservation("untrusted", reason="live gateway command does not match") - return _MarkerObservation("live", payload=payload) def _process_start_identity_error(self, payload: dict[str, object], pid: int) -> str | None: - marker_start = payload.get("proc_start_fingerprint") - live_start = self.proc_start_fingerprint_reader(pid) - if self._process_start_fingerprint_required(): - if not self._valid_marker_start(marker_start) or not self._valid_marker_start( - live_start - ): - return "process start fingerprint is unavailable" - if marker_start != live_start: - return "process start fingerprint does not match" - elif live_start and marker_start != live_start: - return "process start fingerprint does not match" - elif marker_start and live_start != marker_start: - return "process start fingerprint is unavailable" - return None + if "_runtime" in self.__dict__: + return self._runtime.process_start_identity_error(payload, pid) + return _process_identity.process_start_identity_error( + payload.get("proc_start_fingerprint"), + self.proc_start_fingerprint_reader(pid), + fingerprint_required=self._process_start_fingerprint_required(), + ) @staticmethod def _valid_marker_start(value: object) -> bool: - return type(value) is str and bool(value) and len(value) <= 512 + return _process_identity.valid_process_start_fingerprint(value) @staticmethod def _process_start_fingerprint_required() -> bool: - return platform.system() in {"Darwin", "Linux"} + return _process_identity.process_start_fingerprint_required(platform.system()) def _classify_existing_runtime_marker( self, @@ -1391,14 +1253,10 @@ def _classify_existing_runtime_marker( *, expected_pid: int | None = None, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return observed - return self._classify_schema3_runtime_marker( + return self._runtime.classify_existing_runtime_marker( record, - observed.payload, expected_pid=expected_pid, - require_live_command=True, + read_marker=self._read_strict_runtime_marker, ) def _remove_exact_schema3_marker( @@ -1406,61 +1264,23 @@ def _remove_exact_schema3_marker( record: BotRecord, marker: _MarkerObservation, ) -> bool: - if marker.kind != "dead": - return False - generation = self._gateway_generation(marker) - if generation is None: - return False - return self._remove_gateway_generation_marker(record, generation) + return self._runtime.remove_exact_schema3_marker(record, marker) def _gateway_generation( self, marker: _MarkerObservation, ) -> _GatewayGeneration | None: - payload = marker.payload - if marker.kind not in {"live", "dead"} or payload is None: - return None - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - pid = payload.get("pid") - fingerprint = payload.get("command_fingerprint") - process_start = payload.get("proc_start_fingerprint") - if ( - type(operation_id) is not str - or type(revision) is not int - or type(pid) is not int - or type(fingerprint) is not str - or (process_start is not None and type(process_start) is not str) - ): - return None - return _GatewayGeneration( - operation_id=operation_id, - desired_revision=revision, - pid=pid, - command_fingerprint=fingerprint, - proc_start_fingerprint=process_start, - ) + return self._runtime.gateway_generation(marker) def _classify_exact_gateway_generation( self, record: BotRecord, generation: _GatewayGeneration, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return _MarkerObservation("untrusted", reason="previous gateway marker changed") - if observed.payload.get("proc_start_fingerprint") != generation.proc_start_fingerprint: - return _MarkerObservation( - "untrusted", reason="previous gateway process identity changed" - ) - return self._classify_schema3_runtime_marker( + return self._runtime.classify_exact_gateway_generation( record, - observed.payload, - expected_pid=generation.pid, - expected_operation_id=generation.operation_id, - expected_revision=generation.desired_revision, - expected_fingerprint=generation.command_fingerprint, - require_live_command=True, + generation, + read_marker=self._read_strict_runtime_marker, ) def _remove_gateway_generation_marker( @@ -1468,29 +1288,14 @@ def _remove_gateway_generation_marker( record: BotRecord, generation: _GatewayGeneration, ) -> bool: - return remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=generation.operation_id, - desired_revision=generation.desired_revision, - pid=generation.pid, - command_fingerprint=generation.command_fingerprint, - expected_proc_start_fingerprint=generation.proc_start_fingerprint, - lock_timeout_seconds=self.lock_timeout_seconds, - ) + return self._runtime.remove_gateway_generation_marker(record, generation) def _remove_gateway_generation_marker_locked( self, record: BotRecord, generation: _GatewayGeneration, ) -> bool: - return _remove_marker_if_owned_locked( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=generation.operation_id, - desired_revision=generation.desired_revision, - pid=generation.pid, - command_fingerprint=generation.command_fingerprint, - expected_proc_start_fingerprint=generation.proc_start_fingerprint, - ) + return self._runtime.remove_gateway_generation_marker_locked(record, generation) def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: return BotStatusResponse( @@ -1569,25 +1374,32 @@ def _stop_record_effect_locked( context: _LifecycleContext, complete_stop: bool, ) -> BotStatusResponse: - bot_id = record.bot_id - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if ( - observed.kind == "present" - and observed.payload is not None - and self._is_compat_runtime_marker(observed.payload) - ): - return self._pending_action_required( - record, - "schema-v2 or legacy gateway stop requires manual process resolution", - ) - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead - if record.pid and pid_state == _PidState.unknown: - return self._pending_action_required(record, "gateway PID liveness is unknown") - if not record.pid or pid_state == _PidState.dead: - if not self._remove_owned_launch_marker_locked(record, observed=observed): - return self._pending_action_required( - record, "stale gateway marker ownership could not be verified" + effect = self._runtime.stop_locked( + record, + kill_after_timeout=kill_after_timeout, + read_marker=self._read_strict_runtime_marker, + classify_existing=self._classify_existing_runtime_marker, + classify_exact=self._classify_exact_gateway_generation, + remove_owned=self._remove_owned_launch_marker_locked, + remove_generation=self._remove_gateway_generation_marker_locked, + ) + if effect.outcome not in {"not_running", "stopped"}: + if effect.kill_result is not None: + self.store.append_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=effect.pid, + succeeded=bool(effect.kill_succeeded), ) + if effect.outcome == "grace_expired": + reason = ( + "gateway did not stop before grace period expired; " + "Hermes async delegations may still be running" + ) + else: + reason = effect.reason + return self._pending_action_required(record, reason) + if effect.outcome == "not_running": if complete_stop: try: self._complete_stopped_intent( @@ -1599,80 +1411,26 @@ def _stop_record_effect_locked( return self._pending_action_required( record, "stopped state could not be persisted" ) - self.store.append_audit_event("bot.stop", bot_id=bot_id, pid=record.pid) + self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( - bot_id=bot_id, + bot_id=record.bot_id, status=BotStatus.stopped, pid=None, profile_path=record.profile_path, message="not running", ) - - marker = self._classify_existing_runtime_marker(record, expected_pid=record.pid) - generation = self._gateway_generation(marker) - if marker.kind != "live" or generation is None: - return self._pending_action_required( - record, - "refusing to stop process because PID ownership could not be verified", - ) - term_marker = self._classify_exact_gateway_generation(record, generation) - if term_marker.kind != "live": - return self._pending_action_required( - record, - term_marker.reason or "gateway ownership changed before SIGTERM", - ) - - term_result = self._send_signal(record.pid, signal.SIGTERM) - if term_result == _SignalResult.denied: - return self._pending_action_required(record, "could not send SIGTERM to the gateway") - stopped = term_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(bot_id, record.pid) - should_kill = self.kill_after_timeout if kill_after_timeout is None else kill_after_timeout - if not stopped and should_kill: - kill_marker = self._classify_exact_gateway_generation(record, generation) - if kill_marker.kind != "live": - return self._pending_action_required( - record, - kill_marker.reason or "gateway ownership changed before SIGKILL", - ) - kill_result = self._send_signal(record.pid, signal.SIGKILL) - if kill_result == _SignalResult.denied: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=bot_id, - pid=record.pid, - succeeded=False, - ) - return self._pending_action_required( - record, "could not send SIGKILL to the gateway" - ) - stopped = kill_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(bot_id, record.pid) + if effect.kill_result is not None: self.store.append_audit_event( "bot.stop_kill", - bot_id=bot_id, - pid=record.pid, - succeeded=stopped, - ) - if not stopped: - message = ( - "gateway did not stop before grace period expired; " - "Hermes async delegations may still be running" - ) - return self._pending_action_required(record, message) - - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "stopped gateway marker cleanup could not be verified" + bot_id=record.bot_id, + pid=effect.pid, + succeeded=bool(effect.kill_succeeded), ) - self._processes.pop(bot_id, None) if not complete_stop: try: self._update_lifecycle( context, - bot_id, + record.bot_id, BotStatus.stopped, pid=None, action="bot.restart.old_process_stopped", @@ -1693,9 +1451,9 @@ def _stop_record_effect_locked( ) except Exception: return self._pending_action_required(record, "stopped state could not be persisted") - self.store.append_audit_event("bot.stop", bot_id=bot_id, pid=record.pid) + self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( - bot_id=bot_id, + bot_id=record.bot_id, status=BotStatus.stopped, pid=None, profile_path=record.profile_path, @@ -1734,27 +1492,7 @@ def _remove_owned_launch_marker_locked( *, observed: _MarkerObservation | None = None, ) -> bool: - if observed is None: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind == "missing": - return True - if observed.kind != "present" or observed.payload is None or record.pid is None: - return False - if record.pending_action not in {"stop", "restart"}: - return False - marker = self._classify_schema3_runtime_marker( - record, - observed.payload, - expected_pid=record.pid, - expected_revision=record.desired_revision - 1, - require_live_command=True, - ) - generation = self._gateway_generation(marker) - return ( - marker.kind == "dead" - and generation is not None - and self._remove_gateway_generation_marker_locked(record, generation) - ) + return self._runtime.remove_owned_launch_marker_locked(record, observed=observed) def restart( self, @@ -2342,8 +2080,7 @@ def _prepare_reconcile_dead_record_locked( @staticmethod def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: - schema = payload.get("schema") - return (type(schema) is int and schema == 2) or schema is None + return is_compat_runtime_marker(payload) def _recover_pending_intent( self, @@ -2352,123 +2089,86 @@ def _recover_pending_intent( context: _LifecycleContext, allow_launch: bool, ) -> BotStatusResponse: - action = record.pending_action - operation_id = record.pending_operation_id - if action is None or operation_id is None: - return self._pending_action_required(record, "pending lifecycle correlation is invalid") - recovery_context = _LifecycleContext(operation_id, context.source, context.request_id) - if action == "stop": - try: - with self._marker_publication_lock(record): - return self._recover_pending_stop_intent_locked( - record, - context=recovery_context, - allow_stop=allow_launch, - ) - except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return self._intent_recovery.recover( + self, + record, + context=context, + allow_launch=allow_launch, + ) + + @staticmethod + def _recovery_lifecycle_context( + operation_id: str, + context: _LifecycleContext, + ) -> _LifecycleContext: + return _LifecycleContext(operation_id, context.source, context.request_id) - if action not in {"start", "restart"}: - return self._pending_action_required(record, "pending lifecycle action is invalid") + def _pending_launch_preflight( + self, + record: BotRecord, + operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: + probe = self._preflight_start(record, timeout_seconds=None) + expected = self.adapter.launcher_payload( + record.bot_id, + operation_id=operation_id, + desired_revision=record.desired_revision, + readiness_probe=probe, + ) + marker_template = expected["marker"] + if type(marker_template) is not dict: + raise ValueError("invalid expected marker") + return probe, str(marker_template["command_fingerprint"]) + + @staticmethod + def _probe_once(probe: ReadinessProbe) -> ReadinessResult: + return probe_once( + probe.url, + timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), + expected_status=probe.expected_status, + expected_platform=probe.expected_platform, + ) + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse: try: - probe = self._preflight_start(record, timeout_seconds=None) - expected = self.adapter.launcher_payload( - record.bot_id, - operation_id=operation_id, - desired_revision=record.desired_revision, - readiness_probe=probe, - ) - marker_template = expected["marker"] - if type(marker_template) is not dict: - raise ValueError("invalid expected marker") - fingerprint = str(marker_template["command_fingerprint"]) - except (OSError, ValueError, TemplateError, BotDeleteError) as exc: - return self._pending_action_required(record, f"launch preflight failed: {exc}") - if action == "restart": - predecessor_result = self._recover_pending_restart_predecessor( - record, - context=recovery_context, - allow_stop=allow_launch, - ) - if predecessor_result is not None: - return predecessor_result + with self._marker_publication_lock(record): + return self._recover_pending_stop_intent_locked( + record, + context=context, + allow_stop=allow_stop, + ) + except (BotDeleteError, LaunchPayloadError) as exc: + return self._pending_action_required(record, str(exc)) + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _LifecycleContext, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: try: with self._marker_publication_lock(record): - marker = self._matching_runtime_marker( + return self._intent_recovery.recover_pending_launch_locked( + self, record, - expected_fingerprint=fingerprint, - require_live_command=True, + context=context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, ) - if marker.kind == "live" and marker.payload is not None: - marker_pid = marker.payload["pid"] - if isinstance(marker_pid, bool) or not isinstance(marker_pid, int): - return self._pending_action_required(record, "live marker PID is invalid") - pid = marker_pid - status = BotStatus.starting if probe is not None else BotStatus.running - ready_at = None if probe is not None else datetime.now(UTC) - if probe is not None: - readiness = probe_once( - probe.url, - timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), - expected_status=probe.expected_status, - expected_platform=probe.expected_platform, - ) - if readiness.ready: - status = BotStatus.running - ready_at = datetime.now(UTC) - try: - self._complete_started_intent( - record, - context=recovery_context, - status=status, - pid=pid, - ready_at=ready_at, - reset_restart=True, - reason="recovery adopted registered gateway", - ) - except Exception: - return self._pending_action_required( - record, "gateway adoption could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - status, - pid, - record.profile_path, - "recovered registered gateway", - ) - if marker.kind == "untrusted": - return self._pending_action_required(record, marker.reason) - if not allow_launch: - return self._pending_action_required( - record, "desired running gateway is missing; reconcile is required" - ) - if marker.kind == "dead": - generation = self._gateway_generation(marker) - if generation is None or not self._remove_gateway_generation_marker_locked( - record, generation - ): - return self._pending_action_required( - record, "dead gateway marker cleanup failed" - ) - if action == "restart": - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: removed dead gateway marker; " - "launch on next reconcile", - ) except (BotDeleteError, LaunchPayloadError) as exc: return self._pending_action_required(record, str(exc)) - return self._start_record( - record, - reset_restart=action == "restart", - message="recovered interrupted gateway launch", - context=recovery_context, - probe=probe, - ) def _recover_pending_stop_intent_locked( self, @@ -2477,41 +2177,11 @@ def _recover_pending_stop_intent_locked( context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead - if pid_state is _PidState.unknown: - return self._pending_action_required(record, "gateway PID liveness is unknown") - if not record.pid or pid_state is _PidState.dead: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if not self._remove_owned_launch_marker_locked(record, observed=observed): - return self._pending_action_required( - record, "stale gateway marker ownership could not be verified" - ) - try: - self._complete_stopped_intent( - record, - context=context, - reason="recovery confirmed gateway stopped", - ) - except Exception: - return self._pending_action_required(record, "stopped state could not be persisted") - return BotStatusResponse( - record.bot_id, - BotStatus.stopped, - None, - record.profile_path, - "recovered interrupted stop", - ) - if not allow_stop: - if not self._pid_owned(record.profile_path, record.pid, record.bot_id): - return self._pending_action_required( - record, "gateway ownership could not be verified" - ) - return self._pending_action_required(record, "stop intent is pending reconciliation") - return self._stop_record_effect_locked( + return self._intent_recovery.recover_pending_stop_intent_locked( + self, record, - kill_after_timeout=None, context=context, - complete_stop=True, + allow_stop=allow_stop, ) def _pending_restart_old_marker( @@ -2519,27 +2189,10 @@ def _pending_restart_old_marker( record: BotRecord, observed: _MarkerObservation | None = None, ) -> _MarkerObservation | None: - """Return a strictly verified marker from the generation before a restart intent.""" - if observed is None: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return None - payload = observed.payload - marker_operation = payload.get("operation_id") - marker_revision = payload.get("desired_revision") - if ( - type(marker_operation) is not str - or marker_operation == record.pending_operation_id - or type(marker_revision) is not int - or marker_revision != record.desired_revision - 1 - ): - return None - return self._classify_schema3_runtime_marker( + return self._intent_recovery.pending_restart_old_marker( + self, record, - payload, - expected_pid=record.pid, - expected_revision=record.desired_revision - 1, - require_live_command=True, + observed, ) def _recover_pending_restart_predecessor( @@ -2551,61 +2204,11 @@ def _recover_pending_restart_predecessor( ) -> BotStatusResponse | None: try: with self._marker_publication_lock(record): - predecessor = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if ( - record.pid is not None - and predecessor.kind == "present" - and predecessor.payload is not None - and self._is_compat_runtime_marker(predecessor.payload) - ): - return self._pending_action_required( - record, - "schema-v2 or legacy gateway restart requires manual process resolution", - ) - old_marker = self._pending_restart_old_marker(record, predecessor) - if old_marker is not None: - return self._recover_pending_restart_old_gateway( - record, - old_marker, - context=context, - allow_stop=allow_stop, - ) - if record.pid is None or predecessor.kind != "missing": - return None - pid_state = self._pid_state(record.pid) - if pid_state is _PidState.unknown: - return self._pending_action_required( - record, "previous gateway PID liveness is unknown" - ) - if pid_state is _PidState.alive: - return self._pending_action_required( - record, "previous gateway marker is missing" - ) - if not allow_stop: - return self._pending_action_required( - record, "restart intent is pending reconciliation" - ) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_recovered", - stopped_at=datetime.now(UTC), - last_transition_reason=("recovery confirmed the previous gateway stopped"), - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: recovered stopped gateway; launch on next reconcile", + return self._intent_recovery.recover_pending_restart_predecessor_locked( + self, + record, + context=context, + allow_stop=allow_stop, ) except (BotDeleteError, LaunchPayloadError) as exc: return self._pending_action_required(record, str(exc)) @@ -2618,64 +2221,12 @@ def _recover_pending_restart_old_gateway( context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - if marker.kind == "untrusted": - return self._pending_action_required( - record, - marker.reason or "previous gateway ownership could not be verified", - ) - if not allow_stop: - return self._pending_action_required(record, "restart intent is pending reconciliation") - generation = self._gateway_generation(marker) - if generation is None: - return self._pending_action_required( - record, "previous gateway marker correlation is invalid" - ) - if marker.kind == "live": - if record.pid is None: - return self._pending_action_required(record, "previous gateway PID is not recorded") - stopped = self._stop_pending_restart_old_gateway( - record, - generation, - context=context, - ) - if stopped.status is not BotStatus.stopped: - return stopped - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: previous gateway stopped; launch on next reconcile", - ) - if marker.kind != "dead" or marker.payload is None: - return self._pending_action_required( - record, "previous gateway marker ownership could not be verified" - ) - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "previous gateway marker cleanup could not be verified" - ) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_recovered", - stopped_at=datetime.now(UTC), - last_transition_reason="recovery confirmed the previous gateway stopped", - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: recovered stopped gateway; launch on next reconcile", + return self._intent_recovery.recover_pending_restart_old_gateway( + self, + record, + marker, + context=context, + allow_stop=allow_stop, ) def _stop_pending_restart_old_gateway( @@ -2685,88 +2236,29 @@ def _stop_pending_restart_old_gateway( *, context: _LifecycleContext, ) -> BotStatusResponse: - current = self._classify_exact_gateway_generation(record, generation) - if current.kind == "dead": - stopped = True - elif current.kind == "live": - term_result = self._send_signal(generation.pid, signal.SIGTERM) - if term_result == _SignalResult.denied: - return self._pending_action_required( - record, "could not send SIGTERM to the previous gateway" - ) - stopped = term_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(record.bot_id, generation.pid) - else: - return self._pending_action_required( - record, current.reason or "previous gateway marker changed" - ) - - if not stopped and self.kill_after_timeout: - current = self._classify_exact_gateway_generation(record, generation) - if current.kind == "dead": - stopped = True - elif current.kind != "live": - return self._pending_action_required( - record, - current.reason or "previous gateway ownership changed before SIGKILL", - ) - else: - kill_result = self._send_signal(generation.pid, signal.SIGKILL) - if kill_result == _SignalResult.denied: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=record.bot_id, - pid=generation.pid, - succeeded=False, - ) - return self._pending_action_required( - record, "could not send SIGKILL to the previous gateway" - ) - stopped = kill_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(record.bot_id, generation.pid) - self.store.append_audit_event( - "bot.stop_kill", - bot_id=record.bot_id, - pid=generation.pid, - succeeded=stopped, - ) + return self._intent_recovery.stop_pending_restart_old_gateway( + self, + record, + generation, + context=context, + ) - if not stopped: - return self._pending_action_required( - record, - "previous gateway did not stop before the grace period expired", - ) - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "previous gateway marker cleanup could not be verified" - ) - self._processes.pop(record.bot_id, None) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_stopped", - stopped_at=datetime.now(UTC), - last_transition_reason="restart stopped the previous gateway", - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=generation.pid) - return BotStatusResponse( - record.bot_id, - BotStatus.stopped, - None, - record.profile_path, - "gateway shutdown completed", + def _stop_gateway_generation_locked( + self, + record: BotRecord, + generation: _GatewayGeneration, + ) -> StopEffect: + return self._runtime.stop_generation_locked( + record, + generation, + kill_after_timeout=None, + classify_exact=self._classify_exact_gateway_generation, + remove_generation=self._remove_gateway_generation_marker_locked, ) + def _append_recovery_audit_event(self, action: str, **values: object) -> None: + self.store.append_audit_event(action, **values) + def _restart_delay(self, record: BotRecord) -> float: delay = record.restart_backoff_seconds * (2**record.restart_attempts) return float(min(delay, self.restart_backoff_cap_seconds)) @@ -3113,48 +2605,21 @@ def _recover_previously_active_bot( if result.status not in {BotStatus.starting, BotStatus.running}: raise RuntimeError(f"previous bot restart failed after {operation}: {result.message}") - def _assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) -> None: - marker_path = self.pid_marker_path(str(profile_path)) - if not marker_path.exists(): - return - try: - payload = json.loads(marker_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise BotRunningError( - "unregistered bot profile has an unreadable PID marker; refusing replacement" - ) from exc - pid = payload.get("pid") if isinstance(payload, dict) else None - if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: - raise BotRunningError( - "unregistered bot profile has an invalid PID marker; refusing replacement" - ) - if self._pid_state(pid) != _PidState.dead: - raise BotRunningError( - f"unregistered bot profile may still own gateway PID {pid}; refusing replacement" - ) + def _assert_unregistered_profile_inactive( + self, + bot_id: str, + profile_path: Path, + ) -> None: + self._runtime.assert_unregistered_profile_inactive(bot_id, profile_path) def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: - safe_bot_id = validate_id(bot_id, "bot_id") - profile = Path(profile_path).resolve() - profiles_root = (Path(self.adapter.hermes_root) / "profiles").resolve() - try: - relative = profile.relative_to(profiles_root) - except ValueError as exc: - raise BotDeleteError("bot profile path is outside the Hermes profiles root") from exc - if len(relative.parts) != 1 or relative.parts[0] != safe_bot_id: - raise BotDeleteError("bot profile path does not match bot id") - return profile + return self._profile_manager.validate_profile_path(bot_id, profile_path) def _stage_profile_deletion(self, bot_id: str, profile_path: str) -> Path | None: - profile = self._safe_profile_path(bot_id, profile_path) - if not profile.exists(): + deletion = self._profile_manager.stage_delete(bot_id, profile_path) + if deletion is None: return None - tombstone = profile.with_name(f".{profile.name}.deleting-{os.getpid()}-{time.time_ns()}") - try: - os.replace(profile, tombstone) - except OSError as exc: - raise BotDeleteError("could not stage the bot profile for deletion") from exc - return tombstone + return deletion.tombstone_path def _restore_tombstoned_profile( self, @@ -3162,18 +2627,13 @@ def _restore_tombstoned_profile( profile_path: str, tombstone: Path, ) -> None: - profile = self._safe_profile_path(bot_id, profile_path) - if profile.exists() or profile.is_symlink(): - raise BotDeleteError( - "bot state deletion failed and the profile could not be restored because " - "its original path is occupied" + profile = self._profile_manager._pin_profile_path(bot_id, profile_path) + self._profile_manager.rollback_delete( + ProfileDeletion( + profile_path=profile, + tombstone_path=tombstone, ) - try: - os.replace(tombstone, profile) - except OSError as exc: - raise BotDeleteError( - "bot state deletion failed and the profile could not be restored" - ) from exc + ) def _restore_archived_profile( self, @@ -3181,85 +2641,44 @@ def _restore_archived_profile( profile_path: str, archive_path: Path, ) -> None: - profile = self._safe_profile_path(bot_id, profile_path) - if profile.exists() or profile.is_symlink(): - raise BotArchiveError( - "bot state deletion failed and the archived profile could not be restored " - "because its original path is occupied" + profile = self._profile_manager._pin_profile_path(bot_id, profile_path) + self._profile_manager.rollback_archive( + ProfileArchive( + profile_path=profile, + archive_path=archive_path, ) - try: - shutil.move(str(archive_path), str(profile)) - except OSError as exc: - raise BotArchiveError( - "bot state deletion failed and the archived profile could not be restored" - ) from exc + ) def _readiness_probe_for_bot( self, bot_id: str, *, timeout_seconds: float | None = None ) -> ReadinessProbe | None: - _argv, env = self.adapter.command(bot_id, "gateway", "run") - return self._readiness_probe(env, timeout_seconds=timeout_seconds) + return self._runtime.readiness_probe_for_bot( + bot_id, + timeout_seconds=timeout_seconds, + ) def _readiness_probe_for_live_record( self, record: BotRecord ) -> tuple[ReadinessProbe | None, str | None]: - try: - payload = json.loads( - self.pid_marker_path(record.profile_path).read_text(encoding="utf-8") - ) - except (FileNotFoundError, json.JSONDecodeError, OSError): - return None, "readiness provenance is unavailable from the PID marker" - if not isinstance(payload, dict): - return None, "readiness provenance in the PID marker is invalid" - if payload.get("schema") not in {2, 3} or "readiness_probe" not in payload: - # Markers written before readiness provenance was added remain supported. - return self._readiness_probe_for_bot(record.bot_id), None - try: - return _readiness_probe_from_marker(payload["readiness_probe"]), None - except ValueError as exc: - return None, f"readiness provenance in the PID marker is invalid: {exc}" + return self._runtime.readiness_probe_for_live_record(record) def _readiness_probe( self, env: dict[str, str], *, timeout_seconds: float | None = None ) -> ReadinessProbe | None: - resolved_timeout = ( - self.readiness_timeout_seconds if timeout_seconds is None else timeout_seconds - ) - if ( - isinstance(resolved_timeout, bool) - or not isinstance(resolved_timeout, (int, float)) - or not math.isfinite(float(resolved_timeout)) - or not 0.1 <= float(resolved_timeout) <= 300 - ): - raise TemplateError("readiness timeout must be a finite number between 0.1 and 300") - return readiness_probe_from_env( - env, - timeout_seconds=float(resolved_timeout), - interval_seconds=self.readiness_interval_seconds, - ) + return self._runtime.readiness_probe(env, timeout_seconds=timeout_seconds) - def _wait_for_readiness(self, process: PopenLike, probe: ReadinessProbe) -> ReadinessResult: - deadline = time.monotonic() + probe.timeout_seconds - last = ReadinessResult(False, "not probed yet") - while time.monotonic() < deadline: - if process.poll() is not None: - return ReadinessResult(False, "gateway process exited during readiness check") - last = probe_once( - probe.url, - timeout_seconds=min(5.0, max(0.2, probe.interval_seconds)), - expected_status=probe.expected_status, - expected_platform=probe.expected_platform, - ) - if last.ready: - return last - time.sleep(probe.interval_seconds) - return ReadinessResult(False, f"readiness timeout: {last.message}", last.payload) + def _wait_for_readiness( + self, + process: PopenLike, + probe: ReadinessProbe, + ) -> ReadinessResult: + return self._runtime.wait_for_readiness(process, probe) def log_path(self, profile_path: str) -> Path: - return Path(profile_path) / "logs" / "zeus-gateway.log" + return self._runtime.log_path(profile_path) def pid_marker_path(self, profile_path: str) -> Path: - return Path(profile_path) / "logs" / "zeus-gateway.pid.json" + return self._runtime.pid_marker_path(profile_path) def _require_bot(self, bot_id: str) -> BotRecord: record = self.store.get_bot(bot_id) @@ -3268,17 +2687,16 @@ def _require_bot(self, bot_id: str) -> BotRecord: return record def _pid_state(self, pid: int) -> _PidState: - try: - if self.pid_alive_fn is not None: - return _PidState.alive if self.pid_alive_fn(pid) else _PidState.dead - os.kill(pid, 0) - except ProcessLookupError: - return _PidState.dead - except PermissionError: - return _PidState.unknown - except OSError as exc: - return _PidState.dead if exc.errno == errno.ESRCH else _PidState.unknown - return _PidState.alive + if "_runtime" in self.__dict__: + return self._runtime.pid_state(pid) + if self.pid_alive_fn is not None: + return _process_identity.pid_state(pid, pid_alive_fn=self.pid_alive_fn) + + def probe_with_current_kill(probe_pid: int) -> bool: + os.kill(probe_pid, 0) + return True + + return _process_identity.pid_state(pid, pid_alive_fn=probe_with_current_kill) def _unknown_pid_response( self, @@ -3305,19 +2723,7 @@ def _unknown_pid_response( ) def _send_signal(self, pid: int, sig: signal.Signals) -> _SignalResult: - try: - self.kill_fn(pid, sig) - except ProcessLookupError: - return _SignalResult.missing - except PermissionError: - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - return _SignalResult.denied - raise - return _SignalResult.sent + return self._runtime.send_signal(pid, sig) def _write_pid_marker( self, @@ -3328,80 +2734,24 @@ def _write_pid_marker( *, readiness_probe: ReadinessProbe | None | _ReadinessProbeUnset = _READINESS_PROBE_UNSET, ) -> None: - path = self.pid_marker_path(profile_path) - path.parent.mkdir(parents=True, exist_ok=True) - resolved_hermes_bin = self._resolved_hermes_bin() - marker_argv = list(argv) - if resolved_hermes_bin: - marker_argv[0] = resolved_hermes_bin - fingerprint = self.proc_start_fingerprint_reader(pid) - payload = { - "schema": 2, - "pid": pid, - "bot_id": bot_id, - "component": "gateway", - "action": "run", - "argv": marker_argv, - "resolved_hermes_bin": resolved_hermes_bin, - "started_at": time.time(), - } - if not isinstance(readiness_probe, _ReadinessProbeUnset): - payload["readiness_probe"] = _readiness_probe_marker_payload(readiness_probe) - if fingerprint: - payload["proc_start_fingerprint"] = fingerprint - atomic_write_json(path, payload, mode=0o600) + include_readiness_probe = not isinstance(readiness_probe, _ReadinessProbeUnset) + runtime_probe = ( + None if isinstance(readiness_probe, _ReadinessProbeUnset) else readiness_probe + ) + self._runtime.write_pid_marker( + profile_path, + pid, + bot_id, + argv, + readiness_probe=runtime_probe, + include_readiness_probe=include_readiness_probe, + ) def _remove_pid_marker(self, profile_path: str) -> None: - try: - self.pid_marker_path(profile_path).unlink() - except FileNotFoundError: - return + self._runtime.remove_pid_marker(profile_path) def _read_pid_marker(self, profile_path: str) -> dict[str, object]: - path = self.pid_marker_path(profile_path) - if not path.exists(): - return {"exists": False} - marker_mode = _file_mode(path) - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} - if not isinstance(payload, dict): - return { - "exists": True, - "valid": False, - "mode": marker_mode, - "error": "pid marker must be a JSON object", - } - deprecated = payload.get("schema") is None - safe_payload: dict[str, object] = { - "exists": True, - "valid": True, - "mode": marker_mode, - "deprecated": deprecated, - } - for key in ( - "schema", - "pid", - "bot_id", - "component", - "action", - "started_at", - "proc_start_fingerprint", - ): - if key in payload: - safe_payload[key] = payload[key] - if "readiness_probe" in payload: - try: - probe = _readiness_probe_from_marker(payload["readiness_probe"]) - except ValueError: - safe_payload["readiness_probe"] = "invalid" - else: - safe_payload["readiness_probe"] = _readiness_probe_marker_payload(probe) - argv_value = payload.get("argv") - if isinstance(argv_value, list) and all(isinstance(part, str) for part in argv_value): - safe_payload["argv_shape"] = _safe_command_shape(argv_value) - return safe_payload + return self._runtime.read_pid_marker(profile_path) def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: return self._verify_gateway_pid_ownership(profile_path, pid, bot_id).verified @@ -3410,176 +2760,33 @@ def _verify_gateway_pid_ownership( self, profile_path: str, pid: int, bot_id: str ) -> OwnershipCheck: record = self.store.get_bot(bot_id) - if record is not None and record.profile_path != profile_path: - return OwnershipCheck(False, "marker-mismatch") - observed = self._read_strict_runtime_marker(bot_id, profile_path) - if observed.kind == "missing": - return OwnershipCheck(False, "marker-missing") - if observed.kind != "present" or observed.payload is None: - return OwnershipCheck(False, "marker-mismatch") - payload = observed.payload - if payload.get("schema") == 3: - if record is None: - return OwnershipCheck(False, "marker-mismatch") - marker = self._classify_schema3_runtime_marker( - record, - payload, - expected_pid=pid, - require_live_command=True, - ) - if marker.kind != "live": - return OwnershipCheck(False, marker.reason or "marker-mismatch") - live_argv = self.cmdline_reader(pid) - if not live_argv: - return OwnershipCheck(False, "live-cmdline-missing") - live_check = _verify_gateway_command( - live_argv, - bot_id, - self._trusted_hermes_bins(), - require_trusted_path=True, - ) - return OwnershipCheck( - live_check.verified, - live_check.reason, - live_check.classification, - ) - if payload.get("pid") != pid: - return OwnershipCheck(False, "marker-mismatch") - argv_value = payload.get("argv") - if not isinstance(argv_value, list) or not all( - isinstance(part, str) for part in argv_value - ): - return OwnershipCheck(False, "marker-mismatch") - trusted_hermes = self._resolved_hermes_bin() - if trusted_hermes is None: - return OwnershipCheck(False, "untrusted-executable") - marker_check = self._verify_marker_payload(payload, list(argv_value), bot_id) - if not marker_check.verified: - return OwnershipCheck(False, marker_check.reason, marker_check.classification) - live_argv = self.cmdline_reader(pid) - if not live_argv: - return OwnershipCheck(False, "live-cmdline-missing") - live_check = _verify_gateway_command( - live_argv, bot_id, self._trusted_hermes_bins(), require_trusted_path=True - ) - if not live_check.verified: - return OwnershipCheck(False, live_check.reason, live_check.classification) - marker_schema = payload.get("schema") - fingerprint = payload.get("proc_start_fingerprint") - if marker_schema == 2: - live_fingerprint = self.proc_start_fingerprint_reader(pid) - if live_fingerprint and not (isinstance(fingerprint, str) and fingerprint): - return OwnershipCheck( - False, - "pid-start-time-missing", - live_check.classification, - ) - if isinstance(fingerprint, str) and fingerprint and live_fingerprint != fingerprint: - return OwnershipCheck( - False, - "pid-start-time-mismatch", - live_check.classification, - ) - elif isinstance(fingerprint, str) and fingerprint: - live_fingerprint = self.proc_start_fingerprint_reader(pid) - if live_fingerprint != fingerprint: - return OwnershipCheck( - False, - "pid-start-time-mismatch", - live_check.classification, - ) - classification = ( - "legacy-marker-valid" - if marker_check.classification == "legacy-marker-valid" - else live_check.classification + ownership = self._runtime.verify_gateway_pid_ownership( + profile_path, + pid, + bot_id, + expected_record=record, ) - if classification == "legacy-marker-valid": + if ownership.classification == "legacy-marker-valid": self.store.append_audit_event( "bot.pid_marker_legacy_accepted", bot_id=bot_id, pid=pid, ) - return OwnershipCheck(True, "ok", classification) + return ownership def _verify_marker_payload( self, payload: dict[str, object], argv: list[str], bot_id: str ) -> OwnershipCheck: - schema = payload.get("schema") - if schema == 3: - pid = payload.get("pid") - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - fingerprint = payload.get("command_fingerprint") - if ( - type(pid) is not int - or pid <= 0 - or type(operation_id) is not str - or _REQUEST_ID_RE.fullmatch(operation_id) is None - or type(revision) is not int - or revision <= 0 - or type(fingerprint) is not str - ): - return OwnershipCheck(False, "marker-mismatch") - if not _is_owned_runtime_marker( - payload, - bot_id=bot_id, - operation_id=operation_id, - desired_revision=revision, - pid=pid, - expected_fingerprint=fingerprint, - ): - return OwnershipCheck(False, "marker-mismatch") - resolved_hermes_bin = self._resolved_hermes_bin() - marker_hermes = payload.get("resolved_hermes_bin") - if ( - resolved_hermes_bin is None - or type(marker_hermes) is not str - or _resolve_executable(marker_hermes) != resolved_hermes_bin - ): - return OwnershipCheck(False, "untrusted-executable") - marker_check = _verify_gateway_command( - argv, - bot_id, - resolved_hermes_bin, - require_trusted_path=True, - ) - return OwnershipCheck( - marker_check.verified, - marker_check.reason, - marker_check.classification, - ) - if schema == 2: - if payload.get("bot_id") != bot_id: - return OwnershipCheck(False, "wrong-bot-id") - if payload.get("component") != "gateway" or payload.get("action") != "run": - return OwnershipCheck(False, "wrong-command-intent") - resolved_hermes_bin = self._resolved_hermes_bin() - if not isinstance(payload.get("resolved_hermes_bin"), str): - return OwnershipCheck(False, "untrusted-executable") - marker_hermes = _resolve_executable(str(payload["resolved_hermes_bin"])) - if marker_hermes != resolved_hermes_bin: - return OwnershipCheck(False, "untrusted-executable") - marker_check = _verify_gateway_command( - argv, bot_id, resolved_hermes_bin, require_trusted_path=True - ) - return OwnershipCheck( - marker_check.verified, - marker_check.reason, - marker_check.classification, - ) - if schema is not None: - return OwnershipCheck(False, "marker-mismatch") - if not self.allow_legacy_pid_markers: - return OwnershipCheck(False, "legacy-marker-disabled") - marker_check = _verify_gateway_command(argv, bot_id, None, require_trusted_path=False) - if not marker_check.verified: - return OwnershipCheck(False, marker_check.reason, marker_check.classification) - return OwnershipCheck(True, "ok", "legacy-marker-valid") + return self._runtime.verify_marker_payload(payload, argv, bot_id) def _resolved_hermes_bin(self) -> str | None: + if "_runtime" in self.__dict__: + return self._runtime.resolved_hermes_bin() return _resolve_executable(self.adapter.hermes_bin) def _trusted_hermes_bins(self) -> set[str]: + if "_runtime" in self.__dict__: + return self._runtime.trusted_hermes_bins() return _trusted_hermes_paths(self.adapter.hermes_bin) def _cleanup_failed_start_registration( @@ -3650,36 +2857,12 @@ def _cleanup_failed_start_registration( ) return stopped - def _terminate_spawned_process(self, process: PopenLike, cleanup_errors: list[str]) -> bool: - if process.poll() is not None: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - if self._spawned_tree_stopped(process, timeout=0): - return True - term_result = self._signal_spawned_process(process, signal.SIGTERM, cleanup_errors) - if term_result == _SignalResult.missing: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - return self._spawned_tree_stopped(process, timeout=0) - if term_result == _SignalResult.denied: - return False - self._reap_spawned_process( - process, - cleanup_errors, - timeout=self.stop_grace_seconds, - ) - if self._spawned_tree_stopped(process, timeout=0): - return True - kill_result = self._signal_spawned_process(process, signal.SIGKILL, cleanup_errors) - if kill_result == _SignalResult.missing: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - return self._spawned_tree_stopped(process, timeout=0) - if kill_result == _SignalResult.denied: - return False - self._reap_spawned_process( - process, - cleanup_errors, - timeout=self.stop_grace_seconds, - ) - return self._spawned_tree_stopped(process, timeout=self.stop_grace_seconds) + def _terminate_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + ) -> bool: + return self._runtime.terminate_spawned_process(process, cleanup_errors) def _signal_spawned_process( self, @@ -3687,41 +2870,7 @@ def _signal_spawned_process( sig: signal.Signals, cleanup_errors: list[str], ) -> _SignalResult: - if self._cleanup_process_group: - try: - os.killpg(process.pid, sig) - except ProcessLookupError: - return _SignalResult.missing - except PermissionError as exc: - cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") - return _SignalResult.denied - raise - return _SignalResult.sent - method_name = "terminate" if sig == signal.SIGTERM else "kill" - method = getattr(process, method_name, None) - if not callable(method): - return self._send_signal(process.pid, sig) - try: - method() - except ProcessLookupError: - return _SignalResult.missing - except PermissionError as exc: - cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") - return _SignalResult.denied - raise - return _SignalResult.sent + return self._runtime.signal_spawned_process(process, sig, cleanup_errors) def _reap_spawned_process( self, @@ -3730,307 +2879,52 @@ def _reap_spawned_process( *, timeout: float, ) -> bool: - wait = getattr(process, "wait", None) - if callable(wait): - try: - wait(timeout=timeout) - return True - except subprocess.TimeoutExpired: - return False - except Exception as exc: - cleanup_errors.append(f"wait: {type(exc).__name__}: {exc}") - return process.poll() is not None or self._pid_state(process.pid) == _PidState.dead + return self._runtime.reap_spawned_process( + process, + cleanup_errors, + timeout=timeout, + ) def _spawned_tree_stopped(self, process: PopenLike, *, timeout: float) -> bool: - if not self._cleanup_process_group: - return process.poll() is not None or self._pid_state(process.pid) == _PidState.dead - deadline = time.monotonic() + timeout - while True: - try: - os.killpg(process.pid, 0) - except ProcessLookupError: - return True - except PermissionError: - return False - except OSError as exc: - return exc.errno == errno.ESRCH - if time.monotonic() >= deadline: - return False - time.sleep(0.05) + return self._runtime.spawned_tree_stopped(process, timeout=timeout) def _wait_for_exit(self, bot_id: str, pid: int) -> bool: - process = self._processes.get(bot_id) - if process is not None and hasattr(process, "wait"): - try: - process.wait(timeout=self.stop_grace_seconds) - return True - except subprocess.TimeoutExpired: - return False - except Exception: - return False - - deadline = time.monotonic() + self.stop_grace_seconds - while self._pid_state(pid) != _PidState.dead and time.monotonic() < deadline: - time.sleep(0.1) - return self._pid_state(pid) == _PidState.dead + return self._runtime.wait_for_exit(bot_id, pid) def _poll_startup(self, process: PopenLike) -> int | None: - returncode = process.poll() - if returncode is not None or self.startup_grace_seconds <= 0: - return returncode - - deadline = time.monotonic() + self.startup_grace_seconds - while time.monotonic() < deadline: - time.sleep(min(0.01, max(deadline - time.monotonic(), 0))) - returncode = process.poll() - if returncode is not None: - return returncode - return process.poll() + return self._runtime.poll_startup(process) def _read_process_cmdline(pid: int) -> list[str] | None: - system = platform.system() - if system == "Linux": - return _read_linux_cmdline(pid) - if system == "Darwin": - return _read_darwin_cmdline(pid) - return None + return _process_identity.read_process_cmdline( + pid, + system=platform.system(), + run_process=subprocess.run, + ) def _readiness_probe_marker_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: - if probe is None: - return None - return { - "url": probe.url, - "expected_status": probe.expected_status, - "expected_platform": probe.expected_platform, - "timeout_seconds": probe.timeout_seconds, - "interval_seconds": probe.interval_seconds, - } + return readiness_probe_to_payload(probe) def _readiness_probe_from_marker(value: object) -> ReadinessProbe | None: - if value is None: - return None - if not isinstance(value, dict): - raise ValueError("readiness_probe must be an object or null") - url = value.get("url") - expected_status = value.get("expected_status") - expected_platform = value.get("expected_platform") - timeout_seconds = value.get("timeout_seconds") - interval_seconds = value.get("interval_seconds") - if not isinstance(url, str): - raise ValueError("url must be a string") - parsed = urlparse(url) - if ( - parsed.scheme != "http" - or parsed.hostname not in {"127.0.0.1", "localhost", "::1"} - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - ): - raise ValueError("url must be loopback HTTP without credentials or query data") - if not isinstance(expected_status, str) or not expected_status: - raise ValueError("expected_status must be a non-empty string") - if not isinstance(expected_platform, str) or not expected_platform: - raise ValueError("expected_platform must be a non-empty string") - if not _valid_probe_number(timeout_seconds) or not _valid_probe_number(interval_seconds): - raise ValueError("probe timing values must be finite positive numbers") - return ReadinessProbe( - url=url, - expected_status=expected_status, - expected_platform=expected_platform, - timeout_seconds=float(timeout_seconds), - interval_seconds=float(interval_seconds), - ) - - -def _valid_probe_number(value: object) -> TypeGuard[int | float]: - return ( - isinstance(value, int | float) - and not isinstance(value, bool) - and math.isfinite(float(value)) - and 0 < float(value) <= 3600 - ) - - -def _read_linux_cmdline(pid: int, proc_root: Path = Path("/proc")) -> list[str]: - try: - raw = (proc_root / str(pid) / "cmdline").read_bytes() - except FileNotFoundError: - return [] - except OSError: - return [] - return [part.decode("utf-8", errors="surrogateescape") for part in raw.split(b"\0") if part] + return readiness_probe_from_payload(value) def _read_darwin_cmdline(pid: int) -> list[str] | None: - try: - completed = subprocess.run( # nosec B603 - ["/bin/ps", "-p", str(pid), "-o", "command="], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=1, - ) - except (OSError, subprocess.TimeoutExpired): - return None - if completed.returncode != 0: - return [] - command = completed.stdout.strip() - if not command: - return [] - try: - return shlex.split(command) - except ValueError: - return None + return _process_identity.read_darwin_cmdline(pid, run_process=subprocess.run) def _read_process_start_fingerprint(pid: int) -> str | None: - system = platform.system() - if system == "Darwin": - return _read_darwin_process_start_fingerprint(pid) - if system != "Linux": - return None - return _read_linux_process_start_fingerprint(pid) + return _process_identity.read_process_start_fingerprint( + pid, + system=platform.system(), + run_process=subprocess.run, + ) def _read_darwin_process_start_fingerprint(pid: int) -> str | None: - try: - completed = subprocess.run( # nosec B603 - ["/bin/ps", "-p", str(pid), "-o", "lstart="], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=1, - ) - except (OSError, subprocess.TimeoutExpired): - return None - if completed.returncode != 0: - return None - started = " ".join(completed.stdout.split()) - return f"darwin:ps-lstart:{started}" if started else None - - -def _read_linux_process_start_fingerprint(pid: int, proc_root: Path = Path("/proc")) -> str | None: - try: - stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8") - except (FileNotFoundError, OSError, UnicodeDecodeError): - return None - try: - fields = stat.rsplit(") ", 1)[1].split() - except IndexError: - return None - if len(fields) < 20: - return None - return f"linux:/proc-starttime:{fields[19]}" - - -def _verify_gateway_command( - argv: list[str], - bot_id: str, - trusted_hermes_bin: str | set[str] | None, - *, - require_trusted_path: bool, -) -> _CommandCheck: - if not argv: - return _CommandCheck(False, "live-cmdline-missing") - classification = "direct-hermes" - hermes_command = argv[0] - args = argv[1:] - if len(argv) >= 2 and _looks_like_python_interpreter(argv[0]): - classification = "python-script-wrapper" - hermes_command = argv[1] - args = argv[2:] - if len(args) != 4 or args.count("-p") != 1 or args[0] != "-p": - return _CommandCheck(False, "wrong-command-intent", classification) - if args[1] != bot_id: - return _CommandCheck(False, "wrong-bot-id", classification) - if args[2:] != ["gateway", "run"]: - return _CommandCheck(False, "wrong-command-intent", classification) - if require_trusted_path: - resolved_command = _resolve_executable(hermes_command) - if isinstance(trusted_hermes_bin, str): - trusted_hermes_bins = {trusted_hermes_bin} - else: - trusted_hermes_bins = trusted_hermes_bin or set() - if not trusted_hermes_bins or resolved_command not in trusted_hermes_bins: - return _CommandCheck(False, "untrusted-executable", classification) - return _CommandCheck(True, "ok", classification) - - -def _looks_like_python_interpreter(command: str) -> bool: - return bool(_PYTHON_INTERPRETER_RE.fullmatch(Path(command).name.lower())) - - -def _resolve_executable(command: str, path: str | None = None) -> str | None: - if not command: - return None - candidate = command if "/" in command else shutil.which(command, path=path) - if candidate is None: - return None - try: - return str(Path(candidate).expanduser().resolve()) - except (OSError, RuntimeError): - return str(Path(candidate).expanduser().absolute()) - - -def _trusted_hermes_paths(command: str) -> set[str]: - resolved = _resolve_executable(command) - if resolved is None: - return set() - paths = {resolved} - delegated = _resolve_launcher_exec_target(resolved) - if delegated is not None: - paths.add(delegated) - return paths - - -def _resolve_launcher_exec_target(command: str) -> str | None: - path = Path(command) - try: - text = path.read_text(encoding="utf-8", errors="ignore") - except (OSError, UnicodeDecodeError): - return None - if not text.startswith("#!"): - return None - for line in text.splitlines()[1:20]: - stripped = line.strip() - if not stripped.startswith("exec "): - continue - try: - parts = shlex.split(stripped) - except ValueError: - continue - if len(parts) < 2 or parts[0] != "exec": - continue - target = parts[1] - if "/" not in target: - continue - resolved = _resolve_executable(target) - if resolved and Path(resolved).name == "hermes": - return resolved - return None - - -def _safe_command_shape(argv: list[str]) -> str: - if not argv: - return "empty" - classification = "direct-hermes" - args = argv[1:] - if len(argv) >= 2 and _looks_like_python_interpreter(argv[0]): - classification = "python-script-wrapper" - args = argv[2:] - if len(args) == 4 and args[0] == "-p" and args[2:] == ["gateway", "run"]: - return f"{classification} hermes -p gateway run" - return f"{classification} unrecognized" - - -def _file_mode(path: Path) -> str | None: - try: - return f"{stat.S_IMODE(path.stat().st_mode):04o}" - except OSError: - return None + return _process_identity.read_darwin_process_start_fingerprint( + pid, + run_process=subprocess.run, + )