From 5d515eda369dad479665ed8d77feceb4caae2b28 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:52:49 +0200 Subject: [PATCH 01/49] test: characterize public lifecycle contracts --- tests/test_api.py | 183 +++++++++++++++++++++++++++++- tests/test_crash_recovery.py | 85 +++++++++++++- tests/test_lifecycle_ledger.py | 69 ++++++++++- tests/test_supervisor_cli_api.py | 189 ++++++++++++++++++++++++++++++- 4 files changed, 521 insertions(+), 5 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 45132a1..734f940 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -22,7 +22,8 @@ from zeus import api as api_module from zeus.api import make_handler, serve from zeus.config import Settings -from zeus.models import BotRecord, BotStatus, BotStatusResponse +from zeus.errors import ZeusConflictError +from zeus.models import BotRecord, BotStatus, BotStatusResponse, TemplateError from zeus.process_lock import LockTimeoutError from zeus.rate_limit import TokenBucket from zeus.reconciliation import ( @@ -235,6 +236,157 @@ def raw_post_without_content_length(port: int, body: bytes) -> tuple[int, dict[s class ApiBehaviorTests(unittest.TestCase): + def test_exception_status_payload_and_header_contracts(self) -> None: + cases = ( + ( + "unknown_bot", + lambda: KeyError("unknown bot: coder"), + 404, + "unknown_bot", + "unknown bot: coder", + ), + ( + "unknown_template", + lambda: KeyError("unknown template: missing"), + 400, + "unknown_template", + "unknown template: missing", + ), + ( + "missing_field", + lambda: KeyError("bot_id"), + 400, + "invalid_request", + "missing required field: bot_id", + ), + ( + "invalid_bot_id", + lambda: TemplateError("bot_id must match ^[a-z][a-z0-9-]{1,62}$"), + 400, + "invalid_bot_id", + "bot_id must match ^[a-z][a-z0-9-]{1,62}$", + ), + ( + "template_error", + lambda: TemplateError("template contract failed"), + 400, + "invalid_request", + "template contract failed", + ), + ( + "reconcile_lock", + lambda: ReconcileLockTimeoutError(Path("/tmp/reconcile.lock"), 0.1), + 409, + "reconcile_locked", + "reconciliation is already in progress", + ), + ( + "bot_lock", + lambda: LockTimeoutError(Path("/tmp/coder.lock"), 0.1), + 409, + "bot_locked", + "bot lifecycle operation is already in progress", + ), + ( + "conflict", + lambda: ZeusConflictError("conflicting lifecycle state"), + 409, + "conflict", + "conflicting lifecycle state", + ), + ( + "value_error", + lambda: ValueError("invalid lifecycle request"), + 400, + "invalid_request", + "invalid lifecycle request", + ), + ( + "internal_error", + lambda: RuntimeError("private failure detail"), + 500, + "internal_error", + "internal server error", + ), + ) + + class RaisingSupervisor: + failure: Exception + + def __init__(self, *args: object, **kwargs: object) -> None: + return + + def status(self, *args: object, **kwargs: object) -> BotStatusResponse: + raise self.failure + + def start(self, *args: object, **kwargs: object) -> BotStatusResponse: + raise self.failure + + with ( + patch("zeus.api.Supervisor", RaisingSupervisor), + api_server({"ZEUS_API_KEY": "secret"}) as port, + ): + for name, make_error, expected_status, code, message in cases: + expected = { + "error": { + "code": code, + "message": message, + "status": expected_status, + } + } + expected_body = json.dumps(expected, sort_keys=True).encode("utf-8") + requests = ( + ("GET", "/bots/coder/status", {}), + ( + "POST", + "/bots/coder/start", + {"idempotency-key": f"characterization-{name}"}, + ), + ) + for method, path, extra_headers in requests: + with self.subTest(name=name, method=method): + RaisingSupervisor.failure = make_error() + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request( + method, + path, + headers={ + "x-zeus-api-key": "secret", + **extra_headers, + }, + ) + response = conn.getresponse() + body = response.read() + headers = {key.lower(): value for key, value in response.getheaders()} + finally: + conn.close() + + self.assertEqual(expected_status, response.status) + self.assertEqual(expected_body, body) + self.assertEqual( + { + "cache-control": "no-store", + "content-length": str(len(expected_body)), + "content-type": "application/json", + "cross-origin-resource-policy": "same-origin", + "referrer-policy": "no-referrer", + "x-content-type-options": "nosniff", + }, + { + key: headers[key] + for key in ( + "cache-control", + "content-length", + "content-type", + "cross-origin-resource-policy", + "referrer-policy", + "x-content-type-options", + ) + }, + ) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + def test_all_json_responses_include_generated_request_id(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: for method, path in (("GET", "/health"), ("GET", "/bots"), ("PUT", "/bots")): @@ -615,6 +767,35 @@ def test_v1_prefix_routes_health_and_bots(self) -> None: self.assertEqual(200, status) self.assertEqual("coder", body[0]["bot_id"]) + def test_v1_alias_preserves_queries_and_error_contracts(self) -> None: + cases = ( + ("GET", "/", None), + ("GET", "/bots/missing/status", None), + ("GET", "/bots/missing/history?limit=1", None), + ("POST", "/bots/missing/start?wait=1&timeout=2.5", None), + ("POST", "/bots/missing/stop?kill_after_timeout=1", None), + ("POST", "/bots/missing/restart?wait=0", None), + ("POST", "/bots/missing/reconcile?summary=1", None), + ) + with api_server({"ZEUS_API_KEY": "secret"}) as port: + for method, path, body in cases: + with self.subTest(method=method, path=path): + rooted = request_json( + port, + method, + path, + body=body, + headers={"x-zeus-api-key": "secret"}, + ) + aliased = request_json( + port, + method, + "/v1" + path, + body=body, + headers={"x-zeus-api-key": "secret"}, + ) + self.assertEqual(rooted, aliased) + def test_bot_create_and_list_expose_desired_state_and_convergence(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: status, created = request_json( diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index 6ed0137..410d7ae 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -1566,6 +1566,65 @@ def _begin_restart_over_running_gateway( ) return pending, marker_path + def test_start_and_stop_effects_observe_committed_lifecycle_intents(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store, supervisor = self._fixture(Path(tmp)) + observed: list[tuple[str | None, int, str]] = [] + + def spawn_after_intent(*args: object, **kwargs: object) -> _RecoveryPopen: + pending = store.get_bot("coder") + assert pending is not None + event = store.list_lifecycle_events("coder", limit=1, before=None)[0] + observed.append((pending.pending_action, pending.desired_revision, event.action)) + return _RecoveryPopen(*args, **kwargs) + + supervisor.popen_factory = spawn_after_intent + _RecoveryPopen.launches.clear() + + started = supervisor.start("coder") + + self.assertEqual(BotStatus.running, started.status) + self.assertEqual([("start", 1, "bot.start.intent")], observed) + + with tempfile.TemporaryDirectory() as tmp: + alive = True + store, supervisor = self._fixture( + Path(tmp), + desired=DesiredState.running, + pid_alive_fn=lambda pid: alive, + ) + record = store.get_bot("coder") + assert record is not None + running = replace( + record, + status=BotStatus.running, + pid=4321, + desired_revision=1, + ) + store.upsert_bot(running) + self._write_runtime_marker( + supervisor, + running.profile_path, + self._runtime_marker(supervisor, desired_revision=1), + ) + observed = [] + + def signal_after_intent(pid: int, sent_signal: object) -> None: + nonlocal alive + pending = store.get_bot("coder") + assert pending is not None + event = store.list_lifecycle_events("coder", limit=1, before=None)[0] + observed.append((pending.pending_action, pending.desired_revision, event.action)) + alive = False + + supervisor.kill_fn = signal_after_intent + supervisor.stop_grace_seconds = 0 + + stopped = supervisor.stop("coder") + + self.assertEqual(BotStatus.stopped, stopped.status) + self.assertEqual([("stop", 2, "bot.stop.intent")], observed) + def test_pending_compat_restart_fails_closed_without_side_effects(self) -> None: for schema in (2, None): for alive in (True, False): @@ -1752,7 +1811,8 @@ def test_status_never_spawns_to_enforce_desired_running(self) -> None: def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> None: with tempfile.TemporaryDirectory() as tmp: - store, supervisor = self._fixture(Path(tmp)) + root = Path(tmp) + store, _interrupted_supervisor = self._fixture(root) _RecoveryPopen.launches.clear() pending = store.begin_lifecycle_intent( "coder", @@ -1761,6 +1821,27 @@ def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> N source="cli", ) + hermes = root / "bin" / "hermes" + hermes_root = root / "hermes" + recovered_store = StateStore(root / "zeus.db") + recovered_store.init() + supervisor = Supervisor( + recovered_store, + str(hermes), + hermes_root, + popen_factory=_RecoveryPopen, + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [ + str(hermes.resolve()), + "-p", + "coder", + "gateway", + "run", + ], + proc_start_fingerprint_reader=lambda pid: "test-start:4321", + startup_grace_seconds=0, + ) + result = supervisor.reconcile("coder")[0] self.assertEqual(BotStatus.running, result.status) @@ -1768,7 +1849,7 @@ def test_reconcile_crash_before_spawn_reuses_pending_correlation_once(self) -> N marker = _RecoveryPopen.launches[0]["marker"] self.assertEqual("a" * 32, marker["operation_id"]) self.assertEqual(pending.desired_revision, marker["desired_revision"]) - loaded = store.get_bot("coder") + loaded = recovered_store.get_bot("coder") assert loaded is not None self.assertIsNone(loaded.pending_operation_id) diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index f03c0f0..c9169ac 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -11,7 +11,7 @@ from unittest.mock import patch from zeus.lifecycle import LifecycleEvent, LifecycleEventInput, serialize_lifecycle_details -from zeus.models import BotRecord, BotStatus +from zeus.models import BotRecord, BotStatus, DesiredState from zeus.state import StateStore @@ -321,6 +321,73 @@ def test_history_rejects_unsafe_limits_and_cursors(self) -> None: before=invalid_before, # type: ignore[arg-type] ) + def test_pending_intent_projection_and_event_are_committed_as_one_fence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot(self._record(root)) + + pending = store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="api", + request_id="b" * 32, + reason="gateway start requested", + ) + + history = store.list_lifecycle_events("coder", limit=10, before=None) + self.assertEqual(1, len(history)) + intent = history[0] + with closing(sqlite3.connect(store.database_path)) as conn: + last_event_id = conn.execute( + "SELECT last_event_id FROM bots WHERE bot_id = 'coder'" + ).fetchone()[0] + self.assertEqual(intent.event_id, last_event_id) + self.assertEqual(DesiredState.running, pending.desired_state) + self.assertEqual(1, pending.desired_revision) + self.assertEqual("a" * 32, pending.pending_operation_id) + self.assertEqual("start", pending.pending_action) + self.assertEqual( + { + "bot_id": "coder", + "operation_id": "a" * 32, + "request_id": "b" * 32, + "source": "api", + "action": "bot.start.intent", + "outcome": "pending", + "status_before": "stopped", + "status_after": "stopped", + "pid_before": None, + "pid_after": None, + "reason": "gateway start requested", + "error_code": None, + "error_message": None, + "details": { + "action": "start", + "desired_revision": 1, + "desired_state": "running", + }, + }, + { + "bot_id": intent.bot_id, + "operation_id": intent.operation_id, + "request_id": intent.request_id, + "source": intent.source, + "action": intent.action, + "outcome": intent.outcome, + "status_before": intent.status_before, + "status_after": intent.status_after, + "pid_before": intent.pid_before, + "pid_after": intent.pid_after, + "reason": intent.reason, + "error_code": intent.error_code, + "error_message": intent.error_message, + "details": dict(intent.details), + }, + ) + def test_event_input_is_frozen_and_recursively_redacts_bounded_details(self) -> None: nested_items = [{"visible": "value"}] nested: dict[str, object] = { diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index bc199cc..8035d08 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -23,7 +23,7 @@ from zeus.cli import main as cli_main from zeus.config import Settings from zeus.doctor import _check_runtime_paths, run_doctor -from zeus.errors import BotArchiveError, BotDeleteError +from zeus.errors import BotArchiveError, BotDeleteError, BotExistsError from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEventInput from zeus.logging_utils import redact_secrets @@ -372,6 +372,86 @@ def test_adapter_builds_secret_safe_launcher_command_and_private_payload(self) - payload["marker_path"], ) + def test_strict_runtime_contract_accepts_schema3_and_rejects_compat_markers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / "hermes" + profile = hermes_root / "profiles" / "coder" + profile.mkdir(parents=True) + hermes_bin = self._fake_hermes_path(root) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + ) + ) + pending = store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + ) + supervisor = Supervisor( + store, + hermes_bin, + hermes_root, + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: self._gateway_argv(hermes_bin), + proc_start_fingerprint_reader=lambda pid: "test-process-start:4321", + ) + schema3_payload = supervisor.adapter.launcher_payload( + "coder", + operation_id="a" * 32, + desired_revision=pending.desired_revision, + readiness_probe=None, + )["marker"] + assert isinstance(schema3_payload, dict) + schema3 = dict(schema3_payload) + schema3.update( + { + "pid": 4321, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + ) + schema2 = { + "schema": 2, + "pid": 4321, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "argv": self._gateway_argv(hermes_bin), + "resolved_hermes_bin": hermes_bin, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + legacy = { + "pid": 4321, + "argv": self._gateway_argv(hermes_bin), + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } + + for name, marker, expected_kind, expected_reason in ( + ("schema3", schema3, "live", ""), + ("schema2", schema2, "untrusted", "marker correlation is invalid"), + ("legacy", legacy, "untrusted", "marker correlation is invalid"), + ): + with self.subTest(name=name): + observed = supervisor._classify_schema3_runtime_marker( + pending, + marker, + expected_operation_id="a" * 32, + expected_revision=pending.desired_revision, + require_live_command=False, + ) + self.assertEqual(expected_kind, observed.kind) + self.assertEqual(expected_reason, observed.reason) + def test_linux_cmdline_reader_uses_proc_cmdline(self) -> None: with tempfile.TemporaryDirectory() as tmp: proc_root = Path(tmp) / "proc" @@ -1970,6 +2050,113 @@ def test_cli_lifecycle_failures_return_nonzero(self) -> None: self.assertEqual(BotStatus.failed, started["status"]) self.assertEqual(BotStatus.failed, status["status"]) + def test_cli_create_start_stop_and_reconcile_json_exit_contracts(self) -> None: + timestamp = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + created = BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path="/profiles/coder", + created_at=timestamp, + updated_at=timestamp, + ) + started = BotStatusResponse( + bot_id="start-failed", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/start-failed", + message="failed to start gateway: missing hermes", + ) + stopped = BotStatusResponse( + bot_id="coder", + status=BotStatus.stopped, + pid=None, + profile_path="/profiles/coder", + message="gateway shutdown completed", + ) + pending = BotStatusResponse( + bot_id="pending-bot", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/pending-bot", + message="restart scheduled: attempt 1/5 in 5s", + ) + terminal = BotStatusResponse( + bot_id="broken-bot", + status=BotStatus.failed, + pid=None, + profile_path="/profiles/broken-bot", + message="manual policy: not restarting", + ) + + class CliContractSupervisor: + def create_bot(self, request, template, **kwargs): + if request.bot_id == "existing": + raise BotExistsError("bot already exists: existing") + return created + + def start(self, bot_id: str, **kwargs) -> BotStatusResponse: + return started + + def stop(self, bot_id: str, **kwargs) -> BotStatusResponse: + return stopped + + def reconcile(self, bot_id: str | None, **kwargs) -> list[BotStatusResponse]: + return [pending if bot_id == "pending-bot" else terminal] + + def run(argv: list[str]) -> tuple[int, str]: + stdout = io.StringIO() + with redirect_stdout(stdout): + exit_code = cli_main(argv) + return exit_code, stdout.getvalue() + + cases = ( + ( + "create_success", + ["bot", "create", "coder", "--template", "coding-bot", "--json"], + 0, + created.to_dict(), + ), + ( + "create_conflict", + ["bot", "create", "existing", "--template", "coding-bot", "--json"], + 1, + { + "error": { + "code": "bot_exists", + "message": "bot already exists: existing", + } + }, + ), + ("start_failure", ["bot", "start", "start-failed"], 1, started.to_dict()), + ("stop_success", ["bot", "stop", "coder"], 0, stopped.to_dict()), + ( + "reconcile_pending", + ["bot", "reconcile", "pending-bot", "--json"], + 0, + [pending.to_dict()], + ), + ( + "reconcile_terminal", + ["bot", "reconcile", "broken-bot", "--json"], + 1, + [terminal.to_dict()], + ), + ) + with ( + tempfile.TemporaryDirectory() as tmp, + patch.dict(os.environ, {"ZEUS_STATE_DIR": str(Path(tmp) / ".zeus")}), + patch("zeus.cli._services", return_value=(object(), CliContractSupervisor())), + ): + for name, argv, expected_exit, expected_payload in cases: + with self.subTest(name=name): + exit_code, output = run(argv) + self.assertEqual(expected_exit, exit_code) + self.assertEqual( + json.dumps(expected_payload, sort_keys=True) + "\n", + output, + ) + def test_cli_wait_timeout_returns_nonzero_but_async_starting_is_successful(self) -> None: response = BotStatusResponse( bot_id="coder", From 24e8c7c028559544fb9038b7ae766899f46da734 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:01:24 +0200 Subject: [PATCH 02/49] test: strengthen lifecycle characterization fence --- tests/test_api.py | 154 +++++++++++++++++++++----- tests/test_supervisor_cli_api.py | 184 ++++++++++++++++++++++--------- 2 files changed, 260 insertions(+), 78 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 734f940..30e1fa0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -767,34 +767,134 @@ def test_v1_prefix_routes_health_and_bots(self) -> None: self.assertEqual(200, status) self.assertEqual("coder", body[0]["bot_id"]) - def test_v1_alias_preserves_queries_and_error_contracts(self) -> None: - cases = ( - ("GET", "/", None), - ("GET", "/bots/missing/status", None), - ("GET", "/bots/missing/history?limit=1", None), - ("POST", "/bots/missing/start?wait=1&timeout=2.5", None), - ("POST", "/bots/missing/stop?kill_after_timeout=1", None), - ("POST", "/bots/missing/restart?wait=0", None), - ("POST", "/bots/missing/reconcile?summary=1", None), - ) - with api_server({"ZEUS_API_KEY": "secret"}) as port: - for method, path, body in cases: - with self.subTest(method=method, path=path): - rooted = request_json( - port, - method, - path, - body=body, - headers={"x-zeus-api-key": "secret"}, - ) - aliased = request_json( - port, - method, - "/v1" + path, - body=body, - headers={"x-zeus-api-key": "secret"}, + def test_v1_alias_normalizes_roots_and_forwards_lifecycle_queries(self) -> None: + calls: list[tuple[object, ...]] = [] + + class CapturingSupervisor: + def __init__(self, *args: object, **kwargs: object) -> None: + return + + def start( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("start", bot_id, wait, timeout_seconds, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.starting, 123, "/profiles/coder") + + def stop( + self, + bot_id: str, + *, + kill_after_timeout: bool | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("stop", bot_id, kill_after_timeout, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.stopped, None, "/profiles/coder") + + def restart( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + calls.append(("restart", bot_id, wait, timeout_seconds, source, request_id)) + return BotStatusResponse(bot_id, BotStatus.running, 456, "/profiles/coder") + + def reconcile_summary( + self, + bot_id: str | None = None, + *, + source: str = "cli", + request_id: str | None = None, + **kwargs: object, + ) -> object: + calls.append(("reconcile_summary", bot_id, source, request_id)) + + class Summary: + def to_dict(self) -> dict[str, object]: + return {"ok": True, "scope": "bot"} + + return Summary() + + headers = {"x-zeus-api-key": "secret"} + with ( + patch("zeus.api.Supervisor", CapturingSupervisor), + api_server({"ZEUS_API_KEY": "secret"}) as port, + ): + for root in ("/v1", "/v1/"): + with self.subTest(root=root): + status, body = request_json(port, "GET", root, headers=headers) + self.assertEqual(404, status) + self.assertEqual( + { + "error": { + "code": "invalid_request", + "message": "not found", + "status": 404, + } + }, + body, ) - self.assertEqual(rooted, aliased) + + requests = ( + ( + "/v1/bots/coder/start?wait=1&timeout=2.5", + { + "bot_id": "coder", + "message": "", + "pid": 123, + "profile_path": "/profiles/coder", + "status": "starting", + }, + ("start", "coder", True, 2.5, "api"), + ), + ( + "/v1/bots/coder/stop?kill_after_timeout=1", + { + "bot_id": "coder", + "message": "", + "pid": None, + "profile_path": "/profiles/coder", + "status": "stopped", + }, + ("stop", "coder", True, "api"), + ), + ( + "/v1/bots/coder/restart?wait=0&timeout=3.5", + { + "bot_id": "coder", + "message": "", + "pid": 456, + "profile_path": "/profiles/coder", + "status": "running", + }, + ("restart", "coder", False, 3.5, "api"), + ), + ( + "/v1/bots/coder/reconcile?summary=1", + {"ok": True, "scope": "bot"}, + ("reconcile_summary", "coder", "api"), + ), + ) + for path, expected_body, expected_call in requests: + with self.subTest(path=path): + status, body = request_json(port, "POST", path, headers=headers) + self.assertEqual(200, status) + self.assertEqual(expected_body, body) + call = calls.pop(0) + self.assertEqual(expected_call, call[:-1]) + self.assertRegex(str(call[-1]), r"^[0-9a-f]{32}$") + + self.assertEqual([], calls) def test_bot_create_and_list_expose_desired_state_and_convergence(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 8035d08..d29c478 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import http.client import io import json @@ -381,20 +382,18 @@ def test_strict_runtime_contract_accepts_schema3_and_rejects_compat_markers(self hermes_bin = self._fake_hermes_path(root) store = StateStore(root / "zeus.db") store.init() - store.upsert_bot( - BotRecord( - bot_id="coder", - template_id="coding-bot", - display_name="Coder", - profile_path=str(profile), - ) - ) - pending = store.begin_lifecycle_intent( - "coder", - action="start", - operation_id="a" * 32, - source="cli", + running = BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + status=BotStatus.running, + pid=4321, + desired_state=DesiredState.running, + desired_revision=1, ) + store.upsert_bot(running) + signals: list[tuple[int, object]] = [] supervisor = Supervisor( store, hermes_bin, @@ -402,22 +401,27 @@ def test_strict_runtime_contract_accepts_schema3_and_rejects_compat_markers(self pid_alive_fn=lambda pid: True, cmdline_reader=lambda pid: self._gateway_argv(hermes_bin), proc_start_fingerprint_reader=lambda pid: "test-process-start:4321", - ) - schema3_payload = supervisor.adapter.launcher_payload( - "coder", - operation_id="a" * 32, - desired_revision=pending.desired_revision, - readiness_probe=None, - )["marker"] - assert isinstance(schema3_payload, dict) - schema3 = dict(schema3_payload) - schema3.update( - { - "pid": 4321, - "started_at": 1_780_000_000.0, - "proc_start_fingerprint": "test-process-start:4321", - } - ) + kill_fn=lambda pid, sent_signal: signals.append((pid, sent_signal)), + ) + argv = self._gateway_argv(hermes_bin) + command_fingerprint = hashlib.sha256( + json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + schema3 = { + "schema": 3, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "operation_id": "a" * 32, + "desired_revision": 1, + "argv": argv, + "resolved_hermes_bin": hermes_bin, + "command_fingerprint": command_fingerprint, + "readiness_probe": None, + "pid": 4321, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": "test-process-start:4321", + } schema2 = { "schema": 2, "pid": 4321, @@ -435,22 +439,46 @@ def test_strict_runtime_contract_accepts_schema3_and_rejects_compat_markers(self "started_at": 1_780_000_000.0, "proc_start_fingerprint": "test-process-start:4321", } + marker_path = supervisor.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True) - for name, marker, expected_kind, expected_reason in ( - ("schema3", schema3, "live", ""), - ("schema2", schema2, "untrusted", "marker correlation is invalid"), - ("legacy", legacy, "untrusted", "marker correlation is invalid"), - ): + self.assertEqual(3, schema3["schema"]) + marker_path.write_text(json.dumps(schema3, sort_keys=True), encoding="utf-8") + inspected = supervisor.inspect("coder") + + self.assertEqual(3, inspected["pid_marker"]["schema"]) + self.assertIs(True, inspected["live_cmdline_verified"]) + self.assertEqual( + { + "verified": True, + "reason": "ok", + "classification": "direct-hermes", + "expected": { + "bot_id": "coder", + "component": "gateway", + "action": "run", + }, + }, + inspected["ownership"], + ) + + for name, marker in (("schema2", schema2), ("legacy", legacy)): with self.subTest(name=name): - observed = supervisor._classify_schema3_runtime_marker( - pending, - marker, - expected_operation_id="a" * 32, - expected_revision=pending.desired_revision, - require_live_command=False, + store.upsert_bot(running) + marker_path.write_text( + json.dumps(marker, sort_keys=True), + encoding="utf-8", + ) + result = supervisor.stop("coder") + + self.assertEqual(BotStatus.failed, result.status) + self.assertEqual( + "action required: schema-v2 or legacy gateway stop " + "requires manual process resolution", + result.message, ) - self.assertEqual(expected_kind, observed.kind) - self.assertEqual(expected_reason, observed.reason) + self.assertEqual(marker, json.loads(marker_path.read_text(encoding="utf-8"))) + self.assertEqual([], signals) def test_linux_cmdline_reader_uses_proc_cmdline(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -2104,18 +2132,71 @@ def stop(self, bot_id: str, **kwargs) -> BotStatusResponse: def reconcile(self, bot_id: str | None, **kwargs) -> list[BotStatusResponse]: return [pending if bot_id == "pending-bot" else terminal] - def run(argv: list[str]) -> tuple[int, str]: + def run(argv: list[str]) -> tuple[int, str, str]: stdout = io.StringIO() - with redirect_stdout(stdout): + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): exit_code = cli_main(argv) - return exit_code, stdout.getvalue() + return exit_code, stdout.getvalue(), stderr.getvalue() + + created_json = { + "bot_id": "coder", + "template_id": "coding-bot", + "display_name": "Coder", + "profile_path": "/profiles/coder", + "status": "stopped", + "pid": None, + "restart_policy": "manual", + "restart_backoff_seconds": 5.0, + "restart_max_attempts": 5, + "restart_attempts": 0, + "next_restart_at": None, + "started_at": None, + "ready_at": None, + "stopped_at": None, + "last_exit_code": None, + "last_error": None, + "last_transition_reason": None, + "desired_state": "stopped", + "converged": True, + "created_at": "2026-07-21T12:00:00+00:00", + "updated_at": "2026-07-21T12:00:00+00:00", + } + start_json = { + "bot_id": "start-failed", + "status": "failed", + "pid": None, + "profile_path": "/profiles/start-failed", + "message": "failed to start gateway: missing hermes", + } + stop_json = { + "bot_id": "coder", + "status": "stopped", + "pid": None, + "profile_path": "/profiles/coder", + "message": "gateway shutdown completed", + } + pending_json = { + "bot_id": "pending-bot", + "status": "failed", + "pid": None, + "profile_path": "/profiles/pending-bot", + "message": "restart scheduled: attempt 1/5 in 5s", + } + terminal_json = { + "bot_id": "broken-bot", + "status": "failed", + "pid": None, + "profile_path": "/profiles/broken-bot", + "message": "manual policy: not restarting", + } cases = ( ( "create_success", ["bot", "create", "coder", "--template", "coding-bot", "--json"], 0, - created.to_dict(), + created_json, ), ( "create_conflict", @@ -2128,19 +2209,19 @@ def run(argv: list[str]) -> tuple[int, str]: } }, ), - ("start_failure", ["bot", "start", "start-failed"], 1, started.to_dict()), - ("stop_success", ["bot", "stop", "coder"], 0, stopped.to_dict()), + ("start_failure", ["bot", "start", "start-failed"], 1, start_json), + ("stop_success", ["bot", "stop", "coder"], 0, stop_json), ( "reconcile_pending", ["bot", "reconcile", "pending-bot", "--json"], 0, - [pending.to_dict()], + [pending_json], ), ( "reconcile_terminal", ["bot", "reconcile", "broken-bot", "--json"], 1, - [terminal.to_dict()], + [terminal_json], ), ) with ( @@ -2150,12 +2231,13 @@ def run(argv: list[str]) -> tuple[int, str]: ): for name, argv, expected_exit, expected_payload in cases: with self.subTest(name=name): - exit_code, output = run(argv) + exit_code, output, error_output = run(argv) self.assertEqual(expected_exit, exit_code) self.assertEqual( json.dumps(expected_payload, sort_keys=True) + "\n", output, ) + self.assertEqual("", error_output) def test_cli_wait_timeout_returns_nonzero_but_async_starting_is_successful(self) -> None: response = BotStatusResponse( From f9d59f67e17c5ae8de1caf92740c6d282059c07a Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:06:04 +0200 Subject: [PATCH 03/49] docs: align architecture with schema v6 --- docs/ARCHITECTURE.md | 2 +- tests/test_repo_contracts.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 23bbcd4..220ac66 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -130,7 +130,7 @@ The v2-to-v3 migration is also one transaction. It creates a the projection/event invariant, and advances the schema version only after all steps succeed. Additive v3-to-v4 and v4-to-v5 upgrades add durable idempotency and desired/pending intent in forward-only transactions. Databases newer than -schema v5 are rejected rather than downgraded. +schema v6 are rejected rather than downgraded. `$ZEUS_STATE_DIR/logs/audit.jsonl` remains a best-effort compatibility mirror. It is written only after the SQLite transaction commits and is not imported into diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 882a3cc..e5061ed 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -5,6 +5,8 @@ import unittest from pathlib import Path +from zeus.state import SCHEMA_VERSION + class RepoContractTests(unittest.TestCase): def test_publishable_repository_files_exist(self) -> None: @@ -238,6 +240,16 @@ def test_observability_and_lifecycle_history_are_documented(self) -> None: for field in access_fields: self.assertIn(f"`{field}`", api_docs) + def test_architecture_terminal_schema_compatibility_matches_runtime(self) -> None: + architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8") + compatibility_statement = re.search( + r"Databases newer than\s+schema v(?P\d+) are rejected rather than downgraded\.", + architecture, + ) + + self.assertIsNotNone(compatibility_statement) + self.assertEqual(str(SCHEMA_VERSION), compatibility_statement["version"]) + def test_every_openapi_response_documents_request_id_header(self) -> None: document = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) request_id_header = document["components"]["headers"]["XRequestID"] From f63cdb29d161252d9d9cb2e814bca32769cc9dcb Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:09:28 +0200 Subject: [PATCH 04/49] test: pin terminal schema compatibility statement --- tests/test_repo_contracts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index e5061ed..8e4de59 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -242,13 +242,13 @@ def test_observability_and_lifecycle_history_are_documented(self) -> None: def test_architecture_terminal_schema_compatibility_matches_runtime(self) -> None: architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8") - compatibility_statement = re.search( + compatibility_statements = re.findall( r"Databases newer than\s+schema v(?P\d+) are rejected rather than downgraded\.", architecture, ) - self.assertIsNotNone(compatibility_statement) - self.assertEqual(str(SCHEMA_VERSION), compatibility_statement["version"]) + self.assertEqual(1, len(compatibility_statements)) + self.assertEqual(str(SCHEMA_VERSION), compatibility_statements[0]) def test_every_openapi_response_documents_request_id_header(self) -> None: document = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) From ecfac2293c7ff334cb13896d15ddc961d61b6978 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:18:17 +0200 Subject: [PATCH 05/49] security: constrain readiness HTTP probes --- tests/test_lifecycle_regressions.py | 4 +- tests/test_readiness.py | 191 ++++++++++++++++++++++++++++ zeus/readiness.py | 59 +++++++-- 3 files changed, 244 insertions(+), 10 deletions(-) diff --git a/tests/test_lifecycle_regressions.py b/tests/test_lifecycle_regressions.py index 9091ede..194c87c 100644 --- a/tests/test_lifecycle_regressions.py +++ b/tests/test_lifecycle_regressions.py @@ -887,13 +887,13 @@ def test_status_uses_launch_readiness_provenance_in_a_new_process(self) -> None: patch.dict(os.environ, {"ZEUS_ENV_PASSTHROUGH": ""}, clear=False), patch( "zeus.supervisor.probe_once", - return_value=ReadinessResult(False, "connection refused"), + return_value=ReadinessResult(False, "readiness probe failed"), ) as probe_once, ): status = status_supervisor.status("coder") self.assertEqual(BotStatus.starting, status.status) - self.assertEqual("connection refused", status.message) + self.assertEqual("readiness probe failed", status.message) probe_once.assert_called_once_with( "http://127.0.0.1:4312/health", timeout_seconds=0.5, diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 6896b79..e27d4d4 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -1,10 +1,14 @@ from __future__ import annotations +import contextlib import json +import os +import socket import threading import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from unittest.mock import patch from zeus.readiness import ( ReadinessProbe, @@ -29,7 +33,27 @@ def log_message(self, format: str, *args: Any) -> None: return +class _IPv6ThreadingHTTPServer(ThreadingHTTPServer): + address_family = socket.AF_INET6 + + class ReadinessTests(unittest.TestCase): + def _run_server( + self, + handler: type[BaseHTTPRequestHandler], + *, + ipv6: bool = False, + ) -> tuple[ThreadingHTTPServer, threading.Thread]: + server_class = _IPv6ThreadingHTTPServer if ipv6 else ThreadingHTTPServer + host = "::1" if ipv6 else "127.0.0.1" + server = server_class((host, 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server, thread + def test_readiness_probe_from_env_builds_loopback_health_url(self) -> None: probe = readiness_probe_from_env( { @@ -74,6 +98,173 @@ def test_probe_once_accepts_expected_health_payload(self) -> None: self.assertTrue(result.ready) self.assertEqual("ready", result.message) + def test_probe_once_accepts_expected_health_payload_over_ipv6(self) -> None: + try: + server, _thread = self._run_server(_HealthHandler, ipv6=True) + except OSError as exc: + self.skipTest(f"IPv6 loopback is unavailable: {exc}") + + result = probe_once(f"http://[::1]:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_does_not_follow_redirects_or_leak_location(self) -> None: + class RedirectTargetHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + target, _target_thread = self._run_server(RedirectTargetHandler) + secret = "API_KEY=sentinel-secret" + + class RedirectHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(302) + self.send_header( + "location", + f"http://127.0.0.1:{target.server_port}/health?{secret}", + ) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + return + + redirect, _redirect_thread = self._run_server(RedirectHandler) + + result = probe_once(f"http://127.0.0.1:{redirect.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("readiness endpoint returned an HTTP error", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + self.assertEqual(0, RedirectTargetHandler.requests) + + def test_probe_once_ignores_proxy_environment(self) -> None: + secret = "API_KEY=sentinel-secret" + + class TargetHandler(BaseHTTPRequestHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + data = json.dumps({"status": "not-ready", "detail": secret}).encode("utf-8") + self.send_response(200) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + class ProxyHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + target, _target_thread = self._run_server(TargetHandler) + proxy, _proxy_thread = self._run_server(ProxyHandler) + proxy_url = f"http://127.0.0.1:{proxy.server_port}" + proxy_env = { + "HTTP_PROXY": proxy_url, + "http_proxy": proxy_url, + "NO_PROXY": "", + "no_proxy": "", + } + + with patch.dict(os.environ, proxy_env, clear=True), patch("urllib.request._opener", None): + result = probe_once(f"http://127.0.0.1:{target.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("unexpected readiness health payload", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + self.assertEqual(1, TargetHandler.requests) + self.assertEqual(0, ProxyHandler.requests) + + def test_probe_once_rejects_oversized_responses_with_bounded_failures(self) -> None: + max_response_bytes = 64 * 1024 + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + oversized_body = health_payload + b" " * max_response_bytes + + def oversized_handler(include_content_length: bool) -> type[BaseHTTPRequestHandler]: + class OversizedHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + if include_content_length: + self.send_header("content-length", str(len(oversized_body))) + self.end_headers() + with contextlib.suppress(BrokenPipeError): + self.wfile.write(oversized_body) + + def log_message(self, format: str, *args: Any) -> None: + return + + return OversizedHandler + + for include_content_length in (True, False): + with self.subTest(include_content_length=include_content_length): + server, _thread = self._run_server(oversized_handler(include_content_length)) + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("readiness response exceeds size limit", result.message) + self.assertIsNone(result.payload) + + def test_probe_once_rejects_url_data_before_network_io(self) -> None: + class TrackingHandler(_HealthHandler): + requests = 0 + + def do_GET(self) -> None: + self.__class__.requests += 1 + super().do_GET() + + server, _thread = self._run_server(TrackingHandler) + port = server.server_port + urls = ( + f"http://user:password@127.0.0.1:{port}/health", + f"http://127.0.0.1:{port}/health?API_KEY=sentinel-secret", + f"http://127.0.0.1:{port}/health#API_KEY=sentinel-secret", + ) + + for url in urls: + with self.subTest(url=url): + result = probe_once(url) + self.assertFalse(result.ready) + self.assertEqual("readiness URL must be loopback HTTP", result.message) + self.assertIsNone(result.payload) + self.assertNotIn("sentinel-secret", repr(result)) + + self.assertEqual(0, TrackingHandler.requests) + + def test_probe_once_never_returns_unexpected_payload_data(self) -> None: + secret = "API_KEY=sentinel-secret" + + class SecretHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + data = json.dumps({"status": "not-ready", "detail": secret}).encode("utf-8") + self.send_response(200) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(SecretHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("unexpected readiness health payload", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + def test_wait_until_ready_returns_timeout_result(self) -> None: result = wait_until_ready( ReadinessProbe( diff --git a/zeus/readiness.py b/zeus/readiness.py index f5e3195..24cb348 100644 --- a/zeus/readiness.py +++ b/zeus/readiness.py @@ -6,7 +6,21 @@ from dataclasses import dataclass from urllib.error import HTTPError, URLError from urllib.parse import urlparse -from urllib.request import urlopen +from urllib.request import ( + HTTPRedirectHandler, + ProxyHandler, + build_opener, +) + +MAX_READINESS_RESPONSE_BYTES = 64 * 1024 + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, *args: object, **kwargs: object) -> None: + return None + + +_READINESS_OPENER = build_opener(ProxyHandler({}), _RejectRedirects()) @dataclass(frozen=True) @@ -54,12 +68,36 @@ def probe_once( expected_status: str = "ok", expected_platform: str = "hermes-agent", ) -> ReadinessResult: - parsed = urlparse(url) - if parsed.scheme != "http" or parsed.hostname not in {"127.0.0.1", "localhost", "::1"}: + try: + parsed = urlparse(url) + hostname = parsed.hostname + _port = parsed.port + except ValueError: + return ReadinessResult(False, "readiness URL must be loopback HTTP") + if ( + parsed.scheme != "http" + or hostname not in {"127.0.0.1", "localhost", "::1"} + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): return ReadinessResult(False, "readiness URL must be loopback HTTP") try: - with urlopen(url, timeout=timeout_seconds) as response: # nosec B310 - payload = json.loads(response.read().decode("utf-8")) + with _READINESS_OPENER.open(url, timeout=timeout_seconds) as response: # nosec B310 + content_lengths = response.headers.get_all("content-length", []) + try: + declared_lengths = {int(value) for value in content_lengths} + except ValueError: + return ReadinessResult(False, "readiness response has invalid content length") + if any(length < 0 for length in declared_lengths) or len(declared_lengths) > 1: + return ReadinessResult(False, "readiness response has invalid content length") + if any(length > MAX_READINESS_RESPONSE_BYTES for length in declared_lengths): + return ReadinessResult(False, "readiness response exceeds size limit") + body = response.read(MAX_READINESS_RESPONSE_BYTES + 1) + if len(body) > MAX_READINESS_RESPONSE_BYTES: + return ReadinessResult(False, "readiness response exceeds size limit") + payload = json.loads(body.decode("utf-8")) if not isinstance(payload, dict): return ReadinessResult(False, "health payload must be a JSON object") if ( @@ -67,9 +105,14 @@ def probe_once( and payload.get("platform") == expected_platform ): return ReadinessResult(True, "ready", payload) - return ReadinessResult(False, f"unexpected health payload: {payload!r}", payload) - except (HTTPError, URLError, OSError, json.JSONDecodeError, ValueError) as exc: - return ReadinessResult(False, f"{type(exc).__name__}: {exc}") + return ReadinessResult(False, "unexpected readiness health payload") + except HTTPError as exc: + exc.close() + return ReadinessResult(False, "readiness endpoint returned an HTTP error") + except (URLError, OSError): + return ReadinessResult(False, "readiness probe failed") + except (UnicodeDecodeError, json.JSONDecodeError, ValueError): + return ReadinessResult(False, "readiness response is not valid JSON") def wait_until_ready(probe: ReadinessProbe) -> ReadinessResult: From 62c9be2b1e8e5e17d92344131dcd96d547192411 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:31:29 +0200 Subject: [PATCH 06/49] security: close readiness protocol bypasses --- tests/test_readiness.py | 154 +++++++++++++++++++++++++++++++++++++++- zeus/readiness.py | 17 ++++- 2 files changed, 168 insertions(+), 3 deletions(-) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index e27d4d4..aa42d77 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -4,13 +4,18 @@ import json import os import socket +import socketserver import threading import unittest +from email.message import Message +from http.client import HTTPException from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar from unittest.mock import patch +import zeus.readiness as readiness_module from zeus.readiness import ( + MAX_READINESS_RESPONSE_BYTES, ReadinessProbe, probe_once, readiness_probe_from_env, @@ -54,6 +59,18 @@ def _run_server( self.addCleanup(server.shutdown) return server, thread + def _run_raw_server( + self, + handler: type[socketserver.BaseRequestHandler], + ) -> tuple[socketserver.ThreadingTCPServer, threading.Thread]: + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server, thread + def test_readiness_probe_from_env_builds_loopback_health_url(self) -> None: probe = readiness_probe_from_env( { @@ -110,6 +127,48 @@ def test_probe_once_accepts_expected_health_payload_over_ipv6(self) -> None: self.assertEqual("ready", result.message) self.assertEqual(_HealthHandler.payload, result.payload) + def test_probe_once_accepts_health_payload_without_content_length(self) -> None: + class NoContentLengthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + data = json.dumps(_HealthHandler.payload).encode("utf-8") + self.send_response(200) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(NoContentLengthHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + + def test_probe_once_accepts_chunked_health_payload(self) -> None: + class ChunkedHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + data = json.dumps(_HealthHandler.payload).encode("utf-8") + self.send_response(200) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(data):x}\r\n".encode("ascii")) + self.wfile.write(data + b"\r\n0\r\n\r\n") + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(ChunkedHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + def test_probe_once_does_not_follow_redirects_or_leak_location(self) -> None: class RedirectTargetHandler(_HealthHandler): requests = 0 @@ -177,7 +236,14 @@ def do_GET(self) -> None: "no_proxy": "", } - with patch.dict(os.environ, proxy_env, clear=True), patch("urllib.request._opener", None): + with ( + patch.dict(os.environ, proxy_env, clear=True), + patch.object( + readiness_module, + "build_opener", + wraps=readiness_module.build_opener, + ) as build_opener, + ): result = probe_once(f"http://127.0.0.1:{target.server_port}/health") self.assertFalse(result.ready) @@ -186,6 +252,7 @@ def do_GET(self) -> None: self.assertNotIn(secret, repr(result)) self.assertEqual(1, TargetHandler.requests) self.assertEqual(0, ProxyHandler.requests) + build_opener.assert_called_once() def test_probe_once_rejects_oversized_responses_with_bounded_failures(self) -> None: max_response_bytes = 64 * 1024 @@ -216,6 +283,91 @@ def log_message(self, format: str, *args: Any) -> None: self.assertEqual("readiness response exceeds size limit", result.message) self.assertIsNone(result.payload) + def test_probe_once_rejects_trailing_data_after_false_low_content_length(self) -> None: + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + oversized_body = health_payload + b" " * MAX_READINESS_RESPONSE_BYTES + + class FalseLowContentLengthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("content-length", str(len(health_payload))) + self.end_headers() + with contextlib.suppress(BrokenPipeError, ConnectionResetError): + self.wfile.write(oversized_body) + + def log_message(self, format: str, *args: Any) -> None: + return + + server, _thread = self._run_server(FalseLowContentLengthHandler) + + result = probe_once(f"http://127.0.0.1:{server.server_port}/health") + + self.assertFalse(result.ready) + self.assertEqual("readiness response exceeds size limit", result.message) + self.assertIsNone(result.payload) + + def test_probe_once_requests_exactly_one_max_plus_one_bounded_read(self) -> None: + class TrackingResponse: + def __init__(self) -> None: + self.headers = Message() + self.length: int | None = None + self.read_amounts: list[int] = [] + + def __enter__(self) -> TrackingResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def read(self, amount: int) -> bytes: + self.read_amounts.append(amount) + return b"x" * amount + + class TrackingOpener: + def __init__(self, response: TrackingResponse) -> None: + self.response = response + self.requests: list[object] = [] + + def open(self, *args: object, **kwargs: object) -> TrackingResponse: + self.requests.append(args[0]) + return self.response + + response = TrackingResponse() + opener = TrackingOpener(response) + + with patch.object(readiness_module, "_build_readiness_opener", return_value=opener): + result = probe_once("http://127.0.0.1:4312/health") + + self.assertEqual([MAX_READINESS_RESPONSE_BYTES + 1], response.read_amounts) + request = opener.requests[0] + get_header = getattr(request, "get_header", lambda _name: None) + self.assertEqual("close", get_header("Connection")) + self.assertFalse(result.ready) + self.assertEqual("readiness response exceeds size limit", result.message) + self.assertIsNone(result.payload) + + def test_probe_once_maps_malformed_http_to_safe_failure(self) -> None: + secret = "API_KEY=sentinel-secret" + + class MalformedProtocolHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + self.request.recv(4096) + self.request.sendall(f"NOT-HTTP {secret}\r\n\r\n".encode("ascii")) + + server, _thread = self._run_raw_server(MalformedProtocolHandler) + + try: + result = probe_once(f"http://127.0.0.1:{server.server_address[1]}/health") + except HTTPException: + result = None + + self.assertIsNotNone(result) + assert result is not None + self.assertFalse(result.ready) + self.assertEqual("readiness endpoint returned a malformed HTTP response", result.message) + self.assertIsNone(result.payload) + self.assertNotIn(secret, repr(result)) + def test_probe_once_rejects_url_data_before_network_io(self) -> None: class TrackingHandler(_HealthHandler): requests = 0 diff --git a/zeus/readiness.py b/zeus/readiness.py index 24cb348..a635da7 100644 --- a/zeus/readiness.py +++ b/zeus/readiness.py @@ -4,11 +4,14 @@ import time from collections.abc import Mapping from dataclasses import dataclass +from http.client import HTTPException from urllib.error import HTTPError, URLError from urllib.parse import urlparse from urllib.request import ( HTTPRedirectHandler, + OpenerDirector, ProxyHandler, + Request, build_opener, ) @@ -20,7 +23,8 @@ def redirect_request(self, *args: object, **kwargs: object) -> None: return None -_READINESS_OPENER = build_opener(ProxyHandler({}), _RejectRedirects()) +def _build_readiness_opener() -> OpenerDirector: + return build_opener(ProxyHandler({}), _RejectRedirects()) @dataclass(frozen=True) @@ -84,7 +88,8 @@ def probe_once( ): return ReadinessResult(False, "readiness URL must be loopback HTTP") try: - with _READINESS_OPENER.open(url, timeout=timeout_seconds) as response: # nosec B310 + request = Request(url, headers={"Connection": "close"}) # nosec B310 + with _build_readiness_opener().open(request, timeout=timeout_seconds) as response: content_lengths = response.headers.get_all("content-length", []) try: declared_lengths = {int(value) for value in content_lengths} @@ -94,6 +99,12 @@ def probe_once( return ReadinessResult(False, "readiness response has invalid content length") if any(length > MAX_READINESS_RESPONSE_BYTES for length in declared_lengths): return ReadinessResult(False, "readiness response exceeds size limit") + # HTTPResponse.read(amt) clamps amt to its internal Content-Length. The + # connection-close request lets an honest fixed-length response reach EOF, + # while clearing this CPython parser field exposes any understated trailing + # data to the same bounded MAX + 1 read. Chunk framing remains handled by + # HTTPResponse because chunked responses already have length=None. + response.length = None body = response.read(MAX_READINESS_RESPONSE_BYTES + 1) if len(body) > MAX_READINESS_RESPONSE_BYTES: return ReadinessResult(False, "readiness response exceeds size limit") @@ -111,6 +122,8 @@ def probe_once( return ReadinessResult(False, "readiness endpoint returned an HTTP error") except (URLError, OSError): return ReadinessResult(False, "readiness probe failed") + except HTTPException: + return ReadinessResult(False, "readiness endpoint returned a malformed HTTP response") except (UnicodeDecodeError, json.JSONDecodeError, ValueError): return ReadinessResult(False, "readiness response is not valid JSON") From 47604747f9d59272b8317ec506ca7c61fbaf675b Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:08:02 +0200 Subject: [PATCH 07/49] security: add descriptor-safe private file primitives --- tests/test_api_logging.py | 130 +++------ tests/test_private_io.py | 572 ++++++++++++++++++++++++++++++++++++++ zeus/api_logging.py | 37 +-- zeus/private_io.py | 478 +++++++++++++++++++++++++++++++ 4 files changed, 1094 insertions(+), 123 deletions(-) create mode 100644 tests/test_private_io.py create mode 100644 zeus/private_io.py diff --git a/tests/test_api_logging.py b/tests/test_api_logging.py index 23e407c..29dd2ef 100644 --- a/tests/test_api_logging.py +++ b/tests/test_api_logging.py @@ -3,9 +3,7 @@ import http.client import io import json -import os import socket -import stat import tempfile import threading import time @@ -15,11 +13,12 @@ from http.server import ThreadingHTTPServer from pathlib import Path from typing import Any -from unittest.mock import call, patch +from unittest.mock import patch from zeus.api import make_handler from zeus.api_logging import ApiLogWriter from zeus.config import Settings +from zeus.private_io import UnsafeFileError from zeus.request_context import ( IDEMPOTENCY_OUTCOMES, RequestContext, @@ -56,7 +55,7 @@ def raw_http_request(port: int, request: bytes) -> tuple[int, dict[str, str], di class ApiLoggingTests(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() - self.root = Path(self.temp_dir.name) + self.root = Path(self.temp_dir.name).resolve() def tearDown(self) -> None: self.temp_dir.cleanup() @@ -250,6 +249,42 @@ def test_api_log_failure_is_fail_open(self) -> None: ApiLogWriter(disabled_path, enabled=False).access({"status": 200}) self.assertFalse(disabled_path.exists()) + def test_api_log_writer_delegates_exact_canonical_utf8_line(self) -> None: + path = self.root / "api.jsonl" + + with ( + patch("zeus.api_logging.datetime") as clock, + patch("zeus.api_logging.append_private_bytes") as append, + ): + clock.now.return_value.isoformat.return_value = "2026-07-21T12:34:56+00:00" + ApiLogWriter(path, enabled=True).access({"status": 200}) + + append.assert_called_once_with( + path, + b'{"event":"api.access","level":"info","schema_version":1,' + b'"status":200,"ts":"2026-07-21T12:34:56+00:00"}\n', + ) + + def test_api_log_writer_keeps_private_io_failures_fail_open(self) -> None: + path = self.root / "api.jsonl" + + with patch( + "zeus.api_logging.append_private_bytes", + side_effect=UnsafeFileError("unsafe path"), + ) as append: + ApiLogWriter(path, enabled=True).access({"status": 200}) + ApiLogWriter(path, enabled=True).error("a" * 32, RuntimeError("safe")) + + self.assertEqual(2, append.call_count) + + def test_disabled_api_log_writer_does_not_delegate(self) -> None: + with patch("zeus.api_logging.append_private_bytes") as append: + writer = ApiLogWriter(self.root / "api.jsonl", enabled=False) + writer.access({"status": 200}) + writer.error("a" * 32, RuntimeError("safe")) + + append.assert_not_called() + def test_api_access_payload_building_is_fail_open(self) -> None: class BrokenString: def __str__(self) -> str: @@ -272,93 +307,6 @@ def __str__(self) -> str: self.assertEqual("Exception", row["error_type"]) self.assertEqual("Unexpected API error", row["message"]) - def test_api_log_writer_creates_private_directory_and_file(self) -> None: - path = self.root / "logs" / "api.jsonl" - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(0o700, stat.S_IMODE(path.parent.stat().st_mode)) - self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) - - def test_api_log_writer_tightens_existing_directory_and_file_modes(self) -> None: - directory = self.root / "logs" - directory.mkdir(mode=0o755) - path = directory / "api.jsonl" - path.write_text("", encoding="utf-8") - directory.chmod(0o755) - path.chmod(0o644) - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(0o700, stat.S_IMODE(directory.stat().st_mode)) - self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) - self.assertEqual(200, json.loads(path.read_text(encoding="utf-8"))["status"]) - - @unittest.skipUnless(hasattr(os, "O_NOFOLLOW"), "O_NOFOLLOW unavailable") - def test_api_log_writer_does_not_follow_symlinked_log_directory(self) -> None: - outside = self.root / "outside" - outside.mkdir(mode=0o755) - outside.chmod(0o755) - outside_mode = stat.S_IMODE(outside.stat().st_mode) - logs = self.root / "state" / "logs" - logs.parent.mkdir() - logs.symlink_to(outside, target_is_directory=True) - - ApiLogWriter(logs / "api.jsonl", enabled=True).access({"status": 200}) - - self.assertFalse((outside / "api.jsonl").exists()) - self.assertEqual(outside_mode, stat.S_IMODE(outside.stat().st_mode)) - - def test_api_log_writer_closes_descriptor_when_fchmod_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.fchmod", side_effect=OSError("denied")), - patch("zeus.api_logging.os.close", wraps=os.close) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - close.assert_called_once() - self.assertFalse(path.exists()) - - def test_api_log_writer_closes_both_descriptors_when_file_fchmod_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.fchmod", side_effect=[None, OSError("denied")]), - patch("zeus.api_logging.os.close", wraps=os.close) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual(2, close.call_count) - self.assertEqual("", path.read_text(encoding="utf-8")) - - def test_api_log_writer_attempts_directory_close_when_file_close_fails(self) -> None: - path = self.root / "api.jsonl" - - with ( - patch("zeus.api_logging.os.open", side_effect=[101, 202]), - patch("zeus.api_logging.os.fchmod", side_effect=[None, OSError("denied")]), - patch( - "zeus.api_logging.os.close", - side_effect=[OSError("file close failed"), None], - ) as close, - ): - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual([call(202), call(101)], close.call_args_list) - - @unittest.skipUnless(hasattr(os, "O_NOFOLLOW"), "O_NOFOLLOW unavailable") - def test_api_log_writer_does_not_follow_symlinks(self) -> None: - target = self.root / "target.jsonl" - target.write_text("original\n", encoding="utf-8") - path = self.root / "api.jsonl" - path.symlink_to(target) - - ApiLogWriter(path, enabled=True).access({"status": 200}) - - self.assertEqual("original\n", target.read_text(encoding="utf-8")) - def test_api_error_omits_exception_messages_and_tracebacks(self) -> None: path = self.root / "api.jsonl" sensitive_message = ( diff --git a/tests/test_private_io.py b/tests/test_private_io.py new file mode 100644 index 0000000..44920af --- /dev/null +++ b/tests/test_private_io.py @@ -0,0 +1,572 @@ +from __future__ import annotations + +import io +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import BinaryIO +from unittest.mock import patch + +from zeus import private_io +from zeus.private_io import ( + UnsafeFileError, + append_private_bytes, + open_private_append, + read_private_tail, + validate_private_directory, +) + + +class PrivateIOTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name).resolve() + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def assert_all_closed(self, descriptors: set[int]) -> None: + for descriptor in descriptors: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_append_and_tail_preserve_exact_bytes(self) -> None: + path = self.root / "logs" / "events.bin" + first = b"first\x00line\n" + second = b"\xffsecond\n" + + append_private_bytes(path, first) + append_private_bytes(path, second) + + expected = first + second + self.assertEqual(expected, path.read_bytes()) + self.assertEqual(expected, read_private_tail(path, len(expected) + 100)) + self.assertEqual(expected[-7:], read_private_tail(path, 7)) + self.assertEqual(b"", read_private_tail(path, 0)) + + def test_open_private_append_is_unbuffered_and_always_appends(self) -> None: + path = self.root / "logs" / "events.bin" + append_private_bytes(path, b"first") + + with open_private_append(path) as handle: + self.assertIsInstance(handle, io.RawIOBase) + self.assertNotIsInstance(handle, io.BufferedIOBase) + handle.seek(0) + self.assertEqual(6, handle.write(b"second")) + + self.assertEqual(b"firstsecond", path.read_bytes()) + + def test_read_missing_leaf_returns_empty_without_creating_it(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "missing.log" + + self.assertEqual(b"", read_private_tail(path, 128)) + + self.assertFalse(path.exists()) + + def test_tail_size_validation_happens_before_filesystem_mutation(self) -> None: + path = self.root / "missing" / "events.log" + + for value in (True, -1, 1.5, "1", None): + with self.subTest(value=value), self.assertRaises(TypeError): + read_private_tail(path, value) # type: ignore[arg-type] + + self.assertFalse(path.parent.exists()) + + def test_unsupported_path_shapes_fail_before_mutation(self) -> None: + traversal_parent = self.root / "new" / ".." / "escape" + + for path in (Path("relative.log"), traversal_parent / "events.log", Path("/")): + with self.subTest(path=path), self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + + self.assertFalse((self.root / "new").exists()) + + def test_missing_security_flag_fails_before_creation(self) -> None: + path = self.root / "missing" / "events.log" + + with ( + patch.object(private_io.os, "O_NOFOLLOW", 0), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertFalse(path.parent.exists()) + + def test_missing_descriptor_relative_primitive_fails_before_creation(self) -> None: + path = self.root / "missing" / "events.log" + + with ( + patch.object(private_io.os, "supports_dir_fd", frozenset()), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertFalse(path.parent.exists()) + + def test_validate_private_directory_requires_existing_directory(self) -> None: + path = self.root / "missing" + + with self.assertRaises(UnsafeFileError): + validate_private_directory(path) + + self.assertFalse(path.exists()) + + def test_new_and_final_private_directories_are_tightened_without_changing_ancestors( + self, + ) -> None: + ancestor = self.root / "ancestor" + ancestor.mkdir(mode=0o755) + ancestor.chmod(0o755) + private_dir = ancestor / "private" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + + validate_private_directory(private_dir) + append_private_bytes(private_dir / "nested" / "events.log", b"event") + + self.assertEqual(0o755, stat.S_IMODE(ancestor.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE((private_dir / "nested").stat().st_mode)) + + def test_directory_link_count_is_not_restricted_to_one(self) -> None: + private_dir = self.root / "private" + (private_dir / "child").mkdir(parents=True) + self.assertGreater(private_dir.stat().st_nlink, 1) + + validate_private_directory(private_dir) + + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_existing_file_and_parent_modes_are_tightened_before_append(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o755) + path = private_dir / "events.log" + path.write_bytes(b"existing") + private_dir.chmod(0o755) + path.chmod(0o644) + + append_private_bytes(path, b"event") + + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) + self.assertEqual(b"existingevent", path.read_bytes()) + + def test_read_tightens_existing_file_mode(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"event") + path.chmod(0o644) + + self.assertEqual(b"event", read_private_tail(path, 10)) + + self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) + + def test_leaf_symlink_is_rejected_without_touching_target(self) -> None: + target = self.root / "target.log" + target.write_bytes(b"original") + path = self.root / "events.log" + path.symlink_to(target) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + self.assertEqual(b"original", target.read_bytes()) + + def test_intermediate_directory_symlink_is_rejected(self) -> None: + outside = self.root / "outside" + outside.mkdir() + linked = self.root / "linked" + linked.symlink_to(outside, target_is_directory=True) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(linked / "logs" / "events.log", b"event") + + self.assertFalse((outside / "logs").exists()) + + def test_final_parent_symlink_is_rejected_without_tightening_target(self) -> None: + outside = self.root / "outside" + outside.mkdir(mode=0o755) + outside.chmod(0o755) + state = self.root / "state" + state.mkdir() + linked = state / "logs" + linked.symlink_to(outside, target_is_directory=True) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(linked / "events.log", b"event") + + self.assertFalse((outside / "events.log").exists()) + self.assertEqual(0o755, stat.S_IMODE(outside.stat().st_mode)) + + def test_hardlink_is_rejected_before_chmod_or_io(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + target = private_dir / "target.log" + target.write_bytes(b"original") + target.chmod(0o644) + path = private_dir / "events.log" + os.link(target, path) + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + self.assertEqual(b"original", target.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(target.stat().st_mode)) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_fifo_is_rejected_in_bounded_subprocesses(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.fifo" + os.mkfifo(path, mode=0o600) + expressions = ( + "append_private_bytes(path, b'event')", + "read_private_tail(path, 1)", + ) + + for expression in expressions: + script = f""" +import sys +from pathlib import Path +from zeus.private_io import UnsafeFileError, append_private_bytes, read_private_tail +path = Path(sys.argv[1]) +try: + {expression} +except UnsafeFileError: + raise SystemExit(0) +raise SystemExit(1) +""" + with self.subTest(expression=expression): + result = subprocess.run( + [sys.executable, "-c", script, str(path)], + check=False, + capture_output=True, + timeout=2, + ) + self.assertEqual(0, result.returncode, result.stderr.decode("utf-8", "replace")) + + def test_directory_leaf_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.mkdir() + + with self.assertRaises(UnsafeFileError): + append_private_bytes(path, b"event") + with self.assertRaises(UnsafeFileError): + read_private_tail(path, 10) + + def test_final_directory_owner_mismatch_is_rejected_before_chmod(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + + with ( + patch.object(private_io.os, "geteuid", return_value=os.geteuid() + 1), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(0o755, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_file_owner_mismatch_is_rejected_before_chmod_or_io(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + path.chmod(0o644) + real_fstat = os.fstat + + def mismatched_file_owner(fd: int) -> os.stat_result: + result = real_fstat(fd) + if stat.S_ISREG(result.st_mode): + fields = list(result) + fields[4] = result.st_uid + 1 + return os.stat_result(fields) + return result + + with ( + patch.object(private_io.os, "fstat", side_effect=mismatched_file_owner), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(b"original", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + + def test_file_replacement_between_lstat_and_open_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + replacement = private_dir / "replacement.log" + replacement.write_bytes(b"replacement") + displaced = private_dir / "displaced.log" + real_open = os.open + swapped = False + + def racing_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if not swapped and name == path.name and dir_fd is not None: + path.rename(displaced) + replacement.rename(path) + swapped = True + return real_open(name, flags, mode, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=racing_open), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertTrue(swapped) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"replacement", path.read_bytes()) + + def test_file_replacement_before_post_lstat_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + replacement = private_dir / "replacement.log" + replacement.write_bytes(b"replacement") + displaced = private_dir / "displaced.log" + real_lstat = os.lstat + leaf_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal leaf_lstats + if name == path.name and dir_fd is not None: + leaf_lstats += 1 + if leaf_lstats == 2: + path.rename(displaced) + replacement.rename(path) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(2, leaf_lstats) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"replacement", path.read_bytes()) + + def test_directory_replacement_between_lstat_and_open_is_rejected(self) -> None: + private_dir = self.root / "private" + logs = private_dir / "logs" + logs.mkdir(parents=True) + replacement = private_dir / "replacement" + replacement.mkdir() + displaced = private_dir / "displaced" + real_open = os.open + swapped = False + + def racing_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if not swapped and name == logs.name and dir_fd is not None: + logs.rename(displaced) + replacement.rename(logs) + swapped = True + return real_open(name, flags, mode, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=racing_open), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(logs / "events.log", b"event") + + self.assertTrue(swapped) + self.assertFalse((logs / "events.log").exists()) + + def test_fstat_identity_mismatch_is_rejected(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + decoy = private_dir / "decoy.log" + decoy.write_bytes(b"decoy") + decoy_stat = decoy.stat() + real_fstat = os.fstat + + def mismatched_file_identity(fd: int) -> os.stat_result: + result = real_fstat(fd) + if stat.S_ISREG(result.st_mode): + return decoy_stat + return result + + with ( + patch.object(private_io.os, "fstat", side_effect=mismatched_file_identity), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(b"original", path.read_bytes()) + + def test_all_descriptors_close_when_file_fchmod_fails(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + real_fchmod = os.fchmod + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def failing_file_fchmod(fd: int, mode: int) -> None: + if stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("file fchmod failed") + real_fchmod(fd, mode) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fchmod", side_effect=failing_file_fchmod), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + def test_all_descriptors_close_when_fdopen_fails(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=OSError("fdopen failed")), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + def test_file_close_failure_does_not_prevent_parent_cleanup(self) -> None: + path = self.root / "logs" / "events.log" + real_fdopen = os.fdopen + parent_fd: int | None = None + real_open = os.open + + class FailingCloseHandle: + def __init__(self, handle: BinaryIO) -> None: + self.handle = handle + + def write(self, data: bytes) -> int: + return self.handle.write(data) + + def close(self) -> None: + self.handle.close() + raise OSError("file close failed") + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal parent_fd + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + if name == path.name: + parent_fd = dir_fd + return descriptor + + def failing_close_fdopen(fd: int, mode: str, buffering: int = -1) -> FailingCloseHandle: + return FailingCloseHandle(real_fdopen(fd, mode, buffering=buffering)) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=failing_close_fdopen), + patch.object(private_io.os, "close", wraps=os.close) as close, + self.assertRaisesRegex(UnsafeFileError, "close"), + ): + append_private_bytes(path, b"event") + + self.assertIsNotNone(parent_fd) + close.assert_any_call(parent_fd) + with self.assertRaises(OSError): + os.fstat(parent_fd) # type: ignore[arg-type] + + def test_cleanup_errors_do_not_hide_primary_unsafe_error(self) -> None: + path = self.root / "logs" / "events.log" + opened: set[int] = set() + real_open = os.open + real_close = os.close + fdopen_failed = False + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def failing_fdopen(*_args: object, **_kwargs: object) -> BinaryIO: + nonlocal fdopen_failed + fdopen_failed = True + raise UnsafeFileError("primary unsafe error") + + def noisy_close(fd: int) -> None: + real_close(fd) + if fdopen_failed: + raise OSError("cleanup noise") + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "fdopen", side_effect=failing_fdopen), + patch.object(private_io.os, "close", side_effect=noisy_close), + self.assertRaisesRegex(UnsafeFileError, "primary unsafe error"), + ): + append_private_bytes(path, b"event") + + self.assert_all_closed(opened) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/api_logging.py b/zeus/api_logging.py index 7fbf831..41fa9e7 100644 --- a/zeus/api_logging.py +++ b/zeus/api_logging.py @@ -2,13 +2,13 @@ import json import math -import os import re import threading from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path +from zeus.private_io import append_private_bytes from zeus.request_context import AUTH_OUTCOMES, IDEMPOTENCY_OUTCOMES, route_template _HTTP_METHODS = frozenset({"GET", "POST", "UNSUPPORTED"}) @@ -140,37 +140,10 @@ def _write(self, payload: Mapping[str, object]) -> None: if not self.enabled: return try: - line = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + line = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) with self._lock: - self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - directory_flags = os.O_RDONLY - directory_flags |= getattr(os, "O_DIRECTORY", 0) - directory_flags |= getattr(os, "O_CLOEXEC", 0) - directory_flags |= getattr(os, "O_NOFOLLOW", 0) - file_flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT - file_flags |= getattr(os, "O_CLOEXEC", 0) - file_flags |= getattr(os, "O_NOFOLLOW", 0) - directory_fd: int | None = None - file_fd: int | None = None - try: - directory_fd = os.open(self.path.parent, directory_flags) - os.fchmod(directory_fd, 0o700) - file_fd = os.open( - self.path.name, - file_flags, - 0o600, - dir_fd=directory_fd, - ) - os.fchmod(file_fd, 0o600) - with os.fdopen(file_fd, "a", encoding="utf-8") as handle: - file_fd = None - handle.write(line) - finally: - try: - if file_fd is not None: - os.close(file_fd) - finally: - if directory_fd is not None: - os.close(directory_fd) + append_private_bytes(self.path, line) except (OSError, TypeError, ValueError): return diff --git a/zeus/private_io.py b/zeus/private_io.py new file mode 100644 index 0000000..930ef7f --- /dev/null +++ b/zeus/private_io.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +import os +import stat +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, suppress +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, cast + +_DIRECTORY_MODE = 0o700 +_FILE_MODE = 0o600 +_SECURITY_FLAGS = ("O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC", "O_NONBLOCK") +_REQUIRED_FUNCTIONS = ( + "close", + "fchmod", + "fdopen", + "fstat", + "geteuid", + "lseek", + "lstat", + "mkdir", + "open", + "read", + "write", +) +_OPEN_DIR_FD_PROBE = os.open +_MKDIR_DIR_FD_PROBE = os.mkdir +_LSTAT_DIR_FD_PROBE = os.lstat + + +class UnsafeFileError(OSError): + pass + + +@dataclass(frozen=True) +class _Platform: + euid: int + directory_flags: int + append_flags: int + read_flags: int + create_exclusive_flags: int + + +def _required_flag(name: str, *, allow_zero: bool = False) -> int: + value = getattr(os, name, None) + if type(value) is not int or (not allow_zero and value == 0): + raise UnsafeFileError(f"required POSIX flag {name} is unavailable") + return value + + +def _require_platform() -> _Platform: + if os.name != "posix": + raise UnsafeFileError("descriptor-safe private file access requires POSIX") + for name in _REQUIRED_FUNCTIONS: + if not callable(getattr(os, name, None)): + raise UnsafeFileError(f"required POSIX primitive os.{name} is unavailable") + supported = getattr(os, "supports_dir_fd", ()) + for function, name in ( + (_OPEN_DIR_FD_PROBE, "open"), + (_MKDIR_DIR_FD_PROBE, "mkdir"), + (_LSTAT_DIR_FD_PROBE, "lstat"), + ): + if function not in supported: + raise UnsafeFileError(f"descriptor-relative os.{name} is unavailable") + security_flags = {name: _required_flag(name) for name in _SECURITY_FLAGS} + o_rdonly = _required_flag("O_RDONLY", allow_zero=True) + o_wronly = _required_flag("O_WRONLY") + o_append = _required_flag("O_APPEND") + o_creat = _required_flag("O_CREAT") + o_excl = _required_flag("O_EXCL") + euid = os.geteuid() + if isinstance(euid, bool) or not isinstance(euid, int) or euid < 0: + raise UnsafeFileError("effective UID is unavailable") + common_leaf = ( + security_flags["O_NOFOLLOW"] | security_flags["O_CLOEXEC"] | security_flags["O_NONBLOCK"] + ) + return _Platform( + euid=euid, + directory_flags=( + o_rdonly + | security_flags["O_DIRECTORY"] + | security_flags["O_NOFOLLOW"] + | security_flags["O_CLOEXEC"] + ), + append_flags=o_wronly | o_append | common_leaf, + read_flags=o_rdonly | common_leaf, + create_exclusive_flags=o_creat | o_excl, + ) + + +def _validate_path(path: Path, *, file_path: bool) -> tuple[str, ...]: + if not isinstance(path, Path): + raise UnsafeFileError("private path must be a pathlib.Path") + if not path.is_absolute() or path.anchor != "/": + raise UnsafeFileError("private path must be an absolute POSIX path") + parts = path.parts[1:] + if not parts or (file_path and len(parts) < 2): + raise UnsafeFileError("private path cannot use the filesystem root") + if any(part in {"", ".", ".."} or "\0" in part for part in parts): + raise UnsafeFileError("private path contains an unsupported component") + return parts + + +def _same_file(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _same_files(results: tuple[os.stat_result, ...]) -> bool: + return all(_same_file(results[0], result) for result in results[1:]) + + +def _close_suppressing_error(fd: int) -> None: + with suppress(OSError): + os.close(fd) + + +def _close_descriptor(fd: int, description: str) -> None: + try: + os.close(fd) + except OSError as exc: + raise UnsafeFileError(f"{description} descriptor could not be closed") from exc + + +def _lstat_at(parent_fd: int, name: str, description: str) -> os.stat_result: + try: + return os.lstat(name, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be inspected safely") from exc + + +def _validate_directory_snapshots( + snapshots: tuple[os.stat_result, ...], + description: str, +) -> None: + if not all(stat.S_ISDIR(snapshot.st_mode) for snapshot in snapshots): + raise UnsafeFileError(f"{description} is not a directory") + if not _same_files(snapshots): + raise UnsafeFileError(f"{description} changed while it was opened") + + +def _tighten_directory( + parent_fd: int, + name: str, + directory_fd: int, + snapshots: tuple[os.stat_result, ...], + platform: _Platform, + description: str, +) -> None: + if not all(snapshot.st_uid == platform.euid for snapshot in snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + try: + os.fchmod(directory_fd, _DIRECTORY_MODE) + tightened = os.fstat(directory_fd) + current = os.lstat(name, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} permissions could not be tightened") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be validated safely") from exc + final_snapshots = (*snapshots, tightened, current) + _validate_directory_snapshots(final_snapshots, description) + if not all(snapshot.st_uid == platform.euid for snapshot in final_snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + if stat.S_IMODE(tightened.st_mode) != _DIRECTORY_MODE: + raise UnsafeFileError(f"{description} does not have private permissions") + + +def _open_root(platform: _Platform) -> int: + try: + before = os.lstat("/") + root_fd = os.open("/", platform.directory_flags) + except OSError as exc: + raise UnsafeFileError("filesystem root cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("filesystem root cannot be inspected safely") from exc + try: + opened = os.fstat(root_fd) + after = os.lstat("/") + _validate_directory_snapshots((before, opened, after), "filesystem root") + return root_fd + except UnsafeFileError: + _close_suppressing_error(root_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(root_fd) + raise UnsafeFileError("filesystem root changed while it was opened") from exc + except BaseException: + _close_suppressing_error(root_fd) + raise + + +def _open_directory_at( + parent_fd: int, + name: str, + *, + create: bool, + private: bool, + platform: _Platform, +) -> int: + description = f"private path component {name!r}" + created = False + try: + before = os.lstat(name, dir_fd=parent_fd) + except FileNotFoundError as exc: + if not create: + raise UnsafeFileError(f"{description} is unavailable") from exc + try: + os.mkdir(name, mode=_DIRECTORY_MODE, dir_fd=parent_fd) + created = True + except FileExistsError as race: + raise UnsafeFileError(f"{description} appeared while it was created") from race + except OSError as mkdir_error: + raise UnsafeFileError(f"{description} could not be created safely") from mkdir_error + except (TypeError, ValueError) as mkdir_error: + raise UnsafeFileError(f"{description} cannot be created safely") from mkdir_error + before = _lstat_at(parent_fd, name, description) + except OSError as exc: + raise UnsafeFileError(f"{description} is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be inspected safely") from exc + + if not stat.S_ISDIR(before.st_mode): + raise UnsafeFileError(f"{description} is not a directory") + try: + directory_fd = os.open(name, platform.directory_flags, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError(f"{description} cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError(f"{description} cannot be opened safely") from exc + try: + opened = os.fstat(directory_fd) + after = os.lstat(name, dir_fd=parent_fd) + snapshots = (before, opened, after) + _validate_directory_snapshots(snapshots, description) + if created or private: + _tighten_directory( + parent_fd, + name, + directory_fd, + snapshots, + platform, + description, + ) + return directory_fd + except UnsafeFileError: + _close_suppressing_error(directory_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(directory_fd) + raise UnsafeFileError(f"{description} changed while it was opened") from exc + except BaseException: + _close_suppressing_error(directory_fd) + raise + + +@contextmanager +def _open_directory_path( + parts: tuple[str, ...], + *, + create: bool, + platform: _Platform, +) -> Iterator[int]: + current_fd = _open_root(platform) + try: + for index, component in enumerate(parts): + next_fd = _open_directory_at( + current_fd, + component, + create=create, + private=index == len(parts) - 1, + platform=platform, + ) + previous_fd = current_fd + current_fd = -1 + try: + _close_descriptor(previous_fd, "ancestor directory") + except UnsafeFileError: + _close_suppressing_error(next_fd) + raise + current_fd = next_fd + except BaseException: + if current_fd >= 0: + _close_suppressing_error(current_fd) + raise + + result_fd = current_fd + current_fd = -1 + try: + yield result_fd + except BaseException: + _close_suppressing_error(result_fd) + raise + else: + _close_descriptor(result_fd, "private directory") + + +def _validate_file_snapshot(snapshot: os.stat_result, platform: _Platform) -> None: + if not stat.S_ISREG(snapshot.st_mode): + raise UnsafeFileError("private file is not a regular file") + if snapshot.st_uid != platform.euid: + raise UnsafeFileError("private file has an unexpected owner") + if snapshot.st_nlink != 1: + raise UnsafeFileError("private file has unexpected links") + + +def _open_private_file_at( + parent_fd: int, + name: str, + *, + append: bool, + create: bool, + platform: _Platform, +) -> int | None: + before: os.stat_result | None + try: + before = os.lstat(name, dir_fd=parent_fd) + except FileNotFoundError: + before = None + except OSError as exc: + raise UnsafeFileError("private file is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file cannot be inspected safely") from exc + if before is None and not create: + return None + if before is not None: + _validate_file_snapshot(before, platform) + + flags = platform.append_flags if append else platform.read_flags + if before is None: + flags |= platform.create_exclusive_flags + try: + file_fd = os.open(name, flags, _FILE_MODE, dir_fd=parent_fd) + except OSError as exc: + raise UnsafeFileError("private file cannot be opened safely") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file cannot be opened safely") from exc + try: + opened = os.fstat(file_fd) + current = os.lstat(name, dir_fd=parent_fd) + initial_snapshots = (opened, current) if before is None else (before, opened, current) + for snapshot in initial_snapshots: + _validate_file_snapshot(snapshot, platform) + if not _same_files(initial_snapshots): + raise UnsafeFileError("private file changed while it was opened") + + os.fchmod(file_fd, _FILE_MODE) + tightened = os.fstat(file_fd) + final = os.lstat(name, dir_fd=parent_fd) + final_snapshots = (*initial_snapshots, tightened, final) + for snapshot in final_snapshots: + _validate_file_snapshot(snapshot, platform) + if not _same_files(final_snapshots): + raise UnsafeFileError("private file changed while it was opened") + if stat.S_IMODE(tightened.st_mode) != _FILE_MODE: + raise UnsafeFileError("private file does not have private permissions") + return file_fd + except UnsafeFileError: + _close_suppressing_error(file_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(file_fd) + raise UnsafeFileError("private file could not be validated safely") from exc + except BaseException: + _close_suppressing_error(file_fd) + raise + + +@contextmanager +def _private_append_context( + parent_parts: tuple[str, ...], + name: str, + platform: _Platform, +) -> Iterator[BinaryIO]: + with _open_directory_path(parent_parts, create=True, platform=platform) as parent_fd: + file_fd = _open_private_file_at( + parent_fd, + name, + append=True, + create=True, + platform=platform, + ) + if file_fd is None: + raise UnsafeFileError("private file was not created") + try: + try: + handle = cast(BinaryIO, os.fdopen(file_fd, "ab", buffering=0)) + except UnsafeFileError: + _close_suppressing_error(file_fd) + raise + except (OSError, TypeError, ValueError) as exc: + _close_suppressing_error(file_fd) + raise UnsafeFileError("private file descriptor could not be wrapped") from exc + file_fd = -1 + try: + yield handle + except BaseException: + with suppress(OSError): + handle.close() + raise + else: + try: + handle.close() + except OSError as exc: + raise UnsafeFileError("private file descriptor could not be closed") from exc + finally: + if file_fd >= 0: + _close_suppressing_error(file_fd) + + +def append_private_bytes(path: Path, data: bytes) -> None: + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + with open_private_append(path) as handle: + offset = 0 + while offset < len(data): + try: + written = handle.write(data[offset:]) + except OSError as exc: + raise UnsafeFileError("private file write failed") from exc + if written is None or written <= 0 or written > len(data) - offset: + raise UnsafeFileError("private file write was incomplete") + offset += written + + +def open_private_append(path: Path) -> AbstractContextManager[BinaryIO]: + parts = _validate_path(path, file_path=True) + platform = _require_platform() + return _private_append_context(parts[:-1], parts[-1], platform) + + +def read_private_tail(path: Path, max_bytes: int) -> bytes: + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int): + raise TypeError("max_bytes must be an integer") + if max_bytes < 0: + raise TypeError("max_bytes must be non-negative") + parts = _validate_path(path, file_path=True) + platform = _require_platform() + with _open_directory_path(parts[:-1], create=False, platform=platform) as parent_fd: + file_fd = _open_private_file_at( + parent_fd, + parts[-1], + append=False, + create=False, + platform=platform, + ) + if file_fd is None: + return b"" + try: + try: + end = os.lseek(file_fd, 0, os.SEEK_END) + start = max(0, end - max_bytes) + os.lseek(file_fd, start, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = min(max_bytes, end - start) + while remaining: + chunk = os.read(file_fd, min(65536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + result = b"".join(chunks) + except OSError as exc: + raise UnsafeFileError("private file tail could not be read") from exc + except BaseException: + _close_suppressing_error(file_fd) + raise + else: + _close_descriptor(file_fd, "private file") + return result + + +def validate_private_directory(path: Path) -> None: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + with _open_directory_path(parts, create=False, platform=platform): + pass From 1cc07b5152faaac4cead761be2db634204971a83 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:18:06 +0200 Subject: [PATCH 08/49] security: reject post-chmod mode drift --- tests/test_private_io.py | 51 ++++++++++++++++++++++++++++++++++++++++ zeus/private_io.py | 4 ++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/test_private_io.py b/tests/test_private_io.py index 44920af..033a18c 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -143,6 +143,30 @@ def test_directory_link_count_is_not_restricted_to_one(self) -> None: self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + def test_validate_directory_rejects_mode_drift_before_final_lstat(self) -> None: + private_dir = self.root / "mode-drift-dir" + private_dir.mkdir(mode=0o755) + private_dir.chmod(0o755) + real_lstat = os.lstat + directory_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal directory_lstats + if name == private_dir.name and dir_fd is not None: + directory_lstats += 1 + if directory_lstats == 3: + private_dir.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(3, directory_lstats) + self.assertEqual(0o777, stat.S_IMODE(private_dir.stat().st_mode)) + def test_existing_file_and_parent_modes_are_tightened_before_append(self) -> None: private_dir = self.root / "logs" private_dir.mkdir(mode=0o755) @@ -168,6 +192,33 @@ def test_read_tightens_existing_file_mode(self) -> None: self.assertEqual(0o600, stat.S_IMODE(path.stat().st_mode)) + def test_append_rejects_mode_drift_before_final_lstat(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "mode-drift.log" + path.write_bytes(b"original") + path.chmod(0o644) + real_lstat = os.lstat + file_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal file_lstats + if name == path.name and dir_fd is not None: + file_lstats += 1 + if file_lstats == 3: + path.chmod(0o644) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertEqual(3, file_lstats) + self.assertEqual(b"original", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + def test_leaf_symlink_is_rejected_without_touching_target(self) -> None: target = self.root / "target.log" target.write_bytes(b"original") diff --git a/zeus/private_io.py b/zeus/private_io.py index 930ef7f..fbe0c7d 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -163,7 +163,7 @@ def _tighten_directory( _validate_directory_snapshots(final_snapshots, description) if not all(snapshot.st_uid == platform.euid for snapshot in final_snapshots): raise UnsafeFileError(f"{description} has an unexpected owner") - if stat.S_IMODE(tightened.st_mode) != _DIRECTORY_MODE: + if any(stat.S_IMODE(snapshot.st_mode) != _DIRECTORY_MODE for snapshot in (tightened, current)): raise UnsafeFileError(f"{description} does not have private permissions") @@ -353,7 +353,7 @@ def _open_private_file_at( _validate_file_snapshot(snapshot, platform) if not _same_files(final_snapshots): raise UnsafeFileError("private file changed while it was opened") - if stat.S_IMODE(tightened.st_mode) != _FILE_MODE: + if any(stat.S_IMODE(snapshot.st_mode) != _FILE_MODE for snapshot in (tightened, final)): raise UnsafeFileError("private file does not have private permissions") return file_fd except UnsafeFileError: From bc01b9a563cc6b5d48c3cdf71b269bbe23241d90 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:38:49 +0200 Subject: [PATCH 09/49] security: enforce regular private audit and gateway logs --- tests/test_api.py | 38 +++++ tests/test_private_io.py | 7 + tests/test_renderer_state.py | 97 ++++++++++++- tests/test_supervisor_cli_api.py | 230 +++++++++++++++++++++++++++++++ zeus/doctor.py | 64 +++++++-- zeus/logging_utils.py | 11 +- zeus/private_io.py | 89 +++++++----- zeus/renderer.py | 28 +++- zeus/state.py | 7 +- zeus/supervisor.py | 27 +--- 10 files changed, 521 insertions(+), 77 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 30e1fa0..c8aa253 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1767,6 +1767,44 @@ def test_bot_inspect_returns_diagnostics_and_redacts_logs(self) -> None: self.assertEqual(404, status) self.assertEqual("unknown_bot", body["error"]["code"]) + def test_bot_logs_and_inspect_hide_unsafe_gateway_log_path(self) -> None: + with api_server({"ZEUS_API_KEY": "secret"}) as port: + status, created = request_json( + port, + "POST", + "/bots", + body=json_request_body({"bot_id": "coder", "template_id": "coding-bot"}), + headers=auth_json_headers(), + ) + self.assertEqual(200, status) + profile_path = Path(created["profile_path"]) + logs_path = profile_path / "logs" + logs_path.rmdir() + external_logs = profile_path.parent / "external-api-logs" + external_logs.mkdir(mode=0o755) + target_log = external_logs / "zeus-gateway.log" + target_log.write_text("API-TARGET-SENTINEL\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + logs_path.symlink_to(external_logs, target_is_directory=True) + + for endpoint in ("logs", "inspect"): + with self.subTest(endpoint=endpoint): + status, body = request_json( + port, + "GET", + f"/bots/coder/{endpoint}", + headers={"x-zeus-api-key": "secret"}, + ) + self.assertEqual(500, status) + self.assertEqual("internal_error", body["error"]["code"]) + self.assertEqual("internal server error", body["error"]["message"]) + serialized = json.dumps(body) + self.assertNotIn("API-TARGET-SENTINEL", serialized) + self.assertNotIn(str(external_logs), serialized) + + self.assertEqual("API-TARGET-SENTINEL\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + def test_rejects_malformed_json(self) -> None: with api_server({"ZEUS_API_KEY": "secret"}) as port: status, body = request_json( diff --git a/tests/test_private_io.py b/tests/test_private_io.py index 033a18c..ce14259 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -69,6 +69,13 @@ def test_read_missing_leaf_returns_empty_without_creating_it(self) -> None: self.assertFalse(path.exists()) + def test_read_missing_ancestor_returns_empty_without_creating_it(self) -> None: + path = self.root / "missing" / "nested" / "events.log" + + self.assertEqual(b"", read_private_tail(path, 128)) + + self.assertFalse(self.root.joinpath("missing").exists()) + def test_tail_size_validation_happens_before_filesystem_mutation(self) -> None: path = self.root / "missing" / "events.log" diff --git a/tests/test_renderer_state.py b/tests/test_renderer_state.py index fe5937c..05df2b4 100644 --- a/tests/test_renderer_state.py +++ b/tests/test_renderer_state.py @@ -716,12 +716,16 @@ def test_renderer_atomically_replaces_profile_and_preserves_logs(self) -> None: request = BotCreateRequest(bot_id="coder", template_id="coding-bot") renderer.render(request, template) profile = hermes_root / "profiles" / "coder" - (profile / "logs" / "gateway.log").write_text("existing log\n", encoding="utf-8") + logs = profile / "logs" + (logs / "gateway.log").write_text("existing log\n", encoding="utf-8") + logs.chmod(0o755) renderer.render(request, replace(template, soul="replacement soul")) self.assertEqual("replacement soul\n", (profile / "SOUL.md").read_text()) - self.assertEqual("existing log\n", (profile / "logs" / "gateway.log").read_text()) + self.assertEqual("existing log\n", (logs / "gateway.log").read_text()) + self.assertEqual(os.geteuid(), logs.stat().st_uid) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> None: @@ -764,7 +768,7 @@ def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> (profile / "runtime" / "nested" / "state.json").read_text(), ) - def test_renderer_replacement_preserves_symlinks_without_following_managed_links(self) -> None: + def test_renderer_replacement_handles_managed_links_without_following_targets(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) hermes_root = root / ".zeus" / "hermes" @@ -785,7 +789,9 @@ def test_renderer_replacement_preserves_symlinks_without_following_managed_links (external_skills / "SKILL.md").write_text("linked skill\n", encoding="utf-8") external_logs = root / "external-logs" external_logs.mkdir() + external_logs.chmod(0o755) (external_logs / "gateway.log").write_text("linked log\n", encoding="utf-8") + external_logs_mode = external_logs.stat().st_mode & 0o777 try: (profile / "operator-link").symlink_to(external) @@ -816,8 +822,89 @@ def test_renderer_replacement_preserves_symlinks_without_following_managed_links ) self.assertTrue((profile / "skills").is_symlink()) self.assertEqual("linked skill\n", (profile / "skills" / "SKILL.md").read_text()) - self.assertTrue((profile / "logs").is_symlink()) - self.assertEqual("linked log\n", (profile / "logs" / "gateway.log").read_text()) + self.assertFalse((profile / "logs").is_symlink()) + self.assertTrue((profile / "logs").is_dir()) + self.assertEqual(0o700, (profile / "logs").stat().st_mode & 0o777) + self.assertEqual([], list((profile / "logs").iterdir())) + self.assertEqual("linked log\n", (external_logs / "gateway.log").read_text()) + self.assertEqual(external_logs_mode, external_logs.stat().st_mode & 0o777) + + def test_renderer_replacement_replaces_dangling_logs_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + missing_target = root / "missing-external-logs" + logs.symlink_to(missing_target, target_is_directory=True) + + renderer.render(request, replace(template, soul="replacement soul")) + + self.assertFalse(logs.is_symlink()) + self.assertTrue(logs.is_dir()) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) + self.assertFalse(missing_target.exists()) + + def test_renderer_replacement_rejects_regular_logs_file_without_deleting_it(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + logs.write_text("operator-owned logs entry\n", encoding="utf-8") + original_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + + with self.assertRaisesRegex(TemplateError, "profile path must be a directory: logs"): + renderer.render(request, replace(template, soul="must not be installed")) + + self.assertTrue(logs.is_file()) + self.assertEqual("operator-owned logs entry\n", logs.read_text(encoding="utf-8")) + self.assertEqual(original_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + + def test_renderer_transaction_rollback_restores_logs_symlink_exactly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + logs = profile / "logs" + logs.rmdir() + external_logs = root / "external-rollback-logs" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "gateway.log" + sentinel.write_text("rollback target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + logs.symlink_to(external_logs, target_is_directory=True) + original_link = os.readlink(logs) + + with ( + self.assertRaisesRegex(RuntimeError, "injected database failure"), + renderer.transaction(request, replace(template, soul="replacement soul")), + ): + self.assertFalse(logs.is_symlink()) + self.assertTrue(logs.is_dir()) + self.assertEqual(0o700, logs.stat().st_mode & 0o777) + self.assertEqual("rollback target\n", sentinel.read_text(encoding="utf-8")) + raise RuntimeError("injected database failure") + + self.assertTrue(logs.is_symlink()) + self.assertEqual(original_link, os.readlink(logs)) + self.assertEqual("rollback target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) def test_renderer_transaction_commits_replacement_and_cleans_backup(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index d29c478..4432428 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -29,6 +29,7 @@ from zeus.lifecycle import LifecycleEventInput from zeus.logging_utils import redact_secrets from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState, RestartPolicy +from zeus.private_io import UnsafeFileError from zeus.process_lock import LockTimeoutError from zeus.readiness import ReadinessResult from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunSummary @@ -3015,6 +3016,149 @@ def test_audit_write_failure_does_not_break_lifecycle_action(self) -> None: self.assertEqual(2, len(events)) self.assertEqual(["running", "stopped"], [event.status_after for event in events]) + def test_audit_append_delegates_exact_utf8_json_line(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + store = StateStore(root / "state" / "zeus.db") + fixed_now = datetime(2026, 7, 21, 12, 34, 56, tzinfo=UTC) + + with ( + patch("zeus.state.datetime") as clock, + patch("zeus.state.append_private_bytes") as append, + ): + clock.now.return_value = fixed_now + store.append_audit_event( + "bot.test", + label="Δ", + api_key="secret", + ) + + append.assert_called_once_with( + root / "state" / "logs" / "audit.jsonl", + b'{"api_key": "[redacted]", "event": "bot.test", "label": "\\u0394", ' + b'"ts": "2026-07-21T12:34:56+00:00"}\n', + ) + + def test_audit_parent_symlink_is_fail_open_without_mutating_target(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + store = StateStore(state_dir / "zeus.db") + store.init() + external_logs = root / "external-audit-target" + external_logs.mkdir(mode=0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external audit target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + (state_dir / "logs").symlink_to(external_logs, target_is_directory=True) + + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + hermes_bin = self._fake_hermes_path(root) + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor( + store, + hermes_bin, + root / ".zeus" / "hermes", + popen_factory=FakePopen, + pid_alive_fn=lambda pid: bool(FakePopen.launch_count), + cmdline_reader=lambda pid: self._gateway_argv(hermes_bin), + proc_start_fingerprint_reader=lambda pid: f"test-process-start:{pid}", + ) + + status = supervisor.start("coder") + + self.assertEqual(BotStatus.running, status.status) + self.assertEqual("external audit target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_logs.iterdir()) + ) + + def test_unsafe_gateway_log_prevents_pipe_and_process_creation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + logs_path = profile_path / "logs" + logs_path.mkdir(parents=True, mode=0o700) + target_log = root / "external-gateway-target.log" + target_log.write_text("external gateway target\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + (logs_path / "zeus-gateway.log").symlink_to(target_log) + hermes_bin = self._fake_hermes_path(root) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + + def forbidden_popen(*args: object, **kwargs: object) -> FakePopen: + self.fail("process factory must not be called for an unsafe gateway log") + + supervisor = Supervisor( + store, + hermes_bin, + root / ".zeus" / "hermes", + popen_factory=forbidden_popen, + startup_grace_seconds=0, + ) + + with patch( + "zeus.supervisor.os.pipe", + side_effect=AssertionError("pipe allocated before gateway log validation"), + ) as pipe: + status = supervisor.start("coder") + + pipe.assert_not_called() + self.assertEqual(BotStatus.failed, status.status) + self.assertIsNone(status.pid) + self.assertEqual("external gateway target\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + + def test_direct_logs_and_inspect_reject_gateway_log_parent_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-log-target" + external_logs.mkdir(mode=0o755) + target_log = external_logs / "zeus-gateway.log" + target_log.write_text("TARGET-SENTINEL\n", encoding="utf-8") + target_mode = target_log.stat().st_mode & 0o777 + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") + + for operation in (supervisor.logs, supervisor.inspect): + with self.subTest(operation=operation.__name__): + with self.assertRaises(UnsafeFileError) as raised: + operation("coder") + self.assertNotIn("TARGET-SENTINEL", str(raised.exception)) + + self.assertEqual("TARGET-SENTINEL\n", target_log.read_text(encoding="utf-8")) + self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -3308,11 +3452,97 @@ def test_doctor_rejects_world_accessible_state_directory(self) -> None: try: os.chdir(root) check = _check_runtime_paths(settings) + mode_after = state_dir.stat().st_mode & 0o777 finally: os.chdir(old_cwd) self.assertEqual("fail", check.status) self.assertIn("must not be accessible to other users", check.message) + self.assertEqual(0o755, mode_after) + + def test_doctor_rejects_real_and_broken_logs_symlinks_without_target_mutation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + state_dir.chmod(0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + external_logs = root / "external-doctor-target" + external_logs.mkdir(mode=0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("doctor target\n", encoding="utf-8") + target_mode = external_logs.stat().st_mode & 0o777 + logs_link = state_dir / "logs" + + logs_link.symlink_to(external_logs, target_is_directory=True) + real_check = _check_runtime_paths(settings) + logs_link.unlink() + logs_link.symlink_to(root / "missing-doctor-target", target_is_directory=True) + broken_check = _check_runtime_paths(settings) + + self.assertEqual("fail", real_check.status) + self.assertEqual("fail", broken_check.status) + self.assertIn("logs", real_check.message.lower()) + self.assertIn("logs", broken_check.message.lower()) + self.assertNotIn(str(external_logs), real_check.message) + self.assertEqual("doctor target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(target_mode, external_logs.stat().st_mode & 0o777) + + def test_doctor_rejects_permissive_logs_directory_without_repair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + logs_dir = state_dir / "logs" + logs_dir.mkdir(parents=True, mode=0o755) + state_dir.chmod(0o700) + logs_dir.chmod(0o755) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertIn("logs", check.message.lower()) + self.assertEqual(0o755, logs_dir.stat().st_mode & 0o777) + + def test_doctor_rejects_runtime_directory_owned_by_another_user(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + + def foreign_owner(path: Path) -> os.stat_result: + metadata = real_lstat(path) + fields = list(metadata) + fields[4] = os.geteuid() + 1 + return os.stat_result(fields) + + with patch("zeus.doctor.os.lstat", side_effect=foreign_owner): + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertIn("owned by the current user", check.message) + self.assertEqual(0o700, state_dir.stat().st_mode & 0o777) + + def test_doctor_rejects_state_symlink_without_target_mutation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + external_state = root / "external-state-target" + external_state.mkdir(mode=0o755) + sentinel = external_state / "sentinel.txt" + sentinel.write_text("state target\n", encoding="utf-8") + target_mode = external_state.stat().st_mode & 0o777 + state_dir.symlink_to(external_state, target_is_directory=True) + + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertNotIn(str(external_state), check.message) + self.assertEqual("state target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(target_mode, external_state.stat().st_mode & 0o777) def test_api_health_and_auth(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/zeus/doctor.py b/zeus/doctor.py index de2cd71..c4cb5c4 100644 --- a/zeus/doctor.py +++ b/zeus/doctor.py @@ -1,8 +1,10 @@ from __future__ import annotations import json +import os import shutil import socket +import stat import subprocess # nosec B404 import sys from dataclasses import dataclass @@ -10,6 +12,7 @@ from typing import Any from zeus.config import Settings, is_loopback_host +from zeus.private_io import nofollow_absolute_path, validate_private_directory from zeus.state import StateStore from zeus.templates import TemplateStore @@ -100,20 +103,65 @@ def _check_templates() -> DoctorCheck: ) -def _check_runtime_paths(settings: Settings) -> DoctorCheck: - state_dir = settings.state_dir.resolve() - if state_dir.exists() and not state_dir.is_dir(): - return DoctorCheck( +def _existing_runtime_directory_check( + path: Path, + *, + label: str, +) -> tuple[bool, DoctorCheck | None]: + try: + metadata = os.lstat(path) + except FileNotFoundError: + return False, None + except OSError: + return True, DoctorCheck( "runtime_paths", "fail", - f"State path {state_dir} must be a directory", + f"{label} {path} could not be inspected safely", ) - if state_dir.exists() and state_dir.stat().st_mode & 0o007: - return DoctorCheck( + if not stat.S_ISDIR(metadata.st_mode): + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} path {path} must be a directory and not a symlink", + ) + if metadata.st_uid != os.geteuid(): + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} must be owned by the current user", + ) + if stat.S_IMODE(metadata.st_mode) & 0o077: + return True, DoctorCheck( "runtime_paths", "fail", - f"State directory {state_dir} must not be accessible to other users", + f"{label} {path} must not be accessible to other users or groups", + ) + try: + validate_private_directory(path) + except OSError: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} could not be validated safely", + ) + return True, None + + +def _check_runtime_paths(settings: Settings) -> DoctorCheck: + state_dir = nofollow_absolute_path(settings.state_dir) + state_exists, failure = _existing_runtime_directory_check( + state_dir, + label="State directory", + ) + if failure is not None: + return failure + if state_exists: + _, failure = _existing_runtime_directory_check( + state_dir / "logs", + label="State logs directory", ) + if failure is not None: + return failure workspace = Path.cwd().resolve() gitignore = workspace / ".gitignore" if not gitignore.exists(): diff --git a/zeus/logging_utils.py b/zeus/logging_utils.py index 63ca60a..08d3ddf 100644 --- a/zeus/logging_utils.py +++ b/zeus/logging_utils.py @@ -3,6 +3,8 @@ import re from pathlib import Path +from zeus.private_io import read_private_tail + SECRET_KV_RE = re.compile( r"""(?ix) (?P["']?) @@ -31,10 +33,5 @@ def redact_secrets(text: str) -> str: def tail_file(path: Path, max_bytes: int = 20_000) -> str: - if not path.exists(): - return "" - with path.open("rb") as handle: - handle.seek(0, 2) - size = handle.tell() - handle.seek(max(0, size - max_bytes)) - return redact_secrets(handle.read().decode("utf-8", errors="replace")) + data = read_private_tail(path, max_bytes) + return redact_secrets(data.decode("utf-8", errors="replace")) diff --git a/zeus/private_io.py b/zeus/private_io.py index fbe0c7d..4ad7078 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -2,6 +2,7 @@ import os import stat +import sys from collections.abc import Iterator from contextlib import AbstractContextManager, contextmanager, suppress from dataclasses import dataclass @@ -33,6 +34,10 @@ class UnsafeFileError(OSError): pass +class _PrivatePathMissing(Exception): + pass + + @dataclass(frozen=True) class _Platform: euid: int @@ -89,6 +94,18 @@ def _require_platform() -> _Platform: ) +def nofollow_absolute_path(path: Path) -> Path: + """Return an absolute lexical path without resolving supplied components.""" + absolute = Path(os.path.abspath(path.expanduser())) + if ( + sys.platform == "darwin" + and len(absolute.parts) > 1 + and absolute.parts[1] in {"etc", "tmp", "var"} + ): + return Path("/private", *absolute.parts[1:]) + return absolute + + def _validate_path(path: Path, *, file_path: bool) -> tuple[str, ...]: if not isinstance(path, Path): raise UnsafeFileError("private path must be a pathlib.Path") @@ -196,6 +213,7 @@ def _open_directory_at( name: str, *, create: bool, + missing_ok: bool, private: bool, platform: _Platform, ) -> int: @@ -205,6 +223,8 @@ def _open_directory_at( before = os.lstat(name, dir_fd=parent_fd) except FileNotFoundError as exc: if not create: + if missing_ok: + raise _PrivatePathMissing from exc raise UnsafeFileError(f"{description} is unavailable") from exc try: os.mkdir(name, mode=_DIRECTORY_MODE, dir_fd=parent_fd) @@ -260,6 +280,7 @@ def _open_directory_path( parts: tuple[str, ...], *, create: bool, + missing_ok: bool = False, platform: _Platform, ) -> Iterator[int]: current_fd = _open_root(platform) @@ -269,6 +290,7 @@ def _open_directory_path( current_fd, component, create=create, + missing_ok=missing_ok, private=index == len(parts) - 1, platform=platform, ) @@ -437,38 +459,43 @@ def read_private_tail(path: Path, max_bytes: int) -> bytes: raise TypeError("max_bytes must be non-negative") parts = _validate_path(path, file_path=True) platform = _require_platform() - with _open_directory_path(parts[:-1], create=False, platform=platform) as parent_fd: - file_fd = _open_private_file_at( - parent_fd, - parts[-1], - append=False, - create=False, - platform=platform, - ) - if file_fd is None: - return b"" - try: + try: + with _open_directory_path( + parts[:-1], create=False, missing_ok=True, platform=platform + ) as parent_fd: + file_fd = _open_private_file_at( + parent_fd, + parts[-1], + append=False, + create=False, + platform=platform, + ) + if file_fd is None: + return b"" try: - end = os.lseek(file_fd, 0, os.SEEK_END) - start = max(0, end - max_bytes) - os.lseek(file_fd, start, os.SEEK_SET) - chunks: list[bytes] = [] - remaining = min(max_bytes, end - start) - while remaining: - chunk = os.read(file_fd, min(65536, remaining)) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - result = b"".join(chunks) - except OSError as exc: - raise UnsafeFileError("private file tail could not be read") from exc - except BaseException: - _close_suppressing_error(file_fd) - raise - else: - _close_descriptor(file_fd, "private file") - return result + try: + end = os.lseek(file_fd, 0, os.SEEK_END) + start = max(0, end - max_bytes) + os.lseek(file_fd, start, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = min(max_bytes, end - start) + while remaining: + chunk = os.read(file_fd, min(65536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + result = b"".join(chunks) + except OSError as exc: + raise UnsafeFileError("private file tail could not be read") from exc + except BaseException: + _close_suppressing_error(file_fd) + raise + else: + _close_descriptor(file_fd, "private file") + return result + except _PrivatePathMissing: + return b"" def validate_private_directory(path: Path) -> None: diff --git a/zeus/renderer.py b/zeus/renderer.py index 716a72e..0f1de88 100644 --- a/zeus/renderer.py +++ b/zeus/renderer.py @@ -4,6 +4,7 @@ import logging import os import shutil +import stat import tempfile from collections.abc import Iterator from contextlib import contextmanager @@ -133,8 +134,8 @@ def _write_staged_profile( ) _ensure_staged_directory(staging / "cron") - for relative_path in ("logs", "skills"): - _ensure_preserved_directory(staging / relative_path) + _ensure_private_logs_directory(staging / "logs") + _ensure_preserved_directory(staging / "skills") for relative_path, content in rendered_files.items(): _write_staged_file(staging / relative_path, content) @@ -157,6 +158,29 @@ def _ensure_preserved_directory(path: Path) -> None: raise TemplateError(f"profile path must be a directory: {path.name}") +def _ensure_private_logs_directory(path: Path) -> None: + if _path_exists(path): + metadata = os.lstat(path) + if stat.S_ISLNK(metadata.st_mode): + path.unlink() + elif not stat.S_ISDIR(metadata.st_mode): + raise TemplateError(f"profile path must be a directory: {path.name}") + if not _path_exists(path): + path.mkdir(mode=0o700) + + metadata = os.lstat(path) + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid(): + raise TemplateError("profile logs directory must be owned by the current user") + path.chmod(0o700) + validated = os.lstat(path) + if ( + not stat.S_ISDIR(validated.st_mode) + or validated.st_uid != os.geteuid() + or stat.S_IMODE(validated.st_mode) != 0o700 + ): + raise TemplateError("profile logs directory could not be made private") + + def _write_staged_file(path: Path, content: str) -> None: if _path_exists(path): if path.is_dir() and not path.is_symlink(): diff --git a/zeus/state.py b/zeus/state.py index 4562a70..13ad6fb 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -18,6 +18,7 @@ serialize_lifecycle_details, ) from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy +from zeus.private_io import append_private_bytes, nofollow_absolute_path from zeus.reconciliation import ( BotReconcileResult, PersistedReconcileRun, @@ -1760,10 +1761,8 @@ def append_audit_event(self, event: str, **fields: object) -> None: for key, value in fields.items(): payload[key] = _safe_audit_value(key, value) try: - path = self.audit_log_path() - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, sort_keys=True) + "\n") + line = (json.dumps(payload, sort_keys=True) + "\n").encode("utf-8") + append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) except (OSError, TypeError, ValueError): return diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 49e0e1c..49e8040 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -13,7 +13,6 @@ import signal import stat import subprocess # nosec B404 -import sys import threading import time import uuid @@ -60,6 +59,7 @@ TemplateError, validate_id, ) +from zeus.private_io import nofollow_absolute_path, open_private_append from zeus.process_lock import BotProcessLock, LockTimeoutError from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env from zeus.reconciliation import ( @@ -163,19 +163,7 @@ def _gateway_process_launch_kwargs() -> dict[str, object]: def _nofollow_absolute_path(path: Path) -> Path: - absolute = Path(os.path.abspath(path.expanduser())) - if ( - sys.platform == "darwin" - and len(absolute.parts) > 1 - and absolute.parts[1] - in { - "etc", - "tmp", - "var", - } - ): - return Path("/private", *absolute.parts[1:]) - return absolute + return nofollow_absolute_path(path) class Supervisor: @@ -753,7 +741,6 @@ def _start_record( raise RuntimeError("launcher produced an invalid marker payload") expected_fingerprint = str(marker_data["command_fingerprint"]) log_path = self.log_path(record.profile_path) - log_path.parent.mkdir(parents=True, exist_ok=True) process: PopenLike | None = None payload_read = payload_write = ack_read = ack_write = -1 try: @@ -765,10 +752,10 @@ def _start_record( ).encode("utf-8") if not encoded_payload or len(encoded_payload) > MAX_PAYLOAD_BYTES: raise ValueError("launcher payload is too large") - payload_read, payload_write = os.pipe() - ack_read, ack_write = os.pipe() - launcher_argv = self.adapter.launcher_command(payload_read, ack_write) - with log_path.open("ab") as log_file: + with open_private_append(log_path) as log_file: + payload_read, payload_write = os.pipe() + ack_read, ack_write = os.pipe() + launcher_argv = self.adapter.launcher_command(payload_read, ack_write) process = self.popen_factory( launcher_argv, env=dict(os.environ), @@ -3256,7 +3243,7 @@ def _wait_for_readiness(self, process: PopenLike, probe: ReadinessProbe) -> Read return ReadinessResult(False, f"readiness timeout: {last.message}", last.payload) def log_path(self, profile_path: str) -> Path: - return Path(profile_path) / "logs" / "zeus-gateway.log" + return _nofollow_absolute_path(Path(profile_path) / "logs" / "zeus-gateway.log") def pid_marker_path(self, profile_path: str) -> Path: return Path(profile_path) / "logs" / "zeus-gateway.pid.json" From 2507ef2d4702937bd6ff10a9916c6ee4c055f32c Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:07:11 +0200 Subject: [PATCH 10/49] security: prevent private path replacement races --- tests/test_private_io.py | 106 ++++++++++ tests/test_renderer_state.py | 62 ++++++ tests/test_supervisor_cli_api.py | 339 ++++++++++++++++++++++++++++++- zeus/config.py | 14 +- zeus/doctor.py | 18 +- zeus/gateway_launcher.py | 4 + zeus/private_io.py | 161 ++++++++++++--- zeus/renderer.py | 33 ++- zeus/supervisor.py | 71 +++++-- 9 files changed, 740 insertions(+), 68 deletions(-) diff --git a/tests/test_private_io.py b/tests/test_private_io.py index ce14259..6d79348 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -76,6 +76,112 @@ def test_read_missing_ancestor_returns_empty_without_creating_it(self) -> None: self.assertFalse(self.root.joinpath("missing").exists()) + def test_missing_tail_rejects_rebound_open_ancestor_and_closes_descriptors(self) -> None: + ancestor = self.root / "safe" + ancestor.mkdir(mode=0o700) + displaced = self.root / "safe-displaced" + external = self.root / "external" + (external / "missing").mkdir(parents=True) + target = external / "missing" / "events.log" + target.write_bytes(b"external target") + target_mode = stat.S_IMODE(target.stat().st_mode) + path = ancestor / "missing" / "events.log" + ancestor_identity = ancestor.stat() + opened: set[int] = set() + real_lstat = os.lstat + real_open = os.open + swapped = False + + def racing_lstat( + name: str | bytes, + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + if not swapped and name == "missing" and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == ancestor_identity.st_dev + and parent.st_ino == ancestor_identity.st_ino + ): + ancestor.rename(displaced) + ancestor.symlink_to(external, target_is_directory=True) + swapped = True + return real_lstat(name, dir_fd=dir_fd) + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + patch.object(private_io.os, "open", side_effect=tracking_open), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(swapped) + self.assertEqual(b"external target", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + self.assert_all_closed(opened) + + def test_missing_tail_rejects_leaf_appearing_after_observation_and_closes_descriptors( + self, + ) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + opened: set[int] = set() + real_lstat = os.lstat + real_open = os.open + appeared = False + + def racing_lstat( + name: str | bytes, + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal appeared + if not appeared and name == path.name and dir_fd is not None: + try: + return real_lstat(name, dir_fd=dir_fd) + except FileNotFoundError: + path.write_bytes(b"appeared after missing observation") + path.chmod(0o644) + appeared = True + raise + return real_lstat(name, dir_fd=dir_fd) + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + patch.object(private_io.os, "open", side_effect=tracking_open), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(appeared) + self.assertEqual(b"appeared after missing observation", path.read_bytes()) + self.assertEqual(0o644, stat.S_IMODE(path.stat().st_mode)) + self.assert_all_closed(opened) + def test_tail_size_validation_happens_before_filesystem_mutation(self) -> None: path = self.root / "missing" / "events.log" diff --git a/tests/test_renderer_state.py b/tests/test_renderer_state.py index 05df2b4..aef7be0 100644 --- a/tests/test_renderer_state.py +++ b/tests/test_renderer_state.py @@ -11,6 +11,7 @@ from pathlib import Path from unittest.mock import patch +from zeus import private_io from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEventInput from zeus.models import BotCreateRequest, BotRecord, BotStatus, HermesTemplate, TemplateError @@ -728,6 +729,67 @@ def test_renderer_atomically_replaces_profile_and_preserves_logs(self) -> None: self.assertEqual(0o700, logs.stat().st_mode & 0o777) self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + def test_renderer_logs_tightening_does_not_follow_staging_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_log = profile / "logs" / "gateway.log" + live_log.write_text("live log\n", encoding="utf-8") + external_logs = root / "external-race-target" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external target\n", encoding="utf-8") + external_mode = external_logs.stat().st_mode & 0o777 + swapped = False + + def swap_staging_logs() -> None: + nonlocal swapped + if swapped: + return + candidates = list(profile.parent.glob(".coder.staging-*/logs")) + self.assertEqual(1, len(candidates)) + staging_logs = candidates[0] + staging_logs.rename(staging_logs.with_name("logs-displaced")) + staging_logs.symlink_to(external_logs, target_is_directory=True) + swapped = True + + real_chmod = Path.chmod + + def racing_chmod( + path: Path, + mode: int, + *, + follow_symlinks: bool = True, + ) -> None: + if path.name == "logs" and path.parent.name.startswith(".coder.staging-"): + swap_staging_logs() + real_chmod(path, mode, follow_symlinks=follow_symlinks) + + def racing_validate_private_directory(path: Path) -> None: + swap_staging_logs() + private_io.validate_private_directory(path) + + with ( + patch.object(Path, "chmod", new=racing_chmod), + patch( + "zeus.renderer.validate_private_directory", + side_effect=racing_validate_private_directory, + ), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be installed")) + + self.assertTrue(swapped) + self.assertEqual("external target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) + self.assertEqual("live log\n", live_log.read_text(encoding="utf-8")) + def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> None: with tempfile.TemporaryDirectory() as tmp: hermes_root = Path(tmp) / ".zeus" / "hermes" diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 4432428..99cd6da 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -6,6 +6,7 @@ import json import os import sqlite3 +import stat import subprocess import sys import tempfile @@ -3159,6 +3160,144 @@ def test_direct_logs_and_inspect_reject_gateway_log_parent_symlink(self) -> None self.assertEqual("TARGET-SENTINEL\n", target_log.read_text(encoding="utf-8")) self.assertEqual(target_mode, target_log.stat().st_mode & 0o777) + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_direct_inspect_rejects_linked_pid_marker_fifo_without_blocking(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-logs" + external_logs.mkdir(mode=0o755) + fifo = external_logs / "zeus-gateway.pid.json" + os.mkfifo(fifo, mode=0o600) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external marker target\n", encoding="utf-8") + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + script = """ +import sys +from pathlib import Path +from zeus.private_io import UnsafeFileError +from zeus.state import StateStore +from zeus.supervisor import Supervisor + +root = Path(sys.argv[1]) +store = StateStore(root / "state" / "zeus.db") +supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") +try: + supervisor.inspect("coder") +except UnsafeFileError: + raise SystemExit(0) +raise SystemExit(2) +""" + + try: + completed = subprocess.run( + [sys.executable, "-c", script, str(root)], + check=False, + capture_output=True, + timeout=2, + ) + except subprocess.TimeoutExpired: + self.fail("direct inspect blocked on a linked PID-marker FIFO") + + self.assertEqual(0, completed.returncode, completed.stderr.decode("utf-8", "replace")) + self.assertEqual("external marker target\n", sentinel.read_text(encoding="utf-8")) + self.assertTrue(stat.S_ISFIFO(fifo.stat().st_mode)) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "FIFO creation is unavailable") + def test_api_inspect_returns_generic_500_for_linked_pid_marker_fifo_without_blocking( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + profile_path.mkdir(parents=True) + external_logs = root / "external-logs" + external_logs.mkdir(mode=0o755) + fifo = external_logs / "zeus-gateway.pid.json" + os.mkfifo(fifo, mode=0o600) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external API marker target\n", encoding="utf-8") + (profile_path / "logs").symlink_to(external_logs, target_is_directory=True) + store = StateStore(state_dir / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + script = """ +import http.client +import json +import sys +import threading +from http.server import ThreadingHTTPServer +from pathlib import Path +from zeus.api import make_handler +from zeus.config import Settings + +root = Path(sys.argv[1]) +settings = Settings.from_env({ + "ZEUS_STATE_DIR": str(root / "state"), + "ZEUS_API_KEY": "inspect-test-key", + "ZEUS_HOST": "127.0.0.1", + "ZEUS_PORT": "0", +}) +server = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(settings)) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +try: + connection = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=1) + connection.request( + "GET", + "/bots/coder/inspect", + headers={"x-zeus-api-key": "inspect-test-key"}, + ) + response = connection.getresponse() + body = json.loads(response.read()) + expected = { + "error": { + "code": "internal_error", + "message": "internal server error", + "status": 500, + } + } + if response.status != 500 or body != expected: + raise SystemExit(2) +finally: + server.shutdown() + server.server_close() +raise SystemExit(0) +""" + + try: + completed = subprocess.run( + [sys.executable, "-c", script, str(root)], + check=False, + capture_output=True, + timeout=4, + ) + except subprocess.TimeoutExpired: + self.fail("API inspect blocked on a linked PID-marker FIFO") + + self.assertEqual(0, completed.returncode, completed.stderr.decode("utf-8", "replace")) + self.assertEqual("external API marker target\n", sentinel.read_text(encoding="utf-8")) + self.assertTrue(stat.S_ISFIFO(fifo.stat().st_mode)) + def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -3226,6 +3365,35 @@ def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: self.assertNotIn("OPENROUTER_API_KEY", serialized) self.assertNotIn(hermes_bin, serialized) + def test_supervisor_inspect_preserves_invalid_pid_marker_payload(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile_path = root / ".zeus" / "hermes" / "profiles" / "coder" + logs_path = profile_path / "logs" + logs_path.mkdir(parents=True, mode=0o700) + marker = logs_path / "zeus-gateway.pid.json" + marker.write_text("{invalid json\n", encoding="utf-8") + marker.chmod(0o640) + store = StateStore(root / "state" / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile_path), + ) + ) + supervisor = Supervisor(store, "hermes", root / ".zeus" / "hermes") + + inspected = supervisor.inspect("coder") + + self.assertEqual(True, inspected["pid_marker"]["exists"]) + self.assertEqual(False, inspected["pid_marker"]["valid"]) + self.assertEqual("0640", inspected["pid_marker"]["mode"]) + self.assertIn("error", inspected["pid_marker"]) + self.assertEqual("", inspected["recent_logs"]) + def test_settings_parses_stop_kill_after_timeout_flag(self) -> None: with tempfile.TemporaryDirectory() as tmp: disabled = Settings.from_env( @@ -3529,21 +3697,188 @@ def test_doctor_rejects_state_symlink_without_target_mutation(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp).resolve() state_dir = root / "state" - settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) external_state = root / "external-state-target" - external_state.mkdir(mode=0o755) + external_state.mkdir(mode=0o700) + external_state.chmod(0o700) sentinel = external_state / "sentinel.txt" sentinel.write_text("state target\n", encoding="utf-8") target_mode = external_state.stat().st_mode & 0o777 state_dir.symlink_to(external_state, target_is_directory=True) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) check = _check_runtime_paths(settings) + self.assertEqual(state_dir, settings.state_dir) self.assertEqual("fail", check.status) self.assertNotIn(str(external_state), check.message) self.assertEqual("state target\n", sentinel.read_text(encoding="utf-8")) self.assertEqual(target_mode, external_state.stat().st_mode & 0o777) + def test_doctor_does_not_repair_state_replaced_after_preflight(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + displaced = root / "state-displaced" + replacement = root / "state-replacement" + replacement.mkdir(mode=0o777) + replacement.chmod(0o777) + sentinel = replacement / "sentinel.txt" + sentinel.write_text("replacement state\n", encoding="utf-8") + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + swapped = False + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + metadata = real_lstat(path, dir_fd=dir_fd) + if not swapped and dir_fd is None and Path(path) == state_dir: + state_dir.rename(displaced) + replacement.rename(state_dir) + swapped = True + return metadata + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertTrue(swapped) + self.assertEqual("fail", check.status) + self.assertEqual( + "replacement state\n", + (state_dir / sentinel.name).read_text(encoding="utf-8"), + ) + self.assertEqual(0o777, state_dir.stat().st_mode & 0o777) + + def test_doctor_does_not_repair_logs_replaced_after_preflight(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + logs_dir = state_dir / "logs" + logs_dir.mkdir(parents=True, mode=0o700) + state_dir.chmod(0o700) + logs_dir.chmod(0o700) + displaced = state_dir / "logs-displaced" + replacement = state_dir / "logs-replacement" + replacement.mkdir(mode=0o777) + replacement.chmod(0o777) + sentinel = replacement / "sentinel.txt" + sentinel.write_text("replacement logs\n", encoding="utf-8") + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + swapped = False + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal swapped + metadata = real_lstat(path, dir_fd=dir_fd) + if not swapped and dir_fd is None and Path(path) == logs_dir: + logs_dir.rename(displaced) + replacement.rename(logs_dir) + swapped = True + return metadata + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertTrue(swapped) + self.assertEqual("fail", check.status) + self.assertIn("logs", check.message.lower()) + self.assertEqual( + "replacement logs\n", + (logs_dir / sentinel.name).read_text(encoding="utf-8"), + ) + self.assertEqual(0o777, logs_dir.stat().st_mode & 0o777) + + def test_settings_ensure_dirs_creates_missing_private_runtime_tree(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env({"ZEUS_STATE_DIR": str(Path(tmp) / "state")}) + + settings.ensure_dirs() + + for path in ( + settings.state_dir, + settings.hermes_root, + settings.state_dir / "logs", + settings.state_dir / "locks", + settings.state_dir / "locks" / "bots", + ): + with self.subTest(path=path): + self.assertTrue(path.is_dir()) + self.assertEqual(0o700, path.stat().st_mode & 0o777) + + def test_runtime_entrypoints_reject_real_and_dangling_state_links(self) -> None: + for dangling in (False, True): + with self.subTest(dangling=dangling), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + external_state = root / "external-state" + if not dangling: + external_state.mkdir(mode=0o755) + external_state.chmod(0o755) + (external_state / "sentinel.txt").write_text( + "external state\n", encoding="utf-8" + ) + state_dir.symlink_to(external_state, target_is_directory=True) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + with self.assertRaises(UnsafeFileError): + settings.ensure_dirs() + StateStore(settings.database_path).append_audit_event("link.proof") + with self.assertRaises(UnsafeFileError): + make_handler(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(state_dir, settings.state_dir) + if dangling: + self.assertFalse(external_state.exists()) + else: + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_state.iterdir()) + ) + self.assertEqual(0o755, external_state.stat().st_mode & 0o777) + + def test_runtime_entrypoints_reject_real_and_dangling_state_ancestor_links(self) -> None: + for dangling in (False, True): + with self.subTest(dangling=dangling), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + linked_parent = root / "linked-parent" + external_parent = root / "external-parent" + external_state = external_parent / "state" + if not dangling: + external_state.mkdir(parents=True, mode=0o700) + external_state.chmod(0o700) + (external_state / "sentinel.txt").write_text( + "external ancestor state\n", encoding="utf-8" + ) + linked_parent.symlink_to(external_parent, target_is_directory=True) + state_dir = linked_parent / "state" + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + + check = _check_runtime_paths(settings) + with self.assertRaises(UnsafeFileError): + settings.ensure_dirs() + StateStore(settings.database_path).append_audit_event("ancestor-link.proof") + with self.assertRaises(UnsafeFileError): + make_handler(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(state_dir, settings.state_dir) + if dangling: + self.assertFalse(external_parent.exists()) + else: + self.assertEqual( + ["sentinel.txt"], sorted(path.name for path in external_state.iterdir()) + ) + self.assertEqual(0o700, external_state.stat().st_mode & 0o777) + def test_api_health_and_auth(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/zeus/config.py b/zeus/config.py index d5c4307..7f5f9cd 100644 --- a/zeus/config.py +++ b/zeus/config.py @@ -7,6 +7,7 @@ from pathlib import Path from zeus.envfile import parse_env_text +from zeus.private_io import ensure_private_directory, nofollow_absolute_path def load_dotenv(path: Path = Path(".env")) -> dict[str, str]: @@ -45,7 +46,7 @@ class Settings: def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: merged: dict[str, str] = load_dotenv() merged.update(dict(os.environ if env is None else env)) - state_dir = Path(merged.get("ZEUS_STATE_DIR") or ".zeus").resolve() + state_dir = nofollow_absolute_path(Path(merged.get("ZEUS_STATE_DIR") or ".zeus")) port = _port_env(merged, "ZEUS_PORT", default=4311) return cls( state_dir=state_dir, @@ -142,10 +143,13 @@ def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: ) def ensure_dirs(self) -> None: - self.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) - self.hermes_root.mkdir(parents=True, exist_ok=True, mode=0o700) - (self.state_dir / "logs").mkdir(parents=True, exist_ok=True, mode=0o700) - (self.state_dir / "locks" / "bots").mkdir(parents=True, exist_ok=True, mode=0o700) + for path in ( + self.state_dir, + self.hermes_root, + self.state_dir / "logs", + self.state_dir / "locks" / "bots", + ): + ensure_private_directory(path) def is_loopback_host(host: str) -> bool: diff --git a/zeus/doctor.py b/zeus/doctor.py index c4cb5c4..c2d763a 100644 --- a/zeus/doctor.py +++ b/zeus/doctor.py @@ -12,7 +12,7 @@ from typing import Any from zeus.config import Settings, is_loopback_host -from zeus.private_io import nofollow_absolute_path, validate_private_directory +from zeus.private_io import inspect_private_directory, nofollow_absolute_path from zeus.state import StateStore from zeus.templates import TemplateStore @@ -111,6 +111,20 @@ def _existing_runtime_directory_check( try: metadata = os.lstat(path) except FileNotFoundError: + try: + exists = inspect_private_directory(path, missing_ok=True) + except OSError: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} could not be validated safely", + ) + if exists: + return True, DoctorCheck( + "runtime_paths", + "fail", + f"{label} {path} changed while it was inspected", + ) return False, None except OSError: return True, DoctorCheck( @@ -137,7 +151,7 @@ def _existing_runtime_directory_check( f"{label} {path} must not be accessible to other users or groups", ) try: - validate_private_directory(path) + inspect_private_directory(path) except OSError: return True, DoctorCheck( "runtime_paths", diff --git a/zeus/gateway_launcher.py b/zeus/gateway_launcher.py index d96feb2..0edc6d8 100644 --- a/zeus/gateway_launcher.py +++ b/zeus/gateway_launcher.py @@ -535,7 +535,11 @@ def _open_regular_marker(logs_fd: int) -> tuple[int, os.stat_result]: raise LaunchPayloadError("marker is unavailable") from exc if not stat.S_ISREG(before.st_mode): raise LaunchPayloadError("marker is not a regular file") + nonblocking = getattr(os, "O_NONBLOCK", None) + if type(nonblocking) is not int or nonblocking == 0: + raise LaunchPayloadError("bounded marker reads are unavailable") flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= nonblocking try: marker_fd = os.open(MARKER_NAME, flags, dir_fd=logs_fd) except OSError as exc: diff --git a/zeus/private_io.py b/zeus/private_io.py index 4ad7078..bf0295d 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -47,6 +47,32 @@ class _Platform: create_exclusive_flags: int +@dataclass(frozen=True) +class _OpenedDirectoryPath: + descriptors: tuple[int, ...] + names: tuple[str, ...] + + @property + def fd(self) -> int: + return self.descriptors[-1] + + def validate_bindings(self) -> None: + _validate_directory_bindings(self.descriptors, self.names) + + def confirm_missing(self, name: str) -> None: + self.validate_bindings() + try: + os.lstat(name, dir_fd=self.fd) + except FileNotFoundError: + self.validate_bindings() + return + except OSError as exc: + raise UnsafeFileError("missing private path could not be confirmed") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("missing private path cannot be inspected safely") from exc + raise UnsafeFileError("private path appeared while absence was confirmed") + + def _required_flag(name: str, *, allow_zero: bool = False) -> int: value = getattr(os, name, None) if type(value) is not int or (not allow_zero and value == 0): @@ -158,6 +184,42 @@ def _validate_directory_snapshots( raise UnsafeFileError(f"{description} changed while it was opened") +def _validate_directory_bindings( + descriptors: tuple[int, ...], + names: tuple[str, ...], +) -> None: + if len(descriptors) != len(names) + 1: + raise UnsafeFileError("private directory descriptor chain is invalid") + try: + root_snapshots = (os.fstat(descriptors[0]), os.lstat("/")) + _validate_directory_snapshots(root_snapshots, "filesystem root") + for parent_fd, name, directory_fd in zip( + descriptors[:-1], + names, + descriptors[1:], + strict=True, + ): + snapshots = (os.fstat(directory_fd), os.lstat(name, dir_fd=parent_fd)) + _validate_directory_snapshots(snapshots, f"private path component {name!r}") + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("private directory path binding changed") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private directory path cannot be inspected safely") from exc + + +def _inspect_private_directory_snapshots( + snapshots: tuple[os.stat_result, ...], + platform: _Platform, + description: str, +) -> None: + if not all(snapshot.st_uid == platform.euid for snapshot in snapshots): + raise UnsafeFileError(f"{description} has an unexpected owner") + if any(stat.S_IMODE(snapshot.st_mode) & 0o077 for snapshot in snapshots): + raise UnsafeFileError(f"{description} does not have private permissions") + + def _tighten_directory( parent_fd: int, name: str, @@ -215,6 +277,7 @@ def _open_directory_at( create: bool, missing_ok: bool, private: bool, + tighten: bool, platform: _Platform, ) -> int: description = f"private path component {name!r}" @@ -254,7 +317,7 @@ def _open_directory_at( after = os.lstat(name, dir_fd=parent_fd) snapshots = (before, opened, after) _validate_directory_snapshots(snapshots, description) - if created or private: + if created or (private and tighten): _tighten_directory( parent_fd, name, @@ -263,6 +326,8 @@ def _open_directory_at( platform, description, ) + elif private: + _inspect_private_directory_snapshots(snapshots, platform, description) return directory_fd except UnsafeFileError: _close_suppressing_error(directory_fd) @@ -281,41 +346,56 @@ def _open_directory_path( *, create: bool, missing_ok: bool = False, + tighten: bool = True, platform: _Platform, -) -> Iterator[int]: - current_fd = _open_root(platform) +) -> Iterator[_OpenedDirectoryPath]: + descriptors = [_open_root(platform)] + names: list[str] = [] try: for index, component in enumerate(parts): - next_fd = _open_directory_at( - current_fd, - component, - create=create, - missing_ok=missing_ok, - private=index == len(parts) - 1, - platform=platform, - ) - previous_fd = current_fd - current_fd = -1 try: - _close_descriptor(previous_fd, "ancestor directory") - except UnsafeFileError: - _close_suppressing_error(next_fd) + next_fd = _open_directory_at( + descriptors[-1], + component, + create=create, + missing_ok=missing_ok, + private=index == len(parts) - 1, + tighten=tighten, + platform=platform, + ) + except _PrivatePathMissing: + _OpenedDirectoryPath(tuple(descriptors), tuple(names)).confirm_missing(component) raise - current_fd = next_fd + descriptors.append(next_fd) + names.append(component) except BaseException: - if current_fd >= 0: - _close_suppressing_error(current_fd) + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) raise - result_fd = current_fd - current_fd = -1 + opened = _OpenedDirectoryPath(tuple(descriptors), tuple(names)) try: - yield result_fd + yield opened except BaseException: - _close_suppressing_error(result_fd) + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) raise else: - _close_descriptor(result_fd, "private directory") + try: + opened.validate_bindings() + except BaseException: + for descriptor in reversed(descriptors): + _close_suppressing_error(descriptor) + raise + close_error: UnsafeFileError | None = None + for descriptor in reversed(descriptors): + try: + _close_descriptor(descriptor, "private directory") + except UnsafeFileError as exc: + if close_error is None: + close_error = exc + if close_error is not None: + raise close_error def _validate_file_snapshot(snapshot: os.stat_result, platform: _Platform) -> None: @@ -395,9 +475,9 @@ def _private_append_context( name: str, platform: _Platform, ) -> Iterator[BinaryIO]: - with _open_directory_path(parent_parts, create=True, platform=platform) as parent_fd: + with _open_directory_path(parent_parts, create=True, platform=platform) as parent: file_fd = _open_private_file_at( - parent_fd, + parent.fd, name, append=True, create=True, @@ -462,15 +542,16 @@ def read_private_tail(path: Path, max_bytes: int) -> bytes: try: with _open_directory_path( parts[:-1], create=False, missing_ok=True, platform=platform - ) as parent_fd: + ) as parent: file_fd = _open_private_file_at( - parent_fd, + parent.fd, parts[-1], append=False, create=False, platform=platform, ) if file_fd is None: + parent.confirm_missing(parts[-1]) return b"" try: try: @@ -503,3 +584,27 @@ def validate_private_directory(path: Path) -> None: platform = _require_platform() with _open_directory_path(parts, create=False, platform=platform): pass + + +def ensure_private_directory(path: Path) -> None: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + with _open_directory_path(parts, create=True, platform=platform): + pass + + +def inspect_private_directory(path: Path, *, missing_ok: bool = False) -> bool: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + try: + with _open_directory_path( + parts, + create=False, + missing_ok=missing_ok, + tighten=False, + platform=platform, + ): + pass + except _PrivatePathMissing: + return False + return True diff --git a/zeus/renderer.py b/zeus/renderer.py index 0f1de88..420d56c 100644 --- a/zeus/renderer.py +++ b/zeus/renderer.py @@ -14,6 +14,7 @@ from zeus.envfile import dump_env from zeus.models import BotCreateRequest, BotRecord, HermesTemplate, TemplateError +from zeus.private_io import nofollow_absolute_path, validate_private_directory _LOG = logging.getLogger(__name__) @@ -159,26 +160,24 @@ def _ensure_preserved_directory(path: Path) -> None: def _ensure_private_logs_directory(path: Path) -> None: - if _path_exists(path): + try: + if _path_exists(path): + metadata = os.lstat(path) + if stat.S_ISLNK(metadata.st_mode): + path.unlink() + elif not stat.S_ISDIR(metadata.st_mode): + raise TemplateError(f"profile path must be a directory: {path.name}") + if not _path_exists(path): + path.mkdir(mode=0o700) metadata = os.lstat(path) - if stat.S_ISLNK(metadata.st_mode): - path.unlink() - elif not stat.S_ISDIR(metadata.st_mode): - raise TemplateError(f"profile path must be a directory: {path.name}") - if not _path_exists(path): - path.mkdir(mode=0o700) - - metadata = os.lstat(path) + except OSError as exc: + raise TemplateError("profile logs directory could not be prepared safely") from exc if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid(): raise TemplateError("profile logs directory must be owned by the current user") - path.chmod(0o700) - validated = os.lstat(path) - if ( - not stat.S_ISDIR(validated.st_mode) - or validated.st_uid != os.geteuid() - or stat.S_IMODE(validated.st_mode) != 0o700 - ): - raise TemplateError("profile logs directory could not be made private") + try: + validate_private_directory(nofollow_absolute_path(path)) + except OSError as exc: + raise TemplateError("profile logs directory could not be made private") from exc def _write_staged_file(path: Path, content: str) -> None: diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 49e8040..6851292 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -33,6 +33,7 @@ ) from zeus.fs_utils import atomic_write_json from zeus.gateway_launcher import ( + MARKER_NAME, MAX_PAYLOAD_BYTES, LaunchPayloadError, _is_owned_runtime_marker, @@ -59,7 +60,7 @@ TemplateError, validate_id, ) -from zeus.private_io import nofollow_absolute_path, open_private_append +from zeus.private_io import UnsafeFileError, nofollow_absolute_path, open_private_append from zeus.process_lock import BotProcessLock, LockTimeoutError from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env from zeus.reconciliation import ( @@ -166,6 +167,19 @@ def _nofollow_absolute_path(path: Path) -> Path: return nofollow_absolute_path(path) +def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + class Supervisor: def __init__( self, @@ -3345,13 +3359,49 @@ def _remove_pid_marker(self, profile_path: str) -> None: return def _read_pid_marker(self, profile_path: str) -> dict[str, object]: - path = self.pid_marker_path(profile_path) - if not path.exists(): - return {"exists": False} - marker_mode = _file_mode(path) + safe_profile_path = _nofollow_absolute_path(Path(profile_path)) + profile_fd = logs_fd = marker_fd = -1 + try: + try: + profile_fd = _open_profile(safe_profile_path) + logs_fd = _open_logs(profile_fd, create=False) + marker_fd, marker_stat = _open_regular_marker(logs_fd) + except (LaunchPayloadError, OSError, ValueError) as exc: + if _caused_by_missing_path(exc): + return {"exists": False} + raise UnsafeFileError("PID marker cannot be opened safely") from exc + if marker_stat.st_uid != os.geteuid() or marker_stat.st_nlink != 1: + raise UnsafeFileError("PID marker is not a private regular file") + marker_mode = f"{stat.S_IMODE(marker_stat.st_mode):04o}" + try: + raw = _read_bounded_file(marker_fd) + current_marker = os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) + current_logs = os.stat("logs", dir_fd=profile_fd, follow_symlinks=False) + opened_logs = os.fstat(logs_fd) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + return { + "exists": True, + "valid": False, + "mode": marker_mode, + "error": str(exc), + } + if ( + not stat.S_ISREG(current_marker.st_mode) + or current_marker.st_uid != os.geteuid() + or current_marker.st_nlink != 1 + or not _same_identity(marker_stat, current_marker) + or not stat.S_ISDIR(current_logs.st_mode) + or not _same_identity(opened_logs, current_logs) + ): + raise UnsafeFileError("PID marker changed while it was inspected") + finally: + for descriptor in (marker_fd, logs_fd, profile_fd): + if descriptor >= 0: + with contextlib.suppress(OSError): + os.close(descriptor) try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} if not isinstance(payload, dict): return { @@ -4014,10 +4064,3 @@ def _safe_command_shape(argv: list[str]) -> str: if len(args) == 4 and args[0] == "-p" and args[2:] == ["gateway", "run"]: return f"{classification} hermes -p gateway run" return f"{classification} unrecognized" - - -def _file_mode(path: Path) -> str | None: - try: - return f"{stat.S_IMODE(path.stat().st_mode):04o}" - except OSError: - return None From f4c886f5cd49cbbabde32418746c29138975e9c8 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:35:53 +0200 Subject: [PATCH 11/49] security: close remaining private path races --- tests/test_crash_recovery.py | 168 ++++++++++++++ tests/test_private_io.py | 130 +++++++++++ tests/test_renderer_state.py | 93 ++++++++ tests/test_supervisor_cli_api.py | 374 +++++++++++++++++++++++++++++++ zeus/gateway_launcher.py | 284 ++++++++++++++++++++--- zeus/private_io.py | 206 +++++++++++++++-- zeus/renderer.py | 21 +- zeus/supervisor.py | 91 ++++++-- 8 files changed, 1297 insertions(+), 70 deletions(-) diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index 410d7ae..fc7ac45 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import errno import hashlib import json import os @@ -699,6 +700,173 @@ def racing_open(path: object, flags: int, *args: object, **kwargs: object) -> in self.assertTrue(swapped) + def test_profile_chain_rejects_final_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_profile_chain + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / "hermes" / "profiles" / "coder" + profile.parent.mkdir(parents=True) + external = root / "external-profile" + external.mkdir() + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == profile.name and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + profile.symlink_to(external, target_is_directory=True) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaisesRegex(LaunchPayloadError, "appeared"), + ): + _open_profile_chain(profile) + + self.assertTrue(appeared) + self.assertTrue(profile.is_symlink()) + + def test_profile_chain_rejects_intermediate_symlink_appearing_during_missing_lookup( + self, + ) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_profile_chain + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes = root / "hermes" + hermes.mkdir() + profiles = hermes / "profiles" + profile = profiles / "coder" + external = root / "external-profiles" + (external / "coder").mkdir(parents=True) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == profiles.name and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + profiles.symlink_to(external, target_is_directory=True) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaisesRegex(LaunchPayloadError, "appeared"), + ): + _open_profile_chain(profile) + + self.assertTrue(appeared) + self.assertTrue(profiles.is_symlink()) + + def test_directory_open_closes_new_descriptor_when_post_open_fstat_fails(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_directory_at + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + child = root / "child" + child.mkdir() + parent_fd = os.open(root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + real_open = os.open + real_fstat = os.fstat + opened_fd: int | None = None + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal opened_fd + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + if path == child.name and dir_fd == parent_fd: + opened_fd = descriptor + return descriptor + + def failing_fstat(fd: int) -> os.stat_result: + if fd == opened_fd: + raise OSError(errno.EIO, "injected directory fstat failure") + return real_fstat(fd) + + try: + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.gateway_launcher.os.fstat", side_effect=failing_fstat), + self.assertRaises(LaunchPayloadError), + ): + _open_directory_at(parent_fd, child.name) + finally: + os.close(parent_fd) + + self.assertIsNotNone(opened_fd) + with self.assertRaises(OSError): + os.fstat(opened_fd) # type: ignore[arg-type] + + def test_marker_open_closes_new_descriptor_when_post_open_fstat_fails(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _open_regular_marker + + with tempfile.TemporaryDirectory() as tmp: + logs = Path(tmp) / "logs" + logs.mkdir() + marker = logs / "zeus-gateway.pid.json" + marker.write_text("{}", encoding="utf-8") + logs_fd = os.open(logs, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + real_open = os.open + real_fstat = os.fstat + opened_fd: int | None = None + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal opened_fd + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + if path == marker.name and dir_fd == logs_fd: + opened_fd = descriptor + return descriptor + + def failing_fstat(fd: int) -> os.stat_result: + if fd == opened_fd: + raise OSError(errno.EIO, "injected marker fstat failure") + return real_fstat(fd) + + try: + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.gateway_launcher.os.fstat", side_effect=failing_fstat), + self.assertRaises(LaunchPayloadError), + ): + _open_regular_marker(logs_fd) + finally: + os.close(logs_fd) + + self.assertIsNotNone(opened_fd) + with self.assertRaises(OSError): + os.fstat(opened_fd) # type: ignore[arg-type] + def test_ack_failure_removes_only_the_owned_marker(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tests/test_private_io.py b/tests/test_private_io.py index 6d79348..7b2c923 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -15,6 +15,7 @@ from zeus.private_io import ( UnsafeFileError, append_private_bytes, + ensure_private_directory, open_private_append, read_private_tail, validate_private_directory, @@ -280,6 +281,54 @@ def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_res self.assertEqual(3, directory_lstats) self.assertEqual(0o777, stat.S_IMODE(private_dir.stat().st_mode)) + def test_validate_directory_rejects_mode_drift_during_final_binding_check(self) -> None: + private_dir = self.root / "final-mode-drift-dir" + private_dir.mkdir(mode=0o700) + private_dir.chmod(0o700) + real_lstat = os.lstat + directory_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal directory_lstats + if name == private_dir.name and dir_fd is not None: + directory_lstats += 1 + if directory_lstats == 4: + private_dir.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + validate_private_directory(private_dir) + + self.assertEqual(4, directory_lstats) + self.assertEqual(0o777, stat.S_IMODE(private_dir.stat().st_mode)) + + def test_ensure_directory_rechecks_created_intermediate_mode_at_final_binding(self) -> None: + created_parent = self.root / "created-parent" + private_dir = created_parent / "private" + real_lstat = os.lstat + parent_lstats = 0 + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal parent_lstats + if name == created_parent.name and dir_fd is not None: + parent_lstats += 1 + if parent_lstats == 5: + created_parent.chmod(0o777) + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + ): + ensure_private_directory(private_dir) + + self.assertEqual(5, parent_lstats) + self.assertEqual(0o777, stat.S_IMODE(created_parent.stat().st_mode)) + self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + def test_existing_file_and_parent_modes_are_tightened_before_append(self) -> None: private_dir = self.root / "logs" private_dir.mkdir(mode=0o755) @@ -534,6 +583,87 @@ def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_res self.assertEqual(b"original", displaced.read_bytes()) self.assertEqual(b"replacement", path.read_bytes()) + def test_tail_rejects_leaf_replacement_after_read(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + displaced = private_dir / "displaced.log" + target = self.root / "external-target.log" + target.write_bytes(b"external") + target.chmod(0o644) + target_mode = stat.S_IMODE(target.stat().st_mode) + real_read = os.read + swapped = False + + def racing_read(fd: int, size: int) -> bytes: + nonlocal swapped + result = real_read(fd, size) + if result and not swapped and stat.S_ISREG(os.fstat(fd).st_mode): + path.rename(displaced) + path.symlink_to(target) + swapped = True + return result + + with ( + patch.object(private_io.os, "read", side_effect=racing_read), + self.assertRaises(UnsafeFileError), + ): + read_private_tail(path, 128) + + self.assertTrue(swapped) + self.assertTrue(path.is_symlink()) + self.assertEqual(b"original", displaced.read_bytes()) + self.assertEqual(b"external", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + + def test_append_rejects_leaf_replacement_after_write(self) -> None: + private_dir = self.root / "logs" + private_dir.mkdir(mode=0o700) + path = private_dir / "events.log" + path.write_bytes(b"original") + displaced = private_dir / "displaced.log" + target = self.root / "external-target.log" + target.write_bytes(b"external") + target.chmod(0o644) + target_mode = stat.S_IMODE(target.stat().st_mode) + real_fdopen = os.fdopen + swapped = False + + class RacingHandle: + def __init__(self, handle: BinaryIO) -> None: + self.handle = handle + + def write(self, data: bytes) -> int: + nonlocal swapped + written = self.handle.write(data) + if not swapped: + path.rename(displaced) + path.symlink_to(target) + swapped = True + return written + + def fileno(self) -> int: + return self.handle.fileno() + + def close(self) -> None: + self.handle.close() + + def racing_fdopen(fd: int, mode: str, buffering: int = -1) -> RacingHandle: + return RacingHandle(real_fdopen(fd, mode, buffering=buffering)) + + with ( + patch.object(private_io.os, "fdopen", side_effect=racing_fdopen), + self.assertRaises(UnsafeFileError), + ): + append_private_bytes(path, b"event") + + self.assertTrue(swapped) + self.assertTrue(path.is_symlink()) + self.assertEqual(b"originalevent", displaced.read_bytes()) + self.assertEqual(b"external", target.read_bytes()) + self.assertEqual(target_mode, stat.S_IMODE(target.stat().st_mode)) + def test_directory_replacement_between_lstat_and_open_is_rejected(self) -> None: private_dir = self.root / "private" logs = private_dir / "logs" diff --git a/tests/test_renderer_state.py b/tests/test_renderer_state.py index aef7be0..b52b875 100644 --- a/tests/test_renderer_state.py +++ b/tests/test_renderer_state.py @@ -2,6 +2,7 @@ import os import sqlite3 +import stat import tempfile import unittest from concurrent.futures import ThreadPoolExecutor @@ -790,6 +791,98 @@ def racing_validate_private_directory(path: Path) -> None: self.assertEqual(external_mode, external_logs.stat().st_mode & 0o777) self.assertEqual("live log\n", live_log.read_text(encoding="utf-8")) + def test_renderer_rejects_logs_mode_drift_during_final_binding_check(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + real_lstat = os.lstat + logs_lstats = 0 + drifted_mode: int | None = None + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal drifted_mode, logs_lstats + if path == "logs" and dir_fd is not None: + candidates = list(profile.parent.glob(".coder.staging-*/logs")) + if len(candidates) == 1: + staging_logs = candidates[0] + parent = staging_logs.parent.stat() + opened_parent = os.fstat(dir_fd) + if ( + parent.st_dev == opened_parent.st_dev + and parent.st_ino == opened_parent.st_ino + ): + logs_lstats += 1 + if logs_lstats == 4: + staging_logs.chmod(0o777) + drifted_mode = staging_logs.stat().st_mode & 0o777 + return real_lstat(path, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be committed")) + + self.assertEqual(4, logs_lstats) + self.assertEqual(0o777, drifted_mode) + self.assertEqual(live_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + self.assertEqual(0o700, (profile / "logs").stat().st_mode & 0o777) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + + def test_renderer_rejects_logs_replacement_immediately_before_install(self) -> None: + from zeus.renderer import _install_staged_profile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + hermes_root = root / ".zeus" / "hermes" + renderer = ProfileRenderer(hermes_root) + template = TemplateStore().get("coding-bot") + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + renderer.render(request, template) + profile = hermes_root / "profiles" / "coder" + live_soul = (profile / "SOUL.md").read_text(encoding="utf-8") + live_log = profile / "logs" / "gateway.log" + live_log.write_text("live log\n", encoding="utf-8") + external_logs = root / "external-race-target" + external_logs.mkdir(mode=0o755) + external_logs.chmod(0o755) + sentinel = external_logs / "sentinel.txt" + sentinel.write_text("external target\n", encoding="utf-8") + external_mode = stat.S_IMODE(external_logs.stat().st_mode) + swapped = False + + def racing_install(staging: Path, destination: Path) -> Path | None: + nonlocal swapped + staging_logs = staging / "logs" + staging_logs.rename(staging / "logs-displaced") + staging_logs.symlink_to(external_logs, target_is_directory=True) + swapped = True + return _install_staged_profile(staging, destination) + + with ( + patch("zeus.renderer._install_staged_profile", side_effect=racing_install), + self.assertRaises(TemplateError), + ): + renderer.render(request, replace(template, soul="must not be committed")) + + self.assertTrue(swapped) + self.assertEqual(live_soul, (profile / "SOUL.md").read_text(encoding="utf-8")) + self.assertFalse((profile / "logs").is_symlink()) + self.assertEqual("live log\n", live_log.read_text(encoding="utf-8")) + self.assertEqual("external target\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(external_mode, stat.S_IMODE(external_logs.stat().st_mode)) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + def test_renderer_replacement_preserves_complete_existing_profile_tree(self) -> None: with tempfile.TemporaryDirectory() as tmp: hermes_root = Path(tmp) / ".zeus" / "hermes" diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 99cd6da..c047e3d 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -3298,6 +3298,351 @@ def test_api_inspect_returns_generic_500_for_linked_pid_marker_fifo_without_bloc self.assertEqual("external API marker target\n", sentinel.read_text(encoding="utf-8")) self.assertTrue(stat.S_ISFIFO(fifo.stat().st_mode)) + def test_pid_marker_read_rejects_profile_ancestry_replacement_during_read(self) -> None: + from zeus.supervisor import _read_bounded_file + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + external_logs = external / "logs" + external_logs.mkdir(parents=True) + (external_logs / marker.name).write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + opened: set[int] = set() + swapped = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def racing_read(fd: int, limit: int = 64 * 1024) -> bytes: + nonlocal swapped + result = _read_bounded_file(fd, limit) + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + return result + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor._read_bounded_file", side_effect=racing_read), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(swapped) + self.assertTrue(profile.is_symlink()) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_strict_marker_read_rejects_profile_ancestry_replacement_during_read(self) -> None: + from zeus.supervisor import _read_bounded_file + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + external_logs = external / "logs" + external_logs.mkdir(parents=True) + (external_logs / marker.name).write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + opened: set[int] = set() + swapped = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def racing_read(fd: int, limit: int = 64 * 1024) -> bytes: + nonlocal swapped + result = _read_bounded_file(fd, limit) + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + return result + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor._read_bounded_file", side_effect=racing_read), + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(swapped) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(profile.is_symlink()) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_pid_marker_read_revalidates_ancestry_after_read_failure(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = root / "displaced-profile" + external = root / "external-profile" + (external / "logs").mkdir(parents=True) + _store, supervisor = self._supervisor_for_profile_path(root, profile) + swapped = False + + def failing_read(_fd: int, _limit: int = 64 * 1024) -> bytes: + nonlocal swapped + profile.rename(displaced) + profile.symlink_to(external, target_is_directory=True) + swapped = True + raise LaunchPayloadError("injected bounded read failure") + + with ( + patch("zeus.supervisor._read_bounded_file", side_effect=failing_read), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(swapped) + self.assertTrue(profile.is_symlink()) + + def test_strict_marker_read_closes_all_descriptors_when_one_close_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_open = os.open + real_close = os.close + opened: set[int] = set() + close_failed = False + + def tracking_open( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(path, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def noisy_close(fd: int) -> None: + nonlocal close_failed + real_close(fd) + if not close_failed: + close_failed = True + raise OSError("injected close failure") + + with ( + patch("zeus.gateway_launcher.os.open", side_effect=tracking_open), + patch("zeus.supervisor.os.close", side_effect=noisy_close), + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(close_failed) + self.assertEqual("present", observation.kind) + for descriptor in opened: + with self.subTest(descriptor=descriptor), self.assertRaises(OSError): + os.fstat(descriptor) + + def test_strict_marker_read_rejects_current_owner_mismatch(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + marker.write_text('{"pid":4321}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + marker_stats = 0 + + def mismatched_owner_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal marker_stats + result = real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + if path == MARKER_NAME and dir_fd is not None: + marker_stats += 1 + if marker_stats == 2: + fields = list(result) + fields[4] = result.st_uid + 1 + return os.stat_result(fields) + return result + + with patch("zeus.gateway_launcher.os.stat", side_effect=mismatched_owner_stat): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertGreaterEqual(marker_stats, 2) + self.assertEqual("untrusted", observation.kind) + + def test_strict_marker_read_rechecks_leaf_after_final_directory_validation(self) -> None: + from zeus.gateway_launcher import _validate_open_directory_binding + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / "zeus-gateway.pid.json" + marker.write_text('{"pid":4321}', encoding="utf-8") + displaced = logs / "displaced-marker.json" + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + logs_validations = 0 + swapped = False + + def racing_directory_validation( + parent_fd: int, + name: str, + directory_fd: int, + description: str, + ) -> os.stat_result: + nonlocal logs_validations, swapped + result = _validate_open_directory_binding( + parent_fd, + name, + directory_fd, + description, + ) + if name == "logs": + logs_validations += 1 + if logs_validations == 2: + marker.rename(displaced) + marker.symlink_to(target) + swapped = True + return result + + with patch( + "zeus.gateway_launcher._validate_open_directory_binding", + side_effect=racing_directory_validation, + ): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(swapped) + self.assertEqual(2, logs_validations) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(marker.is_symlink()) + + def test_pid_marker_read_rejects_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == MARKER_NAME and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + marker.symlink_to(target) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(str(profile)) + + self.assertTrue(appeared) + self.assertTrue(marker.is_symlink()) + self.assertEqual('{"pid":9876}', target.read_text(encoding="utf-8")) + + def test_strict_marker_read_rejects_symlink_appearing_during_missing_lookup(self) -> None: + from zeus.gateway_launcher import MARKER_NAME + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + profile = root / ".zeus" / "hermes" / "profiles" / "coder" + logs = profile / "logs" + logs.mkdir(parents=True, mode=0o700) + marker = logs / MARKER_NAME + target = root / "external-marker.json" + target.write_text('{"pid":9876}', encoding="utf-8") + _store, supervisor = self._supervisor_for_profile_path(root, profile) + real_stat = os.stat + appeared = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal appeared + if path == MARKER_NAME and dir_fd is not None and not appeared: + try: + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + except FileNotFoundError: + marker.symlink_to(target) + appeared = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat): + observation = supervisor._read_strict_runtime_marker("coder", str(profile)) + + self.assertTrue(appeared) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(marker.is_symlink()) + self.assertEqual('{"pid":9876}', target.read_text(encoding="utf-8")) + def test_supervisor_inspect_reports_metadata_without_env_contents(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -3796,6 +4141,35 @@ def racing_lstat( ) self.assertEqual(0o777, logs_dir.stat().st_mode & 0o777) + def test_doctor_rejects_same_inode_mode_drift_at_final_binding(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + state_dir = root / "state" + state_dir.mkdir(mode=0o700) + state_dir.chmod(0o700) + settings = Settings.from_env({"ZEUS_STATE_DIR": str(state_dir)}) + real_lstat = os.lstat + state_lstats = 0 + + def racing_lstat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal state_lstats + if path == state_dir.name and dir_fd is not None: + state_lstats += 1 + if state_lstats == 3: + state_dir.chmod(0o777) + return real_lstat(path, dir_fd=dir_fd) + + with patch("zeus.doctor.os.lstat", side_effect=racing_lstat): + check = _check_runtime_paths(settings) + + self.assertEqual("fail", check.status) + self.assertEqual(3, state_lstats) + self.assertEqual(0o777, state_dir.stat().st_mode & 0o777) + def test_settings_ensure_dirs_creates_missing_private_runtime_tree(self) -> None: with tempfile.TemporaryDirectory() as tmp: settings = Settings.from_env({"ZEUS_STATE_DIR": str(Path(tmp) / "state")}) diff --git a/zeus/gateway_launcher.py b/zeus/gateway_launcher.py index 0edc6d8..f1388c7 100644 --- a/zeus/gateway_launcher.py +++ b/zeus/gateway_launcher.py @@ -11,6 +11,7 @@ import subprocess # nosec B404 import sys import time +from dataclasses import dataclass from pathlib import Path from types import TracebackType from typing import NoReturn @@ -239,6 +240,108 @@ def _same_file(before: os.stat_result, after: os.stat_result) -> bool: return before.st_dev == after.st_dev and before.st_ino == after.st_ino +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + +def _validate_open_directory_binding( + parent_fd: int, + name: str, + directory_fd: int, + description: str, +) -> os.stat_result: + try: + opened = os.fstat(directory_fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError(f"{description} changed while it was used") from exc + if ( + not stat.S_ISDIR(opened.st_mode) + or not stat.S_ISDIR(current.st_mode) + or not _same_file(opened, current) + ): + raise LaunchPayloadError(f"{description} changed while it was used") + return opened + + +@dataclass +class _OpenedProfile: + descriptors: list[int] + names: tuple[str, ...] + + @property + def fd(self) -> int: + if not self.descriptors: + raise LaunchPayloadError("profile descriptor chain is closed") + return self.descriptors[-1] + + def validate_bindings(self, *, require_profile_owner: bool = True) -> None: + if len(self.descriptors) != len(self.names) + 1: + raise LaunchPayloadError("profile descriptor chain is invalid") + try: + opened_root = os.fstat(self.descriptors[0]) + current_root = os.stat("/", follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError("filesystem root changed while it was used") from exc + if ( + not stat.S_ISDIR(opened_root.st_mode) + or not stat.S_ISDIR(current_root.st_mode) + or not _same_file(opened_root, current_root) + ): + raise LaunchPayloadError("filesystem root changed while it was used") + for parent_fd, name, directory_fd in zip( + self.descriptors[:-1], + self.names, + self.descriptors[1:], + strict=True, + ): + _validate_open_directory_binding( + parent_fd, + name, + directory_fd, + "profile path component", + ) + try: + profile_stat = os.fstat(self.fd) + except OSError as exc: + raise LaunchPayloadError("profile directory changed while it was used") from exc + if require_profile_owner and hasattr(os, "geteuid") and profile_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("profile directory has an unexpected owner") + + def confirm_missing(self, name: str) -> None: + for _attempt in range(2): + self.validate_bindings(require_profile_owner=False) + try: + os.stat(name, dir_fd=self.fd, follow_symlinks=False) + except FileNotFoundError: + continue + except OSError as exc: + raise LaunchPayloadError("missing profile entry cannot be confirmed") from exc + raise LaunchPayloadError("profile entry appeared while absence was confirmed") + self.validate_bindings(require_profile_owner=False) + + def detach_fd(self) -> int: + result = self.fd + ancestors = self.descriptors[:-1] + self.descriptors = [] + for descriptor in reversed(ancestors): + with contextlib.suppress(OSError): + os.close(descriptor) + return result + + def close(self) -> None: + descriptors = self.descriptors + self.descriptors = [] + for descriptor in reversed(descriptors): + with contextlib.suppress(OSError): + os.close(descriptor) + + def _open_directory_at(parent_fd: int, name: str) -> int: try: before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) @@ -250,34 +353,56 @@ def _open_directory_at(parent_fd: int, name: str) -> int: opened_fd = os.open(name, _directory_flags(), dir_fd=parent_fd) except OSError as exc: raise LaunchPayloadError("profile path component cannot be opened safely") from exc - after = os.fstat(opened_fd) - if not stat.S_ISDIR(after.st_mode) or not _same_file(before, after): - os.close(opened_fd) - raise LaunchPayloadError("profile path changed while it was opened") - return opened_fd + try: + after = os.fstat(opened_fd) + if not stat.S_ISDIR(after.st_mode) or not _same_file(before, after): + raise LaunchPayloadError("profile path changed while it was opened") + return opened_fd + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise LaunchPayloadError("profile path could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(opened_fd) + raise -def _open_profile(profile_path: Path) -> int: +def _open_profile_chain(profile_path: Path) -> _OpenedProfile: if not profile_path.is_absolute() or not profile_path.parts or profile_path.parts[0] != "/": raise LaunchPayloadError("profile path must be absolute") try: - current_fd = os.open("/", _directory_flags()) + root_fd = os.open("/", _directory_flags()) except OSError as exc: raise LaunchPayloadError("filesystem root cannot be opened safely") from exc + descriptors = [root_fd] + names: list[str] = [] try: for component in profile_path.parts[1:]: - next_fd = _open_directory_at(current_fd, component) - os.close(current_fd) - current_fd = next_fd - profile_stat = os.fstat(current_fd) - if hasattr(os, "geteuid") and profile_stat.st_uid != os.geteuid(): - raise LaunchPayloadError("profile directory has an unexpected owner") - result = current_fd - current_fd = -1 - return result - finally: - if current_fd >= 0: - os.close(current_fd) + try: + next_fd = _open_directory_at(descriptors[-1], component) + except LaunchPayloadError as exc: + if _caused_by_missing_path(exc): + _OpenedProfile(descriptors, tuple(names)).confirm_missing(component) + raise + descriptors.append(next_fd) + names.append(component) + opened = _OpenedProfile(descriptors, tuple(names)) + opened.validate_bindings() + return opened + except BaseException: + for descriptor in reversed(descriptors): + with contextlib.suppress(OSError): + os.close(descriptor) + raise + + +def _open_profile(profile_path: Path) -> int: + return _open_profile_chain(profile_path).detach_fd() class _MarkerPublicationLock: @@ -400,14 +525,25 @@ def _open_logs(profile_fd: int, *, create: bool) -> int: logs_fd = _open_directory_at(profile_fd, "logs") except OSError as exc: raise LaunchPayloadError("marker directory cannot be opened safely") from exc - logs_stat = os.fstat(logs_fd) - if not stat.S_ISDIR(logs_stat.st_mode): - os.close(logs_fd) - raise LaunchPayloadError("marker directory is not a directory") - if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): - os.close(logs_fd) - raise LaunchPayloadError("marker directory has an unexpected owner") - return logs_fd + try: + logs_stat = os.fstat(logs_fd) + if not stat.S_ISDIR(logs_stat.st_mode): + raise LaunchPayloadError("marker directory is not a directory") + if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("marker directory has an unexpected owner") + return logs_fd + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise LaunchPayloadError("marker directory could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(logs_fd) + raise def _write_all(fd: int, data: bytes) -> None: @@ -544,11 +680,95 @@ def _open_regular_marker(logs_fd: int) -> tuple[int, os.stat_result]: marker_fd = os.open(MARKER_NAME, flags, dir_fd=logs_fd) except OSError as exc: raise LaunchPayloadError("marker cannot be opened safely") from exc - after = os.fstat(marker_fd) - if not stat.S_ISREG(after.st_mode) or not _same_file(before, after): - os.close(marker_fd) - raise LaunchPayloadError("marker changed while it was opened") - return marker_fd, after + try: + after = os.fstat(marker_fd) + if not stat.S_ISREG(after.st_mode) or not _same_file(before, after): + raise LaunchPayloadError("marker changed while it was opened") + return marker_fd, after + except LaunchPayloadError: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise + except OSError as exc: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise LaunchPayloadError("marker could not be validated safely") from exc + except BaseException: + with contextlib.suppress(OSError): + os.close(marker_fd) + raise + + +def _validate_open_marker_binding( + logs_fd: int, + marker_fd: int, + marker_stat: os.stat_result, +) -> os.stat_result: + try: + opened_marker = os.fstat(marker_fd) + current_marker = os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) + except OSError as exc: + raise LaunchPayloadError("marker changed while it was read") from exc + snapshots = (marker_stat, opened_marker, current_marker) + if not all(stat.S_ISREG(snapshot.st_mode) for snapshot in snapshots) or not all( + _same_file(marker_stat, snapshot) for snapshot in snapshots[1:] + ): + raise LaunchPayloadError("marker changed while it was read") + if hasattr(os, "geteuid") and any(snapshot.st_uid != os.geteuid() for snapshot in snapshots): + raise LaunchPayloadError("marker has an unexpected owner") + if any(snapshot.st_nlink != 1 for snapshot in snapshots): + raise LaunchPayloadError("marker has unexpected links") + return current_marker + + +def _validate_marker_bindings( + profile: _OpenedProfile, + logs_fd: int, + marker_fd: int, + marker_stat: os.stat_result, +) -> os.stat_result: + current_marker = marker_stat + for _attempt in range(2): + profile.validate_bindings() + logs_stat = _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) + if hasattr(os, "geteuid") and logs_stat.st_uid != os.geteuid(): + raise LaunchPayloadError("marker directory has an unexpected owner") + current_marker = _validate_open_marker_binding( + logs_fd, + marker_fd, + marker_stat, + ) + return current_marker + + +def _confirm_marker_missing(profile: _OpenedProfile, logs_fd: int) -> None: + for _attempt in range(2): + profile.validate_bindings() + _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) + try: + os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) + except FileNotFoundError: + continue + except OSError as exc: + raise LaunchPayloadError("missing marker cannot be confirmed") from exc + raise LaunchPayloadError("marker appeared while absence was confirmed") + profile.validate_bindings() + _validate_open_directory_binding( + profile.fd, + "logs", + logs_fd, + "marker directory", + ) def _remove_marker_if_owned_locked( diff --git a/zeus/private_io.py b/zeus/private_io.py index bf0295d..da8ee34 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -6,6 +6,7 @@ from collections.abc import Iterator from contextlib import AbstractContextManager, contextmanager, suppress from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import BinaryIO, cast @@ -14,6 +15,7 @@ _SECURITY_FLAGS = ("O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC", "O_NONBLOCK") _REQUIRED_FUNCTIONS = ( "close", + "dup", "fchmod", "fdopen", "fstat", @@ -38,6 +40,12 @@ class _PrivatePathMissing(Exception): pass +class _DirectoryRequirement(Enum): + identity = "identity" + exact_private = "exact_private" + inspect_private = "inspect_private" + + @dataclass(frozen=True) class _Platform: euid: int @@ -51,13 +59,20 @@ class _Platform: class _OpenedDirectoryPath: descriptors: tuple[int, ...] names: tuple[str, ...] + requirements: tuple[_DirectoryRequirement, ...] + euid: int @property def fd(self) -> int: return self.descriptors[-1] def validate_bindings(self) -> None: - _validate_directory_bindings(self.descriptors, self.names) + _validate_directory_bindings( + self.descriptors, + self.names, + self.requirements, + self.euid, + ) def confirm_missing(self, name: str) -> None: self.validate_bindings() @@ -73,6 +88,68 @@ def confirm_missing(self, name: str) -> None: raise UnsafeFileError("private path appeared while absence was confirmed") +@dataclass(frozen=True) +class _OpenedPrivateFile: + fd: int + parent_fd: int + name: str + identity: os.stat_result + platform: _Platform + + def validate_binding(self) -> None: + try: + opened = os.fstat(self.fd) + current = os.lstat(self.name, dir_fd=self.parent_fd) + snapshots = (self.identity, opened, current) + for snapshot in snapshots: + _validate_file_snapshot(snapshot, self.platform) + if not _same_files(snapshots): + raise UnsafeFileError("private file changed while it was used") + if any(stat.S_IMODE(snapshot.st_mode) != _FILE_MODE for snapshot in (opened, current)): + raise UnsafeFileError("private file does not have private permissions") + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("private file binding changed while it was used") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError("private file binding cannot be inspected safely") from exc + + +@dataclass(frozen=True) +class _PinnedPrivateDirectory: + fd: int + identity: os.stat_result + platform: _Platform + + def validate_at(self, path: Path) -> None: + parts = _validate_path(path, file_path=False) + with _open_directory_path( + parts, + create=False, + tighten=False, + platform=self.platform, + ) as current: + try: + pinned = os.fstat(self.fd) + installed = os.fstat(current.fd) + snapshots = (self.identity, pinned, installed) + _validate_directory_snapshots(snapshots, "pinned private directory") + _validate_directory_requirement( + snapshots, + _DirectoryRequirement.exact_private, + self.platform.euid, + "pinned private directory", + ) + except UnsafeFileError: + raise + except OSError as exc: + raise UnsafeFileError("pinned private directory is unavailable") from exc + except (TypeError, ValueError) as exc: + raise UnsafeFileError( + "pinned private directory cannot be inspected safely" + ) from exc + + def _required_flag(name: str, *, allow_zero: bool = False) -> int: value = getattr(os, name, None) if type(value) is not int or (not allow_zero and value == 0): @@ -187,20 +264,31 @@ def _validate_directory_snapshots( def _validate_directory_bindings( descriptors: tuple[int, ...], names: tuple[str, ...], + requirements: tuple[_DirectoryRequirement, ...], + euid: int, ) -> None: - if len(descriptors) != len(names) + 1: + if len(descriptors) != len(names) + 1 or len(requirements) != len(descriptors): raise UnsafeFileError("private directory descriptor chain is invalid") try: root_snapshots = (os.fstat(descriptors[0]), os.lstat("/")) _validate_directory_snapshots(root_snapshots, "filesystem root") - for parent_fd, name, directory_fd in zip( + _validate_directory_requirement( + root_snapshots, + requirements[0], + euid, + "filesystem root", + ) + for parent_fd, name, directory_fd, requirement in zip( descriptors[:-1], names, descriptors[1:], + requirements[1:], strict=True, ): + description = f"private path component {name!r}" snapshots = (os.fstat(directory_fd), os.lstat(name, dir_fd=parent_fd)) - _validate_directory_snapshots(snapshots, f"private path component {name!r}") + _validate_directory_snapshots(snapshots, description) + _validate_directory_requirement(snapshots, requirement, euid, description) except UnsafeFileError: raise except OSError as exc: @@ -209,14 +297,22 @@ def _validate_directory_bindings( raise UnsafeFileError("private directory path cannot be inspected safely") from exc -def _inspect_private_directory_snapshots( +def _validate_directory_requirement( snapshots: tuple[os.stat_result, ...], - platform: _Platform, + requirement: _DirectoryRequirement, + euid: int, description: str, ) -> None: - if not all(snapshot.st_uid == platform.euid for snapshot in snapshots): + if requirement is _DirectoryRequirement.identity: + return + if not all(snapshot.st_uid == euid for snapshot in snapshots): raise UnsafeFileError(f"{description} has an unexpected owner") - if any(stat.S_IMODE(snapshot.st_mode) & 0o077 for snapshot in snapshots): + modes = tuple(stat.S_IMODE(snapshot.st_mode) for snapshot in snapshots) + if requirement is _DirectoryRequirement.exact_private: + unsafe = any(mode != _DIRECTORY_MODE for mode in modes) + else: + unsafe = any(mode & 0o077 for mode in modes) + if unsafe: raise UnsafeFileError(f"{description} does not have private permissions") @@ -279,7 +375,7 @@ def _open_directory_at( private: bool, tighten: bool, platform: _Platform, -) -> int: +) -> tuple[int, _DirectoryRequirement]: description = f"private path component {name!r}" created = False try: @@ -326,9 +422,18 @@ def _open_directory_at( platform, description, ) + requirement = _DirectoryRequirement.exact_private elif private: - _inspect_private_directory_snapshots(snapshots, platform, description) - return directory_fd + requirement = _DirectoryRequirement.inspect_private + _validate_directory_requirement( + snapshots, + requirement, + platform.euid, + description, + ) + else: + requirement = _DirectoryRequirement.identity + return directory_fd, requirement except UnsafeFileError: _close_suppressing_error(directory_fd) raise @@ -351,10 +456,11 @@ def _open_directory_path( ) -> Iterator[_OpenedDirectoryPath]: descriptors = [_open_root(platform)] names: list[str] = [] + requirements = [_DirectoryRequirement.identity] try: for index, component in enumerate(parts): try: - next_fd = _open_directory_at( + next_fd, requirement = _open_directory_at( descriptors[-1], component, create=create, @@ -364,16 +470,27 @@ def _open_directory_path( platform=platform, ) except _PrivatePathMissing: - _OpenedDirectoryPath(tuple(descriptors), tuple(names)).confirm_missing(component) + _OpenedDirectoryPath( + tuple(descriptors), + tuple(names), + tuple(requirements), + platform.euid, + ).confirm_missing(component) raise descriptors.append(next_fd) names.append(component) + requirements.append(requirement) except BaseException: for descriptor in reversed(descriptors): _close_suppressing_error(descriptor) raise - opened = _OpenedDirectoryPath(tuple(descriptors), tuple(names)) + opened = _OpenedDirectoryPath( + tuple(descriptors), + tuple(names), + tuple(requirements), + platform.euid, + ) try: yield opened except BaseException: @@ -414,7 +531,7 @@ def _open_private_file_at( append: bool, create: bool, platform: _Platform, -) -> int | None: +) -> _OpenedPrivateFile | None: before: os.stat_result | None try: before = os.lstat(name, dir_fd=parent_fd) @@ -457,7 +574,13 @@ def _open_private_file_at( raise UnsafeFileError("private file changed while it was opened") if any(stat.S_IMODE(snapshot.st_mode) != _FILE_MODE for snapshot in (tightened, final)): raise UnsafeFileError("private file does not have private permissions") - return file_fd + return _OpenedPrivateFile( + fd=file_fd, + parent_fd=parent_fd, + name=name, + identity=tightened, + platform=platform, + ) except UnsafeFileError: _close_suppressing_error(file_fd) raise @@ -476,15 +599,16 @@ def _private_append_context( platform: _Platform, ) -> Iterator[BinaryIO]: with _open_directory_path(parent_parts, create=True, platform=platform) as parent: - file_fd = _open_private_file_at( + opened_file = _open_private_file_at( parent.fd, name, append=True, create=True, platform=platform, ) - if file_fd is None: + if opened_file is None: raise UnsafeFileError("private file was not created") + file_fd = opened_file.fd try: try: handle = cast(BinaryIO, os.fdopen(file_fd, "ab", buffering=0)) @@ -502,6 +626,12 @@ def _private_append_context( handle.close() raise else: + try: + opened_file.validate_binding() + except BaseException: + with suppress(OSError): + handle.close() + raise try: handle.close() except OSError as exc: @@ -543,16 +673,17 @@ def read_private_tail(path: Path, max_bytes: int) -> bytes: with _open_directory_path( parts[:-1], create=False, missing_ok=True, platform=platform ) as parent: - file_fd = _open_private_file_at( + opened_file = _open_private_file_at( parent.fd, parts[-1], append=False, create=False, platform=platform, ) - if file_fd is None: + if opened_file is None: parent.confirm_missing(parts[-1]) return b"" + file_fd = opened_file.fd try: try: end = os.lseek(file_fd, 0, os.SEEK_END) @@ -569,6 +700,7 @@ def read_private_tail(path: Path, max_bytes: int) -> bytes: result = b"".join(chunks) except OSError as exc: raise UnsafeFileError("private file tail could not be read") from exc + opened_file.validate_binding() except BaseException: _close_suppressing_error(file_fd) raise @@ -586,6 +718,40 @@ def validate_private_directory(path: Path) -> None: pass +@contextmanager +def pin_private_directory(path: Path) -> Iterator[_PinnedPrivateDirectory]: + parts = _validate_path(path, file_path=False) + platform = _require_platform() + pinned_fd = -1 + with _open_directory_path(parts, create=False, platform=platform) as opened: + try: + pinned_fd = os.dup(opened.fd) + identity = os.fstat(pinned_fd) + _validate_directory_snapshots((identity,), "pinned private directory") + _validate_directory_requirement( + (identity,), + _DirectoryRequirement.exact_private, + platform.euid, + "pinned private directory", + ) + except UnsafeFileError: + if pinned_fd >= 0: + _close_suppressing_error(pinned_fd) + raise + except (OSError, TypeError, ValueError) as exc: + if pinned_fd >= 0: + _close_suppressing_error(pinned_fd) + raise UnsafeFileError("private directory could not be pinned safely") from exc + pinned = _PinnedPrivateDirectory(pinned_fd, identity, platform) + try: + yield pinned + except BaseException: + _close_suppressing_error(pinned_fd) + raise + else: + _close_descriptor(pinned_fd, "pinned private directory") + + def ensure_private_directory(path: Path) -> None: parts = _validate_path(path, file_path=False) platform = _require_platform() diff --git a/zeus/renderer.py b/zeus/renderer.py index 420d56c..864a1e1 100644 --- a/zeus/renderer.py +++ b/zeus/renderer.py @@ -14,7 +14,12 @@ from zeus.envfile import dump_env from zeus.models import BotCreateRequest, BotRecord, HermesTemplate, TemplateError -from zeus.private_io import nofollow_absolute_path, validate_private_directory +from zeus.private_io import ( + UnsafeFileError, + nofollow_absolute_path, + pin_private_directory, + validate_private_directory, +) _LOG = logging.getLogger(__name__) @@ -97,7 +102,19 @@ def transaction( staging = Path(tempfile.mkdtemp(prefix=f".{request.bot_id}.staging-", dir=profiles_root)) try: _write_staged_profile(staging, profile, rendered_files) - backup = _install_staged_profile(staging, profile) + backup: Path | None = None + installed = False + try: + with pin_private_directory(nofollow_absolute_path(staging / "logs")) as pinned: + backup = _install_staged_profile(staging, profile) + installed = True + pinned.validate_at(nofollow_absolute_path(profile / "logs")) + except UnsafeFileError as exc: + if installed: + _rollback_installed_profile(profile, backup) + raise TemplateError( + "installed profile logs directory could not be verified safely" + ) from exc finally: try: _remove_path(staging) diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 6851292..460b9a1 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -33,16 +33,17 @@ ) from zeus.fs_utils import atomic_write_json from zeus.gateway_launcher import ( - MARKER_NAME, MAX_PAYLOAD_BYTES, LaunchPayloadError, + _confirm_marker_missing, _is_owned_runtime_marker, _open_logs, - _open_profile, + _open_profile_chain, _open_regular_marker, _read_bounded_file, _reject_duplicate_keys, _remove_marker_if_owned_locked, + _validate_marker_bindings, marker_publication_lock, remove_marker_if_owned, ) @@ -1220,9 +1221,10 @@ def _read_strict_runtime_marker( "untrusted", reason="registered profile path does not match the trusted Hermes profile", ) - profile_fd = logs_fd = marker_fd = -1 + profile = None + logs_fd = marker_fd = -1 try: - profile_fd = _open_profile(profile_path) + profile = _open_profile_chain(profile_path) except (OSError, ValueError) as exc: if isinstance(exc.__cause__, FileNotFoundError): return _MarkerObservation("missing", reason="marker is missing") @@ -1231,9 +1233,16 @@ def _read_strict_runtime_marker( ) try: try: - logs_fd = _open_logs(profile_fd, create=False) + logs_fd = _open_logs(profile.fd, create=False) except ValueError as exc: if isinstance(exc.__cause__, FileNotFoundError): + try: + profile.confirm_missing("logs") + except (OSError, ValueError) as confirm_error: + return _MarkerObservation( + "untrusted", + reason=f"marker directory absence is untrusted: {confirm_error}", + ) return _MarkerObservation("missing", reason="marker is missing") return _MarkerObservation( "untrusted", reason=f"marker directory cannot be opened safely: {exc}" @@ -1241,11 +1250,31 @@ def _read_strict_runtime_marker( try: marker_fd, marker_stat = _open_regular_marker(logs_fd) raw = _read_bounded_file(marker_fd) + marker_stat = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) except FileNotFoundError: + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return _MarkerObservation( + "untrusted", + reason=f"marker absence is untrusted: {confirm_error}", + ) return _MarkerObservation("missing", reason="marker is missing") except ValueError as exc: if isinstance(exc.__cause__, FileNotFoundError): + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return _MarkerObservation( + "untrusted", + reason=f"marker absence is untrusted: {confirm_error}", + ) return _MarkerObservation("missing", reason="marker is missing") return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: @@ -1253,9 +1282,12 @@ def _read_strict_runtime_marker( except FileNotFoundError: return _MarkerObservation("missing", reason="marker is missing") finally: - for fd in (marker_fd, logs_fd, profile_fd): + for fd in (marker_fd, logs_fd): if fd >= 0: - os.close(fd) + with contextlib.suppress(OSError): + os.close(fd) + if profile is not None: + profile.close() if type(value) is not dict: return _MarkerObservation("untrusted", reason="marker is not an object") if marker_stat.st_nlink != 1: @@ -3360,14 +3392,24 @@ def _remove_pid_marker(self, profile_path: str) -> None: def _read_pid_marker(self, profile_path: str) -> dict[str, object]: safe_profile_path = _nofollow_absolute_path(Path(profile_path)) - profile_fd = logs_fd = marker_fd = -1 + profile = None + logs_fd = marker_fd = -1 try: try: - profile_fd = _open_profile(safe_profile_path) - logs_fd = _open_logs(profile_fd, create=False) + profile = _open_profile_chain(safe_profile_path) + logs_fd = _open_logs(profile.fd, create=False) marker_fd, marker_stat = _open_regular_marker(logs_fd) except (LaunchPayloadError, OSError, ValueError) as exc: if _caused_by_missing_path(exc): + try: + if profile is not None and logs_fd >= 0: + _confirm_marker_missing(profile, logs_fd) + elif profile is not None: + profile.confirm_missing("logs") + except (LaunchPayloadError, OSError, ValueError) as confirm_error: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from confirm_error return {"exists": False} raise UnsafeFileError("PID marker cannot be opened safely") from exc if marker_stat.st_uid != os.geteuid() or marker_stat.st_nlink != 1: @@ -3375,30 +3417,47 @@ def _read_pid_marker(self, profile_path: str) -> dict[str, object]: marker_mode = f"{stat.S_IMODE(marker_stat.st_mode):04o}" try: raw = _read_bounded_file(marker_fd) - current_marker = os.stat(MARKER_NAME, dir_fd=logs_fd, follow_symlinks=False) - current_logs = os.stat("logs", dir_fd=profile_fd, follow_symlinks=False) - opened_logs = os.fstat(logs_fd) except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + try: + _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + except (LaunchPayloadError, OSError, TypeError, ValueError) as binding_error: + raise UnsafeFileError( + "PID marker changed while it was inspected" + ) from binding_error return { "exists": True, "valid": False, "mode": marker_mode, "error": str(exc), } + try: + current_marker = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + raise UnsafeFileError("PID marker changed while it was inspected") from exc if ( not stat.S_ISREG(current_marker.st_mode) or current_marker.st_uid != os.geteuid() or current_marker.st_nlink != 1 or not _same_identity(marker_stat, current_marker) - or not stat.S_ISDIR(current_logs.st_mode) - or not _same_identity(opened_logs, current_logs) ): raise UnsafeFileError("PID marker changed while it was inspected") finally: - for descriptor in (marker_fd, logs_fd, profile_fd): + for descriptor in (marker_fd, logs_fd): if descriptor >= 0: with contextlib.suppress(OSError): os.close(descriptor) + if profile is not None: + profile.close() try: payload = json.loads(raw.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: From beef5b7a236cf56fa4894795e38e61ec9b47a78e Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:50:28 +0200 Subject: [PATCH 12/49] security: sanitize and bound audit events --- tests/test_lifecycle_ledger.py | 114 ++++++++++++++ tests/test_supervisor_cli_api.py | 126 ++++++++++++++- zeus/lifecycle.py | 122 ++------------- zeus/logging_utils.py | 28 +--- zeus/reconciliation.py | 14 +- zeus/sanitization.py | 258 +++++++++++++++++++++++++++++++ zeus/state.py | 63 ++++---- 7 files changed, 559 insertions(+), 166 deletions(-) create mode 100644 zeus/sanitization.py diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index c9169ac..223aee5 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -12,6 +12,7 @@ from zeus.lifecycle import LifecycleEvent, LifecycleEventInput, serialize_lifecycle_details from zeus.models import BotRecord, BotStatus, DesiredState +from zeus.sanitization import sanitize_details from zeus.state import StateStore @@ -499,6 +500,119 @@ def test_event_details_normalize_forbidden_names_and_are_json_compatible(self) - ) json.dumps(stored.to_dict(), sort_keys=True) + def test_sanitizer_bounds_cycles_nonfinite_numbers_and_free_text_secrets(self) -> None: + secret = "lifecycle-sentinel-9a21" + cycle: list[object] = [] + cycle.append(cycle) + deep: object = "leaf" + for _ in range(12): + deep = {"next": deep} + + sanitized = sanitize_details( + { + "api_key": secret, + "messages": [ + f"API_KEY={secret}", + f"authorization=Bearer {secret}", + f"request failed with Bearer {secret}", + ], + "cycle": cycle, + "deep": deep, + "nan": float("nan"), + "positive_infinity": float("inf"), + "negative_infinity": float("-inf"), + } + ) + encoded = json.dumps( + sanitized, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + self.assertFalse(secret.encode("utf-8") in encoded) + self.assertLessEqual(len(encoded), 8192) + self.assertIsNone(sanitized["nan"]) # type: ignore[index] + self.assertIsNone(sanitized["positive_infinity"]) # type: ignore[index] + self.assertIsNone(sanitized["negative_infinity"]) # type: ignore[index] + self.assertIn(b"[cycle]", encoded) + self.assertIn(b"[truncated]", encoded) + + oversized = json.dumps( + sanitize_details({"message": "Δ" * 20_000}), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + self.assertLessEqual(len(oversized), 8192) + + bounded_collection = sanitize_details({"items": list(range(1_000))}) + self.assertLessEqual(len(bounded_collection["items"]), 64) # type: ignore[index] + + def test_sanitizer_redacts_values_for_oversized_sensitive_keys(self) -> None: + secret = "long-key-sentinel-cb30" + sanitized = sanitize_details({f"{'x' * 256}_api_key": secret}) + encoded = json.dumps(sanitized, allow_nan=False).encode("utf-8") + + self.assertFalse(secret.encode("utf-8") in encoded) + + def test_sanitizer_never_invokes_hostile_string_or_repr_methods(self) -> None: + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + class HostileString(str): + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + hostile_key = Hostile() + hostile_value = Hostile() + + sanitized = sanitize_details( + { + "value": hostile_value, + "text": HostileString("TOKEN=hostile-string-sentinel"), + hostile_key: "untrusted key value", + } + ) + + self.assertEqual("[unsupported]", sanitized["value"]) # type: ignore[index] + self.assertEqual("[unsupported]", sanitized["text"]) # type: ignore[index] + json.dumps(sanitized, allow_nan=False) + + def test_lifecycle_text_fields_are_redacted_before_sqlite_persistence(self) -> None: + secret = "event-sentinel-84c2" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="a" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=f"readiness failed with Bearer {secret}", + error_code="readiness_failed", + error_message=f"cleanup failed: API_KEY={secret}", + ) + + store.upsert_bot_with_event(self._record(root), event=event) + + with closing(sqlite3.connect(store.database_path)) as conn: + row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + assert row is not None + persisted = "\n".join(str(value) for value in row if value is not None) + self.assertFalse(secret in persisted) + def test_upsert_bot_with_event_rolls_back_projection_when_event_insert_fails(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index c047e3d..dddf449 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -14,7 +14,7 @@ import time import unittest import uuid -from contextlib import redirect_stderr, redirect_stdout +from contextlib import closing, redirect_stderr, redirect_stdout from datetime import UTC, datetime, timedelta from pathlib import Path from unittest.mock import patch @@ -33,7 +33,12 @@ from zeus.private_io import UnsafeFileError from zeus.process_lock import LockTimeoutError from zeus.readiness import ReadinessResult -from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunSummary +from zeus.reconciliation import ( + BotReconcileResult, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, +) from zeus.state import StateStore from zeus.supervisor import ( Supervisor, @@ -2980,6 +2985,123 @@ def test_audit_log_records_lifecycle_and_redacts_secret_like_fields(self) -> Non self.assertNotIn("plain-secret", store.audit_log_path().read_text(encoding="utf-8")) self.assertIn("[redacted]", store.audit_log_path().read_text(encoding="utf-8")) + def test_audit_sanitization_is_bounded_recursive_and_best_effort(self) -> None: + secret = "audit-sentinel-71b4" + + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + store.append_audit_event( + "bot.start_registration_failed", + cleanup_errors=[ + f"cleanup failed: API_KEY={secret}", + {"message": f"authorization=Bearer {secret}"}, + ], + readiness_message=f"probe returned Bearer {secret}", + nonfinite=float("nan"), + hostile=Hostile(), + ) + store.append_audit_event("bot.oversized", oversized="Δ" * 20_000) + store.append_audit_event( + "e" * 2_048, + first="x" * 1_800, + second="x" * 1_800, + third="x" * 1_800, + fourth="x" * 1_800, + ) + + raw = store.audit_log_path().read_bytes() + lines = raw.splitlines() + payload = json.loads(lines[0]) + self.assertFalse(secret.encode("utf-8") in raw) + self.assertNotIn(b"NaN", raw) + self.assertTrue(all(len(line) <= 8192 for line in lines)) + self.assertEqual("bot.start_registration_failed", payload["event"]) + self.assertEqual("[unsupported]", payload["hostile"]) + self.assertIsNone(payload["nonfinite"]) + self.assertTrue(json.loads(lines[1])["truncated"]) + self.assertTrue(json.loads(lines[2])["truncated"]) + + def test_projection_and_reconcile_messages_are_sanitized_before_persistence(self) -> None: + secret = "projection-sentinel-2f96" + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=f"readiness API_KEY={secret}", + last_transition_reason=f"cleanup Bearer {secret}", + ) + ) + initial_record = store.get_bot("coder") + assert initial_record is not None + self.assertFalse(secret in (initial_record.last_error or "")) + self.assertFalse(secret in (initial_record.last_transition_reason or "")) + store.update_lifecycle_state( + "coder", + BotStatus.failed, + last_error=f"updated readiness API_KEY={secret}", + last_transition_reason=f"updated cleanup Bearer {secret}", + ) + run = ReconcileRunStart( + run_id="security-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message=f"readiness API_KEY={secret}; cleanup Bearer {secret}", + error_code="readiness_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + loaded_record = store.get_bot("coder") + loaded_run = store.get_reconcile_run(run.run_id) + assert loaded_record is not None + assert loaded_run is not None + self.assertFalse(secret in (loaded_record.last_error or "")) + self.assertFalse(secret in (loaded_record.last_transition_reason or "")) + self.assertFalse(secret in result.message) + self.assertFalse(secret in loaded_run.results[0].message) + self.assertEqual(result, loaded_run.results[0]) + with closing(sqlite3.connect(store.database_path)) as conn: + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert bot_row is not None + assert reconcile_row is not None + persisted = "\n".join((*bot_row, *reconcile_row)) + self.assertFalse(secret in persisted) + def test_audit_write_failure_does_not_break_lifecycle_action(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/zeus/lifecycle.py b/zeus/lifecycle.py index 87a3503..f54c18b 100644 --- a/zeus/lifecycle.py +++ b/zeus/lifecycle.py @@ -1,114 +1,33 @@ from __future__ import annotations import json -import math -import re from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime from types import MappingProxyType -from zeus.logging_utils import redact_secrets +from zeus.sanitization import JSONValue, sanitize_details, sanitize_text -MAX_DETAILS_JSON_LENGTH = 8192 -MAX_DETAIL_DEPTH = 4 -MAX_DETAIL_ITEMS = 32 -MAX_DETAIL_STRING_LENGTH = 2048 MAX_EVENT_TEXT_LENGTH = 2048 -_CAMEL_ACRONYM_BOUNDARY_RE = re.compile(r"([A-Z]+)([A-Z][a-z])") -_CAMEL_WORD_BOUNDARY_RE = re.compile(r"([a-z0-9])([A-Z])") -_NON_NAME_CHARACTER_RE = re.compile(r"[^a-z0-9]+") -_SECRET_NAME_PARTS = frozenset({"apikey", "key", "token", "secret", "password"}) -_FORBIDDEN_DETAIL_NAMES = frozenset( - { - "authorization", - "authorization_header", - "authorization_headers", - "header", - "headers", - "request_body", - "response_body", - "body", - "raw_query", - "query_string", - "query", - "forwarded_for", - "x_forwarded_for", - "client_address", - "client_ip", - "client_port", - "remote_addr", - "traceback", - "exception_trace", - "idempotency_key", - } -) - def _safe_text(value: str, *, maximum: int = MAX_EVENT_TEXT_LENGTH) -> str: - return redact_secrets(value)[:maximum] - - -def _normalize_detail_name(key: str) -> str: - separated = _CAMEL_ACRONYM_BOUNDARY_RE.sub(r"\1_\2", key) - separated = _CAMEL_WORD_BOUNDARY_RE.sub(r"\1_\2", separated).lower() - return _NON_NAME_CHARACTER_RE.sub("_", separated).strip("_") - - -def _contains_name(normalized: str, candidate: str) -> bool: - return f"_{candidate}_" in f"_{normalized}_" - - -def _is_forbidden_detail_name(key: str) -> bool: - normalized = _normalize_detail_name(key) - if any(_contains_name(normalized, name) for name in _FORBIDDEN_DETAIL_NAMES): - return True - return any(_contains_name(normalized, part) for part in _SECRET_NAME_PARTS) - - -def _safe_detail_value(key: str, value: object, *, depth: int) -> object: - if _is_forbidden_detail_name(key): - return "[redacted]" - if depth >= MAX_DETAIL_DEPTH: - return "[truncated]" - if value is None or isinstance(value, bool | int): - return value - if isinstance(value, float): - return value if math.isfinite(value) else None - if isinstance(value, str): - return _safe_text(value, maximum=MAX_DETAIL_STRING_LENGTH) - if isinstance(value, Mapping): - result: dict[str, object] = {} - for raw_child_key, child_value in list(value.items())[:MAX_DETAIL_ITEMS]: - child_key = _safe_text(str(raw_child_key), maximum=128) - result[child_key] = _safe_detail_value( - child_key, - child_value, - depth=depth + 1, - ) - return result - if isinstance(value, list | tuple): - return [_safe_detail_value(key, item, depth=depth + 1) for item in value[:MAX_DETAIL_ITEMS]] - return "[unsupported]" - - -def safe_lifecycle_details(details: Mapping[str, object]) -> dict[str, object]: - result: dict[str, object] = {} - for raw_key, value in list(details.items())[:MAX_DETAIL_ITEMS]: - key = _safe_text(str(raw_key), maximum=128) - result[key] = _safe_detail_value(key, value, depth=0) - if len(json.dumps(result, sort_keys=True, separators=(",", ":"))) > MAX_DETAILS_JSON_LENGTH: - return {"truncated": True} + return sanitize_text(value, max_length=maximum) + + +def safe_lifecycle_details(details: Mapping[str, object]) -> dict[str, JSONValue]: + result = sanitize_details(details) + if type(result) is not dict: + raise TypeError("lifecycle event details must be a mapping") return result -def _freeze_detail_value(value: object) -> object: - if isinstance(value, Mapping): +def _freeze_detail_value(value: JSONValue) -> object: + if type(value) is dict: return MappingProxyType( - {str(key): _freeze_detail_value(child_value) for key, child_value in value.items()} + {key: _freeze_detail_value(child_value) for key, child_value in value.items()} ) - if isinstance(value, list | tuple): + if type(value) is list: return tuple(_freeze_detail_value(child_value) for child_value in value) return value @@ -120,27 +39,20 @@ def freeze_lifecycle_details(details: Mapping[str, object]) -> Mapping[str, obje return frozen -def _thaw_detail_value(value: object) -> object: - if isinstance(value, Mapping): - return {str(key): _thaw_detail_value(child_value) for key, child_value in value.items()} - if isinstance(value, list | tuple): - return [_thaw_detail_value(child_value) for child_value in value] - return value - - -def thaw_lifecycle_details(details: Mapping[str, object]) -> dict[str, object]: - return {str(key): _thaw_detail_value(value) for key, value in details.items()} +def thaw_lifecycle_details(details: Mapping[str, object]) -> dict[str, JSONValue]: + return safe_lifecycle_details(details) def serialize_lifecycle_details(details: Mapping[str, object]) -> str: return json.dumps( - thaw_lifecycle_details(safe_lifecycle_details(details)), + safe_lifecycle_details(details), sort_keys=True, separators=(",", ":"), + allow_nan=False, ) -def deserialize_lifecycle_details(value: str) -> dict[str, object]: +def deserialize_lifecycle_details(value: str) -> dict[str, JSONValue]: parsed = json.loads(value) if not isinstance(parsed, dict): raise ValueError("lifecycle event details must be a JSON object") diff --git a/zeus/logging_utils.py b/zeus/logging_utils.py index 08d3ddf..6ef7502 100644 --- a/zeus/logging_utils.py +++ b/zeus/logging_utils.py @@ -1,35 +1,9 @@ from __future__ import annotations -import re from pathlib import Path from zeus.private_io import read_private_tail - -SECRET_KV_RE = re.compile( - r"""(?ix) - (?P["']?) - (?P[A-Z0-9_.-]*(?:API[_-]?KEY|KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_.-]*) - (?P=prefix) - (?P\s*[:=]\s*) - (?P - "(?:[^"\\]|\\.)*" | - '(?:[^'\\]|\\.)*' | - [^\s,}]+ - ) - """ -) -BEARER_RE = re.compile(r"(?i)(Authorization\s*:\s*Bearer\s+)([A-Za-z0-9._~+/=-]+)") - - -def redact_secrets(text: str) -> str: - text = BEARER_RE.sub(r"\1[redacted]", text) - return SECRET_KV_RE.sub( - lambda match: ( - f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" - f"{match.group('sep')}[redacted]" - ), - text, - ) +from zeus.sanitization import redact_secrets def tail_file(path: Path, max_bytes: int = 20_000) -> str: diff --git a/zeus/reconciliation.py b/zeus/reconciliation.py index fea3b1b..0e52453 100644 --- a/zeus/reconciliation.py +++ b/zeus/reconciliation.py @@ -11,6 +11,7 @@ from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState from zeus.process_lock import BotProcessLock, LockTimeoutError +from zeus.sanitization import sanitize_text MAX_RECONCILE_ID_LENGTH = 128 MAX_RECONCILE_TEXT_LENGTH = 2048 @@ -48,7 +49,7 @@ def _validate_required_text(value: str, name: str, *, maximum: int) -> str: def _bound_text(value: str, name: str) -> str: - if not isinstance(value, str): + if type(value) is not str: raise ValueError(f"{name} must be a string") return value[:MAX_RECONCILE_TEXT_LENGTH] @@ -56,7 +57,7 @@ def _bound_text(value: str, name: str) -> str: def _bound_optional_text(value: str | None, name: str) -> str | None: if value is None: return None - if not isinstance(value, str) or not value: + if type(value) is not str or not value: raise ValueError(f"{name} must be a non-empty string when provided") return value[:MAX_RECONCILE_TEXT_LENGTH] @@ -144,7 +145,14 @@ def __post_init__(self) -> None: object.__setattr__(self, "observed_status", observed_status) object.__setattr__(self, "pid", _validate_positive_optional_int(self.pid, "pid")) object.__setattr__(self, "action", _bound_text(self.action, "action")) - object.__setattr__(self, "message", _bound_text(self.message, "message")) + object.__setattr__( + self, + "message", + sanitize_text( + _bound_text(self.message, "message"), + max_length=MAX_RECONCILE_TEXT_LENGTH, + ), + ) object.__setattr__( self, "error_code", diff --git a/zeus/sanitization.py b/zeus/sanitization.py new file mode 100644 index 0000000..6a79597 --- /dev/null +++ b/zeus/sanitization.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import json +import math +import re +from types import MappingProxyType +from typing import TypeAlias + +JSONScalar: TypeAlias = None | bool | int | float | str +JSONValue: TypeAlias = JSONScalar | list["JSONValue"] | dict[str, "JSONValue"] + +MAX_SANITIZED_JSON_BYTES = 8192 +MAX_DETAIL_KEY_LENGTH = 128 +MAX_EVENT_TEXT_LENGTH = 2048 + +REDACTED_VALUE = "[redacted]" +TRUNCATED_VALUE = "[truncated]" +CYCLE_VALUE = "[cycle]" +UNSUPPORTED_VALUE = "[unsupported]" + +_CAMEL_ACRONYM_BOUNDARY_RE = re.compile(r"([A-Z]+)([A-Z][a-z])") +_CAMEL_WORD_BOUNDARY_RE = re.compile(r"([a-z0-9])([A-Z])") +_NON_NAME_CHARACTER_RE = re.compile(r"[^a-z0-9]+") +_SECRET_NAME_PARTS = frozenset({"apikey", "key", "token", "secret", "password"}) +_FORBIDDEN_DETAIL_NAMES = frozenset( + { + "authorization", + "authorization_header", + "authorization_headers", + "header", + "headers", + "request_body", + "response_body", + "body", + "raw_query", + "query_string", + "query", + "forwarded_for", + "x_forwarded_for", + "client_address", + "client_ip", + "client_port", + "remote_addr", + "traceback", + "exception_trace", + "idempotency_key", + } +) + +_SECRET_KV_RE = re.compile( + r"""(?ix) + (?P["']?) + (?P + [A-Z0-9_.-]*(?:API[_-]?KEY|KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_.-]* + | AUTHORIZATION + ) + (?P=prefix) + (?P\s*[:=]\s*) + (?P + "(?:[^"\\]|\\.)*" | + '(?:[^'\\]|\\.)*' | + [^\s,}]+ + ) + """ +) +_BEARER_RE = re.compile(r"(?i)(\bBearer\s+)([A-Za-z0-9._~+/=-]+)") + + +def redact_secrets(text: str) -> str: + """Redact common secret assignments and bearer credentials from free text.""" + + redacted = _BEARER_RE.sub(r"\1[redacted]", text) + return _SECRET_KV_RE.sub( + lambda match: ( + f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" + f"{match.group('sep')}[redacted]" + ), + redacted, + ) + + +def sanitize_text(value: str, *, max_length: int = MAX_EVENT_TEXT_LENGTH) -> str: + """Return bounded, redacted text without coercing non-string objects.""" + + if type(value) is not str: + return UNSUPPORTED_VALUE + if type(max_length) is not int or max_length < 0: + raise ValueError("max_length must be a non-negative integer") + return redact_secrets(value[:max_length])[:max_length] + + +def _normalize_detail_name(key: str) -> str: + separated = _CAMEL_ACRONYM_BOUNDARY_RE.sub(r"\1_\2", key) + separated = _CAMEL_WORD_BOUNDARY_RE.sub(r"\1_\2", separated).lower() + return _NON_NAME_CHARACTER_RE.sub("_", separated).strip("_") + + +def _contains_name(normalized: str, candidate: str) -> bool: + return f"_{candidate}_" in f"_{normalized}_" + + +def _is_forbidden_detail_name(key: str) -> bool: + normalized = _normalize_detail_name(key) + if any(_contains_name(normalized, name) for name in _FORBIDDEN_DETAIL_NAMES): + return True + return any(_contains_name(normalized, part) for part in _SECRET_NAME_PARTS) + + +def _sanitize_key(value: object, *, max_length: int) -> str | None: + if type(value) is str: + return sanitize_text(value, max_length=max_length) + if type(value) is int: + try: + return str(value)[:max_length] + except ValueError: + return None + if type(value) is float and math.isfinite(value): + return str(value)[:max_length] + if value is None: + return "null" + if type(value) is bool: + return "true" if value else "false" + return None + + +class _SanitizationState: + def __init__( + self, + *, + max_depth: int, + max_items: int, + max_string_length: int, + ) -> None: + self.max_depth = max_depth + self.remaining_items = max_items + self.max_string_length = max_string_length + self.active_container_ids: set[int] = set() + + def consume_item(self) -> bool: + if self.remaining_items <= 0: + return False + self.remaining_items -= 1 + return True + + +def _sanitize_value( + value: object, + *, + state: _SanitizationState, + depth: int, + key: str | None, +) -> JSONValue: + if key is not None and _is_forbidden_detail_name(key): + return REDACTED_VALUE + if value is None: + return None + if type(value) is bool: + return value + if type(value) is int: + return value + if type(value) is float: + return value if math.isfinite(value) else None + if type(value) is str: + return sanitize_text(value, max_length=state.max_string_length) + + if type(value) is dict or type(value) is MappingProxyType: + if depth >= state.max_depth: + return TRUNCATED_VALUE + container_id = id(value) + if container_id in state.active_container_ids: + return CYCLE_VALUE + state.active_container_ids.add(container_id) + try: + result: dict[str, JSONValue] = {} + for raw_key, child_value in value.items(): + if not state.consume_item(): + break + child_key = _sanitize_key(raw_key, max_length=MAX_DETAIL_KEY_LENGTH) + if child_key is None: + continue + if type(raw_key) is str and len(raw_key) > MAX_DETAIL_KEY_LENGTH: + result[child_key] = REDACTED_VALUE + else: + result[child_key] = _sanitize_value( + child_value, + state=state, + depth=depth + 1, + key=child_key, + ) + return result + finally: + state.active_container_ids.remove(container_id) + + if type(value) is list or type(value) is tuple: + if depth >= state.max_depth: + return TRUNCATED_VALUE + container_id = id(value) + if container_id in state.active_container_ids: + return CYCLE_VALUE + state.active_container_ids.add(container_id) + try: + result_list: list[JSONValue] = [] + for child_value in value: + if not state.consume_item(): + break + result_list.append( + _sanitize_value( + child_value, + state=state, + depth=depth + 1, + key=key, + ) + ) + return result_list + finally: + state.active_container_ids.remove(container_id) + + return UNSUPPORTED_VALUE + + +def _validate_limit(value: int, name: str) -> int: + if type(value) is not int or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + return value + + +def _canonical_json_bytes(value: JSONValue) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def sanitize_details( + value: object, + *, + max_depth: int = 6, + max_items: int = 64, + max_string_length: int = 2048, +) -> JSONValue: + """Produce a bounded JSON value without invoking arbitrary conversions.""" + + state = _SanitizationState( + max_depth=_validate_limit(max_depth, "max_depth"), + max_items=_validate_limit(max_items, "max_items"), + max_string_length=_validate_limit(max_string_length, "max_string_length"), + ) + sanitized = _sanitize_value(value, state=state, depth=0, key=None) + try: + if len(_canonical_json_bytes(sanitized)) <= MAX_SANITIZED_JSON_BYTES: + return sanitized + except (TypeError, ValueError, RecursionError, UnicodeError): + pass + if type(sanitized) is dict: + return {"truncated": True} + return TRUNCATED_VALUE diff --git a/zeus/state.py b/zeus/state.py index 13ad6fb..593bae7 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -3,7 +3,6 @@ import json import re import sqlite3 -from collections.abc import Mapping from contextlib import closing from dataclasses import dataclass from datetime import UTC, datetime @@ -26,9 +25,9 @@ ReconcileRunStart, ReconcileRunSummary, ) +from zeus.sanitization import MAX_SANITIZED_JSON_BYTES, sanitize_details, sanitize_text SCHEMA_VERSION = 6 -AUDIT_SECRET_RE = re.compile(r"(KEY|TOKEN|SECRET|PASSWORD)$") LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") @@ -46,21 +45,10 @@ } -def _safe_audit_value(key: str, value: object) -> object: - if AUDIT_SECRET_RE.search(key.upper()): - return "[redacted]" - if value is None or isinstance(value, str | int | float | bool): - return value - if isinstance(value, Path): - return str(value) - if isinstance(value, Mapping): - return { - str(child_key): _safe_audit_value(str(child_key), child_value) - for child_key, child_value in value.items() - } - if isinstance(value, list): - return [_safe_audit_value(key, child_value) for child_value in value] - return str(value) +def _sanitize_optional_persisted_text(value: str | None) -> str | None: + if value is None: + return None + return sanitize_text(value) def _validate_idempotency_hash(value: str, label: str) -> str: @@ -1754,16 +1742,33 @@ def audit_log_path(self) -> Path: return self.database_path.parent / "logs" / "audit.jsonl" def append_audit_event(self, event: str, **fields: object) -> None: - payload: dict[str, object] = { - "ts": datetime.now(UTC).isoformat(), - "event": event, - } - for key, value in fields.items(): - payload[key] = _safe_audit_value(key, value) try: - line = (json.dumps(payload, sort_keys=True) + "\n").encode("utf-8") + safe_fields = sanitize_details(fields) + if type(safe_fields) is not dict: + return + timestamp = datetime.now(UTC).isoformat() + safe_event = sanitize_text(event) + payload = { + "ts": timestamp, + "event": safe_event, + **safe_fields, + } + line = (json.dumps(payload, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + line = ( + json.dumps( + { + "ts": timestamp, + "event": safe_event, + "truncated": True, + }, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) - except (OSError, TypeError, ValueError): + except Exception: return def upsert_bot(self, record: BotRecord) -> None: @@ -1843,8 +1848,8 @@ def _upsert_bot_row(self, conn: sqlite3.Connection, record: BotRecord) -> None: record.ready_at.isoformat() if record.ready_at else None, record.stopped_at.isoformat() if record.stopped_at else None, record.last_exit_code, - record.last_error, - record.last_transition_reason, + _sanitize_optional_persisted_text(record.last_error), + _sanitize_optional_persisted_text(record.last_transition_reason), record.desired_state.value, record.desired_revision, record.desired_updated_at.isoformat() if record.desired_updated_at else None, @@ -2010,8 +2015,8 @@ def _update_lifecycle_row( int(clear_stopped_at), stopped_at.isoformat() if stopped_at else None, last_exit_code, - last_error, - last_transition_reason, + _sanitize_optional_persisted_text(last_error), + _sanitize_optional_persisted_text(last_transition_reason), now, int(reset_restart), int(reset_restart), From ec06b35a577c8b2d196877ba2b0df4e55d3991e8 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:57:06 +0200 Subject: [PATCH 13/49] security: redact and checksum-gate VPS verification --- README.md | 4 +- docs/FRESH_VPS_TEST.md | 22 +- scripts/fresh_vps_verify.sh | 342 +++++++++++++++++++++++++++--- tests/test_fresh_vps_script.py | 376 +++++++++++++++++++++++++++++++++ 4 files changed, 709 insertions(+), 35 deletions(-) create mode 100644 tests/test_fresh_vps_script.py diff --git a/README.md b/README.md index 50c8632..5e0b5c3 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,9 @@ passes a random per-run API key, probes `/health`, and then stops the bot. For a clean Debian/Ubuntu host, use the fresh VPS harness: ```bash -ZEUS_VPS_INSTALL_PACKAGES=1 ZEUS_VPS_INSTALL_HERMES=1 bash scripts/fresh_vps_verify.sh +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ +ZEUS_VPS_INSTALL_PACKAGES=1 ZEUS_VPS_INSTALL_HERMES=1 \ +bash scripts/fresh_vps_verify.sh ``` See [Fresh VPS test](docs/FRESH_VPS_TEST.md) for gateway and async-delegation probes. diff --git a/docs/FRESH_VPS_TEST.md b/docs/FRESH_VPS_TEST.md index 5769a9e..4a3968c 100644 --- a/docs/FRESH_VPS_TEST.md +++ b/docs/FRESH_VPS_TEST.md @@ -15,25 +15,33 @@ Use this runbook to verify Zeus on a clean Debian or Ubuntu VPS with a real Herm ## Threat Model -The VPS test downloads and executes the official Hermes installer only when `ZEUS_VPS_INSTALL_HERMES=1` is set. That crosses the network boundary and trusts the Hermes installer HTTPS endpoint. Run it only on a disposable host or after reviewing the downloaded installer saved in the evidence directory. +The VPS test downloads the official Hermes installer only when `ZEUS_VPS_INSTALL_HERMES=1` is set. Remote bootstrap also requires `ZEUS_VPS_HERMES_INSTALLER_SHA256` to contain the exact 64-hex-character SHA-256 digest of an installer you reviewed. The script verifies the downloaded bytes with a constant-time comparison and refuses to execute a missing, malformed, or mismatched digest. This still crosses the network boundary, so use a digest obtained through a trusted review process and run the bootstrap only on a disposable host. -Do not put provider tokens in command history. Configure Hermes credentials through Hermes' own setup flow or a secure environment mechanism. Review `.tmp/fresh-vps-verify/.../run.log` before sharing it, because Hermes tools may print environment-specific diagnostics. +Do not put provider tokens in command history. Configure Hermes credentials through Hermes' own setup flow or a secure environment mechanism. The API smoke key is generated ephemerally by default; authenticated curl commands use a redacted transcript label and receive the key through standard input instead of an argument. Review `.tmp/fresh-vps-verify/.../run.log` before sharing it, because Hermes tools may print environment-specific diagnostics. + +`ZEUS_VPS_ASYNC_PROMPT` is also replaced by a fixed label in the transcript. Hermes still receives that operator-supplied prompt through its established `-z ` argument, so it may be visible to local process inspection while that optional probe runs. ## Fresh Host Flow Clone or copy this repository onto the VPS, then run from the repository root: ```bash +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ ZEUS_VPS_INSTALL_PACKAGES=1 \ ZEUS_VPS_INSTALL_HERMES=1 \ bash scripts/fresh_vps_verify.sh ``` -This installs basic Debian/Ubuntu packages, creates `.venv/`, installs Zeus editable, installs Hermes if missing, runs the local gates, checks Hermes, renders all templates, and runs an API smoke test. +Obtain the pinned digest separately: download the installer without executing it, review the saved file, and calculate its SHA-256 with a trusted local tool such as `sha256sum`. The verifier never learns or trusts a digest from the download endpoint itself. + +The command installs basic Debian/Ubuntu packages, creates `.venv/`, installs Zeus editable, installs the digest-verified Hermes payload if missing, runs the local gates, checks Hermes, renders all templates, and runs an API smoke test. + +On a minimal supported apt host where `python3` is not yet available, `ZEUS_VPS_INSTALL_PACKAGES=1` permits a minimal Python 3 prerequisite install before the private evidence log can be initialized. That early package-manager output is console-only; the normal package step is still recorded afterward. Without that explicit package-bootstrap opt-in, the verifier fails with a prerequisite error. To also start a real Hermes gateway for the default real-Hermes bot: ```bash +ZEUS_VPS_HERMES_INSTALLER_SHA256='<64-hex SHA-256 of the reviewed installer>' \ ZEUS_VPS_INSTALL_PACKAGES=1 \ ZEUS_VPS_INSTALL_HERMES=1 \ ZEUS_VPS_START_GATEWAY=1 \ @@ -66,7 +74,7 @@ The pass criteria are: ## Evidence -The script writes logs under: +The script writes logs under a workspace-relative directory with mode `0700`: ```text .tmp/fresh-vps-verify// @@ -74,10 +82,12 @@ The script writes logs under: Important files: -- `run.log`: full command transcript, including `git rev-parse HEAD` and `git status --short` when run from a Git checkout. -- `hermes-install.sh`: downloaded installer, when Hermes installation is enabled. +- `run.log`: command transcript, including `git rev-parse HEAD` and `git status --short` when run from a Git checkout; sensitive invocations use fixed redacted labels. +- `hermes-install.sh`: downloaded and digest-verified installer, when Hermes installation is enabled. - `zeus-api.log`: API server log from the loopback smoke test. +Evidence files use mode `0600`. The verifier rejects absolute, escaping, symlinked, or otherwise unsafe evidence paths instead of changing permissions on their targets. `ZEUS_VPS_LOG_DIR` must remain a workspace-relative scratch directory. + Runtime state is created under ignored workspace directories such as `.zeus-real-hermes-check/`, `.zeus-vps-multi/`, and `.zeus-vps-api/`. Clean up after collecting evidence: diff --git a/scripts/fresh_vps_verify.sh b/scripts/fresh_vps_verify.sh index 73e9955..78123ae 100755 --- a/scripts/fresh_vps_verify.sh +++ b/scripts/fresh_vps_verify.sh @@ -2,15 +2,235 @@ # Zeus Hermes Orchestrator # Maintained by BrainX: https://github.com/brainx set -Eeuo pipefail +umask 077 + +fail() { + echo "fresh VPS verification failed: $*" >&2 + exit 1 +} + +safe_relative_dir() { + case "$1" in + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) + fail "$2 must be a workspace-relative scratch directory" + ;; + esac +} repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" +sudo_prefix=() +if [ "${EUID:-$(id -u)}" -ne 0 ]; then + sudo_prefix=(sudo) +fi + +if ! command -v python3 >/dev/null 2>&1; then + if [ "${ZEUS_VPS_INSTALL_PACKAGES:-0}" != "1" ] || ! command -v apt-get >/dev/null 2>&1; then + fail "Python 3 is required; install it first or enable the supported apt package bootstrap" + fi + echo "Bootstrapping the Python 3 prerequisite before private evidence logging." + "${sudo_prefix[@]}" apt-get update + "${sudo_prefix[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y python3 + hash -r + command -v python3 >/dev/null 2>&1 || fail "Python 3 bootstrap completed without a python3 executable" +fi + timestamp="$(date -u '+%Y%m%dT%H%M%SZ')" log_dir="${ZEUS_VPS_LOG_DIR:-.tmp/fresh-vps-verify/$timestamp}" -mkdir -p "$log_dir" +safe_relative_dir "$log_dir" "ZEUS_VPS_LOG_DIR" + +private_evidence() { + python3 -I -B - "$repo_root" "$@" <<'PY' +import os +import stat +import sys + + +def fail(message: str) -> None: + raise ValueError(message) + + +def relative_parts(value: str) -> list[str]: + if not value or os.path.isabs(value): + fail("path must be relative") + parts = value.split("/") + if any(part in {"", ".", ".."} for part in parts): + fail("path contains an unsafe component") + if any("\r" in part or "\n" in part for part in parts): + fail("path contains a line break") + return parts + + +def same_identity(left: os.stat_result, right: os.stat_result) -> bool: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + + +def validate_directory_bindings( + bindings: list[tuple[int, str, int]], + *, + exact_final_mode: bool, +) -> None: + for index, (parent_fd, name, child_fd) in enumerate(bindings): + opened = os.fstat(child_fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if not stat.S_ISDIR(opened.st_mode) or not same_identity(opened, current): + fail("directory binding changed") + if opened.st_uid != os.geteuid(): + fail("directory is not owned by the current user") + if exact_final_mode and index == len(bindings) - 1: + if stat.S_IMODE(opened.st_mode) != 0o700: + fail("evidence directory is not private") + + +def open_directory_chain( + root: str, + parts: list[str], + *, + create: bool, +) -> tuple[list[int], list[tuple[int, str, int]]]: + if not hasattr(os, "O_NOFOLLOW"): + fail("O_NOFOLLOW is required for private evidence") + directory_flags = os.O_RDONLY | os.O_DIRECTORY + directory_flags |= os.O_NOFOLLOW + root_fd = os.open(root, directory_flags) + descriptors = [root_fd] + bindings: list[tuple[int, str, int]] = [] + parent_fd = root_fd + try: + for index, name in enumerate(parts): + created = False + try: + child_fd = os.open(name, directory_flags, dir_fd=parent_fd) + except FileNotFoundError: + if not create: + raise + os.mkdir(name, 0o700, dir_fd=parent_fd) + created = True + child_fd = os.open(name, directory_flags, dir_fd=parent_fd) + descriptors.append(child_fd) + bindings.append((parent_fd, name, child_fd)) + opened = os.fstat(child_fd) + if not stat.S_ISDIR(opened.st_mode) or opened.st_uid != os.geteuid(): + fail("unsafe evidence directory") + if created or index == len(parts) - 1: + validate_directory_bindings(bindings, exact_final_mode=False) + os.fchmod(child_fd, 0o700) + parent_fd = child_fd + validate_directory_bindings(bindings, exact_final_mode=True) + return descriptors, bindings + except BaseException: + for descriptor in reversed(descriptors): + os.close(descriptor) + raise + + +def validate_file(opened: os.stat_result, current: os.stat_result) -> None: + if not stat.S_ISREG(opened.st_mode) or not same_identity(opened, current): + fail("unsafe evidence file binding") + if opened.st_uid != os.geteuid() or opened.st_nlink != 1: + fail("unsafe evidence file ownership") + if stat.S_IMODE(opened.st_mode) != 0o600: + fail("evidence file is not private") + + +def prepare(root: str, relative_directory: str, names: list[str]) -> None: + descriptors, bindings = open_directory_chain( + root, + relative_parts(relative_directory), + create=True, + ) + directory_fd = descriptors[-1] + if not hasattr(os, "O_NOFOLLOW"): + fail("O_NOFOLLOW is required for private evidence") + file_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + file_flags |= os.O_NOFOLLOW + try: + for name in names: + if "/" in name or name in {"", ".", ".."}: + fail("invalid evidence filename") + file_fd = os.open(name, file_flags, 0o600, dir_fd=directory_fd) + try: + opened = os.fstat(file_fd) + current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if not stat.S_ISREG(opened.st_mode) or not same_identity(opened, current): + fail("unsafe evidence file binding") + if opened.st_uid != os.geteuid() or opened.st_nlink != 1: + fail("unsafe evidence file ownership") + os.fchmod(file_fd, 0o600) + validate_file( + os.fstat(file_fd), + os.stat(name, dir_fd=directory_fd, follow_symlinks=False), + ) + validate_directory_bindings(bindings, exact_final_mode=True) + finally: + os.close(file_fd) + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def validate_descriptor( + root: str, + relative_file: str, + descriptor: int, + *, + truncate: bool, +) -> None: + parts = relative_parts(relative_file) + if len(parts) < 2: + fail("evidence file must be inside an evidence directory") + descriptors, bindings = open_directory_chain(root, parts[:-1], create=False) + directory_fd = descriptors[-1] + try: + opened = os.fstat(descriptor) + current = os.stat(parts[-1], dir_fd=directory_fd, follow_symlinks=False) + validate_file(opened, current) + validate_directory_bindings(bindings, exact_final_mode=True) + if truncate: + os.ftruncate(descriptor, 0) + validate_file( + os.fstat(descriptor), + os.stat(parts[-1], dir_fd=directory_fd, follow_symlinks=False), + ) + validate_directory_bindings(bindings, exact_final_mode=True) + finally: + for opened_fd in reversed(descriptors): + os.close(opened_fd) + + +try: + repository_root = sys.argv[1] + action = sys.argv[2] + relative_path = sys.argv[3] + if action == "prepare": + prepare(repository_root, relative_path, sys.argv[4:]) + elif action == "validate-file": + validate_descriptor( + repository_root, + relative_path, + int(sys.argv[4]), + truncate=sys.argv[5] == "truncate", + ) + else: + fail("unknown private evidence operation") +except (OSError, ValueError) as error: + print(f"private evidence validation failed: {error}", file=sys.stderr) + raise SystemExit(1) from None +PY +} + +if ! private_evidence prepare "$log_dir" run.log zeus-api.log; then + fail "could not create a private evidence directory" +fi log_file="$log_dir/run.log" -exec > >(tee -a "$log_file") 2>&1 +run_log_fd=9 +exec 9<>"$log_file" +if ! private_evidence validate-file "$log_file" "$run_log_fd" truncate; then + exec 9>&- + fail "could not open the private evidence transcript" +fi +exec > >(tee -a "/dev/fd/$run_log_fd") 2>&1 api_pid="" @@ -25,9 +245,20 @@ run() { "$@" } -fail() { - echo "fresh VPS verification failed: $*" >&2 - exit 1 +run_stdout_to_fd() { + local output_fd="$1" + shift + printf '+' + printf ' %q' "$@" + printf '\n' + "$@" 1>&"$output_fd" +} + +run_sensitive() { + local label="$1" + shift + printf '+ %s\n' "$label" + "$@" } cleanup() { @@ -38,19 +269,6 @@ cleanup() { } trap cleanup EXIT INT TERM -safe_relative_dir() { - case "$1" in - "" | "/" | "." | ".." | /* | ../* | */../*) - fail "$2 must be a workspace-relative scratch directory" - ;; - esac -} - -sudo_prefix=() -if [ "${EUID:-$(id -u)}" -ne 0 ]; then - sudo_prefix=(sudo) -fi - section "Repository" test -f pyproject.toml || fail "run this script from a Zeus checkout" test -f zeus/cli.py || fail "missing zeus/cli.py" @@ -93,9 +311,52 @@ hash -r if command -v hermes >/dev/null 2>&1; then run command -v hermes elif [ "${ZEUS_VPS_INSTALL_HERMES:-0}" = "1" ]; then + installer_sha256="${ZEUS_VPS_HERMES_INSTALLER_SHA256:-}" + if [[ ! "$installer_sha256" =~ ^[0-9A-Fa-f]{64}$ ]]; then + fail "ZEUS_VPS_HERMES_INSTALLER_SHA256 must be exactly 64 hexadecimal characters" + fi + if ! private_evidence prepare "$log_dir" hermes-install.sh; then + fail "could not create a private Hermes installer evidence file" + fi installer="$log_dir/hermes-install.sh" - run curl -fsSL https://hermes-agent.nousresearch.com/install.sh -o "$installer" - run bash "$installer" + installer_fd=8 + exec 8<>"$installer" + if ! private_evidence validate-file "$installer" "$installer_fd" truncate; then + exec 8>&- + fail "could not open the private Hermes installer evidence file" + fi + run_stdout_to_fd "$installer_fd" curl -fsSL https://hermes-agent.nousresearch.com/install.sh + if ! private_evidence validate-file "$installer" "$installer_fd" preserve; then + exec 8>&- + fail "Hermes installer evidence changed during download" + fi + if ! python3 -I -B - "$installer_fd" "$installer_sha256" <<'PY' +import hashlib +import hmac +import os +import sys + +descriptor = int(sys.argv[1]) +expected = sys.argv[2].lower() +digest = hashlib.sha256() +offset = 0 +while True: + chunk = os.pread(descriptor, 1024 * 1024, offset) + if not chunk: + break + digest.update(chunk) + offset += len(chunk) +matches = hmac.compare_digest(digest.hexdigest(), expected) +if matches: + os.lseek(descriptor, 0, os.SEEK_SET) +raise SystemExit(0 if matches else 1) +PY + then + exec 8>&- + fail "Hermes installer SHA-256 mismatch; refusing to execute it" + fi + run bash "/dev/fd/$installer_fd" + exec 8>&- export PATH="$HOME/.local/bin:$HOME/.hermes/bin:$PATH" hash -r run command -v hermes @@ -133,7 +394,9 @@ done if [ -n "${ZEUS_VPS_ASYNC_PROMPT:-}" ]; then section "Optional Async Delegation Prompt" - run env HERMES_HOME="$repo_root/$multi_state/hermes" hermes -p vps-coder -z "$ZEUS_VPS_ASYNC_PROMPT" + run_sensitive "Hermes async prompt [redacted]" env \ + HERMES_HOME="$repo_root/$multi_state/hermes" \ + hermes -p vps-coder -z "$ZEUS_VPS_ASYNC_PROMPT" else echo "Skipping live async prompt. Set ZEUS_VPS_ASYNC_PROMPT to exercise delegate_task/background behavior with real credentials." fi @@ -143,9 +406,31 @@ api_state="${ZEUS_VPS_API_STATE_DIR:-.zeus-vps-api}" safe_relative_dir "$api_state" "ZEUS_VPS_API_STATE_DIR" rm -rf -- "$api_state" api_port="${ZEUS_VPS_API_PORT:-4311}" -api_key="${ZEUS_VPS_API_KEY:-vps-local-check}" +api_key="${ZEUS_VPS_API_KEY:-$(python -I -B -c 'import secrets; print(secrets.token_hex(32))')}" +if [[ "$api_key" == *$'\r'* || "$api_key" == *$'\n'* ]]; then + fail "ZEUS_VPS_API_KEY must not contain a line break" +fi + +write_api_curl_config() { + local include_json_header="$1" + local escaped_key + escaped_key=${api_key//\\/\\\\} + escaped_key=${escaped_key//\"/\\\"} + printf 'header = "x-zeus-api-key: %s"\n' "$escaped_key" + if [ "$include_json_header" = "1" ]; then + printf 'header = "content-type: application/json"\n' + fi +} + +api_log_file="$log_dir/zeus-api.log" +api_log_fd=7 +exec 7<>"$api_log_file" +if ! private_evidence validate-file "$api_log_file" "$api_log_fd" truncate; then + exec 7>&- + fail "could not open the private API evidence log" +fi ZEUS_STATE_DIR="$api_state" ZEUS_API_KEY="$api_key" \ - python -B -m zeus.api --host 127.0.0.1 --port "$api_port" >"$log_dir/zeus-api.log" 2>&1 & + python -B -m zeus.api --host 127.0.0.1 --port "$api_port" 1>&"$api_log_fd" 2>&1 & api_pid="$!" for _ in $(seq 1 40); do @@ -156,17 +441,18 @@ for _ in $(seq 1 40); do done run curl -fsS "http://127.0.0.1:$api_port/health" -run curl -fsS -H "x-zeus-api-key: $api_key" "http://127.0.0.1:$api_port/doctor" -run curl -fsS \ - -H "x-zeus-api-key: $api_key" \ - -H "content-type: application/json" \ +write_api_curl_config 0 | run_sensitive "authenticated API request [redacted]" \ + curl -fsS --config - "http://127.0.0.1:$api_port/doctor" +write_api_curl_config 1 | run_sensitive "authenticated API request [redacted]" \ + curl -fsS --config - \ --data '{"bot_id":"api-vps","template_id":"support-gateway"}' \ "http://127.0.0.1:$api_port/bots" cleanup api_pid="" +exec 7>&- section "Summary" echo "Fresh VPS verification completed." echo "Evidence log: $log_file" -echo "API log: $log_dir/zeus-api.log" +echo "API log: $api_log_file" diff --git a/tests/test_fresh_vps_script.py b/tests/test_fresh_vps_script.py new file mode 100644 index 0000000..1279e6a --- /dev/null +++ b/tests/test_fresh_vps_script.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "fresh_vps_verify.sh" +INSTALLER_URL = "https://hermes-agent.nousresearch.com/install.sh" + + +class FreshVPSVerifierTests(unittest.TestCase): + def setUp(self) -> None: + self._temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self._temporary_directory.cleanup) + self.root = Path(self._temporary_directory.name) + self.checkout = self.root / "checkout" + self.checkout.mkdir() + (self.checkout / "scripts").mkdir() + (self.checkout / "zeus").mkdir() + shutil.copy2(SCRIPT, self.checkout / "scripts" / SCRIPT.name) + (self.checkout / "pyproject.toml").write_text("[project]\nname='test'\n") + (self.checkout / "zeus" / "cli.py").write_text("# verifier fixture\n") + + self.home = self.root / "home" + self.home.mkdir() + self.stub_bin = self.root / "bin" + self.stub_bin.mkdir() + self.curl_capture = self.root / "curl.jsonl" + self.apt_capture = self.root / "apt.log" + self.installer_executed = self.root / "installer-executed" + self.fake_installer = self.root / "fake-hermes-installer.sh" + self.fake_installer.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -eu", + 'mkdir -p "$HOME/.local/bin"', + "cat > \"$HOME/.local/bin/hermes\" <<'HERMES'", + "#!/usr/bin/env bash", + "exit 0", + "HERMES", + 'chmod 700 "$HOME/.local/bin/hermes"', + "printf 'executed\\n' > \"$INSTALLER_EXECUTED\"", + "", + ] + ), + encoding="utf-8", + ) + self.installer_digest = hashlib.sha256(self.fake_installer.read_bytes()).hexdigest() + self._write_stubs() + + def _write_executable(self, name: str, content: str) -> None: + path = self.stub_bin / name + path.write_text(content, encoding="utf-8") + path.chmod(0o700) + + def _write_stubs(self) -> None: + python_stub = textwrap.dedent( + f"""\ + #!/usr/bin/env bash + set -eu + real_python={str(Path(sys.executable))!r} + if [ "${{1:-}}" = "-I" ]; then + exec "$real_python" "$@" + fi + if [ "${{1:-}}" = "-B" ] && [ "${{2:-}}" = "-c" ]; then + exec "$real_python" "$@" + fi + if [ "${{1:-}}" = "-m" ] && [ "${{2:-}}" = "venv" ]; then + mkdir -p "$3/bin" + : > "$3/bin/activate" + exit 0 + fi + if [ "${{1:-}}" = "--version" ]; then + printf 'Python fixture\n' + fi + exit 0 + """ + ) + self._write_executable("python3", python_stub) + self._write_executable("python", python_stub) + self._write_executable( + "sh", + "#!/usr/bin/env bash\nset -eu\nexit 0\n", + ) + + curl_stub = textwrap.dedent( + f"""\ + #!{sys.executable} + import json + import os + from pathlib import Path + import shutil + import sys + + args = sys.argv[1:] + config_stdin = sys.stdin.read() if "--config" in args and "-" in args else "" + capture = Path(os.environ["CURL_CAPTURE"]) + with capture.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({{"argv": args, "stdin": config_stdin}}) + "\\n") + + if {INSTALLER_URL!r} in args: + source = Path(os.environ["FAKE_INSTALLER_SOURCE"]) + if "-o" in args: + output = Path(args[args.index("-o") + 1]) + shutil.copyfile(source, output) + else: + sys.stdout.buffer.write(source.read_bytes()) + else: + print('{{"status":"ok"}}') + """ + ) + self._write_executable("curl", curl_stub) + + def _base_environment(self) -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.stub_bin}:/usr/bin:/bin", + "HOME": str(self.home), + "CURL_CAPTURE": str(self.curl_capture), + "FAKE_INSTALLER_SOURCE": str(self.fake_installer), + "INSTALLER_EXECUTED": str(self.installer_executed), + "ZEUS_VPS_INSTALL_HERMES": "1", + "ZEUS_VPS_LOG_DIR": "evidence", + "ZEUS_VPS_VENV_DIR": ".fixture-venv", + "ZEUS_VPS_MULTI_STATE_DIR": ".fixture-multi", + "ZEUS_VPS_API_STATE_DIR": ".fixture-api", + } + ) + environment.pop("ZEUS_VPS_API_KEY", None) + environment.pop("ZEUS_VPS_HERMES_INSTALLER_SHA256", None) + return environment + + def _pythonless_bootstrap_path(self) -> str: + bootstrap_bin = self.root / "bootstrap-bin" + bootstrap_bin.mkdir() + for name in ("curl", "sh"): + (bootstrap_bin / name).symlink_to(self.stub_bin / name) + for name in ("dirname", "env", "seq", "tee", "uname"): + (bootstrap_bin / name).symlink_to(Path("/usr/bin") / name) + + sudo = bootstrap_bin / "sudo" + sudo.write_text('#!/bin/bash\nset -eu\nexec "$@"\n', encoding="utf-8") + sudo.chmod(0o700) + apt_get = bootstrap_bin / "apt-get" + apt_get.write_text( + "\n".join( + [ + "#!/bin/bash", + "set -eu", + 'printf \'%s\\n\' "$*" >> "$APT_CAPTURE"', + 'if [ "${1:-}" = "install" ]; then', + ' ln -sf "$PYTHON_STUB_SOURCE" "$BOOTSTRAP_BIN/python3"', + ' ln -sf "$PYTHON_STUB_SOURCE" "$BOOTSTRAP_BIN/python"', + "fi", + "", + ] + ), + encoding="utf-8", + ) + apt_get.chmod(0o700) + return f"{bootstrap_bin}:/bin" + + def _run(self, **overrides: str) -> subprocess.CompletedProcess[str]: + environment = self._base_environment() + environment.update(overrides) + return subprocess.run( + ["/bin/bash", "scripts/fresh_vps_verify.sh"], + cwd=self.checkout, + env=environment, + input="", + text=True, + capture_output=True, + check=False, + timeout=20, + ) + + def _curl_records(self) -> list[dict[str, object]]: + if not self.curl_capture.exists(): + return [] + return [ + json.loads(line) for line in self.curl_capture.read_text(encoding="utf-8").splitlines() + ] + + def test_missing_installer_digest_fails_before_download(self) -> None: + result = self._run() + + self.assertNotEqual(0, result.returncode) + self.assertIn("ZEUS_VPS_HERMES_INSTALLER_SHA256", result.stdout + result.stderr) + self.assertFalse(self.installer_executed.exists()) + self.assertFalse(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_missing_python_is_bootstrapped_before_private_evidence(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + result = self._run( + PATH=bootstrap_path, + APT_CAPTURE=str(self.apt_capture), + BOOTSTRAP_BIN=bootstrap_path.removesuffix(":/bin"), + PYTHON_STUB_SOURCE=str(self.stub_bin / "python3"), + ZEUS_VPS_INSTALL_PACKAGES="1", + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="bootstrap-test-key", + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("Bootstrapping the Python 3 prerequisite", result.stdout) + apt_commands = self.apt_capture.read_text(encoding="utf-8").splitlines() + self.assertIn("install -y python3", apt_commands) + self.assertTrue(any("python3-venv" in command for command in apt_commands)) + self.assertEqual(0o700, stat.S_IMODE((self.checkout / "evidence").stat().st_mode)) + + def test_missing_python_without_package_bootstrap_fails_deterministically(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + result = self._run( + PATH=bootstrap_path, + APT_CAPTURE=str(self.apt_capture), + BOOTSTRAP_BIN=bootstrap_path.removesuffix(":/bin"), + PYTHON_STUB_SOURCE=str(self.stub_bin / "python3"), + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("Python 3 is required", result.stdout + result.stderr) + self.assertFalse((self.checkout / "evidence").exists()) + + def test_malformed_installer_digest_fails_before_download(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256="g" * 64) + + self.assertNotEqual(0, result.returncode) + self.assertIn("64", result.stdout + result.stderr) + self.assertFalse(self.installer_executed.exists()) + self.assertFalse(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_mismatched_installer_digest_never_executes(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256="0" * 64) + + self.assertNotEqual(0, result.returncode) + self.assertIn("mismatch", (result.stdout + result.stderr).lower()) + self.assertFalse(self.installer_executed.exists()) + self.assertTrue(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + + def test_matching_digest_executes_and_sensitive_curl_uses_stdin(self) -> None: + api_key = 'sentinel-api-key-"quote\\nslash' + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY=api_key, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertTrue(self.installer_executed.exists()) + transcript = result.stdout + result.stderr + run_log = (self.checkout / "evidence" / "run.log").read_text(encoding="utf-8") + self.assertNotIn(api_key, transcript) + self.assertNotIn(api_key, run_log) + self.assertIn("authenticated API request [redacted]", transcript) + + records = self._curl_records() + argv_text = "\n".join(" ".join(record["argv"]) for record in records) + self.assertNotIn(api_key, argv_text) + authenticated = [record for record in records if record["stdin"]] + self.assertEqual(2, len(authenticated)) + escaped = api_key.replace("\\", "\\\\").replace('"', '\\"') + for record in authenticated: + self.assertIn("--config", record["argv"]) + self.assertNotIn("-H", record["argv"]) + self.assertIn(f'header = "x-zeus-api-key: {escaped}"', record["stdin"]) + + evidence = self.checkout / "evidence" + self.assertEqual(0o700, stat.S_IMODE(evidence.stat().st_mode)) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + with self.subTest(name=name): + self.assertEqual(0o600, stat.S_IMODE((evidence / name).stat().st_mode)) + + def test_existing_evidence_permissions_are_repaired(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir() + evidence.chmod(0o777) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + path = evidence / name + path.write_text("old evidence\n", encoding="utf-8") + path.chmod(0o666) + + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="permission-test-key", + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertEqual(0o700, stat.S_IMODE(evidence.stat().st_mode)) + for name in ("run.log", "zeus-api.log", "hermes-install.sh"): + with self.subTest(name=name): + self.assertEqual(0o600, stat.S_IMODE((evidence / name).stat().st_mode)) + + def test_async_prompt_uses_a_fixed_sensitive_transcript_label(self) -> None: + prompt = "SENTINEL_ASYNC_PROMPT_must_not_be_logged" + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY="async-test-key", + ZEUS_VPS_ASYNC_PROMPT=prompt, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + transcript = result.stdout + result.stderr + run_log = (self.checkout / "evidence" / "run.log").read_text(encoding="utf-8") + self.assertIn("Hermes async prompt [redacted]", transcript) + self.assertNotIn(prompt, transcript) + self.assertNotIn(prompt, run_log) + + def test_default_api_key_is_ephemeral_and_generated_by_secrets(self) -> None: + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + authenticated = [record for record in self._curl_records() if record["stdin"]] + self.assertEqual(2, len(authenticated)) + headers = [] + for record in authenticated: + for line in str(record["stdin"]).splitlines(): + if line.startswith('header = "x-zeus-api-key: '): + header = line.removeprefix('header = "x-zeus-api-key: ') + headers.append(header.removesuffix('"')) + self.assertEqual(2, len(headers)) + self.assertEqual(headers[0], headers[1]) + self.assertRegex(headers[0], r"^[0-9a-f]{64}$") + self.assertNotIn(headers[0], result.stdout + result.stderr) + + def test_api_key_with_line_breaks_is_rejected_without_disclosure(self) -> None: + api_key = "sentinel\r\ninjected-header" + result = self._run( + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ZEUS_VPS_API_KEY=api_key, + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("line break", (result.stdout + result.stderr).lower()) + self.assertNotIn(api_key, result.stdout + result.stderr) + self.assertFalse(any(record["stdin"] for record in self._curl_records())) + + def test_symlinked_evidence_directory_is_rejected_without_mutating_target(self) -> None: + external = self.root / "external" + external.mkdir(mode=0o755) + (self.checkout / "evidence").symlink_to(external, target_is_directory=True) + + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertNotEqual(0, result.returncode) + self.assertEqual([], list(external.iterdir())) + self.assertEqual(0o755, stat.S_IMODE(external.stat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + + def test_symlinked_evidence_file_is_rejected_without_mutating_target(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir(mode=0o700) + external = self.root / "external.log" + external.write_text("preserve me\n", encoding="utf-8") + external.chmod(0o644) + (evidence / "run.log").symlink_to(external) + + result = self._run(ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("preserve me\n", external.read_text(encoding="utf-8")) + self.assertEqual(0o644, stat.S_IMODE(external.stat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + + +if __name__ == "__main__": + unittest.main() From 308fd0c103f478eed4169ef69c19d55ed5cc29ca Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:05:58 +0200 Subject: [PATCH 14/49] security: close audit sanitization gaps --- tests/test_lifecycle_ledger.py | 111 ++++++++++++++++++++++++++++++++- zeus/sanitization.py | 23 ++++++- zeus/state.py | 4 +- 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index 223aee5..2994ee7 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -6,13 +6,14 @@ import unittest from contextlib import closing from dataclasses import FrozenInstanceError -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from unittest.mock import patch from zeus.lifecycle import LifecycleEvent, LifecycleEventInput, serialize_lifecycle_details from zeus.models import BotRecord, BotStatus, DesiredState -from zeus.sanitization import sanitize_details +from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunStart +from zeus.sanitization import sanitize_details, sanitize_text from zeus.state import StateStore @@ -613,6 +614,112 @@ def test_lifecycle_text_fields_are_redacted_before_sqlite_persistence(self) -> N persisted = "\n".join(str(value) for value in row if value is not None) self.assertFalse(secret in persisted) + def test_authorization_credentials_are_redacted_from_all_persistence_sinks(self) -> None: + credentials = ( + "basic-credential-sentinel-47d1", + "digest-credential-sentinel-8a20", + "custom-credential-sentinel-53fe", + ) + messages = ( + f"Authorization: Basic {credentials[0]}", + f'authorization=Digest username="operator", response="{credentials[1]}"', + f"Authorization: CustomScheme {credentials[2]}", + ) + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="a" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=messages[0], + error_code="authorization_failed", + error_message=messages[1], + details={"message": messages[2]}, + ) + store.upsert_bot_with_event( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=messages[0], + last_transition_reason=messages[1], + ), + event=event, + ) + store.append_audit_event( + "bot.authorization_failed", + cleanup_errors=list(messages), + ) + run = ReconcileRunStart( + run_id="authorization-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message="; ".join(messages), + error_code="authorization_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + with closing(sqlite3.connect(store.database_path)) as conn: + lifecycle_row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert lifecycle_row is not None + assert bot_row is not None + assert reconcile_row is not None + observed = ( + *(sanitize_text(message) for message in messages), + *lifecycle_row, + *bot_row, + *reconcile_row, + result.message, + store.audit_log_path().read_text(encoding="utf-8"), + ) + for credential in credentials: + self.assertFalse( + any(credential in value for value in observed if isinstance(value, str)) + ) + + def test_audit_non_bmp_event_fallback_is_strictly_byte_bounded(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + + store.append_audit_event(chr(0x1F512) * 2_048, message="ordinary") + + line = store.audit_log_path().read_bytes() + self.assertLessEqual(len(line), 8192) + self.assertEqual(1, len(line.splitlines())) + self.assertTrue(json.loads(line)["truncated"]) + def test_upsert_bot_with_event_rolls_back_projection_when_event_insert_fails(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/zeus/sanitization.py b/zeus/sanitization.py index 6a79597..f07bb46 100644 --- a/zeus/sanitization.py +++ b/zeus/sanitization.py @@ -52,7 +52,6 @@ (?P["']?) (?P [A-Z0-9_.-]*(?:API[_-]?KEY|KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_.-]* - | AUTHORIZATION ) (?P=prefix) (?P\s*[:=]\s*) @@ -63,13 +62,33 @@ ) """ ) +_AUTHORIZATION_RE = re.compile( + r"""(?ix) + (?P["']?) + (?PAUTHORIZATION) + (?P=prefix) + (?P\s*[:=]\s*) + (?P + "(?:[^"\\\r\n]|\\.)*" | + '(?:[^'\\\r\n]|\\.)*' | + [^\r\n]+ + ) + """ +) _BEARER_RE = re.compile(r"(?i)(\bBearer\s+)([A-Za-z0-9._~+/=-]+)") def redact_secrets(text: str) -> str: """Redact common secret assignments and bearer credentials from free text.""" - redacted = _BEARER_RE.sub(r"\1[redacted]", text) + redacted = _AUTHORIZATION_RE.sub( + lambda match: ( + f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" + f"{match.group('sep')}[redacted]" + ), + text, + ) + redacted = _BEARER_RE.sub(r"\1[redacted]", redacted) return _SECRET_KV_RE.sub( lambda match: ( f"{match.group('prefix')}{match.group('name')}{match.group('prefix')}" diff --git a/zeus/state.py b/zeus/state.py index 593bae7..313a010 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -1759,7 +1759,7 @@ def append_audit_event(self, event: str, **fields: object) -> None: json.dumps( { "ts": timestamp, - "event": safe_event, + "event": "audit.truncated", "truncated": True, }, sort_keys=True, @@ -1767,6 +1767,8 @@ def append_audit_event(self, event: str, **fields: object) -> None: ) + "\n" ).encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + return append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) except Exception: return From 2a0ca6a6d2ae49b1d59e2922b69e9a630092ea66 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:07:39 +0200 Subject: [PATCH 15/49] security: require confirmed marker absence --- tests/test_crash_recovery.py | 114 +++++++++++++++++++++++++++++++++++ tests/test_private_io.py | 63 +++++++++++++++++++ zeus/gateway_launcher.py | 5 ++ zeus/private_io.py | 39 ++++++------ zeus/supervisor.py | 15 +++-- 5 files changed, 213 insertions(+), 23 deletions(-) diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index fc7ac45..eb959bb 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -1643,6 +1643,120 @@ def _write_runtime_marker( marker_path.write_text(json.dumps(marker), encoding="utf-8") return marker_path + def test_strict_marker_final_profile_validation_enoent_is_untrusted(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + _store, supervisor = self._fixture(root) + record = _store.get_bot("coder") + assert record is not None + profile = Path(record.profile_path) + marker_path = self._write_runtime_marker( + supervisor, + record.profile_path, + self._runtime_marker(supervisor), + ) + displaced = root / "profile-displaced-during-validation" + profiles_stat = profile.parent.stat() + real_stat = os.stat + profile_observations = 0 + restored = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal profile_observations, restored + if path == profile.name and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == profiles_stat.st_dev + and parent.st_ino == profiles_stat.st_ino + ): + profile_observations += 1 + if profile_observations == 2: + profile.rename(displaced) + try: + return real_stat( + path, + dir_fd=dir_fd, + follow_symlinks=follow_symlinks, + ) + except FileNotFoundError: + displaced.rename(profile) + restored = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat): + observation = supervisor._read_strict_runtime_marker("coder", record.profile_path) + + self.assertEqual(2, profile_observations) + self.assertTrue(restored) + self.assertEqual("untrusted", observation.kind) + self.assertTrue(profile.is_dir()) + self.assertTrue(marker_path.is_file()) + + def test_inspect_marker_final_profile_validation_enoent_is_unsafe(self) -> None: + from zeus.private_io import UnsafeFileError + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + _store, supervisor = self._fixture(root) + record = _store.get_bot("coder") + assert record is not None + profile = Path(record.profile_path) + marker_path = self._write_runtime_marker( + supervisor, + record.profile_path, + self._runtime_marker(supervisor), + ) + displaced = root / "profile-displaced-during-inspect" + profiles_stat = profile.parent.stat() + real_stat = os.stat + profile_observations = 0 + restored = False + + def racing_stat( + path: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> os.stat_result: + nonlocal profile_observations, restored + if path == profile.name and dir_fd is not None: + parent = os.fstat(dir_fd) + if ( + parent.st_dev == profiles_stat.st_dev + and parent.st_ino == profiles_stat.st_ino + ): + profile_observations += 1 + if profile_observations == 2: + profile.rename(displaced) + try: + return real_stat( + path, + dir_fd=dir_fd, + follow_symlinks=follow_symlinks, + ) + except FileNotFoundError: + displaced.rename(profile) + restored = True + raise + return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks) + + with ( + patch("zeus.gateway_launcher.os.stat", side_effect=racing_stat), + self.assertRaises(UnsafeFileError), + ): + supervisor._read_pid_marker(record.profile_path) + + self.assertEqual(2, profile_observations) + self.assertTrue(restored) + self.assertTrue(profile.is_dir()) + self.assertTrue(marker_path.is_file()) + def _compat_marker( self, supervisor: Supervisor, diff --git a/tests/test_private_io.py b/tests/test_private_io.py index 7b2c923..cb7f73c 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -329,6 +329,69 @@ def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_res self.assertEqual(0o777, stat.S_IMODE(created_parent.stat().st_mode)) self.assertEqual(0o700, stat.S_IMODE(private_dir.stat().st_mode)) + def test_pin_directory_closes_duplicate_when_exit_binding_validation_fails(self) -> None: + private_dir = self.root / "private" + private_dir.mkdir(mode=0o700) + displaced = self.root / "private-displaced" + parent_stat = private_dir.parent.stat() + opened: set[int] = set() + duplicated: set[int] = set() + real_close = os.close + real_dup = os.dup + real_fstat = os.fstat + real_lstat = os.lstat + real_open = os.open + raced = False + + def tracking_open( + name: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + descriptor = real_open(name, flags, mode, dir_fd=dir_fd) + opened.add(descriptor) + return descriptor + + def tracking_dup(fd: int) -> int: + descriptor = real_dup(fd) + opened.add(descriptor) + duplicated.add(descriptor) + return descriptor + + def racing_lstat(name: str | bytes, *, dir_fd: int | None = None) -> os.stat_result: + nonlocal raced + if duplicated and not raced and name == private_dir.name and dir_fd is not None: + parent = real_fstat(dir_fd) + if parent.st_dev == parent_stat.st_dev and parent.st_ino == parent_stat.st_ino: + private_dir.rename(displaced) + raced = True + return real_lstat(name, dir_fd=dir_fd) + + with ( + patch.object(private_io.os, "open", side_effect=tracking_open), + patch.object(private_io.os, "dup", side_effect=tracking_dup), + patch.object(private_io.os, "lstat", side_effect=racing_lstat), + self.assertRaises(UnsafeFileError), + private_io.pin_private_directory(private_dir), + ): + self.fail("pin context must not be entered after binding drift") + + leaked: list[int] = [] + for descriptor in opened: + try: + real_fstat(descriptor) + except OSError: + continue + leaked.append(descriptor) + for descriptor in leaked: + real_close(descriptor) + + self.assertTrue(raced) + self.assertEqual(1, len(duplicated)) + self.assertEqual([], leaked) + def test_existing_file_and_parent_modes_are_tightened_before_append(self) -> None: private_dir = self.root / "logs" private_dir.mkdir(mode=0o755) diff --git a/zeus/gateway_launcher.py b/zeus/gateway_launcher.py index f1388c7..1b0d0eb 100644 --- a/zeus/gateway_launcher.py +++ b/zeus/gateway_launcher.py @@ -66,6 +66,10 @@ class LaunchPayloadError(ValueError): pass +class _ConfirmedMissing(LaunchPayloadError): + """A missing path component whose absence was rechecked on retained descriptors.""" + + def command_fingerprint(argv: list[str]) -> str: encoded = json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() @@ -388,6 +392,7 @@ def _open_profile_chain(profile_path: Path) -> _OpenedProfile: except LaunchPayloadError as exc: if _caused_by_missing_path(exc): _OpenedProfile(descriptors, tuple(names)).confirm_missing(component) + raise _ConfirmedMissing("profile path component is missing") from exc raise descriptors.append(next_fd) names.append(component) diff --git a/zeus/private_io.py b/zeus/private_io.py index da8ee34..8ec43d5 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -723,25 +723,26 @@ def pin_private_directory(path: Path) -> Iterator[_PinnedPrivateDirectory]: parts = _validate_path(path, file_path=False) platform = _require_platform() pinned_fd = -1 - with _open_directory_path(parts, create=False, platform=platform) as opened: - try: - pinned_fd = os.dup(opened.fd) - identity = os.fstat(pinned_fd) - _validate_directory_snapshots((identity,), "pinned private directory") - _validate_directory_requirement( - (identity,), - _DirectoryRequirement.exact_private, - platform.euid, - "pinned private directory", - ) - except UnsafeFileError: - if pinned_fd >= 0: - _close_suppressing_error(pinned_fd) - raise - except (OSError, TypeError, ValueError) as exc: - if pinned_fd >= 0: - _close_suppressing_error(pinned_fd) - raise UnsafeFileError("private directory could not be pinned safely") from exc + try: + with _open_directory_path(parts, create=False, platform=platform) as opened: + try: + pinned_fd = os.dup(opened.fd) + identity = os.fstat(pinned_fd) + _validate_directory_snapshots((identity,), "pinned private directory") + _validate_directory_requirement( + (identity,), + _DirectoryRequirement.exact_private, + platform.euid, + "pinned private directory", + ) + except UnsafeFileError: + raise + except (OSError, TypeError, ValueError) as exc: + raise UnsafeFileError("private directory could not be pinned safely") from exc + except BaseException: + if pinned_fd >= 0: + _close_suppressing_error(pinned_fd) + raise pinned = _PinnedPrivateDirectory(pinned_fd, identity, platform) try: yield pinned diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 460b9a1..d0ee4f7 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -36,6 +36,7 @@ MAX_PAYLOAD_BYTES, LaunchPayloadError, _confirm_marker_missing, + _ConfirmedMissing, _is_owned_runtime_marker, _open_logs, _open_profile_chain, @@ -1225,9 +1226,9 @@ def _read_strict_runtime_marker( logs_fd = marker_fd = -1 try: profile = _open_profile_chain(profile_path) + except _ConfirmedMissing: + return _MarkerObservation("missing", reason="marker is missing") except (OSError, ValueError) as exc: - if isinstance(exc.__cause__, FileNotFoundError): - return _MarkerObservation("missing", reason="marker is missing") return _MarkerObservation( "untrusted", reason=f"registered profile cannot be opened safely: {exc}" ) @@ -1279,8 +1280,8 @@ def _read_strict_runtime_marker( return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - except FileNotFoundError: - return _MarkerObservation("missing", reason="marker is missing") + except FileNotFoundError as exc: + return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") finally: for fd in (marker_fd, logs_fd): if fd >= 0: @@ -3399,6 +3400,8 @@ def _read_pid_marker(self, profile_path: str) -> dict[str, object]: profile = _open_profile_chain(safe_profile_path) logs_fd = _open_logs(profile.fd, create=False) marker_fd, marker_stat = _open_regular_marker(logs_fd) + except _ConfirmedMissing: + return {"exists": False} except (LaunchPayloadError, OSError, ValueError) as exc: if _caused_by_missing_path(exc): try: @@ -3406,6 +3409,10 @@ def _read_pid_marker(self, profile_path: str) -> dict[str, object]: _confirm_marker_missing(profile, logs_fd) elif profile is not None: profile.confirm_missing("logs") + else: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from exc except (LaunchPayloadError, OSError, ValueError) as confirm_error: raise UnsafeFileError( "PID marker absence cannot be confirmed safely" From dd1094a07159cca328b4b656383391bd9e60903a Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:08:19 +0200 Subject: [PATCH 16/49] security: reject FIFO evidence without blocking --- scripts/fresh_vps_verify.sh | 6 ++-- tests/test_fresh_vps_script.py | 52 +++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/scripts/fresh_vps_verify.sh b/scripts/fresh_vps_verify.sh index 78123ae..2ee1104 100755 --- a/scripts/fresh_vps_verify.sh +++ b/scripts/fresh_vps_verify.sh @@ -141,10 +141,10 @@ def prepare(root: str, relative_directory: str, names: list[str]) -> None: create=True, ) directory_fd = descriptors[-1] - if not hasattr(os, "O_NOFOLLOW"): - fail("O_NOFOLLOW is required for private evidence") + if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_NONBLOCK"): + fail("O_NOFOLLOW and O_NONBLOCK are required for private evidence") file_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND - file_flags |= os.O_NOFOLLOW + file_flags |= os.O_NOFOLLOW | os.O_NONBLOCK try: for name in names: if "/" in name or name in {"", ".", ".."}: diff --git a/tests/test_fresh_vps_script.py b/tests/test_fresh_vps_script.py index 1279e6a..9165368 100644 --- a/tests/test_fresh_vps_script.py +++ b/tests/test_fresh_vps_script.py @@ -4,6 +4,7 @@ import json import os import shutil +import signal import stat import subprocess import sys @@ -171,19 +172,37 @@ def _pythonless_bootstrap_path(self) -> str: apt_get.chmod(0o700) return f"{bootstrap_bin}:/bin" - def _run(self, **overrides: str) -> subprocess.CompletedProcess[str]: + def _run( + self, + *, + timeout: float = 20, + **overrides: str, + ) -> subprocess.CompletedProcess[str]: environment = self._base_environment() environment.update(overrides) - return subprocess.run( - ["/bin/bash", "scripts/fresh_vps_verify.sh"], + arguments = ["/bin/bash", "scripts/fresh_vps_verify.sh"] + process = subprocess.Popen( + arguments, cwd=self.checkout, env=environment, - input="", text=True, - capture_output=True, - check=False, - timeout=20, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, ) + try: + stdout, stderr = process.communicate("", timeout=timeout) + except subprocess.TimeoutExpired as error: + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + error.cmd, + error.timeout, + output=stdout, + stderr=stderr, + ) from None + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) def _curl_records(self) -> list[dict[str, object]]: if not self.curl_capture.exists(): @@ -371,6 +390,25 @@ def test_symlinked_evidence_file_is_rejected_without_mutating_target(self) -> No self.assertEqual(0o644, stat.S_IMODE(external.stat().st_mode)) self.assertFalse(self.installer_executed.exists()) + def test_fifo_evidence_file_is_rejected_without_blocking(self) -> None: + evidence = self.checkout / "evidence" + evidence.mkdir(mode=0o700) + fifo = evidence / "run.log" + os.mkfifo(fifo, mode=0o600) + + try: + result = self._run( + timeout=1.0, + ZEUS_VPS_HERMES_INSTALLER_SHA256=self.installer_digest, + ) + except subprocess.TimeoutExpired: + self.fail("verifier blocked while opening a pre-existing evidence FIFO") + + self.assertNotEqual(0, result.returncode) + self.assertTrue(stat.S_ISFIFO(fifo.lstat().st_mode)) + self.assertEqual(0o600, stat.S_IMODE(fifo.lstat().st_mode)) + self.assertFalse(self.installer_executed.exists()) + if __name__ == "__main__": unittest.main() From 494ade3af6129eff80859f62ad3d7f2a38c896dc Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:20:39 +0200 Subject: [PATCH 17/49] security: redact folded authorization variants --- tests/test_lifecycle_ledger.py | 102 +++++++++++++++++++++++++++++++++ zeus/sanitization.py | 5 +- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index 2994ee7..0104232 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -709,6 +709,108 @@ def test_authorization_credentials_are_redacted_from_all_persistence_sinks(self) any(credential in value for value in observed if isinstance(value, str)) ) + def test_authorization_label_variants_and_folded_credentials_are_redacted(self) -> None: + credentials = tuple(f"authorization-variant-sentinel-{index:02d}" for index in range(12)) + messages = ( + f"Authorization Header: Basic {credentials[0]}", + f'authorization_header=Digest response="{credentials[1]}"', + f"Authorization Headers: CustomScheme {credentials[2]}", + f"authorizationHeaders=Bearer {credentials[3]}", + f"authorization-header: Basic {credentials[4]}", + f'authorization.headers=Digest response="{credentials[5]}"', + f"authorizationHeader=CustomScheme {credentials[6]}", + f"authorization/header: Basic {credentials[7]}", + f"Authorization: Basic\r\n {credentials[8]}", + f'Authorization Header: Digest\n\tresponse="{credentials[9]}"', + f"authorization_headers=CustomScheme\r\n {credentials[10]}", + f"authorizationHeaders: Bearer\n\t{credentials[11]}", + ) + combined = "\nnext authorization case\n".join(messages) + started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + event = LifecycleEventInput( + bot_id="coder", + operation_id="b" * 32, + source="cli", + action="bot.start", + outcome="failure", + reason=combined, + error_code="authorization_failed", + error_message=combined, + details={"message": combined}, + ) + store.upsert_bot_with_event( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(root / "profiles" / "coder"), + last_error=combined, + last_transition_reason=combined, + ), + event=event, + ) + store.append_audit_event( + "bot.authorization_failed", + cleanup_errors=list(messages), + ) + run = ReconcileRunStart( + run_id="authorization-variant-sanitization", + scope="fleet", + requested_bot_id=None, + source="cli", + force=False, + reset_restart=False, + started_at=started_at, + ) + result = BotReconcileResult( + bot_id="coder", + outcome=ReconcileOutcome.error, + desired_state="running", + observed_status="failed", + pid=None, + action="inspect", + message=combined, + error_code="authorization_failed", + event_id=None, + started_at=started_at, + finished_at=started_at + timedelta(seconds=1), + ) + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + + with closing(sqlite3.connect(store.database_path)) as conn: + lifecycle_row = conn.execute( + "SELECT reason, error_message, details_json FROM lifecycle_events" + ).fetchone() + bot_row = conn.execute( + "SELECT last_error, last_transition_reason FROM bots WHERE bot_id = 'coder'" + ).fetchone() + reconcile_row = conn.execute( + "SELECT message FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone() + assert lifecycle_row is not None + assert bot_row is not None + assert reconcile_row is not None + observed_by_sink = { + "direct": tuple(sanitize_text(message) for message in messages), + "lifecycle": lifecycle_row, + "bot_projection": bot_row, + "reconciliation": (*reconcile_row, result.message), + "audit": (store.audit_log_path().read_text(encoding="utf-8"),), + } + for case_index, credential in enumerate(credentials): + for sink_name, observed in observed_by_sink.items(): + with self.subTest(case=case_index, sink=sink_name): + self.assertFalse( + any(credential in value for value in observed if isinstance(value, str)) + ) + def test_audit_non_bmp_event_fallback_is_strictly_byte_bounded(self) -> None: with tempfile.TemporaryDirectory() as tmp: store = StateStore(Path(tmp) / "zeus.db") diff --git a/zeus/sanitization.py b/zeus/sanitization.py index f07bb46..f7b3475 100644 --- a/zeus/sanitization.py +++ b/zeus/sanitization.py @@ -65,14 +65,15 @@ _AUTHORIZATION_RE = re.compile( r"""(?ix) (?P["']?) - (?PAUTHORIZATION) + (?PAUTHORIZATION(?:[^A-Z0-9\r\n:=]*HEADERS?)?) (?P=prefix) - (?P\s*[:=]\s*) + (?P[ \t]*[:=][ \t]*) (?P "(?:[^"\\\r\n]|\\.)*" | '(?:[^'\\\r\n]|\\.)*' | [^\r\n]+ ) + (?P(?:(?:\r\n|[\r\n])[ \t]+[^\r\n]*)*) """ ) _BEARER_RE = re.compile(r"(?i)(\bBearer\s+)([A-Za-z0-9._~+/=-]+)") From 5930546af1aaeb4d03ad046192c36ce53042f3e1 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:22:06 +0200 Subject: [PATCH 18/49] feat(cli): import bot secrets without argv exposure --- .env.example | 7 +- README.md | 24 ++- docs/OPERATIONS.md | 17 ++ tests/test_supervisor_cli_api.py | 326 ++++++++++++++++++++++++++++++- zeus/cli.py | 56 +++++- 5 files changed, 423 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 7351323..6aef88e 100644 --- a/.env.example +++ b/.env.example @@ -50,7 +50,12 @@ ZEUS_ALLOW_LEGACY_PID_MARKERS=1 # Keep empty unless required for proxies or certificate bundles. ZEUS_ENV_PASSTHROUGH= -# Provider and gateway secrets referenced by bundled templates. +# Provider and gateway secrets referenced by bundled templates. Copy this file +# to the trusted workspace ./.env, then import only the names a bot needs with +# `zeus bot create ... --env-from NAME`. A process environment value takes +# precedence over ./.env. Do not pass secrets with legacy `--env NAME=VALUE`; +# command arguments may be exposed in shell history and process listings. After +# copying this file, run `chmod 0600 .env` before adding real secrets. OPENROUTER_API_KEY= NOUS_API_KEY= OPENAI_API_KEY= diff --git a/README.md b/README.md index 5e0b5c3..62b6439 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,18 @@ cp .env.example .env zeus doctor zeus template list -zeus bot create coder --template coding-bot +zeus bot create coder --template coding-bot --env-from OPENROUTER_API_KEY zeus bot doctor coder ``` +`--env-from NAME` imports a named value from the process environment first and +then the trusted workspace `./.env`; the value never enters the Zeus argument +list or command output. A present but empty process value is an error and does +not fall back to `.env`. Keep the workspace `.env` private with `chmod 0600 .env`. +The legacy `--env NAME=VALUE` form remains available +for non-secret compatibility values, but is unsafe for secrets because command +arguments can be retained in shell history and exposed in process listings. + Safety model: Zeus is a local process orchestrator, not a sandbox. Use Docker or another Hermes terminal backend for untrusted tasks. Do not expose the API directly to a network; keep it on loopback or behind a separately hardened @@ -288,6 +296,20 @@ source tree. Rendered `.env` values are serialized with quoting when needed so whitespace, `#`, quotes, and backslashes cannot create extra assignments. +Import template secrets by name instead of placing their values in argv: + +```bash +zeus bot create coder \ + --template coding-bot \ + --env-from OPENROUTER_API_KEY +``` + +Zeus looks for each imported name in the process environment and then the +trusted workspace `./.env`. Process values take precedence. Missing and empty +values fail bot creation without printing the value. Keep `./.env` mode `0600`; +Zeus also writes the imported values only to the selected profile's mode-`0600` +`.env` file. + Built-in templates include OpenRouter-backed bots and `deepseek-coding-bot`, which uses Hermes' native DeepSeek provider with `DEEPSEEK_API_KEY`. Example templates also cover gateway operations, log triage, and documentation writing. Each template should set a bounded async delegation cap: diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0405af3..c36a5d9 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -354,6 +354,23 @@ Hermes child processes receive a minimal environment by default plus variables rendered into the bot profile `.env`. Zeus does not pass the full API service or operator shell environment to child processes. +Import bot secrets without putting their values in command arguments: + +```bash +zeus bot create coder \ + --template coding-bot \ + --env-from OPENROUTER_API_KEY +``` + +For every `--env-from NAME`, Zeus reads the process environment first and then +the trusted workspace `./.env`. A present but empty process value fails closed; +it does not fall back to `.env`. Missing and empty errors identify only the +variable name, and imported values are not printed. Keep the trusted source +private with `chmod 0600 .env`. Zeus persists imported values only in the +selected bot profile's `.env`, which Zeus writes with mode `0600`. The legacy +`--env NAME=VALUE` form remains compatible for non-secret values but is unsafe +for secrets because argv can be visible in shell history and process listings. + To pass selected host variables, set an explicit allowlist: ```dotenv diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index dddf449..6b7c132 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import hmac import http.client import io import json @@ -21,10 +22,11 @@ from zeus.api import main as api_main from zeus.api import make_handler -from zeus.cli import _services, build_parser +from zeus.cli import _parse_env, _services, build_parser from zeus.cli import main as cli_main from zeus.config import Settings from zeus.doctor import _check_runtime_paths, run_doctor +from zeus.envfile import parse_env_text from zeus.errors import BotArchiveError, BotDeleteError, BotExistsError from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEventInput @@ -127,6 +129,19 @@ def _run_cli_failure(self, argv: list[str]) -> str: self.assertEqual(1, cli_main(argv)) return stdout.getvalue() + def _run_cli_result(self, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = cli_main(argv) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def _copy_cli_template(self, root: Path, template_id: str = "coding-bot") -> None: + templates = root / "templates" + templates.mkdir() + source = Path(__file__).resolve().parents[1] / "templates" / f"{template_id}.toml" + (templates / source.name).write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + def _fake_hermes_path(self, root: Path) -> str: hermes = root / "bin" / "hermes" hermes.parent.mkdir(parents=True, exist_ok=True) @@ -2070,6 +2085,315 @@ def test_cli_creates_bot(self) -> None: finally: os.chdir(old_cwd) + def test_cli_create_help_recommends_env_from_and_warns_legacy_env_is_unsafe(self) -> None: + stdout = io.StringIO() + with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: + build_parser().parse_args(["bot", "create", "--help"]) + + self.assertEqual(0, raised.exception.code) + help_text = stdout.getvalue() + self.assertIn("--env-from", help_text) + self.assertIn("process environment", help_text) + self.assertIn("unsafe for secrets", help_text) + + def test_cli_env_from_process_value_wins_without_entering_parser_namespace_or_output( + self, + ) -> None: + process_secret = "process-precedence-sentinel" + dotenv_secret = "dotenv-precedence-sentinel" + parsed = build_parser().parse_args( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + self.assertEqual(["OPENROUTER_API_KEY"], parsed.env_from) + self.assertFalse(process_secret in repr(vars(parsed))) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + (root / ".env").write_text(f"OPENROUTER_API_KEY={dotenv_secret}\n", encoding="utf-8") + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENROUTER_API_KEY": process_secret, + }, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], process_secret) + ) + self.assertFalse(dotenv_secret in profile_env) + self.assertFalse(process_secret in stdout + stderr) + self.assertFalse(dotenv_secret in stdout + stderr) + audit_text = (root / ".zeus" / "logs" / "audit.jsonl").read_text(encoding="utf-8") + self.assertFalse(process_secret in audit_text) + self.assertFalse(dotenv_secret in audit_text) + database_bytes = (root / ".zeus" / "zeus.db").read_bytes() + self.assertFalse(process_secret.encode() in database_bytes) + self.assertFalse(dotenv_secret.encode() in database_bytes) + + def test_cli_env_from_uses_trusted_dotenv_fallback_without_echoing_value(self) -> None: + dotenv_secret = "trusted-dotenv-fallback-sentinel" + unrequested_secret = "unrequested-dotenv-sentinel" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + (root / ".env").write_text( + f"OPENROUTER_API_KEY={dotenv_secret}\nOPENAI_API_KEY={unrequested_secret}\n", + encoding="utf-8", + ) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + {"ZEUS_STATE_DIR": str(root / ".zeus")}, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], dotenv_secret) + ) + self.assertFalse(dotenv_secret in stdout + stderr) + self.assertFalse(unrequested_secret in profile_env) + self.assertFalse(unrequested_secret in stdout + stderr) + profile_env_path = root / ".zeus" / "hermes" / "profiles" / "coder" / ".env" + self.assertEqual(0o600, stat.S_IMODE(profile_env_path.stat().st_mode)) + + def test_cli_env_from_missing_and_empty_values_fail_before_service_creation(self) -> None: + fallback_secret = "must-not-fall-back-sentinel" + for value_state in ("missing", "empty"): + for as_json in (False, True): + with ( + self.subTest(value_state=value_state, as_json=as_json), + tempfile.TemporaryDirectory() as tmp, + ): + root = Path(tmp) + self._copy_cli_template(root) + if value_state == "empty": + (root / ".env").write_text( + f"OPENROUTER_API_KEY={fallback_secret}\n", encoding="utf-8" + ) + env = {"ZEUS_STATE_DIR": str(root / ".zeus")} + if value_state == "empty": + env["OPENROUTER_API_KEY"] = "" + argv = [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENROUTER_API_KEY", + ] + if as_json: + argv.append("--json") + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict(os.environ, env, clear=True): + exit_code, stdout, stderr = self._run_cli_result(argv) + finally: + os.chdir(old_cwd) + + self.assertEqual(1, exit_code) + output = stdout + stderr + self.assertIn("OPENROUTER_API_KEY", output) + self.assertIn("missing or empty", output) + self.assertFalse(fallback_secret in output) + self.assertFalse((root / ".zeus").exists()) + if as_json: + payload = json.loads(stdout) + self.assertEqual("invalid_request", payload["error"]["code"]) + self.assertEqual("", stderr) + else: + self.assertEqual("", stdout) + + def test_cli_env_from_rejects_duplicate_and_invalid_names_without_values(self) -> None: + imported_secret = "imported-duplicate-sentinel" + legacy_secret = "legacy-duplicate-sentinel" + invalid_name = "invalid-name\ninjected-control-text" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENROUTER_API_KEY": imported_secret, + }, + clear=True, + ): + duplicate = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env", + f"OPENROUTER_API_KEY={legacy_secret}", + "--env-from", + "OPENROUTER_API_KEY", + ] + ) + invalid_import = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + invalid_name, + ] + ) + finally: + os.chdir(old_cwd) + + for exit_code, stdout, stderr in (duplicate, invalid_import): + self.assertEqual(1, exit_code) + self.assertFalse(imported_secret in stdout + stderr) + self.assertFalse(legacy_secret in stdout + stderr) + self.assertIn("provided by both --env and --env-from", duplicate[2]) + self.assertIn("valid environment variable name", invalid_import[2]) + self.assertFalse(invalid_name in invalid_import[2]) + self.assertNotIn("injected-control-text", invalid_import[2]) + self.assertFalse((root / ".zeus").exists()) + + def test_cli_env_from_value_is_not_disclosed_when_template_rejects_key(self) -> None: + secret = "controlled-failure-sentinel" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(root / ".zeus"), + "OPENAI_API_KEY": secret, + }, + clear=True, + ): + exit_code, stdout, stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env-from", + "OPENAI_API_KEY", + "--json", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(1, exit_code) + payload = json.loads(stdout) + self.assertEqual("invalid_request", payload["error"]["code"]) + self.assertIn("OPENAI_API_KEY", payload["error"]["message"]) + self.assertFalse(secret in stdout + stderr) + self.assertFalse((root / ".zeus" / "hermes" / "profiles" / "coder").exists()) + + def test_cli_legacy_env_syntax_remains_compatible(self) -> None: + legacy_value = "legacy-compatible-value" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._copy_cli_template(root) + old_cwd = Path.cwd() + try: + os.chdir(root) + with patch.dict( + os.environ, + {"ZEUS_STATE_DIR": str(root / ".zeus")}, + clear=True, + ): + exit_code, _stdout, _stderr = self._run_cli_result( + [ + "bot", + "create", + "coder", + "--template", + "coding-bot", + "--env", + f"OPENROUTER_API_KEY={legacy_value}", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(0, exit_code) + profile_env = (root / ".zeus" / "hermes" / "profiles" / "coder" / ".env").read_text( + encoding="utf-8" + ) + parsed_profile_env = parse_env_text(profile_env) + self.assertTrue( + hmac.compare_digest(parsed_profile_env["OPENROUTER_API_KEY"], legacy_value) + ) + + def test_cli_legacy_env_parser_keeps_historical_permissive_and_error_contract(self) -> None: + self.assertEqual({"legacy-key": "value"}, _parse_env(["legacy-key=value"])) + + malformed = "legacy-key" + with self.assertRaises(SystemExit) as raised: + _parse_env([malformed]) + self.assertEqual(f"--env must be NAME=VALUE, got {malformed!r}", str(raised.exception)) + def test_cli_lifecycle_failures_return_nonzero(self) -> None: with tempfile.TemporaryDirectory() as tmp: state_dir = Path(tmp) / ".zeus" diff --git a/zeus/cli.py b/zeus/cli.py index 100312a..a8decfa 100644 --- a/zeus/cli.py +++ b/zeus/cli.py @@ -2,16 +2,18 @@ import argparse import json +import os import shutil import sys import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import cast from zeus.api import serve, template_to_dict -from zeus.config import Settings +from zeus.config import Settings, load_dotenv from zeus.doctor import report_to_json, report_to_text, run_doctor +from zeus.envfile import ENV_KEY_RE from zeus.errors import ZeusConflictError from zeus.models import BotCreateRequest, BotStatus, BotStatusResponse, RestartPolicy, TemplateError from zeus.process_lock import LockTimeoutError @@ -60,7 +62,17 @@ def build_parser() -> argparse.ArgumentParser: "--env", action="append", default=[], - help="NAME=VALUE for env keys declared by the selected template", + help=( + "NAME=VALUE for env keys declared by the selected template; " + "unsafe for secrets because values enter argv" + ), + ) + create.add_argument( + "--env-from", + action="append", + default=[], + metavar="NAME", + help="import NAME from the process environment, then trusted ./.env, without argv values", ) create.add_argument("--restart-policy", choices=["manual", "on-failure"], default="manual") create.add_argument("--restart-backoff-seconds", type=float, default=5.0) @@ -234,8 +246,44 @@ def _parse_env(pairs: list[str]) -> dict[str, str]: return values +def _resolve_create_env( + pairs: list[str], + imported_names: list[str], + *, + process_env: Mapping[str, str] | None = None, +) -> dict[str, str]: + values = _parse_env(pairs) + for name in imported_names: + if not ENV_KEY_RE.fullmatch(name): + raise ValueError("--env-from requires a valid environment variable name") + + duplicate_names = sorted(set(values).intersection(imported_names)) + if duplicate_names: + raise ValueError( + "environment variable provided by both --env and --env-from: " + + ", ".join(duplicate_names) + ) + + source_env = os.environ if process_env is None else process_env + needs_dotenv = any(name not in source_env for name in imported_names) + dotenv = load_dotenv(Path(".env")) if needs_dotenv else {} + for name in imported_names: + value = source_env[name] if name in source_env else dotenv.get(name) + if value is None or value == "": + raise ValueError(f"environment variable {name} is missing or empty") + values[name] = value + return values + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + create_env: dict[str, str] = {} + if args.resource == "bot" and args.action == "create": + try: + create_env = _resolve_create_env(args.env, args.env_from) + except ValueError as exc: + return _print_cli_error("invalid_request", str(exc), as_json=args.as_json) + try: settings = Settings.from_env() except ValueError as exc: @@ -305,7 +353,7 @@ def main(argv: list[str] | None = None) -> int: bot_id=args.bot_id, template_id=args.template_id, display_name=args.display_name, - env=_parse_env(args.env), + env=create_env, restart_policy=RestartPolicy(args.restart_policy), restart_backoff_seconds=args.restart_backoff_seconds, restart_max_attempts=args.restart_max_attempts, From 987c4983f6952b372a0573c95abfadc0c62e5dae Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:34:46 +0200 Subject: [PATCH 19/49] security: redact delimiter-first authorization folds --- tests/test_lifecycle_ledger.py | 6 +++++- zeus/sanitization.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index 0104232..547cd19 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -710,7 +710,7 @@ def test_authorization_credentials_are_redacted_from_all_persistence_sinks(self) ) def test_authorization_label_variants_and_folded_credentials_are_redacted(self) -> None: - credentials = tuple(f"authorization-variant-sentinel-{index:02d}" for index in range(12)) + credentials = tuple(f"authorization-variant-sentinel-{index:02d}" for index in range(16)) messages = ( f"Authorization Header: Basic {credentials[0]}", f'authorization_header=Digest response="{credentials[1]}"', @@ -724,6 +724,10 @@ def test_authorization_label_variants_and_folded_credentials_are_redacted(self) f'Authorization Header: Digest\n\tresponse="{credentials[9]}"', f"authorization_headers=CustomScheme\r\n {credentials[10]}", f"authorizationHeaders: Bearer\n\t{credentials[11]}", + f"Authorization:\r\n Basic {credentials[12]}", + f'Authorization Header:\n\tDigest response="{credentials[13]}"', + f"authorization_headers=\r\n CustomScheme {credentials[14]}", + f'"Authorization":\n "Basic {credentials[15]}"', ) combined = "\nnext authorization case\n".join(messages) started_at = datetime(2026, 7, 21, 12, 0, tzinfo=UTC) diff --git a/zeus/sanitization.py b/zeus/sanitization.py index f7b3475..b455e8c 100644 --- a/zeus/sanitization.py +++ b/zeus/sanitization.py @@ -68,6 +68,7 @@ (?PAUTHORIZATION(?:[^A-Z0-9\r\n:=]*HEADERS?)?) (?P=prefix) (?P[ \t]*[:=][ \t]*) + (?P(?:(?:\r\n|[\r\n])[ \t]+)?) (?P "(?:[^"\\\r\n]|\\.)*" | '(?:[^'\\\r\n]|\\.)*' | From 8587b0405160e164396f6cf7ee91a89a25506c5c Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:41:15 +0200 Subject: [PATCH 20/49] docs: make CLI secret setup explicit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 62b6439..048f9a8 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ python3 -m venv .venv . .venv/bin/activate pip install -e . cp .env.example .env +chmod 0600 .env +# Edit .env and set OPENROUTER_API_KEY to a non-empty provider key. zeus doctor zeus template list From 2701b87c159ff4f45c5d284ec0bd34b3d0803f4c Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:45:47 +0200 Subject: [PATCH 21/49] feat(cli): add version and descriptive help --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- CHANGELOG.md | 3 + tests/test_supervisor_cli_api.py | 158 ++++++++++++ zeus/cli.py | 353 +++++++++++++++++++++----- 4 files changed, 455 insertions(+), 61 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index af86253..d0a3bb1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -7,7 +7,7 @@ body: id: zeus-version attributes: label: Zeus version - description: Output from `zeus --version` or the installed package version. + description: Run `zeus --version` and paste the complete output. validations: required: true - type: input diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bcea3..b08a42f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Added descriptive CLI help and a state-free `zeus --version` command backed by the package + version. + ## 0.3.0 - Relaunched the public repository from an audited, single-root history on the `main` branch while diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 6b7c132..4f0c119 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -20,6 +20,7 @@ from pathlib import Path from unittest.mock import patch +from zeus import __version__ from zeus.api import main as api_main from zeus.api import make_handler from zeus.cli import _parse_env, _services, build_parser @@ -2085,6 +2086,163 @@ def test_cli_creates_bot(self) -> None: finally: os.chdir(old_cwd) + def test_cli_version_is_single_sourced_and_has_no_runtime_side_effects(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + old_cwd = Path.cwd() + stdout = io.StringIO() + stderr = io.StringIO() + try: + os.chdir(root) + with ( + patch("zeus.cli.Settings.from_env") as from_env, + patch("zeus.cli._services") as services, + patch("zeus.cli._demo_services") as demo_services, + redirect_stdout(stdout), + redirect_stderr(stderr), + self.assertRaises(SystemExit) as raised, + ): + cli_main(["--version"]) + + self.assertEqual(0, raised.exception.code) + self.assertEqual(f"zeus {__version__}\n", stdout.getvalue()) + self.assertEqual("", stderr.getvalue()) + from_env.assert_not_called() + services.assert_not_called() + demo_services.assert_not_called() + self.assertFalse((root / ".zeus").exists()) + finally: + os.chdir(old_cwd) + + def test_cli_help_is_descriptive_and_has_no_runtime_side_effects(self) -> None: + help_cases = ( + ( + ["--help"], + ( + "Manage local Hermes bot gateways and the Zeus API.", + "Run the local Zeus HTTP API server.", + "Check Zeus configuration, runtime paths, and dependencies.", + "Inspect available Hermes bot templates.", + "Run the bundled fake-Hermes lifecycle demo.", + "Create, inspect, and manage Hermes bot gateways.", + ), + ), + ( + ["serve", "--help"], + ("Run the local Zeus HTTP API server.", "bind host", "listen port", "4311"), + ), + ( + ["doctor", "--help"], + ( + "Check Zeus configuration, runtime paths, and dependencies.", + "machine-readable JSON", + "treat warnings as failures", + ), + ), + ( + ["template", "--help"], + ("Inspect available Hermes bot templates.", "List available bot templates."), + ), + ( + ["demo", "--help"], + ( + "Run the bundled fake-Hermes lifecycle demo.", + "Create and start the demo bot.", + "Show the demo bot status.", + "Stop the demo bot.", + ), + ), + ( + ["bot", "--help"], + ( + "Create, inspect, and manage Hermes bot gateways.", + "Create a bot profile from a template.", + "List registered bots.", + "Delete a bot registration.", + "Archive a bot profile.", + "Reconcile desired and observed bot state.", + "Show bot diagnostics.", + "Show immutable lifecycle history.", + "Start a bot gateway.", + "Stop a bot gateway gracefully.", + "Restart a bot gateway.", + "Show current bot status.", + "Show redacted bot logs.", + "Run Hermes doctor for a bot profile.", + ), + ), + ( + ["bot", "create", "--help"], + ( + "Create a bot profile from a template.", + "--replace", + "replace an existing bot", + "stop a running bot before replacement", + ), + ), + ( + ["bot", "start", "--help"], + ("Start a bot gateway.", "--wait", "wait for readiness", "--no-wait"), + ), + ( + ["bot", "reconcile", "--help"], + ( + "Reconcile desired and observed bot state.", + "--force", + "eligible restart now", + "--reset-restart", + "reset restart backoff", + ), + ), + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + old_cwd = Path.cwd() + try: + os.chdir(root) + with ( + patch("zeus.cli.Settings.from_env") as from_env, + patch("zeus.cli._services") as services, + patch("zeus.cli._demo_services") as demo_services, + ): + for argv, expected_fragments in help_cases: + with self.subTest(argv=argv): + stdout = io.StringIO() + stderr = io.StringIO() + with ( + redirect_stdout(stdout), + redirect_stderr(stderr), + self.assertRaises(SystemExit) as raised, + ): + cli_main(argv) + + self.assertEqual(0, raised.exception.code) + self.assertEqual("", stderr.getvalue()) + help_text = " ".join(stdout.getvalue().split()) + for fragment in expected_fragments: + self.assertIn(fragment, help_text) + + from_env.assert_not_called() + services.assert_not_called() + demo_services.assert_not_called() + self.assertFalse((root / ".zeus").exists()) + finally: + os.chdir(old_cwd) + + def test_bug_report_version_command_and_changelog_are_current(self) -> None: + bug_template = Path(".github/ISSUE_TEMPLATE/bug_report.yml").read_text(encoding="utf-8") + changelog = Path("CHANGELOG.md").read_text(encoding="utf-8") + unreleased = changelog.split("## 0.3.0", 1)[0] + + self.assertIn( + "description: Run `zeus --version` and paste the complete output.", + bug_template, + ) + self.assertNotIn("or the installed package version", bug_template) + self.assertIn("descriptive CLI help", unreleased) + self.assertIn("`zeus --version`", unreleased) + def test_cli_create_help_recommends_env_from_and_warns_legacy_env_is_unsafe(self) -> None: stdout = io.StringIO() with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: diff --git a/zeus/cli.py b/zeus/cli.py index a8decfa..7fd7408 100644 --- a/zeus/cli.py +++ b/zeus/cli.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import cast +from zeus import __version__ from zeus.api import serve, template_to_dict from zeus.config import Settings, load_dotenv from zeus.doctor import report_to_json, report_to_text, run_doctor @@ -28,36 +29,112 @@ def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="zeus") + parser = argparse.ArgumentParser( + prog="zeus", + description="Manage local Hermes bot gateways and the Zeus API.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {__version__}", + ) sub = parser.add_subparsers(dest="resource", required=True) - serve_cmd = sub.add_parser("serve") - serve_cmd.add_argument("--host") - serve_cmd.add_argument("--port", type=int) - - doctor = sub.add_parser("doctor") - doctor.add_argument("--json", action="store_true", dest="as_json") - doctor.add_argument("--strict", action="store_true") + serve_description = "Run the local Zeus HTTP API server." + serve_cmd = sub.add_parser( + "serve", + help=serve_description, + description=serve_description, + ) + serve_cmd.add_argument("--host", help="bind host (default: ZEUS_HOST or 127.0.0.1)") + serve_cmd.add_argument("--port", type=int, help="listen port (default: ZEUS_PORT or 4311)") + + doctor_description = "Check Zeus configuration, runtime paths, and dependencies." + doctor = sub.add_parser( + "doctor", + help=doctor_description, + description=doctor_description, + ) + doctor.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + doctor.add_argument( + "--strict", + action="store_true", + help="treat warnings as failures", + ) - template = sub.add_parser("template") + template_description = "Inspect available Hermes bot templates." + template = sub.add_parser( + "template", + help=template_description, + description=template_description, + ) template_sub = template.add_subparsers(dest="action", required=True) - template_list = template_sub.add_parser("list") - template_list.add_argument("--json", action="store_true", dest="as_json") + template_list_description = "List available bot templates." + template_list = template_sub.add_parser( + "list", + help=template_list_description, + description=template_list_description, + ) + template_list.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) - demo = sub.add_parser("demo") + demo_description = "Run the bundled fake-Hermes lifecycle demo." + demo = sub.add_parser( + "demo", + help=demo_description, + description=demo_description, + ) demo_sub = demo.add_subparsers(dest="action", required=True) - for action in ["up", "status", "down"]: - command = demo_sub.add_parser(action) - command.add_argument("--bot-id", default=DEMO_BOT_ID) - command.add_argument("--json", action="store_true", dest="as_json") + demo_actions = { + "up": "Create and start the demo bot.", + "status": "Show the demo bot status.", + "down": "Stop the demo bot.", + } + for action, description in demo_actions.items(): + command = demo_sub.add_parser(action, help=description, description=description) + command.add_argument( + "--bot-id", + default=DEMO_BOT_ID, + help=f"demo bot ID (default: {DEMO_BOT_ID})", + ) + command.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) - bot = sub.add_parser("bot") + bot_description = "Create, inspect, and manage Hermes bot gateways." + bot = sub.add_parser( + "bot", + help=bot_description, + description=bot_description, + ) bot_sub = bot.add_subparsers(dest="action", required=True) - create = bot_sub.add_parser("create") - create.add_argument("bot_id") - create.add_argument("--template", required=True, dest="template_id") - create.add_argument("--name", dest="display_name") + create_description = "Create a bot profile from a template." + create = bot_sub.add_parser( + "create", + help=create_description, + description=create_description, + ) + create.add_argument("bot_id", help="unique bot ID") + create.add_argument( + "--template", + required=True, + dest="template_id", + help="template ID used to render the bot profile", + ) + create.add_argument("--name", dest="display_name", help="display name (defaults to bot ID)") create.add_argument( "--env", action="append", @@ -74,59 +151,215 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAME", help="import NAME from the process environment, then trusted ./.env, without argv values", ) - create.add_argument("--restart-policy", choices=["manual", "on-failure"], default="manual") - create.add_argument("--restart-backoff-seconds", type=float, default=5.0) - create.add_argument("--restart-max-attempts", type=int, default=5) - create.add_argument("--replace", action="store_true", dest="replace_existing") - create.add_argument("--stop", action="store_true", dest="stop_if_running") - create.add_argument("--json", action="store_true", dest="as_json") - - bot_list = bot_sub.add_parser("list") - bot_list.add_argument("--json", action="store_true", dest="as_json") - delete = bot_sub.add_parser("delete") - delete.add_argument("bot_id") - delete.add_argument("--stop", action="store_true", dest="stop_if_running") - delete.add_argument("--remove-profile", action="store_true") - delete.add_argument("--json", action="store_true", dest="as_json") - archive = bot_sub.add_parser("archive") - archive.add_argument("bot_id") - archive.add_argument("--stop", action="store_true", dest="stop_if_running") - archive.add_argument("--json", action="store_true", dest="as_json") - reconcile = bot_sub.add_parser("reconcile") - reconcile.add_argument("bot_id", nargs="?") - reconcile.add_argument("--json", action="store_true", dest="as_json") - reconcile.add_argument("--summary", action="store_true") - reconcile.add_argument("--force", action="store_true") - reconcile.add_argument("--reset-restart", action="store_true") - inspect = bot_sub.add_parser("inspect") - inspect.add_argument("bot_id") - inspect.add_argument("--json", action="store_true", dest="as_json") - history = bot_sub.add_parser("history") - history.add_argument("bot_id") - history.add_argument("--limit", type=_history_limit, default=50) - history.add_argument("--before", type=_positive_event_id) - history.add_argument("--json", action="store_true", dest="as_json") - for action in ["start", "stop", "restart", "status", "logs", "doctor"]: - command = bot_sub.add_parser(action) - command.add_argument("bot_id") + create.add_argument( + "--restart-policy", + choices=["manual", "on-failure"], + default="manual", + help="automatic restart policy (default: manual)", + ) + create.add_argument( + "--restart-backoff-seconds", + type=float, + default=5.0, + help="initial on-failure restart backoff in seconds (default: 5)", + ) + create.add_argument( + "--restart-max-attempts", + type=int, + default=5, + help="maximum consecutive restart attempts (default: 5)", + ) + create.add_argument( + "--replace", + action="store_true", + dest="replace_existing", + help="replace an existing bot (requires --stop if it is running)", + ) + create.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop a running bot before replacement", + ) + create.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + + list_description = "List registered bots." + bot_list = bot_sub.add_parser( + "list", + help=list_description, + description=list_description, + ) + bot_list.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + delete_description = "Delete a bot registration." + delete = bot_sub.add_parser( + "delete", + help=delete_description, + description=delete_description, + ) + delete.add_argument("bot_id", help="bot ID") + delete.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop the bot before deletion", + ) + delete.add_argument( + "--remove-profile", + action="store_true", + help="also remove the managed profile directory", + ) + delete.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + archive_description = "Archive a bot profile." + archive = bot_sub.add_parser( + "archive", + help=archive_description, + description=archive_description, + ) + archive.add_argument("bot_id", help="bot ID") + archive.add_argument( + "--stop", + action="store_true", + dest="stop_if_running", + help="stop the bot before archiving", + ) + archive.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + reconcile_description = "Reconcile desired and observed bot state." + reconcile = bot_sub.add_parser( + "reconcile", + help=reconcile_description, + description=reconcile_description, + ) + reconcile.add_argument("bot_id", nargs="?", help="bot ID; omit to reconcile all bots") + reconcile.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + reconcile.add_argument( + "--summary", + action="store_true", + help="emit persisted run metadata and per-bot results", + ) + reconcile.add_argument( + "--force", + action="store_true", + help="run an eligible restart now instead of waiting for backoff", + ) + reconcile.add_argument( + "--reset-restart", + action="store_true", + help="reset restart backoff and retry budget before reconciling", + ) + inspect_description = "Show bot diagnostics." + inspect = bot_sub.add_parser( + "inspect", + help=inspect_description, + description=inspect_description, + ) + inspect.add_argument("bot_id", help="bot ID") + inspect.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + history_description = "Show immutable lifecycle history." + history = bot_sub.add_parser( + "history", + help=history_description, + description=history_description, + ) + history.add_argument("bot_id", help="bot ID") + history.add_argument( + "--limit", + type=_history_limit, + default=50, + help="maximum events to return, 1-1000 (default: 50)", + ) + history.add_argument( + "--before", + type=_positive_event_id, + help="return events before this exclusive event ID", + ) + history.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) + lifecycle_descriptions = { + "start": "Start a bot gateway.", + "stop": "Stop a bot gateway gracefully.", + "restart": "Restart a bot gateway.", + "status": "Show current bot status.", + "logs": "Show redacted bot logs.", + "doctor": "Run Hermes doctor for a bot profile.", + } + for action, description in lifecycle_descriptions.items(): + command = bot_sub.add_parser(action, help=description, description=description) + command.add_argument("bot_id", help="bot ID") if action in {"start", "restart"}: - command.add_argument("--wait", dest="wait", action="store_true", default=False) - command.add_argument("--no-wait", dest="wait", action="store_false") - command.add_argument("--timeout", type=float, dest="timeout_seconds") + command.add_argument( + "--wait", + dest="wait", + action="store_true", + default=False, + help="wait for readiness before returning", + ) + command.add_argument( + "--no-wait", + dest="wait", + action="store_false", + help="return after launch without waiting for readiness (default)", + ) + command.add_argument( + "--timeout", + type=float, + dest="timeout_seconds", + help="readiness timeout in seconds", + ) if action == "stop": command.add_argument( "--kill-after-timeout", dest="kill_after_timeout", action="store_true", default=None, + help="send SIGKILL if graceful shutdown times out", ) command.add_argument( "--no-kill-after-timeout", dest="kill_after_timeout", action="store_false", + help="never escalate a timed-out graceful shutdown", ) if action == "logs": - command.add_argument("--json", action="store_true", dest="as_json") + command.add_argument( + "--json", + action="store_true", + dest="as_json", + help="emit machine-readable JSON", + ) return parser From 7a3cd857ab6b7dd1fd0240a33c2071ab6627adbd Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:51:05 +0200 Subject: [PATCH 22/49] docs: clarify onboarding compatibility and roadmap --- CONTRIBUTING.md | 12 +++++-- README.md | 56 +++++++++++++++++++++-------- docs/COMPATIBILITY.md | 58 ++++++++++++++++++++++++++++++ docs/ROADMAP.md | 42 +++++++++++++++------- tests/test_repo_contracts.py | 70 ++++++++++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 30 deletions(-) create mode 100644 docs/COMPATIBILITY.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba3bf58..e68619c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,10 +7,15 @@ Zeus is intentionally small and workspace-local. Changes should keep the project ```bash python3 -m venv .venv . .venv/bin/activate -pip install -e . -sh scripts/test.sh +python -m pip install -e ".[dev]" +make check ``` +See the [compatibility policy](docs/COMPATIBILITY.md) for the operating systems, +Python versions, and Hermes boundary covered by committed automation. The +real-Hermes check below is a separate release gate because it depends on the +operator's installed Hermes version. + ## Quality Bar - Keep the core runtime dependency-free unless there is a clear operational reason. @@ -25,7 +30,8 @@ sh scripts/test.sh ## Real Hermes Verification -The default test suite uses a fake Hermes executable to verify Zeus process handling. Before release, run: +The default test suite uses a fake Hermes executable to verify Zeus process handling. +Before release, separately run: ```bash sh scripts/verify_real_hermes.sh diff --git a/README.md b/README.md index 048f9a8..7f7fd51 100644 --- a/README.md +++ b/README.md @@ -48,14 +48,45 @@ run summaries at that boundary. ## Quick Start +### 1. Credential-free offline demo + +The fastest first success needs neither Hermes nor provider credentials. From a +checkout: + ```bash python3 -m venv .venv . .venv/bin/activate -pip install -e . +python -m pip install -e . + +zeus demo up +zeus demo status +zeus demo down +``` + +The demo uses Zeus' packaged fake-Hermes executable and stores its disposable +runtime under `ZEUS_STATE_DIR` (the workspace-local `.zeus/` directory by +default). It exercises real profile rendering and process lifecycle behavior +without contacting a provider. + +### 2. Real Hermes setup + +Check the installed Hermes version, then prepare a private workspace secret +file: + +```bash +hermes version cp .env.example .env chmod 0600 .env -# Edit .env and set OPENROUTER_API_KEY to a non-empty provider key. +``` + +`.env.example` contains empty placeholders and is not ready to import. Stop here +until `.env` contains a real, non-empty provider key required by the selected +template, such as `OPENROUTER_API_KEY` for `coding-bot`. As an alternative, +provide the same named secret through a secure process-environment mechanism. + +Then validate Zeus and render the real Hermes profile: +```bash zeus doctor zeus template list zeus bot create coder --template coding-bot --env-from OPENROUTER_API_KEY @@ -84,8 +115,10 @@ ZEUS_API_KEY=change-me sh scripts/start.sh ## 60-Second Demo -The asciinema recording in [docs/assets/demo.cast](docs/assets/demo.cast) mirrors the local -operator flow: +The pre-recorded asciinema cast in [docs/assets/demo.cast](docs/assets/demo.cast) +illustrates the local operator flow. It is not evidence that the current Zeus +checkout is compatible with whichever Hermes version is installed today; use +the live verification steps below for that evidence. ```bash zeus doctor @@ -108,6 +141,7 @@ zeus bot stop coder - [Operations](docs/OPERATIONS.md) - [Reconcile scheduling](docs/RECONCILE.md) - [Release process](docs/RELEASE.md) +- [Compatibility policy](docs/COMPATIBILITY.md) - [Roadmap](docs/ROADMAP.md) - [Contributing](CONTRIBUTING.md) - [Code of conduct](CODE_OF_CONDUCT.md) @@ -122,16 +156,10 @@ Zeus is maintained by [BrainX](https://github.com/brainx). See [Credits](CREDITS - Hermes Agent installed as `hermes` for real bot startup - Optional Docker or another Hermes terminal backend for stronger execution isolation -No Python package dependencies are required for the current MVP. - -## Setup - -```bash -python3 -m venv .venv -. .venv/bin/activate -pip install -e . -cp .env.example .env -``` +Zeus has no required third-party Python runtime dependencies. Development and +build tools are available separately through the optional `dev` dependency +group; see [Contributing](CONTRIBUTING.md). The exact automated platform and +Python matrix is recorded in the [compatibility policy](docs/COMPATIBILITY.md). ## Install Modes diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md new file mode 100644 index 0000000..e094f3a --- /dev/null +++ b/docs/COMPATIBILITY.md @@ -0,0 +1,58 @@ +# Compatibility Policy + +This document records compatibility evidence produced by the current committed +automation. It distinguishes repeatable CI from manual checks and does not turn +an untested platform or external Hermes release into a support claim. + +## Automated matrix + +| Gate | Committed runner | Python | Scope | +| --- | --- | --- | --- | +| Main CI matrix | Linux `ubuntu-latest` | Python 3.11, 3.12, and 3.13 | Unit and integration tests, repository contracts, source-and-branch coverage, formatting, lint, typing, Bandit, and ShellCheck | +| Subprocess lifecycle | Linux `ubuntu-latest` | Python 3.11 | Focused multi-process lifecycle and locking behavior | +| Package build | Linux `ubuntu-latest` | Python 3.11 | Wheel and source build, installed-wheel smoke test, and metadata checks | +| Tagged release build | Linux `ubuntu-latest` | Python 3.11 | Full release gate, artifact checksums, and GitHub release artifacts | + +In short, the focused lifecycle and package jobs use Python 3.11. The +`ubuntu-latest` label is the exact committed runner selection, but GitHub manages +the underlying Linux image and may update it over time. macOS and Windows are +not currently automated. Results from an individual developer machine are local +evidence for that run, not an automated platform guarantee. + +The package metadata requires Python 3.11 or newer, while committed CI currently +tests the versions listed above. A version absent from that matrix is not covered +by the current automated compatibility claim. + +## Manual clean-host evidence + +[`scripts/fresh_vps_verify.sh`](../scripts/fresh_vps_verify.sh) provides a manual +clean-host runbook for Debian and Ubuntu. It can bootstrap OS packages, install +Zeus into a virtual environment, run local gates, render multiple profiles, and +exercise the loopback API. Optional Hermes installation and live probes cross an +external network and credential boundary, so their logs are evidence for that +specific host and invocation rather than deterministic CI. + +Local development checks such as `make check` and `sh scripts/wheel_smoke.sh` +remain useful evidence, but they do not add the developer's operating system to +the automated matrix. + +## Hermes boundary + +No Hermes version is pinned by this repository. There is no deterministic +real-Hermes CI gate today. The manual +[`scripts/verify_real_hermes.sh`](../scripts/verify_real_hermes.sh) check uses +whichever `hermes` executable is installed on `PATH`: it runs strict diagnostics, +renders a profile, invokes Hermes doctor, and can optionally start a loopback +gateway and probe its health. + +Record `hermes version` with manual verification evidence. Passing against one +installed version does not establish compatibility with every Hermes release. +Before a Hermes baseline becomes required automation, the repository must name +the exact verified version or immutable source, install it reproducibly, and run +the real-Hermes gate without provider secrets in logs or command arguments. + +## Updating this policy + +Update this file in the same change that adds or removes a CI runner, Python +version, package gate, or reproducible Hermes baseline. Aspirational platforms +belong in the [roadmap](ROADMAP.md), not in the automated matrix. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 11aff9a..eb5105e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,20 +1,36 @@ # Roadmap -## v0.1.x +## Current status -- Stable local CLI -- Template validation -- Real Hermes verification -- API hardening -- Restart policies +Zeus v0.3.0 is alpha software with a local-first, host-local scope. It owns +profiles, processes, lifecycle safety, and reconciliation evidence on one host. -## v0.2.x +## Shipped -- Export/import -- TUI dashboard +- Workspace-local CLI and loopback API. +- Bundled and custom Hermes template validation and rendering. +- PID ownership checks, lifecycle locking, crash recovery, and restart policy. +- SQLite lifecycle and reconciliation evidence. +- Credential-free fake-Hermes demo and manual real-Hermes verification scripts. +- Wheel builds, installed-wheel smoke checks, and GitHub release artifacts. -## v0.3.x +## Near term -- Plugin registry -- Distributed agent groups -- Remote-safe API mode +- Keep local and CI quality gates aligned with the measured coverage baseline. +- Strengthen installed-package behavior and compatibility evidence. +- Decompose large internal modules without changing the public CLI, API, or + persisted schemas. +- Improve local operational readiness, backup guidance, and health evidence. + +## Under evaluation + +- Workspace-local configuration export and import that never exports secrets. +- A local TUI for lifecycle status and reconciliation history. +- Local plugin discovery with explicit trust and compatibility boundaries. + +## Out of scope + +The out-of-scope responsibilities are cross-host placement, distributed +approvals, fleet rollout policy, and control-plane ownership. They belong to +[Olymp](https://github.com/brainx/olymp), not Zeus. Zeus will keep a narrow +host-local API and durable evidence boundary that Olymp can consume. diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 8e4de59..62c5753 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -27,6 +27,7 @@ def test_publishable_repository_files_exist(self) -> None: "docs/OPERATIONS.md", "docs/RECONCILE.md", "docs/RELEASE.md", + "docs/COMPATIBILITY.md", "docs/openapi.json", "docs/ROADMAP.md", "docs/assets/demo.cast", @@ -188,6 +189,75 @@ def test_readme_has_informative_github_landing_sections(self) -> None: self.assertIn("Do not expose the API", readme) self.assertIn("## Known Limitations", readme) + def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> None: + readme = Path("README.md").read_text(encoding="utf-8") + contributing = Path("CONTRIBUTING.md").read_text(encoding="utf-8") + roadmap = Path("docs/ROADMAP.md").read_text(encoding="utf-8") + roadmap_text = " ".join(roadmap.split()) + compatibility_path = Path("docs/COMPATIBILITY.md") + + self.assertTrue(compatibility_path.is_file()) + compatibility = compatibility_path.read_text(encoding="utf-8") + compatibility_text = " ".join(compatibility.split()) + + offline_heading = "### 1. Credential-free offline demo" + hermes_heading = "### 2. Real Hermes setup" + self.assertLess(readme.index(offline_heading), readme.index(hermes_heading)) + for command in ( + "python3 -m venv .venv", + "python -m pip install -e .", + "zeus demo up", + "zeus demo status", + "zeus demo down", + ): + self.assertIn(command, readme) + for command in ( + "hermes version", + "cp .env.example .env", + "chmod 0600 .env", + "zeus doctor", + "--env-from OPENROUTER_API_KEY", + "zeus bot doctor coder", + ): + self.assertIn(command, readme) + self.assertIn("real, non-empty provider key", readme) + self.assertIn("docs/COMPATIBILITY.md", readme) + self.assertIn("no required third-party Python runtime dependencies", readme) + + self.assertIn('python -m pip install -e ".[dev]"', contributing) + self.assertIn("make check", contributing) + self.assertIn("docs/COMPATIBILITY.md", contributing) + self.assertIn("sh scripts/verify_real_hermes.sh", contributing) + + for heading in ( + "## Current status", + "## Shipped", + "## Near term", + "## Under evaluation", + "## Out of scope", + ): + self.assertIn(heading, roadmap) + for statement in ( + "v0.3.0", + "alpha", + "host-local", + "cross-host placement", + "distributed approvals", + "fleet rollout policy", + "control-plane ownership", + "Olymp", + ): + self.assertIn(statement, roadmap_text) + + self.assertIn("`ubuntu-latest`", compatibility_text) + self.assertIn("Python 3.11, 3.12, and 3.13", compatibility_text) + self.assertIn("focused lifecycle and package jobs use Python 3.11", compatibility_text) + self.assertIn("Debian and Ubuntu", compatibility_text) + self.assertIn("No Hermes version is pinned", compatibility_text) + self.assertIn("whichever `hermes` executable is installed", compatibility_text) + self.assertIn("macOS and Windows are not currently automated", compatibility_text) + self.assertNotIn("Python 3.14", compatibility_text) + def test_env_example_lists_deepseek_and_api_auth(self) -> None: env = Path(".env.example").read_text(encoding="utf-8") api_docs = Path("docs/API.md").read_text(encoding="utf-8") From 706ee5b55415faf0bff00662bb8457c99673534d Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:57:48 +0200 Subject: [PATCH 23/49] test: tighten onboarding compatibility contracts --- docs/COMPATIBILITY.md | 6 +++--- tests/test_repo_contracts.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index e094f3a..6da2ec0 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -19,9 +19,9 @@ the underlying Linux image and may update it over time. macOS and Windows are not currently automated. Results from an individual developer machine are local evidence for that run, not an automated platform guarantee. -The package metadata requires Python 3.11 or newer, while committed CI currently -tests the versions listed above. A version absent from that matrix is not covered -by the current automated compatibility claim. +The package metadata declares `requires-python = ">=3.11"`, while committed CI +currently tests the versions listed above. A version absent from that matrix is +not covered by the current automated compatibility claim. ## Manual clean-host evidence diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 62c5753..ccf6d1e 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -194,6 +194,9 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No contributing = Path("CONTRIBUTING.md").read_text(encoding="utf-8") roadmap = Path("docs/ROADMAP.md").read_text(encoding="utf-8") roadmap_text = " ".join(roadmap.split()) + ci_workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + release_workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + pyproject = Path("pyproject.toml").read_text(encoding="utf-8") compatibility_path = Path("docs/COMPATIBILITY.md") self.assertTrue(compatibility_path.is_file()) @@ -203,6 +206,8 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No offline_heading = "### 1. Credential-free offline demo" hermes_heading = "### 2. Real Hermes setup" self.assertLess(readme.index(offline_heading), readme.index(hermes_heading)) + offline_path = readme.split(offline_heading, 1)[1].split(hermes_heading, 1)[0] + hermes_path = readme.split(hermes_heading, 1)[1].split("\n## ", 1)[0] for command in ( "python3 -m venv .venv", "python -m pip install -e .", @@ -210,7 +215,7 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No "zeus demo status", "zeus demo down", ): - self.assertIn(command, readme) + self.assertIn(command, offline_path) for command in ( "hermes version", "cp .env.example .env", @@ -219,8 +224,8 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No "--env-from OPENROUTER_API_KEY", "zeus bot doctor coder", ): - self.assertIn(command, readme) - self.assertIn("real, non-empty provider key", readme) + self.assertIn(command, hermes_path) + self.assertIn("real, non-empty provider key", hermes_path) self.assertIn("docs/COMPATIBILITY.md", readme) self.assertIn("no required third-party Python runtime dependencies", readme) @@ -249,8 +254,24 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No ): self.assertIn(statement, roadmap_text) - self.assertIn("`ubuntu-latest`", compatibility_text) - self.assertIn("Python 3.11, 3.12, and 3.13", compatibility_text) + workflow_text = ci_workflow + "\n" + release_workflow + workflow_runners = set( + re.findall(r"(?m)^\s*runs-on:\s*([^\s#]+)", workflow_text) + ) + workflow_python_versions = set(re.findall(r'"(3\.\d+)"', workflow_text)) + documented_python_versions = set( + re.findall(r"(? Date: Tue, 21 Jul 2026 23:07:08 +0200 Subject: [PATCH 24/49] test: ratchet coverage and align quality gates --- .coveragerc | 2 +- .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++--- Makefile | 3 +++ docs/COMPATIBILITY.md | 24 +++++++++++------ pyproject.toml | 1 - scripts/test.sh | 15 +++++++++-- tests/test_api.py | 11 ++++---- tests/test_repo_contracts.py | 46 ++++++++++++++++++++++++-------- tests/test_supervisor_cli_api.py | 2 +- 9 files changed, 114 insertions(+), 32 deletions(-) diff --git a/.coveragerc b/.coveragerc index f0149e9..e0a33c5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ source = zeus [report] -fail_under = 70 +fail_under = 79 precision = 2 show_missing = True skip_covered = True diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b05889c..9a9064c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,8 @@ concurrency: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -50,8 +51,23 @@ jobs: coverage run -m unittest discover -s tests coverage report + python-3-14: + continue-on-error: true + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.14" + - name: Install package and test tools + run: python -m pip install -e ".[dev]" + - name: Run Zeus test suite + run: sh scripts/test.sh + lifecycle-subprocess: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -62,8 +78,26 @@ jobs: - name: Run subprocess lifecycle tests run: python -m unittest tests.test_subprocess_lifecycle + macos-process-lifecycle: + runs-on: macos-26 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + - name: Install package and test tools + run: python -m pip install -e ".[dev]" + - name: Run focused process lifecycle tests + run: | + python -m unittest -v \ + tests.test_subprocess_lifecycle \ + tests.test_fake_hermes_integration \ + tests.test_crash_recovery.GatewayLauncherTests + package: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -71,6 +105,8 @@ jobs: python-version: "3.11" - name: Install package build tools run: python -m pip install -e ".[dev]" + - name: Check installed dependencies + run: python -m pip check - name: Build package run: | rm -rf dist diff --git a/Makefile b/Makefile index fc9a699..2ec7e7a 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,10 @@ check: ruff check . mypy zeus bandit -r zeus + shellcheck scripts/*.sh build: + python -m pip check python -m build twine check dist/* @@ -44,6 +46,7 @@ release-check: mypy zeus bandit -r zeus shellcheck scripts/*.sh + python -m pip check rm -rf dist python -m build ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 6da2ec0..d4af822 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -8,16 +8,24 @@ an untested platform or external Hermes release into a support claim. | Gate | Committed runner | Python | Scope | | --- | --- | --- | --- | -| Main CI matrix | Linux `ubuntu-latest` | Python 3.11, 3.12, and 3.13 | Unit and integration tests, repository contracts, source-and-branch coverage, formatting, lint, typing, Bandit, and ShellCheck | -| Subprocess lifecycle | Linux `ubuntu-latest` | Python 3.11 | Focused multi-process lifecycle and locking behavior | -| Package build | Linux `ubuntu-latest` | Python 3.11 | Wheel and source build, installed-wheel smoke test, and metadata checks | +| Main CI matrix | Linux `ubuntu-24.04` | Python 3.11, 3.12, and 3.13 | Unit and integration tests, repository contracts, source-and-branch coverage, formatting, lint, typing, Bandit, and ShellCheck | +| Provisional Python compatibility | Linux `ubuntu-24.04` | Python 3.14 | Full Zeus test suite; non-required and Zeus-only because no Hermes baseline is pinned | +| Subprocess lifecycle | Linux `ubuntu-24.04` | Python 3.11 | Focused multi-process lifecycle and locking behavior | +| macOS process lifecycle | macOS `macos-26` | Python 3.13 | Focused process, fake-Hermes integration, and gateway-launcher recovery tests | +| Package build | Linux `ubuntu-24.04` | Python 3.11 | Wheel and source build, installed-wheel smoke test, dependency consistency, and metadata checks | | Tagged release build | Linux `ubuntu-latest` | Python 3.11 | Full release gate, artifact checksums, and GitHub release artifacts | -In short, the focused lifecycle and package jobs use Python 3.11. The -`ubuntu-latest` label is the exact committed runner selection, but GitHub manages -the underlying Linux image and may update it over time. macOS and Windows are -not currently automated. Results from an individual developer machine are local -evidence for that run, not an automated platform guarantee. +In short, the focused Linux lifecycle and package jobs use Python 3.11. Main CI +uses the explicit `ubuntu-24.04` image, while the separate tagged-release +workflow still uses `ubuntu-latest`. The focused macOS lane uses `macos-26` and +Python 3.13. Windows is not currently automated. GitHub manages the contents of +all hosted runner images and may update them over time; results from an +individual developer machine remain local evidence rather than an automated +platform guarantee. + +Python 3.14 is a provisional Zeus-only lane with `continue-on-error` behavior. +It does not promote Python 3.14 to required Hermes compatibility: the repository +has not yet pinned and passed a Hermes baseline on that interpreter. The package metadata declares `requires-python = ">=3.11"`, while committed CI currently tests the versions listed above. A version absent from that matrix is diff --git a/pyproject.toml b/pyproject.toml index 672ebc3..d615921 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ dev = [ "build>=1.2.1", "coverage>=7.0.0", "twine>=5.1.1", - "pytest>=8.3.0", ] [project.scripts] diff --git a/scripts/test.sh b/scripts/test.sh index 6a42de8..4656e27 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,13 +4,24 @@ set -eu tmp_dir=".tmp/test" +warning_log="$tmp_dir/unittest-stderr.log" cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT INT TERM -python3 -B -m compileall zeus tests -python3 -B -m unittest discover -s tests -v mkdir -p "$tmp_dir" +python3 -B -m compileall zeus tests +unittest_status=0 +python3 -B -W error::ResourceWarning -m unittest discover -s tests -v \ + 2>"$warning_log" || unittest_status=$? +cat "$warning_log" >&2 +if grep -F "ResourceWarning" "$warning_log" >/dev/null; then + echo "ResourceWarning detected in the test suite." >&2 + exit 1 +fi +if [ "$unittest_status" -ne 0 ]; then + exit "$unittest_status" +fi python3 -B -m zeus.cli doctor --json >"$tmp_dir/zeus-doctor.json" python3 -B -m zeus.cli template list >"$tmp_dir/zeus-templates.txt" diff --git a/tests/test_api.py b/tests/test_api.py index c8aa253..b02659e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1052,7 +1052,7 @@ def test_reconcile_summary_returns_canonical_fleet_and_bot_run_payloads(self) -> }, set(payload["results"][0]), ) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 2, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -1114,7 +1114,7 @@ def test_reconcile_summary_query_is_strict_and_missing_bot_remains_404(self) -> ) self.assertEqual(404, status) self.assertEqual("unknown_bot", body["error"]["code"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 0, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -1291,7 +1291,7 @@ def claim_after_registry_growth( self.assertEqual(["coder"], [item["bot_id"] for item in first["results"]]) self.assertNotIn("idempotency-replayed", first_headers) self.assertEqual("true", replay_headers["idempotency-replayed"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 1, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], @@ -1417,7 +1417,7 @@ def reconcile_summary(self, *args: object, **kwargs: object) -> ReconcileRunSumm ) self.assertEqual(1, effects) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( (0, 1), ( @@ -1449,7 +1449,7 @@ def test_keyed_summary_over_replay_budget_is_rejected_before_run(self) -> None: self.assertEqual(422, status) self.assertEqual("idempotency_response_too_large", body["error"]["code"]) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( (0, 0), ( @@ -2382,6 +2382,7 @@ def test_second_api_process_cannot_overwrite_pid_marker(self) -> None: except subprocess.TimeoutExpired: first.kill() first.wait(timeout=5) + first.communicate() class ApiRateLimitTests(unittest.TestCase): diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index ccf6d1e..1444a61 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -69,10 +69,21 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") self.assertIn("workflow_dispatch:", workflow) + self.assertIn("runs-on: ubuntu-24.04", workflow) + self.assertNotIn("runs-on: ubuntu-latest", workflow) self.assertIn("3.11", workflow) self.assertIn("3.12", workflow) self.assertIn("3.13", workflow) + self.assertIn("python-3-14:", workflow) + self.assertIn('python-version: "3.14"', workflow) + self.assertIn("continue-on-error: true", workflow) + self.assertIn("macos-process-lifecycle:", workflow) + self.assertIn("runs-on: macos-26", workflow) + self.assertIn("tests.test_subprocess_lifecycle", workflow) + self.assertIn("tests.test_fake_hermes_integration", workflow) + self.assertIn("tests.test_crash_recovery.GatewayLauncherTests", workflow) self.assertIn('pip install -e ".[dev]"', workflow) + self.assertIn("python -m pip check", workflow) self.assertIn("ruff format --check .", workflow) self.assertIn("ruff check .", workflow) self.assertIn("mypy zeus", workflow) @@ -92,6 +103,9 @@ def test_test_script_runs_compile_unittest_and_doctor(self) -> None: self.assertIn("compileall zeus tests", script) self.assertIn("unittest discover -s tests -v", script) + self.assertIn("-W error::ResourceWarning", script) + self.assertIn('warning_log="$tmp_dir/unittest-stderr.log"', script) + self.assertIn('grep -F "ResourceWarning" "$warning_log"', script) self.assertIn("trap cleanup EXIT INT TERM", script) self.assertIn('mkdir -p "$tmp_dir"', script) self.assertIn("zeus.cli doctor --json", script) @@ -118,7 +132,8 @@ def test_coverage_gate_measures_production_source_and_branches(self) -> None: self.assertIn("branch = True", config) self.assertRegex(config, r"(?m)^source =\s*$") self.assertRegex(config, r"(?m)^[ \t]+zeus[ \t]*$") - self.assertIn("fail_under = 70", config) + self.assertIn("fail_under = 79", config) + self.assertNotIn("fail_under = 70", config) self.assertIn("precision = 2", config) def test_workflow_actions_are_pinned_to_immutable_commits(self) -> None: @@ -255,13 +270,9 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No self.assertIn(statement, roadmap_text) workflow_text = ci_workflow + "\n" + release_workflow - workflow_runners = set( - re.findall(r"(?m)^\s*runs-on:\s*([^\s#]+)", workflow_text) - ) + workflow_runners = set(re.findall(r"(?m)^\s*runs-on:\s*([^\s#]+)", workflow_text)) workflow_python_versions = set(re.findall(r'"(3\.\d+)"', workflow_text)) - documented_python_versions = set( - re.findall(r"(? No f'`requires-python = "{requires_python.group(1)}"`', compatibility_text, ) - self.assertIn("focused lifecycle and package jobs use Python 3.11", compatibility_text) + self.assertIn("lifecycle and package jobs use Python 3.11", compatibility_text) self.assertIn("Debian and Ubuntu", compatibility_text) self.assertIn("No Hermes version is pinned", compatibility_text) self.assertIn("whichever `hermes` executable is installed", compatibility_text) - self.assertIn("macOS and Windows are not currently automated", compatibility_text) - self.assertNotIn("Python 3.14", compatibility_text) + self.assertIn("Python 3.14", compatibility_text) + self.assertIn("provisional Zeus-only", compatibility_text) + self.assertIn("focused process", compatibility_text.lower()) + self.assertIn("Windows is not currently automated", compatibility_text) def test_env_example_lists_deepseek_and_api_auth(self) -> None: env = Path(".env.example").read_text(encoding="utf-8") @@ -334,7 +347,8 @@ def test_observability_and_lifecycle_history_are_documented(self) -> None: def test_architecture_terminal_schema_compatibility_matches_runtime(self) -> None: architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8") compatibility_statements = re.findall( - r"Databases newer than\s+schema v(?P\d+) are rejected rather than downgraded\.", + r"Databases newer than\s+schema v(?P\d+) " + r"are rejected rather than downgraded\.", architecture, ) @@ -411,6 +425,7 @@ def test_pyproject_has_no_placeholder_repository_urls(self) -> None: self.assertIn("mypy>=1.11.0", pyproject) self.assertIn("bandit>=1.7.9", pyproject) self.assertIn("coverage>=7.0.0", pyproject) + self.assertNotIn('"pytest', pyproject) self.assertIn('dynamic = ["version"]', pyproject) self.assertIn('version = {attr = "zeus.__version__"}', pyproject) self.assertIn('Repository = "https://github.com/brainx/zeus"', pyproject) @@ -565,6 +580,9 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: def test_makefile_has_release_check_target(self) -> None: makefile = Path("Makefile").read_text(encoding="utf-8") + check_recipe = re.search(r"(?m)^check:\n(?P(?:\t[^\n]*\n)+)", makefile) + build_recipe = re.search(r"(?m)^build:\n(?P(?:\t[^\n]*\n)+)", makefile) + release_recipe = re.search(r"(?m)^release-check:\n(?P(?:\t[^\n]*\n)+)", makefile) self.assertIn("release-check:", makefile) self.assertIn("coverage:", makefile) @@ -574,6 +592,12 @@ def test_makefile_has_release_check_target(self) -> None: self.assertIn("coverage report", makefile) self.assertNotIn("coverage report --fail-under", makefile) self.assertIn("shellcheck scripts/*.sh", makefile) + self.assertIsNotNone(check_recipe) + self.assertIsNotNone(build_recipe) + self.assertIsNotNone(release_recipe) + self.assertIn("shellcheck scripts/*.sh", check_recipe.group("body")) + self.assertIn("python -m pip check", build_recipe.group("body")) + self.assertIn("python -m pip check", release_recipe.group("body")) self.assertIn("rm -rf dist", makefile) self.assertIn("python -m build", makefile) self.assertIn("ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", makefile) diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 4f0c119..84eda6e 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -2848,7 +2848,7 @@ def test_cli_reconcile_summary_json_runs_once_and_human_output_is_deterministic( }, payload["counts"], ) - with sqlite3.connect(state_dir / "zeus.db") as conn: + with closing(sqlite3.connect(state_dir / "zeus.db")) as conn: self.assertEqual( 1, conn.execute("SELECT COUNT(*) FROM reconcile_runs").fetchone()[0], From f695096c0c0cab515285edb15cecdbb5d7a0e954 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:13:00 +0200 Subject: [PATCH 25/49] build: verify installed CLI behavior --- .github/workflows/ci.yml | 2 +- scripts/wheel_smoke.sh | 65 ++++++++++++++++++++++++++++++++---- tests/test_repo_contracts.py | 61 ++++++++++++++++++++++++++++++--- 3 files changed, 116 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a9064c..5306bc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: run: | rm -rf dist python -m build - - name: Smoke test built wheel + - name: Verify installed wheel behavior run: ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh - name: Check package metadata run: twine check dist/* diff --git a/scripts/wheel_smoke.sh b/scripts/wheel_smoke.sh index 26c05a5..c313729 100755 --- a/scripts/wheel_smoke.sh +++ b/scripts/wheel_smoke.sh @@ -6,11 +6,19 @@ set -eu repo_root="$(pwd)" tmp_dir="$repo_root/.tmp/wheel-smoke" build_artifacts="${ZEUS_WHEEL_SMOKE_BUILD:-1}" +venv_python="" +venv_zeus="" +venv_fake_hermes="" +state_dir="$tmp_dir/state" +demo_started=0 fail() { echo "wheel smoke failed: $*" >&2 exit 1 } cleanup() { + if [ "$demo_started" = "1" ] && [ -n "$venv_zeus" ] && [ -x "$venv_zeus" ]; then + ZEUS_STATE_DIR="$state_dir" "$venv_zeus" demo down --json >/dev/null 2>&1 || true + fi rm -rf "$tmp_dir" } trap cleanup EXIT INT TERM @@ -39,20 +47,63 @@ wheel_path="$1" "$python_cmd" -m venv "$tmp_dir/venv" venv_python="$tmp_dir/venv/bin/python" venv_zeus="$tmp_dir/venv/bin/zeus" -"$venv_python" -m pip install "$wheel_path" +venv_fake_hermes="$tmp_dir/venv/bin/zeus-fake-hermes" +PIP_NO_INDEX=1 "$venv_python" -m pip install --no-deps "$wheel_path" cd "$tmp_dir" +unset PYTHONPATH PYTHONHOME +export PYTHONNOUSERSITE=1 +export PATH="$tmp_dir/venv/bin:$PATH" +export ZEUS_STATE_DIR="$state_dir" + +installed_fake_hermes="$(command -v zeus-fake-hermes || true)" +[ "$installed_fake_hermes" = "$venv_fake_hermes" ] || + fail "PATH did not select the installed zeus-fake-hermes entry point" + +[ ! -e "$state_dir" ] || fail "state directory exists before stateless CLI checks" +"$venv_zeus" --help >zeus-help.txt +grep -F "usage: zeus" zeus-help.txt >/dev/null +metadata_version=$("$venv_python" -c \ + 'from importlib.metadata import version; print(version("zeus-hermes-orchestrator"))') +module_version=$("$venv_python" -c 'import zeus; print(zeus.__version__)') +cli_version=$("$venv_zeus" --version) +module_path=$("$venv_python" -c \ + 'from pathlib import Path; import zeus; print(Path(zeus.__file__).resolve())') +[ "$metadata_version" = "$module_version" ] || + fail "distribution and module versions differ: $metadata_version != $module_version" +[ "$cli_version" = "zeus $metadata_version" ] || + fail "CLI version does not match package metadata: $cli_version" +case "$module_path" in + "$tmp_dir"/venv/*) ;; + *) fail "zeus imported outside the isolated virtual environment: $module_path" ;; +esac +[ ! -e "$state_dir" ] || fail "help or version checks created runtime state" + +"$venv_python" -m pip check "$venv_zeus" template list >template-list.txt "$venv_zeus" doctor --json >doctor.json -"$venv_python" -c "import zeus; print(zeus.__version__)" -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo up --json >demo-up.json -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo status --json >demo-status.json -ZEUS_STATE_DIR="$tmp_dir/state" "$venv_zeus" demo down --json >demo-down.json +for template_id in \ + coding-bot \ + deepseek-coding-bot \ + docs-writer-bot \ + gateway-operator \ + log-triage-bot \ + research-bot \ + support-gateway; do + grep "^${template_id}[[:space:]]" template-list.txt >/dev/null || + fail "installed wheel is missing bundled template: $template_id" +done -grep "coding-bot" template-list.txt >/dev/null -grep "deepseek-coding-bot" template-list.txt >/dev/null grep '"checks"' doctor.json >/dev/null +demo_started=1 +"$venv_zeus" demo up --json >demo-up.json +"$venv_zeus" demo status --json >demo-status.json +"$venv_zeus" demo down --json >demo-down.json +demo_started=0 + grep '"fake_hermes_bin"' demo-up.json >/dev/null +grep -F "\"fake_hermes_bin\": \"$venv_fake_hermes\"" demo-up.json >/dev/null +grep '"status": "running"' demo-up.json >/dev/null grep '"status": "running"' demo-status.json >/dev/null grep '"status": "stopped"' demo-down.json >/dev/null diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 1444a61..7eec1d7 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -96,7 +96,9 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: self.assertNotIn("coverage report --fail-under", workflow) self.assertIn("python -m build", workflow) self.assertIn("twine check dist/*", workflow) - self.assertIn("sh scripts/wheel_smoke.sh", workflow) + self.assertIn("- name: Verify installed wheel behavior", workflow) + self.assertIn("ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", workflow) + self.assertNotIn("- name: Smoke test built wheel", workflow) def test_test_script_runs_compile_unittest_and_doctor(self) -> None: script = Path("scripts/test.sh").read_text(encoding="utf-8") @@ -113,9 +115,60 @@ def test_test_script_runs_compile_unittest_and_doctor(self) -> None: def test_wheel_smoke_exercises_installed_demo_entrypoint(self) -> None: script = Path("scripts/wheel_smoke.sh").read_text(encoding="utf-8") - self.assertIn('"$venv_zeus" demo up --json', script) - self.assertIn('"$venv_zeus" demo status --json', script) - self.assertIn('"$venv_zeus" demo down --json', script) + trap_index = script.index("trap cleanup EXIT INT TERM") + for initialization in ('venv_zeus=""', 'state_dir="$tmp_dir/state"', "demo_started=0"): + self.assertIn(initialization, script) + self.assertLess(script.index(initialization), trap_index) + + cleanup = script[script.index("cleanup() {") : trap_index] + self.assertIn('[ "$demo_started" = "1" ]', cleanup) + self.assertIn('ZEUS_STATE_DIR="$state_dir" "$venv_zeus" demo down --json', cleanup) + self.assertLess(cleanup.index("demo down --json"), cleanup.index('rm -rf "$tmp_dir"')) + + help_command = '"$venv_zeus" --help' + pip_check_command = '"$venv_python" -m pip check' + self.assertIn(help_command, script) + self.assertIn(pip_check_command, script) + cd_index = script.index('cd "$tmp_dir"') + help_index = script.index(help_command) + pip_check_index = script.index(pip_check_command) + self.assertLess(cd_index, help_index) + self.assertLess(cd_index, pip_check_index) + self.assertIn("unset PYTHONPATH PYTHONHOME", script) + self.assertIn("export PYTHONNOUSERSITE=1", script) + self.assertIn('export PATH="$tmp_dir/venv/bin:$PATH"', script) + self.assertIn("command -v zeus-fake-hermes", script) + self.assertIn('"$venv_zeus" --help >zeus-help.txt', script) + self.assertIn('grep -F "usage: zeus" zeus-help.txt', script) + + self.assertIn('version("zeus-hermes-orchestrator")', script) + self.assertIn("import zeus; print(zeus.__version__)", script) + self.assertIn('cli_version=$("$venv_zeus" --version)', script) + self.assertIn('[ "$metadata_version" = "$module_version" ]', script) + self.assertIn('[ "$cli_version" = "zeus $metadata_version" ]', script) + self.assertIn("zeus.__file__", script) + self.assertGreaterEqual(script.count('[ ! -e "$state_dir" ]'), 2) + + for template_id in ( + "coding-bot", + "deepseek-coding-bot", + "docs-writer-bot", + "gateway-operator", + "log-triage-bot", + "research-bot", + "support-gateway", + ): + self.assertIn(template_id, script) + + self.assertIn('export ZEUS_STATE_DIR="$state_dir"', script) + self.assertIn('"$venv_zeus" doctor --json', script) + up_index = script.index('"$venv_zeus" demo up --json') + status_index = script.index('"$venv_zeus" demo status --json') + down_index = script.index('"$venv_zeus" demo down --json', up_index) + self.assertLess(up_index, status_index) + self.assertLess(status_index, down_index) + self.assertNotEqual(-1, script.rfind("demo_started=1", 0, up_index)) + self.assertNotEqual(-1, script.find("demo_started=0", down_index)) self.assertIn('"fake_hermes_bin"', script) self.assertIn('"status": "running"', script) self.assertIn('"status": "stopped"', script) From b7bfe0380931477d97327b009a3f78ea1df13d58 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:20:13 +0200 Subject: [PATCH 26/49] build: canonicalize wheel smoke paths --- scripts/wheel_smoke.sh | 2 +- tests/test_repo_contracts.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/wheel_smoke.sh b/scripts/wheel_smoke.sh index c313729..08b4cad 100755 --- a/scripts/wheel_smoke.sh +++ b/scripts/wheel_smoke.sh @@ -3,7 +3,7 @@ # Maintained by BrainX: https://github.com/brainx set -eu -repo_root="$(pwd)" +repo_root="$(pwd -P)" tmp_dir="$repo_root/.tmp/wheel-smoke" build_artifacts="${ZEUS_WHEEL_SMOKE_BUILD:-1}" venv_python="" diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 7eec1d7..1ac7826 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -115,6 +115,13 @@ def test_test_script_runs_compile_unittest_and_doctor(self) -> None: def test_wheel_smoke_exercises_installed_demo_entrypoint(self) -> None: script = Path("scripts/wheel_smoke.sh").read_text(encoding="utf-8") + canonical_root = 'repo_root="$(pwd -P)"' + tmp_dir = 'tmp_dir="$repo_root/.tmp/wheel-smoke"' + self.assertIn(canonical_root, script) + self.assertIn(tmp_dir, script) + self.assertLess(script.index(canonical_root), script.index(tmp_dir)) + self.assertNotIn('repo_root="$(pwd)"', script) + trap_index = script.index("trap cleanup EXIT INT TERM") for initialization in ('venv_zeus=""', 'state_dir="$tmp_dir/state"', "demo_started=0"): self.assertIn(initialization, script) From 323695d665a2915e42cb5ec58cdfa95cca79ead9 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:32:51 +0200 Subject: [PATCH 27/49] test: exercise quality gate contracts --- tests/test_repo_contracts.py | 447 +++++++++++++++++++++++++++++++---- 1 file changed, 398 insertions(+), 49 deletions(-) diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 1ac7826..87598e0 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -1,13 +1,110 @@ from __future__ import annotations import json +import os import re +import shutil +import subprocess +import tempfile +import textwrap import unittest from pathlib import Path from zeus.state import SCHEMA_VERSION +def _workflow_job_bodies(workflow: str) -> dict[str, str]: + jobs_header = re.search(r"(?m)^jobs:\s*$", workflow) + if jobs_header is None: + raise ValueError("workflow has no jobs mapping") + + jobs_text = workflow[jobs_header.end() :] + next_top_level = re.search(r"(?m)^[A-Za-z_][A-Za-z0-9_-]*:\s*", jobs_text) + if next_top_level is not None: + jobs_text = jobs_text[: next_top_level.start()] + + headers = list(re.finditer(r"(?m)^ (?P[A-Za-z0-9_-]+):\s*$", jobs_text)) + if not headers: + raise ValueError("workflow jobs mapping is empty") + + jobs: dict[str, str] = {} + for index, header in enumerate(headers): + name = header.group("name") + if name in jobs: + raise ValueError(f"duplicate workflow job: {name}") + end = headers[index + 1].start() if index + 1 < len(headers) else len(jobs_text) + jobs[name] = jobs_text[header.end() : end] + return jobs + + +def _job_level_scalar(job_body: str, key: str) -> str: + values = re.findall( + rf"(?m)^ {re.escape(key)}:\s*(?P[^\s#][^\n#]*?)\s*(?:#.*)?$", + job_body, + ) + if len(values) != 1: + raise ValueError(f"expected one job-level {key}, found {len(values)}") + return values[0] + + +def _job_python_versions(job_body: str) -> tuple[str, ...]: + matrix = re.search(r"(?m)^ python-version:\s*(?P\[[^\n]+\])\s*$", job_body) + if matrix is not None: + versions = json.loads(matrix.group("versions")) + if not isinstance(versions, list) or not all(isinstance(item, str) for item in versions): + raise ValueError("python-version matrix must be a list of strings") + return tuple(versions) + + setup_versions = re.findall( + r'(?m)^ python-version:\s*"(?P3\.\d+)"\s*$', job_body + ) + if len(setup_versions) != 1: + raise ValueError(f"expected one setup-python version, found {len(setup_versions)}") + return tuple(setup_versions) + + +def _job_run_commands(job_body: str) -> tuple[str, ...]: + lines = job_body.splitlines() + commands: list[str] = [] + index = 0 + while index < len(lines): + match = re.match(r"^ run:\s*(?P.*)$", lines[index]) + if match is None: + index += 1 + continue + + value = match.group("value").strip() + if value not in {"|", "|-"}: + commands.append(value) + index += 1 + continue + + index += 1 + block: list[str] = [] + while index < len(lines): + line = lines[index] + if line.strip() and len(line) - len(line.lstrip(" ")) <= 8: + break + block.append(line) + index += 1 + commands.append(textwrap.dedent("\n".join(block)).strip()) + return tuple(commands) + + +def _markdown_table_rows(markdown: str) -> dict[str, tuple[str, ...]]: + rows: dict[str, tuple[str, ...]] = {} + for line in markdown.splitlines(): + if not line.startswith("|"): + continue + cells = tuple(cell.strip() for cell in line.strip().strip("|").split("|")) + if not cells or cells[0] == "Gate" or set(cells[0]) <= {"-", ":"}: + continue + if cells[0] in rows: + raise ValueError(f"duplicate Markdown table row: {cells[0]}") + rows[cells[0]] = cells[1:] + return rows + + class RepoContractTests(unittest.TestCase): def test_publishable_repository_files_exist(self) -> None: required = [ @@ -67,50 +164,224 @@ def test_publishable_repository_files_exist(self) -> None: def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + jobs = _workflow_job_bodies(workflow) + + expected_runners = { + "test": "ubuntu-24.04", + "python-3-14": "ubuntu-24.04", + "lifecycle-subprocess": "ubuntu-24.04", + "macos-process-lifecycle": "macos-26", + "package": "ubuntu-24.04", + } + expected_python_versions = { + "test": ("3.11", "3.12", "3.13"), + "python-3-14": ("3.14",), + "lifecycle-subprocess": ("3.11",), + "macos-process-lifecycle": ("3.13",), + "package": ("3.11",), + } + expected_setup_python = { + "test": "${{ matrix.python-version }}", + "python-3-14": '"3.14"', + "lifecycle-subprocess": '"3.11"', + "macos-process-lifecycle": '"3.13"', + "package": '"3.11"', + } + expected_commands = { + "test": ( + "sudo apt-get update\nsudo apt-get install -y shellcheck", + 'python -m pip install -e ".[dev]"', + "ruff format --check .", + "ruff check .", + "mypy zeus", + "bandit -r zeus", + "shellcheck scripts/*.sh", + "sh scripts/test.sh", + "sh scripts/repo_check.sh", + "coverage erase\ncoverage run -m unittest discover -s tests\ncoverage report", + ), + "python-3-14": ( + 'python -m pip install -e ".[dev]"', + "sh scripts/test.sh", + ), + "lifecycle-subprocess": ( + 'python -m pip install -e ".[dev]"', + "python -m unittest tests.test_subprocess_lifecycle", + ), + "macos-process-lifecycle": ( + 'python -m pip install -e ".[dev]"', + "python -m unittest -v \\\n" + " tests.test_subprocess_lifecycle \\\n" + " tests.test_fake_hermes_integration \\\n" + " tests.test_crash_recovery.GatewayLauncherTests", + ), + "package": ( + 'python -m pip install -e ".[dev]"', + "python -m pip check", + "rm -rf dist\npython -m build", + "ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", + "twine check dist/*", + ), + } + self.assertEqual(set(expected_runners), set(jobs)) self.assertIn("workflow_dispatch:", workflow) - self.assertIn("runs-on: ubuntu-24.04", workflow) - self.assertNotIn("runs-on: ubuntu-latest", workflow) - self.assertIn("3.11", workflow) - self.assertIn("3.12", workflow) - self.assertIn("3.13", workflow) - self.assertIn("python-3-14:", workflow) - self.assertIn('python-version: "3.14"', workflow) - self.assertIn("continue-on-error: true", workflow) - self.assertIn("macos-process-lifecycle:", workflow) - self.assertIn("runs-on: macos-26", workflow) - self.assertIn("tests.test_subprocess_lifecycle", workflow) - self.assertIn("tests.test_fake_hermes_integration", workflow) - self.assertIn("tests.test_crash_recovery.GatewayLauncherTests", workflow) - self.assertIn('pip install -e ".[dev]"', workflow) - self.assertIn("python -m pip check", workflow) - self.assertIn("ruff format --check .", workflow) - self.assertIn("ruff check .", workflow) - self.assertIn("mypy zeus", workflow) - self.assertIn("bandit -r zeus", workflow) - self.assertIn("shellcheck scripts/*.sh", workflow) - self.assertIn("sh scripts/test.sh", workflow) - self.assertIn("coverage erase", workflow) - self.assertIn("coverage run -m unittest discover -s tests", workflow) - self.assertIn("coverage report", workflow) - self.assertNotIn("coverage report --fail-under", workflow) - self.assertIn("python -m build", workflow) - self.assertIn("twine check dist/*", workflow) - self.assertIn("- name: Verify installed wheel behavior", workflow) - self.assertIn("ZEUS_WHEEL_SMOKE_BUILD=0 sh scripts/wheel_smoke.sh", workflow) - self.assertNotIn("- name: Smoke test built wheel", workflow) - - def test_test_script_runs_compile_unittest_and_doctor(self) -> None: - script = Path("scripts/test.sh").read_text(encoding="utf-8") - - self.assertIn("compileall zeus tests", script) - self.assertIn("unittest discover -s tests -v", script) - self.assertIn("-W error::ResourceWarning", script) - self.assertIn('warning_log="$tmp_dir/unittest-stderr.log"', script) - self.assertIn('grep -F "ResourceWarning" "$warning_log"', script) - self.assertIn("trap cleanup EXIT INT TERM", script) - self.assertIn('mkdir -p "$tmp_dir"', script) - self.assertIn("zeus.cli doctor --json", script) + for job_name, job_body in jobs.items(): + with self.subTest(job=job_name): + self.assertEqual(expected_runners[job_name], _job_level_scalar(job_body, "runs-on")) + self.assertEqual(expected_python_versions[job_name], _job_python_versions(job_body)) + self.assertEqual( + [expected_setup_python[job_name]], + re.findall(r"(?m)^ python-version:\s*(.+?)\s*$", job_body), + ) + self.assertEqual(expected_commands[job_name], _job_run_commands(job_body)) + + job_level_continue_on_error: dict[str, str] = {} + for job_name, job_body in jobs.items(): + values = re.findall(r"(?m)^ continue-on-error:\s*([^\s#]+)\s*$", job_body) + self.assertLessEqual(len(values), 1, msg=f"duplicate setting in {job_name}") + if values: + job_level_continue_on_error[job_name] = values[0] + self.assertEqual({"python-3-14": "true"}, job_level_continue_on_error) + + python_314 = jobs["python-3-14"].lower() + self.assertNotIn("hermes", python_314) + self.assertNotIn("verify_real_hermes", python_314) + + macos_selectors = re.findall( + r"(? None: + script_source = Path("scripts/test.sh").read_text(encoding="utf-8") + fake_python_source = textwrap.dedent( + """\ + #!/bin/sh + set -eu + : "${FAKE_PYTHON_LOG:?}" + : "${FAKE_UNITTEST_MODE:?}" + printf '%s\\n' "$*" >> "$FAKE_PYTHON_LOG" + case "$*" in + *"-m compileall zeus tests") + exit 0 + ;; + *"-m unittest discover -s tests -v") + case "$FAKE_UNITTEST_MODE" in + warning) + printf '%s\\n' "Exception ignored in: <_io.FileIO name='fixture'>" >&2 + printf '%s\\n' "ResourceWarning: unclosed file " >&2 + exit 0 + ;; + exit7) + printf '%s\\n' "synthetic unittest failure" >&2 + exit 7 + ;; + clean) + exit 0 + ;; + *) + exit 98 + ;; + esac + ;; + *"-m zeus.cli doctor --json") + printf '%s\\n' '{"ok": true}' + ;; + *"-m zeus.cli template list") + printf '%s\\n' 'coding-bot' + ;; + *) + printf '%s\\n' "unexpected python invocation: $*" >&2 + exit 97 + ;; + esac + """ + ) + + def run_case( + root: Path, mode: str + ) -> tuple[subprocess.CompletedProcess[str], tuple[str, ...]]: + case_root = root / mode + scripts_dir = case_root / "scripts" + bin_dir = case_root / "bin" + scripts_dir.mkdir(parents=True) + bin_dir.mkdir() + script = scripts_dir / "test.sh" + fake_python = bin_dir / "python3" + log = case_root / "python-calls.log" + script.write_text(script_source, encoding="utf-8") + fake_python.write_text(fake_python_source, encoding="utf-8") + fake_python.chmod(0o755) + + environment = os.environ.copy() + environment.update( + { + "FAKE_PYTHON_LOG": str(log), + "FAKE_UNITTEST_MODE": mode, + "PATH": str(bin_dir) + os.pathsep + environment.get("PATH", ""), + } + ) + shell = shutil.which("sh", path=environment["PATH"]) + if shell is None: + self.fail("POSIX sh is required for the shell contract") + result = subprocess.run( + [shell, str(script)], + cwd=case_root, + env=environment, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + calls = tuple(log.read_text(encoding="utf-8").splitlines()) + return result, calls + + pre_gate_calls = ( + "-B -m compileall zeus tests", + "-B -W error::ResourceWarning -m unittest discover -s tests -v", + ) + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + + warning_result, warning_calls = run_case(root, "warning") + self.assertNotEqual(0, warning_result.returncode) + self.assertIn("Exception ignored in:", warning_result.stderr) + self.assertIn("ResourceWarning", warning_result.stderr) + self.assertEqual(pre_gate_calls, warning_calls) + + failed_result, failed_calls = run_case(root, "exit7") + self.assertEqual(7, failed_result.returncode) + self.assertNotIn("ResourceWarning", failed_result.stderr) + self.assertEqual(pre_gate_calls, failed_calls) + + clean_result, clean_calls = run_case(root, "clean") + self.assertEqual(0, clean_result.returncode) + self.assertEqual( + ( + *pre_gate_calls, + "-B -m zeus.cli doctor --json", + "-B -m zeus.cli template list", + ), + clean_calls, + ) def test_wheel_smoke_exercises_installed_demo_entrypoint(self) -> None: script = Path("scripts/wheel_smoke.sh").read_text(encoding="utf-8") @@ -329,16 +600,94 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No ): self.assertIn(statement, roadmap_text) - workflow_text = ci_workflow + "\n" + release_workflow - workflow_runners = set(re.findall(r"(?m)^\s*runs-on:\s*([^\s#]+)", workflow_text)) - workflow_python_versions = set(re.findall(r'"(3\.\d+)"', workflow_text)) - documented_python_versions = set(re.findall(r"(? Date: Tue, 21 Jul 2026 23:37:27 +0200 Subject: [PATCH 28/49] refactor(api): centralize exception mapping --- tests/test_api_errors.py | 112 +++++++++++++++++++++++++++++++++++++++ zeus/api.py | 93 ++++++-------------------------- zeus/api_errors.py | 61 +++++++++++++++++++++ 3 files changed, 188 insertions(+), 78 deletions(-) create mode 100644 tests/test_api_errors.py create mode 100644 zeus/api_errors.py diff --git a/tests/test_api_errors.py b/tests/test_api_errors.py new file mode 100644 index 0000000..b118ef4 --- /dev/null +++ b/tests/test_api_errors.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import unittest +from http import HTTPStatus +from pathlib import Path + +from zeus.api_errors import ApiError, map_api_exception +from zeus.errors import ZeusConflictError +from zeus.models import TemplateError +from zeus.process_lock import LockTimeoutError +from zeus.reconciliation import ReconcileLockTimeoutError + + +class ApiErrorMappingTests(unittest.TestCase): + def test_maps_public_exception_contract(self) -> None: + cases = ( + ( + "unknown_bot", + KeyError("unknown bot: coder"), + ApiError(HTTPStatus.NOT_FOUND, "unknown_bot", "unknown bot: coder"), + ), + ( + "unknown_template", + KeyError("unknown template: missing"), + ApiError( + HTTPStatus.BAD_REQUEST, + "unknown_template", + "unknown template: missing", + ), + ), + ( + "missing_field", + KeyError("bot_id"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "missing required field: bot_id", + ), + ), + ( + "invalid_bot_id", + TemplateError("bot_id must match ^[a-z][a-z0-9-]{1,62}$"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_bot_id", + "bot_id must match ^[a-z][a-z0-9-]{1,62}$", + ), + ), + ( + "template_error", + TemplateError("template contract failed"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "template contract failed", + ), + ), + ( + "reconcile_lock", + ReconcileLockTimeoutError(Path("/tmp/reconcile.lock"), 0.1), + ApiError( + HTTPStatus.CONFLICT, + "reconcile_locked", + "reconciliation is already in progress", + ), + ), + ( + "bot_lock", + LockTimeoutError(Path("/tmp/coder.lock"), 0.1), + ApiError( + HTTPStatus.CONFLICT, + "bot_locked", + "bot lifecycle operation is already in progress", + ), + ), + ( + "conflict", + ZeusConflictError("conflicting lifecycle state"), + ApiError( + HTTPStatus.CONFLICT, + "conflict", + "conflicting lifecycle state", + ), + ), + ( + "value_error", + ValueError("invalid lifecycle request"), + ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "invalid lifecycle request", + ), + ), + ( + "internal_error", + RuntimeError("private failure detail"), + ApiError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "internal_error", + "internal server error", + log_exception=True, + ), + ), + ) + + for name, exception, expected in cases: + with self.subTest(name=name): + self.assertEqual(expected, map_api_exception(exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/api.py b/zeus/api.py index e3b4846..aa72e81 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -19,24 +19,22 @@ from typing import Any, NoReturn from urllib.parse import parse_qs, urlparse +from zeus.api_errors import map_api_exception from zeus.api_logging import ApiLogWriter from zeus.config import Settings, validate_api_exposure from zeus.doctor import run_doctor -from zeus.errors import ZeusConflictError from zeus.idempotency import IdempotencyClaim, canonical_request_hash, hash_key from zeus.models import ( BotCreateRequest, BotStatus, HermesTemplate, RestartPolicy, - TemplateError, validate_id, ) from zeus.process_lock import BotProcessLock, LockTimeoutError from zeus.rate_limit import TokenBucket from zeus.reconciliation import ( MAX_RECONCILE_TEXT_LENGTH, - ReconcileLockTimeoutError, ReconcileOutcome, ) from zeus.request_context import RequestContext, new_request_id, route_template @@ -1057,39 +1055,10 @@ def _consume_mutation_capacity(self) -> None: def _json_error(self, exc: Exception) -> None: if isinstance(exc, _ResponseSent): return - if isinstance(exc, KeyError): - status, code, message = _key_error_response(exc) - self._json_error_response(status, code, message) - elif isinstance(exc, TemplateError): - code = ( - "invalid_bot_id" - if str(exc).startswith("bot_id must match") - else "invalid_request" - ) - self._json_error_response(HTTPStatus.BAD_REQUEST, code, str(exc)) - elif isinstance(exc, ReconcileLockTimeoutError): - self._json_error_response( - HTTPStatus.CONFLICT, - "reconcile_locked", - "reconciliation is already in progress", - ) - elif isinstance(exc, LockTimeoutError): - self._json_error_response( - HTTPStatus.CONFLICT, - "bot_locked", - "bot lifecycle operation is already in progress", - ) - elif isinstance(exc, ZeusConflictError): - self._json_error_response(HTTPStatus.CONFLICT, exc.code, str(exc)) - elif isinstance(exc, ValueError): - self._json_error_response(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) - else: + error = map_api_exception(exc) + if error.log_exception: api_log_writer.error(self._request_context.request_id, exc) - self._json_error_response( - HTTPStatus.INTERNAL_SERVER_ERROR, - "internal_error", - "internal server error", - ) + self._json_error_response(error.status, error.code, error.message) def _handle_idempotency_outcome(self, claim: IdempotencyClaim) -> bool: if claim.kind == "claimed": @@ -1150,41 +1119,19 @@ def _handle_idempotency_outcome(self, claim: IdempotencyClaim) -> bool: raise RuntimeError("invalid idempotency claim outcome") def _buffer_error(self, exc: Exception) -> BufferedJsonResponse: - if isinstance(exc, KeyError): - status, code, message = _key_error_response(exc) - elif isinstance(exc, TemplateError): - status = HTTPStatus.BAD_REQUEST - code = ( - "invalid_bot_id" - if str(exc).startswith("bot_id must match") - else "invalid_request" - ) - message = str(exc) - elif isinstance(exc, ReconcileLockTimeoutError): - status = HTTPStatus.CONFLICT - code = "reconcile_locked" - message = "reconciliation is already in progress" - elif isinstance(exc, LockTimeoutError): - status = HTTPStatus.CONFLICT - code = "bot_locked" - message = "bot lifecycle operation is already in progress" - elif isinstance(exc, ZeusConflictError): - status = HTTPStatus.CONFLICT - code = exc.code - message = str(exc) - elif isinstance(exc, ValueError): - status = HTTPStatus.BAD_REQUEST - code = "invalid_request" - message = str(exc) - else: - status = HTTPStatus.INTERNAL_SERVER_ERROR - code = "internal_error" - message = "internal server error" + error = map_api_exception(exc) + if error.log_exception: api_log_writer.error(self._request_context.request_id, exc) - self._response_error_code = code + self._response_error_code = error.code return BufferedJsonResponse( - status, - {"error": {"code": code, "message": message, "status": status.value}}, + error.status, + { + "error": { + "code": error.code, + "message": error.message, + "status": error.status.value, + } + }, ) def _buffer_internal_error(self, exc: Exception) -> BufferedJsonResponse: @@ -1353,16 +1300,6 @@ def _safe_route_template(target: str) -> str | None: return None -def _key_error_response(exc: KeyError) -> tuple[HTTPStatus, str, str]: - detail = exc.args[0] if exc.args else "missing required field" - message = str(detail) - if message.startswith("unknown bot:"): - return HTTPStatus.NOT_FOUND, "unknown_bot", message - if message.startswith("unknown template:"): - return HTTPStatus.BAD_REQUEST, "unknown_template", message - return HTTPStatus.BAD_REQUEST, "invalid_request", f"missing required field: {message}" - - def template_to_dict(template: HermesTemplate) -> dict[str, Any]: return { "id": template.id, diff --git a/zeus/api_errors.py b/zeus/api_errors.py new file mode 100644 index 0000000..5ac2802 --- /dev/null +++ b/zeus/api_errors.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from dataclasses import dataclass +from http import HTTPStatus + +from zeus.errors import ZeusConflictError +from zeus.models import TemplateError +from zeus.process_lock import LockTimeoutError +from zeus.reconciliation import ReconcileLockTimeoutError + + +@dataclass(frozen=True) +class ApiError: + status: HTTPStatus + code: str + message: str + log_exception: bool = False + + +def map_api_exception(exc: Exception) -> ApiError: + if isinstance(exc, KeyError): + return _map_key_error(exc) + if isinstance(exc, TemplateError): + code = "invalid_bot_id" if str(exc).startswith("bot_id must match") else "invalid_request" + return ApiError(HTTPStatus.BAD_REQUEST, code, str(exc)) + if isinstance(exc, ReconcileLockTimeoutError): + return ApiError( + HTTPStatus.CONFLICT, + "reconcile_locked", + "reconciliation is already in progress", + ) + if isinstance(exc, LockTimeoutError): + return ApiError( + HTTPStatus.CONFLICT, + "bot_locked", + "bot lifecycle operation is already in progress", + ) + if isinstance(exc, ZeusConflictError): + return ApiError(HTTPStatus.CONFLICT, exc.code, str(exc)) + if isinstance(exc, ValueError): + return ApiError(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) + return ApiError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "internal_error", + "internal server error", + log_exception=True, + ) + + +def _map_key_error(exc: KeyError) -> ApiError: + detail = exc.args[0] if exc.args else "missing required field" + message = str(detail) + if message.startswith("unknown bot:"): + return ApiError(HTTPStatus.NOT_FOUND, "unknown_bot", message) + if message.startswith("unknown template:"): + return ApiError(HTTPStatus.BAD_REQUEST, "unknown_template", message) + return ApiError( + HTTPStatus.BAD_REQUEST, + "invalid_request", + f"missing required field: {message}", + ) From 6f482ef9d63b3974b36bfdfaf8d1357f8b1be28d Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:41:40 +0200 Subject: [PATCH 29/49] refactor(api): extract strict request parsing --- tests/test_api_request.py | 127 ++++++++++++++++++++++++++++++++++++++ zeus/api.py | 79 +++++------------------- zeus/api_request.py | 78 +++++++++++++++++++++++ 3 files changed, 220 insertions(+), 64 deletions(-) create mode 100644 tests/test_api_request.py create mode 100644 zeus/api_request.py diff --git a/tests/test_api_request.py b/tests/test_api_request.py new file mode 100644 index 0000000..defc8d1 --- /dev/null +++ b/tests/test_api_request.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import unittest + +from zeus.api_request import decode_json_object, normalize_api_path, parse_query + + +class ApiPathTests(unittest.TestCase): + def test_normalizes_only_the_v1_alias(self) -> None: + cases = ( + ("/v1", "/"), + ("/v1/", "/"), + ("/v1/bots", "/bots"), + ("/health", "/health"), + ("/v10/bots", "/v10/bots"), + ) + + for target, expected in cases: + with self.subTest(target=target): + self.assertEqual(expected, normalize_api_path(target)) + + def test_rejects_request_target_fragments(self) -> None: + with self.assertRaisesRegex( + ValueError, + "^request target must not include a fragment$", + ): + normalize_api_path("/health#internal") + + +class ApiQueryTests(unittest.TestCase): + def test_preserves_blank_values_and_field_order(self) -> None: + values = parse_query( + "/bots?second=2&first=", + frozenset({"first", "second"}), + ) + + self.assertEqual(["second", "first"], list(values)) + self.assertEqual({"second": ["2"], "first": [""]}, values) + + def test_accepts_sixteen_query_fields(self) -> None: + names = tuple(f"field{index}" for index in range(16)) + query = "&".join(f"{name}=1" for name in names) + + values = parse_query(f"/health?{query}", frozenset(names)) + + self.assertEqual(16, len(values)) + + def test_rejects_seventeen_query_fields(self) -> None: + names = tuple(f"field{index}" for index in range(17)) + query = "&".join(f"{name}=1" for name in names) + + with self.assertRaisesRegex(ValueError, "^too many query parameters$"): + parse_query(f"/health?{query}", frozenset(names)) + + def test_reports_unknown_field_before_duplicate_field(self) -> None: + with self.assertRaisesRegex(ValueError, "^unknown query parameter: debug$"): + parse_query( + "/health?known=1&known=2&debug=1", + frozenset({"known"}), + ) + + def test_rejects_duplicate_fields(self) -> None: + with self.assertRaisesRegex( + ValueError, + "^query parameter replace must be specified once$", + ): + parse_query( + "/bots?replace=0&replace=1", + frozenset({"replace"}), + ) + + +class ApiJsonTests(unittest.TestCase): + def test_decodes_a_json_object(self) -> None: + self.assertEqual( + {"bot_id": "coder", "env": {"MODE": "safe"}}, + decode_json_object(b'{"bot_id":"coder","env":{"MODE":"safe"}}'), + ) + + def test_rejects_duplicate_fields_at_every_object_depth(self) -> None: + cases = ( + (b'{"bot_id":"coder","bot_id":"other"}', "bot_id"), + (b'{"env":{"MODE":"safe","MODE":"unsafe"}}', "MODE"), + ) + + for data, field in cases: + with ( + self.subTest(field=field), + self.assertRaisesRegex( + ValueError, + f"^duplicate JSON field: {field}$", + ), + ): + decode_json_object(data) + + def test_rejects_nonstandard_json_constants(self) -> None: + for constant in ("NaN", "Infinity", "-Infinity"): + with ( + self.subTest(constant=constant), + self.assertRaisesRegex( + ValueError, + f"^invalid JSON constant: {constant}$", + ), + ): + decode_json_object(f'{{"value":{constant}}}'.encode()) + + def test_rejects_array_and_scalar_roots(self) -> None: + for data in (b"[]", b'"value"', b"1", b"null"): + with ( + self.subTest(data=data), + self.assertRaisesRegex( + ValueError, + "^request body must be a JSON object$", + ), + ): + decode_json_object(data) + + def test_accepts_and_rejects_exact_depth_boundary(self) -> None: + data = b'{"nested":[0]}' + + self.assertEqual({"nested": [0]}, decode_json_object(data, max_depth=3)) + with self.assertRaisesRegex(ValueError, "^request JSON nesting exceeds 2$"): + decode_json_object(data, max_depth=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/api.py b/zeus/api.py index aa72e81..44b6b35 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -17,10 +17,16 @@ from http.server import ThreadingHTTPServer as _ThreadingHTTPServer from pathlib import Path from typing import Any, NoReturn -from urllib.parse import parse_qs, urlparse +from urllib.parse import urlparse +from zeus import api_request from zeus.api_errors import map_api_exception from zeus.api_logging import ApiLogWriter +from zeus.api_request import ( + decode_json_object, + normalize_api_path, + parse_query, +) from zeus.config import Settings, validate_api_exposure from zeus.doctor import run_doctor from zeus.idempotency import IdempotencyClaim, canonical_request_hash, hash_key @@ -53,8 +59,8 @@ "restart_max_attempts", } ) -MAX_JSON_DEPTH = 64 -MAX_QUERY_FIELDS = 16 +MAX_JSON_DEPTH = api_request.MAX_JSON_DEPTH +MAX_QUERY_FIELDS = api_request.MAX_QUERY_FIELDS MAX_IDEMPOTENCY_MESSAGE_JSON_BYTES = 4_096 _IDEMPOTENCY_MESSAGE_REPLACEMENT = "response message omitted because it exceeded the replay budget" _IDEMPOTENCY_OWNER_LOCK = threading.Lock() @@ -216,31 +222,6 @@ def _bound_idempotent_messages(value: object) -> tuple[object, bool]: return value, False -def _json_object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate JSON field: {key}") - result[key] = value - return result - - -def _reject_json_constant(value: str) -> Any: - raise ValueError(f"invalid JSON constant: {value}") - - -def _validate_json_depth(value: Any) -> None: - stack = [(value, 1)] - while stack: - current, depth = stack.pop() - if depth > MAX_JSON_DEPTH: - raise ValueError(f"request JSON nesting exceeds {MAX_JSON_DEPTH}") - if isinstance(current, dict): - stack.extend((child, depth + 1) for child in current.values()) - elif isinstance(current, list): - stack.extend((child, depth + 1) for child in current) - - class ThreadingHTTPServer(_ThreadingHTTPServer): daemon_threads = True @@ -411,6 +392,7 @@ class ZeusHandler(BaseHTTPRequestHandler): _request_context: RequestContext _response_status: int _response_error_code: str | None + _validated_query_values: dict[str, list[str]] def send_error( self, @@ -818,18 +800,7 @@ def _read_json(self) -> dict[str, Any]: if length > 1_000_000: raise ValueError("request body too large") data = self.rfile.read(length) if length else b"{}" - try: - parsed = json.loads( - data.decode("utf-8"), - object_pairs_hook=_json_object_without_duplicates, - parse_constant=_reject_json_constant, - ) - except RecursionError as exc: - raise ValueError(f"request JSON nesting exceeds {MAX_JSON_DEPTH}") from exc - _validate_json_depth(parsed) - if not isinstance(parsed, dict): - raise ValueError("request body must be a JSON object") - return parsed + return decode_json_object(data) def _require_json_content_type(self) -> None: content_encoding = self.headers.get("content-encoding", "").strip().lower() @@ -880,34 +851,13 @@ def _bot_id_from_path(self, path: str, action: str) -> str: return validate_id(parts[1], "bot_id") def _normalized_path(self) -> str: - parsed = urlparse(self.path) - if parsed.fragment: - raise ValueError("request target must not include a fragment") - path = parsed.path - if path == "/v1": - return "/" - if path.startswith("/v1/"): - return path[3:] - return path + return normalize_api_path(self.path) def _query_values(self) -> dict[str, list[str]]: - try: - return parse_qs( - urlparse(self.path).query, - keep_blank_values=True, - max_num_fields=MAX_QUERY_FIELDS, - ) - except ValueError as exc: - raise ValueError("too many query parameters") from exc + return self._validated_query_values def _validate_query_parameters(self, allowed: set[str]) -> None: - values = self._query_values() - unknown = sorted(set(values) - allowed) - if unknown: - raise ValueError(f"unknown query parameter: {unknown[0]}") - duplicates = sorted(name for name, entries in values.items() if len(entries) > 1) - if duplicates: - raise ValueError(f"query parameter {duplicates[0]} must be specified once") + self._validated_query_values = parse_query(self.path, frozenset(allowed)) def _post_query_parameters(self, path: str) -> set[str] | None: if path == "/bots": @@ -1264,6 +1214,7 @@ def _handle_request(self, dispatch: Callable[[], None]) -> None: self._request_context.route = _safe_route_template(self.path) self._response_status = HTTPStatus.INTERNAL_SERVER_ERROR.value self._response_error_code = None + self._validated_query_values = {} try: dispatch() except Exception as exc: diff --git a/zeus/api_request.py b/zeus/api_request.py new file mode 100644 index 0000000..480f07c --- /dev/null +++ b/zeus/api_request.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import json +from typing import Any, cast +from urllib.parse import parse_qs, urlparse + +MAX_JSON_DEPTH = 64 +MAX_QUERY_FIELDS = 16 + + +def normalize_api_path(target: str) -> str: + parsed = urlparse(target) + if parsed.fragment: + raise ValueError("request target must not include a fragment") + path = parsed.path + if path == "/v1": + return "/" + if path.startswith("/v1/"): + return path[3:] + return path + + +def parse_query(target: str, allowed: frozenset[str]) -> dict[str, list[str]]: + try: + values = parse_qs( + urlparse(target).query, + keep_blank_values=True, + max_num_fields=MAX_QUERY_FIELDS, + ) + except ValueError as exc: + raise ValueError("too many query parameters") from exc + unknown = sorted(set(values) - allowed) + if unknown: + raise ValueError(f"unknown query parameter: {unknown[0]}") + duplicates = sorted(name for name, entries in values.items() if len(entries) > 1) + if duplicates: + raise ValueError(f"query parameter {duplicates[0]} must be specified once") + return values + + +def decode_json_object(data: bytes, *, max_depth: int = MAX_JSON_DEPTH) -> dict[str, Any]: + try: + parsed: object = json.loads( + data.decode("utf-8"), + object_pairs_hook=_json_object_without_duplicates, + parse_constant=_reject_json_constant, + ) + except RecursionError as exc: + raise ValueError(f"request JSON nesting exceeds {max_depth}") from exc + _validate_json_depth(parsed, max_depth=max_depth) + if not isinstance(parsed, dict): + raise ValueError("request body must be a JSON object") + return cast(dict[str, Any], parsed) + + +def _json_object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON field: {key}") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"invalid JSON constant: {value}") + + +def _validate_json_depth(value: object, *, max_depth: int) -> None: + stack = [(value, 1)] + while stack: + current, depth = stack.pop() + if depth > max_depth: + raise ValueError(f"request JSON nesting exceeds {max_depth}") + if isinstance(current, dict): + stack.extend((child, depth + 1) for child in current.values()) + elif isinstance(current, list): + stack.extend((child, depth + 1) for child in current) From 4b86773fcde1b01ea18f41dae9bbca3f7ade978d Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:51:09 +0200 Subject: [PATCH 30/49] refactor(api): isolate server lifecycle --- docs/ARCHITECTURE.md | 5 +- tests/test_api.py | 186 +---------------------------- tests/test_api_server.py | 216 +++++++++++++++++++++++++++++++++ zeus/api.py | 232 ++---------------------------------- zeus/api_server.py | 251 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 482 insertions(+), 408 deletions(-) create mode 100644 tests/test_api_server.py create mode 100644 zeus/api_server.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 220ac66..02645be 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -35,12 +35,15 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.state`: SQLite bot projection and authoritative lifecycle ledger. - `zeus.lifecycle`: Bounded lifecycle event types and recursively redacted details. - `zeus.request_context`: Locally generated request IDs and normalized route templates. +- `zeus.api_errors`: Transport-neutral API exception classification. +- `zeus.api_request`: Strict path, query, and JSON request parsing. +- `zeus.api_server`: Bounded HTTP concurrency and graceful server lifecycle. - `zeus.api_logging`: Locked, fail-open, secret-safe API JSONL output. - `zeus.idempotency`: Key validation and canonical request hashing. - `zeus.hermes_adapter`: Subprocess command construction for Hermes. - `zeus.gateway_launcher`: Descriptor-only marker-before-exec helper. - `zeus.supervisor`: Gateway lifecycle, PID ownership markers, logs, and status. -- `zeus.api`: Local HTTP API. +- `zeus.api`: Local HTTP routes and compatibility facade. - `zeus.cli`: Operator CLI. - `zeus.doctor`: Readiness diagnostics. diff --git a/tests/test_api.py b/tests/test_api.py index b02659e..eff9d82 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -14,7 +14,7 @@ from collections.abc import Iterator from contextlib import closing, contextmanager from datetime import UTC, datetime -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from http.server import ThreadingHTTPServer from pathlib import Path from typing import Any from unittest.mock import patch @@ -439,190 +439,6 @@ def test_api_resource_limits_are_configurable_and_bounded(self) -> None: with self.subTest(env=env), self.assertRaisesRegex(ValueError, message): Settings.from_env(env) - def test_http_server_rejects_requests_above_concurrency_limit(self) -> None: - first_entered = threading.Event() - release_first = threading.Event() - call_lock = threading.Lock() - call_count = 0 - - class LimitedHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 1 - api_request_timeout_seconds = 2.0 - - def do_GET(self) -> None: - nonlocal call_count - with call_lock: - call_count += 1 - current_call = call_count - if current_call == 1: - first_entered.set() - release_first.wait(timeout=3) - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - server = api_module.ThreadingHTTPServer(("127.0.0.1", 0), LimitedHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - first_result: list[tuple[int, dict[str, Any]]] = [] - first_thread = threading.Thread( - target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) - ) - server_thread.start() - first_thread.start() - try: - self.assertTrue(first_entered.wait(timeout=2)) - conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) - try: - conn.request("GET", "/health") - response = conn.getresponse() - raw_body = response.read() - finally: - conn.close() - - self.assertEqual(503, response.status) - self.assertEqual("1", response.getheader("retry-after")) - self.assertRegex( - response.getheader("x-request-id") or "", - r"^[0-9a-f]{32}$", - ) - body = json.loads(raw_body.decode("utf-8")) - self.assertEqual("server_busy", body["error"]["code"]) - finally: - release_first.set() - first_thread.join(timeout=3) - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - - self.assertFalse(first_thread.is_alive()) - self.assertEqual(200, first_result[0][0]) - - def test_http_server_drains_active_requests_and_rejects_new_work(self) -> None: - first_entered = threading.Event() - release_first = threading.Event() - - class DrainHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 2 - api_request_timeout_seconds = 2.0 - - def do_GET(self) -> None: - first_entered.set() - release_first.wait(timeout=3) - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - server = api_module.ThreadingHTTPServer(("127.0.0.1", 0), DrainHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - first_result: list[tuple[int, dict[str, Any]]] = [] - first_thread = threading.Thread( - target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) - ) - server_thread.start() - first_thread.start() - try: - self.assertTrue(first_entered.wait(timeout=2)) - request_graceful_shutdown = getattr(server, "request_graceful_shutdown", None) - wait_until_draining = getattr(server, "wait_until_draining", None) - wait_for_drain = getattr(server, "wait_for_drain", None) - self.assertTrue(callable(request_graceful_shutdown)) - self.assertTrue(callable(wait_until_draining)) - self.assertTrue(callable(wait_for_drain)) - request_graceful_shutdown(1.0) - self.assertTrue(wait_until_draining(1.0)) - - conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) - try: - conn.request("GET", "/health") - response = conn.getresponse() - raw_body = response.read() - finally: - conn.close() - - self.assertEqual(503, response.status) - self.assertEqual("1", response.getheader("retry-after")) - self.assertRegex( - response.getheader("x-request-id") or "", - r"^[0-9a-f]{32}$", - ) - body = json.loads(raw_body.decode("utf-8")) - self.assertEqual("server_draining", body["error"]["code"]) - self.assertFalse(wait_for_drain(0.05)) - - release_first.set() - first_thread.join(timeout=3) - self.assertTrue(wait_for_drain(1.0)) - server_thread.join(timeout=3) - finally: - release_first.set() - first_thread.join(timeout=3) - if server_thread.is_alive(): - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - - self.assertFalse(first_thread.is_alive()) - self.assertFalse(server_thread.is_alive()) - self.assertEqual(200, first_result[0][0]) - - def test_http_server_times_out_incomplete_requests_and_releases_capacity(self) -> None: - accepted = threading.Event() - - class TimeoutHandler(BaseHTTPRequestHandler): - api_max_concurrent_requests = 1 - api_request_timeout_seconds = 0.1 - - def do_GET(self) -> None: - data = b'{"status":"ok"}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, format: str, *args: Any) -> None: - return - - class TrackingServer(api_module.ThreadingHTTPServer): - def process_request(self, request: Any, client_address: Any) -> None: - accepted.set() - super().process_request(request, client_address) - - server = TrackingServer(("127.0.0.1", 0), TimeoutHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - server_thread.start() - slow_client = socket.create_connection(("127.0.0.1", server.server_port), timeout=2) - try: - slow_client.sendall(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n") - self.assertTrue(accepted.wait(timeout=2)) - slow_client.settimeout(1) - try: - closed_data = slow_client.recv(1) - except TimeoutError: - self.fail("incomplete API request was not closed after the configured timeout") - self.assertEqual(b"", closed_data) - - status, body = request_json(server.server_port, "GET", "/health") - self.assertEqual(200, status) - self.assertEqual({"status": "ok"}, body) - finally: - slow_client.close() - server.shutdown() - server.server_close() - server_thread.join(timeout=3) - def test_health_is_public_but_missing_api_key_rejects_bots_list(self) -> None: with api_server() as port: status, body = request_json(port, "GET", "/health") diff --git a/tests/test_api_server.py b/tests/test_api_server.py new file mode 100644 index 0000000..bd2cc02 --- /dev/null +++ b/tests/test_api_server.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import http.client +import json +import socket +import threading +import unittest +from http.server import BaseHTTPRequestHandler +from typing import Any + +from zeus.api import ThreadingHTTPServer as public_server +from zeus.api_server import ThreadingHTTPServer as extracted_server + + +def request_json(port: int, method: str, path: str) -> tuple[int, dict[str, Any]]: + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request(method, path) + response = conn.getresponse() + body = json.loads(response.read().decode("utf-8")) + return response.status, body + finally: + conn.close() + + +class ApiServerTests(unittest.TestCase): + def test_public_server_is_extracted_server(self) -> None: + self.assertIs(public_server, extracted_server) + + def test_rejects_requests_above_concurrency_limit(self) -> None: + first_entered = threading.Event() + release_first = threading.Event() + call_lock = threading.Lock() + call_count = 0 + + class LimitedHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 1 + api_request_timeout_seconds = 2.0 + + def do_GET(self) -> None: + nonlocal call_count + with call_lock: + call_count += 1 + current_call = call_count + if current_call == 1: + first_entered.set() + release_first.wait(timeout=3) + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server = extracted_server(("127.0.0.1", 0), LimitedHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + first_result: list[tuple[int, dict[str, Any]]] = [] + first_thread = threading.Thread( + target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) + ) + server_thread.start() + first_thread.start() + try: + self.assertTrue(first_entered.wait(timeout=2)) + conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) + try: + conn.request("GET", "/health") + response = conn.getresponse() + raw_body = response.read() + finally: + conn.close() + + self.assertEqual(503, response.status) + self.assertEqual("1", response.getheader("retry-after")) + self.assertRegex( + response.getheader("x-request-id") or "", + r"^[0-9a-f]{32}$", + ) + body = json.loads(raw_body.decode("utf-8")) + self.assertEqual("server_busy", body["error"]["code"]) + finally: + release_first.set() + first_thread.join(timeout=3) + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + self.assertFalse(first_thread.is_alive()) + self.assertEqual(200, first_result[0][0]) + + def test_drains_active_requests_and_rejects_new_work(self) -> None: + first_entered = threading.Event() + release_first = threading.Event() + + class DrainHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 2 + api_request_timeout_seconds = 2.0 + + def do_GET(self) -> None: + first_entered.set() + release_first.wait(timeout=3) + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + server = extracted_server(("127.0.0.1", 0), DrainHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + first_result: list[tuple[int, dict[str, Any]]] = [] + first_thread = threading.Thread( + target=lambda: first_result.append(request_json(server.server_port, "GET", "/health")) + ) + server_thread.start() + first_thread.start() + try: + self.assertTrue(first_entered.wait(timeout=2)) + request_graceful_shutdown = getattr(server, "request_graceful_shutdown", None) + wait_until_draining = getattr(server, "wait_until_draining", None) + wait_for_drain = getattr(server, "wait_for_drain", None) + self.assertTrue(callable(request_graceful_shutdown)) + self.assertTrue(callable(wait_until_draining)) + self.assertTrue(callable(wait_for_drain)) + request_graceful_shutdown(1.0) + self.assertTrue(wait_until_draining(1.0)) + + conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2) + try: + conn.request("GET", "/health") + response = conn.getresponse() + raw_body = response.read() + finally: + conn.close() + + self.assertEqual(503, response.status) + self.assertEqual("1", response.getheader("retry-after")) + self.assertRegex( + response.getheader("x-request-id") or "", + r"^[0-9a-f]{32}$", + ) + body = json.loads(raw_body.decode("utf-8")) + self.assertEqual("server_draining", body["error"]["code"]) + self.assertFalse(wait_for_drain(0.05)) + + release_first.set() + first_thread.join(timeout=3) + self.assertTrue(wait_for_drain(1.0)) + server_thread.join(timeout=3) + finally: + release_first.set() + first_thread.join(timeout=3) + if server_thread.is_alive(): + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + self.assertFalse(first_thread.is_alive()) + self.assertFalse(server_thread.is_alive()) + self.assertEqual(200, first_result[0][0]) + + def test_times_out_incomplete_requests_and_releases_capacity(self) -> None: + accepted = threading.Event() + + class TimeoutHandler(BaseHTTPRequestHandler): + api_max_concurrent_requests = 1 + api_request_timeout_seconds = 0.1 + + def do_GET(self) -> None: + data = b'{"status":"ok"}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + return + + class TrackingServer(extracted_server): + def process_request(self, request: Any, client_address: Any) -> None: + accepted.set() + super().process_request(request, client_address) + + server = TrackingServer(("127.0.0.1", 0), TimeoutHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + slow_client = socket.create_connection(("127.0.0.1", server.server_port), timeout=2) + try: + slow_client.sendall(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n") + self.assertTrue(accepted.wait(timeout=2)) + slow_client.settimeout(1) + try: + closed_data = slow_client.recv(1) + except TimeoutError: + self.fail("incomplete API request was not closed after the configured timeout") + self.assertEqual(b"", closed_data) + + status, body = request_json(server.server_port, "GET", "/health") + self.assertEqual(200, status) + self.assertEqual({"status": "ok"}, body) + finally: + slow_client.close() + server.shutdown() + server.server_close() + server_thread.join(timeout=3) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/api.py b/zeus/api.py index 44b6b35..a3286cf 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -5,7 +5,6 @@ import hmac import json import os -import signal import sys import threading import time @@ -14,8 +13,6 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus from http.server import BaseHTTPRequestHandler -from http.server import ThreadingHTTPServer as _ThreadingHTTPServer -from pathlib import Path from typing import Any, NoReturn from urllib.parse import urlparse @@ -27,7 +24,9 @@ normalize_api_path, parse_query, ) -from zeus.config import Settings, validate_api_exposure +from zeus.api_server import ThreadingHTTPServer +from zeus.api_server import serve as _serve_server +from zeus.config import Settings from zeus.doctor import run_doctor from zeus.idempotency import IdempotencyClaim, canonical_request_hash, hash_key from zeus.models import ( @@ -37,7 +36,7 @@ RestartPolicy, validate_id, ) -from zeus.process_lock import BotProcessLock, LockTimeoutError +from zeus.process_lock import LockTimeoutError from zeus.rate_limit import TokenBucket from zeus.reconciliation import ( MAX_RECONCILE_TEXT_LENGTH, @@ -222,141 +221,6 @@ def _bound_idempotent_messages(value: object) -> tuple[object, bool]: return value, False -class ThreadingHTTPServer(_ThreadingHTTPServer): - daemon_threads = True - - def __init__( - self, - server_address: Any, - RequestHandlerClass: type[BaseHTTPRequestHandler], - bind_and_activate: bool = True, - ) -> None: - self.api_max_concurrent_requests = int( - getattr(RequestHandlerClass, "api_max_concurrent_requests", 32) - ) - self.api_request_timeout_seconds = float( - getattr(RequestHandlerClass, "api_request_timeout_seconds", 10.0) - ) - self._request_slots = threading.BoundedSemaphore(self.api_max_concurrent_requests) - self._request_state = threading.Condition() - self._active_requests: set[int] = set() - self._draining = False - self._drain_started = threading.Event() - self._graceful_shutdown_requested = False - self._graceful_shutdown_timeout_seconds = 0.0 - self._graceful_shutdown_thread: threading.Thread | None = None - self._graceful_shutdown_result: bool | None = None - super().__init__(server_address, RequestHandlerClass, bind_and_activate) - - def process_request(self, request: Any, client_address: Any) -> None: - with self._request_state: - if self._draining: - rejection = ("server_draining", "API server is draining") - elif not self._request_slots.acquire(blocking=False): - rejection = ("server_busy", "API request capacity is exhausted") - else: - self._active_requests.add(id(request)) - rejection = None - - if rejection is not None: - self._reject_unavailable_request(request, *rejection) - self.shutdown_request(request) - return - try: - super().process_request(request, client_address) - except Exception: - self._finish_request(request) - raise - - def process_request_thread(self, request: Any, client_address: Any) -> None: - try: - request.settimeout(self.api_request_timeout_seconds) - super().process_request_thread(request, client_address) - finally: - self._finish_request(request) - - def begin_draining(self) -> None: - with self._request_state: - self._draining = True - self._drain_started.set() - - def request_graceful_shutdown(self, timeout_seconds: float) -> None: - self._graceful_shutdown_timeout_seconds = timeout_seconds - self._graceful_shutdown_requested = True - - def wait_until_draining(self, timeout_seconds: float) -> bool: - return self._drain_started.wait(timeout_seconds) - - def service_actions(self) -> None: - super().service_actions() - if not self._graceful_shutdown_requested or self._graceful_shutdown_thread is not None: - return - self.begin_draining() - coordinator = threading.Thread(target=self._complete_graceful_shutdown, daemon=True) - coordinator.start() - self._graceful_shutdown_thread = coordinator - - def finish_graceful_shutdown(self) -> bool | None: - coordinator = self._graceful_shutdown_thread - if coordinator is None: - return None - coordinator.join() - return self._graceful_shutdown_result - - def wait_for_drain(self, timeout_seconds: float) -> bool: - deadline = time.monotonic() + timeout_seconds - with self._request_state: - while self._active_requests: - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._request_state.wait(remaining) - return True - - def _finish_request(self, request: Any) -> None: - with self._request_state: - request_id = id(request) - if request_id not in self._active_requests: - return - self._active_requests.remove(request_id) - self._request_slots.release() - if not self._active_requests: - self._request_state.notify_all() - - def _complete_graceful_shutdown(self) -> None: - try: - self._graceful_shutdown_result = self.wait_for_drain( - self._graceful_shutdown_timeout_seconds - ) - finally: - self.shutdown() - - def _reject_unavailable_request(self, request: Any, code: str, message: str) -> None: - request_id = new_request_id() - body = json.dumps( - { - "error": { - "code": code, - "message": message, - "status": HTTPStatus.SERVICE_UNAVAILABLE.value, - } - }, - sort_keys=True, - ).encode("utf-8") - response = ( - b"HTTP/1.1 503 Service Unavailable\r\n" - b"connection: close\r\n" - b"content-type: application/json\r\n" - b"cache-control: no-store\r\n" - + f"x-request-id: {request_id}\r\n".encode("ascii") - + b"retry-after: 1\r\n" - + f"content-length: {len(body)}\r\n\r\n".encode("ascii") - + body - ) - with contextlib.suppress(OSError): - request.sendall(response) - - def make_handler(settings: Settings) -> type[BaseHTTPRequestHandler]: settings.ensure_dirs() api_log_writer = ApiLogWriter( @@ -1264,89 +1128,13 @@ def template_to_dict(template: HermesTemplate) -> dict[str, Any]: def serve(host: str, port: int, settings: Settings | None = None) -> None: base_settings = settings or Settings.from_env() - runtime_settings = replace(base_settings, host=host, port=port) - if not 0 <= port <= 65535: - raise ValueError("port must be between 0 and 65535") - validate_api_exposure( - runtime_settings.host, - runtime_settings.api_key, - runtime_settings.allow_unauth_reads, + _serve_server( + host, + port, + base_settings, + handler_factory=make_handler, + server_factory=ThreadingHTTPServer, ) - runtime_settings.ensure_dirs() - pid = os.getpid() - pid_path = runtime_settings.state_dir / "zeus.pid" - api_lock = BotProcessLock( - runtime_settings.state_dir / "locks" / "api.lock", - timeout_seconds=min(runtime_settings.lock_timeout_seconds, 1.0), - ) - with api_lock: - handler = make_handler(runtime_settings) - server = ThreadingHTTPServer((host, port), handler) - previous_signal_handlers: dict[signal.Signals, Any] = {} - try: - if threading.current_thread() is threading.main_thread(): - for signum in (signal.SIGTERM, signal.SIGINT): - previous_signal_handlers[signum] = signal.getsignal(signum) - signal.signal( - signum, - lambda _signum, _frame: server.request_graceful_shutdown( - runtime_settings.api_shutdown_drain_seconds - ), - ) - with contextlib.suppress(KeyboardInterrupt): - _write_api_pid(pid_path, pid) - server.serve_forever() - finally: - for signum, previous_handler in previous_signal_handlers.items(): - signal.signal(signum, previous_handler) - finish_graceful_shutdown = getattr(server, "finish_graceful_shutdown", None) - drain_result = ( - finish_graceful_shutdown() if callable(finish_graceful_shutdown) else None - ) - if drain_result is None: - begin_draining = getattr(server, "begin_draining", None) - if callable(begin_draining): - begin_draining() - server.server_close() - wait_for_drain = getattr(server, "wait_for_drain", None) - drain_result = ( - wait_for_drain(runtime_settings.api_shutdown_drain_seconds) - if callable(wait_for_drain) - else True - ) - else: - server.server_close() - if not drain_result: - print("warning: API shutdown drain deadline expired", file=sys.stderr) - _remove_api_pid_if_owned(pid_path, pid) - - -def _write_api_pid(path: Path, pid: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - tmp = path.with_name(f".{path.name}.{pid}.{time.time_ns()}.tmp") - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(f"{pid}\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp, path) - with contextlib.suppress(OSError): - path.chmod(0o600) - finally: - with contextlib.suppress(FileNotFoundError): - tmp.unlink() - - -def _remove_api_pid_if_owned(path: Path, pid: int) -> None: - try: - recorded_pid = int(path.read_text(encoding="utf-8").strip()) - except (FileNotFoundError, OSError, ValueError): - return - if recorded_pid != pid: - return - with contextlib.suppress(FileNotFoundError): - path.unlink() def main(argv: list[str] | None = None) -> int: diff --git a/zeus/api_server.py b/zeus/api_server.py new file mode 100644 index 0000000..d93e430 --- /dev/null +++ b/zeus/api_server.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import contextlib +import json +import os +import signal +import sys +import threading +import time +from collections.abc import Callable +from dataclasses import replace +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler +from http.server import ThreadingHTTPServer as _ThreadingHTTPServer +from pathlib import Path +from typing import Any + +from zeus.config import Settings, validate_api_exposure +from zeus.process_lock import BotProcessLock +from zeus.request_context import new_request_id + +HandlerFactory = Callable[[Settings], type[BaseHTTPRequestHandler]] +ServerFactory = Callable[[tuple[str, int], type[BaseHTTPRequestHandler]], Any] + + +class ThreadingHTTPServer(_ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + server_address: Any, + RequestHandlerClass: type[BaseHTTPRequestHandler], + bind_and_activate: bool = True, + ) -> None: + self.api_max_concurrent_requests = int( + getattr(RequestHandlerClass, "api_max_concurrent_requests", 32) + ) + self.api_request_timeout_seconds = float( + getattr(RequestHandlerClass, "api_request_timeout_seconds", 10.0) + ) + self._request_slots = threading.BoundedSemaphore(self.api_max_concurrent_requests) + self._request_state = threading.Condition() + self._active_requests: set[int] = set() + self._draining = False + self._drain_started = threading.Event() + self._graceful_shutdown_requested = False + self._graceful_shutdown_timeout_seconds = 0.0 + self._graceful_shutdown_thread: threading.Thread | None = None + self._graceful_shutdown_result: bool | None = None + super().__init__(server_address, RequestHandlerClass, bind_and_activate) + + def process_request(self, request: Any, client_address: Any) -> None: + with self._request_state: + if self._draining: + rejection = ("server_draining", "API server is draining") + elif not self._request_slots.acquire(blocking=False): + rejection = ("server_busy", "API request capacity is exhausted") + else: + self._active_requests.add(id(request)) + rejection = None + + if rejection is not None: + self._reject_unavailable_request(request, *rejection) + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._finish_request(request) + raise + + def process_request_thread(self, request: Any, client_address: Any) -> None: + try: + request.settimeout(self.api_request_timeout_seconds) + super().process_request_thread(request, client_address) + finally: + self._finish_request(request) + + def begin_draining(self) -> None: + with self._request_state: + self._draining = True + self._drain_started.set() + + def request_graceful_shutdown(self, timeout_seconds: float) -> None: + self._graceful_shutdown_timeout_seconds = timeout_seconds + self._graceful_shutdown_requested = True + + def wait_until_draining(self, timeout_seconds: float) -> bool: + return self._drain_started.wait(timeout_seconds) + + def service_actions(self) -> None: + super().service_actions() + if not self._graceful_shutdown_requested or self._graceful_shutdown_thread is not None: + return + self.begin_draining() + coordinator = threading.Thread(target=self._complete_graceful_shutdown, daemon=True) + coordinator.start() + self._graceful_shutdown_thread = coordinator + + def finish_graceful_shutdown(self) -> bool | None: + coordinator = self._graceful_shutdown_thread + if coordinator is None: + return None + coordinator.join() + return self._graceful_shutdown_result + + def wait_for_drain(self, timeout_seconds: float) -> bool: + deadline = time.monotonic() + timeout_seconds + with self._request_state: + while self._active_requests: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._request_state.wait(remaining) + return True + + def _finish_request(self, request: Any) -> None: + with self._request_state: + request_id = id(request) + if request_id not in self._active_requests: + return + self._active_requests.remove(request_id) + self._request_slots.release() + if not self._active_requests: + self._request_state.notify_all() + + def _complete_graceful_shutdown(self) -> None: + try: + self._graceful_shutdown_result = self.wait_for_drain( + self._graceful_shutdown_timeout_seconds + ) + finally: + self.shutdown() + + def _reject_unavailable_request(self, request: Any, code: str, message: str) -> None: + request_id = new_request_id() + body = json.dumps( + { + "error": { + "code": code, + "message": message, + "status": HTTPStatus.SERVICE_UNAVAILABLE.value, + } + }, + sort_keys=True, + ).encode("utf-8") + response = ( + b"HTTP/1.1 503 Service Unavailable\r\n" + b"connection: close\r\n" + b"content-type: application/json\r\n" + b"cache-control: no-store\r\n" + + f"x-request-id: {request_id}\r\n".encode("ascii") + + b"retry-after: 1\r\n" + + f"content-length: {len(body)}\r\n\r\n".encode("ascii") + + body + ) + with contextlib.suppress(OSError): + request.sendall(response) + + +def serve( + host: str, + port: int, + settings: Settings, + *, + handler_factory: HandlerFactory, + server_factory: ServerFactory, +) -> None: + runtime_settings = replace(settings, host=host, port=port) + if not 0 <= port <= 65535: + raise ValueError("port must be between 0 and 65535") + validate_api_exposure( + runtime_settings.host, + runtime_settings.api_key, + runtime_settings.allow_unauth_reads, + ) + runtime_settings.ensure_dirs() + pid = os.getpid() + pid_path = runtime_settings.state_dir / "zeus.pid" + api_lock = BotProcessLock( + runtime_settings.state_dir / "locks" / "api.lock", + timeout_seconds=min(runtime_settings.lock_timeout_seconds, 1.0), + ) + with api_lock: + handler = handler_factory(runtime_settings) + server = server_factory((host, port), handler) + previous_signal_handlers: dict[signal.Signals, Any] = {} + try: + if threading.current_thread() is threading.main_thread(): + for signum in (signal.SIGTERM, signal.SIGINT): + previous_signal_handlers[signum] = signal.getsignal(signum) + signal.signal( + signum, + lambda _signum, _frame: server.request_graceful_shutdown( + runtime_settings.api_shutdown_drain_seconds + ), + ) + with contextlib.suppress(KeyboardInterrupt): + _write_api_pid(pid_path, pid) + server.serve_forever() + finally: + for signum, previous_handler in previous_signal_handlers.items(): + signal.signal(signum, previous_handler) + finish_graceful_shutdown = getattr(server, "finish_graceful_shutdown", None) + drain_result = ( + finish_graceful_shutdown() if callable(finish_graceful_shutdown) else None + ) + if drain_result is None: + begin_draining = getattr(server, "begin_draining", None) + if callable(begin_draining): + begin_draining() + server.server_close() + wait_for_drain = getattr(server, "wait_for_drain", None) + drain_result = ( + wait_for_drain(runtime_settings.api_shutdown_drain_seconds) + if callable(wait_for_drain) + else True + ) + else: + server.server_close() + if not drain_result: + print("warning: API shutdown drain deadline expired", file=sys.stderr) + _remove_api_pid_if_owned(pid_path, pid) + + +def _write_api_pid(path: Path, pid: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + tmp = path.with_name(f".{path.name}.{pid}.{time.time_ns()}.tmp") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(f"{pid}\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + with contextlib.suppress(OSError): + path.chmod(0o600) + finally: + with contextlib.suppress(FileNotFoundError): + tmp.unlink() + + +def _remove_api_pid_if_owned(path: Path, pid: int) -> None: + try: + recorded_pid = int(path.read_text(encoding="utf-8").strip()) + except (FileNotFoundError, OSError, ValueError): + return + if recorded_pid != pid: + return + with contextlib.suppress(FileNotFoundError): + path.unlink() From e6dffbd77b6278c4d49dcd71005b437712f391b3 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:09:55 +0200 Subject: [PATCH 31/49] refactor(state): extract SQLite schema management --- tests/test_sqlite_schema.py | 832 ++++++++++++++++++++++++++++++++++++ zeus/reconciliation.py | 3 +- zeus/schema.py | 504 ++++++++++++++++++++++ zeus/sqlite_db.py | 41 ++ zeus/state.py | 519 +--------------------- 5 files changed, 1391 insertions(+), 508 deletions(-) create mode 100644 tests/test_sqlite_schema.py create mode 100644 zeus/schema.py create mode 100644 zeus/sqlite_db.py diff --git a/tests/test_sqlite_schema.py b/tests/test_sqlite_schema.py new file mode 100644 index 0000000..53403b8 --- /dev/null +++ b/tests/test_sqlite_schema.py @@ -0,0 +1,832 @@ +from __future__ import annotations + +import hashlib +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, call, patch + +from zeus.schema import SCHEMA_VERSION, SchemaManager, _preflight_schema_compatibility +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import StateStore + +_ORIGINAL_SQLITE_CONNECT = sqlite3.connect + +DDL_OBJECT_NAMES = ( + "bots", + "schema_version", + "lifecycle_events", + "idx_lifecycle_events_bot", + "idx_lifecycle_events_operation", + "lifecycle_events_reject_update", + "lifecycle_events_reject_delete", + "idempotency_records", + "idx_idempotency_records_expires", + "bots_desired_intent_reject_partial_insert", + "bots_desired_intent_reject_partial_update", + "reconcile_runs", + "reconcile_results", +) + +BASE_BOTS_SCHEMA = """ +CREATE TABLE bots ( + bot_id TEXT PRIMARY KEY, + template_id TEXT NOT NULL, + display_name TEXT NOT NULL, + profile_path TEXT NOT NULL, + status TEXT NOT NULL, + pid INTEGER, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +) +""" + +# Captured from the pre-extraction implementation. Every value is the exact +# sqlite_master SQL BLOB rendered as uppercase hexadecimal, not normalized SQL. +_FROZEN_COMMON_DDL_ROWS = ( + ( + "index", + "idx_idempotency_records_expires", + "idempotency_records", + ( + "43524541544520494E444558206964785F6964656D706F74656E63795F7265636F7264735F65" + "7870697265730A2020202020202020202020204F4E206964656D706F74656E63795F7265636F" + "7264732028657870697265735F6174290A202020202020202020202020" + ), + ), + ( + "index", + "idx_lifecycle_events_bot", + "lifecycle_events", + ( + "43524541544520494E444558206964785F6C6966656379636C655F6576656E74735F626F740A" + "2020202020202020202020204F4E206C6966656379636C655F6576656E74732028626F745F69" + "642C206576656E745F69642044455343290A202020202020202020202020" + ), + ), + ( + "index", + "idx_lifecycle_events_operation", + "lifecycle_events", + ( + "43524541544520494E444558206964785F6C6966656379636C655F6576656E74735F6F706572" + "6174696F6E0A2020202020202020202020204F4E206C6966656379636C655F6576656E747320" + "286F7065726174696F6E5F69642C206576656E745F6964290A202020202020202020202020" + ), + ), + ( + "table", + "idempotency_records", + "idempotency_records", + ( + "435245415445205441424C45206964656D706F74656E63795F7265636F72647320280A202020" + "202020202020202020202020206B65795F686173682054455854205052494D415259204B4559" + "2C0A20202020202020202020202020202020726571756573745F686173682054455854204E4F" + "54204E554C4C2C0A2020202020202020202020202020202073746174652054455854204E4F54" + "204E554C4C20434845434B2028737461746520494E202827696E5F70726F6772657373272C20" + "27636F6D706C657465642729292C0A202020202020202020202020202020206F776E65725F69" + "6E7374616E63655F69642054455854204E4F54204E554C4C2C0A202020202020202020202020" + "20202020726573706F6E73655F73746174757320494E5445474552204E554C4C2C0A20202020" + "202020202020202020202020726573706F6E73655F6A736F6E2054455854204E554C4C2C0A20" + "202020202020202020202020202020637265617465645F61742054455854204E4F54204E554C" + "4C2C0A20202020202020202020202020202020757064617465645F61742054455854204E4F54" + "204E554C4C2C0A20202020202020202020202020202020657870697265735F61742054455854" + "204E4F54204E554C4C0A20202020202020202020202029" + ), + ), + ( + "table", + "lifecycle_events", + "lifecycle_events", + ( + "435245415445205441424C45206C6966656379636C655F6576656E747320280A202020202020" + "202020202020202020206576656E745F696420494E5445474552205052494D415259204B4559" + "204155544F494E4352454D454E542C0A20202020202020202020202020202020626F745F6964" + "2054455854204E4F54204E554C4C2C0A202020202020202020202020202020206F7065726174" + "696F6E5F69642054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "726571756573745F696420544558542C0A202020202020202020202020202020206F63637572" + "7265645F61742054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "736F757263652054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "616374696F6E2054455854204E4F54204E554C4C2C0A20202020202020202020202020202020" + "6F7574636F6D652054455854204E4F54204E554C4C2C0A202020202020202020202020202020" + "207374617475735F6265666F726520544558542C0A2020202020202020202020202020202073" + "74617475735F616674657220544558542C0A202020202020202020202020202020207069645F" + "6265666F726520494E54454745522C0A202020202020202020202020202020207069645F6166" + "74657220494E54454745522C0A20202020202020202020202020202020726561736F6E205445" + "5854204E4F54204E554C4C2044454641554C542027272C0A2020202020202020202020202020" + "20206572726F725F636F646520544558542C0A20202020202020202020202020202020657272" + "6F725F6D65737361676520544558542C0A202020202020202020202020202020206465746169" + "6C735F6A736F6E2054455854204E4F54204E554C4C2044454641554C5420277B7D270A202020" + "20202020202020202029" + ), + ), + ( + "table", + "reconcile_results", + "reconcile_results", + ( + "435245415445205441424C45207265636F6E63696C655F726573756C747320280A2020202020" + "202020202020202020202072756E5F69642054455854204E4F54204E554C4C2C0A2020202020" + "2020202020202020202020626F745F69642054455854204E4F54204E554C4C2C0A2020202020" + "20202020202020202020206F7264696E616C20494E5445474552204E4F54204E554C4C204348" + "45434B20286F7264696E616C203E3D2030292C0A202020202020202020202020202020206F75" + "74636F6D652054455854204E4F54204E554C4C20434845434B20280A20202020202020202020" + "202020202020202020206F7574636F6D6520494E20280A202020202020202020202020202020" + "202020202020202020276865616C746879272C20276368616E676564272C202770656E64696E" + "67272C0A20202020202020202020202020202020202020202020202027616374696F6E5F7265" + "717569726564272C20276572726F72272C2027736B6970706564270A20202020202020202020" + "20202020202020202020290A20202020202020202020202020202020292C0A20202020202020" + "202020202020202020646573697265645F7374617465205445585420434845434B20280A2020" + "202020202020202020202020202020202020646573697265645F7374617465204953204E554C" + "4C204F5220646573697265645F737461746520494E20282772756E6E696E67272C202773746F" + "7070656427290A20202020202020202020202020202020292C0A202020202020202020202020" + "202020206F627365727665645F737461747573205445585420434845434B20280A2020202020" + "2020202020202020202020202020206F627365727665645F737461747573204953204E554C4C" + "0A20202020202020202020202020202020202020204F52206F627365727665645F7374617475" + "7320494E20282773746F70706564272C20277374617274696E67272C202772756E6E696E6727" + "2C20276661696C6564272C2027756E6B6E6F776E27290A202020202020202020202020202020" + "20292C0A2020202020202020202020202020202070696420494E544547455220434845434B20" + "28706964204953204E554C4C204F5220706964203E2030292C0A202020202020202020202020" + "20202020616374696F6E2054455854204E4F54204E554C4C20434845434B20286C656E677468" + "28616374696F6E29203C3D2032303438292C0A202020202020202020202020202020206D6573" + "736167652054455854204E4F54204E554C4C20434845434B20286C656E677468286D65737361" + "676529203C3D2032303438292C0A202020202020202020202020202020206572726F725F636F" + "6465205445585420434845434B20286572726F725F636F6465204953204E554C4C204F52206C" + "656E677468286572726F725F636F646529203C3D2032303438292C0A20202020202020202020" + "2020202020206576656E745F696420494E544547455220434845434B20286576656E745F6964" + "204953204E554C4C204F52206576656E745F6964203E2030292C0A2020202020202020202020" + "2020202020737461727465645F61742054455854204E4F54204E554C4C2C0A20202020202020" + "20202020202020202066696E69736865645F61742054455854204E4F54204E554C4C2C0A2020" + "20202020202020202020202020205052494D415259204B4559202872756E5F69642C20626F74" + "5F6964292C0A20202020202020202020202020202020554E49515545202872756E5F69642C20" + "6F7264696E616C292C0A20202020202020202020202020202020434845434B202866696E6973" + "6865645F6174203E3D20737461727465645F6174292C0A202020202020202020202020202020" + "20464F524549474E204B4559202872756E5F696429205245464552454E434553207265636F6E" + "63696C655F72756E732872756E5F696429204F4E2044454C45544520434153434144450A2020" + "2020202020202020202029" + ), + ), + ( + "table", + "reconcile_runs", + "reconcile_runs", + ( + "435245415445205441424C45207265636F6E63696C655F72756E7320280A2020202020202020" + "202020202020202072756E5F69642054455854205052494D415259204B45592C0A2020202020" + "202020202020202020202073636F70652054455854204E4F54204E554C4C20434845434B2028" + "73636F706520494E202827666C656574272C2027626F742729292C0A20202020202020202020" + "2020202020207265717565737465645F626F745F696420544558542C0A202020202020202020" + "20202020202020736F757263652054455854204E4F54204E554C4C2C0A202020202020202020" + "20202020202020666F72636520494E5445474552204E4F54204E554C4C20434845434B202866" + "6F72636520494E2028302C203129292C0A202020202020202020202020202020207265736574" + "5F7265737461727420494E5445474552204E4F54204E554C4C20434845434B20287265736574" + "5F7265737461727420494E2028302C203129292C0A2020202020202020202020202020202073" + "7461727465645F61742054455854204E4F54204E554C4C2C0A20202020202020202020202020" + "20202066696E69736865645F617420544558542C0A202020202020202020202020202020206F" + "7574636F6D652054455854204E4F54204E554C4C20434845434B20280A202020202020202020" + "20202020202020202020206F7574636F6D6520494E20282772756E6E696E67272C2027737563" + "636565646564272C2027636F6D706C657465645F776974685F6572726F7273272C2027696E74" + "657272757074656427290A20202020202020202020202020202020292C0A2020202020202020" + "2020202020202020746F74616C20494E5445474552204E4F54204E554C4C2044454641554C54" + "203020434845434B2028746F74616C203E3D2030292C0A202020202020202020202020202020" + "206865616C7468795F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C" + "54203020434845434B20286865616C7468795F636F756E74203E3D2030292C0A202020202020" + "202020202020202020206368616E6765645F636F756E7420494E5445474552204E4F54204E55" + "4C4C2044454641554C54203020434845434B20286368616E6765645F636F756E74203E3D2030" + "292C0A2020202020202020202020202020202070656E64696E675F636F756E7420494E544547" + "4552204E4F54204E554C4C2044454641554C54203020434845434B202870656E64696E675F63" + "6F756E74203E3D2030292C0A20202020202020202020202020202020616374696F6E5F726571" + "75697265645F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C542030" + "0A2020202020202020202020202020202020202020434845434B2028616374696F6E5F726571" + "75697265645F636F756E74203E3D2030292C0A20202020202020202020202020202020657272" + "6F725F636F756E7420494E5445474552204E4F54204E554C4C2044454641554C542030204348" + "45434B20286572726F725F636F756E74203E3D2030292C0A2020202020202020202020202020" + "2020736B69707065645F636F756E7420494E5445474552204E4F54204E554C4C204445464155" + "4C54203020434845434B2028736B69707065645F636F756E74203E3D2030292C0A2020202020" + "2020202020202020202020434845434B20280A20202020202020202020202020202020202020" + "20746F74616C203D206865616C7468795F636F756E74202B206368616E6765645F636F756E74" + "202B2070656E64696E675F636F756E740A202020202020202020202020202020202020202020" + "20202020202B20616374696F6E5F72657175697265645F636F756E74202B206572726F725F63" + "6F756E74202B20736B69707065645F636F756E740A2020202020202020202020202020202029" + "2C0A20202020202020202020202020202020434845434B20280A202020202020202020202020" + "20202020202020202873636F7065203D2027666C6565742720414E4420726571756573746564" + "5F626F745F6964204953204E554C4C290A20202020202020202020202020202020202020204F" + "52202873636F7065203D2027626F742720414E44207265717565737465645F626F745F696420" + "4953204E4F54204E554C4C290A20202020202020202020202020202020292C0A202020202020" + "20202020202020202020434845434B20280A2020202020202020202020202020202020202020" + "286F7574636F6D65203D202772756E6E696E672720414E442066696E69736865645F61742049" + "53204E554C4C290A20202020202020202020202020202020202020204F5220286F7574636F6D" + "6520213D202772756E6E696E672720414E442066696E69736865645F6174204953204E4F5420" + "4E554C4C290A20202020202020202020202020202020292C0A20202020202020202020202020" + "202020434845434B202866696E69736865645F6174204953204E554C4C204F522066696E6973" + "6865645F6174203E3D20737461727465645F6174290A20202020202020202020202029" + ), + ), + ( + "trigger", + "bots_desired_intent_reject_partial_insert", + "bots", + ( + "435245415445205452494747455220626F74735F646573697265645F696E74656E745F72656A" + "6563745F7061727469616C5F696E736572740A2020202020202020202020204245464F524520" + "494E53455254204F4E20626F74730A2020202020202020202020205748454E204E4F5420280A" + "20202020202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E" + "5F6964204953204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64696E675F" + "73696E6365204953204E554C4C290A202020202020202020202020202020204F520A20202020" + "202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E5F696420" + "4953204E4F54204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E4F54204E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64" + "696E675F73696E6365204953204E4F54204E554C4C290A202020202020202020202020290A20" + "2020202020202020202020424547494E0A2020202020202020202020202020202053454C4543" + "542052414953452841424F52542C202770656E64696E67206C6966656379636C6520696E7465" + "6E74206D75737420626520616C6C206E756C6C206F7220616C6C20706F70756C617465642729" + "3B0A202020202020202020202020454E44" + ), + ), + ( + "trigger", + "bots_desired_intent_reject_partial_update", + "bots", + ( + "435245415445205452494747455220626F74735F646573697265645F696E74656E745F72656A" + "6563745F7061727469616C5F7570646174650A2020202020202020202020204245464F524520" + "555044415445204F4E20626F74730A2020202020202020202020205748454E204E4F5420280A" + "20202020202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E" + "5F6964204953204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64696E675F" + "73696E6365204953204E554C4C290A202020202020202020202020202020204F520A20202020" + "202020202020202020202020284E45572E70656E64696E675F6F7065726174696F6E5F696420" + "4953204E4F54204E554C4C20414E44204E45572E70656E64696E675F616374696F6E20495320" + "4E4F54204E554C4C0A2020202020202020202020202020202020414E44204E45572E70656E64" + "696E675F73696E6365204953204E4F54204E554C4C290A202020202020202020202020290A20" + "2020202020202020202020424547494E0A2020202020202020202020202020202053454C4543" + "542052414953452841424F52542C202770656E64696E67206C6966656379636C6520696E7465" + "6E74206D75737420626520616C6C206E756C6C206F7220616C6C20706F70756C617465642729" + "3B0A202020202020202020202020454E44" + ), + ), + ( + "trigger", + "lifecycle_events_reject_delete", + "lifecycle_events", + ( + "4352454154452054524947474552206C6966656379636C655F6576656E74735F72656A656374" + "5F64656C6574650A2020202020202020202020204245464F52452044454C455445204F4E206C" + "6966656379636C655F6576656E74730A202020202020202020202020424547494E0A20202020" + "20202020202020202020202053454C4543542052414953452841424F52542C20276C69666563" + "79636C65206576656E74732061726520696D6D757461626C6527293B0A202020202020202020" + "202020454E44" + ), + ), + ( + "trigger", + "lifecycle_events_reject_update", + "lifecycle_events", + ( + "4352454154452054524947474552206C6966656379636C655F6576656E74735F72656A656374" + "5F7570646174650A2020202020202020202020204245464F524520555044415445204F4E206C" + "6966656379636C655F6576656E74730A202020202020202020202020424547494E0A20202020" + "20202020202020202020202053454C4543542052414953452841424F52542C20276C69666563" + "79636C65206576656E74732061726520696D6D757461626C6527293B0A202020202020202020" + "202020454E44" + ), + ), +) +_FROZEN_FRESH_BOTS_SQL_HEX = ( + "435245415445205441424C4520626F747320280A20202020202020202020202020202020626F" + "745F69642054455854205052494D415259204B45592C0A202020202020202020202020202020" + "2074656D706C6174655F69642054455854204E4F54204E554C4C2C0A20202020202020202020" + "202020202020646973706C61795F6E616D652054455854204E4F54204E554C4C2C0A20202020" + "20202020202020202020202070726F66696C655F706174682054455854204E4F54204E554C4C" + "2C0A202020202020202020202020202020207374617475732054455854204E4F54204E554C4C" + "2C0A2020202020202020202020202020202070696420494E54454745522C0A20202020202020" + "202020202020202020726573746172745F706F6C6963792054455854204E4F54204E554C4C20" + "44454641554C5420276D616E75616C272C0A2020202020202020202020202020202072657374" + "6172745F6261636B6F66665F7365636F6E6473205245414C204E4F54204E554C4C2044454641" + "554C5420352E302C0A20202020202020202020202020202020726573746172745F6D61785F61" + "7474656D70747320494E5445474552204E4F54204E554C4C2044454641554C5420352C0A2020" + "2020202020202020202020202020726573746172745F617474656D70747320494E5445474552" + "204E4F54204E554C4C2044454641554C5420302C0A202020202020202020202020202020206E" + "6578745F726573746172745F617420544558542C0A2020202020202020202020202020202073" + "7461727465645F617420544558542C0A2020202020202020202020202020202072656164795F" + "617420544558542C0A2020202020202020202020202020202073746F707065645F6174205445" + "58542C0A202020202020202020202020202020206C6173745F657869745F636F646520494E54" + "454745522C0A202020202020202020202020202020206C6173745F6572726F7220544558542C" + "0A202020202020202020202020202020206C6173745F7472616E736974696F6E5F726561736F" + "6E20544558542C0A202020202020202020202020202020206C6173745F6576656E745F696420" + "494E54454745522C0A20202020202020202020202020202020646573697265645F7374617465" + "2054455854204E4F54204E554C4C2044454641554C54202773746F70706564270A2020202020" + "202020202020202020202020202020434845434B2028646573697265645F737461746520494E" + "20282772756E6E696E67272C202773746F707065642729292C0A202020202020202020202020" + "20202020646573697265645F7265766973696F6E20494E5445474552204E4F54204E554C4C20" + "44454641554C54203020434845434B2028646573697265645F7265766973696F6E203E3D2030" + "292C0A20202020202020202020202020202020646573697265645F757064617465645F617420" + "544558542C0A2020202020202020202020202020202070656E64696E675F6F7065726174696F" + "6E5F696420544558542C0A2020202020202020202020202020202070656E64696E675F616374" + "696F6E205445585420434845434B20280A202020202020202020202020202020202020202070" + "656E64696E675F616374696F6E204953204E554C4C204F522070656E64696E675F616374696F" + "6E20494E2028277374617274272C202773746F70272C20277265737461727427290A20202020" + "202020202020202020202020292C0A2020202020202020202020202020202070656E64696E67" + "5F73696E636520544558542C0A20202020202020202020202020202020637265617465645F61" + "742054455854204E4F54204E554C4C2C0A202020202020202020202020202020207570646174" + "65645F61742054455854204E4F54204E554C4C2C0A2020202020202020202020202020202043" + "4845434B20280A20202020202020202020202020202020202020202870656E64696E675F6F70" + "65726174696F6E5F6964204953204E554C4C0A20202020202020202020202020202020202020" + "2020414E442070656E64696E675F616374696F6E204953204E554C4C0A202020202020202020" + "202020202020202020202020414E442070656E64696E675F73696E6365204953204E554C4C29" + "0A20202020202020202020202020202020202020204F520A2020202020202020202020202020" + "2020202020202870656E64696E675F6F7065726174696F6E5F6964204953204E4F54204E554C" + "4C20414E442070656E64696E675F616374696F6E204953204E4F54204E554C4C0A2020202020" + "20202020202020202020202020202020414E442070656E64696E675F73696E6365204953204E" + "4F54204E554C4C290A20202020202020202020202020202020290A2020202020202020202020" + "2029" +) +_FROZEN_LEGACY_BOTS_SQL_HEX = ( + "435245415445205441424C4520626F747320280A20202020626F745F69642054455854205052" + "494D415259204B45592C0A2020202074656D706C6174655F69642054455854204E4F54204E55" + "4C4C2C0A20202020646973706C61795F6E616D652054455854204E4F54204E554C4C2C0A2020" + "202070726F66696C655F706174682054455854204E4F54204E554C4C2C0A2020202073746174" + "75732054455854204E4F54204E554C4C2C0A2020202070696420494E54454745522C0A202020" + "20637265617465645F61742054455854204E4F54204E554C4C2C0A2020202075706461746564" + "5F61742054455854204E4F54204E554C4C0A2C20726573746172745F706F6C69637920544558" + "54204E4F54204E554C4C2044454641554C5420276D616E75616C272C20726573746172745F62" + "61636B6F66665F7365636F6E6473205245414C204E4F54204E554C4C2044454641554C542035" + "2E302C20726573746172745F6D61785F617474656D70747320494E5445474552204E4F54204E" + "554C4C2044454641554C5420352C20726573746172745F617474656D70747320494E54454745" + "52204E4F54204E554C4C2044454641554C5420302C206E6578745F726573746172745F617420" + "544558542C20737461727465645F617420544558542C2072656164795F617420544558542C20" + "73746F707065645F617420544558542C206C6173745F657869745F636F646520494E54454745" + "522C206C6173745F6572726F7220544558542C206C6173745F7472616E736974696F6E5F7265" + "61736F6E20544558542C206C6173745F6576656E745F696420494E54454745522C2064657369" + "7265645F73746174652054455854204E4F54204E554C4C2044454641554C54202773746F7070" + "65642720434845434B2028646573697265645F737461746520494E20282772756E6E696E6727" + "2C202773746F707065642729292C20646573697265645F7265766973696F6E20494E54454745" + "52204E4F54204E554C4C2044454641554C54203020434845434B2028646573697265645F7265" + "766973696F6E203E3D2030292C20646573697265645F757064617465645F617420544558542C" + "2070656E64696E675F6F7065726174696F6E5F696420544558542C2070656E64696E675F6163" + "74696F6E205445585420434845434B202870656E64696E675F616374696F6E204953204E554C" + "4C204F522070656E64696E675F616374696F6E20494E2028277374617274272C202773746F70" + "272C2027726573746172742729292C2070656E64696E675F73696E6365205445585429" +) +_FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX = ( + "435245415445205441424C4520736368656D615F76657273696F6E20280A2020202020202020" + "202020202020202076657273696F6E20494E5445474552204E4F54204E554C4C0A2020202020" + "2020202020202029" +) +_FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX = ( + "435245415445205441424C4520736368656D615F76657273696F6E20280A2020202020202020" + "20202020202020202020202076657273696F6E20494E5445474552204E4F54204E554C4C0A20" + "20202020202020202020202020202029" +) +EXPECTED_DDL_CATALOGS: dict[str, tuple[tuple[str, str, str, str], ...]] = { + "fresh": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_FRESH_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v0": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_CREATED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v1": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v2": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v3": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v4": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), + "v5": tuple( + sorted( + ( + *_FROZEN_COMMON_DDL_ROWS, + ("table", "bots", "bots", _FROZEN_LEGACY_BOTS_SQL_HEX), + ( + "table", + "schema_version", + "schema_version", + _FROZEN_SEEDED_SCHEMA_VERSION_SQL_HEX, + ), + ) + ) + ), +} + + +def _ddl_catalog(database_path: Path) -> tuple[tuple[str, str, str, str], ...]: + placeholders = ", ".join("?" for _name in DDL_OBJECT_NAMES) + with closing(sqlite3.connect(database_path)) as conn: + rows = conn.execute( + f""" + SELECT type, name, tbl_name, hex(CAST(sql AS BLOB)) + FROM sqlite_master + WHERE name IN ({placeholders}) + ORDER BY type, name + """, + DDL_OBJECT_NAMES, + ).fetchall() + return tuple((str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows) + + +def _seed_legacy_database(database_path: Path, version: int, *, with_bot: bool = False) -> None: + if not 0 <= version <= 5: + raise ValueError("legacy schema version must be between 0 and 5") + manager = SchemaManager(SQLiteDatabase(database_path)) + with closing(sqlite3.connect(database_path)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("BEGIN IMMEDIATE") + conn.execute(BASE_BOTS_SCHEMA) + if version >= 1: + manager._ensure_restart_schema(conn) + if version >= 2: + manager._ensure_lifecycle_schema(conn) + if with_bot: + conn.execute( + """ + INSERT INTO bots ( + bot_id, template_id, display_name, profile_path, status, pid, + created_at, updated_at + ) VALUES ( + 'coder', 'coding-bot', 'Coder', '/profiles/coder', 'running', 4321, + '2026-01-01T00:00:00+00:00', '2026-01-01T00:00:00+00:00' + ) + """ + ) + if version >= 3: + manager._migrate_v2_to_v3(conn) + if version >= 4: + manager._migrate_v3_to_v4(conn) + if version >= 5: + manager._migrate_v4_to_v5(conn) + if version > 0: + conn.execute( + """ + CREATE TABLE schema_version ( + version INTEGER NOT NULL + ) + """ + ) + conn.execute("INSERT INTO schema_version (version) VALUES (?)", (version,)) + conn.commit() + + +def _database_snapshot( + database_path: Path, +) -> tuple[ + str, + tuple[str, ...], + str, + tuple[tuple[str, bool], ...], +]: + digest = hashlib.sha256(database_path.read_bytes()).hexdigest() + uri = f"{database_path.resolve().as_uri()}?mode=ro" + with closing(_ORIGINAL_SQLITE_CONNECT(uri, uri=True)) as conn: + dump = tuple(conn.iterdump()) + journal_mode = str(conn.execute("PRAGMA journal_mode").fetchone()[0]) + sidecars = tuple( + (suffix, database_path.with_name(database_path.name + suffix).exists()) + for suffix in ("-wal", "-shm", "-journal") + ) + return digest, dump, journal_mode, sidecars + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +class SQLiteSchemaTests(unittest.TestCase): + def test_new_schema_components_construct_without_opening_the_database(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "nested" / "zeus.db" + + database = SQLiteDatabase(database_path) + SchemaManager(database) + + self.assertEqual(database_path, database.database_path) + self.assertFalse(database_path.parent.exists()) + + def test_connection_preserves_row_factory_and_all_pragmas(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase(Path(tmp) / "nested" / "zeus.db") + + with closing(database.connect()) as conn: + values = { + "foreign_keys": conn.execute("PRAGMA foreign_keys").fetchone()[0], + "journal_mode": conn.execute("PRAGMA journal_mode").fetchone()[0], + "synchronous": conn.execute("PRAGMA synchronous").fetchone()[0], + "busy_timeout": conn.execute("PRAGMA busy_timeout").fetchone()[0], + } + row = conn.execute("SELECT 1 AS value").fetchone() + + self.assertIs(sqlite3.Row, conn.row_factory) + self.assertIsInstance(row, sqlite3.Row) + self.assertEqual( + { + "foreign_keys": 1, + "journal_mode": "wal", + "synchronous": 1, + "busy_timeout": 5000, + }, + values, + ) + + def test_immediate_uses_one_connection_and_commits_or_rolls_back(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = _TracingSQLiteDatabase(Path(tmp) / "zeus.db") + with database.immediate() as conn: + conn.execute("CREATE TABLE committed (value TEXT NOT NULL)") + conn.execute("INSERT INTO committed VALUES ('kept')") + + successful_trace = list(database.traces) + database.traces.clear() + with ( + self.assertRaisesRegex(RuntimeError, "injected rollback"), + database.immediate() as conn, + ): + conn.execute("INSERT INTO committed VALUES ('discarded')") + raise RuntimeError("injected rollback") + rollback_trace = list(database.traces) + + with closing(database.connect()) as conn: + values = [row[0] for row in conn.execute("SELECT value FROM committed")] + + successful_sql = [sql for _identity, sql in successful_trace] + rollback_sql = [sql for _identity, sql in rollback_trace] + self.assertEqual(1, len({identity for identity, _sql in successful_trace})) + self.assertEqual("BEGIN IMMEDIATE", successful_sql[0]) + self.assertEqual("COMMIT", successful_sql[-1]) + self.assertEqual("BEGIN IMMEDIATE", rollback_sql[0]) + self.assertEqual("ROLLBACK", rollback_sql[-1]) + self.assertEqual(["kept"], values) + + def test_state_store_delegates_without_opening_during_construction(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + connection = MagicMock(spec=sqlite3.Connection) + database.connect.return_value = connection + schema = MagicMock(spec=SchemaManager) + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + ): + store = StateStore(database_path) + database.connect.assert_not_called() + schema.init.assert_not_called() + schema.migrate.assert_not_called() + + self.assertEqual(database_path, store.database_path) + self.assertIs(connection, store.connect()) + store.init() + store.migrate() + + self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual([call(database)], schema_type.call_args_list) + database.connect.assert_called_once_with() + schema.init.assert_called_once_with() + schema.migrate.assert_called_once_with() + + def test_fresh_and_every_legacy_branch_match_frozen_ddl_catalogs(self) -> None: + for case in ("fresh", "v0", "v1", "v2", "v3", "v4", "v5"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + store = StateStore(database_path) + if case == "fresh": + store.init() + else: + _seed_legacy_database(database_path, int(case[1:])) + store.migrate() + + self.assertEqual(EXPECTED_DDL_CATALOGS[case], _ddl_catalog(database_path)) + with closing(store.connect()) as conn: + self.assertEqual( + SCHEMA_VERSION, + conn.execute("SELECT version FROM schema_version").fetchone()[0], + ) + + def test_schema_init_and_migrate_each_use_one_immediate_transaction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + init_database = _TracingSQLiteDatabase(Path(tmp) / "fresh.db") + SchemaManager(init_database).init() + init_sql = [sql for _identity, sql in init_database.traces] + + migrate_path = Path(tmp) / "legacy.db" + _seed_legacy_database(migrate_path, 5) + migrate_database = _TracingSQLiteDatabase(migrate_path) + SchemaManager(migrate_database).migrate() + migrate_sql = [sql for _identity, sql in migrate_database.traces] + + self.assertEqual(1, len({identity for identity, _sql in init_database.traces})) + self.assertEqual("BEGIN IMMEDIATE", init_sql[0]) + self.assertEqual("COMMIT", init_sql[-1]) + self.assertLess( + next(index for index, sql in enumerate(init_sql) if "bots (" in sql), + next(index for index, sql in enumerate(init_sql) if "UPDATE schema_version" in sql), + ) + self.assertEqual(1, len({identity for identity, _sql in migrate_database.traces})) + self.assertEqual("BEGIN IMMEDIATE", migrate_sql[0]) + self.assertIn("CREATE TABLE reconcile_runs", "\n".join(migrate_sql)) + self.assertIn("UPDATE schema_version SET version = 6", migrate_sql) + self.assertEqual("COMMIT", migrate_sql[-1]) + + def test_migration_failure_rolls_back_ddl_backfill_and_version_together(self) -> None: + class FailingSchemaManager(SchemaManager): + def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: + super()._migrate_v5_to_v6(conn) + raise sqlite3.OperationalError("injected migration failure") + + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + _seed_legacy_database(database_path, 2, with_bot=True) + before_dump = _database_snapshot(database_path)[1] + database = _TracingSQLiteDatabase(database_path) + + with self.assertRaisesRegex(sqlite3.OperationalError, "injected migration failure"): + FailingSchemaManager(database).migrate() + + after_dump = _database_snapshot(database_path)[1] + with closing(sqlite3.connect(database_path)) as conn: + version = conn.execute("SELECT version FROM schema_version").fetchone()[0] + last_event_column = [ + row[1] + for row in conn.execute("PRAGMA table_info(bots)") + if row[1] == "last_event_id" + ] + lifecycle_table = conn.execute( + "SELECT 1 FROM sqlite_master WHERE name = 'lifecycle_events'" + ).fetchone() + bot = conn.execute( + "SELECT status, pid, updated_at FROM bots WHERE bot_id = 'coder'" + ).fetchone() + + sql = [statement for _identity, statement in database.traces] + self.assertEqual(before_dump, after_dump) + self.assertEqual(2, version) + self.assertEqual([], last_event_column) + self.assertIsNone(lifecycle_table) + self.assertEqual(("running", 4321, "2026-01-01T00:00:00+00:00"), bot) + self.assertEqual("BEGIN IMMEDIATE", sql[0]) + self.assertIn("INSERT INTO lifecycle_events", "\n".join(sql)) + self.assertIn("UPDATE schema_version SET version = 5", sql) + self.assertEqual("ROLLBACK", sql[-1]) + + def test_v7_rejection_is_read_only_for_connect_init_and_migrate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + with closing(sqlite3.connect(database_path)) as conn: + conn.execute("UPDATE schema_version SET version = 7") + conn.commit() + before = _database_snapshot(database_path) + expected_uri = f"{database_path.resolve().as_uri()}?mode=ro" + real_connect = sqlite3.connect + + for entry_point in ("connect", "init", "migrate"): + calls: list[tuple[object, bool]] = [] + + def connect_spy( + *args: object, + _calls: list[tuple[object, bool]] = calls, + **kwargs: object, + ) -> sqlite3.Connection: + _calls.append((args[0], bool(kwargs.get("uri", False)))) + return real_connect(*args, **kwargs) + + with ( + self.subTest(entry_point=entry_point), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy), + self.assertRaisesRegex(RuntimeError, "newer than supported"), + ): + getattr(StateStore(database_path), entry_point)() + + self.assertEqual([(expected_uri, True)], calls) + self.assertEqual(before, _database_snapshot(database_path)) + + def test_toctou_second_guard_rejects_before_pragmas_or_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + real_connect = sqlite3.connect + traces: list[str] = [] + post_swap_snapshot: list[object] = [] + calls: list[tuple[object, bool]] = [] + + def swap_after_preflight(path: Path) -> None: + _preflight_schema_compatibility(path) + with closing(real_connect(path)) as conn: + conn.execute("UPDATE schema_version SET version = 7") + conn.commit() + post_swap_snapshot.append(_database_snapshot(path)) + + def connect_spy(*args: object, **kwargs: object) -> sqlite3.Connection: + calls.append((args[0], bool(kwargs.get("uri", False)))) + conn = real_connect(*args, **kwargs) + if not kwargs.get("uri", False): + conn.set_trace_callback(traces.append) + return conn + + with ( + patch( + "zeus.sqlite_db._preflight_schema_compatibility", + side_effect=swap_after_preflight, + ), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy), + self.assertRaisesRegex(RuntimeError, "newer than supported"), + ): + SQLiteDatabase(database_path).connect() + + self.assertEqual(1, len(post_swap_snapshot)) + self.assertEqual(post_swap_snapshot[0], _database_snapshot(database_path)) + self.assertEqual(2, len(calls)) + self.assertTrue(calls[0][1]) + self.assertFalse(calls[1][1]) + self.assertTrue(any("schema_version" in sql for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("PRAGMA") for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("BEGIN") for sql in traces)) + self.assertFalse(any(sql.lstrip().upper().startswith("UPDATE") for sql in traces)) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/reconciliation.py b/zeus/reconciliation.py index 0e52453..606bcd3 100644 --- a/zeus/reconciliation.py +++ b/zeus/reconciliation.py @@ -440,7 +440,8 @@ def summarize_results( class _ReconciliationStore(Protocol): - database_path: Path + @property + def database_path(self) -> Path: ... def list_bots(self) -> list[BotRecord]: ... diff --git a/zeus/schema.py b/zeus/schema.py new file mode 100644 index 0000000..83be196 --- /dev/null +++ b/zeus/schema.py @@ -0,0 +1,504 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import AbstractContextManager, closing +from pathlib import Path +from typing import Protocol + +from zeus.lifecycle import serialize_lifecycle_details + +SCHEMA_VERSION = 6 + + +class _SchemaDatabase(Protocol): + def connect(self) -> sqlite3.Connection: ... + + def immediate(self) -> AbstractContextManager[sqlite3.Connection]: ... + + +def _assert_schema_compatible(conn: sqlite3.Connection) -> None: + table = conn.execute( + """ + SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'schema_version' + """ + ).fetchone() + if table is None: + return + row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() + if row is not None and int(row["version"]) > SCHEMA_VERSION: + raise RuntimeError( + f"database schema version {int(row['version'])} is newer than supported " + f"version {SCHEMA_VERSION}" + ) + + +def _preflight_schema_compatibility(database_path: Path) -> None: + if not database_path.exists(): + return + uri = f"{database_path.resolve().as_uri()}?mode=ro" + with closing(sqlite3.connect(uri, uri=True)) as conn: + conn.row_factory = sqlite3.Row + _assert_schema_compatible(conn) + + +class SchemaManager: + def __init__(self, database: _SchemaDatabase) -> None: + self._database = database + + def init(self) -> None: + with self._database.immediate() as conn: + self._create_bots_table(conn) + self._migrate(conn) + + def migrate(self) -> None: + with self._database.immediate() as conn: + self._migrate(conn) + + def _create_bots_table(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS bots ( + bot_id TEXT PRIMARY KEY, + template_id TEXT NOT NULL, + display_name TEXT NOT NULL, + profile_path TEXT NOT NULL, + status TEXT NOT NULL, + pid INTEGER, + restart_policy TEXT NOT NULL DEFAULT 'manual', + restart_backoff_seconds REAL NOT NULL DEFAULT 5.0, + restart_max_attempts INTEGER NOT NULL DEFAULT 5, + restart_attempts INTEGER NOT NULL DEFAULT 0, + next_restart_at TEXT, + started_at TEXT, + ready_at TEXT, + stopped_at TEXT, + last_exit_code INTEGER, + last_error TEXT, + last_transition_reason TEXT, + last_event_id INTEGER, + desired_state TEXT NOT NULL DEFAULT 'stopped' + CHECK (desired_state IN ('running', 'stopped')), + desired_revision INTEGER NOT NULL DEFAULT 0 CHECK (desired_revision >= 0), + desired_updated_at TEXT, + pending_operation_id TEXT, + pending_action TEXT CHECK ( + pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart') + ), + pending_since TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ( + (pending_operation_id IS NULL + AND pending_action IS NULL + AND pending_since IS NULL) + OR + (pending_operation_id IS NOT NULL AND pending_action IS NOT NULL + AND pending_since IS NOT NULL) + ) + ) + """ + ) + + def _migrate(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER NOT NULL + ) + """ + ) + row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() + current_version = int(row["version"]) if row else 0 + if row is None: + conn.execute("INSERT INTO schema_version (version) VALUES (0)") + if current_version > SCHEMA_VERSION: + raise RuntimeError( + f"database schema version {current_version} is newer than supported " + f"version {SCHEMA_VERSION}" + ) + if current_version < 1: + self._ensure_restart_schema(conn) + conn.execute("UPDATE schema_version SET version = ?", (1,)) + current_version = 1 + if current_version < 2: + self._ensure_lifecycle_schema(conn) + conn.execute("UPDATE schema_version SET version = ?", (2,)) + current_version = 2 + if current_version < 3: + self._migrate_v2_to_v3(conn) + conn.execute("UPDATE schema_version SET version = ?", (3,)) + current_version = 3 + if current_version < 4: + self._migrate_v3_to_v4(conn) + conn.execute("UPDATE schema_version SET version = ?", (4,)) + current_version = 4 + if current_version < 5: + self._migrate_v4_to_v5(conn) + conn.execute("UPDATE schema_version SET version = ?", (5,)) + current_version = 5 + if current_version < 6: + self._migrate_v5_to_v6(conn) + conn.execute("UPDATE schema_version SET version = ?", (6,)) + + def _ensure_restart_schema(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "restart_policy": ( + "ALTER TABLE bots ADD COLUMN restart_policy TEXT NOT NULL DEFAULT 'manual'" + ), + "restart_backoff_seconds": ( + "ALTER TABLE bots ADD COLUMN restart_backoff_seconds REAL NOT NULL DEFAULT 5.0" + ), + "restart_max_attempts": ( + "ALTER TABLE bots ADD COLUMN restart_max_attempts INTEGER NOT NULL DEFAULT 5" + ), + "restart_attempts": ( + "ALTER TABLE bots ADD COLUMN restart_attempts INTEGER NOT NULL DEFAULT 0" + ), + "next_restart_at": "ALTER TABLE bots ADD COLUMN next_restart_at TEXT", + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + def _ensure_lifecycle_schema(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "started_at": "ALTER TABLE bots ADD COLUMN started_at TEXT", + "ready_at": "ALTER TABLE bots ADD COLUMN ready_at TEXT", + "stopped_at": "ALTER TABLE bots ADD COLUMN stopped_at TEXT", + "last_exit_code": "ALTER TABLE bots ADD COLUMN last_exit_code INTEGER", + "last_error": "ALTER TABLE bots ADD COLUMN last_error TEXT", + "last_transition_reason": ("ALTER TABLE bots ADD COLUMN last_transition_reason TEXT"), + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + def _migrate_v2_to_v3(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + if "last_event_id" not in columns: + conn.execute("ALTER TABLE bots ADD COLUMN last_event_id INTEGER") + conn.execute( + """ + CREATE TABLE lifecycle_events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + bot_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + request_id TEXT, + occurred_at TEXT NOT NULL, + source TEXT NOT NULL, + action TEXT NOT NULL, + outcome TEXT NOT NULL, + status_before TEXT, + status_after TEXT, + pid_before INTEGER, + pid_after INTEGER, + reason TEXT NOT NULL DEFAULT '', + error_code TEXT, + error_message TEXT, + details_json TEXT NOT NULL DEFAULT '{}' + ) + """ + ) + conn.execute( + """ + CREATE INDEX idx_lifecycle_events_bot + ON lifecycle_events (bot_id, event_id DESC) + """ + ) + conn.execute( + """ + CREATE INDEX idx_lifecycle_events_operation + ON lifecycle_events (operation_id, event_id) + """ + ) + conn.execute( + """ + CREATE TRIGGER lifecycle_events_reject_update + BEFORE UPDATE ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'lifecycle events are immutable'); + END + """ + ) + conn.execute( + """ + CREATE TRIGGER lifecycle_events_reject_delete + BEFORE DELETE ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'lifecycle events are immutable'); + END + """ + ) + + rows = conn.execute( + "SELECT bot_id, status, pid, updated_at FROM bots ORDER BY bot_id" + ).fetchall() + for row in rows: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, + operation_id, + occurred_at, + source, + action, + outcome, + status_before, + status_after, + pid_before, + pid_after, + reason, + details_json + ) VALUES (?, 'migration-v3', ?, 'migration', 'migration.snapshot', + 'success', ?, ?, ?, ?, 'schema v3 snapshot', '{}') + """, + ( + row["bot_id"], + row["updated_at"], + row["status"], + row["status"], + row["pid"], + row["pid"], + ), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (cursor.lastrowid, row["bot_id"]), + ) + + invalid_projection = conn.execute( + """ + SELECT bots.bot_id + FROM bots + LEFT JOIN lifecycle_events + ON lifecycle_events.event_id = bots.last_event_id + AND lifecycle_events.bot_id = bots.bot_id + WHERE lifecycle_events.event_id IS NULL + LIMIT 1 + """ + ).fetchone() + if invalid_projection is not None: + raise sqlite3.IntegrityError("lifecycle snapshot invariant failed") + + def _migrate_v3_to_v4(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE idempotency_records ( + key_hash TEXT PRIMARY KEY, + request_hash TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('in_progress', 'completed')), + owner_instance_id TEXT NOT NULL, + response_status INTEGER NULL, + response_json TEXT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + expires_at TEXT NOT NULL + ) + """ + ) + conn.execute( + """ + CREATE INDEX idx_idempotency_records_expires + ON idempotency_records (expires_at) + """ + ) + + def _migrate_v4_to_v5(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} + migrations = { + "desired_state": ( + "ALTER TABLE bots ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'stopped' " + "CHECK (desired_state IN ('running', 'stopped'))" + ), + "desired_revision": ( + "ALTER TABLE bots ADD COLUMN desired_revision INTEGER NOT NULL DEFAULT 0 " + "CHECK (desired_revision >= 0)" + ), + "desired_updated_at": "ALTER TABLE bots ADD COLUMN desired_updated_at TEXT", + "pending_operation_id": "ALTER TABLE bots ADD COLUMN pending_operation_id TEXT", + "pending_action": ( + "ALTER TABLE bots ADD COLUMN pending_action TEXT CHECK " + "(pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart'))" + ), + "pending_since": "ALTER TABLE bots ADD COLUMN pending_since TEXT", + } + for column, statement in migrations.items(): + if column not in columns: + conn.execute(statement) + + conn.execute( + """ + CREATE TRIGGER bots_desired_intent_reject_partial_insert + BEFORE INSERT ON bots + WHEN NOT ( + (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL + AND NEW.pending_since IS NULL) + OR + (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL + AND NEW.pending_since IS NOT NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); + END + """ + ) + conn.execute( + """ + CREATE TRIGGER bots_desired_intent_reject_partial_update + BEFORE UPDATE ON bots + WHEN NOT ( + (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL + AND NEW.pending_since IS NULL) + OR + (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL + AND NEW.pending_since IS NOT NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); + END + """ + ) + + conn.execute( + """ + UPDATE bots + SET desired_state = CASE + WHEN status IN ('running', 'starting') THEN 'running' + WHEN status IN ('failed', 'unknown') AND restart_policy = 'on-failure' + THEN 'running' + ELSE 'stopped' + END, + desired_revision = 0, + desired_updated_at = updated_at, + pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL + """ + ) + + rows = conn.execute( + """ + SELECT bot_id, status, pid, desired_state, desired_revision, desired_updated_at + FROM bots ORDER BY bot_id + """ + ).fetchall() + for row in rows: + details = serialize_lifecycle_details( + { + "desired_state": str(row["desired_state"]), + "desired_revision": int(row["desired_revision"]), + } + ) + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, operation_id, occurred_at, source, action, outcome, + status_before, status_after, pid_before, pid_after, reason, details_json + ) VALUES (?, 'migration-v5', ?, 'migration', + 'migration.desired_state_snapshot', 'success', ?, ?, ?, ?, + 'schema v5 desired-state snapshot', ?) + """, + ( + row["bot_id"], + row["desired_updated_at"], + row["status"], + row["status"], + row["pid"], + row["pid"], + details, + ), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (cursor.lastrowid, row["bot_id"]), + ) + + invalid_projection = conn.execute( + """ + SELECT bots.bot_id + FROM bots + LEFT JOIN lifecycle_events + ON lifecycle_events.event_id = bots.last_event_id + AND lifecycle_events.bot_id = bots.bot_id + WHERE lifecycle_events.event_id IS NULL + LIMIT 1 + """ + ).fetchone() + if invalid_projection is not None: + raise sqlite3.IntegrityError("desired-state snapshot invariant failed") + + def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE reconcile_runs ( + run_id TEXT PRIMARY KEY, + scope TEXT NOT NULL CHECK (scope IN ('fleet', 'bot')), + requested_bot_id TEXT, + source TEXT NOT NULL, + force INTEGER NOT NULL CHECK (force IN (0, 1)), + reset_restart INTEGER NOT NULL CHECK (reset_restart IN (0, 1)), + started_at TEXT NOT NULL, + finished_at TEXT, + outcome TEXT NOT NULL CHECK ( + outcome IN ('running', 'succeeded', 'completed_with_errors', 'interrupted') + ), + total INTEGER NOT NULL DEFAULT 0 CHECK (total >= 0), + healthy_count INTEGER NOT NULL DEFAULT 0 CHECK (healthy_count >= 0), + changed_count INTEGER NOT NULL DEFAULT 0 CHECK (changed_count >= 0), + pending_count INTEGER NOT NULL DEFAULT 0 CHECK (pending_count >= 0), + action_required_count INTEGER NOT NULL DEFAULT 0 + CHECK (action_required_count >= 0), + error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), + skipped_count INTEGER NOT NULL DEFAULT 0 CHECK (skipped_count >= 0), + CHECK ( + total = healthy_count + changed_count + pending_count + + action_required_count + error_count + skipped_count + ), + CHECK ( + (scope = 'fleet' AND requested_bot_id IS NULL) + OR (scope = 'bot' AND requested_bot_id IS NOT NULL) + ), + CHECK ( + (outcome = 'running' AND finished_at IS NULL) + OR (outcome != 'running' AND finished_at IS NOT NULL) + ), + CHECK (finished_at IS NULL OR finished_at >= started_at) + ) + """ + ) + conn.execute( + """ + CREATE TABLE reconcile_results ( + run_id TEXT NOT NULL, + bot_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + outcome TEXT NOT NULL CHECK ( + outcome IN ( + 'healthy', 'changed', 'pending', + 'action_required', 'error', 'skipped' + ) + ), + desired_state TEXT CHECK ( + desired_state IS NULL OR desired_state IN ('running', 'stopped') + ), + observed_status TEXT CHECK ( + observed_status IS NULL + OR observed_status IN ('stopped', 'starting', 'running', 'failed', 'unknown') + ), + pid INTEGER CHECK (pid IS NULL OR pid > 0), + action TEXT NOT NULL CHECK (length(action) <= 2048), + message TEXT NOT NULL CHECK (length(message) <= 2048), + error_code TEXT CHECK (error_code IS NULL OR length(error_code) <= 2048), + event_id INTEGER CHECK (event_id IS NULL OR event_id > 0), + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + PRIMARY KEY (run_id, bot_id), + UNIQUE (run_id, ordinal), + CHECK (finished_at >= started_at), + FOREIGN KEY (run_id) REFERENCES reconcile_runs(run_id) ON DELETE CASCADE + ) + """ + ) diff --git a/zeus/sqlite_db.py b/zeus/sqlite_db.py new file mode 100644 index 0000000..4710d8f --- /dev/null +++ b/zeus/sqlite_db.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterator +from contextlib import closing, contextmanager +from pathlib import Path + +from zeus.schema import _assert_schema_compatible, _preflight_schema_compatibility + + +class SQLiteDatabase: + def __init__(self, database_path: Path | str) -> None: + self.database_path = Path(database_path) + + def connect(self) -> sqlite3.Connection: + self.database_path.parent.mkdir(parents=True, exist_ok=True) + _preflight_schema_compatibility(self.database_path) + conn = sqlite3.connect(self.database_path) + conn.row_factory = sqlite3.Row + try: + _assert_schema_compatible(conn) + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA busy_timeout=5000") + return conn + except Exception: + conn.close() + raise + + @contextmanager + def immediate(self) -> Iterator[sqlite3.Connection]: + with closing(self.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + yield conn + except Exception: + conn.rollback() + raise + else: + conn.commit() diff --git a/zeus/state.py b/zeus/state.py index 313a010..9bf2853 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -26,8 +26,10 @@ ReconcileRunSummary, ) from zeus.sanitization import MAX_SANITIZED_JSON_BYTES, sanitize_details, sanitize_text +from zeus.schema import SCHEMA_VERSION as SCHEMA_VERSION +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase -SCHEMA_VERSION = 6 LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") @@ -243,518 +245,21 @@ def _idempotency_claim_from_row( class StateStore: def __init__(self, database_path: Path | str) -> None: - self.database_path = Path(database_path) + self._database = SQLiteDatabase(database_path) + self._schema = SchemaManager(self._database) - def connect(self) -> sqlite3.Connection: - self.database_path.parent.mkdir(parents=True, exist_ok=True) - self._preflight_schema_compatibility() - conn = sqlite3.connect(self.database_path) - conn.row_factory = sqlite3.Row - try: - self._assert_schema_compatible(conn) - conn.execute("PRAGMA foreign_keys=ON") - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") - conn.execute("PRAGMA busy_timeout=5000") - return conn - except Exception: - conn.close() - raise - - def _preflight_schema_compatibility(self) -> None: - if not self.database_path.exists(): - return - uri = f"{self.database_path.resolve().as_uri()}?mode=ro" - with closing(sqlite3.connect(uri, uri=True)) as conn: - conn.row_factory = sqlite3.Row - self._assert_schema_compatible(conn) + @property + def database_path(self) -> Path: + return self._database.database_path - def _assert_schema_compatible(self, conn: sqlite3.Connection) -> None: - table = conn.execute( - """ - SELECT 1 FROM sqlite_master - WHERE type = 'table' AND name = 'schema_version' - """ - ).fetchone() - if table is None: - return - row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - if row is not None and int(row["version"]) > SCHEMA_VERSION: - raise RuntimeError( - f"database schema version {int(row['version'])} is newer than supported " - f"version {SCHEMA_VERSION}" - ) + def connect(self) -> sqlite3.Connection: + return self._database.connect() def init(self) -> None: - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - self._create_bots_table(conn) - self._migrate(conn) - except Exception: - conn.rollback() - raise - else: - conn.commit() + self._schema.init() def migrate(self) -> None: - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - self._migrate(conn) - except Exception: - conn.rollback() - raise - else: - conn.commit() - - def _create_bots_table(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS bots ( - bot_id TEXT PRIMARY KEY, - template_id TEXT NOT NULL, - display_name TEXT NOT NULL, - profile_path TEXT NOT NULL, - status TEXT NOT NULL, - pid INTEGER, - restart_policy TEXT NOT NULL DEFAULT 'manual', - restart_backoff_seconds REAL NOT NULL DEFAULT 5.0, - restart_max_attempts INTEGER NOT NULL DEFAULT 5, - restart_attempts INTEGER NOT NULL DEFAULT 0, - next_restart_at TEXT, - started_at TEXT, - ready_at TEXT, - stopped_at TEXT, - last_exit_code INTEGER, - last_error TEXT, - last_transition_reason TEXT, - last_event_id INTEGER, - desired_state TEXT NOT NULL DEFAULT 'stopped' - CHECK (desired_state IN ('running', 'stopped')), - desired_revision INTEGER NOT NULL DEFAULT 0 CHECK (desired_revision >= 0), - desired_updated_at TEXT, - pending_operation_id TEXT, - pending_action TEXT CHECK ( - pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart') - ), - pending_since TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - CHECK ( - (pending_operation_id IS NULL - AND pending_action IS NULL - AND pending_since IS NULL) - OR - (pending_operation_id IS NOT NULL AND pending_action IS NOT NULL - AND pending_since IS NOT NULL) - ) - ) - """ - ) - - def _migrate(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER NOT NULL - ) - """ - ) - row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - current_version = int(row["version"]) if row else 0 - if row is None: - conn.execute("INSERT INTO schema_version (version) VALUES (0)") - if current_version > SCHEMA_VERSION: - raise RuntimeError( - f"database schema version {current_version} is newer than supported " - f"version {SCHEMA_VERSION}" - ) - if current_version < 1: - self._ensure_restart_schema(conn) - conn.execute("UPDATE schema_version SET version = ?", (1,)) - current_version = 1 - if current_version < 2: - self._ensure_lifecycle_schema(conn) - conn.execute("UPDATE schema_version SET version = ?", (2,)) - current_version = 2 - if current_version < 3: - self._migrate_v2_to_v3(conn) - conn.execute("UPDATE schema_version SET version = ?", (3,)) - current_version = 3 - if current_version < 4: - self._migrate_v3_to_v4(conn) - conn.execute("UPDATE schema_version SET version = ?", (4,)) - current_version = 4 - if current_version < 5: - self._migrate_v4_to_v5(conn) - conn.execute("UPDATE schema_version SET version = ?", (5,)) - current_version = 5 - if current_version < 6: - self._migrate_v5_to_v6(conn) - conn.execute("UPDATE schema_version SET version = ?", (6,)) - - def _ensure_restart_schema(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "restart_policy": ( - "ALTER TABLE bots ADD COLUMN restart_policy TEXT NOT NULL DEFAULT 'manual'" - ), - "restart_backoff_seconds": ( - "ALTER TABLE bots ADD COLUMN restart_backoff_seconds REAL NOT NULL DEFAULT 5.0" - ), - "restart_max_attempts": ( - "ALTER TABLE bots ADD COLUMN restart_max_attempts INTEGER NOT NULL DEFAULT 5" - ), - "restart_attempts": ( - "ALTER TABLE bots ADD COLUMN restart_attempts INTEGER NOT NULL DEFAULT 0" - ), - "next_restart_at": "ALTER TABLE bots ADD COLUMN next_restart_at TEXT", - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - def _ensure_lifecycle_schema(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "started_at": "ALTER TABLE bots ADD COLUMN started_at TEXT", - "ready_at": "ALTER TABLE bots ADD COLUMN ready_at TEXT", - "stopped_at": "ALTER TABLE bots ADD COLUMN stopped_at TEXT", - "last_exit_code": "ALTER TABLE bots ADD COLUMN last_exit_code INTEGER", - "last_error": "ALTER TABLE bots ADD COLUMN last_error TEXT", - "last_transition_reason": ("ALTER TABLE bots ADD COLUMN last_transition_reason TEXT"), - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - def _migrate_v2_to_v3(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - if "last_event_id" not in columns: - conn.execute("ALTER TABLE bots ADD COLUMN last_event_id INTEGER") - conn.execute( - """ - CREATE TABLE lifecycle_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - bot_id TEXT NOT NULL, - operation_id TEXT NOT NULL, - request_id TEXT, - occurred_at TEXT NOT NULL, - source TEXT NOT NULL, - action TEXT NOT NULL, - outcome TEXT NOT NULL, - status_before TEXT, - status_after TEXT, - pid_before INTEGER, - pid_after INTEGER, - reason TEXT NOT NULL DEFAULT '', - error_code TEXT, - error_message TEXT, - details_json TEXT NOT NULL DEFAULT '{}' - ) - """ - ) - conn.execute( - """ - CREATE INDEX idx_lifecycle_events_bot - ON lifecycle_events (bot_id, event_id DESC) - """ - ) - conn.execute( - """ - CREATE INDEX idx_lifecycle_events_operation - ON lifecycle_events (operation_id, event_id) - """ - ) - conn.execute( - """ - CREATE TRIGGER lifecycle_events_reject_update - BEFORE UPDATE ON lifecycle_events - BEGIN - SELECT RAISE(ABORT, 'lifecycle events are immutable'); - END - """ - ) - conn.execute( - """ - CREATE TRIGGER lifecycle_events_reject_delete - BEFORE DELETE ON lifecycle_events - BEGIN - SELECT RAISE(ABORT, 'lifecycle events are immutable'); - END - """ - ) - - rows = conn.execute( - "SELECT bot_id, status, pid, updated_at FROM bots ORDER BY bot_id" - ).fetchall() - for row in rows: - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, - operation_id, - occurred_at, - source, - action, - outcome, - status_before, - status_after, - pid_before, - pid_after, - reason, - details_json - ) VALUES (?, 'migration-v3', ?, 'migration', 'migration.snapshot', - 'success', ?, ?, ?, ?, 'schema v3 snapshot', '{}') - """, - ( - row["bot_id"], - row["updated_at"], - row["status"], - row["status"], - row["pid"], - row["pid"], - ), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (cursor.lastrowid, row["bot_id"]), - ) - - invalid_projection = conn.execute( - """ - SELECT bots.bot_id - FROM bots - LEFT JOIN lifecycle_events - ON lifecycle_events.event_id = bots.last_event_id - AND lifecycle_events.bot_id = bots.bot_id - WHERE lifecycle_events.event_id IS NULL - LIMIT 1 - """ - ).fetchone() - if invalid_projection is not None: - raise sqlite3.IntegrityError("lifecycle snapshot invariant failed") - - def _migrate_v3_to_v4(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE idempotency_records ( - key_hash TEXT PRIMARY KEY, - request_hash TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('in_progress', 'completed')), - owner_instance_id TEXT NOT NULL, - response_status INTEGER NULL, - response_json TEXT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - expires_at TEXT NOT NULL - ) - """ - ) - conn.execute( - """ - CREATE INDEX idx_idempotency_records_expires - ON idempotency_records (expires_at) - """ - ) - - def _migrate_v4_to_v5(self, conn: sqlite3.Connection) -> None: - columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} - migrations = { - "desired_state": ( - "ALTER TABLE bots ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'stopped' " - "CHECK (desired_state IN ('running', 'stopped'))" - ), - "desired_revision": ( - "ALTER TABLE bots ADD COLUMN desired_revision INTEGER NOT NULL DEFAULT 0 " - "CHECK (desired_revision >= 0)" - ), - "desired_updated_at": "ALTER TABLE bots ADD COLUMN desired_updated_at TEXT", - "pending_operation_id": "ALTER TABLE bots ADD COLUMN pending_operation_id TEXT", - "pending_action": ( - "ALTER TABLE bots ADD COLUMN pending_action TEXT CHECK " - "(pending_action IS NULL OR pending_action IN ('start', 'stop', 'restart'))" - ), - "pending_since": "ALTER TABLE bots ADD COLUMN pending_since TEXT", - } - for column, statement in migrations.items(): - if column not in columns: - conn.execute(statement) - - conn.execute( - """ - CREATE TRIGGER bots_desired_intent_reject_partial_insert - BEFORE INSERT ON bots - WHEN NOT ( - (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL - AND NEW.pending_since IS NULL) - OR - (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL - AND NEW.pending_since IS NOT NULL) - ) - BEGIN - SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); - END - """ - ) - conn.execute( - """ - CREATE TRIGGER bots_desired_intent_reject_partial_update - BEFORE UPDATE ON bots - WHEN NOT ( - (NEW.pending_operation_id IS NULL AND NEW.pending_action IS NULL - AND NEW.pending_since IS NULL) - OR - (NEW.pending_operation_id IS NOT NULL AND NEW.pending_action IS NOT NULL - AND NEW.pending_since IS NOT NULL) - ) - BEGIN - SELECT RAISE(ABORT, 'pending lifecycle intent must be all null or all populated'); - END - """ - ) - - conn.execute( - """ - UPDATE bots - SET desired_state = CASE - WHEN status IN ('running', 'starting') THEN 'running' - WHEN status IN ('failed', 'unknown') AND restart_policy = 'on-failure' - THEN 'running' - ELSE 'stopped' - END, - desired_revision = 0, - desired_updated_at = updated_at, - pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL - """ - ) - - rows = conn.execute( - """ - SELECT bot_id, status, pid, desired_state, desired_revision, desired_updated_at - FROM bots ORDER BY bot_id - """ - ).fetchall() - for row in rows: - details = serialize_lifecycle_details( - { - "desired_state": str(row["desired_state"]), - "desired_revision": int(row["desired_revision"]), - } - ) - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, operation_id, occurred_at, source, action, outcome, - status_before, status_after, pid_before, pid_after, reason, details_json - ) VALUES (?, 'migration-v5', ?, 'migration', - 'migration.desired_state_snapshot', 'success', ?, ?, ?, ?, - 'schema v5 desired-state snapshot', ?) - """, - ( - row["bot_id"], - row["desired_updated_at"], - row["status"], - row["status"], - row["pid"], - row["pid"], - details, - ), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (cursor.lastrowid, row["bot_id"]), - ) - - invalid_projection = conn.execute( - """ - SELECT bots.bot_id - FROM bots - LEFT JOIN lifecycle_events - ON lifecycle_events.event_id = bots.last_event_id - AND lifecycle_events.bot_id = bots.bot_id - WHERE lifecycle_events.event_id IS NULL - LIMIT 1 - """ - ).fetchone() - if invalid_projection is not None: - raise sqlite3.IntegrityError("desired-state snapshot invariant failed") - - def _migrate_v5_to_v6(self, conn: sqlite3.Connection) -> None: - conn.execute( - """ - CREATE TABLE reconcile_runs ( - run_id TEXT PRIMARY KEY, - scope TEXT NOT NULL CHECK (scope IN ('fleet', 'bot')), - requested_bot_id TEXT, - source TEXT NOT NULL, - force INTEGER NOT NULL CHECK (force IN (0, 1)), - reset_restart INTEGER NOT NULL CHECK (reset_restart IN (0, 1)), - started_at TEXT NOT NULL, - finished_at TEXT, - outcome TEXT NOT NULL CHECK ( - outcome IN ('running', 'succeeded', 'completed_with_errors', 'interrupted') - ), - total INTEGER NOT NULL DEFAULT 0 CHECK (total >= 0), - healthy_count INTEGER NOT NULL DEFAULT 0 CHECK (healthy_count >= 0), - changed_count INTEGER NOT NULL DEFAULT 0 CHECK (changed_count >= 0), - pending_count INTEGER NOT NULL DEFAULT 0 CHECK (pending_count >= 0), - action_required_count INTEGER NOT NULL DEFAULT 0 - CHECK (action_required_count >= 0), - error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), - skipped_count INTEGER NOT NULL DEFAULT 0 CHECK (skipped_count >= 0), - CHECK ( - total = healthy_count + changed_count + pending_count - + action_required_count + error_count + skipped_count - ), - CHECK ( - (scope = 'fleet' AND requested_bot_id IS NULL) - OR (scope = 'bot' AND requested_bot_id IS NOT NULL) - ), - CHECK ( - (outcome = 'running' AND finished_at IS NULL) - OR (outcome != 'running' AND finished_at IS NOT NULL) - ), - CHECK (finished_at IS NULL OR finished_at >= started_at) - ) - """ - ) - conn.execute( - """ - CREATE TABLE reconcile_results ( - run_id TEXT NOT NULL, - bot_id TEXT NOT NULL, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - outcome TEXT NOT NULL CHECK ( - outcome IN ( - 'healthy', 'changed', 'pending', - 'action_required', 'error', 'skipped' - ) - ), - desired_state TEXT CHECK ( - desired_state IS NULL OR desired_state IN ('running', 'stopped') - ), - observed_status TEXT CHECK ( - observed_status IS NULL - OR observed_status IN ('stopped', 'starting', 'running', 'failed', 'unknown') - ), - pid INTEGER CHECK (pid IS NULL OR pid > 0), - action TEXT NOT NULL CHECK (length(action) <= 2048), - message TEXT NOT NULL CHECK (length(message) <= 2048), - error_code TEXT CHECK (error_code IS NULL OR length(error_code) <= 2048), - event_id INTEGER CHECK (event_id IS NULL OR event_id > 0), - started_at TEXT NOT NULL, - finished_at TEXT NOT NULL, - PRIMARY KEY (run_id, bot_id), - UNIQUE (run_id, ordinal), - CHECK (finished_at >= started_at), - FOREIGN KEY (run_id) REFERENCES reconcile_runs(run_id) ON DELETE CASCADE - ) - """ - ) + self._schema.migrate() def begin_reconcile_run(self, run: ReconcileRunStart) -> None: if not isinstance(run, ReconcileRunStart): From e3df2b3ba0d957f0f30230dc3b916c9f7991b3e3 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:21:12 +0200 Subject: [PATCH 32/49] refactor(state): isolate idempotency persistence --- tests/test_idempotency_store.py | 481 ++++++++++++++++++++++++++++++++ zeus/idempotency_store.py | 360 ++++++++++++++++++++++++ zeus/state.py | 400 ++++---------------------- 3 files changed, 897 insertions(+), 344 deletions(-) create mode 100644 tests/test_idempotency_store.py create mode 100644 zeus/idempotency_store.py diff --git a/tests/test_idempotency_store.py b/tests/test_idempotency_store.py new file mode 100644 index 0000000..29e5cc7 --- /dev/null +++ b/tests/test_idempotency_store.py @@ -0,0 +1,481 @@ +from __future__ import annotations + +import inspect +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, call, patch + +from zeus import api as api_module +from zeus.idempotency import IdempotencyClaim +from zeus.idempotency_store import ( + IDEMPOTENCY_HASH_RE, + IDEMPOTENCY_OWNER_RE, + MAX_IDEMPOTENCY_RESPONSE_BYTES, + IdempotencyStore, +) +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import ( + IDEMPOTENCY_HASH_RE as STATE_IDEMPOTENCY_HASH_RE, +) +from zeus.state import ( + IDEMPOTENCY_OWNER_RE as STATE_IDEMPOTENCY_OWNER_RE, +) +from zeus.state import ( + MAX_IDEMPOTENCY_RESPONSE_BYTES as STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, +) +from zeus.state import StateStore + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +class _UnavailableSQLiteDatabase(SQLiteDatabase): + def connect(self) -> sqlite3.Connection: + raise sqlite3.OperationalError("database unavailable") + + +def _row_bytes(database_path: Path, key_hash: str) -> tuple[object, ...] | None: + columns = ( + "key_hash", + "request_hash", + "state", + "owner_instance_id", + "response_status", + "response_json", + "created_at", + "updated_at", + "expires_at", + ) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + with closing(sqlite3.connect(database_path)) as conn: + return conn.execute( + f"SELECT {fields} FROM idempotency_records WHERE key_hash = ?", + (key_hash,), + ).fetchone() + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _identity, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statement_index(statements: list[str], prefix: str) -> int: + return next( + index + for index, statement in enumerate(statements) + if statement.lstrip().upper().startswith(prefix) + ) + + +class IdempotencyStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.database_path = Path(self.temporary_directory.name) / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = IdempotencyStore(self.database) + self.key_hash = "a" * 64 + self.request_hash = "b" * 64 + self.future = datetime.now(UTC) + timedelta(hours=1) + + def claim( + self, + *, + key_hash: str | None = None, + request_hash: str | None = None, + owner_instance_id: str = "owner-a", + max_records: int = 10_000, + ) -> IdempotencyClaim: + return self.store.claim_idempotency( + key_hash=key_hash or self.key_hash, + request_hash=request_hash or self.request_hash, + owner_instance_id=owner_instance_id, + expires_at=self.future, + max_records=max_records, + ) + + def complete(self, *, key_hash: str | None = None) -> None: + self.store.complete_idempotency( + key_hash=key_hash or self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=202, + response_json='{"ok":true}', + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + + def test_constructs_without_opening_the_database(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + IdempotencyStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + def test_direct_store_preserves_every_claim_outcome(self) -> None: + self.assertEqual("claimed", self.claim().kind) + self.assertEqual("in_progress", self.claim().kind) + self.assertEqual("conflict", self.claim(request_hash="c" * 64).kind) + self.assertEqual( + "indeterminate", + self.claim(owner_instance_id="owner-b").kind, + ) + + self.complete() + replay = self.claim() + self.assertEqual("replay", replay.kind) + self.assertEqual(202, replay.response_status) + self.assertEqual('{"ok":true}', replay.response_json) + + unavailable = IdempotencyStore( + _UnavailableSQLiteDatabase(self.database_path.with_name("unavailable.db")) + ) + unavailable_claim = unavailable.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + unavailable_lookup = unavailable.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + self.assertEqual("unavailable", unavailable_claim.kind) + self.assertIsNotNone(unavailable_lookup) + self.assertEqual("unavailable", unavailable_lookup.kind if unavailable_lookup else None) + + def test_validation_precedes_connection_and_completion_connection_errors_propagate( + self, + ) -> None: + database = MagicMock(spec=SQLiteDatabase) + store = IdempotencyStore(database) + + with self.assertRaisesRegex(ValueError, "key hash is invalid"): + store.claim_idempotency( + key_hash="invalid", + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + database.connect.assert_not_called() + + unavailable = IdempotencyStore( + _UnavailableSQLiteDatabase(self.database_path.with_name("unavailable.db")) + ) + with self.assertRaisesRegex(sqlite3.OperationalError, "database unavailable"): + unavailable.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + + def test_claim_complete_and_lookup_keep_exact_transaction_boundaries(self) -> None: + tracing_database = _TracingSQLiteDatabase( + self.database_path.with_name("transaction-trace.db") + ) + SchemaManager(tracing_database).init() + tracing_database.traces.clear() + store = IdempotencyStore(tracing_database) + + claimed = store.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + ) + claim_trace = list(tracing_database.traces) + tracing_database.traces.clear() + + store.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=datetime.now(UTC), + expires_at=self.future, + ) + complete_trace = list(tracing_database.traces) + tracing_database.traces.clear() + + replay = store.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + lookup_trace = list(tracing_database.traces) + + claim_sql = [sql for _identity, sql in claim_trace] + complete_sql = [sql for _identity, sql in complete_trace] + lookup_sql = [sql for _identity, sql in lookup_trace] + self.assertEqual("claimed", claimed.kind) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(claim_trace)) + self.assertEqual(1, len({identity for identity, _sql in claim_trace})) + self.assertLess( + _statement_index(claim_sql, "SELECT *"), + _statement_index(claim_sql, "DELETE FROM"), + ) + self.assertLess( + _statement_index(claim_sql, "DELETE FROM"), + _statement_index(claim_sql, "SELECT COUNT"), + ) + self.assertLess( + _statement_index(claim_sql, "SELECT COUNT"), + _statement_index(claim_sql, "INSERT INTO"), + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(complete_trace)) + self.assertEqual(1, len({identity for identity, _sql in complete_trace})) + self.assertLess( + _statement_index(complete_sql, "SELECT *"), + _statement_index(complete_sql, "UPDATE"), + ) + + self.assertIsNotNone(replay) + self.assertEqual("replay", replay.kind if replay else None) + self.assertEqual([], _control_statements(lookup_trace)) + self.assertEqual(1, len({identity for identity, _sql in lookup_trace})) + self.assertTrue(lookup_sql) + self.assertTrue(all(sql.lstrip().upper().startswith("SELECT") for sql in lookup_sql)) + + def test_capacity_exhaustion_is_an_explicit_normal_commit(self) -> None: + tracing_database = _TracingSQLiteDatabase(self.database_path.with_name("capacity.db")) + SchemaManager(tracing_database).init() + store = IdempotencyStore(tracing_database) + store.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=1, + ) + tracing_database.traces.clear() + + result = store.claim_idempotency( + key_hash="c" * 64, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=1, + ) + trace = list(tracing_database.traces) + + self.assertEqual("unavailable", result.kind) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(trace)) + self.assertFalse(any(sql.lstrip().upper().startswith("INSERT") for _identity, sql in trace)) + + def test_corrupt_claim_rolls_back_without_changing_any_stored_bytes(self) -> None: + self.assertEqual("claimed", self.claim().kind) + with closing(sqlite3.connect(self.database_path)) as conn: + conn.execute( + "UPDATE idempotency_records SET request_hash = 'corrupt' WHERE key_hash = ?", + (self.key_hash,), + ) + conn.commit() + before = _row_bytes(self.database_path, self.key_hash) + + result = self.claim(owner_instance_id="owner-b") + + self.assertEqual("unavailable", result.kind) + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + with closing(sqlite3.connect(self.database_path)) as conn: + count = conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] + self.assertEqual(1, count) + + def test_failed_completion_rolls_back_without_changing_any_stored_bytes(self) -> None: + self.assertEqual("claimed", self.claim().kind) + with closing(sqlite3.connect(self.database_path)) as conn: + conn.execute( + """ + CREATE TRIGGER reject_test_completion + BEFORE UPDATE ON idempotency_records + WHEN NEW.state = 'completed' + BEGIN + SELECT RAISE(ABORT, 'injected completion failure'); + END + """ + ) + conn.commit() + before = _row_bytes(self.database_path, self.key_hash) + + with self.assertRaisesRegex(sqlite3.DatabaseError, "injected completion failure"): + self.complete() + + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + + def test_lookup_leaves_an_expired_record_byte_for_byte_unchanged(self) -> None: + self.assertEqual("claimed", self.claim().kind) + completed_at = datetime.now(UTC) + self.store.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=200, + response_json="{}", + completed_at=completed_at, + expires_at=completed_at, + ) + before = _row_bytes(self.database_path, self.key_hash) + + result = self.store.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + + self.assertIsNone(result) + self.assertEqual(before, _row_bytes(self.database_path, self.key_hash)) + + def test_state_facade_delegates_exact_keywords_and_propagates_results_and_errors(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + delegate = MagicMock(spec=IdempotencyStore) + claim_result = IdempotencyClaim("claimed") + delegate.claim_idempotency.return_value = claim_result + delegate.lookup_idempotency.return_value = None + completion_error = RuntimeError("completion sentinel") + delegate.complete_idempotency.side_effect = completion_error + completed_at = datetime(2026, 7, 22, 12, tzinfo=UTC) + expires_at = completed_at + timedelta(hours=1) + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=delegate) as store_type, + ): + facade = StateStore(database_path) + actual_claim = facade.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=expires_at, + max_records=321, + ) + actual_lookup = facade.lookup_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + with self.assertRaises(RuntimeError) as caught: + facade.complete_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=201, + response_json='{"created":true}', + completed_at=completed_at, + expires_at=expires_at, + ) + + self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], store_type.call_args_list) + database.connect.assert_not_called() + self.assertIs(claim_result, actual_claim) + self.assertIsNone(actual_lookup) + self.assertIs(completion_error, caught.exception) + delegate.claim_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=expires_at, + max_records=321, + ) + delegate.lookup_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + ) + delegate.complete_idempotency.assert_called_once_with( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + response_status=201, + response_json='{"created":true}', + completed_at=completed_at, + expires_at=expires_at, + ) + self.assertEqual( + inspect.signature(IdempotencyStore.claim_idempotency), + inspect.signature(StateStore.claim_idempotency), + ) + self.assertEqual( + inspect.signature(IdempotencyStore.lookup_idempotency), + inspect.signature(StateStore.lookup_idempotency), + ) + self.assertEqual( + inspect.signature(IdempotencyStore.complete_idempotency), + inspect.signature(StateStore.complete_idempotency), + ) + + def test_state_compatibility_aliases_and_class_monkeypatch_remain_supported(self) -> None: + self.assertIs(IDEMPOTENCY_HASH_RE, STATE_IDEMPOTENCY_HASH_RE) + self.assertIs(IDEMPOTENCY_OWNER_RE, STATE_IDEMPOTENCY_OWNER_RE) + self.assertIs( + MAX_IDEMPOTENCY_RESPONSE_BYTES, + STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, + ) + self.assertIs( + STATE_MAX_IDEMPOTENCY_RESPONSE_BYTES, + api_module.MAX_IDEMPOTENCY_RESPONSE_BYTES, + ) + facade = StateStore(self.database_path.with_name("monkeypatch.db")) + patched_result = IdempotencyClaim("unavailable") + + with patch.object( + StateStore, + "claim_idempotency", + autospec=True, + return_value=patched_result, + ) as patched_claim: + result = facade.claim_idempotency( + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=55, + ) + + self.assertIs(patched_result, result) + patched_claim.assert_called_once_with( + facade, + key_hash=self.key_hash, + request_hash=self.request_hash, + owner_instance_id="owner-a", + expires_at=self.future, + max_records=55, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/idempotency_store.py b/zeus/idempotency_store.py new file mode 100644 index 0000000..09396dc --- /dev/null +++ b/zeus/idempotency_store.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from dataclasses import dataclass +from datetime import UTC, datetime + +from zeus.idempotency import IdempotencyClaim +from zeus.sqlite_db import SQLiteDatabase + +IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") +IDEMPOTENCY_OWNER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$", re.ASCII) +MAX_IDEMPOTENCY_RESPONSE_BYTES = 1_000_000 + + +def _validate_idempotency_hash(value: str, label: str) -> str: + if not isinstance(value, str): + raise TypeError(f"idempotency {label} must be a string") + if IDEMPOTENCY_HASH_RE.fullmatch(value) is None: + raise ValueError(f"idempotency {label} is invalid") + return value + + +def _validate_idempotency_owner(value: str) -> str: + if not isinstance(value, str): + raise TypeError("idempotency owner must be a string") + if IDEMPOTENCY_OWNER_RE.fullmatch(value) is None: + raise ValueError("idempotency owner is invalid") + return value + + +def _validate_idempotency_timestamp(value: datetime, label: str) -> str: + if not isinstance(value, datetime): + raise TypeError(f"idempotency {label} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"idempotency {label} is invalid") + return value.astimezone(UTC).isoformat() + + +def _validate_idempotency_status(value: int) -> int: + if type(value) is not int or not 200 <= value <= 599: + raise ValueError("idempotency response status is invalid") + return value + + +def _reject_non_finite_json(_value: str) -> object: + raise ValueError("idempotency response JSON is invalid") + + +def _validate_idempotency_response_json(value: str) -> str: + if not isinstance(value, str): + raise TypeError("idempotency response JSON must be a string") + try: + encoded = value.encode("utf-8") + except UnicodeError: + raise ValueError("idempotency response JSON is invalid") from None + if len(encoded) > MAX_IDEMPOTENCY_RESPONSE_BYTES: + raise ValueError("idempotency response JSON is too large") + try: + json.loads(value, parse_constant=_reject_non_finite_json) + except (json.JSONDecodeError, TypeError, ValueError, RecursionError): + raise ValueError("idempotency response JSON is invalid") from None + return value + + +def _parse_idempotency_timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str): + raise TypeError(f"stored idempotency {label} must be a string") + try: + parsed = datetime.fromisoformat(value) + except ValueError: + raise ValueError(f"stored idempotency {label} is invalid") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError(f"stored idempotency {label} is invalid") + normalized = parsed.astimezone(UTC) + if normalized.isoformat() != value: + raise ValueError(f"stored idempotency {label} is invalid") + return normalized + + +@dataclass(frozen=True) +class _ValidatedIdempotencyRow: + request_hash: str + state: str + owner_instance_id: str + response_status: int | None + response_json: str | None + created_at: datetime + updated_at: datetime + expires_at: datetime + + +def _validate_idempotency_row(row: sqlite3.Row) -> _ValidatedIdempotencyRow: + _validate_idempotency_hash(row["key_hash"], "stored key hash") + request_hash = _validate_idempotency_hash(row["request_hash"], "stored request hash") + owner = _validate_idempotency_owner(row["owner_instance_id"]) + state = row["state"] + if state not in {"in_progress", "completed"}: + raise ValueError("stored idempotency state is invalid") + created_at = _parse_idempotency_timestamp(row["created_at"], "creation timestamp") + updated_at = _parse_idempotency_timestamp(row["updated_at"], "update timestamp") + expires_at = _parse_idempotency_timestamp(row["expires_at"], "expiry timestamp") + response_status = row["response_status"] + response_json = row["response_json"] + if state == "in_progress": + if created_at > updated_at or updated_at > expires_at: + raise ValueError("stored in-progress idempotency timestamps are invalid") + if response_status is not None or response_json is not None: + raise ValueError("stored in-progress idempotency response is invalid") + safe_status = None + safe_json = None + else: + if created_at > updated_at or updated_at > expires_at: + raise ValueError("stored completed idempotency timestamps are invalid") + if response_status is None or response_json is None: + raise ValueError("stored completed idempotency response is incomplete") + safe_status = _validate_idempotency_status(response_status) + safe_json = _validate_idempotency_response_json(response_json) + return _ValidatedIdempotencyRow( + request_hash=request_hash, + state=state, + owner_instance_id=owner, + response_status=safe_status, + response_json=safe_json, + created_at=created_at, + updated_at=updated_at, + expires_at=expires_at, + ) + + +def _idempotency_claim_from_row( + row: sqlite3.Row, + *, + request_hash: str, + owner_instance_id: str, +) -> IdempotencyClaim: + validated = _validate_idempotency_row(row) + if validated.request_hash != request_hash: + return IdempotencyClaim("conflict") + if validated.state == "completed": + if validated.response_status is None or validated.response_json is None: + raise AssertionError("validated completed idempotency response is incomplete") + return IdempotencyClaim( + "replay", + validated.response_status, + validated.response_json, + ) + if validated.owner_instance_id == owner_instance_id: + return IdempotencyClaim("in_progress") + return IdempotencyClaim("indeterminate") + + +class IdempotencyStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def claim_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + expires_at: datetime, + max_records: int = 10_000, + ) -> IdempotencyClaim: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") + if type(max_records) is not int or not 1 <= max_records <= 1_000_000: + raise ValueError("idempotency capacity is invalid") + current_time = datetime.now(UTC) + if datetime.fromisoformat(safe_expiry) < current_time: + raise ValueError("idempotency expiry timestamp is invalid") + now = current_time.isoformat() + + try: + conn = self._database.connect() + except (OSError, sqlite3.Error): + return IdempotencyClaim("unavailable") + with closing(conn): + try: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is not None: + validated = _validate_idempotency_row(row) + reclaimable = validated.expires_at <= current_time and ( + validated.state == "completed" or validated.owner_instance_id != safe_owner + ) + if reclaimable: + conn.execute( + "DELETE FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ) + row = None + conn.execute( + """ + DELETE FROM idempotency_records + WHERE expires_at <= ? + AND (state = 'completed' OR owner_instance_id != ?) + AND key_hash != ? + """, + (now, safe_owner, safe_key_hash), + ) + if row is None: + count = int( + conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] + ) + if count >= max_records: + conn.commit() + return IdempotencyClaim("unavailable") + conn.execute( + """ + INSERT INTO idempotency_records ( + key_hash, + request_hash, + state, + owner_instance_id, + response_status, + response_json, + created_at, + updated_at, + expires_at + ) VALUES (?, ?, 'in_progress', ?, NULL, NULL, ?, ?, ?) + """, + ( + safe_key_hash, + safe_request_hash, + safe_owner, + now, + now, + safe_expiry, + ), + ) + conn.commit() + return IdempotencyClaim("claimed") + + result = _idempotency_claim_from_row( + row, + request_hash=safe_request_hash, + owner_instance_id=safe_owner, + ) + conn.commit() + return result + except (OSError, sqlite3.Error, ValueError, TypeError): + conn.rollback() + return IdempotencyClaim("unavailable") + + def lookup_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + ) -> IdempotencyClaim | None: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + current_time = datetime.now(UTC) + + try: + conn = self._database.connect() + except (OSError, sqlite3.Error): + return IdempotencyClaim("unavailable") + with closing(conn): + try: + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is None: + return None + validated = _validate_idempotency_row(row) + if validated.expires_at <= current_time and ( + validated.state == "completed" or validated.owner_instance_id != safe_owner + ): + return None + return _idempotency_claim_from_row( + row, + request_hash=safe_request_hash, + owner_instance_id=safe_owner, + ) + except (OSError, sqlite3.Error, ValueError, TypeError): + return IdempotencyClaim("unavailable") + + def complete_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + response_status: int, + response_json: str, + completed_at: datetime, + expires_at: datetime, + ) -> None: + safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") + safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") + safe_owner = _validate_idempotency_owner(owner_instance_id) + safe_status = _validate_idempotency_status(response_status) + safe_json = _validate_idempotency_response_json(response_json) + safe_completed = _validate_idempotency_timestamp(completed_at, "completion timestamp") + safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") + if safe_expiry < safe_completed: + raise ValueError("idempotency expiry timestamp is invalid") + completed_time = datetime.fromisoformat(safe_completed) + + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT * FROM idempotency_records WHERE key_hash = ?", + (safe_key_hash,), + ).fetchone() + if row is None: + raise RuntimeError("idempotency completion failed") + validated = _validate_idempotency_row(row) + if ( + validated.request_hash != safe_request_hash + or validated.owner_instance_id != safe_owner + or validated.state != "in_progress" + ): + raise RuntimeError("idempotency completion failed") + if completed_time < validated.created_at: + raise ValueError("idempotency completion timestamp is invalid") + cursor = conn.execute( + """ + UPDATE idempotency_records + SET state = 'completed', + response_status = ?, + response_json = ?, + updated_at = ?, + expires_at = ? + WHERE key_hash = ? + AND request_hash = ? + AND owner_instance_id = ? + AND state = 'in_progress' + """, + ( + safe_status, + safe_json, + safe_completed, + safe_expiry, + safe_key_hash, + safe_request_hash, + safe_owner, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("idempotency completion failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() diff --git a/zeus/state.py b/zeus/state.py index 9bf2853..0e0a49e 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -4,12 +4,15 @@ import re import sqlite3 from contextlib import closing -from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Literal from zeus.idempotency import IdempotencyClaim +from zeus.idempotency_store import IDEMPOTENCY_HASH_RE as IDEMPOTENCY_HASH_RE +from zeus.idempotency_store import IDEMPOTENCY_OWNER_RE as IDEMPOTENCY_OWNER_RE +from zeus.idempotency_store import MAX_IDEMPOTENCY_RESPONSE_BYTES as MAX_IDEMPOTENCY_RESPONSE_BYTES +from zeus.idempotency_store import IdempotencyStore from zeus.lifecycle import ( LifecycleEvent, LifecycleEventInput, @@ -32,9 +35,6 @@ LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") -IDEMPOTENCY_HASH_RE = re.compile(r"^[0-9a-f]{64}$") -IDEMPOTENCY_OWNER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$", re.ASCII) -MAX_IDEMPOTENCY_RESPONSE_BYTES = 1_000_000 LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) RECONCILE_COUNTER_COLUMNS = { @@ -53,71 +53,6 @@ def _sanitize_optional_persisted_text(value: str | None) -> str | None: return sanitize_text(value) -def _validate_idempotency_hash(value: str, label: str) -> str: - if not isinstance(value, str): - raise TypeError(f"idempotency {label} must be a string") - if IDEMPOTENCY_HASH_RE.fullmatch(value) is None: - raise ValueError(f"idempotency {label} is invalid") - return value - - -def _validate_idempotency_owner(value: str) -> str: - if not isinstance(value, str): - raise TypeError("idempotency owner must be a string") - if IDEMPOTENCY_OWNER_RE.fullmatch(value) is None: - raise ValueError("idempotency owner is invalid") - return value - - -def _validate_idempotency_timestamp(value: datetime, label: str) -> str: - if not isinstance(value, datetime): - raise TypeError(f"idempotency {label} must be a datetime") - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"idempotency {label} is invalid") - return value.astimezone(UTC).isoformat() - - -def _validate_idempotency_status(value: int) -> int: - if type(value) is not int or not 200 <= value <= 599: - raise ValueError("idempotency response status is invalid") - return value - - -def _reject_non_finite_json(_value: str) -> object: - raise ValueError("idempotency response JSON is invalid") - - -def _validate_idempotency_response_json(value: str) -> str: - if not isinstance(value, str): - raise TypeError("idempotency response JSON must be a string") - try: - encoded = value.encode("utf-8") - except UnicodeError: - raise ValueError("idempotency response JSON is invalid") from None - if len(encoded) > MAX_IDEMPOTENCY_RESPONSE_BYTES: - raise ValueError("idempotency response JSON is too large") - try: - json.loads(value, parse_constant=_reject_non_finite_json) - except (json.JSONDecodeError, TypeError, ValueError, RecursionError): - raise ValueError("idempotency response JSON is invalid") from None - return value - - -def _parse_idempotency_timestamp(value: object, label: str) -> datetime: - if not isinstance(value, str): - raise TypeError(f"stored idempotency {label} must be a string") - try: - parsed = datetime.fromisoformat(value) - except ValueError: - raise ValueError(f"stored idempotency {label} is invalid") from None - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise ValueError(f"stored idempotency {label} is invalid") - normalized = parsed.astimezone(UTC) - if normalized.isoformat() != value: - raise ValueError(f"stored idempotency {label} is invalid") - return normalized - - def _validate_reconcile_run_id(value: str) -> str: if not isinstance(value, str): raise TypeError("reconciliation run_id must be a string") @@ -171,82 +106,11 @@ def _validate_stored_boolean_flag(value: object, label: str) -> bool: return value == 1 -@dataclass(frozen=True) -class _ValidatedIdempotencyRow: - request_hash: str - state: str - owner_instance_id: str - response_status: int | None - response_json: str | None - created_at: datetime - updated_at: datetime - expires_at: datetime - - -def _validate_idempotency_row(row: sqlite3.Row) -> _ValidatedIdempotencyRow: - _validate_idempotency_hash(row["key_hash"], "stored key hash") - request_hash = _validate_idempotency_hash(row["request_hash"], "stored request hash") - owner = _validate_idempotency_owner(row["owner_instance_id"]) - state = row["state"] - if state not in {"in_progress", "completed"}: - raise ValueError("stored idempotency state is invalid") - created_at = _parse_idempotency_timestamp(row["created_at"], "creation timestamp") - updated_at = _parse_idempotency_timestamp(row["updated_at"], "update timestamp") - expires_at = _parse_idempotency_timestamp(row["expires_at"], "expiry timestamp") - response_status = row["response_status"] - response_json = row["response_json"] - if state == "in_progress": - if created_at > updated_at or updated_at > expires_at: - raise ValueError("stored in-progress idempotency timestamps are invalid") - if response_status is not None or response_json is not None: - raise ValueError("stored in-progress idempotency response is invalid") - safe_status = None - safe_json = None - else: - if created_at > updated_at or updated_at > expires_at: - raise ValueError("stored completed idempotency timestamps are invalid") - if response_status is None or response_json is None: - raise ValueError("stored completed idempotency response is incomplete") - safe_status = _validate_idempotency_status(response_status) - safe_json = _validate_idempotency_response_json(response_json) - return _ValidatedIdempotencyRow( - request_hash=request_hash, - state=state, - owner_instance_id=owner, - response_status=safe_status, - response_json=safe_json, - created_at=created_at, - updated_at=updated_at, - expires_at=expires_at, - ) - - -def _idempotency_claim_from_row( - row: sqlite3.Row, - *, - request_hash: str, - owner_instance_id: str, -) -> IdempotencyClaim: - validated = _validate_idempotency_row(row) - if validated.request_hash != request_hash: - return IdempotencyClaim("conflict") - if validated.state == "completed": - if validated.response_status is None or validated.response_json is None: - raise AssertionError("validated completed idempotency response is incomplete") - return IdempotencyClaim( - "replay", - validated.response_status, - validated.response_json, - ) - if validated.owner_instance_id == owner_instance_id: - return IdempotencyClaim("in_progress") - return IdempotencyClaim("indeterminate") - - class StateStore: def __init__(self, database_path: Path | str) -> None: self._database = SQLiteDatabase(database_path) self._schema = SchemaManager(self._database) + self._idempotency = IdempotencyStore(self._database) @property def database_path(self) -> Path: @@ -261,6 +125,57 @@ def init(self) -> None: def migrate(self) -> None: self._schema.migrate() + def claim_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + expires_at: datetime, + max_records: int = 10_000, + ) -> IdempotencyClaim: + return self._idempotency.claim_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + expires_at=expires_at, + max_records=max_records, + ) + + def lookup_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + ) -> IdempotencyClaim | None: + return self._idempotency.lookup_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + ) + + def complete_idempotency( + self, + *, + key_hash: str, + request_hash: str, + owner_instance_id: str, + response_status: int, + response_json: str, + completed_at: datetime, + expires_at: datetime, + ) -> None: + self._idempotency.complete_idempotency( + key_hash=key_hash, + request_hash=request_hash, + owner_instance_id=owner_instance_id, + response_status=response_status, + response_json=response_json, + completed_at=completed_at, + expires_at=expires_at, + ) + def begin_reconcile_run(self, run: ReconcileRunStart) -> None: if not isinstance(run, ReconcileRunStart): raise TypeError("run must be a ReconcileRunStart") @@ -648,209 +563,6 @@ def _reconcile_counts_from_results( counts[result.outcome.value] += 1 return counts - def claim_idempotency( - self, - *, - key_hash: str, - request_hash: str, - owner_instance_id: str, - expires_at: datetime, - max_records: int = 10_000, - ) -> IdempotencyClaim: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") - if type(max_records) is not int or not 1 <= max_records <= 1_000_000: - raise ValueError("idempotency capacity is invalid") - current_time = datetime.now(UTC) - if datetime.fromisoformat(safe_expiry) < current_time: - raise ValueError("idempotency expiry timestamp is invalid") - now = current_time.isoformat() - - try: - conn = self.connect() - except (OSError, sqlite3.Error): - return IdempotencyClaim("unavailable") - with closing(conn): - try: - conn.execute("BEGIN IMMEDIATE") - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is not None: - validated = _validate_idempotency_row(row) - reclaimable = validated.expires_at <= current_time and ( - validated.state == "completed" or validated.owner_instance_id != safe_owner - ) - if reclaimable: - conn.execute( - "DELETE FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ) - row = None - conn.execute( - """ - DELETE FROM idempotency_records - WHERE expires_at <= ? - AND (state = 'completed' OR owner_instance_id != ?) - AND key_hash != ? - """, - (now, safe_owner, safe_key_hash), - ) - if row is None: - count = int( - conn.execute("SELECT COUNT(*) FROM idempotency_records").fetchone()[0] - ) - if count >= max_records: - conn.commit() - return IdempotencyClaim("unavailable") - conn.execute( - """ - INSERT INTO idempotency_records ( - key_hash, - request_hash, - state, - owner_instance_id, - response_status, - response_json, - created_at, - updated_at, - expires_at - ) VALUES (?, ?, 'in_progress', ?, NULL, NULL, ?, ?, ?) - """, - ( - safe_key_hash, - safe_request_hash, - safe_owner, - now, - now, - safe_expiry, - ), - ) - conn.commit() - return IdempotencyClaim("claimed") - - result = _idempotency_claim_from_row( - row, - request_hash=safe_request_hash, - owner_instance_id=safe_owner, - ) - conn.commit() - return result - except (OSError, sqlite3.Error, ValueError, TypeError): - conn.rollback() - return IdempotencyClaim("unavailable") - - def lookup_idempotency( - self, - *, - key_hash: str, - request_hash: str, - owner_instance_id: str, - ) -> IdempotencyClaim | None: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - current_time = datetime.now(UTC) - - try: - conn = self.connect() - except (OSError, sqlite3.Error): - return IdempotencyClaim("unavailable") - with closing(conn): - try: - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is None: - return None - validated = _validate_idempotency_row(row) - if validated.expires_at <= current_time and ( - validated.state == "completed" or validated.owner_instance_id != safe_owner - ): - return None - return _idempotency_claim_from_row( - row, - request_hash=safe_request_hash, - owner_instance_id=safe_owner, - ) - except (OSError, sqlite3.Error, ValueError, TypeError): - return IdempotencyClaim("unavailable") - - def complete_idempotency( - self, - *, - key_hash: str, - request_hash: str, - owner_instance_id: str, - response_status: int, - response_json: str, - completed_at: datetime, - expires_at: datetime, - ) -> None: - safe_key_hash = _validate_idempotency_hash(key_hash, "key hash") - safe_request_hash = _validate_idempotency_hash(request_hash, "request hash") - safe_owner = _validate_idempotency_owner(owner_instance_id) - safe_status = _validate_idempotency_status(response_status) - safe_json = _validate_idempotency_response_json(response_json) - safe_completed = _validate_idempotency_timestamp(completed_at, "completion timestamp") - safe_expiry = _validate_idempotency_timestamp(expires_at, "expiry timestamp") - if safe_expiry < safe_completed: - raise ValueError("idempotency expiry timestamp is invalid") - completed_time = datetime.fromisoformat(safe_completed) - - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - row = conn.execute( - "SELECT * FROM idempotency_records WHERE key_hash = ?", - (safe_key_hash,), - ).fetchone() - if row is None: - raise RuntimeError("idempotency completion failed") - validated = _validate_idempotency_row(row) - if ( - validated.request_hash != safe_request_hash - or validated.owner_instance_id != safe_owner - or validated.state != "in_progress" - ): - raise RuntimeError("idempotency completion failed") - if completed_time < validated.created_at: - raise ValueError("idempotency completion timestamp is invalid") - cursor = conn.execute( - """ - UPDATE idempotency_records - SET state = 'completed', - response_status = ?, - response_json = ?, - updated_at = ?, - expires_at = ? - WHERE key_hash = ? - AND request_hash = ? - AND owner_instance_id = ? - AND state = 'in_progress' - """, - ( - safe_status, - safe_json, - safe_completed, - safe_expiry, - safe_key_hash, - safe_request_hash, - safe_owner, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("idempotency completion failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() - def begin_lifecycle_intent( self, bot_id: str, From d727b01a9e644acd5d22aa60bd5e82c10a1840d1 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:30:56 +0200 Subject: [PATCH 33/49] refactor(state): isolate reconciliation persistence --- tests/test_reconcile_store.py | 533 ++++++++++++++++++++++++++++++++++ zeus/reconcile_store.py | 468 +++++++++++++++++++++++++++++ zeus/state.py | 440 +--------------------------- 3 files changed, 1009 insertions(+), 432 deletions(-) create mode 100644 tests/test_reconcile_store.py create mode 100644 zeus/reconcile_store.py diff --git a/tests/test_reconcile_store.py b/tests/test_reconcile_store.py new file mode 100644 index 0000000..c72ab2f --- /dev/null +++ b/tests/test_reconcile_store.py @@ -0,0 +1,533 @@ +from __future__ import annotations + +import ast +import inspect +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, call, patch + +from zeus.idempotency_store import IdempotencyStore +from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS, ReconcileStore +from zeus.reconciliation import ( + BotReconcileResult, + PersistedReconcileRun, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, + summarize_results, +) +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import RECONCILE_COUNTER_COLUMNS as STATE_RECONCILE_COUNTER_COLUMNS +from zeus.state import StateStore + +RUN_STARTED_AT = datetime(2026, 7, 12, 11, 59, tzinfo=UTC) +RESULT_STARTED_AT = datetime(2026, 7, 12, 12, 0, tzinfo=UTC) +RESULT_FINISHED_AT = RESULT_STARTED_AT + timedelta(seconds=1) +RUN_FINISHED_AT = datetime(2026, 7, 12, 12, 1, tzinfo=UTC) + + +class _TracingSQLiteDatabase(SQLiteDatabase): + traces: ClassVar[list[tuple[int, str]]] + + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces = [] + + def connect(self) -> sqlite3.Connection: + conn = super().connect() + connection_id = id(conn) + conn.set_trace_callback(lambda sql: self.traces.append((connection_id, sql))) + return conn + + +def _run( + run_id: str = "run-1", + *, + scope: str = "fleet", + requested_bot_id: str | None = None, +) -> ReconcileRunStart: + return ReconcileRunStart( + run_id=run_id, + scope=scope, + requested_bot_id=requested_bot_id, + source="cli", + force=False, + reset_restart=False, + started_at=RUN_STARTED_AT, + ) + + +def _result( + bot_id: str = "coder", + outcome: ReconcileOutcome = ReconcileOutcome.healthy, + *, + event_id: int | None = None, +) -> BotReconcileResult: + return BotReconcileResult( + bot_id=bot_id, + outcome=outcome, + desired_state="running", + observed_status="running", + pid=1234, + action="none" if outcome is ReconcileOutcome.healthy else "start", + message="reconciled", + error_code=None, + event_id=event_id, + started_at=RESULT_STARTED_AT, + finished_at=RESULT_FINISHED_AT, + ) + + +def _summary( + run: ReconcileRunStart, + results: list[BotReconcileResult], +) -> ReconcileRunSummary: + return summarize_results( + run.run_id, + run.scope, + results, + started_at=run.started_at, + finished_at=RUN_FINISHED_AT, + ) + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _connection_id, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statements(trace: list[tuple[int, str]]) -> list[str]: + return [sql for _connection_id, sql in trace] + + +def _statement_index(statements: list[str], prefix: str, *, start: int = 0) -> int: + return next( + index + for index, statement in enumerate(statements[start:], start=start) + if statement.lstrip().upper().startswith(prefix) + ) + + +def _run_bytes(database_path: Path, run_id: str) -> tuple[object, ...] | None: + columns = ( + "run_id", + "scope", + "requested_bot_id", + "source", + "force", + "reset_restart", + "started_at", + "finished_at", + "outcome", + "total", + "healthy_count", + "changed_count", + "pending_count", + "action_required_count", + "error_count", + "skipped_count", + ) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + with closing(sqlite3.connect(database_path)) as conn: + return conn.execute( + f"SELECT {fields} FROM reconcile_runs WHERE run_id = ?", + (run_id,), + ).fetchone() + + +class ReconcileStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.database_path = Path(self.temporary_directory.name) / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = ReconcileStore(self.database) + + def test_constructs_without_opening_database_and_owns_no_schema_or_lifecycle_store( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + ReconcileStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + import zeus.reconcile_store as reconcile_store_module + + source = inspect.getsource(reconcile_store_module) + tree = ast.parse(source) + imported_modules = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + } + self.assertTrue( + imported_modules.isdisjoint( + { + "zeus.bot_lifecycle_store", + "zeus.lifecycle", + "zeus.schema", + "zeus.state", + } + ) + ) + self.assertNotIn("CREATE TABLE", source.upper()) + self.assertNotIn("CREATE TRIGGER", source.upper()) + + def test_direct_store_round_trip_and_interruption(self) -> None: + run = _run() + result = _result(outcome=ReconcileOutcome.changed) + + self.store.begin_reconcile_run(run) + self.store.append_reconcile_result(run.run_id, result) + finished = self.store.finish_reconcile_run(_summary(run, [result])) + loaded = self.store.get_reconcile_run(run.run_id) + + self.assertIsInstance(finished, PersistedReconcileRun) + self.assertEqual(finished, loaded) + self.assertEqual("succeeded", finished.outcome) + self.assertEqual((result,), finished.results) + self.assertEqual(1, finished.counts[ReconcileOutcome.changed.value]) + + stale = _run("run-stale") + self.store.begin_reconcile_run(stale) + interrupted_at = RUN_FINISHED_AT + timedelta(minutes=1) + self.assertEqual( + 1, + self.store.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at), + ) + interrupted = self.store.get_reconcile_run(stale.run_id) + self.assertIsNotNone(interrupted) + self.assertEqual("interrupted", interrupted.outcome if interrupted else None) + self.assertEqual(interrupted_at, interrupted.finished_at if interrupted else None) + + def test_all_five_operations_keep_exact_transaction_boundaries(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("trace.db")) + SchemaManager(database).init() + database.traces.clear() + store = ReconcileStore(database) + run = _run() + result = _result() + + store.begin_reconcile_run(run) + begin_trace = list(database.traces) + database.traces.clear() + + store.append_reconcile_result(run.run_id, result) + append_trace = list(database.traces) + database.traces.clear() + + store.finish_reconcile_run(_summary(run, [result])) + finish_trace = list(database.traces) + database.traces.clear() + + loaded = store.get_reconcile_run(run.run_id) + get_trace = list(database.traces) + + store.begin_reconcile_run(_run("run-stale-a")) + store.begin_reconcile_run(_run("run-stale-b")) + database.traces.clear() + interrupted = store.interrupt_stale_reconcile_runs(interrupted_at=RUN_FINISHED_AT) + interrupt_trace = list(database.traces) + + self.assertIsNotNone(loaded) + self.assertEqual(2, interrupted) + for trace in (begin_trace, append_trace, finish_trace, get_trace, interrupt_trace): + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(begin_trace)) + begin_sql = _statements(begin_trace) + self.assertLess( + _statement_index(begin_sql, "BEGIN IMMEDIATE"), + _statement_index(begin_sql, "INSERT INTO RECONCILE_RUNS"), + ) + self.assertLess( + _statement_index(begin_sql, "INSERT INTO RECONCILE_RUNS"), + _statement_index(begin_sql, "COMMIT"), + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(append_trace)) + append_sql = _statements(append_trace) + result_insert = _statement_index(append_sql, "INSERT INTO RECONCILE_RESULTS") + counter_update = _statement_index(append_sql, "UPDATE RECONCILE_RUNS") + self.assertLess(result_insert, counter_update) + self.assertLess(counter_update, _statement_index(append_sql, "COMMIT")) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(finish_trace)) + finish_sql = _statements(finish_trace) + finish_update = _statement_index(finish_sql, "UPDATE RECONCILE_RUNS") + run_selects = [ + index + for index, sql in enumerate(finish_sql) + if sql.lstrip().upper().startswith("SELECT * FROM RECONCILE_RUNS") + ] + result_selects = [ + index + for index, sql in enumerate(finish_sql) + if sql.lstrip().upper().startswith("SELECT * FROM RECONCILE_RESULTS") + ] + self.assertEqual(2, len(run_selects)) + self.assertEqual(2, len(result_selects)) + self.assertLess(run_selects[0], result_selects[0]) + self.assertLess(result_selects[0], finish_update) + self.assertLess(finish_update, run_selects[1]) + self.assertLess(run_selects[1], result_selects[1]) + self.assertLess(result_selects[1], _statement_index(finish_sql, "COMMIT")) + + self.assertEqual(["BEGIN", "COMMIT"], _control_statements(get_trace)) + get_sql = _statements(get_trace) + self.assertEqual("BEGIN", get_sql[0].strip().upper()) + self.assertTrue( + all(sql.lstrip().upper().startswith(("BEGIN", "SELECT", "COMMIT")) for sql in get_sql) + ) + + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(interrupt_trace)) + interrupt_sql = _statements(interrupt_trace) + interrupt_updates = [ + index + for index, sql in enumerate(interrupt_sql) + if sql.lstrip().upper().startswith("UPDATE RECONCILE_RUNS") + ] + self.assertEqual(2, len(interrupt_updates)) + self.assertLess(interrupt_updates[-1], _statement_index(interrupt_sql, "COMMIT")) + + def test_result_insert_and_counter_update_roll_back_together(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("append-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + run = _run() + store.begin_reconcile_run(run) + with closing(database.connect()) as conn: + conn.execute( + """ + CREATE TRIGGER reject_reconcile_counter_update + BEFORE UPDATE ON reconcile_runs + WHEN NEW.total = OLD.total + 1 + BEGIN + SELECT RAISE(ABORT, 'injected counter failure'); + END + """ + ) + conn.commit() + database.traces.clear() + + with self.assertRaisesRegex(sqlite3.DatabaseError, "injected counter failure"): + store.append_reconcile_result(run.run_id, _result()) + append_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(append_trace)) + append_sql = _statements(append_trace) + self.assertLess( + _statement_index(append_sql, "INSERT INTO RECONCILE_RESULTS"), + _statement_index(append_sql, "UPDATE RECONCILE_RUNS"), + ) + with closing(sqlite3.connect(database.database_path)) as conn: + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + run_row = conn.execute( + """ + SELECT outcome, total, healthy_count, changed_count, pending_count, + action_required_count, error_count, skipped_count + FROM reconcile_runs + WHERE run_id = ? + """, + (run.run_id,), + ).fetchone() + self.assertEqual(0, result_count) + self.assertEqual(("running", 0, 0, 0, 0, 0, 0, 0), run_row) + + def test_finish_validation_rolls_back_and_leaves_run_open_byte_for_byte(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("finish-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + run = _run() + result = _result() + store.begin_reconcile_run(run) + store.append_reconcile_result(run.run_id, result) + before = _run_bytes(database.database_path, run.run_id) + database.traces.clear() + + with self.assertRaisesRegex(RuntimeError, "persisted reconciliation results"): + store.finish_reconcile_run(_summary(run, [])) + finish_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(finish_trace)) + self.assertEqual(before, _run_bytes(database.database_path, run.run_id)) + with closing(sqlite3.connect(database.database_path)) as conn: + outcome, finished_at = conn.execute( + "SELECT outcome, finished_at FROM reconcile_runs WHERE run_id = ?", + (run.run_id,), + ).fetchone() + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + self.assertEqual(("running", None), (outcome, finished_at)) + self.assertEqual(1, result_count) + + def test_interrupt_rolls_back_every_running_run_when_one_has_a_late_result(self) -> None: + database = _TracingSQLiteDatabase(self.database_path.with_name("interrupt-rollback.db")) + SchemaManager(database).init() + store = ReconcileStore(database) + first = _run("run-a-empty") + late = _run("run-z-late") + store.begin_reconcile_run(first) + store.begin_reconcile_run(late) + store.append_reconcile_result(late.run_id, _result()) + first_before = _run_bytes(database.database_path, first.run_id) + late_before = _run_bytes(database.database_path, late.run_id) + interrupted_at = RESULT_STARTED_AT + timedelta(microseconds=500_000) + database.traces.clear() + + with self.assertRaisesRegex(ValueError, "finishes after"): + store.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at) + interrupt_trace = list(database.traces) + + self.assertEqual(["BEGIN IMMEDIATE", "ROLLBACK"], _control_statements(interrupt_trace)) + interrupt_sql = _statements(interrupt_trace) + self.assertEqual( + 1, + sum(sql.lstrip().upper().startswith("UPDATE RECONCILE_RUNS") for sql in interrupt_sql), + ) + self.assertEqual(first_before, _run_bytes(database.database_path, first.run_id)) + self.assertEqual(late_before, _run_bytes(database.database_path, late.run_id)) + + def test_event_link_rejects_missing_and_wrong_bot_without_partial_results(self) -> None: + run = _run() + self.store.begin_reconcile_run(run) + + with self.assertRaisesRegex(ValueError, "lifecycle event"): + self.store.append_reconcile_result( + run.run_id, + _result(outcome=ReconcileOutcome.changed, event_id=999), + ) + + with closing(sqlite3.connect(self.database_path)) as conn: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, operation_id, occurred_at, source, action, outcome + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + "other", + "a" * 32, + RUN_STARTED_AT.isoformat(), + "cli", + "bot.create", + "success", + ), + ) + wrong_bot_event_id = int(cursor.lastrowid) + conn.commit() + + with self.assertRaisesRegex(ValueError, "lifecycle event"): + self.store.append_reconcile_result( + run.run_id, + _result( + outcome=ReconcileOutcome.changed, + event_id=wrong_bot_event_id, + ), + ) + + with closing(sqlite3.connect(self.database_path)) as conn: + result_count = conn.execute( + "SELECT COUNT(*) FROM reconcile_results WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + total = conn.execute( + "SELECT total FROM reconcile_runs WHERE run_id = ?", + (run.run_id,), + ).fetchone()[0] + self.assertEqual(0, result_count) + self.assertEqual(0, total) + + def test_state_facade_uses_shared_database_and_delegates_verbatim_signatures(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + idempotency = MagicMock(spec=IdempotencyStore) + delegate = MagicMock(spec=ReconcileStore) + run = _run() + result = _result() + summary = _summary(run, [result]) + finished = MagicMock(spec=PersistedReconcileRun) + loaded = MagicMock(spec=PersistedReconcileRun) + delegate.finish_reconcile_run.return_value = finished + delegate.interrupt_stale_reconcile_runs.return_value = 3 + delegate.get_reconcile_run.return_value = loaded + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=idempotency) as idempotency_type, + patch("zeus.state.ReconcileStore", return_value=delegate) as reconcile_type, + ): + facade = StateStore(database_path) + facade.begin_reconcile_run(run) + facade.append_reconcile_result(run.run_id, result) + actual_finished = facade.finish_reconcile_run(summary) + actual_interrupted = facade.interrupt_stale_reconcile_runs( + interrupted_at=RUN_FINISHED_AT + ) + actual_loaded = facade.get_reconcile_run(run.run_id) + + self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], idempotency_type.call_args_list) + self.assertEqual([call(database)], reconcile_type.call_args_list) + database.connect.assert_not_called() + delegate.begin_reconcile_run.assert_called_once_with(run) + delegate.append_reconcile_result.assert_called_once_with(run.run_id, result) + delegate.finish_reconcile_run.assert_called_once_with(summary) + delegate.interrupt_stale_reconcile_runs.assert_called_once_with( + interrupted_at=RUN_FINISHED_AT + ) + delegate.get_reconcile_run.assert_called_once_with(run.run_id) + self.assertIs(finished, actual_finished) + self.assertEqual(3, actual_interrupted) + self.assertIs(loaded, actual_loaded) + for method_name in ( + "begin_reconcile_run", + "append_reconcile_result", + "finish_reconcile_run", + "interrupt_stale_reconcile_runs", + "get_reconcile_run", + ): + self.assertEqual( + inspect.signature(getattr(ReconcileStore, method_name)), + inspect.signature(getattr(StateStore, method_name)), + ) + + def test_reconcile_counter_columns_remain_a_state_compatibility_alias(self) -> None: + self.assertIs(RECONCILE_COUNTER_COLUMNS, STATE_RECONCILE_COUNTER_COLUMNS) + self.assertEqual( + { + ReconcileOutcome.healthy: "healthy_count", + ReconcileOutcome.changed: "changed_count", + ReconcileOutcome.pending: "pending_count", + ReconcileOutcome.action_required: "action_required_count", + ReconcileOutcome.error: "error_count", + ReconcileOutcome.skipped: "skipped_count", + }, + RECONCILE_COUNTER_COLUMNS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/reconcile_store.py b/zeus/reconcile_store.py new file mode 100644 index 0000000..4e2f16e --- /dev/null +++ b/zeus/reconcile_store.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime + +from zeus.reconciliation import ( + BotReconcileResult, + PersistedReconcileRun, + ReconcileOutcome, + ReconcileRunStart, + ReconcileRunSummary, +) +from zeus.sqlite_db import SQLiteDatabase + +RECONCILE_COUNTER_COLUMNS = { + ReconcileOutcome.healthy: "healthy_count", + ReconcileOutcome.changed: "changed_count", + ReconcileOutcome.pending: "pending_count", + ReconcileOutcome.action_required: "action_required_count", + ReconcileOutcome.error: "error_count", + ReconcileOutcome.skipped: "skipped_count", +} + + +def _validate_reconcile_run_id(value: str) -> str: + if not isinstance(value, str): + raise TypeError("reconciliation run_id must be a string") + if not value or len(value) > 128: + raise ValueError("reconciliation run_id must be between 1 and 128 characters") + if any(ord(character) < 0x20 or 0x7F <= ord(character) <= 0x9F for character in value): + raise ValueError("reconciliation run_id must not contain control characters") + return value + + +def _serialize_reconcile_timestamp(value: datetime, label: str) -> str: + if not isinstance(value, datetime): + raise TypeError(f"reconciliation {label} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"reconciliation {label} must be timezone-aware") + return value.astimezone(UTC).isoformat() + + +def _parse_reconcile_timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str): + raise TypeError(f"stored reconciliation {label} must be a string") + try: + parsed = datetime.fromisoformat(value) + except ValueError: + raise ValueError(f"stored reconciliation {label} is invalid") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError(f"stored reconciliation {label} is invalid") + normalized = parsed.astimezone(UTC) + if normalized.isoformat() != value: + raise ValueError(f"stored reconciliation {label} is not normalized") + return normalized + + +def _validate_stored_nonnegative_integer(value: object, label: str) -> int: + if type(value) is not int or value < 0: + raise ValueError(f"stored reconciliation {label} must be a non-negative integer") + return value + + +def _validate_stored_optional_positive_integer(value: object, label: str) -> int | None: + if value is None: + return None + if type(value) is not int or value <= 0: + raise ValueError(f"stored reconciliation {label} must be a positive integer or null") + return value + + +def _validate_stored_boolean_flag(value: object, label: str) -> bool: + if type(value) is not int or value not in {0, 1}: + raise ValueError(f"stored reconciliation {label} must be exactly 0 or 1") + return value == 1 + + +class ReconcileStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def begin_reconcile_run(self, run: ReconcileRunStart) -> None: + if not isinstance(run, ReconcileRunStart): + raise TypeError("run must be a ReconcileRunStart") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + """ + INSERT INTO reconcile_runs ( + run_id, + scope, + requested_bot_id, + source, + force, + reset_restart, + started_at, + finished_at, + outcome + ) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'running') + """, + ( + run.run_id, + run.scope, + run.requested_bot_id, + run.source, + int(run.force), + int(run.reset_restart), + _serialize_reconcile_timestamp(run.started_at, "start timestamp"), + ), + ) + except Exception: + conn.rollback() + raise + else: + conn.commit() + + def append_reconcile_result( + self, + run_id: str, + result: BotReconcileResult, + ) -> None: + safe_run_id = _validate_reconcile_run_id(run_id) + if not isinstance(result, BotReconcileResult): + raise TypeError("result must be a BotReconcileResult") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + run_row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (safe_run_id,), + ).fetchone() + if run_row is None: + raise KeyError(f"unknown reconciliation run: {safe_run_id}") + if run_row["outcome"] != "running": + raise RuntimeError("reconciliation run is not running") + current_run = self._materialize_reconcile_run(conn, safe_run_id) + if current_run is None: + raise sqlite3.IntegrityError("reconciliation run disappeared") + if current_run.scope == "bot" and result.bot_id != current_run.requested_bot_id: + raise ValueError("reconciliation result must match the requested bot") + if result.started_at < current_run.started_at: + raise ValueError("reconciliation result starts before its run") + if result.event_id is not None: + event_row = conn.execute( + "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", + (result.event_id,), + ).fetchone() + if event_row is None or event_row["bot_id"] != result.bot_id: + raise ValueError( + "reconciliation lifecycle event must exist and match the result bot" + ) + conn.execute( + """ + INSERT INTO reconcile_results ( + run_id, + bot_id, + ordinal, + outcome, + desired_state, + observed_status, + pid, + action, + message, + error_code, + event_id, + started_at, + finished_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + safe_run_id, + result.bot_id, + current_run.total, + result.outcome.value, + result.desired_state, + result.observed_status, + result.pid, + result.action, + result.message, + result.error_code, + result.event_id, + _serialize_reconcile_timestamp( + result.started_at, + "result start timestamp", + ), + _serialize_reconcile_timestamp( + result.finished_at, + "result finish timestamp", + ), + ), + ) + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET total = total + 1, + healthy_count = healthy_count + ?, + changed_count = changed_count + ?, + pending_count = pending_count + ?, + action_required_count = action_required_count + ?, + error_count = error_count + ?, + skipped_count = skipped_count + ? + WHERE run_id = ? AND outcome = 'running' + """, + ( + int(result.outcome is ReconcileOutcome.healthy), + int(result.outcome is ReconcileOutcome.changed), + int(result.outcome is ReconcileOutcome.pending), + int(result.outcome is ReconcileOutcome.action_required), + int(result.outcome is ReconcileOutcome.error), + int(result.outcome is ReconcileOutcome.skipped), + safe_run_id, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation result counter update failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() + + def finish_reconcile_run( + self, + summary: ReconcileRunSummary, + ) -> PersistedReconcileRun: + if not isinstance(summary, ReconcileRunSummary): + raise TypeError("summary must be a ReconcileRunSummary") + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + run_row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (summary.run_id,), + ).fetchone() + if run_row is None: + raise KeyError(f"unknown reconciliation run: {summary.run_id}") + if run_row["outcome"] != "running": + raise RuntimeError("reconciliation run is not running") + if run_row["scope"] != summary.scope: + raise RuntimeError("reconciliation summary scope does not match run") + stored_started_at = _parse_reconcile_timestamp( + run_row["started_at"], + "run start timestamp", + ) + if stored_started_at != summary.started_at: + raise RuntimeError("reconciliation summary start timestamp does not match run") + persisted_results = self._reconcile_results_in_transaction(conn, summary.run_id) + if tuple(summary.results) != persisted_results: + raise RuntimeError("persisted reconciliation results do not match summary") + observed_counts = self._reconcile_counts_from_results(persisted_results) + stored_counts = self._reconcile_counts_from_run_row(run_row) + stored_total = _validate_stored_nonnegative_integer( + run_row["total"], + "total", + ) + if stored_counts != observed_counts or stored_total != len(persisted_results): + raise RuntimeError("persisted reconciliation counters do not match results") + if dict(summary.counts) != observed_counts or summary.total != len( + persisted_results + ): + raise RuntimeError("reconciliation summary counters do not match results") + if any(result.finished_at > summary.finished_at for result in persisted_results): + raise RuntimeError("reconciliation summary finishes before a persisted result") + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET finished_at = ?, + outcome = ?, + total = ?, + healthy_count = ?, + changed_count = ?, + pending_count = ?, + action_required_count = ?, + error_count = ?, + skipped_count = ? + WHERE run_id = ? AND outcome = 'running' + """, + ( + _serialize_reconcile_timestamp( + summary.finished_at, + "finish timestamp", + ), + summary.outcome, + summary.total, + summary.counts[ReconcileOutcome.healthy.value], + summary.counts[ReconcileOutcome.changed.value], + summary.counts[ReconcileOutcome.pending.value], + summary.counts[ReconcileOutcome.action_required.value], + summary.counts[ReconcileOutcome.error.value], + summary.counts[ReconcileOutcome.skipped.value], + summary.run_id, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation run finalization failed") + finished = self._materialize_reconcile_run(conn, summary.run_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + if finished is None: + raise sqlite3.IntegrityError("finished reconciliation run is missing") + return finished + + def interrupt_stale_reconcile_runs(self, *, interrupted_at: datetime) -> int: + safe_interrupted_at = _serialize_reconcile_timestamp( + interrupted_at, + "interruption timestamp", + ) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + "SELECT * FROM reconcile_runs WHERE outcome = 'running' ORDER BY run_id" + ).fetchall() + for row in rows: + run_id = str(row["run_id"]) + current_run = self._materialize_reconcile_run(conn, run_id) + if current_run is None: + raise sqlite3.IntegrityError("reconciliation run disappeared") + if interrupted_at < current_run.started_at: + raise ValueError("interruption timestamp precedes a running reconciliation") + if any(result.finished_at > interrupted_at for result in current_run.results): + raise ValueError( + "reconciliation result finishes after the interruption timestamp" + ) + cursor = conn.execute( + """ + UPDATE reconcile_runs + SET outcome = 'interrupted', finished_at = ? + WHERE run_id = ? AND outcome = 'running' + """, + (safe_interrupted_at, run_id), + ) + if cursor.rowcount != 1: + raise sqlite3.IntegrityError("reconciliation interruption update failed") + except Exception: + conn.rollback() + raise + else: + conn.commit() + return len(rows) + + def get_reconcile_run(self, run_id: str) -> PersistedReconcileRun | None: + safe_run_id = _validate_reconcile_run_id(run_id) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN") + run = self._materialize_reconcile_run(conn, safe_run_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + return run + + def _materialize_reconcile_run( + self, + conn: sqlite3.Connection, + run_id: str, + ) -> PersistedReconcileRun | None: + row = conn.execute( + "SELECT * FROM reconcile_runs WHERE run_id = ?", + (run_id,), + ).fetchone() + if row is None: + return None + results = self._reconcile_results_in_transaction(conn, run_id) + finished_at = row["finished_at"] + return PersistedReconcileRun( + run_id=str(row["run_id"]), + scope=str(row["scope"]), + requested_bot_id=( + str(row["requested_bot_id"]) if row["requested_bot_id"] is not None else None + ), + source=str(row["source"]), + force=_validate_stored_boolean_flag(row["force"], "force flag"), + reset_restart=_validate_stored_boolean_flag( + row["reset_restart"], + "reset-restart flag", + ), + started_at=_parse_reconcile_timestamp(row["started_at"], "run start timestamp"), + finished_at=( + _parse_reconcile_timestamp(finished_at, "run finish timestamp") + if finished_at is not None + else None + ), + outcome=str(row["outcome"]), + total=_validate_stored_nonnegative_integer(row["total"], "total"), + counts=self._reconcile_counts_from_run_row(row), + results=results, + ) + + def _reconcile_results_in_transaction( + self, + conn: sqlite3.Connection, + run_id: str, + ) -> tuple[BotReconcileResult, ...]: + rows = conn.execute( + "SELECT * FROM reconcile_results WHERE run_id = ? ORDER BY ordinal", + (run_id,), + ).fetchall() + results: list[BotReconcileResult] = [] + for expected_ordinal, row in enumerate(rows): + ordinal = _validate_stored_nonnegative_integer(row["ordinal"], "result ordinal") + if ordinal != expected_ordinal: + raise ValueError("stored reconciliation ordinals are not contiguous") + bot_id = str(row["bot_id"]) + event_id = _validate_stored_optional_positive_integer( + row["event_id"], + "result event id", + ) + if event_id is not None: + event_row = conn.execute( + "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", + (event_id,), + ).fetchone() + if event_row is None or event_row["bot_id"] != bot_id: + raise ValueError( + "stored reconciliation lifecycle event does not match the result bot" + ) + results.append( + BotReconcileResult( + bot_id=bot_id, + outcome=ReconcileOutcome(str(row["outcome"])), + desired_state=( + str(row["desired_state"]) if row["desired_state"] is not None else None + ), + observed_status=( + str(row["observed_status"]) if row["observed_status"] is not None else None + ), + pid=_validate_stored_optional_positive_integer(row["pid"], "result pid"), + action=str(row["action"]), + message=str(row["message"]), + error_code=(str(row["error_code"]) if row["error_code"] is not None else None), + event_id=event_id, + started_at=_parse_reconcile_timestamp( + row["started_at"], + "result start timestamp", + ), + finished_at=_parse_reconcile_timestamp( + row["finished_at"], + "result finish timestamp", + ), + ) + ) + return tuple(results) + + def _reconcile_counts_from_run_row(self, row: sqlite3.Row) -> dict[str, int]: + return { + outcome.value: _validate_stored_nonnegative_integer( + row[column], + f"{outcome.value} counter", + ) + for outcome, column in RECONCILE_COUNTER_COLUMNS.items() + } + + def _reconcile_counts_from_results( + self, + results: tuple[BotReconcileResult, ...], + ) -> dict[str, int]: + counts = {outcome.value: 0 for outcome in ReconcileOutcome} + for result in results: + counts[result.outcome.value] += 1 + return counts diff --git a/zeus/state.py b/zeus/state.py index 0e0a49e..374f6f8 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -21,10 +21,11 @@ ) from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy from zeus.private_io import append_private_bytes, nofollow_absolute_path +from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS as RECONCILE_COUNTER_COLUMNS +from zeus.reconcile_store import ReconcileStore from zeus.reconciliation import ( BotReconcileResult, PersistedReconcileRun, - ReconcileOutcome, ReconcileRunStart, ReconcileRunSummary, ) @@ -37,14 +38,6 @@ LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) -RECONCILE_COUNTER_COLUMNS = { - ReconcileOutcome.healthy: "healthy_count", - ReconcileOutcome.changed: "changed_count", - ReconcileOutcome.pending: "pending_count", - ReconcileOutcome.action_required: "action_required_count", - ReconcileOutcome.error: "error_count", - ReconcileOutcome.skipped: "skipped_count", -} def _sanitize_optional_persisted_text(value: str | None) -> str | None: @@ -53,64 +46,12 @@ def _sanitize_optional_persisted_text(value: str | None) -> str | None: return sanitize_text(value) -def _validate_reconcile_run_id(value: str) -> str: - if not isinstance(value, str): - raise TypeError("reconciliation run_id must be a string") - if not value or len(value) > 128: - raise ValueError("reconciliation run_id must be between 1 and 128 characters") - if any(ord(character) < 0x20 or 0x7F <= ord(character) <= 0x9F for character in value): - raise ValueError("reconciliation run_id must not contain control characters") - return value - - -def _serialize_reconcile_timestamp(value: datetime, label: str) -> str: - if not isinstance(value, datetime): - raise TypeError(f"reconciliation {label} must be a datetime") - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"reconciliation {label} must be timezone-aware") - return value.astimezone(UTC).isoformat() - - -def _parse_reconcile_timestamp(value: object, label: str) -> datetime: - if not isinstance(value, str): - raise TypeError(f"stored reconciliation {label} must be a string") - try: - parsed = datetime.fromisoformat(value) - except ValueError: - raise ValueError(f"stored reconciliation {label} is invalid") from None - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise ValueError(f"stored reconciliation {label} is invalid") - normalized = parsed.astimezone(UTC) - if normalized.isoformat() != value: - raise ValueError(f"stored reconciliation {label} is not normalized") - return normalized - - -def _validate_stored_nonnegative_integer(value: object, label: str) -> int: - if type(value) is not int or value < 0: - raise ValueError(f"stored reconciliation {label} must be a non-negative integer") - return value - - -def _validate_stored_optional_positive_integer(value: object, label: str) -> int | None: - if value is None: - return None - if type(value) is not int or value <= 0: - raise ValueError(f"stored reconciliation {label} must be a positive integer or null") - return value - - -def _validate_stored_boolean_flag(value: object, label: str) -> bool: - if type(value) is not int or value not in {0, 1}: - raise ValueError(f"stored reconciliation {label} must be exactly 0 or 1") - return value == 1 - - class StateStore: def __init__(self, database_path: Path | str) -> None: self._database = SQLiteDatabase(database_path) self._schema = SchemaManager(self._database) self._idempotency = IdempotencyStore(self._database) + self._reconcile = ReconcileStore(self._database) @property def database_path(self) -> Path: @@ -177,391 +118,26 @@ def complete_idempotency( ) def begin_reconcile_run(self, run: ReconcileRunStart) -> None: - if not isinstance(run, ReconcileRunStart): - raise TypeError("run must be a ReconcileRunStart") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - conn.execute( - """ - INSERT INTO reconcile_runs ( - run_id, - scope, - requested_bot_id, - source, - force, - reset_restart, - started_at, - finished_at, - outcome - ) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'running') - """, - ( - run.run_id, - run.scope, - run.requested_bot_id, - run.source, - int(run.force), - int(run.reset_restart), - _serialize_reconcile_timestamp(run.started_at, "start timestamp"), - ), - ) - except Exception: - conn.rollback() - raise - else: - conn.commit() + self._reconcile.begin_reconcile_run(run) def append_reconcile_result( self, run_id: str, result: BotReconcileResult, ) -> None: - safe_run_id = _validate_reconcile_run_id(run_id) - if not isinstance(result, BotReconcileResult): - raise TypeError("result must be a BotReconcileResult") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - run_row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (safe_run_id,), - ).fetchone() - if run_row is None: - raise KeyError(f"unknown reconciliation run: {safe_run_id}") - if run_row["outcome"] != "running": - raise RuntimeError("reconciliation run is not running") - current_run = self._materialize_reconcile_run(conn, safe_run_id) - if current_run is None: - raise sqlite3.IntegrityError("reconciliation run disappeared") - if current_run.scope == "bot" and result.bot_id != current_run.requested_bot_id: - raise ValueError("reconciliation result must match the requested bot") - if result.started_at < current_run.started_at: - raise ValueError("reconciliation result starts before its run") - if result.event_id is not None: - event_row = conn.execute( - "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", - (result.event_id,), - ).fetchone() - if event_row is None or event_row["bot_id"] != result.bot_id: - raise ValueError( - "reconciliation lifecycle event must exist and match the result bot" - ) - conn.execute( - """ - INSERT INTO reconcile_results ( - run_id, - bot_id, - ordinal, - outcome, - desired_state, - observed_status, - pid, - action, - message, - error_code, - event_id, - started_at, - finished_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - safe_run_id, - result.bot_id, - current_run.total, - result.outcome.value, - result.desired_state, - result.observed_status, - result.pid, - result.action, - result.message, - result.error_code, - result.event_id, - _serialize_reconcile_timestamp( - result.started_at, - "result start timestamp", - ), - _serialize_reconcile_timestamp( - result.finished_at, - "result finish timestamp", - ), - ), - ) - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET total = total + 1, - healthy_count = healthy_count + ?, - changed_count = changed_count + ?, - pending_count = pending_count + ?, - action_required_count = action_required_count + ?, - error_count = error_count + ?, - skipped_count = skipped_count + ? - WHERE run_id = ? AND outcome = 'running' - """, - ( - int(result.outcome is ReconcileOutcome.healthy), - int(result.outcome is ReconcileOutcome.changed), - int(result.outcome is ReconcileOutcome.pending), - int(result.outcome is ReconcileOutcome.action_required), - int(result.outcome is ReconcileOutcome.error), - int(result.outcome is ReconcileOutcome.skipped), - safe_run_id, - ), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation result counter update failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() + self._reconcile.append_reconcile_result(run_id, result) def finish_reconcile_run( self, summary: ReconcileRunSummary, ) -> PersistedReconcileRun: - if not isinstance(summary, ReconcileRunSummary): - raise TypeError("summary must be a ReconcileRunSummary") - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - run_row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (summary.run_id,), - ).fetchone() - if run_row is None: - raise KeyError(f"unknown reconciliation run: {summary.run_id}") - if run_row["outcome"] != "running": - raise RuntimeError("reconciliation run is not running") - if run_row["scope"] != summary.scope: - raise RuntimeError("reconciliation summary scope does not match run") - stored_started_at = _parse_reconcile_timestamp( - run_row["started_at"], - "run start timestamp", - ) - if stored_started_at != summary.started_at: - raise RuntimeError("reconciliation summary start timestamp does not match run") - persisted_results = self._reconcile_results_in_transaction(conn, summary.run_id) - if tuple(summary.results) != persisted_results: - raise RuntimeError("persisted reconciliation results do not match summary") - observed_counts = self._reconcile_counts_from_results(persisted_results) - stored_counts = self._reconcile_counts_from_run_row(run_row) - stored_total = _validate_stored_nonnegative_integer( - run_row["total"], - "total", - ) - if stored_counts != observed_counts or stored_total != len(persisted_results): - raise RuntimeError("persisted reconciliation counters do not match results") - if dict(summary.counts) != observed_counts or summary.total != len( - persisted_results - ): - raise RuntimeError("reconciliation summary counters do not match results") - if any(result.finished_at > summary.finished_at for result in persisted_results): - raise RuntimeError("reconciliation summary finishes before a persisted result") - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET finished_at = ?, - outcome = ?, - total = ?, - healthy_count = ?, - changed_count = ?, - pending_count = ?, - action_required_count = ?, - error_count = ?, - skipped_count = ? - WHERE run_id = ? AND outcome = 'running' - """, - ( - _serialize_reconcile_timestamp( - summary.finished_at, - "finish timestamp", - ), - summary.outcome, - summary.total, - summary.counts[ReconcileOutcome.healthy.value], - summary.counts[ReconcileOutcome.changed.value], - summary.counts[ReconcileOutcome.pending.value], - summary.counts[ReconcileOutcome.action_required.value], - summary.counts[ReconcileOutcome.error.value], - summary.counts[ReconcileOutcome.skipped.value], - summary.run_id, - ), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation run finalization failed") - finished = self._materialize_reconcile_run(conn, summary.run_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - if finished is None: - raise sqlite3.IntegrityError("finished reconciliation run is missing") - return finished + return self._reconcile.finish_reconcile_run(summary) def interrupt_stale_reconcile_runs(self, *, interrupted_at: datetime) -> int: - safe_interrupted_at = _serialize_reconcile_timestamp( - interrupted_at, - "interruption timestamp", - ) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - rows = conn.execute( - "SELECT * FROM reconcile_runs WHERE outcome = 'running' ORDER BY run_id" - ).fetchall() - for row in rows: - run_id = str(row["run_id"]) - current_run = self._materialize_reconcile_run(conn, run_id) - if current_run is None: - raise sqlite3.IntegrityError("reconciliation run disappeared") - if interrupted_at < current_run.started_at: - raise ValueError("interruption timestamp precedes a running reconciliation") - if any(result.finished_at > interrupted_at for result in current_run.results): - raise ValueError( - "reconciliation result finishes after the interruption timestamp" - ) - cursor = conn.execute( - """ - UPDATE reconcile_runs - SET outcome = 'interrupted', finished_at = ? - WHERE run_id = ? AND outcome = 'running' - """, - (safe_interrupted_at, run_id), - ) - if cursor.rowcount != 1: - raise sqlite3.IntegrityError("reconciliation interruption update failed") - except Exception: - conn.rollback() - raise - else: - conn.commit() - return len(rows) + return self._reconcile.interrupt_stale_reconcile_runs(interrupted_at=interrupted_at) def get_reconcile_run(self, run_id: str) -> PersistedReconcileRun | None: - safe_run_id = _validate_reconcile_run_id(run_id) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN") - run = self._materialize_reconcile_run(conn, safe_run_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - return run - - def _materialize_reconcile_run( - self, - conn: sqlite3.Connection, - run_id: str, - ) -> PersistedReconcileRun | None: - row = conn.execute( - "SELECT * FROM reconcile_runs WHERE run_id = ?", - (run_id,), - ).fetchone() - if row is None: - return None - results = self._reconcile_results_in_transaction(conn, run_id) - finished_at = row["finished_at"] - return PersistedReconcileRun( - run_id=str(row["run_id"]), - scope=str(row["scope"]), - requested_bot_id=( - str(row["requested_bot_id"]) if row["requested_bot_id"] is not None else None - ), - source=str(row["source"]), - force=_validate_stored_boolean_flag(row["force"], "force flag"), - reset_restart=_validate_stored_boolean_flag( - row["reset_restart"], - "reset-restart flag", - ), - started_at=_parse_reconcile_timestamp(row["started_at"], "run start timestamp"), - finished_at=( - _parse_reconcile_timestamp(finished_at, "run finish timestamp") - if finished_at is not None - else None - ), - outcome=str(row["outcome"]), - total=_validate_stored_nonnegative_integer(row["total"], "total"), - counts=self._reconcile_counts_from_run_row(row), - results=results, - ) - - def _reconcile_results_in_transaction( - self, - conn: sqlite3.Connection, - run_id: str, - ) -> tuple[BotReconcileResult, ...]: - rows = conn.execute( - "SELECT * FROM reconcile_results WHERE run_id = ? ORDER BY ordinal", - (run_id,), - ).fetchall() - results: list[BotReconcileResult] = [] - for expected_ordinal, row in enumerate(rows): - ordinal = _validate_stored_nonnegative_integer(row["ordinal"], "result ordinal") - if ordinal != expected_ordinal: - raise ValueError("stored reconciliation ordinals are not contiguous") - bot_id = str(row["bot_id"]) - event_id = _validate_stored_optional_positive_integer( - row["event_id"], - "result event id", - ) - if event_id is not None: - event_row = conn.execute( - "SELECT bot_id FROM lifecycle_events WHERE event_id = ?", - (event_id,), - ).fetchone() - if event_row is None or event_row["bot_id"] != bot_id: - raise ValueError( - "stored reconciliation lifecycle event does not match the result bot" - ) - results.append( - BotReconcileResult( - bot_id=bot_id, - outcome=ReconcileOutcome(str(row["outcome"])), - desired_state=( - str(row["desired_state"]) if row["desired_state"] is not None else None - ), - observed_status=( - str(row["observed_status"]) if row["observed_status"] is not None else None - ), - pid=_validate_stored_optional_positive_integer(row["pid"], "result pid"), - action=str(row["action"]), - message=str(row["message"]), - error_code=(str(row["error_code"]) if row["error_code"] is not None else None), - event_id=event_id, - started_at=_parse_reconcile_timestamp( - row["started_at"], - "result start timestamp", - ), - finished_at=_parse_reconcile_timestamp( - row["finished_at"], - "result finish timestamp", - ), - ) - ) - return tuple(results) - - def _reconcile_counts_from_run_row(self, row: sqlite3.Row) -> dict[str, int]: - return { - outcome.value: _validate_stored_nonnegative_integer( - row[column], - f"{outcome.value} counter", - ) - for outcome, column in RECONCILE_COUNTER_COLUMNS.items() - } - - def _reconcile_counts_from_results( - self, - results: tuple[BotReconcileResult, ...], - ) -> dict[str, int]: - counts = {outcome.value: 0 for outcome in ReconcileOutcome} - for result in results: - counts[result.outcome.value] += 1 - return counts + return self._reconcile.get_reconcile_run(run_id) def begin_lifecycle_intent( self, From 717771ea49eec84a47c99bd3098168d1c9af0b54 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:53:11 +0200 Subject: [PATCH 34/49] refactor(state): isolate bot lifecycle persistence --- tests/test_bot_lifecycle_store.py | 1262 +++++++++++++++++++++++++++++ tests/test_crash_recovery.py | 9 +- tests/test_lifecycle_ledger.py | 13 +- tests/test_supervisor_cli_api.py | 4 +- zeus/bot_lifecycle_store.py | 1174 +++++++++++++++++++++++++++ zeus/state.py | 1080 +++--------------------- 6 files changed, 2544 insertions(+), 998 deletions(-) create mode 100644 tests/test_bot_lifecycle_store.py create mode 100644 zeus/bot_lifecycle_store.py diff --git a/tests/test_bot_lifecycle_store.py b/tests/test_bot_lifecycle_store.py new file mode 100644 index 0000000..4ce50a7 --- /dev/null +++ b/tests/test_bot_lifecycle_store.py @@ -0,0 +1,1262 @@ +from __future__ import annotations + +import ast +import inspect +import json +import sqlite3 +import tempfile +import unittest +from collections.abc import Callable +from contextlib import closing +from dataclasses import FrozenInstanceError +from datetime import UTC, datetime, timedelta +from operator import methodcaller +from pathlib import Path +from unittest.mock import MagicMock, call, patch + +from zeus.bot_lifecycle_store import ( + LIFECYCLE_ERROR_CODE_RE, + LIFECYCLE_ID_RE, + LIFECYCLE_INTENT_ACTIONS, + LIFECYCLE_SOURCES, + BotLifecycleStore, +) +from zeus.idempotency_store import IdempotencyStore +from zeus.lifecycle import LifecycleEvent, LifecycleEventInput +from zeus.models import BotRecord, BotStatus +from zeus.reconcile_store import ReconcileStore +from zeus.sanitization import MAX_SANITIZED_JSON_BYTES +from zeus.schema import SchemaManager +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import LIFECYCLE_ERROR_CODE_RE as STATE_LIFECYCLE_ERROR_CODE_RE +from zeus.state import LIFECYCLE_ID_RE as STATE_LIFECYCLE_ID_RE +from zeus.state import LIFECYCLE_INTENT_ACTIONS as STATE_LIFECYCLE_INTENT_ACTIONS +from zeus.state import LIFECYCLE_SOURCES as STATE_LIFECYCLE_SOURCES +from zeus.state import StateStore + +FIXED_NOW = datetime(2026, 7, 21, 12, 34, 56, tzinfo=UTC) +NEXT_RESTART = FIXED_NOW + timedelta(minutes=5) + +PUBLIC_METHODS = ( + "begin_lifecycle_intent", + "complete_lifecycle_intent", + "clear_stale_intent", + "audit_log_path", + "append_audit_event", + "upsert_bot", + "upsert_bot_with_event", + "get_bot", + "list_bots", + "update_status", + "update_lifecycle_state", + "update_lifecycle_with_event", + "update_restart_state", + "update_restart_with_event", + "delete_bot", + "delete_bot_with_event", + "list_lifecycle_events", + "history_payload", +) + + +class _TracingSQLiteDatabase(SQLiteDatabase): + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.traces: list[tuple[int, str]] = [] + self.connection_count = 0 + + def connect(self) -> sqlite3.Connection: + self.connection_count += 1 + conn = super().connect() + connection_serial = self.connection_count + conn.set_trace_callback(lambda sql: self.traces.append((connection_serial, sql))) + return conn + + +class _CountingSQLiteDatabase(SQLiteDatabase): + def __init__(self, database_path: Path | str) -> None: + super().__init__(database_path) + self.connection_count = 0 + + def connect(self) -> sqlite3.Connection: + self.connection_count += 1 + return super().connect() + + +def _record(root: Path, *, bot_id: str = "coder") -> BotRecord: + return BotRecord( + bot_id=bot_id, + template_id="coding-bot", + display_name=bot_id.title(), + profile_path=str(root / "profiles" / bot_id), + ) + + +def _event( + action: str, + *, + bot_id: str = "coder", + operation_character: str = "a", +) -> LifecycleEventInput: + return LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_character * 32, + source="cli", + action=action, + outcome="success", + reason="test transition", + details={"kind": action}, + ) + + +def _control_statements(trace: list[tuple[int, str]]) -> list[str]: + controls = {"BEGIN", "COMMIT", "ROLLBACK"} + return [ + sql.strip().upper() + for _connection_id, sql in trace + if sql.strip().upper().split(maxsplit=1)[0] in controls + ] + + +def _statements(trace: list[tuple[int, str]]) -> list[str]: + return [sql for _connection_id, sql in trace] + + +def _statement_index(statements: list[str], prefix: str, *, start: int = 0) -> int: + normalized_prefix = " ".join(prefix.split()).upper() + return next( + index + for index, statement in enumerate(statements[start:], start=start) + if " ".join(statement.split()).upper().startswith(normalized_prefix) + ) + + +def _table_bytes(database_path: Path, table: str, order_by: str) -> tuple[tuple[object, ...], ...]: + with closing(sqlite3.connect(database_path)) as conn: + columns = tuple(row[1] for row in conn.execute(f"PRAGMA table_info({table})")) + fields = ", ".join(f"typeof({column}), hex(CAST({column} AS BLOB))" for column in columns) + rows = conn.execute(f"SELECT {fields} FROM {table} ORDER BY {order_by}").fetchall() + return tuple(rows) + + +def _database_snapshot( + database_path: Path, +) -> tuple[tuple[tuple[object, ...], ...], tuple[tuple[object, ...], ...]]: + return ( + _table_bytes(database_path, "bots", "bot_id"), + _table_bytes(database_path, "lifecycle_events", "event_id"), + ) + + +def _install_event_rejection(database_path: Path) -> None: + with closing(sqlite3.connect(database_path)) as conn: + conn.execute( + """ + CREATE TRIGGER reject_test_lifecycle_event + BEFORE INSERT ON lifecycle_events + BEGIN + SELECT RAISE(ABORT, 'injected lifecycle event failure'); + END + """ + ) + conn.commit() + + +class BotLifecycleStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.root = Path(self.temporary_directory.name) + self.database_path = self.root / "zeus.db" + self.database = SQLiteDatabase(self.database_path) + SchemaManager(self.database).init() + self.store = BotLifecycleStore(self.database) + + def _new_tracing_store( + self, + name: str, + ) -> tuple[_TracingSQLiteDatabase, BotLifecycleStore, Path]: + root = self.root / name + database = _TracingSQLiteDatabase(root / "zeus.db") + SchemaManager(database).init() + database.traces.clear() + return database, BotLifecycleStore(database), root + + def _prepare_atomic( + self, + store: BotLifecycleStore, + root: Path, + operation: str, + ) -> None: + if operation != "create": + store.upsert_bot(_record(root)) + if operation in {"complete_intent", "clear_intent"}: + store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="b" * 32, + source="cli", + ) + + def _invoke_atomic( + self, + store: BotLifecycleStore, + root: Path, + operation: str, + ) -> object: + if operation == "create": + return store.upsert_bot_with_event( + _record(root), + event=_event("bot.create"), + ) + if operation == "lifecycle": + return store.update_lifecycle_with_event( + "coder", + BotStatus.running, + 4321, + event=_event("bot.start"), + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + ) + if operation == "restart": + return store.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event("bot.restart.schedule"), + ) + if operation == "delete": + return store.delete_bot_with_event( + "coder", + event=_event("bot.delete"), + ) + if operation == "begin_intent": + return store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + ) + if operation == "complete_intent": + return store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="b" * 32, + desired_revision=1, + status=BotStatus.running, + pid=4321, + source="cli", + ) + if operation == "clear_intent": + return store.clear_stale_intent( + "coder", + action="start", + operation_id="b" * 32, + desired_revision=1, + source="recovery", + reason="stale intent", + ) + raise AssertionError(f"unknown atomic operation: {operation}") + + def test_ownership_construction_and_compatibility_aliases(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + + BotLifecycleStore(SQLiteDatabase(database_path)) + + self.assertFalse(database_path.exists()) + + import zeus.bot_lifecycle_store as bot_lifecycle_store_module + + source = inspect.getsource(bot_lifecycle_store_module) + tree = ast.parse(source) + imported_modules = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + } + self.assertTrue( + imported_modules.isdisjoint( + { + "zeus.reconcile_store", + "zeus.reconciliation", + "zeus.schema", + "zeus.state", + } + ) + ) + self.assertNotIn("CREATE TABLE", source.upper()) + self.assertNotIn("CREATE TRIGGER", source.upper()) + self.assertIs(LIFECYCLE_ID_RE, STATE_LIFECYCLE_ID_RE) + self.assertIs(LIFECYCLE_ERROR_CODE_RE, STATE_LIFECYCLE_ERROR_CODE_RE) + self.assertIs(LIFECYCLE_SOURCES, STATE_LIFECYCLE_SOURCES) + self.assertIs(LIFECYCLE_INTENT_ACTIONS, STATE_LIFECYCLE_INTENT_ACTIONS) + + def test_direct_store_round_trip_covers_every_public_family(self) -> None: + coder = _record(self.root) + self.store.upsert_bot(coder) + self.assertEqual(coder, self.store.get_bot("coder")) + self.assertEqual([coder], self.store.list_bots()) + + self.store.update_status("coder", BotStatus.starting, 1111, reset_restart=True) + self.store.update_lifecycle_state( + "coder", + BotStatus.running, + 2222, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + last_transition_reason="ready", + ) + self.store.update_restart_state( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + ) + loaded = self.store.get_bot("coder") + assert loaded is not None + self.assertEqual(BotStatus.failed, loaded.status) + self.assertEqual(2, loaded.restart_attempts) + self.assertEqual(NEXT_RESTART, loaded.next_restart_at) + + begun = self.store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + ) + self.assertEqual(1, begun.desired_revision) + completed = self.store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=3333, + source="cli", + ) + self.assertTrue(completed.converged) + restarted = self.store.begin_lifecycle_intent( + "coder", + action="restart", + operation_id="b" * 32, + source="cli", + ) + cleared = self.store.clear_stale_intent( + "coder", + action="restart", + operation_id="b" * 32, + desired_revision=restarted.desired_revision, + source="recovery", + reason="stale intent", + ) + self.assertIsNone(cleared.pending_operation_id) + + evented = _record(self.root, bot_id="evented") + created = self.store.upsert_bot_with_event( + evented, + event=_event("bot.create", bot_id="evented", operation_character="c"), + ) + transitioned = self.store.update_lifecycle_with_event( + "evented", + BotStatus.running, + 4444, + event=_event("bot.start", bot_id="evented", operation_character="d"), + ) + scheduled = self.store.update_restart_with_event( + "evented", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event( + "bot.restart.schedule", + bot_id="evented", + operation_character="e", + ), + ) + deleted = self.store.delete_bot_with_event( + "evented", + event=_event("bot.delete", bot_id="evented", operation_character="f"), + ) + self.assertEqual( + ["bot.delete", "bot.restart.schedule", "bot.start", "bot.create"], + [event.action for event in self.store.list_lifecycle_events("evented", 10, None)], + ) + self.assertIsInstance(created, LifecycleEvent) + self.assertIsInstance(transitioned, LifecycleEvent) + self.assertIsInstance(scheduled, LifecycleEvent) + self.assertIs(deleted, True) + self.assertEqual("evented", self.store.history_payload("evented", 2, None)["bot_id"]) + self.assertEqual(self.root / "logs" / "audit.jsonl", self.store.audit_log_path()) + + self.store.update_status("missing", BotStatus.running, 9999) + with self.assertRaises(KeyError): + self.store.update_lifecycle_state("missing", BotStatus.failed) + with self.assertRaises(KeyError): + self.store.update_restart_state( + "missing", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=None, + ) + self.assertIs(self.store.delete_bot("missing"), False) + with self.assertRaises(KeyError): + self.store.delete_bot_with_event( + "missing", + event=_event("bot.delete", bot_id="missing"), + ) + self.assertIs(self.store.delete_bot("coder"), True) + + def test_invalid_inputs_are_rejected_before_connecting(self) -> None: + database = _CountingSQLiteDatabase(self.root / "validation.db") + store = BotLifecycleStore(database) + record = _record(self.root) + + invalid_calls = ( + lambda: store.begin_lifecycle_intent( + "coder", + action="delete", + operation_id="a" * 32, + source="cli", + ), + lambda: store.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="api", + ), + lambda: store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=0, + status=BotStatus.running, + pid=1, + source="cli", + ), + lambda: store.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.stopped, + pid=None, + source="cli", + ), + lambda: store.upsert_bot_with_event( + record, + event=_event("bot.create", bot_id="other"), + ), + lambda: store.list_lifecycle_events("coder", 0, None), + lambda: store.history_payload("coder", 10, 0), + ) + for invalid_call in invalid_calls: + with self.subTest(call=invalid_call), self.assertRaises((TypeError, ValueError)): + invalid_call() + self.assertEqual(0, database.connection_count) + + def test_all_seven_atomic_paths_have_one_immediate_transaction_and_required_order(self) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + pre_event_prefixes = { + "create": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "INSERT INTO BOTS", + ), + "lifecycle": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + ), + "restart": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + ), + "delete": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "DELETE FROM BOTS", + ), + "begin_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET DESIRED_STATE", + ), + "complete_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET STATUS", + "UPDATE BOTS SET PENDING_OPERATION_ID", + ), + "clear_intent": ( + "SELECT * FROM BOTS WHERE BOT_ID", + "UPDATE BOTS SET PENDING_OPERATION_ID", + ), + } + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"atomic-{operation}") + self._prepare_atomic(store, root, operation) + database.traces.clear() + database.connection_count = 0 + + self._invoke_atomic(store, root, operation) + trace = list(database.traces) + statements = _statements(trace) + + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual(["BEGIN IMMEDIATE", "COMMIT"], _control_statements(trace)) + begin = _statement_index(statements, "BEGIN IMMEDIATE") + ordered_indices: list[int] = [] + search_start = begin + 1 + for prefix in pre_event_prefixes[operation]: + index = _statement_index(statements, prefix, start=search_start) + ordered_indices.append(index) + search_start = index + 1 + event_insert = _statement_index( + statements, + "INSERT INTO LIFECYCLE_EVENTS", + start=search_start, + ) + event_select = _statement_index( + statements, + "SELECT * FROM LIFECYCLE_EVENTS WHERE EVENT_ID", + start=event_insert, + ) + commit = _statement_index(statements, "COMMIT") + self.assertEqual(sorted(ordered_indices), ordered_indices) + self.assertLess(ordered_indices[-1], event_insert) + self.assertLess(event_insert, event_select) + self.assertLess(event_select, commit) + last_event_updates = [ + index + for index, sql in enumerate(statements) + if sql.lstrip().upper().startswith("UPDATE BOTS SET LAST_EVENT_ID") + ] + if operation == "delete": + self.assertEqual([], last_event_updates) + else: + self.assertTrue(last_event_updates) + self.assertLess(event_insert, last_event_updates[0]) + self.assertLess(last_event_updates[0], event_select) + if operation.endswith("intent"): + projection_selects = [ + index + for index, sql in enumerate(statements) + if sql.lstrip().upper().startswith("SELECT * FROM BOTS WHERE BOT_ID") + ] + self.assertGreaterEqual(len(projection_selects), 2) + self.assertLess(event_select, projection_selects[-1]) + self.assertLess(projection_selects[-1], commit) + + def test_plain_writes_keep_implicit_transactions_and_reads_remain_transaction_free( + self, + ) -> None: + write_operations = ( + "upsert", + "status", + "lifecycle", + "restart", + "delete", + ) + for operation in write_operations: + with self.subTest(write=operation): + database, store, root = self._new_tracing_store(f"plain-{operation}") + if operation != "upsert": + store.upsert_bot(_record(root)) + database.traces.clear() + database.connection_count = 0 + + if operation == "upsert": + store.upsert_bot(_record(root)) + elif operation == "status": + store.update_status("coder", BotStatus.starting, 1001) + elif operation == "lifecycle": + store.update_lifecycle_state( + "coder", + BotStatus.running, + 1002, + started_at=FIXED_NOW, + ) + elif operation == "restart": + store.update_restart_state( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=3, + next_restart_at=NEXT_RESTART, + ) + else: + self.assertIs(store.delete_bot("coder"), True) + + trace = list(database.traces) + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual(["BEGIN", "COMMIT"], _control_statements(trace)) + self.assertNotIn("BEGIN IMMEDIATE", _control_statements(trace)) + + database, store, root = self._new_tracing_store("reads") + store.upsert_bot_with_event(_record(root), event=_event("bot.create")) + read_operations = ( + lambda: store.get_bot("coder"), + store.list_bots, + lambda: store.list_lifecycle_events("coder", 10, None), + lambda: store.history_payload("coder", 10, None), + ) + for read_operation in read_operations: + with self.subTest(read=read_operation): + database.traces.clear() + database.connection_count = 0 + read_operation() + trace = list(database.traces) + self.assertEqual(1, database.connection_count) + self.assertEqual(1, len({connection_id for connection_id, _sql in trace})) + self.assertEqual([], _control_statements(trace)) + self.assertTrue( + all(sql.lstrip().upper().startswith("SELECT") for _identity, sql in trace) + ) + database.traces.clear() + database.connection_count = 0 + self.assertEqual(root / "logs" / "audit.jsonl", store.audit_log_path()) + self.assertEqual([], database.traces) + self.assertEqual(0, database.connection_count) + + def test_sql_trigger_rolls_back_all_seven_atomic_paths_byte_for_byte(self) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"rollback-{operation}") + self._prepare_atomic(store, root, operation) + _install_event_rejection(database.database_path) + before = _database_snapshot(database.database_path) + database.traces.clear() + database.connection_count = 0 + + with self.assertRaisesRegex( + sqlite3.DatabaseError, + "injected lifecycle event failure", + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual( + ["BEGIN IMMEDIATE", "ROLLBACK"], + _control_statements(database.traces), + ) + self.assertEqual(1, database.connection_count) + self.assertEqual(before, _database_snapshot(database.database_path)) + + def test_late_materialization_failures_roll_back_event_and_projection(self) -> None: + for operation in ("create", "lifecycle", "restart", "delete"): + with self.subTest(event=operation): + database, store, root = self._new_tracing_store(f"late-event-{operation}") + self._prepare_atomic(store, root, operation) + before = _database_snapshot(database.database_path) + + with ( + patch.object( + store, + "_row_to_lifecycle_event", + side_effect=sqlite3.DatabaseError("event materialization failed"), + ), + self.assertRaisesRegex(sqlite3.DatabaseError, "event materialization failed"), + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual(before, _database_snapshot(database.database_path)) + + for operation in ("begin_intent", "complete_intent", "clear_intent"): + with self.subTest(record=operation): + database, store, root = self._new_tracing_store(f"late-record-{operation}") + self._prepare_atomic(store, root, operation) + before = _database_snapshot(database.database_path) + + with ( + patch.object( + store, + "_row_to_record", + side_effect=sqlite3.DatabaseError("record materialization failed"), + ), + self.assertRaisesRegex(sqlite3.DatabaseError, "record materialization failed"), + ): + self._invoke_atomic(store, root, operation) + + self.assertEqual(before, _database_snapshot(database.database_path)) + + def test_audit_callback_observes_commit_and_failure_is_fail_open_for_every_atomic_path( + self, + ) -> None: + operations = ( + "create", + "lifecycle", + "restart", + "delete", + "begin_intent", + "complete_intent", + "clear_intent", + ) + expected_projection = { + "create": ("stopped", "stopped", None, None, 0), + "lifecycle": ("running", "stopped", None, None, 0), + "restart": ("failed", "stopped", None, None, 1), + "delete": None, + "begin_intent": ("stopped", "running", "a" * 32, "start", 0), + "complete_intent": ("running", "running", None, None, 0), + "clear_intent": ("stopped", "running", None, None, 0), + } + for operation in operations: + with self.subTest(operation=operation): + database, store, root = self._new_tracing_store(f"audit-order-{operation}") + self._prepare_atomic(store, root, operation) + before_events = len(_database_snapshot(database.database_path)[1]) + database.traces.clear() + database.connection_count = 0 + callback_observations: list[tuple[list[str], int, tuple[object, ...] | None]] = [] + + def observe_commit( + _event: LifecycleEvent, + *, + _database: _TracingSQLiteDatabase = database, + _observations: list[ + tuple[list[str], int, tuple[object, ...] | None] + ] = callback_observations, + ) -> None: + with closing(sqlite3.connect(_database.database_path)) as conn: + event_count = conn.execute( + "SELECT COUNT(*) FROM lifecycle_events" + ).fetchone()[0] + projection_row = conn.execute( + """ + SELECT status, + desired_state, + pending_operation_id, + pending_action, + restart_attempts + FROM bots + WHERE bot_id = 'coder' + """ + ).fetchone() + projection = tuple(projection_row) if projection_row is not None else None + _observations.append( + (_control_statements(_database.traces), event_count, projection) + ) + raise RuntimeError("audit callback failed") + + with patch.object(store, "_append_lifecycle_audit", side_effect=observe_commit): + result = self._invoke_atomic(store, root, operation) + + self.assertEqual( + [ + ( + ["BEGIN IMMEDIATE", "COMMIT"], + before_events + 1, + expected_projection[operation], + ) + ], + callback_observations, + ) + self.assertEqual(1, database.connection_count) + self.assertEqual( + before_events + 1, len(_database_snapshot(database.database_path)[1]) + ) + if operation == "delete": + self.assertIs(result, True) + + def test_history_cursor_deleted_bot_and_event_immutability(self) -> None: + self.store.upsert_bot_with_event( + _record(self.root), event=_event("bot.create", operation_character="a") + ) + self.store.update_lifecycle_with_event( + "coder", + BotStatus.running, + 4321, + event=_event("bot.start", operation_character="b"), + ) + self.store.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=1, + next_restart_at=NEXT_RESTART, + event=_event("bot.restart.schedule", operation_character="c"), + ) + self.store.delete_bot_with_event( + "coder", + event=_event("bot.delete", operation_character="d"), + ) + + first_page = self.store.list_lifecycle_events("coder", 2, None) + second_page = self.store.list_lifecycle_events( + "coder", + 2, + first_page[-1].event_id, + ) + self.assertEqual(["bot.delete", "bot.restart.schedule"], [e.action for e in first_page]) + self.assertEqual(["bot.start", "bot.create"], [e.action for e in second_page]) + self.assertTrue( + {event.event_id for event in first_page}.isdisjoint( + event.event_id for event in second_page + ) + ) + payload = self.store.history_payload("coder", 2, None) + self.assertIsNotNone(payload["next_before"]) + self.assertEqual("coder", payload["bot_id"]) + self.assertEqual([], self.store.list_lifecycle_events("unknown", 10, None)) + with self.assertRaisesRegex(KeyError, "unknown bot"): + self.store.history_payload("unknown", 10, None) + + event = first_page[0] + with self.assertRaises(FrozenInstanceError): + event.action = "mutated" # type: ignore[misc] + with self.assertRaises(TypeError): + event.details["kind"] = "mutated" # type: ignore[index] + before = _database_snapshot(self.database_path) + for statement in ( + "UPDATE lifecycle_events SET action = 'mutated' WHERE event_id = ?", + "DELETE FROM lifecycle_events WHERE event_id = ?", + ): + with ( + self.subTest(statement=statement), + closing(sqlite3.connect(self.database_path)) as conn, + ): + with self.assertRaisesRegex(sqlite3.DatabaseError, "immutable"): + conn.execute(statement, (event.event_id,)) + conn.rollback() + self.assertEqual(before, _database_snapshot(self.database_path)) + + def test_audit_exact_bytes_mapping_bounds_hostile_input_and_symlink_fail_open(self) -> None: + exact_root = (self.root / "exact").resolve() + store = BotLifecycleStore(SQLiteDatabase(exact_root / "zeus.db")) + with ( + patch("zeus.bot_lifecycle_store.datetime") as clock, + patch("zeus.bot_lifecycle_store.append_private_bytes") as append, + ): + clock.now.return_value = FIXED_NOW + store.append_audit_event("bot.test", label="Δ", api_key="secret") + append.assert_called_once_with( + exact_root / "logs" / "audit.jsonl", + b'{"api_key": "[redacted]", "event": "bot.test", "label": "\\u0394", ' + b'"ts": "2026-07-21T12:34:56+00:00"}\n', + ) + + lifecycle_event = LifecycleEvent( + event_id=7, + bot_id="coder", + operation_id="a" * 32, + request_id=None, + occurred_at=FIXED_NOW, + source="cli", + action="bot.start", + outcome="success", + status_before="stopped", + status_after="running", + pid_before=None, + pid_after=4321, + reason="operator request", + error_code=None, + error_message=None, + details={"phase": "ready"}, + ) + with patch.object(store, "append_audit_event") as append_event: + store._append_lifecycle_audit(lifecycle_event) + append_event.assert_called_once_with( + "bot.start", + bot_id="coder", + operation_id="a" * 32, + request_id=None, + source="cli", + outcome="success", + status_before="stopped", + status_after="running", + pid_before=None, + pid_after=4321, + reason="operator request", + error_code=None, + error_message=None, + details=lifecycle_event.details, + ) + + class Hostile: + def __str__(self) -> str: + raise AssertionError("string conversion invoked") + + def __repr__(self) -> str: + raise AssertionError("representation invoked") + + bounded = BotLifecycleStore(SQLiteDatabase(self.root / "bounded" / "zeus.db")) + secret = "audit-sentinel" + bounded.append_audit_event( + "bot.hostile", + api_key=secret, + nonfinite=float("nan"), + hostile=Hostile(), + ) + bounded.append_audit_event(chr(0x1F512) * 2_048, message="ordinary") + lines = bounded.audit_log_path().read_bytes().splitlines() + self.assertEqual("[redacted]", json.loads(lines[0])["api_key"]) + self.assertIsNone(json.loads(lines[0])["nonfinite"]) + self.assertEqual("[unsupported]", json.loads(lines[0])["hostile"]) + self.assertNotIn(secret.encode(), b"\n".join(lines)) + self.assertNotIn(b"NaN", b"\n".join(lines)) + self.assertTrue(all(len(line) <= MAX_SANITIZED_JSON_BYTES for line in lines)) + self.assertTrue(json.loads(lines[1])["truncated"]) + + symlink_root = self.root / "symlink" + state_dir = symlink_root / "state" + state_dir.mkdir(parents=True) + external = symlink_root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("unchanged\n", encoding="utf-8") + mode_before = external.stat().st_mode + (state_dir / "logs").symlink_to(external, target_is_directory=True) + unsafe = BotLifecycleStore(SQLiteDatabase(state_dir / "zeus.db")) + + unsafe.append_audit_event("bot.test", message="fail open") + + self.assertEqual("unchanged\n", sentinel.read_text(encoding="utf-8")) + self.assertEqual(mode_before, external.stat().st_mode) + self.assertFalse((external / "audit.jsonl").exists()) + + def test_state_facade_forwards_all_eighteen_methods_with_exact_signatures(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + database.database_path = database_path + schema = MagicMock(spec=SchemaManager) + idempotency = MagicMock(spec=IdempotencyStore) + reconcile = MagicMock(spec=ReconcileStore) + delegate = MagicMock(spec=BotLifecycleStore) + record = _record(Path("profiles")) + event_input = _event("bot.test") + event = MagicMock(spec=LifecycleEvent) + history: dict[str, object] = { + "bot_id": "coder", + "events": [], + "next_before": None, + } + audit_path = Path("state") / "logs" / "audit.jsonl" + delegate.begin_lifecycle_intent.return_value = record + delegate.complete_lifecycle_intent.return_value = record + delegate.clear_stale_intent.return_value = record + delegate.audit_log_path.return_value = audit_path + void_sentinel = object() + delegate.append_audit_event.return_value = void_sentinel + delegate.upsert_bot.return_value = void_sentinel + delegate.update_status.return_value = void_sentinel + delegate.update_lifecycle_state.return_value = void_sentinel + delegate.update_restart_state.return_value = void_sentinel + delegate.upsert_bot_with_event.return_value = event + delegate.get_bot.return_value = record + delegate.list_bots.return_value = [record] + delegate.update_lifecycle_with_event.return_value = event + delegate.update_restart_with_event.return_value = event + delegate.delete_bot.return_value = True + delegate.delete_bot_with_event.return_value = True + delegate.list_lifecycle_events.return_value = [event] + delegate.history_payload.return_value = history + + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager", return_value=schema) as schema_type, + patch("zeus.state.IdempotencyStore", return_value=idempotency) as idempotency_type, + patch("zeus.state.ReconcileStore", return_value=reconcile) as reconcile_type, + patch("zeus.state.BotLifecycleStore", return_value=delegate) as lifecycle_type, + ): + facade = StateStore(database_path) + actual_begin = facade.begin_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + request_id=None, + reason="begin", + ) + actual_complete = facade.complete_lifecycle_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=1234, + source="cli", + outcome="success", + request_id=None, + reason="complete", + error_code=None, + error_message=None, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=None, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + actual_clear = facade.clear_stale_intent( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + source="recovery", + reason="clear", + request_id=None, + ) + actual_audit_path = facade.audit_log_path() + actual_audit = methodcaller("append_audit_event", "bot.test", bot_id="coder")(facade) + actual_upsert = methodcaller("upsert_bot", record)(facade) + actual_created = facade.upsert_bot_with_event(record, event=event_input) + actual_record = facade.get_bot("coder") + actual_records = facade.list_bots() + actual_status = methodcaller( + "update_status", + "coder", + BotStatus.running, + 1234, + reset_restart=True, + )(facade) + actual_lifecycle_state = methodcaller( + "update_lifecycle_state", + "coder", + BotStatus.running, + 1234, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=0, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + )(facade) + actual_lifecycle = facade.update_lifecycle_with_event( + "coder", + BotStatus.running, + 1234, + event=event_input, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=0, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + actual_restart_state = methodcaller( + "update_restart_state", + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + )(facade) + actual_restart = facade.update_restart_with_event( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + event=event_input, + ) + actual_delete = facade.delete_bot("coder") + actual_delete_event = facade.delete_bot_with_event("coder", event=event_input) + actual_events = facade.list_lifecycle_events("coder", 10, 20) + actual_history = facade.history_payload("coder", 10, 20) + + self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual([call(database)], schema_type.call_args_list) + self.assertEqual([call(database)], idempotency_type.call_args_list) + self.assertEqual([call(database)], reconcile_type.call_args_list) + self.assertEqual([call(database)], lifecycle_type.call_args_list) + database.connect.assert_not_called() + self.assertIs(database_path, facade.database_path) + self.assertIs(record, actual_begin) + self.assertIs(record, actual_complete) + self.assertIs(record, actual_clear) + self.assertIs(audit_path, actual_audit_path) + self.assertIsNone(actual_audit) + self.assertIsNone(actual_upsert) + self.assertIs(event, actual_created) + self.assertIs(record, actual_record) + self.assertIs(delegate.list_bots.return_value, actual_records) + self.assertIsNone(actual_status) + self.assertIsNone(actual_lifecycle_state) + self.assertIs(event, actual_lifecycle) + self.assertIsNone(actual_restart_state) + self.assertIs(event, actual_restart) + self.assertIs(actual_delete, True) + self.assertIs(actual_delete_event, True) + self.assertIs(delegate.list_lifecycle_events.return_value, actual_events) + self.assertIs(history, actual_history) + + delegate.begin_lifecycle_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + source="cli", + request_id=None, + reason="begin", + ) + delegate.complete_lifecycle_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + status=BotStatus.running, + pid=1234, + source="cli", + outcome="success", + request_id=None, + reason="complete", + error_code=None, + error_message=None, + started_at=FIXED_NOW, + ready_at=FIXED_NOW, + stopped_at=None, + last_exit_code=None, + last_error=None, + last_transition_reason="ready", + reset_restart=True, + clear_ready_at=False, + clear_stopped_at=True, + ) + delegate.clear_stale_intent.assert_called_once_with( + "coder", + action="start", + operation_id="a" * 32, + desired_revision=1, + source="recovery", + reason="clear", + request_id=None, + ) + delegate.audit_log_path.assert_called_once_with() + delegate.append_audit_event.assert_called_once_with("bot.test", bot_id="coder") + delegate.upsert_bot.assert_called_once_with(record) + delegate.upsert_bot_with_event.assert_called_once_with(record, event=event_input) + delegate.get_bot.assert_called_once_with("coder") + delegate.list_bots.assert_called_once_with() + delegate.update_status.assert_called_once_with( + "coder", BotStatus.running, 1234, reset_restart=True + ) + lifecycle_kwargs = { + "started_at": FIXED_NOW, + "ready_at": FIXED_NOW, + "stopped_at": None, + "last_exit_code": 0, + "last_error": None, + "last_transition_reason": "ready", + "reset_restart": True, + "clear_ready_at": False, + "clear_stopped_at": True, + } + delegate.update_lifecycle_state.assert_called_once_with( + "coder", BotStatus.running, 1234, **lifecycle_kwargs + ) + delegate.update_lifecycle_with_event.assert_called_once_with( + "coder", + BotStatus.running, + 1234, + event=event_input, + **lifecycle_kwargs, + ) + delegate.update_restart_state.assert_called_once_with( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + ) + delegate.update_restart_with_event.assert_called_once_with( + "coder", + status=BotStatus.failed, + pid=None, + restart_attempts=2, + next_restart_at=NEXT_RESTART, + event=event_input, + ) + delegate.delete_bot.assert_called_once_with("coder") + delegate.delete_bot_with_event.assert_called_once_with("coder", event=event_input) + delegate.list_lifecycle_events.assert_called_once_with("coder", 10, 20) + delegate.history_payload.assert_called_once_with("coder", 10, 20) + for method_name in PUBLIC_METHODS: + self.assertEqual( + inspect.signature(getattr(BotLifecycleStore, method_name)), + inspect.signature(getattr(StateStore, method_name)), + ) + + def assert_exception_identity( + method_name: str, + invoke: Callable[[], object], + ) -> None: + error = RuntimeError(f"facade sentinel: {method_name}") + method = getattr(delegate, method_name) + method.side_effect = error + try: + with self.assertRaises(RuntimeError) as caught: + invoke() + self.assertIs(error, caught.exception) + finally: + method.side_effect = None + + exception_cases: tuple[tuple[str, Callable[[], object]], ...] = ( + ( + "begin_lifecycle_intent", + lambda: facade.begin_lifecycle_intent( + "other", + action="start", + operation_id="c" * 32, + source="cli", + ), + ), + ("audit_log_path", facade.audit_log_path), + ( + "append_audit_event", + lambda: facade.append_audit_event("bot.test", bot_id="other"), + ), + ( + "upsert_bot_with_event", + lambda: facade.upsert_bot_with_event(record, event=event_input), + ), + ("get_bot", lambda: facade.get_bot("other")), + ("list_bots", facade.list_bots), + ( + "update_status", + lambda: facade.update_status("other", BotStatus.failed), + ), + ("delete_bot", lambda: facade.delete_bot("other")), + ( + "list_lifecycle_events", + lambda: facade.list_lifecycle_events("other", 10, None), + ), + ( + "history_payload", + lambda: facade.history_payload("other", 10, None), + ), + ) + for method_name, invoke in exception_cases: + with self.subTest(exception=method_name): + assert_exception_identity(method_name, invoke) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index eb959bb..9a3cb90 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -21,6 +21,7 @@ from unittest.mock import patch from zeus import models +from zeus.bot_lifecycle_store import BotLifecycleStore from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy from zeus.state import StateStore from zeus.supervisor import Supervisor @@ -1476,7 +1477,9 @@ def test_intent_event_failure_rolls_back_projection(self) -> None: with ( patch.object( - store, "_insert_lifecycle_event", side_effect=sqlite3.DatabaseError("boom") + BotLifecycleStore, + "_insert_lifecycle_event", + side_effect=sqlite3.DatabaseError("boom"), ), self.assertRaisesRegex(sqlite3.DatabaseError, "boom"), ): @@ -1504,7 +1507,9 @@ def test_intent_completion_and_clear_failures_roll_back_projection(self) -> None with ( patch.object( - store, "_insert_lifecycle_event", side_effect=sqlite3.DatabaseError("boom") + BotLifecycleStore, + "_insert_lifecycle_event", + side_effect=sqlite3.DatabaseError("boom"), ), self.assertRaisesRegex(sqlite3.DatabaseError, "boom"), ): diff --git a/tests/test_lifecycle_ledger.py b/tests/test_lifecycle_ledger.py index 547cd19..da3feff 100644 --- a/tests/test_lifecycle_ledger.py +++ b/tests/test_lifecycle_ledger.py @@ -10,6 +10,7 @@ from pathlib import Path from unittest.mock import patch +from zeus.bot_lifecycle_store import BotLifecycleStore from zeus.lifecycle import LifecycleEvent, LifecycleEventInput, serialize_lifecycle_details from zeus.models import BotRecord, BotStatus, DesiredState from zeus.reconciliation import BotReconcileResult, ReconcileOutcome, ReconcileRunStart @@ -834,7 +835,7 @@ def test_upsert_bot_with_event_rolls_back_projection_when_event_insert_fails(sel with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -854,7 +855,7 @@ def test_update_lifecycle_with_event_rolls_back_when_event_insert_fails(self) -> with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -882,7 +883,7 @@ def test_update_restart_with_event_rolls_back_projection_when_event_insert_fails with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -913,7 +914,7 @@ def test_delete_bot_with_event_rolls_back_projection_when_event_insert_fails(sel with ( patch.object( - store, + BotLifecycleStore, "_insert_lifecycle_event", side_effect=sqlite3.Error("boom"), ), @@ -935,7 +936,7 @@ def test_atomic_mutations_roll_back_when_event_materialization_fails(self) -> No with ( patch.object( - store, + BotLifecycleStore, "_row_to_lifecycle_event", side_effect=sqlite3.Error("materialization failed"), ), @@ -970,7 +971,7 @@ def test_atomic_mutations_ignore_post_commit_audit_failures(self) -> None: store.upsert_bot(self._record(root)) with patch.object( - store, + BotLifecycleStore, "_append_lifecycle_audit", side_effect=RuntimeError("audit failed"), ): diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index 84eda6e..e1081ca 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -3628,8 +3628,8 @@ def test_audit_append_delegates_exact_utf8_json_line(self) -> None: fixed_now = datetime(2026, 7, 21, 12, 34, 56, tzinfo=UTC) with ( - patch("zeus.state.datetime") as clock, - patch("zeus.state.append_private_bytes") as append, + patch("zeus.bot_lifecycle_store.datetime") as clock, + patch("zeus.bot_lifecycle_store.append_private_bytes") as append, ): clock.now.return_value = fixed_now store.append_audit_event( diff --git a/zeus/bot_lifecycle_store.py b/zeus/bot_lifecycle_store.py new file mode 100644 index 0000000..ef77106 --- /dev/null +++ b/zeus/bot_lifecycle_store.py @@ -0,0 +1,1174 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal + +from zeus.lifecycle import ( + LifecycleEvent, + LifecycleEventInput, + deserialize_lifecycle_details, + serialize_lifecycle_details, +) +from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy +from zeus.private_io import append_private_bytes, nofollow_absolute_path +from zeus.sanitization import MAX_SANITIZED_JSON_BYTES, sanitize_details, sanitize_text +from zeus.sqlite_db import SQLiteDatabase + +LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") +LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) +LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) + + +def _sanitize_optional_persisted_text(value: str | None) -> str | None: + if value is None: + return None + return sanitize_text(value) + + +class BotLifecycleStore: + def __init__(self, database: SQLiteDatabase) -> None: + self._database = database + + def begin_lifecycle_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + source: str, + request_id: str | None = None, + reason: str = "", + ) -> BotRecord: + self._validate_intent_action(action) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent", + outcome="pending", + request_id=request_id, + reason=reason, + details={"action": action, "desired_state": desired_state.value}, + ) + self._validate_event_target(bot_id, event) + now = datetime.now(UTC) + now_text = now.isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + if prior_row["pending_operation_id"] is not None: + raise RuntimeError("bot already has a pending lifecycle intent") + desired_revision = int(prior_row["desired_revision"]) + 1 + cursor = conn.execute( + """ + UPDATE bots + SET desired_state = ?, + desired_revision = ?, + desired_updated_at = ?, + pending_operation_id = ?, + pending_action = ?, + pending_since = ?, + updated_at = ? + WHERE bot_id = ? AND pending_operation_id IS NULL + """, + ( + desired_state.value, + desired_revision, + now_text, + operation_id, + action, + now_text, + now_text, + bot_id, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("lifecycle intent could not be started") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent", + outcome="pending", + request_id=request_id, + reason=reason, + details={ + "action": action, + "desired_state": desired_state.value, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=str(prior_row["status"]), + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def complete_lifecycle_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + desired_revision: int, + status: BotStatus, + pid: int | None, + source: str, + outcome: Literal["success", "failure"] = "success", + request_id: str | None = None, + reason: str = "", + error_code: str | None = None, + error_message: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> BotRecord: + self._validate_intent_action(action) + self._validate_intent_revision(desired_revision) + self._validate_intent_correlation(bot_id, operation_id, source, request_id) + self._validate_intent_completion( + action, + outcome=outcome, + status=status, + pid=pid, + error_code=error_code, + error_message=error_message, + ) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._require_matching_intent( + prior_row, + action, + operation_id, + desired_revision, + ) + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + cursor = conn.execute( + """ + UPDATE bots + SET pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL + WHERE bot_id = ? + AND pending_operation_id = ? + AND pending_action = ? + AND desired_revision = ? + AND desired_state = ? + """, + (bot_id, operation_id, action, desired_revision, desired_state.value), + ) + if cursor.rowcount != 1: + raise RuntimeError("pending lifecycle intent does not match") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.complete", + outcome=outcome, + request_id=request_id, + reason=reason, + error_code=error_code, + error_message=error_message, + details={ + "action": action, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def clear_stale_intent( + self, + bot_id: str, + *, + action: str, + operation_id: str, + desired_revision: int, + source: str, + reason: str, + request_id: str | None = None, + ) -> BotRecord: + self._validate_intent_action(action) + self._validate_intent_revision(desired_revision) + self._validate_intent_correlation(bot_id, operation_id, source, request_id) + desired_state = DesiredState.stopped if action == "stop" else DesiredState.running + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._require_matching_intent( + prior_row, + action, + operation_id, + desired_revision, + ) + cursor = conn.execute( + """ + UPDATE bots + SET pending_operation_id = NULL, + pending_action = NULL, + pending_since = NULL, + updated_at = ? + WHERE bot_id = ? + AND pending_operation_id = ? + AND pending_action = ? + AND desired_revision = ? + AND desired_state = ? + """, + ( + now, + bot_id, + operation_id, + action, + desired_revision, + desired_state.value, + ), + ) + if cursor.rowcount != 1: + raise RuntimeError("pending lifecycle intent does not match") + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action=f"bot.{action}.intent.clear", + outcome="cleared", + request_id=request_id, + reason=reason, + details={ + "action": action, + "desired_revision": desired_revision, + }, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=str(prior_row["status"]), + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored_event = self._materialize_lifecycle_event(conn, event_id) + record = self._record_in_transaction(conn, bot_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored_event) + return record + + def _validate_intent_action(self, action: str) -> None: + if type(action) is not str or action not in LIFECYCLE_INTENT_ACTIONS: + raise ValueError("invalid lifecycle intent action") + + def _validate_intent_completion( + self, + action: str, + *, + outcome: Literal["success", "failure"], + status: BotStatus, + pid: int | None, + error_code: str | None, + error_message: str | None, + ) -> None: + if outcome not in {"success", "failure"}: + raise ValueError("invalid lifecycle intent outcome") + if not isinstance(status, BotStatus): + raise TypeError("lifecycle intent status must be a BotStatus") + if pid is not None and (type(pid) is not int or pid <= 0): + raise ValueError("lifecycle intent terminal state has an invalid PID") + if error_code is not None and ( + type(error_code) is not str or LIFECYCLE_ERROR_CODE_RE.fullmatch(error_code) is None + ): + raise ValueError("invalid lifecycle intent error code") + if error_message is not None and type(error_message) is not str: + raise TypeError("lifecycle intent error message must be a string") + if outcome == "success": + if error_code is not None or error_message is not None: + raise ValueError("successful lifecycle intent cannot include error metadata") + if action == "stop": + compatible = status is BotStatus.stopped and pid is None + else: + compatible = status in {BotStatus.starting, BotStatus.running} and pid is not None + else: + compatible = status in {BotStatus.failed, BotStatus.unknown, BotStatus.stopped} + if status in {BotStatus.failed, BotStatus.stopped} and pid is not None: + compatible = False + if not compatible: + raise ValueError("lifecycle intent terminal state is incompatible with action outcome") + + def _validate_intent_revision(self, desired_revision: int) -> None: + if type(desired_revision) is not int or desired_revision < 1: + raise ValueError("desired revision must be a positive integer") + + def _validate_intent_correlation( + self, + bot_id: str, + operation_id: str, + source: str, + request_id: str | None, + ) -> None: + event = LifecycleEventInput( + bot_id=bot_id, + operation_id=operation_id, + source=source, + action="lifecycle.intent.validate", + outcome="validation", + request_id=request_id, + ) + self._validate_event_target(bot_id, event) + + def _require_matching_intent( + self, + row: sqlite3.Row, + action: str, + operation_id: str, + desired_revision: int, + ) -> None: + desired_state = ( + DesiredState.stopped.value if action == "stop" else DesiredState.running.value + ) + if ( + row["pending_operation_id"] != operation_id + or int(row["desired_revision"]) != desired_revision + or row["pending_action"] != action + or row["desired_state"] != desired_state + ): + raise RuntimeError("pending lifecycle intent does not match") + + def _record_in_transaction(self, conn: sqlite3.Connection, bot_id: str) -> BotRecord: + row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() + if row is None: + raise sqlite3.IntegrityError("updated bot projection is missing") + return self._row_to_record(row) + + def audit_log_path(self) -> Path: + return self._database.database_path.parent / "logs" / "audit.jsonl" + + def append_audit_event(self, event: str, **fields: object) -> None: + try: + safe_fields = sanitize_details(fields) + if type(safe_fields) is not dict: + return + timestamp = datetime.now(UTC).isoformat() + safe_event = sanitize_text(event) + payload = { + "ts": timestamp, + "event": safe_event, + **safe_fields, + } + line = (json.dumps(payload, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + line = ( + json.dumps( + { + "ts": timestamp, + "event": "audit.truncated", + "truncated": True, + }, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + if len(line) > MAX_SANITIZED_JSON_BYTES: + return + append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) + except Exception: + return + + def upsert_bot(self, record: BotRecord) -> None: + with closing(self._database.connect()) as conn: + self._upsert_bot_row(conn, record) + conn.commit() + + def _upsert_bot_row(self, conn: sqlite3.Connection, record: BotRecord) -> None: + conn.execute( + """ + INSERT INTO bots ( + bot_id, + template_id, + display_name, + profile_path, + status, + pid, + restart_policy, + restart_backoff_seconds, + restart_max_attempts, + restart_attempts, + next_restart_at, + started_at, + ready_at, + stopped_at, + last_exit_code, + last_error, + last_transition_reason, + desired_state, + desired_revision, + desired_updated_at, + pending_operation_id, + pending_action, + pending_since, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(bot_id) DO UPDATE SET + template_id = excluded.template_id, + display_name = excluded.display_name, + profile_path = excluded.profile_path, + status = excluded.status, + pid = excluded.pid, + restart_policy = excluded.restart_policy, + restart_backoff_seconds = excluded.restart_backoff_seconds, + restart_max_attempts = excluded.restart_max_attempts, + restart_attempts = excluded.restart_attempts, + next_restart_at = excluded.next_restart_at, + started_at = excluded.started_at, + ready_at = excluded.ready_at, + stopped_at = excluded.stopped_at, + last_exit_code = excluded.last_exit_code, + last_error = excluded.last_error, + last_transition_reason = excluded.last_transition_reason, + desired_state = excluded.desired_state, + desired_revision = excluded.desired_revision, + desired_updated_at = excluded.desired_updated_at, + pending_operation_id = excluded.pending_operation_id, + pending_action = excluded.pending_action, + pending_since = excluded.pending_since, + updated_at = excluded.updated_at + """, + ( + record.bot_id, + record.template_id, + record.display_name, + record.profile_path, + record.status.value, + record.pid, + record.restart_policy.value, + record.restart_backoff_seconds, + record.restart_max_attempts, + record.restart_attempts, + record.next_restart_at.isoformat() if record.next_restart_at else None, + record.started_at.isoformat() if record.started_at else None, + record.ready_at.isoformat() if record.ready_at else None, + record.stopped_at.isoformat() if record.stopped_at else None, + record.last_exit_code, + _sanitize_optional_persisted_text(record.last_error), + _sanitize_optional_persisted_text(record.last_transition_reason), + record.desired_state.value, + record.desired_revision, + record.desired_updated_at.isoformat() if record.desired_updated_at else None, + record.pending_operation_id, + record.pending_action, + record.pending_since.isoformat() if record.pending_since else None, + record.created_at.isoformat(), + record.updated_at.isoformat(), + ), + ) + + def upsert_bot_with_event( + self, + record: BotRecord, + *, + event: LifecycleEventInput, + ) -> LifecycleEvent: + self._validate_event_target(record.bot_id, event) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (record.bot_id,) + ).fetchone() + self._upsert_bot_row(conn, record) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]) if prior_row else None, + status_after=record.status.value, + pid_before=int(prior_row["pid"]) if prior_row and prior_row["pid"] else None, + pid_after=record.pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, record.bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def get_bot(self, bot_id: str) -> BotRecord | None: + with closing(self._database.connect()) as conn: + row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() + return self._row_to_record(row) if row else None + + def list_bots(self) -> list[BotRecord]: + with closing(self._database.connect()) as conn: + rows = conn.execute("SELECT * FROM bots ORDER BY bot_id").fetchall() + return [self._row_to_record(row) for row in rows] + + def update_status( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + reset_restart: bool = False, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + if reset_restart: + conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + restart_attempts = 0, + next_restart_at = NULL, + updated_at = ? + WHERE bot_id = ? + """, + (status.value, pid, now, bot_id), + ) + else: + conn.execute( + "UPDATE bots SET status = ?, pid = ?, updated_at = ? WHERE bot_id = ?", + (status.value, pid, now, bot_id), + ) + conn.commit() + + def update_lifecycle_state( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + conn.commit() + + def _update_lifecycle_row( + self, + conn: sqlite3.Connection, + bot_id: str, + status: BotStatus, + pid: int | None, + *, + started_at: datetime | None, + ready_at: datetime | None, + stopped_at: datetime | None, + last_exit_code: int | None, + last_error: str | None, + last_transition_reason: str | None, + reset_restart: bool, + clear_ready_at: bool, + clear_stopped_at: bool, + now: str, + ) -> None: + cursor = conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + started_at = COALESCE(?, started_at), + ready_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, ready_at) END, + stopped_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, stopped_at) END, + last_exit_code = ?, + last_error = ?, + last_transition_reason = COALESCE(?, last_transition_reason), + updated_at = ?, + restart_attempts = CASE WHEN ? THEN 0 ELSE restart_attempts END, + next_restart_at = CASE WHEN ? THEN NULL ELSE next_restart_at END + WHERE bot_id = ? + """, + ( + status.value, + pid, + started_at.isoformat() if started_at else None, + int(clear_ready_at), + ready_at.isoformat() if ready_at else None, + int(clear_stopped_at), + stopped_at.isoformat() if stopped_at else None, + last_exit_code, + _sanitize_optional_persisted_text(last_error), + _sanitize_optional_persisted_text(last_transition_reason), + now, + int(reset_restart), + int(reset_restart), + bot_id, + ), + ) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + + def update_lifecycle_with_event( + self, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + event: LifecycleEventInput, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + ) -> LifecycleEvent: + self._validate_event_target(bot_id, event) + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._update_lifecycle_row( + conn, + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + now=now, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def update_restart_state( + self, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + ) -> None: + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + self._update_restart_row( + conn, + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + now=now, + ) + conn.commit() + + def _update_restart_row( + self, + conn: sqlite3.Connection, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + now: str, + ) -> None: + cursor = conn.execute( + """ + UPDATE bots + SET status = ?, + pid = ?, + restart_attempts = ?, + next_restart_at = ?, + updated_at = ? + WHERE bot_id = ? + """, + ( + status.value, + pid, + restart_attempts, + next_restart_at.isoformat() if next_restart_at else None, + now, + bot_id, + ), + ) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + + def update_restart_with_event( + self, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + event: LifecycleEventInput, + ) -> LifecycleEvent: + self._validate_event_target(bot_id, event) + now = datetime.now(UTC).isoformat() + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + self._update_restart_row( + conn, + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + now=now, + ) + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=status.value, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=pid, + ) + conn.execute( + "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", + (event_id, bot_id), + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return stored + + def delete_bot(self, bot_id: str) -> bool: + with closing(self._database.connect()) as conn: + cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) + conn.commit() + return cursor.rowcount > 0 + + def delete_bot_with_event( + self, + bot_id: str, + *, + event: LifecycleEventInput, + ) -> bool: + self._validate_event_target(bot_id, event) + with closing(self._database.connect()) as conn: + try: + conn.execute("BEGIN IMMEDIATE") + prior_row = conn.execute( + "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) + ).fetchone() + if prior_row is None: + raise KeyError(f"unknown bot: {bot_id}") + cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) + if cursor.rowcount != 1: + raise KeyError(f"unknown bot: {bot_id}") + event_id = self._insert_lifecycle_event( + conn, + event, + status_before=str(prior_row["status"]), + status_after=None, + pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), + pid_after=None, + ) + stored = self._materialize_lifecycle_event(conn, event_id) + except Exception: + conn.rollback() + raise + else: + conn.commit() + self._append_lifecycle_audit_fail_open(stored) + return True + + def _validate_event_target(self, bot_id: str, event: LifecycleEventInput) -> None: + if event.bot_id != bot_id: + raise ValueError("lifecycle event bot_id must match the projection target") + if LIFECYCLE_ID_RE.fullmatch(event.operation_id) is None: + raise ValueError("lifecycle operation_id must be generated UUID hex") + if event.source not in LIFECYCLE_SOURCES: + raise ValueError("invalid lifecycle event source") + if event.source == "api": + if event.request_id is None or LIFECYCLE_ID_RE.fullmatch(event.request_id) is None: + raise ValueError("API lifecycle events require a generated request ID") + elif event.request_id is not None: + raise ValueError("only API lifecycle events may carry a request ID") + + def _insert_lifecycle_event( + self, + conn: sqlite3.Connection, + event: LifecycleEventInput, + *, + status_before: str | None, + status_after: str | None, + pid_before: int | None, + pid_after: int | None, + ) -> int: + cursor = conn.execute( + """ + INSERT INTO lifecycle_events ( + bot_id, + operation_id, + request_id, + occurred_at, + source, + action, + outcome, + status_before, + status_after, + pid_before, + pid_after, + reason, + error_code, + error_message, + details_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + event.bot_id, + event.operation_id, + event.request_id, + datetime.now(UTC).isoformat(), + event.source, + event.action, + event.outcome, + status_before, + status_after, + pid_before, + pid_after, + event.reason, + event.error_code, + event.error_message, + serialize_lifecycle_details(event.details), + ), + ) + if cursor.lastrowid is None: + raise sqlite3.IntegrityError("lifecycle event insert returned no event id") + return int(cursor.lastrowid) + + def _materialize_lifecycle_event( + self, + conn: sqlite3.Connection, + event_id: int, + ) -> LifecycleEvent: + row = conn.execute( + "SELECT * FROM lifecycle_events WHERE event_id = ?", (event_id,) + ).fetchone() + if row is None: + raise sqlite3.IntegrityError("inserted lifecycle event is missing") + return self._row_to_lifecycle_event(row) + + def _append_lifecycle_audit_fail_open(self, event: LifecycleEvent) -> None: + try: + self._append_lifecycle_audit(event) + except Exception: + return + + def _append_lifecycle_audit(self, event: LifecycleEvent) -> None: + self.append_audit_event( + event.action, + bot_id=event.bot_id, + operation_id=event.operation_id, + request_id=event.request_id, + source=event.source, + outcome=event.outcome, + status_before=event.status_before, + status_after=event.status_after, + pid_before=event.pid_before, + pid_after=event.pid_after, + reason=event.reason, + error_code=event.error_code, + error_message=event.error_message, + details=event.details, + ) + + def list_lifecycle_events( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> list[LifecycleEvent]: + self._validate_history_page(bot_id, limit, before) + with closing(self._database.connect()) as conn: + rows = self._list_lifecycle_event_rows(conn, bot_id, limit, before) + return [self._row_to_lifecycle_event(row) for row in rows] + + def history_payload( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> dict[str, object]: + self._validate_history_page(bot_id, limit, before) + with closing(self._database.connect()) as conn: + rows = self._list_lifecycle_event_rows(conn, bot_id, limit + 1, before) + if not rows: + bot_is_known = ( + conn.execute( + """ + SELECT 1 FROM bots WHERE bot_id = ? + UNION ALL + SELECT 1 FROM lifecycle_events WHERE bot_id = ? + LIMIT 1 + """, + (bot_id, bot_id), + ).fetchone() + is not None + ) + if not bot_is_known: + raise KeyError(f"unknown bot: {bot_id}") + has_more = len(rows) > limit + events = [self._row_to_lifecycle_event(row) for row in rows[:limit]] + return { + "bot_id": bot_id, + "events": [event.to_dict() for event in events], + "next_before": events[-1].event_id if has_more else None, + } + + def _validate_history_page( + self, + bot_id: str, + limit: int, + before: int | None, + ) -> None: + if not isinstance(bot_id, str): + raise TypeError("bot_id must be a string") + if isinstance(limit, bool) or not isinstance(limit, int): + raise TypeError("limit must be an integer") + if limit < 1 or limit > 1000: + raise ValueError("limit must be between 1 and 1000") + if before is not None: + if isinstance(before, bool) or not isinstance(before, int): + raise TypeError("before must be an integer or null") + if before < 1: + raise ValueError("before must be positive") + + def _list_lifecycle_event_rows( + self, + conn: sqlite3.Connection, + bot_id: str, + limit: int, + before: int | None, + ) -> list[sqlite3.Row]: + if before is None: + return conn.execute( + """ + SELECT * FROM lifecycle_events + WHERE bot_id = ? + ORDER BY event_id DESC + LIMIT ? + """, + (bot_id, limit), + ).fetchall() + return conn.execute( + """ + SELECT * FROM lifecycle_events + WHERE bot_id = ? AND event_id < ? + ORDER BY event_id DESC + LIMIT ? + """, + (bot_id, before, limit), + ).fetchall() + + def _row_to_lifecycle_event(self, row: sqlite3.Row) -> LifecycleEvent: + return LifecycleEvent( + event_id=int(row["event_id"]), + bot_id=str(row["bot_id"]), + operation_id=str(row["operation_id"]), + request_id=str(row["request_id"]) if row["request_id"] is not None else None, + occurred_at=datetime.fromisoformat(str(row["occurred_at"])), + source=str(row["source"]), + action=str(row["action"]), + outcome=str(row["outcome"]), + status_before=(str(row["status_before"]) if row["status_before"] is not None else None), + status_after=(str(row["status_after"]) if row["status_after"] is not None else None), + pid_before=int(row["pid_before"]) if row["pid_before"] is not None else None, + pid_after=int(row["pid_after"]) if row["pid_after"] is not None else None, + reason=str(row["reason"]), + error_code=str(row["error_code"]) if row["error_code"] is not None else None, + error_message=(str(row["error_message"]) if row["error_message"] is not None else None), + details=deserialize_lifecycle_details(str(row["details_json"])), + ) + + def _row_to_record(self, row: sqlite3.Row) -> BotRecord: + next_restart_at = row["next_restart_at"] + started_at = row["started_at"] + ready_at = row["ready_at"] + stopped_at = row["stopped_at"] + desired_updated_at = row["desired_updated_at"] + pending_since = row["pending_since"] + return BotRecord( + bot_id=row["bot_id"], + template_id=row["template_id"], + display_name=row["display_name"], + profile_path=row["profile_path"], + status=BotStatus(row["status"]), + pid=row["pid"], + restart_policy=RestartPolicy(row["restart_policy"]), + restart_backoff_seconds=float(row["restart_backoff_seconds"]), + restart_max_attempts=int(row["restart_max_attempts"]), + restart_attempts=int(row["restart_attempts"]), + next_restart_at=datetime.fromisoformat(next_restart_at) if next_restart_at else None, + started_at=datetime.fromisoformat(started_at) if started_at else None, + ready_at=datetime.fromisoformat(ready_at) if ready_at else None, + stopped_at=datetime.fromisoformat(stopped_at) if stopped_at else None, + last_exit_code=row["last_exit_code"], + last_error=row["last_error"], + last_transition_reason=row["last_transition_reason"], + desired_state=DesiredState(row["desired_state"]), + desired_revision=int(row["desired_revision"]), + desired_updated_at=( + datetime.fromisoformat(desired_updated_at) if desired_updated_at else None + ), + pending_operation_id=row["pending_operation_id"], + pending_action=row["pending_action"], + pending_since=datetime.fromisoformat(pending_since) if pending_since else None, + created_at=datetime.fromisoformat(row["created_at"]), + updated_at=datetime.fromisoformat(row["updated_at"]), + ) diff --git a/zeus/state.py b/zeus/state.py index 374f6f8..3a61555 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -1,13 +1,15 @@ from __future__ import annotations -import json -import re import sqlite3 -from contextlib import closing -from datetime import UTC, datetime +from datetime import datetime from pathlib import Path from typing import Literal +from zeus.bot_lifecycle_store import LIFECYCLE_ERROR_CODE_RE as LIFECYCLE_ERROR_CODE_RE +from zeus.bot_lifecycle_store import LIFECYCLE_ID_RE as LIFECYCLE_ID_RE +from zeus.bot_lifecycle_store import LIFECYCLE_INTENT_ACTIONS as LIFECYCLE_INTENT_ACTIONS +from zeus.bot_lifecycle_store import LIFECYCLE_SOURCES as LIFECYCLE_SOURCES +from zeus.bot_lifecycle_store import BotLifecycleStore from zeus.idempotency import IdempotencyClaim from zeus.idempotency_store import IDEMPOTENCY_HASH_RE as IDEMPOTENCY_HASH_RE from zeus.idempotency_store import IDEMPOTENCY_OWNER_RE as IDEMPOTENCY_OWNER_RE @@ -16,11 +18,8 @@ from zeus.lifecycle import ( LifecycleEvent, LifecycleEventInput, - deserialize_lifecycle_details, - serialize_lifecycle_details, ) -from zeus.models import BotRecord, BotStatus, DesiredState, RestartPolicy -from zeus.private_io import append_private_bytes, nofollow_absolute_path +from zeus.models import BotRecord, BotStatus from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS as RECONCILE_COUNTER_COLUMNS from zeus.reconcile_store import ReconcileStore from zeus.reconciliation import ( @@ -29,22 +28,10 @@ ReconcileRunStart, ReconcileRunSummary, ) -from zeus.sanitization import MAX_SANITIZED_JSON_BYTES, sanitize_details, sanitize_text from zeus.schema import SCHEMA_VERSION as SCHEMA_VERSION from zeus.schema import SchemaManager from zeus.sqlite_db import SQLiteDatabase -LIFECYCLE_ID_RE = re.compile(r"^[0-9a-f]{32}$") -LIFECYCLE_ERROR_CODE_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") -LIFECYCLE_SOURCES = frozenset({"api", "cli", "migration", "reconcile", "recovery", "system"}) -LIFECYCLE_INTENT_ACTIONS = frozenset({"start", "stop", "restart"}) - - -def _sanitize_optional_persisted_text(value: str | None) -> str | None: - if value is None: - return None - return sanitize_text(value) - class StateStore: def __init__(self, database_path: Path | str) -> None: @@ -52,6 +39,7 @@ def __init__(self, database_path: Path | str) -> None: self._schema = SchemaManager(self._database) self._idempotency = IdempotencyStore(self._database) self._reconcile = ReconcileStore(self._database) + self._bot_lifecycle = BotLifecycleStore(self._database) @property def database_path(self) -> Path: @@ -149,92 +137,14 @@ def begin_lifecycle_intent( request_id: str | None = None, reason: str = "", ) -> BotRecord: - self._validate_intent_action(action) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - event = LifecycleEventInput( - bot_id=bot_id, + return self._bot_lifecycle.begin_lifecycle_intent( + bot_id, + action=action, operation_id=operation_id, source=source, - action=f"bot.{action}.intent", - outcome="pending", request_id=request_id, reason=reason, - details={"action": action, "desired_state": desired_state.value}, ) - self._validate_event_target(bot_id, event) - now = datetime.now(UTC) - now_text = now.isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - if prior_row["pending_operation_id"] is not None: - raise RuntimeError("bot already has a pending lifecycle intent") - desired_revision = int(prior_row["desired_revision"]) + 1 - cursor = conn.execute( - """ - UPDATE bots - SET desired_state = ?, - desired_revision = ?, - desired_updated_at = ?, - pending_operation_id = ?, - pending_action = ?, - pending_since = ?, - updated_at = ? - WHERE bot_id = ? AND pending_operation_id IS NULL - """, - ( - desired_state.value, - desired_revision, - now_text, - operation_id, - action, - now_text, - now_text, - bot_id, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("lifecycle intent could not be started") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.intent", - outcome="pending", - request_id=request_id, - reason=reason, - details={ - "action": action, - "desired_state": desired_state.value, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=str(prior_row["status"]), - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record def complete_lifecycle_intent( self, @@ -261,101 +171,29 @@ def complete_lifecycle_intent( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> BotRecord: - self._validate_intent_action(action) - self._validate_intent_revision(desired_revision) - self._validate_intent_correlation(bot_id, operation_id, source, request_id) - self._validate_intent_completion( - action, - outcome=outcome, + return self._bot_lifecycle.complete_lifecycle_intent( + bot_id, + action=action, + operation_id=operation_id, + desired_revision=desired_revision, status=status, pid=pid, + source=source, + outcome=outcome, + request_id=request_id, + reason=reason, error_code=error_code, error_message=error_message, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, ) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._require_matching_intent( - prior_row, - action, - operation_id, - desired_revision, - ) - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - cursor = conn.execute( - """ - UPDATE bots - SET pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL - WHERE bot_id = ? - AND pending_operation_id = ? - AND pending_action = ? - AND desired_revision = ? - AND desired_state = ? - """, - (bot_id, operation_id, action, desired_revision, desired_state.value), - ) - if cursor.rowcount != 1: - raise RuntimeError("pending lifecycle intent does not match") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.complete", - outcome=outcome, - request_id=request_id, - reason=reason, - error_code=error_code, - error_message=error_message, - details={ - "action": action, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record def clear_stale_intent( self, @@ -368,293 +206,24 @@ def clear_stale_intent( reason: str, request_id: str | None = None, ) -> BotRecord: - self._validate_intent_action(action) - self._validate_intent_revision(desired_revision) - self._validate_intent_correlation(bot_id, operation_id, source, request_id) - desired_state = DesiredState.stopped if action == "stop" else DesiredState.running - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._require_matching_intent( - prior_row, - action, - operation_id, - desired_revision, - ) - cursor = conn.execute( - """ - UPDATE bots - SET pending_operation_id = NULL, - pending_action = NULL, - pending_since = NULL, - updated_at = ? - WHERE bot_id = ? - AND pending_operation_id = ? - AND pending_action = ? - AND desired_revision = ? - AND desired_state = ? - """, - ( - now, - bot_id, - operation_id, - action, - desired_revision, - desired_state.value, - ), - ) - if cursor.rowcount != 1: - raise RuntimeError("pending lifecycle intent does not match") - event = LifecycleEventInput( - bot_id=bot_id, - operation_id=operation_id, - source=source, - action=f"bot.{action}.intent.clear", - outcome="cleared", - request_id=request_id, - reason=reason, - details={ - "action": action, - "desired_revision": desired_revision, - }, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=str(prior_row["status"]), - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored_event = self._materialize_lifecycle_event(conn, event_id) - record = self._record_in_transaction(conn, bot_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored_event) - return record - - def _validate_intent_action(self, action: str) -> None: - if type(action) is not str or action not in LIFECYCLE_INTENT_ACTIONS: - raise ValueError("invalid lifecycle intent action") - - def _validate_intent_completion( - self, - action: str, - *, - outcome: Literal["success", "failure"], - status: BotStatus, - pid: int | None, - error_code: str | None, - error_message: str | None, - ) -> None: - if outcome not in {"success", "failure"}: - raise ValueError("invalid lifecycle intent outcome") - if not isinstance(status, BotStatus): - raise TypeError("lifecycle intent status must be a BotStatus") - if pid is not None and (type(pid) is not int or pid <= 0): - raise ValueError("lifecycle intent terminal state has an invalid PID") - if error_code is not None and ( - type(error_code) is not str or LIFECYCLE_ERROR_CODE_RE.fullmatch(error_code) is None - ): - raise ValueError("invalid lifecycle intent error code") - if error_message is not None and type(error_message) is not str: - raise TypeError("lifecycle intent error message must be a string") - if outcome == "success": - if error_code is not None or error_message is not None: - raise ValueError("successful lifecycle intent cannot include error metadata") - if action == "stop": - compatible = status is BotStatus.stopped and pid is None - else: - compatible = status in {BotStatus.starting, BotStatus.running} and pid is not None - else: - compatible = status in {BotStatus.failed, BotStatus.unknown, BotStatus.stopped} - if status in {BotStatus.failed, BotStatus.stopped} and pid is not None: - compatible = False - if not compatible: - raise ValueError("lifecycle intent terminal state is incompatible with action outcome") - - def _validate_intent_revision(self, desired_revision: int) -> None: - if type(desired_revision) is not int or desired_revision < 1: - raise ValueError("desired revision must be a positive integer") - - def _validate_intent_correlation( - self, - bot_id: str, - operation_id: str, - source: str, - request_id: str | None, - ) -> None: - event = LifecycleEventInput( - bot_id=bot_id, + return self._bot_lifecycle.clear_stale_intent( + bot_id, + action=action, operation_id=operation_id, + desired_revision=desired_revision, source=source, - action="lifecycle.intent.validate", - outcome="validation", + reason=reason, request_id=request_id, ) - self._validate_event_target(bot_id, event) - - def _require_matching_intent( - self, - row: sqlite3.Row, - action: str, - operation_id: str, - desired_revision: int, - ) -> None: - desired_state = ( - DesiredState.stopped.value if action == "stop" else DesiredState.running.value - ) - if ( - row["pending_operation_id"] != operation_id - or int(row["desired_revision"]) != desired_revision - or row["pending_action"] != action - or row["desired_state"] != desired_state - ): - raise RuntimeError("pending lifecycle intent does not match") - - def _record_in_transaction(self, conn: sqlite3.Connection, bot_id: str) -> BotRecord: - row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() - if row is None: - raise sqlite3.IntegrityError("updated bot projection is missing") - return self._row_to_record(row) def audit_log_path(self) -> Path: - return self.database_path.parent / "logs" / "audit.jsonl" + return self._bot_lifecycle.audit_log_path() def append_audit_event(self, event: str, **fields: object) -> None: - try: - safe_fields = sanitize_details(fields) - if type(safe_fields) is not dict: - return - timestamp = datetime.now(UTC).isoformat() - safe_event = sanitize_text(event) - payload = { - "ts": timestamp, - "event": safe_event, - **safe_fields, - } - line = (json.dumps(payload, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") - if len(line) > MAX_SANITIZED_JSON_BYTES: - line = ( - json.dumps( - { - "ts": timestamp, - "event": "audit.truncated", - "truncated": True, - }, - sort_keys=True, - allow_nan=False, - ) - + "\n" - ).encode("utf-8") - if len(line) > MAX_SANITIZED_JSON_BYTES: - return - append_private_bytes(nofollow_absolute_path(self.audit_log_path()), line) - except Exception: - return + self._bot_lifecycle.append_audit_event(event, **fields) def upsert_bot(self, record: BotRecord) -> None: - with closing(self.connect()) as conn: - self._upsert_bot_row(conn, record) - conn.commit() - - def _upsert_bot_row(self, conn: sqlite3.Connection, record: BotRecord) -> None: - conn.execute( - """ - INSERT INTO bots ( - bot_id, - template_id, - display_name, - profile_path, - status, - pid, - restart_policy, - restart_backoff_seconds, - restart_max_attempts, - restart_attempts, - next_restart_at, - started_at, - ready_at, - stopped_at, - last_exit_code, - last_error, - last_transition_reason, - desired_state, - desired_revision, - desired_updated_at, - pending_operation_id, - pending_action, - pending_since, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(bot_id) DO UPDATE SET - template_id = excluded.template_id, - display_name = excluded.display_name, - profile_path = excluded.profile_path, - status = excluded.status, - pid = excluded.pid, - restart_policy = excluded.restart_policy, - restart_backoff_seconds = excluded.restart_backoff_seconds, - restart_max_attempts = excluded.restart_max_attempts, - restart_attempts = excluded.restart_attempts, - next_restart_at = excluded.next_restart_at, - started_at = excluded.started_at, - ready_at = excluded.ready_at, - stopped_at = excluded.stopped_at, - last_exit_code = excluded.last_exit_code, - last_error = excluded.last_error, - last_transition_reason = excluded.last_transition_reason, - desired_state = excluded.desired_state, - desired_revision = excluded.desired_revision, - desired_updated_at = excluded.desired_updated_at, - pending_operation_id = excluded.pending_operation_id, - pending_action = excluded.pending_action, - pending_since = excluded.pending_since, - updated_at = excluded.updated_at - """, - ( - record.bot_id, - record.template_id, - record.display_name, - record.profile_path, - record.status.value, - record.pid, - record.restart_policy.value, - record.restart_backoff_seconds, - record.restart_max_attempts, - record.restart_attempts, - record.next_restart_at.isoformat() if record.next_restart_at else None, - record.started_at.isoformat() if record.started_at else None, - record.ready_at.isoformat() if record.ready_at else None, - record.stopped_at.isoformat() if record.stopped_at else None, - record.last_exit_code, - _sanitize_optional_persisted_text(record.last_error), - _sanitize_optional_persisted_text(record.last_transition_reason), - record.desired_state.value, - record.desired_revision, - record.desired_updated_at.isoformat() if record.desired_updated_at else None, - record.pending_operation_id, - record.pending_action, - record.pending_since.isoformat() if record.pending_since else None, - record.created_at.isoformat(), - record.updated_at.isoformat(), - ), - ) + self._bot_lifecycle.upsert_bot(record) def upsert_bot_with_event( self, @@ -662,44 +231,13 @@ def upsert_bot_with_event( *, event: LifecycleEventInput, ) -> LifecycleEvent: - self._validate_event_target(record.bot_id, event) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (record.bot_id,) - ).fetchone() - self._upsert_bot_row(conn, record) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]) if prior_row else None, - status_after=record.status.value, - pid_before=int(prior_row["pid"]) if prior_row and prior_row["pid"] else None, - pid_after=record.pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, record.bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.upsert_bot_with_event(record, event=event) def get_bot(self, bot_id: str) -> BotRecord | None: - with closing(self.connect()) as conn: - row = conn.execute("SELECT * FROM bots WHERE bot_id = ?", (bot_id,)).fetchone() - return self._row_to_record(row) if row else None + return self._bot_lifecycle.get_bot(bot_id) def list_bots(self) -> list[BotRecord]: - with closing(self.connect()) as conn: - rows = conn.execute("SELECT * FROM bots ORDER BY bot_id").fetchall() - return [self._row_to_record(row) for row in rows] + return self._bot_lifecycle.list_bots() def update_status( self, @@ -709,27 +247,12 @@ def update_status( *, reset_restart: bool = False, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - if reset_restart: - conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - restart_attempts = 0, - next_restart_at = NULL, - updated_at = ? - WHERE bot_id = ? - """, - (status.value, pid, now, bot_id), - ) - else: - conn.execute( - "UPDATE bots SET status = ?, pid = ?, updated_at = ? WHERE bot_id = ?", - (status.value, pid, now, bot_id), - ) - conn.commit() + self._bot_lifecycle.update_status( + bot_id, + status, + pid, + reset_restart=reset_restart, + ) def update_lifecycle_state( self, @@ -747,79 +270,20 @@ def update_lifecycle_state( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - conn.commit() - - def _update_lifecycle_row( - self, - conn: sqlite3.Connection, - bot_id: str, - status: BotStatus, - pid: int | None, - *, - started_at: datetime | None, - ready_at: datetime | None, - stopped_at: datetime | None, - last_exit_code: int | None, - last_error: str | None, - last_transition_reason: str | None, - reset_restart: bool, - clear_ready_at: bool, - clear_stopped_at: bool, - now: str, - ) -> None: - cursor = conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - started_at = COALESCE(?, started_at), - ready_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, ready_at) END, - stopped_at = CASE WHEN ? THEN NULL ELSE COALESCE(?, stopped_at) END, - last_exit_code = ?, - last_error = ?, - last_transition_reason = COALESCE(?, last_transition_reason), - updated_at = ?, - restart_attempts = CASE WHEN ? THEN 0 ELSE restart_attempts END, - next_restart_at = CASE WHEN ? THEN NULL ELSE next_restart_at END - WHERE bot_id = ? - """, - ( - status.value, - pid, - started_at.isoformat() if started_at else None, - int(clear_ready_at), - ready_at.isoformat() if ready_at else None, - int(clear_stopped_at), - stopped_at.isoformat() if stopped_at else None, - last_exit_code, - _sanitize_optional_persisted_text(last_error), - _sanitize_optional_persisted_text(last_transition_reason), - now, - int(reset_restart), - int(reset_restart), - bot_id, - ), + self._bot_lifecycle.update_lifecycle_state( + bot_id, + status, + pid, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, ) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") def update_lifecycle_with_event( self, @@ -838,52 +302,21 @@ def update_lifecycle_with_event( clear_ready_at: bool = False, clear_stopped_at: bool = False, ) -> LifecycleEvent: - self._validate_event_target(bot_id, event) - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._update_lifecycle_row( - conn, - bot_id, - status, - pid, - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - now=now, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.update_lifecycle_with_event( + bot_id, + status, + pid, + event=event, + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + ) def update_restart_state( self, @@ -894,51 +327,13 @@ def update_restart_state( restart_attempts: int, next_restart_at: datetime | None, ) -> None: - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - self._update_restart_row( - conn, - bot_id, - status=status, - pid=pid, - restart_attempts=restart_attempts, - next_restart_at=next_restart_at, - now=now, - ) - conn.commit() - - def _update_restart_row( - self, - conn: sqlite3.Connection, - bot_id: str, - *, - status: BotStatus, - pid: int | None, - restart_attempts: int, - next_restart_at: datetime | None, - now: str, - ) -> None: - cursor = conn.execute( - """ - UPDATE bots - SET status = ?, - pid = ?, - restart_attempts = ?, - next_restart_at = ?, - updated_at = ? - WHERE bot_id = ? - """, - ( - status.value, - pid, - restart_attempts, - next_restart_at.isoformat() if next_restart_at else None, - now, - bot_id, - ), + self._bot_lifecycle.update_restart_state( + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, ) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") def update_restart_with_event( self, @@ -950,51 +345,17 @@ def update_restart_with_event( next_restart_at: datetime | None, event: LifecycleEventInput, ) -> LifecycleEvent: - self._validate_event_target(bot_id, event) - now = datetime.now(UTC).isoformat() - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - self._update_restart_row( - conn, - bot_id, - status=status, - pid=pid, - restart_attempts=restart_attempts, - next_restart_at=next_restart_at, - now=now, - ) - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=status.value, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=pid, - ) - conn.execute( - "UPDATE bots SET last_event_id = ? WHERE bot_id = ?", - (event_id, bot_id), - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return stored + return self._bot_lifecycle.update_restart_with_event( + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + event=event, + ) def delete_bot(self, bot_id: str) -> bool: - with closing(self.connect()) as conn: - cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) - conn.commit() - return cursor.rowcount > 0 + return self._bot_lifecycle.delete_bot(bot_id) def delete_bot_with_event( self, @@ -1002,135 +363,7 @@ def delete_bot_with_event( *, event: LifecycleEventInput, ) -> bool: - self._validate_event_target(bot_id, event) - with closing(self.connect()) as conn: - try: - conn.execute("BEGIN IMMEDIATE") - prior_row = conn.execute( - "SELECT * FROM bots WHERE bot_id = ?", (bot_id,) - ).fetchone() - if prior_row is None: - raise KeyError(f"unknown bot: {bot_id}") - cursor = conn.execute("DELETE FROM bots WHERE bot_id = ?", (bot_id,)) - if cursor.rowcount != 1: - raise KeyError(f"unknown bot: {bot_id}") - event_id = self._insert_lifecycle_event( - conn, - event, - status_before=str(prior_row["status"]), - status_after=None, - pid_before=(int(prior_row["pid"]) if prior_row["pid"] is not None else None), - pid_after=None, - ) - stored = self._materialize_lifecycle_event(conn, event_id) - except Exception: - conn.rollback() - raise - else: - conn.commit() - self._append_lifecycle_audit_fail_open(stored) - return True - - def _validate_event_target(self, bot_id: str, event: LifecycleEventInput) -> None: - if event.bot_id != bot_id: - raise ValueError("lifecycle event bot_id must match the projection target") - if LIFECYCLE_ID_RE.fullmatch(event.operation_id) is None: - raise ValueError("lifecycle operation_id must be generated UUID hex") - if event.source not in LIFECYCLE_SOURCES: - raise ValueError("invalid lifecycle event source") - if event.source == "api": - if event.request_id is None or LIFECYCLE_ID_RE.fullmatch(event.request_id) is None: - raise ValueError("API lifecycle events require a generated request ID") - elif event.request_id is not None: - raise ValueError("only API lifecycle events may carry a request ID") - - def _insert_lifecycle_event( - self, - conn: sqlite3.Connection, - event: LifecycleEventInput, - *, - status_before: str | None, - status_after: str | None, - pid_before: int | None, - pid_after: int | None, - ) -> int: - cursor = conn.execute( - """ - INSERT INTO lifecycle_events ( - bot_id, - operation_id, - request_id, - occurred_at, - source, - action, - outcome, - status_before, - status_after, - pid_before, - pid_after, - reason, - error_code, - error_message, - details_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - event.bot_id, - event.operation_id, - event.request_id, - datetime.now(UTC).isoformat(), - event.source, - event.action, - event.outcome, - status_before, - status_after, - pid_before, - pid_after, - event.reason, - event.error_code, - event.error_message, - serialize_lifecycle_details(event.details), - ), - ) - if cursor.lastrowid is None: - raise sqlite3.IntegrityError("lifecycle event insert returned no event id") - return int(cursor.lastrowid) - - def _materialize_lifecycle_event( - self, - conn: sqlite3.Connection, - event_id: int, - ) -> LifecycleEvent: - row = conn.execute( - "SELECT * FROM lifecycle_events WHERE event_id = ?", (event_id,) - ).fetchone() - if row is None: - raise sqlite3.IntegrityError("inserted lifecycle event is missing") - return self._row_to_lifecycle_event(row) - - def _append_lifecycle_audit_fail_open(self, event: LifecycleEvent) -> None: - try: - self._append_lifecycle_audit(event) - except Exception: - return - - def _append_lifecycle_audit(self, event: LifecycleEvent) -> None: - self.append_audit_event( - event.action, - bot_id=event.bot_id, - operation_id=event.operation_id, - request_id=event.request_id, - source=event.source, - outcome=event.outcome, - status_before=event.status_before, - status_after=event.status_after, - pid_before=event.pid_before, - pid_after=event.pid_after, - reason=event.reason, - error_code=event.error_code, - error_message=event.error_message, - details=event.details, - ) + return self._bot_lifecycle.delete_bot_with_event(bot_id, event=event) def list_lifecycle_events( self, @@ -1138,10 +371,7 @@ def list_lifecycle_events( limit: int, before: int | None, ) -> list[LifecycleEvent]: - self._validate_history_page(bot_id, limit, before) - with closing(self.connect()) as conn: - rows = self._list_lifecycle_event_rows(conn, bot_id, limit, before) - return [self._row_to_lifecycle_event(row) for row in rows] + return self._bot_lifecycle.list_lifecycle_events(bot_id, limit, before) def history_payload( self, @@ -1149,130 +379,4 @@ def history_payload( limit: int, before: int | None, ) -> dict[str, object]: - self._validate_history_page(bot_id, limit, before) - with closing(self.connect()) as conn: - rows = self._list_lifecycle_event_rows(conn, bot_id, limit + 1, before) - if not rows: - bot_is_known = ( - conn.execute( - """ - SELECT 1 FROM bots WHERE bot_id = ? - UNION ALL - SELECT 1 FROM lifecycle_events WHERE bot_id = ? - LIMIT 1 - """, - (bot_id, bot_id), - ).fetchone() - is not None - ) - if not bot_is_known: - raise KeyError(f"unknown bot: {bot_id}") - has_more = len(rows) > limit - events = [self._row_to_lifecycle_event(row) for row in rows[:limit]] - return { - "bot_id": bot_id, - "events": [event.to_dict() for event in events], - "next_before": events[-1].event_id if has_more else None, - } - - def _validate_history_page( - self, - bot_id: str, - limit: int, - before: int | None, - ) -> None: - if not isinstance(bot_id, str): - raise TypeError("bot_id must be a string") - if isinstance(limit, bool) or not isinstance(limit, int): - raise TypeError("limit must be an integer") - if limit < 1 or limit > 1000: - raise ValueError("limit must be between 1 and 1000") - if before is not None: - if isinstance(before, bool) or not isinstance(before, int): - raise TypeError("before must be an integer or null") - if before < 1: - raise ValueError("before must be positive") - - def _list_lifecycle_event_rows( - self, - conn: sqlite3.Connection, - bot_id: str, - limit: int, - before: int | None, - ) -> list[sqlite3.Row]: - if before is None: - return conn.execute( - """ - SELECT * FROM lifecycle_events - WHERE bot_id = ? - ORDER BY event_id DESC - LIMIT ? - """, - (bot_id, limit), - ).fetchall() - return conn.execute( - """ - SELECT * FROM lifecycle_events - WHERE bot_id = ? AND event_id < ? - ORDER BY event_id DESC - LIMIT ? - """, - (bot_id, before, limit), - ).fetchall() - - def _row_to_lifecycle_event(self, row: sqlite3.Row) -> LifecycleEvent: - return LifecycleEvent( - event_id=int(row["event_id"]), - bot_id=str(row["bot_id"]), - operation_id=str(row["operation_id"]), - request_id=str(row["request_id"]) if row["request_id"] is not None else None, - occurred_at=datetime.fromisoformat(str(row["occurred_at"])), - source=str(row["source"]), - action=str(row["action"]), - outcome=str(row["outcome"]), - status_before=(str(row["status_before"]) if row["status_before"] is not None else None), - status_after=(str(row["status_after"]) if row["status_after"] is not None else None), - pid_before=int(row["pid_before"]) if row["pid_before"] is not None else None, - pid_after=int(row["pid_after"]) if row["pid_after"] is not None else None, - reason=str(row["reason"]), - error_code=str(row["error_code"]) if row["error_code"] is not None else None, - error_message=(str(row["error_message"]) if row["error_message"] is not None else None), - details=deserialize_lifecycle_details(str(row["details_json"])), - ) - - def _row_to_record(self, row: sqlite3.Row) -> BotRecord: - next_restart_at = row["next_restart_at"] - started_at = row["started_at"] - ready_at = row["ready_at"] - stopped_at = row["stopped_at"] - desired_updated_at = row["desired_updated_at"] - pending_since = row["pending_since"] - return BotRecord( - bot_id=row["bot_id"], - template_id=row["template_id"], - display_name=row["display_name"], - profile_path=row["profile_path"], - status=BotStatus(row["status"]), - pid=row["pid"], - restart_policy=RestartPolicy(row["restart_policy"]), - restart_backoff_seconds=float(row["restart_backoff_seconds"]), - restart_max_attempts=int(row["restart_max_attempts"]), - restart_attempts=int(row["restart_attempts"]), - next_restart_at=datetime.fromisoformat(next_restart_at) if next_restart_at else None, - started_at=datetime.fromisoformat(started_at) if started_at else None, - ready_at=datetime.fromisoformat(ready_at) if ready_at else None, - stopped_at=datetime.fromisoformat(stopped_at) if stopped_at else None, - last_exit_code=row["last_exit_code"], - last_error=row["last_error"], - last_transition_reason=row["last_transition_reason"], - desired_state=DesiredState(row["desired_state"]), - desired_revision=int(row["desired_revision"]), - desired_updated_at=( - datetime.fromisoformat(desired_updated_at) if desired_updated_at else None - ), - pending_operation_id=row["pending_operation_id"], - pending_action=row["pending_action"], - pending_since=datetime.fromisoformat(pending_since) if pending_since else None, - created_at=datetime.fromisoformat(row["created_at"]), - updated_at=datetime.fromisoformat(row["updated_at"]), - ) + return self._bot_lifecycle.history_payload(bot_id, limit, before) From a562779c0c1e8c7ffc8ca06e6287c626d333e30f Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:04:21 +0200 Subject: [PATCH 35/49] feat(state): make SQLite durability explicit --- .env.example | 4 + docs/ARCHITECTURE.md | 26 ++++- docs/COMPATIBILITY.md | 11 ++ docs/OPERATIONS.md | 37 ++++++- docs/SYSTEMD.md | 15 +++ systemd/zeus-api.service | 1 + systemd/zeus-reconcile.service | 1 + tests/test_api.py | 62 ++++++++++- tests/test_bot_lifecycle_store.py | 11 +- tests/test_idempotency_store.py | 11 +- tests/test_reconcile_store.py | 11 +- tests/test_repo_contracts.py | 34 ++++++ tests/test_sqlite_schema.py | 165 +++++++++++++++++++++++++++++- tests/test_supervisor_cli_api.py | 102 +++++++++++++++++- zeus/api.py | 5 +- zeus/cli.py | 10 +- zeus/config.py | 18 ++++ zeus/doctor.py | 5 +- zeus/sqlite_db.py | 16 ++- zeus/state.py | 10 +- 20 files changed, 534 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 6aef88e..f43cadf 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,10 @@ ZEUS_API_LOG_ENABLED=1 # Optional workspace-local runtime path. ZEUS_STATE_DIR=.zeus +# SQLite commit durability. NORMAL is the compatibility/default local setting; +# FULL adds power-loss durability at the cost of higher commit latency. +ZEUS_SQLITE_SYNCHRONOUS=NORMAL + # Optional: send SIGKILL if SIGTERM does not stop Hermes before the grace timeout. # Disabled by default to avoid interrupting a still-shutting-down gateway. ZEUS_STOP_KILL_AFTER_TIMEOUT=0 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 02645be..f3d70e1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,12 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.models`: Template, bot, and status validation. - `zeus.templates`: Bundled plus local TOML template discovery with duplicate ID checks. - `zeus.renderer`: Hermes profile rendering. -- `zeus.state`: SQLite bot projection and authoritative lifecycle ledger. +- `zeus.sqlite_db`: Shared SQLite connection factory and per-connection durability policy. +- `zeus.schema`: Schema-v6 initialization, compatibility guards, and forward migrations. +- `zeus.idempotency_store`: Durable API mutation claims and replay responses. +- `zeus.reconcile_store`: Persisted fleet reconciliation runs and ordered results. +- `zeus.bot_lifecycle_store`: Bot projection, intent, lifecycle ledger, history, and audit mirror. +- `zeus.state`: Compatibility facade composing the shared database and persistence stores. - `zeus.lifecycle`: Bounded lifecycle event types and recursively redacted details. - `zeus.request_context`: Locally generated request IDs and normalized route templates. - `zeus.api_errors`: Transport-neutral API exception classification. @@ -47,6 +52,25 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.cli`: Operator CLI. - `zeus.doctor`: Readiness diagnostics. +## SQLite Durability + +Zeus uses persistent SQLite WAL mode. `SQLiteDatabase` installs the selected +`ZEUS_SQLITE_SYNCHRONOUS` policy on every returned operational connection after +both newer-schema guards, foreign-key enforcement, and WAL setup. The raw +read-only schema preflight cannot commit and is intentionally not configured. + +Committed transactions survive an application or Zeus process crash under both +NORMAL and FULL. With NORMAL, SQLite omits a WAL sync on most commits, so a host +OS crash, hard reset, or power loss can roll back recently reported commits +after recovery while retaining WAL consistency. With FULL, SQLite syncs the WAL +at each commit to provide durability across OS crash or power loss, at the cost +of commit latency. + +The setting is per connection: every process that writes the same database must +select the intended mode. It covers SQLite only. Rendered profile files, PID +markers, locks, and the best-effort audit JSONL remain separate filesystem +state, and neither mode replaces backup and restore procedures. + ## Process Lifecycle 1. `zeus bot create` precomputes a profile, stages it under the profiles directory, diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index d4af822..ae427ad 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -31,6 +31,17 @@ The package metadata declares `requires-python = ">=3.11"`, while committed CI currently tests the versions listed above. A version absent from that matrix is not covered by the current automated compatibility claim. +## SQLite durability compatibility + +Unset or empty `ZEUS_SQLITE_SYNCHRONOUS` configuration remains NORMAL, as do +direct `StateStore(path)` and `SQLiteDatabase(path)` calls. Upgrading therefore +does not silently change local commit latency. FULL is an explicit +higher-durability option for deployments that accept its additional commit +latency. + +This policy does not change database structure or require a migration: the +schema remains schema v6 and every existing v6 database stays compatible. + ## Manual clean-host evidence [`scripts/fresh_vps_verify.sh`](../scripts/fresh_vps_verify.sh) provides a manual diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index c36a5d9..41c12be 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,5 +1,40 @@ # Operations +## SQLite Durability + +Zeus uses SQLite WAL mode and applies `ZEUS_SQLITE_SYNCHRONOUS` to every +operational connection. Unset or empty configuration defaults to NORMAL for +upgrade compatibility. The bundled API and reconcile services select FULL. + +Committed transactions survive an application or Zeus process crash under both +NORMAL and FULL. With NORMAL, SQLite omits a WAL sync on most commits; a host OS +crash, hard reset, or power loss can roll back recently reported commits after +recovery while WAL consistency is retained. With FULL, SQLite syncs the WAL at +each commit to provide durability across OS crash or power loss, at the cost of +commit latency. + +The policy is per connection. Every process that writes the same database, +including manual CLI and doctor commands capable of migration, must select the +intended mode. A NORMAL writer does not change an existing FULL connection, but +its own commits retain NORMAL power-loss semantics. For hosted manual work where +FULL durability is expected, carry the setting explicitly: + +```bash +sudo -u zeus env \ + ZEUS_STATE_DIR=/var/lib/zeus \ + ZEUS_SQLITE_SYNCHRONOUS=FULL \ + /opt/zeus/.venv/bin/zeus bot reconcile +sudo -u zeus env \ + ZEUS_STATE_DIR=/var/lib/zeus \ + ZEUS_SQLITE_SYNCHRONOUS=FULL \ + /opt/zeus/.venv/bin/zeus doctor --strict +``` + +The setting covers SQLite only. It does not make rendered profile files, PID +markers, locks, or the best-effort audit JSONL atomic with the database, replace +backups, or overcome storage that lies about flushes. Continue the backup and +restore procedures below under either mode. + ## Backup Back up `ZEUS_STATE_DIR` regularly. For the sample systemd deployment, that is @@ -59,7 +94,7 @@ Start services and verify the restored host: ```bash sudo systemctl start zeus-api sudo systemctl start zeus-reconcile.timer -sudo -u zeus env ZEUS_STATE_DIR=/var/lib/zeus \ +sudo -u zeus env ZEUS_STATE_DIR=/var/lib/zeus ZEUS_SQLITE_SYNCHRONOUS=FULL \ /opt/zeus/.venv/bin/zeus doctor --strict ``` diff --git a/docs/SYSTEMD.md b/docs/SYSTEMD.md index 4523728..d69b280 100644 --- a/docs/SYSTEMD.md +++ b/docs/SYSTEMD.md @@ -28,6 +28,7 @@ ZEUS_HERMES_BIN=/usr/local/bin/hermes ZEUS_HOST=127.0.0.1 ZEUS_PORT=4311 ZEUS_STATE_DIR=/var/lib/zeus +ZEUS_SQLITE_SYNCHRONOUS=FULL ``` Protect the file: @@ -39,6 +40,20 @@ sudo chmod 0640 /etc/zeus/zeus.env Provider keys such as `DEEPSEEK_API_KEY` can also live in this env file. Do not commit real keys. +Both bundled writer units, `zeus-api.service` and +`zeus-reconcile.service`, select `ZEUS_SQLITE_SYNCHRONOUS=FULL`. Keep the +setting consistent for every process that writes `/var/lib/zeus/zeus.db`, +including manual CLI and doctor commands. After changing the mode, restart both +writer units so newly opened connections receive it: + +```bash +sudo systemctl restart zeus-api zeus-reconcile.service +``` + +FULL applies to SQLite commits only and can add commit latency. It does not make +rendered profiles, PID markers, locks, or audit JSONL writes power-loss atomic; +retain the backup plan in [Operations](OPERATIONS.md). + Leave `ZEUS_ENV_PASSTHROUGH` unset unless Hermes needs selected proxy or certificate variables from the service environment. Profile `.env` files remain the preferred place for provider keys used by a bot. diff --git a/systemd/zeus-api.service b/systemd/zeus-api.service index 76c0a0a..0b96dde 100644 --- a/systemd/zeus-api.service +++ b/systemd/zeus-api.service @@ -15,6 +15,7 @@ EnvironmentFile=/etc/zeus/zeus.env Environment=ZEUS_HOST=127.0.0.1 Environment=ZEUS_PORT=4311 Environment=ZEUS_STATE_DIR=/var/lib/zeus +Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL ExecStart=/opt/zeus/.venv/bin/python -m zeus.api Restart=on-failure RestartSec=5s diff --git a/systemd/zeus-reconcile.service b/systemd/zeus-reconcile.service index 1ba1d6a..a9a8a53 100644 --- a/systemd/zeus-reconcile.service +++ b/systemd/zeus-reconcile.service @@ -9,6 +9,7 @@ User=zeus Group=zeus WorkingDirectory=/opt/zeus EnvironmentFile=/etc/zeus/zeus.env +Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL ExecStart=/opt/zeus/.venv/bin/zeus bot reconcile NoNewPrivileges=true PrivateTmp=true diff --git a/tests/test_api.py b/tests/test_api.py index eff9d82..7943e7e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import http.client +import io import json import os import socket @@ -12,7 +13,7 @@ import time import unittest from collections.abc import Iterator -from contextlib import closing, contextmanager +from contextlib import closing, contextmanager, redirect_stderr from datetime import UTC, datetime from http.server import ThreadingHTTPServer from pathlib import Path @@ -21,7 +22,7 @@ from zeus import api as api_module from zeus.api import make_handler, serve -from zeus.config import Settings +from zeus.config import Settings, SQLiteSynchronous from zeus.errors import ZeusConflictError from zeus.models import BotRecord, BotStatus, BotStatusResponse, TemplateError from zeus.process_lock import LockTimeoutError @@ -2522,5 +2523,62 @@ def bucket_factory(rate_per_minute: int, burst: int) -> TokenBucket: self.assertEqual(404, refilled_status) +class ApiSQLiteDurabilityTests(unittest.TestCase): + def test_make_handler_forwards_full_to_state_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env( + { + "ZEUS_STATE_DIR": str(Path(tmp) / "state"), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL", + } + ) + with ( + patch.object(Settings, "ensure_dirs"), + patch("zeus.api.StateStore") as store_type, + patch("zeus.api.ApiLogWriter"), + ): + make_handler(settings) + + store_type.assert_called_once_with( + settings.database_path, + synchronous=SQLiteSynchronous.FULL, + ) + + def test_api_main_rejects_invalid_mode_before_runtime_setup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + stderr = io.StringIO() + with ( + patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL; VACUUM", + }, + clear=True, + ), + patch("zeus.config.load_dotenv", return_value={}), + patch("zeus.api.serve") as serve_entrypoint, + patch("zeus.api.StateStore") as store_type, + patch("zeus.api.make_handler") as handler_factory, + patch("zeus.api.ThreadingHTTPServer") as server_type, + patch.object(Settings, "ensure_dirs") as ensure_dirs, + redirect_stderr(stderr), + ): + result = api_module.main([]) + + self.assertEqual(1, result) + self.assertEqual( + "Invalid Zeus configuration: ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL\n", + stderr.getvalue(), + ) + serve_entrypoint.assert_not_called() + store_type.assert_not_called() + handler_factory.assert_not_called() + server_type.assert_not_called() + ensure_dirs.assert_not_called() + self.assertFalse(state_dir.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_bot_lifecycle_store.py b/tests/test_bot_lifecycle_store.py index 4ce50a7..7eebeca 100644 --- a/tests/test_bot_lifecycle_store.py +++ b/tests/test_bot_lifecycle_store.py @@ -21,6 +21,7 @@ LIFECYCLE_SOURCES, BotLifecycleStore, ) +from zeus.config import SQLiteSynchronous from zeus.idempotency_store import IdempotencyStore from zeus.lifecycle import LifecycleEvent, LifecycleEventInput from zeus.models import BotRecord, BotStatus @@ -1083,7 +1084,15 @@ def test_state_facade_forwards_all_eighteen_methods_with_exact_signatures(self) actual_events = facade.list_lifecycle_events("coder", 10, 20) actual_history = facade.history_payload("coder", 10, 20) - self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) self.assertEqual([call(database)], schema_type.call_args_list) self.assertEqual([call(database)], idempotency_type.call_args_list) self.assertEqual([call(database)], reconcile_type.call_args_list) diff --git a/tests/test_idempotency_store.py b/tests/test_idempotency_store.py index 29e5cc7..5e9905e 100644 --- a/tests/test_idempotency_store.py +++ b/tests/test_idempotency_store.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock, call, patch from zeus import api as api_module +from zeus.config import SQLiteSynchronous from zeus.idempotency import IdempotencyClaim from zeus.idempotency_store import ( IDEMPOTENCY_HASH_RE, @@ -397,7 +398,15 @@ def test_state_facade_delegates_exact_keywords_and_propagates_results_and_errors expires_at=expires_at, ) - self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) self.assertEqual([call(database)], schema_type.call_args_list) self.assertEqual([call(database)], store_type.call_args_list) database.connect.assert_not_called() diff --git a/tests/test_reconcile_store.py b/tests/test_reconcile_store.py index c72ab2f..b235c68 100644 --- a/tests/test_reconcile_store.py +++ b/tests/test_reconcile_store.py @@ -11,6 +11,7 @@ from typing import ClassVar from unittest.mock import MagicMock, call, patch +from zeus.config import SQLiteSynchronous from zeus.idempotency_store import IdempotencyStore from zeus.reconcile_store import RECONCILE_COUNTER_COLUMNS, ReconcileStore from zeus.reconciliation import ( @@ -487,7 +488,15 @@ def test_state_facade_uses_shared_database_and_delegates_verbatim_signatures(sel ) actual_loaded = facade.get_reconcile_run(run.run_id) - self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) self.assertEqual([call(database)], schema_type.call_args_list) self.assertEqual([call(database)], idempotency_type.call_args_list) self.assertEqual([call(database)], reconcile_type.call_args_list) diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 87598e0..06dd9f0 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -1241,6 +1241,40 @@ def test_brainx_maintainer_is_credited(self) -> None: self.assertIn("https://github.com/brainx", architecture) self.assertIn('Maintainer = "https://github.com/brainx"', pyproject) + def test_sqlite_durability_policy_is_consistent_across_runtime_and_docs(self) -> None: + env_example = Path(".env.example").read_text(encoding="utf-8") + api_service = Path("systemd/zeus-api.service").read_text(encoding="utf-8") + reconcile_service = Path("systemd/zeus-reconcile.service").read_text(encoding="utf-8") + timer = Path("systemd/zeus-reconcile.timer").read_text(encoding="utf-8") + systemd_docs = Path("docs/SYSTEMD.md").read_text(encoding="utf-8") + compatibility = Path("docs/COMPATIBILITY.md").read_text(encoding="utf-8").lower() + architecture = Path("docs/ARCHITECTURE.md").read_text(encoding="utf-8").lower() + operations = Path("docs/OPERATIONS.md").read_text(encoding="utf-8").lower() + + self.assertEqual(1, env_example.count("ZEUS_SQLITE_SYNCHRONOUS=NORMAL")) + self.assertNotIn("ZEUS_SQLITE_SYNCHRONOUS=FULL", env_example) + for service in (api_service, reconcile_service): + with self.subTest(service=service[:32]): + self.assertEqual( + 1, + service.count("Environment=ZEUS_SQLITE_SYNCHRONOUS=FULL"), + ) + self.assertIn("EnvironmentFile=/etc/zeus/zeus.env", service) + self.assertNotIn("ZEUS_SQLITE_SYNCHRONOUS", timer) + self.assertIn("ZEUS_SQLITE_SYNCHRONOUS=FULL", systemd_docs) + self.assertIn("normal", compatibility) + self.assertIn("unset", compatibility) + self.assertIn("empty", compatibility) + self.assertIn("schema v6", compatibility) + for text in (architecture, operations): + self.assertIn("process crash", text) + self.assertIn("power loss", text) + self.assertIn("every process that writes the same database", operations) + self.assertIn("sqlite only", operations) + self.assertIn("rendered profile files", operations) + self.assertIn("audit jsonl", operations) + self.assertIn("backup", operations) + def test_root_scripts_credit_repo_maintainers(self) -> None: script_paths = [Path("Makefile"), *sorted(Path("scripts").glob("*.sh"))] diff --git a/tests/test_sqlite_schema.py b/tests/test_sqlite_schema.py index 53403b8..5435d41 100644 --- a/tests/test_sqlite_schema.py +++ b/tests/test_sqlite_schema.py @@ -6,9 +6,10 @@ import unittest from contextlib import closing from pathlib import Path -from typing import ClassVar +from typing import Any, ClassVar, cast from unittest.mock import MagicMock, call, patch +from zeus.config import SQLiteSynchronous from zeus.schema import SCHEMA_VERSION, SchemaManager, _preflight_schema_compatibility from zeus.sqlite_db import SQLiteDatabase from zeus.state import StateStore @@ -662,7 +663,15 @@ def test_state_store_delegates_without_opening_during_construction(self) -> None store.init() store.migrate() - self.assertEqual([call(database_path)], database_type.call_args_list) + self.assertEqual( + [ + call( + database_path, + synchronous=SQLiteSynchronous.NORMAL, + ) + ], + database_type.call_args_list, + ) self.assertEqual([call(database)], schema_type.call_args_list) database.connect.assert_called_once_with() schema.init.assert_called_once_with() @@ -828,5 +837,157 @@ def connect_spy(*args: object, **kwargs: object) -> sqlite3.Connection: self.assertFalse(any(sql.lstrip().upper().startswith("UPDATE") for sql in traces)) +class SQLiteDurabilityTests(unittest.TestCase): + def test_database_default_is_normal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase(Path(tmp) / "zeus.db") + self.assertIs(SQLiteSynchronous.NORMAL, database.synchronous) + SchemaManager(database).init() + with closing(database.connect()) as conn: + synchronous = conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, synchronous) + + def test_database_full_is_applied_to_every_open(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database = SQLiteDatabase( + Path(tmp) / "zeus.db", + synchronous="FULL", + ) + self.assertIs(SQLiteSynchronous.FULL, database.synchronous) + SchemaManager(database).init() + with ( + closing(database.connect()) as first, + closing(database.connect()) as second, + database.immediate() as immediate, + ): + observed = [ + conn.execute("PRAGMA synchronous").fetchone()[0] + for conn in (first, second, immediate) + ] + + self.assertEqual([2, 2, 2], observed) + + def test_operational_connection_installs_full_in_guarded_order(self) -> None: + events: list[str] = [] + connection = MagicMock(spec=sqlite3.Connection) + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "nested" / "zeus.db" + database = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + + def preflight(path: Path) -> None: + self.assertTrue(path.parent.is_dir()) + events.append("preflight") + + def open_connection(path: Path) -> sqlite3.Connection: + self.assertEqual(database_path, path) + events.append("open") + return connection + + def guard(conn: sqlite3.Connection) -> None: + self.assertIs(connection, conn) + events.append("guard") + + def execute(sql: str) -> MagicMock: + events.append(sql) + return MagicMock() + + connection.execute.side_effect = execute + with ( + patch("zeus.sqlite_db._preflight_schema_compatibility", side_effect=preflight), + patch("zeus.sqlite_db.sqlite3.connect", side_effect=open_connection), + patch("zeus.sqlite_db._assert_schema_compatible", side_effect=guard), + ): + actual = database.connect() + + self.assertIs(connection, actual) + self.assertIs(sqlite3.Row, connection.row_factory) + self.assertEqual( + [ + "preflight", + "open", + "guard", + "PRAGMA foreign_keys=ON", + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=FULL", + "PRAGMA busy_timeout=5000", + ], + events, + ) + + def test_connections_keep_independent_synchronous_modes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + normal = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.NORMAL) + full = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + SchemaManager(normal).init() + with closing(normal.connect()) as normal_conn, closing(full.connect()) as full_conn: + normal_value = normal_conn.execute("PRAGMA synchronous").fetchone()[0] + full_value = full_conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, normal_value) + self.assertEqual(2, full_value) + + def test_state_store_default_remains_normal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = StateStore(Path(tmp) / "zeus.db") + store.init() + with closing(store.connect()) as conn: + synchronous = conn.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(1, synchronous) + + def test_state_store_rejects_invalid_mode_before_parent_creation(self) -> None: + invalid_values: tuple[object, ...] = ( + "OFF", + "EXTRA", + "full", + "FULL ", + "1", + "FULL; VACUUM", + 1, + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for index, value in enumerate(invalid_values): + with self.subTest(value=value): + database_path = root / str(index) / "zeus.db" + with self.assertRaises(ValueError): + StateStore(database_path, synchronous=cast(Any, value)) + self.assertFalse(database_path.parent.exists()) + + def test_state_store_passes_one_configured_database_to_every_delegate(self) -> None: + database_path = Path("state") / "zeus.db" + database = MagicMock(spec=SQLiteDatabase) + with ( + patch("zeus.state.SQLiteDatabase", return_value=database) as database_type, + patch("zeus.state.SchemaManager") as schema_type, + patch("zeus.state.IdempotencyStore") as idempotency_type, + patch("zeus.state.ReconcileStore") as reconcile_type, + patch("zeus.state.BotLifecycleStore") as lifecycle_type, + ): + StateStore(database_path, synchronous=SQLiteSynchronous.FULL) + + database_type.assert_called_once_with( + database_path, + synchronous=SQLiteSynchronous.FULL, + ) + for child_type in ( + schema_type, + idempotency_type, + reconcile_type, + lifecycle_type, + ): + child_type.assert_called_once_with(database) + + def test_durability_configuration_does_not_change_schema_version(self) -> None: + self.assertEqual(6, SCHEMA_VERSION) + for mode in (SQLiteSynchronous.NORMAL, SQLiteSynchronous.FULL): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path, synchronous=mode).init() + self.assertEqual(EXPECTED_DDL_CATALOGS["fresh"], _ddl_catalog(database_path)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_supervisor_cli_api.py b/tests/test_supervisor_cli_api.py index e1081ca..98221a1 100644 --- a/tests/test_supervisor_cli_api.py +++ b/tests/test_supervisor_cli_api.py @@ -18,15 +18,15 @@ from contextlib import closing, redirect_stderr, redirect_stdout from datetime import UTC, datetime, timedelta from pathlib import Path -from unittest.mock import patch +from unittest.mock import call, patch from zeus import __version__ from zeus.api import main as api_main from zeus.api import make_handler -from zeus.cli import _parse_env, _services, build_parser +from zeus.cli import _demo_services, _parse_env, _services, build_parser from zeus.cli import main as cli_main -from zeus.config import Settings -from zeus.doctor import _check_runtime_paths, run_doctor +from zeus.config import Settings, SQLiteSynchronous +from zeus.doctor import _check_bots, _check_runtime_paths, run_doctor from zeus.envfile import parse_env_text from zeus.errors import BotArchiveError, BotDeleteError, BotExistsError from zeus.hermes_adapter import HermesAdapter @@ -5120,5 +5120,99 @@ def test_api_allow_unauth_reads_keeps_mutations_locked(self) -> None: server.server_close() +class SQLiteDurabilityConfigurationTests(unittest.TestCase): + def test_settings_accept_exact_modes_and_default_missing_or_empty_to_normal(self) -> None: + cases = ( + (None, SQLiteSynchronous.NORMAL), + ("", SQLiteSynchronous.NORMAL), + ("NORMAL", SQLiteSynchronous.NORMAL), + ("FULL", SQLiteSynchronous.FULL), + ) + for raw_value, expected in cases: + with self.subTest(raw_value=raw_value): + env: dict[str, str] = {} + if raw_value is not None: + env["ZEUS_SQLITE_SYNCHRONOUS"] = raw_value + with patch("zeus.config.load_dotenv", return_value={}): + settings = Settings.from_env(env) + self.assertIs(expected, settings.sqlite_synchronous) + + def test_invalid_modes_fail_exactly_before_creating_state(self) -> None: + invalid_values = ("OFF", "EXTRA", "full", "FULL ", "1", "FULL; VACUUM") + for value in invalid_values: + with self.subTest(value=value), tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + with ( + patch("zeus.config.load_dotenv", return_value={}), + self.assertRaisesRegex( + ValueError, + r"^ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL$", + ), + ): + Settings.from_env( + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": value, + } + ) + self.assertFalse(state_dir.exists()) + + def test_cli_demo_and_doctor_forward_full_by_name(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = Settings.from_env( + { + "ZEUS_STATE_DIR": str(Path(tmp) / "state"), + "ZEUS_SQLITE_SYNCHRONOUS": "FULL", + } + ) + with ( + patch.object(Settings, "ensure_dirs"), + patch("zeus.cli.StateStore") as cli_store_type, + patch("zeus.cli._demo_hermes_bin", return_value="fake-hermes"), + patch("zeus.cli._demo_cmdline_reader", return_value=lambda _pid: []), + ): + _services(settings) + _demo_services(settings, "coder") + + with patch("zeus.doctor.StateStore") as doctor_store_type: + _check_bots(settings) + + expected = call( + settings.database_path, + synchronous=SQLiteSynchronous.FULL, + ) + self.assertEqual([expected, expected], cli_store_type.call_args_list) + self.assertEqual([expected], doctor_store_type.call_args_list) + + def test_cli_main_rejects_invalid_mode_before_runtime_state_or_services(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state_dir = Path(tmp) / "missing-state" + stderr = io.StringIO() + with ( + patch.dict( + os.environ, + { + "ZEUS_STATE_DIR": str(state_dir), + "ZEUS_SQLITE_SYNCHRONOUS": "OFF", + }, + clear=True, + ), + patch("zeus.config.load_dotenv", return_value={}), + patch("zeus.cli.StateStore") as store_type, + patch.object(Settings, "ensure_dirs") as ensure_dirs, + redirect_stderr(stderr), + ): + result = cli_main(["bot", "list"]) + + self.assertEqual(1, result) + self.assertEqual( + "Invalid Zeus configuration: ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL\n", + stderr.getvalue(), + ) + store_type.assert_not_called() + ensure_dirs.assert_not_called() + self.assertFalse(state_dir.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/zeus/api.py b/zeus/api.py index a3286cf..980c188 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -227,7 +227,10 @@ def make_handler(settings: Settings) -> type[BaseHTTPRequestHandler]: settings.state_dir / "logs" / "api.jsonl", enabled=settings.api_log_enabled, ) - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() supervisor = Supervisor( store, diff --git a/zeus/cli.py b/zeus/cli.py index 7fd7408..829b2b3 100644 --- a/zeus/cli.py +++ b/zeus/cli.py @@ -386,7 +386,10 @@ def _positive_event_id(value: str) -> int: def _services(settings: Settings) -> tuple[StateStore, Supervisor]: settings.ensure_dirs() - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() return store, Supervisor( store, @@ -403,7 +406,10 @@ def _services(settings: Settings) -> tuple[StateStore, Supervisor]: def _demo_services(settings: Settings, bot_id: str) -> tuple[StateStore, Supervisor, str]: settings.ensure_dirs() fake_hermes = _demo_hermes_bin(settings) - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) store.init() return ( store, diff --git a/zeus/config.py b/zeus/config.py index 7f5f9cd..a40c31b 100644 --- a/zeus/config.py +++ b/zeus/config.py @@ -3,6 +3,7 @@ import os from collections.abc import Mapping from dataclasses import dataclass +from enum import StrEnum from ipaddress import ip_address from pathlib import Path @@ -10,6 +11,11 @@ from zeus.private_io import ensure_private_directory, nofollow_absolute_path +class SQLiteSynchronous(StrEnum): + NORMAL = "NORMAL" + FULL = "FULL" + + def load_dotenv(path: Path = Path(".env")) -> dict[str, str]: if not path.exists(): return {} @@ -41,11 +47,13 @@ class Settings: api_idempotency_retention_seconds: int api_idempotency_max_records: int api_log_enabled: bool = True + sqlite_synchronous: SQLiteSynchronous = SQLiteSynchronous.NORMAL @classmethod def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: merged: dict[str, str] = load_dotenv() merged.update(dict(os.environ if env is None else env)) + sqlite_synchronous = _sqlite_synchronous_env(merged) state_dir = nofollow_absolute_path(Path(merged.get("ZEUS_STATE_DIR") or ".zeus")) port = _port_env(merged, "ZEUS_PORT", default=4311) return cls( @@ -140,6 +148,7 @@ def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: maximum=1_000_000, ), api_log_enabled=merged.get("ZEUS_API_LOG_ENABLED", "1") == "1", + sqlite_synchronous=sqlite_synchronous, ) def ensure_dirs(self) -> None: @@ -173,6 +182,15 @@ def validate_api_exposure(host: str, api_key: str | None, allow_unauth_reads: bo raise ValueError("non-loopback API bind requires ZEUS_API_KEY with at least 16 characters") +def _sqlite_synchronous_env(env: Mapping[str, str]) -> SQLiteSynchronous: + raw_value = env.get("ZEUS_SQLITE_SYNCHRONOUS") + value = SQLiteSynchronous.NORMAL.value if raw_value is None or raw_value == "" else raw_value + try: + return SQLiteSynchronous(value) + except ValueError as exc: + raise ValueError("ZEUS_SQLITE_SYNCHRONOUS must be NORMAL or FULL") from exc + + def _port_env(env: Mapping[str, str], name: str, *, default: int) -> int: raw_value = env.get(name) if raw_value is None or raw_value == "": diff --git a/zeus/doctor.py b/zeus/doctor.py index c2d763a..ad3d3ba 100644 --- a/zeus/doctor.py +++ b/zeus/doctor.py @@ -315,7 +315,10 @@ def _check_api_auth(settings: Settings) -> DoctorCheck: def _check_bots(settings: Settings) -> list[DoctorCheck]: - store = StateStore(settings.database_path) + store = StateStore( + settings.database_path, + synchronous=settings.sqlite_synchronous, + ) if not settings.database_path.exists(): return [DoctorCheck("bots", "pass", "No bot registry exists yet")] try: diff --git a/zeus/sqlite_db.py b/zeus/sqlite_db.py index 4710d8f..1402d9a 100644 --- a/zeus/sqlite_db.py +++ b/zeus/sqlite_db.py @@ -5,12 +5,24 @@ from contextlib import closing, contextmanager from pathlib import Path +from zeus.config import SQLiteSynchronous from zeus.schema import _assert_schema_compatible, _preflight_schema_compatibility +_SYNCHRONOUS_PRAGMA = { + SQLiteSynchronous.NORMAL: "PRAGMA synchronous=NORMAL", + SQLiteSynchronous.FULL: "PRAGMA synchronous=FULL", +} + class SQLiteDatabase: - def __init__(self, database_path: Path | str) -> None: + def __init__( + self, + database_path: Path | str, + *, + synchronous: SQLiteSynchronous | str = SQLiteSynchronous.NORMAL, + ) -> None: self.database_path = Path(database_path) + self.synchronous = SQLiteSynchronous(synchronous) def connect(self) -> sqlite3.Connection: self.database_path.parent.mkdir(parents=True, exist_ok=True) @@ -21,7 +33,7 @@ def connect(self) -> sqlite3.Connection: _assert_schema_compatible(conn) conn.execute("PRAGMA foreign_keys=ON") conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA synchronous=NORMAL") + conn.execute(_SYNCHRONOUS_PRAGMA[self.synchronous]) conn.execute("PRAGMA busy_timeout=5000") return conn except Exception: diff --git a/zeus/state.py b/zeus/state.py index 3a61555..94f5d40 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -10,6 +10,7 @@ from zeus.bot_lifecycle_store import LIFECYCLE_INTENT_ACTIONS as LIFECYCLE_INTENT_ACTIONS from zeus.bot_lifecycle_store import LIFECYCLE_SOURCES as LIFECYCLE_SOURCES from zeus.bot_lifecycle_store import BotLifecycleStore +from zeus.config import SQLiteSynchronous from zeus.idempotency import IdempotencyClaim from zeus.idempotency_store import IDEMPOTENCY_HASH_RE as IDEMPOTENCY_HASH_RE from zeus.idempotency_store import IDEMPOTENCY_OWNER_RE as IDEMPOTENCY_OWNER_RE @@ -34,8 +35,13 @@ class StateStore: - def __init__(self, database_path: Path | str) -> None: - self._database = SQLiteDatabase(database_path) + def __init__( + self, + database_path: Path | str, + *, + synchronous: SQLiteSynchronous | str = SQLiteSynchronous.NORMAL, + ) -> None: + self._database = SQLiteDatabase(database_path, synchronous=synchronous) self._schema = SchemaManager(self._database) self._idempotency = IdempotencyStore(self._database) self._reconcile = ReconcileStore(self._database) From d3d4d39d07efd25774dd411f882d84db0777b797 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:22:05 +0200 Subject: [PATCH 36/49] refactor(supervisor): extract process identity checks --- tests/test_process_identity.py | 518 +++++++++++++++++++++++++++++++++ zeus/process_identity.py | 292 +++++++++++++++++++ zeus/supervisor.py | 261 +++-------------- 3 files changed, 855 insertions(+), 216 deletions(-) create mode 100644 tests/test_process_identity.py create mode 100644 zeus/process_identity.py diff --git a/tests/test_process_identity.py b/tests/test_process_identity.py new file mode 100644 index 0000000..31b381f --- /dev/null +++ b/tests/test_process_identity.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import ast +import errno +import os +import subprocess +import sys +import tempfile +import unittest +from dataclasses import FrozenInstanceError +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import zeus.process_identity as identity +import zeus.supervisor as supervisor_module + + +class ProcessReaderTests(unittest.TestCase): + def test_linux_cmdline_parses_proc_bytes_and_handles_missing_process(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + proc_root = Path(tmp) / "proc" + pid_dir = proc_root / "4321" + pid_dir.mkdir(parents=True) + (pid_dir / "cmdline").write_bytes(b"hermes\0-p\0coder\0gateway\0run\0invalid-\xff\0") + + argv = identity.read_linux_cmdline(4321, proc_root=proc_root) + + self.assertEqual( + ["hermes", "-p", "coder", "gateway", "run", "invalid-\udcff"], + argv, + ) + self.assertEqual([], identity.read_linux_cmdline(9999, proc_root=proc_root)) + + def test_linux_start_fingerprint_reads_proc_stat_field_twenty_two(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + proc_root = Path(tmp) / "proc" + pid_dir = proc_root / "4321" + pid_dir.mkdir(parents=True) + fields = ["S", *["0"] * 18, "987654321", "0"] + (pid_dir / "stat").write_text( + f"4321 (hermes gateway) {' '.join(fields)}\n", + encoding="utf-8", + ) + + fingerprint = identity.read_linux_process_start_fingerprint( + 4321, + proc_root=proc_root, + ) + + self.assertEqual("linux:/proc-starttime:987654321", fingerprint) + self.assertIsNone( + identity.read_linux_process_start_fingerprint(9999, proc_root=proc_root) + ) + + def test_darwin_cmdline_uses_fixed_ps_command_and_parses_quotes(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout='"/Applications/Hermes CLI/hermes" -p coder gateway run\n', + ) + with patch("zeus.process_identity.subprocess.run", return_value=completed) as run: + argv = identity.read_darwin_cmdline(4321) + + self.assertEqual( + ["/Applications/Hermes CLI/hermes", "-p", "coder", "gateway", "run"], + argv, + ) + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_darwin_start_fingerprint_normalizes_ps_lstart(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout=" Mon Jun 29 16:54:50 2026\n", + ) + with patch("zeus.process_identity.subprocess.run", return_value=completed) as run: + fingerprint = identity.read_darwin_process_start_fingerprint(4321) + + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "lstart="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_platform_dispatchers_select_linux_darwin_or_unsupported(self) -> None: + with ( + patch("zeus.process_identity.platform.system", return_value="Linux"), + patch( + "zeus.process_identity.read_linux_cmdline", + return_value=["linux"], + ) as linux_cmdline, + patch( + "zeus.process_identity.read_linux_process_start_fingerprint", + return_value="linux-start", + ) as linux_start, + ): + self.assertEqual(["linux"], identity.read_process_cmdline(11)) + self.assertEqual("linux-start", identity.read_process_start_fingerprint(11)) + linux_cmdline.assert_called_once_with(11) + linux_start.assert_called_once_with(11) + + with ( + patch("zeus.process_identity.platform.system", return_value="Darwin"), + patch( + "zeus.process_identity.read_darwin_cmdline", + return_value=["darwin"], + ) as darwin_cmdline, + patch( + "zeus.process_identity.read_darwin_process_start_fingerprint", + return_value="darwin-start", + ) as darwin_start, + ): + self.assertEqual(["darwin"], identity.read_process_cmdline(22)) + self.assertEqual("darwin-start", identity.read_process_start_fingerprint(22)) + darwin_cmdline.assert_called_once_with(22) + darwin_start.assert_called_once_with(22) + + with patch("zeus.process_identity.platform.system", return_value="FreeBSD"): + self.assertIsNone(identity.read_process_cmdline(33)) + self.assertIsNone(identity.read_process_start_fingerprint(33)) + + +class CommandIdentityTests(unittest.TestCase): + @staticmethod + def _write_executable(path: Path, text: str = "#!/bin/sh\n") -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + return str(path.resolve()) + + def test_gateway_command_classifier_preserves_complete_shape_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_bin = self._write_executable(root / "bin" / "hermes") + fake_hermes = self._write_executable(root / "bin" / "fake-hermes") + cases = ( + (["hermes", "-p", "coder", "gateway", "run"], True, "direct-hermes", "ok"), + ([hermes_bin, "-p", "coder", "gateway", "run"], True, "direct-hermes", "ok"), + ( + [sys.executable, hermes_bin, "-p", "coder", "gateway", "run"], + True, + "python-script-wrapper", + "ok", + ), + ( + [sys.executable, fake_hermes, "-p", "coder", "gateway", "run"], + False, + "python-script-wrapper", + "untrusted-executable", + ), + ( + [sys.executable, hermes_bin, "-p", "other", "gateway", "run"], + False, + "python-script-wrapper", + "wrong-bot-id", + ), + ( + [sys.executable, hermes_bin, "-p", "coder", "doctor"], + False, + "python-script-wrapper", + "wrong-command-intent", + ), + ( + ["python", "-c", "sleep", "-p", "coder", "gateway", "run"], + False, + "python-script-wrapper", + "wrong-command-intent", + ), + (["sleep", "60"], False, "direct-hermes", "wrong-command-intent"), + ( + [hermes_bin, "gateway", "run"], + False, + "direct-hermes", + "wrong-command-intent", + ), + ( + [hermes_bin, "-p", "coder", "-p", "other", "gateway", "run"], + False, + "direct-hermes", + "wrong-command-intent", + ), + ) + with patch.dict(os.environ, {"PATH": str(root / "bin")}): + for argv, verified, classification, reason in cases: + with self.subTest(argv=argv): + check = identity.verify_gateway_command( + argv, + "coder", + hermes_bin, + require_trusted_path=True, + ) + self.assertEqual(verified, check.verified) + self.assertEqual(classification, check.classification) + self.assertEqual(reason, check.reason) + + def test_command_shape_and_python_interpreter_classification_are_bounded(self) -> None: + for command in ("python", "python3", "PYTHON3.11", "/usr/bin/python3.12"): + with self.subTest(command=command): + self.assertTrue(identity.looks_like_python_interpreter(command)) + for command in ("pypy3", "python3.11-config", "python-script", ""): + with self.subTest(command=command): + self.assertFalse(identity.looks_like_python_interpreter(command)) + + self.assertEqual("empty", identity.safe_command_shape([])) + self.assertEqual( + "direct-hermes hermes -p gateway run", + identity.safe_command_shape(["hermes", "-p", "secret-bot", "gateway", "run"]), + ) + self.assertEqual( + "python-script-wrapper hermes -p gateway run", + identity.safe_command_shape( + [sys.executable, "/opt/hermes", "-p", "secret-bot", "gateway", "run"] + ), + ) + self.assertEqual( + "direct-hermes unrecognized", + identity.safe_command_shape(["hermes", "doctor"]), + ) + + def test_executable_launcher_and_trusted_hermes_resolution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_bin = self._write_executable(root / "real" / "hermes") + path_hermes = self._write_executable(root / "bin" / "path-hermes") + launcher = root / "bin" / "hermes-launcher" + launcher_path = self._write_executable( + launcher, + f'#!/bin/sh\n# delegated install\nexec "{hermes_bin}" "$@"\n', + ) + + self.assertIsNone(identity.resolve_executable("")) + self.assertEqual( + path_hermes, identity.resolve_executable("path-hermes", str(root / "bin")) + ) + self.assertEqual(launcher_path, identity.resolve_executable(launcher_path)) + self.assertEqual(hermes_bin, identity.resolve_launcher_exec_target(launcher_path)) + self.assertEqual( + {launcher_path, hermes_bin}, + identity.trusted_hermes_paths(launcher_path), + ) + self.assertEqual({hermes_bin}, identity.trusted_hermes_paths(hermes_bin)) + + def test_launcher_resolution_rejects_untrusted_or_malformed_scripts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + not_hermes = self._write_executable(root / "bin" / "not-hermes") + cases = ( + ("plain", f'exec "{not_hermes}" "$@"\n'), + ("no-exec", "#!/bin/sh\nprintf ready\n"), + ("path-lookup", '#!/bin/sh\nexec hermes "$@"\n'), + ("malformed", "#!/bin/sh\nexec 'unterminated\n"), + ("wrong-target", f'#!/bin/sh\nexec "{not_hermes}" "$@"\n'), + ) + for name, text in cases: + with self.subTest(name=name): + script = root / name + script.write_text(text, encoding="utf-8") + self.assertIsNone(identity.resolve_launcher_exec_target(str(script))) + + +class PidStateTests(unittest.TestCase): + @staticmethod + def _raises(exc: BaseException) -> identity.PidAliveFn: + def raise_error(_pid: int) -> bool: + raise exc + + return raise_error + + def test_pid_state_classifies_callbacks_and_os_errors_exactly(self) -> None: + cases = ( + ("alive", lambda _pid: True, identity.PidState.alive), + ("false", lambda _pid: False, identity.PidState.dead), + ( + "esrch", + self._raises(OSError(errno.ESRCH, "no such process")), + identity.PidState.dead, + ), + ( + "eperm", + self._raises(OSError(errno.EPERM, "operation not permitted")), + identity.PidState.unknown, + ), + ( + "process-lookup", + self._raises(ProcessLookupError(errno.ESRCH, "no such process")), + identity.PidState.dead, + ), + ( + "permission", + self._raises(PermissionError(errno.EPERM, "operation not permitted")), + identity.PidState.unknown, + ), + ) + for name, callback, expected in cases: + with self.subTest(name=name): + self.assertIs(expected, identity.pid_state(4321, pid_alive_fn=callback)) + + def test_pid_state_default_probes_with_signal_zero(self) -> None: + with patch("zeus.process_identity.os.kill") as kill: + self.assertIs(identity.PidState.alive, identity.pid_state(4321)) + kill.assert_called_once_with(4321, 0) + + +class ProcessStartIdentityTests(unittest.TestCase): + def test_process_start_requirement_is_explicit_and_platform_bounded(self) -> None: + self.assertTrue(identity.process_start_fingerprint_required("Linux")) + self.assertTrue(identity.process_start_fingerprint_required("Darwin")) + self.assertFalse(identity.process_start_fingerprint_required("FreeBSD")) + self.assertFalse(identity.process_start_fingerprint_required("linux")) + + def test_process_start_fingerprint_validation_is_exact(self) -> None: + self.assertTrue(identity.valid_process_start_fingerprint("x")) + self.assertTrue(identity.valid_process_start_fingerprint("x" * 512)) + for value in (None, "", "x" * 513, 1, True, b"start"): + with self.subTest(value=value): + self.assertFalse(identity.valid_process_start_fingerprint(value)) + + def test_process_start_comparison_preserves_required_and_optional_semantics(self) -> None: + cases = ( + ("same", "same", True, None), + (None, "same", True, "process start fingerprint is unavailable"), + ("same", None, True, "process start fingerprint is unavailable"), + ("old", "new", True, "process start fingerprint does not match"), + (None, None, False, None), + ("same", "same", False, None), + (None, "live", False, "process start fingerprint does not match"), + ("marker", None, False, "process start fingerprint is unavailable"), + ("old", "new", False, "process start fingerprint does not match"), + ) + for marker, live, required, expected in cases: + with self.subTest(marker=marker, live=live, required=required): + self.assertEqual( + expected, + identity.process_start_identity_error( + marker, + live, + fingerprint_required=required, + ), + ) + + +class FrozenBoundaryTests(unittest.TestCase): + def test_identity_types_are_frozen_and_finite(self) -> None: + self.assertEqual( + {"alive", "dead", "unknown"}, + {state.value for state in identity.PidState}, + ) + check = identity.CommandCheck(True, "ok", "direct-hermes") + with self.assertRaises(FrozenInstanceError): + check.reason = "changed" # type: ignore[misc] + + def test_process_identity_has_only_standard_library_dependencies(self) -> None: + module_path = Path(identity.__file__) + source = module_path.read_text(encoding="utf-8") + tree = ast.parse(source) + imported_roots: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imported_roots.add(node.module.split(".", 1)[0]) + + self.assertNotIn("zeus", imported_roots) + self.assertNotIn("sqlite3", imported_roots) + self.assertNotIn("signal", imported_roots) + self.assertNotIn("StateStore", source) + self.assertNotIn("BotRecord", source) + + +class SupervisorCompatibilityTests(unittest.TestCase): + def test_legacy_types_and_pure_helpers_remain_direct_aliases(self) -> None: + self.assertIs(identity.PidState, supervisor_module._PidState) + self.assertIs(identity.CommandCheck, supervisor_module._CommandCheck) + self.assertIs(identity.PidAliveFn, supervisor_module.PidAliveFn) + self.assertIs(identity.CmdlineReader, supervisor_module.CmdlineReader) + self.assertIs( + identity.ProcStartFingerprintReader, + supervisor_module.ProcStartFingerprintReader, + ) + aliases = { + "_read_linux_cmdline": identity.read_linux_cmdline, + "_read_linux_process_start_fingerprint": ( + identity.read_linux_process_start_fingerprint + ), + "_verify_gateway_command": identity.verify_gateway_command, + "_looks_like_python_interpreter": identity.looks_like_python_interpreter, + "_resolve_executable": identity.resolve_executable, + "_trusted_hermes_paths": identity.trusted_hermes_paths, + "_resolve_launcher_exec_target": identity.resolve_launcher_exec_target, + "_safe_command_shape": identity.safe_command_shape, + } + for old_name, new_value in aliases.items(): + with self.subTest(old_name=old_name): + self.assertIs(new_value, getattr(supervisor_module, old_name)) + + def test_legacy_cmdline_dispatch_uses_current_supervisor_globals(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout='"/patched/hermes" -p coder gateway run\n', + ) + with ( + patch("zeus.supervisor.platform.system", return_value="Darwin") as system, + patch("zeus.supervisor.subprocess.run", return_value=completed) as run, + ): + argv = supervisor_module._read_process_cmdline(4321) + + self.assertEqual(["/patched/hermes", "-p", "coder", "gateway", "run"], argv) + system.assert_called_once_with() + run.assert_called_once_with( + ["/bin/ps", "-p", "4321", "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + + def test_legacy_direct_darwin_readers_use_current_supervisor_subprocess(self) -> None: + command = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="/patched/hermes -p coder gateway run\n", + ) + started = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="Mon Jun 29 16:54:50 2026\n", + ) + with patch("zeus.supervisor.subprocess.run", side_effect=[command, started]) as run: + argv = supervisor_module._read_darwin_cmdline(4321) + fingerprint = supervisor_module._read_darwin_process_start_fingerprint(4321) + + self.assertEqual(["/patched/hermes", "-p", "coder", "gateway", "run"], argv) + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + self.assertEqual(2, run.call_count) + + def test_legacy_start_dispatch_uses_current_platform_and_subprocess(self) -> None: + completed = subprocess.CompletedProcess( + args=["ps"], + returncode=0, + stdout="Mon Jun 29 16:54:50 2026\n", + ) + with ( + patch("zeus.supervisor.platform.system", return_value="Darwin") as system, + patch("zeus.supervisor.subprocess.run", return_value=completed) as run, + ): + fingerprint = supervisor_module._read_process_start_fingerprint(4321) + + self.assertEqual("darwin:ps-lstart:Mon Jun 29 16:54:50 2026", fingerprint) + system.assert_called_once_with() + run.assert_called_once() + + def test_supervisor_pid_state_shim_uses_live_callback_or_current_os_kill(self) -> None: + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.pid_alive_fn = None + with patch("zeus.supervisor.os.kill") as kill: + self.assertIs(identity.PidState.alive, supervisor._pid_state(4321)) + supervisor.pid_alive_fn = lambda _pid: False + self.assertIs(identity.PidState.dead, supervisor._pid_state(4321)) + kill.assert_called_once_with(4321, 0) + + supervisor.pid_alive_fn = self._permission_denied + self.assertIs(identity.PidState.unknown, supervisor._pid_state(4321)) + + @staticmethod + def _permission_denied(_pid: int) -> bool: + raise PermissionError(errno.EPERM, "operation not permitted") + + def test_supervisor_process_start_shims_read_live_callback_and_platform(self) -> None: + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.proc_start_fingerprint_reader = lambda _pid: "same" + with patch("zeus.supervisor.platform.system", return_value="Linux"): + self.assertTrue(supervisor._process_start_fingerprint_required()) + self.assertTrue(supervisor._valid_marker_start("same")) + self.assertIsNone( + supervisor._process_start_identity_error( + {"proc_start_fingerprint": "same"}, + 4321, + ) + ) + supervisor.proc_start_fingerprint_reader = lambda _pid: "reused" + self.assertEqual( + "process start fingerprint does not match", + supervisor._process_start_identity_error( + {"proc_start_fingerprint": "same"}, + 4321, + ), + ) + + with patch("zeus.supervisor.platform.system", return_value="FreeBSD"): + self.assertFalse(supervisor._process_start_fingerprint_required()) + + def test_supervisor_hermes_resolution_shims_use_current_adapter_value(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + hermes = Path(tmp) / "hermes" + hermes.write_text("#!/bin/sh\n", encoding="utf-8") + supervisor = object.__new__(supervisor_module.Supervisor) + supervisor.adapter = SimpleNamespace(hermes_bin=str(hermes)) + + self.assertEqual(str(hermes.resolve()), supervisor._resolved_hermes_bin()) + self.assertEqual({str(hermes.resolve())}, supervisor._trusted_hermes_bins()) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/process_identity.py b/zeus/process_identity.py new file mode 100644 index 0000000..6bd9753 --- /dev/null +++ b/zeus/process_identity.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import errno +import os +import platform +import re +import shlex +import shutil +import subprocess # nosec B404 +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import cast + +PidAliveFn = Callable[[int], bool] +CmdlineReader = Callable[[int], list[str] | None] +ProcStartFingerprintReader = Callable[[int], str | None] + +_SubprocessRun = Callable[..., subprocess.CompletedProcess[str]] +_PYTHON_INTERPRETER_RE = re.compile(r"^python(?:\d+(?:\.\d+)?)?$") + + +@dataclass(frozen=True) +class CommandCheck: + verified: bool + reason: str + classification: str | None = None + + +class PidState(Enum): + alive = "alive" + dead = "dead" + unknown = "unknown" + + +def read_process_cmdline( + pid: int, + *, + system: str | None = None, + run_process: _SubprocessRun | None = None, +) -> list[str] | None: + current_system = platform.system() if system is None else system + if current_system == "Linux": + return read_linux_cmdline(pid) + if current_system == "Darwin": + if run_process is None: + return read_darwin_cmdline(pid) + return read_darwin_cmdline(pid, run_process=run_process) + return None + + +def read_linux_cmdline(pid: int, proc_root: Path = Path("/proc")) -> list[str]: + try: + raw = (proc_root / str(pid) / "cmdline").read_bytes() + except FileNotFoundError: + return [] + except OSError: + return [] + return [part.decode("utf-8", errors="surrogateescape") for part in raw.split(b"\0") if part] + + +def read_darwin_cmdline( + pid: int, + *, + run_process: _SubprocessRun | None = None, +) -> list[str] | None: + runner = _subprocess_runner(run_process) + try: + completed = runner( + ["/bin/ps", "-p", str(pid), "-o", "command="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return [] + command = completed.stdout.strip() + if not command: + return [] + try: + return shlex.split(command) + except ValueError: + return None + + +def read_process_start_fingerprint( + pid: int, + *, + system: str | None = None, + run_process: _SubprocessRun | None = None, +) -> str | None: + current_system = platform.system() if system is None else system + if current_system == "Darwin": + if run_process is None: + return read_darwin_process_start_fingerprint(pid) + return read_darwin_process_start_fingerprint(pid, run_process=run_process) + if current_system != "Linux": + return None + return read_linux_process_start_fingerprint(pid) + + +def read_darwin_process_start_fingerprint( + pid: int, + *, + run_process: _SubprocessRun | None = None, +) -> str | None: + runner = _subprocess_runner(run_process) + try: + completed = runner( + ["/bin/ps", "-p", str(pid), "-o", "lstart="], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=1, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return None + started = " ".join(completed.stdout.split()) + return f"darwin:ps-lstart:{started}" if started else None + + +def read_linux_process_start_fingerprint( + pid: int, + proc_root: Path = Path("/proc"), +) -> str | None: + try: + stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8") + except (FileNotFoundError, OSError, UnicodeDecodeError): + return None + try: + fields = stat.rsplit(") ", 1)[1].split() + except IndexError: + return None + if len(fields) < 20: + return None + return f"linux:/proc-starttime:{fields[19]}" + + +def pid_state(pid: int, *, pid_alive_fn: PidAliveFn | None = None) -> PidState: + try: + if pid_alive_fn is not None: + return PidState.alive if pid_alive_fn(pid) else PidState.dead + os.kill(pid, 0) + except ProcessLookupError: + return PidState.dead + except PermissionError: + return PidState.unknown + except OSError as exc: + return PidState.dead if exc.errno == errno.ESRCH else PidState.unknown + return PidState.alive + + +def valid_process_start_fingerprint(value: object) -> bool: + return type(value) is str and bool(value) and len(value) <= 512 + + +def process_start_fingerprint_required(system: str) -> bool: + return system in {"Darwin", "Linux"} + + +def process_start_identity_error( + marker_fingerprint: object, + live_fingerprint: str | None, + *, + fingerprint_required: bool, +) -> str | None: + if fingerprint_required: + marker_is_valid = valid_process_start_fingerprint(marker_fingerprint) + live_is_valid = valid_process_start_fingerprint(live_fingerprint) + if not marker_is_valid or not live_is_valid: + return "process start fingerprint is unavailable" + if marker_fingerprint != live_fingerprint: + return "process start fingerprint does not match" + elif live_fingerprint and marker_fingerprint != live_fingerprint: + return "process start fingerprint does not match" + elif marker_fingerprint and live_fingerprint != marker_fingerprint: + return "process start fingerprint is unavailable" + return None + + +def verify_gateway_command( + argv: list[str], + bot_id: str, + trusted_hermes_bin: str | set[str] | None, + *, + require_trusted_path: bool, +) -> CommandCheck: + if not argv: + return CommandCheck(False, "live-cmdline-missing") + classification = "direct-hermes" + hermes_command = argv[0] + args = argv[1:] + if len(argv) >= 2 and looks_like_python_interpreter(argv[0]): + classification = "python-script-wrapper" + hermes_command = argv[1] + args = argv[2:] + if len(args) != 4 or args.count("-p") != 1 or args[0] != "-p": + return CommandCheck(False, "wrong-command-intent", classification) + if args[1] != bot_id: + return CommandCheck(False, "wrong-bot-id", classification) + if args[2:] != ["gateway", "run"]: + return CommandCheck(False, "wrong-command-intent", classification) + if require_trusted_path: + resolved_command = resolve_executable(hermes_command) + if isinstance(trusted_hermes_bin, str): + trusted_hermes_bins = {trusted_hermes_bin} + else: + trusted_hermes_bins = trusted_hermes_bin or set() + if not trusted_hermes_bins or resolved_command not in trusted_hermes_bins: + return CommandCheck(False, "untrusted-executable", classification) + return CommandCheck(True, "ok", classification) + + +def looks_like_python_interpreter(command: str) -> bool: + return bool(_PYTHON_INTERPRETER_RE.fullmatch(Path(command).name.lower())) + + +def resolve_executable(command: str, path: str | None = None) -> str | None: + if not command: + return None + candidate = command if "/" in command else shutil.which(command, path=path) + if candidate is None: + return None + try: + return str(Path(candidate).expanduser().resolve()) + except (OSError, RuntimeError): + return str(Path(candidate).expanduser().absolute()) + + +def trusted_hermes_paths(command: str) -> set[str]: + resolved = resolve_executable(command) + if resolved is None: + return set() + paths = {resolved} + delegated = resolve_launcher_exec_target(resolved) + if delegated is not None: + paths.add(delegated) + return paths + + +def resolve_launcher_exec_target(command: str) -> str | None: + path = Path(command) + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except (OSError, UnicodeDecodeError): + return None + if not text.startswith("#!"): + return None + for line in text.splitlines()[1:20]: + stripped = line.strip() + if not stripped.startswith("exec "): + continue + try: + parts = shlex.split(stripped) + except ValueError: + continue + if len(parts) < 2 or parts[0] != "exec": + continue + target = parts[1] + if "/" not in target: + continue + resolved = resolve_executable(target) + if resolved and Path(resolved).name == "hermes": + return resolved + return None + + +def safe_command_shape(argv: list[str]) -> str: + if not argv: + return "empty" + classification = "direct-hermes" + args = argv[1:] + if len(argv) >= 2 and looks_like_python_interpreter(argv[0]): + classification = "python-script-wrapper" + args = argv[2:] + if len(args) == 4 and args[0] == "-p" and args[2:] == ["gateway", "run"]: + return f"{classification} hermes -p gateway run" + return f"{classification} unrecognized" + + +def _subprocess_runner(run_process: _SubprocessRun | None) -> _SubprocessRun: + if run_process is not None: + return run_process + return cast(_SubprocessRun, subprocess.run) diff --git a/zeus/supervisor.py b/zeus/supervisor.py index d0ee4f7..a4a46c0 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -8,7 +8,6 @@ import platform import re import select -import shlex import shutil import signal import stat @@ -24,6 +23,7 @@ from typing import Protocol, TypeGuard from urllib.parse import urlparse +from zeus import process_identity as _process_identity from zeus.errors import ( BotArchiveError, BotDeleteError, @@ -86,9 +86,20 @@ def poll(self) -> int | None: ... PopenFactory = Callable[..., PopenLike] KillFn = Callable[[int, signal.Signals], None] -PidAliveFn = Callable[[int], bool] -CmdlineReader = Callable[[int], list[str] | None] -ProcStartFingerprintReader = Callable[[int], str | None] +PidAliveFn = _process_identity.PidAliveFn +CmdlineReader = _process_identity.CmdlineReader +ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader + +_CommandCheck = _process_identity.CommandCheck +_PidState = _process_identity.PidState +_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter +_read_linux_cmdline = _process_identity.read_linux_cmdline +_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint +_resolve_executable = _process_identity.resolve_executable +_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target +_safe_command_shape = _process_identity.safe_command_shape +_trusted_hermes_paths = _process_identity.trusted_hermes_paths +_verify_gateway_command = _process_identity.verify_gateway_command @dataclass(frozen=True) @@ -98,19 +109,6 @@ class OwnershipCheck: classification: str | None = None -@dataclass(frozen=True) -class _CommandCheck: - verified: bool - reason: str - classification: str | None = None - - -class _PidState(Enum): - alive = "alive" - dead = "dead" - unknown = "unknown" - - class _SignalResult(Enum): sent = "sent" missing = "missing" @@ -121,7 +119,6 @@ class _ReadinessProbeUnset: pass -_PYTHON_INTERPRETER_RE = re.compile(r"^python(?:\d+(?:\.\d+)?)?$") _REQUEST_ID_RE = re.compile(r"^[0-9a-f]{32}$") _LIFECYCLE_SOURCES = frozenset({"api", "cli", "reconcile", "recovery", "system"}) _READINESS_PROBE_UNSET = _ReadinessProbeUnset() @@ -1396,28 +1393,19 @@ def _classify_schema3_runtime_marker( return _MarkerObservation("live", payload=payload) def _process_start_identity_error(self, payload: dict[str, object], pid: int) -> str | None: - marker_start = payload.get("proc_start_fingerprint") - live_start = self.proc_start_fingerprint_reader(pid) - if self._process_start_fingerprint_required(): - if not self._valid_marker_start(marker_start) or not self._valid_marker_start( - live_start - ): - return "process start fingerprint is unavailable" - if marker_start != live_start: - return "process start fingerprint does not match" - elif live_start and marker_start != live_start: - return "process start fingerprint does not match" - elif marker_start and live_start != marker_start: - return "process start fingerprint is unavailable" - return None + return _process_identity.process_start_identity_error( + payload.get("proc_start_fingerprint"), + self.proc_start_fingerprint_reader(pid), + fingerprint_required=self._process_start_fingerprint_required(), + ) @staticmethod def _valid_marker_start(value: object) -> bool: - return type(value) is str and bool(value) and len(value) <= 512 + return _process_identity.valid_process_start_fingerprint(value) @staticmethod def _process_start_fingerprint_required() -> bool: - return platform.system() in {"Darwin", "Linux"} + return _process_identity.process_start_fingerprint_required(platform.system()) def _classify_existing_runtime_marker( self, @@ -3302,17 +3290,14 @@ def _require_bot(self, bot_id: str) -> BotRecord: return record def _pid_state(self, pid: int) -> _PidState: - try: - if self.pid_alive_fn is not None: - return _PidState.alive if self.pid_alive_fn(pid) else _PidState.dead - os.kill(pid, 0) - except ProcessLookupError: - return _PidState.dead - except PermissionError: - return _PidState.unknown - except OSError as exc: - return _PidState.dead if exc.errno == errno.ESRCH else _PidState.unknown - return _PidState.alive + if self.pid_alive_fn is not None: + return _process_identity.pid_state(pid, pid_alive_fn=self.pid_alive_fn) + + def probe_with_current_kill(probe_pid: int) -> bool: + os.kill(probe_pid, 0) + return True + + return _process_identity.pid_state(pid, pid_alive_fn=probe_with_current_kill) def _unknown_pid_response( self, @@ -3892,12 +3877,11 @@ def _poll_startup(self, process: PopenLike) -> int | None: def _read_process_cmdline(pid: int) -> list[str] | None: - system = platform.system() - if system == "Linux": - return _read_linux_cmdline(pid) - if system == "Darwin": - return _read_darwin_cmdline(pid) - return None + return _process_identity.read_process_cmdline( + pid, + system=platform.system(), + run_process=subprocess.run, + ) def _readiness_probe_marker_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: @@ -3958,175 +3942,20 @@ def _valid_probe_number(value: object) -> TypeGuard[int | float]: ) -def _read_linux_cmdline(pid: int, proc_root: Path = Path("/proc")) -> list[str]: - try: - raw = (proc_root / str(pid) / "cmdline").read_bytes() - except FileNotFoundError: - return [] - except OSError: - return [] - return [part.decode("utf-8", errors="surrogateescape") for part in raw.split(b"\0") if part] - - def _read_darwin_cmdline(pid: int) -> list[str] | None: - try: - completed = subprocess.run( # nosec B603 - ["/bin/ps", "-p", str(pid), "-o", "command="], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=1, - ) - except (OSError, subprocess.TimeoutExpired): - return None - if completed.returncode != 0: - return [] - command = completed.stdout.strip() - if not command: - return [] - try: - return shlex.split(command) - except ValueError: - return None + return _process_identity.read_darwin_cmdline(pid, run_process=subprocess.run) def _read_process_start_fingerprint(pid: int) -> str | None: - system = platform.system() - if system == "Darwin": - return _read_darwin_process_start_fingerprint(pid) - if system != "Linux": - return None - return _read_linux_process_start_fingerprint(pid) + return _process_identity.read_process_start_fingerprint( + pid, + system=platform.system(), + run_process=subprocess.run, + ) def _read_darwin_process_start_fingerprint(pid: int) -> str | None: - try: - completed = subprocess.run( # nosec B603 - ["/bin/ps", "-p", str(pid), "-o", "lstart="], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=1, - ) - except (OSError, subprocess.TimeoutExpired): - return None - if completed.returncode != 0: - return None - started = " ".join(completed.stdout.split()) - return f"darwin:ps-lstart:{started}" if started else None - - -def _read_linux_process_start_fingerprint(pid: int, proc_root: Path = Path("/proc")) -> str | None: - try: - stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8") - except (FileNotFoundError, OSError, UnicodeDecodeError): - return None - try: - fields = stat.rsplit(") ", 1)[1].split() - except IndexError: - return None - if len(fields) < 20: - return None - return f"linux:/proc-starttime:{fields[19]}" - - -def _verify_gateway_command( - argv: list[str], - bot_id: str, - trusted_hermes_bin: str | set[str] | None, - *, - require_trusted_path: bool, -) -> _CommandCheck: - if not argv: - return _CommandCheck(False, "live-cmdline-missing") - classification = "direct-hermes" - hermes_command = argv[0] - args = argv[1:] - if len(argv) >= 2 and _looks_like_python_interpreter(argv[0]): - classification = "python-script-wrapper" - hermes_command = argv[1] - args = argv[2:] - if len(args) != 4 or args.count("-p") != 1 or args[0] != "-p": - return _CommandCheck(False, "wrong-command-intent", classification) - if args[1] != bot_id: - return _CommandCheck(False, "wrong-bot-id", classification) - if args[2:] != ["gateway", "run"]: - return _CommandCheck(False, "wrong-command-intent", classification) - if require_trusted_path: - resolved_command = _resolve_executable(hermes_command) - if isinstance(trusted_hermes_bin, str): - trusted_hermes_bins = {trusted_hermes_bin} - else: - trusted_hermes_bins = trusted_hermes_bin or set() - if not trusted_hermes_bins or resolved_command not in trusted_hermes_bins: - return _CommandCheck(False, "untrusted-executable", classification) - return _CommandCheck(True, "ok", classification) - - -def _looks_like_python_interpreter(command: str) -> bool: - return bool(_PYTHON_INTERPRETER_RE.fullmatch(Path(command).name.lower())) - - -def _resolve_executable(command: str, path: str | None = None) -> str | None: - if not command: - return None - candidate = command if "/" in command else shutil.which(command, path=path) - if candidate is None: - return None - try: - return str(Path(candidate).expanduser().resolve()) - except (OSError, RuntimeError): - return str(Path(candidate).expanduser().absolute()) - - -def _trusted_hermes_paths(command: str) -> set[str]: - resolved = _resolve_executable(command) - if resolved is None: - return set() - paths = {resolved} - delegated = _resolve_launcher_exec_target(resolved) - if delegated is not None: - paths.add(delegated) - return paths - - -def _resolve_launcher_exec_target(command: str) -> str | None: - path = Path(command) - try: - text = path.read_text(encoding="utf-8", errors="ignore") - except (OSError, UnicodeDecodeError): - return None - if not text.startswith("#!"): - return None - for line in text.splitlines()[1:20]: - stripped = line.strip() - if not stripped.startswith("exec "): - continue - try: - parts = shlex.split(stripped) - except ValueError: - continue - if len(parts) < 2 or parts[0] != "exec": - continue - target = parts[1] - if "/" not in target: - continue - resolved = _resolve_executable(target) - if resolved and Path(resolved).name == "hermes": - return resolved - return None - - -def _safe_command_shape(argv: list[str]) -> str: - if not argv: - return "empty" - classification = "direct-hermes" - args = argv[1:] - if len(argv) >= 2 and _looks_like_python_interpreter(argv[0]): - classification = "python-script-wrapper" - args = argv[2:] - if len(args) == 4 and args[0] == "-p" and args[2:] == ["gateway", "run"]: - return f"{classification} hermes -p gateway run" - return f"{classification} unrecognized" + return _process_identity.read_darwin_process_start_fingerprint( + pid, + run_process=subprocess.run, + ) From 94451e8c119056652757a79d397dddf795409f85 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:44:53 +0200 Subject: [PATCH 37/49] refactor(supervisor): centralize gateway marker protocol --- tests/test_gateway_marker.py | 549 +++++++++++++++++++++++++++++++++++ zeus/gateway_launcher.py | 178 +++--------- zeus/gateway_marker.py | 419 ++++++++++++++++++++++++++ zeus/hermes_adapter.py | 13 +- zeus/supervisor.py | 78 +---- 5 files changed, 1030 insertions(+), 207 deletions(-) create mode 100644 tests/test_gateway_marker.py create mode 100644 zeus/gateway_marker.py diff --git a/tests/test_gateway_marker.py b/tests/test_gateway_marker.py new file mode 100644 index 0000000..f7de218 --- /dev/null +++ b/tests/test_gateway_marker.py @@ -0,0 +1,549 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import unittest +from dataclasses import FrozenInstanceError +from typing import Any + +from zeus.gateway_marker import ( + GatewayGeneration, + GatewayLaunchMarker, + GatewayRuntimeMarker, + MarkerValidationError, + command_fingerprint, + is_compat_runtime_marker, + is_owned_runtime_marker, + parse_launch_marker, + parse_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.readiness import ReadinessProbe + + +def _probe_payload() -> dict[str, object]: + return { + "url": "http://127.0.0.1:4312/health", + "expected_status": "ok", + "expected_platform": "hermes", + "timeout_seconds": 10, + "interval_seconds": 0.25, + } + + +def _launch_payload(*, probe: object = ...) -> dict[str, object]: + argv = ["/opt/hermes/bin/hermes", "-p", "coder", "gateway", "run"] + return { + "schema": 3, + "bot_id": "coder", + "component": "gateway", + "action": "run", + "operation_id": "a" * 32, + "desired_revision": 7, + "argv": argv, + "resolved_hermes_bin": argv[0], + "command_fingerprint": command_fingerprint(argv), + "readiness_probe": _probe_payload() if probe is ... else probe, + } + + +def _runtime_payload(*, include_start_fingerprint: bool = True) -> dict[str, object]: + payload = _launch_payload() + payload.update({"pid": 4321, "started_at": 1_780_000_000}) + if include_start_fingerprint: + payload["proc_start_fingerprint"] = "linux:/proc-starttime:987654321" + return payload + + +class GatewayMarkerTests(unittest.TestCase): + def assert_marker_error(self, expected: str, payload: object, *, runtime: bool = False) -> None: + parser = parse_runtime_marker if runtime else parse_launch_marker + with self.assertRaisesRegex(MarkerValidationError, f"^{expected}$"): + parser(payload) + + def test_launch_and_runtime_round_trip_as_immutable_models(self) -> None: + launch_payload = _launch_payload() + expected_launch_payload = _launch_payload() + launch = parse_launch_marker(launch_payload) + + self.assertIsInstance(launch, GatewayLaunchMarker) + self.assertIs(type(launch.argv), tuple) + self.assertEqual(launch_payload, launch.to_payload()) + with self.assertRaises(FrozenInstanceError): + launch.bot_id = "other" # type: ignore[misc] + + source_argv = launch_payload["argv"] + source_probe = launch_payload["readiness_probe"] + assert isinstance(source_argv, list) + assert isinstance(source_probe, dict) + source_argv[0] = "/tmp/source-was-mutated" + source_probe["url"] = "http://localhost:1/source-was-mutated" + self.assertEqual(expected_launch_payload, launch.to_payload()) + + first = launch.to_payload() + second = launch.to_payload() + self.assertIsNot(first, second) + self.assertIsNot(first["argv"], second["argv"]) + self.assertIsNot(first["readiness_probe"], second["readiness_probe"]) + first_argv = first["argv"] + first_probe = first["readiness_probe"] + assert isinstance(first_argv, list) + assert isinstance(first_probe, dict) + first_argv[0] = "/tmp/replaced" + first_probe["url"] = "http://localhost:1/replaced" + self.assertEqual(expected_launch_payload, launch.to_payload()) + + runtime_payload = _runtime_payload() + runtime = parse_runtime_marker(runtime_payload) + self.assertIsInstance(runtime, GatewayRuntimeMarker) + self.assertEqual(runtime_payload, runtime.to_payload()) + self.assertEqual( + GatewayGeneration( + operation_id="a" * 32, + desired_revision=7, + pid=4321, + command_fingerprint=str(runtime_payload["command_fingerprint"]), + proc_start_fingerprint="linux:/proc-starttime:987654321", + ), + runtime.generation(), + ) + + without_start = _runtime_payload(include_start_fingerprint=False) + parsed_without_start = parse_runtime_marker(without_start) + self.assertEqual(without_start, parsed_without_start.to_payload()) + self.assertIsNone(parsed_without_start.generation().proc_start_fingerprint) + + def test_command_fingerprint_uses_canonical_utf8_json(self) -> None: + argv = ["/opt/hérmes", "-p", "coder", "gateway", "run"] + expected = hashlib.sha256( + json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + self.assertEqual(expected, command_fingerprint(argv)) + + def test_parsers_require_exact_launch_runtime_and_probe_key_sets(self) -> None: + self.assert_marker_error("marker must be an object", []) + with self.assertRaises(MarkerValidationError): + parse_runtime_marker([]) + + launch = _launch_payload() + for key in tuple(launch): + invalid = dict(launch) + del invalid[key] + with self.subTest(kind="launch-missing", key=key): + self.assert_marker_error("marker has invalid keys", invalid) + with_extra = dict(launch, extra=True) + self.assert_marker_error("marker has invalid keys", with_extra) + + runtime = _runtime_payload() + for key in tuple(runtime): + invalid = dict(runtime) + del invalid[key] + if key == "proc_start_fingerprint": + self.assertEqual( + _runtime_payload(include_start_fingerprint=False), + parse_runtime_marker(invalid).to_payload(), + ) + else: + with self.subTest(kind="runtime-missing", key=key): + self.assert_marker_error( + "runtime marker has invalid keys", invalid, runtime=True + ) + self.assert_marker_error( + "runtime marker has invalid keys", dict(runtime, extra=True), runtime=True + ) + + invalid_probe = _launch_payload() + probe = _probe_payload() + del probe["interval_seconds"] + invalid_probe["readiness_probe"] = probe + self.assert_marker_error("readiness_probe has invalid keys", invalid_probe) + + def test_launch_parser_rejects_invalid_intent_correlation_and_bool_integers(self) -> None: + cases: tuple[tuple[str, str, object], ...] = ( + ("schema-bool", "schema", True), + ("schema-old", "schema", 2), + ("component", "component", "worker"), + ("action", "action", "stop"), + ("operation-uppercase", "operation_id", "A" * 32), + ("operation-length", "operation_id", "a" * 31), + ("revision-bool", "desired_revision", True), + ("revision-zero", "desired_revision", 0), + ("revision-large", "desired_revision", 2**63), + ) + expected = { + "schema-bool": "marker schema is invalid", + "schema-old": "marker schema is invalid", + "component": "marker command intent is invalid", + "action": "marker command intent is invalid", + "operation-uppercase": "operation_id is invalid", + "operation-length": "operation_id is invalid", + "revision-bool": "desired_revision is invalid", + "revision-zero": "desired_revision is invalid", + "revision-large": "desired_revision is invalid", + } + for name, key, value in cases: + payload = _launch_payload() + payload[key] = value + with self.subTest(name=name): + self.assert_marker_error(expected[name], payload) + + def test_runtime_parser_rejects_invalid_pid_time_and_start_fingerprint(self) -> None: + cases: tuple[tuple[str, str, object, str], ...] = ( + ("pid-bool", "pid", True, "marker PID is invalid"), + ("pid-zero", "pid", 0, "marker PID is invalid"), + ("started-bool", "started_at", True, "marker started_at is invalid"), + ("started-zero", "started_at", 0, "marker started_at is invalid"), + ("started-nan", "started_at", math.nan, "marker started_at is invalid"), + ("started-inf", "started_at", math.inf, "marker started_at is invalid"), + ( + "start-null", + "proc_start_fingerprint", + None, + "process start fingerprint is invalid", + ), + ( + "start-non-string", + "proc_start_fingerprint", + 123, + "process start fingerprint is invalid", + ), + ( + "start-empty", + "proc_start_fingerprint", + "", + "process start fingerprint is invalid", + ), + ( + "start-long", + "proc_start_fingerprint", + "x" * 513, + "process start fingerprint is invalid", + ), + ) + for name, key, value, error in cases: + payload = _runtime_payload() + payload[key] = value + with self.subTest(name=name): + self.assert_marker_error(error, payload, runtime=True) + + def test_parser_rejects_invalid_argv_path_and_fingerprint(self) -> None: + cases: list[tuple[str, dict[str, object], str]] = [] + + not_a_list = _launch_payload() + not_a_list["argv"] = tuple(not_a_list["argv"]) # type: ignore[arg-type] + cases.append(("argv-type", not_a_list, "argv must be a bounded non-empty list")) + + empty_argv = _launch_payload() + empty_argv["argv"] = [] + cases.append(("argv-empty", empty_argv, "argv must be a bounded non-empty list")) + + too_many_parts = _launch_payload() + too_many_parts["argv"] = ["x"] * 65 + cases.append(("argv-too-many", too_many_parts, "argv must be a bounded non-empty list")) + + empty_part = _launch_payload() + empty_part["argv"] = ["/opt/hermes/bin/hermes", "", "coder", "gateway", "run"] + cases.append( + ("argv-empty-part", empty_part, "argv item must be a bounded non-empty string") + ) + + nul_part = _launch_payload() + nul_part["argv"] = ["/opt/hermes/bin/hermes\0", "-p", "coder", "gateway", "run"] + cases.append(("argv-nul-part", nul_part, "argv item must be a bounded non-empty string")) + + oversized_part = _launch_payload() + oversized_part["argv"] = ["x" * (16 * 1024 + 1)] + cases.append( + ( + "argv-oversized-part", + oversized_part, + "argv item must be a bounded non-empty string", + ) + ) + + oversized_argv = _launch_payload() + oversized_argv["argv"] = ["x" * 14_000] * 5 + cases.append(("argv-total-bytes", oversized_argv, "argv is too large")) + + wrong_command = _launch_payload() + wrong_command["argv"] = ["/opt/hermes/bin/hermes", "-p", "coder", "gateway", "stop"] + cases.append(("argv-command", wrong_command, "argv is not a Hermes gateway command")) + + wrong_profile = _launch_payload() + wrong_profile["argv"] = [ + "/opt/hermes/bin/hermes", + "-p", + "other", + "gateway", + "run", + ] + cases.append(("argv-profile", wrong_profile, "argv is not a Hermes gateway command")) + + non_absolute = _launch_payload() + non_absolute["resolved_hermes_bin"] = "hermes" + cases.append( + ( + "relative-path", + non_absolute, + "resolved_hermes_bin must be an absolute path without traversal", + ) + ) + + traversal = _launch_payload() + traversal["resolved_hermes_bin"] = "/opt/hermes/../bin/hermes" + cases.append( + ( + "traversal", + traversal, + "resolved_hermes_bin must be an absolute path without traversal", + ) + ) + + mismatched_path = _launch_payload() + mismatched_path["resolved_hermes_bin"] = "/usr/bin/hermes" + cases.append( + ( + "path-mismatch", + mismatched_path, + "exec argv does not use the resolved Hermes binary", + ) + ) + + invalid_fingerprint = _launch_payload() + invalid_fingerprint["command_fingerprint"] = "f" * 64 + cases.append( + ("fingerprint-mismatch", invalid_fingerprint, "command fingerprint is invalid") + ) + + for name, payload, error in cases: + with self.subTest(name=name): + self.assert_marker_error(error, payload) + + def test_resolved_path_comparison_preserves_baseline_lexical_normalization(self) -> None: + for resolved_path in ( + "/opt//hermes/bin/hermes", + "/opt/./hermes/bin/hermes", + "/opt/hermes/bin/hermes/", + ): + with self.subTest(kind="canonical-argv", resolved_path=resolved_path): + payload = _launch_payload() + payload["resolved_hermes_bin"] = resolved_path + parsed = parse_launch_marker(payload) + self.assertEqual(resolved_path, parsed.resolved_hermes_bin) + self.assertEqual(resolved_path, parsed.to_payload()["resolved_hermes_bin"]) + + with self.subTest(kind="noncanonical-argv", resolved_path=resolved_path): + payload = _launch_payload() + argv = payload["argv"] + assert isinstance(argv, list) + argv[0] = resolved_path + payload["resolved_hermes_bin"] = resolved_path + payload["command_fingerprint"] = command_fingerprint(argv) + self.assert_marker_error( + "exec argv does not use the resolved Hermes binary", payload + ) + + def test_readiness_conversion_round_trips_and_validates_the_contract(self) -> None: + payload = _probe_payload() + probe = readiness_probe_from_payload(payload) + + self.assertIsInstance(probe, ReadinessProbe) + self.assertEqual(payload, readiness_probe_to_payload(probe)) + self.assertIsNone(readiness_probe_from_payload(None)) + self.assertIsNone(readiness_probe_to_payload(None)) + + invalid_cases: tuple[tuple[str, object], ...] = ( + ("non-object", []), + ("https", dict(payload, url="https://127.0.0.1:4312/health")), + ("remote", dict(payload, url="http://example.com/health")), + ("credentials", dict(payload, url="http://user@localhost:4312/health")), + ("query", dict(payload, url="http://localhost:4312/health?secret=yes")), + ("fragment", dict(payload, url="http://localhost:4312/health#fragment")), + ("empty-status", dict(payload, expected_status="")), + ("empty-platform", dict(payload, expected_platform="")), + ("bool-timeout", dict(payload, timeout_seconds=True)), + ("zero-interval", dict(payload, interval_seconds=0)), + ("nan-timeout", dict(payload, timeout_seconds=math.nan)), + ("infinite-timeout", dict(payload, timeout_seconds=math.inf)), + ("large-timeout", dict(payload, timeout_seconds=3600.1)), + ) + for name, invalid in invalid_cases: + with self.subTest(name=name), self.assertRaises(MarkerValidationError): + readiness_probe_from_payload(invalid) + + strict_cases = ( + ("url-nul", "url", "http://localhost:4312/health\0", "readiness URL"), + ("url-long", "url", "http://localhost/" + "x" * 2049, "readiness URL"), + ("status-long", "expected_status", "x" * 129, "expected status"), + ("platform-nul", "expected_platform", "hermes\0agent", "expected platform"), + ) + for name, key, value, error_prefix in strict_cases: + marker_payload = _launch_payload() + marker_probe = marker_payload["readiness_probe"] + assert isinstance(marker_probe, dict) + marker_probe[key] = value + with ( + self.subTest(name=name), + self.assertRaisesRegex( + MarkerValidationError, f"^{error_prefix} must be a bounded non-empty string$" + ), + ): + parse_launch_marker(marker_payload) + + def test_strict_parsers_reject_compat_markers_while_recognizer_accepts_them(self) -> None: + schema2 = _runtime_payload() + schema2["schema"] = 2 + schemaless = _runtime_payload() + del schemaless["schema"] + explicit_null = _runtime_payload() + explicit_null["schema"] = None + + for name, payload in ( + ("schema2", schema2), + ("schemaless", schemaless), + ("explicit-null", explicit_null), + ): + with self.subTest(name=name): + with self.assertRaises(MarkerValidationError): + parse_runtime_marker(payload) + self.assertTrue(is_compat_runtime_marker(payload)) + + self.assertFalse(is_compat_runtime_marker(_runtime_payload())) + self.assertFalse(is_compat_runtime_marker([])) + self.assertFalse(is_compat_runtime_marker({"schema": True})) + + def test_owned_runtime_check_matches_full_schema_and_generation(self) -> None: + payload = _runtime_payload() + expected: dict[str, Any] = { + "bot_id": "coder", + "operation_id": "a" * 32, + "desired_revision": 7, + "pid": 4321, + "expected_fingerprint": payload["command_fingerprint"], + } + self.assertTrue(is_owned_runtime_marker(payload, **expected)) + + mismatches = { + "bot_id": "other", + "operation_id": "b" * 32, + "desired_revision": 8, + "pid": 4322, + "expected_fingerprint": "f" * 64, + } + for key, value in mismatches.items(): + call = dict(expected) + call[key] = value + with self.subTest(key=key): + self.assertFalse(is_owned_runtime_marker(payload, **call)) + + malformed = dict(payload, started_at=math.nan) + self.assertFalse(is_owned_runtime_marker(malformed, **expected)) + + def test_gateway_launcher_keeps_fingerprint_and_ownership_compatibility_exports(self) -> None: + from zeus.gateway_launcher import ( + _is_owned_runtime_marker as launcher_is_owned_runtime_marker, + ) + from zeus.gateway_launcher import command_fingerprint as launcher_command_fingerprint + + self.assertIs(command_fingerprint, launcher_command_fingerprint) + self.assertEqual( + inspect.signature(is_owned_runtime_marker), + inspect.signature(launcher_is_owned_runtime_marker), + ) + payload = _runtime_payload() + arguments: dict[str, Any] = { + "bot_id": "coder", + "operation_id": "a" * 32, + "desired_revision": 7, + "pid": 4321, + "expected_fingerprint": str(payload["command_fingerprint"]), + } + self.assertEqual( + is_owned_runtime_marker(payload, **arguments), + launcher_is_owned_runtime_marker(payload, **arguments), + ) + + def test_launcher_preserves_marker_bytes_and_validation_precedence(self) -> None: + from zeus.gateway_launcher import LaunchPayloadError, _validate_payload + + marker = _launch_payload() + marker_argv = marker["argv"] + assert isinstance(marker_argv, list) + root_payload: dict[str, object] = { + "profile_path": "/opt/hermes/profiles/coder", + "marker_path": "/opt/hermes/profiles/coder/logs/zeus-gateway.pid.json", + "marker": marker, + "argv": list(marker_argv), + "env": {"HERMES_HOME": "/opt/hermes", "PATH": "/usr/bin"}, + } + _profile, validated_marker, _argv, _env = _validate_payload(root_payload) + self.assertEqual(marker, validated_marker) + runtime_marker = dict(validated_marker) + runtime_marker.update( + { + "pid": 4321, + "started_at": 1_780_000_000, + "proc_start_fingerprint": "linux:/proc-starttime:123", + } + ) + marker_bytes = ( + json.dumps(runtime_marker, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + self.assertEqual( + b'{"action":"run","argv":["/opt/hermes/bin/hermes","-p","coder",' + b'"gateway","run"],"bot_id":"coder","command_fingerprint":' + b'"7d203f8c4831e34eccfed055c8a8b82f9d68601ed15135a68dfde76c32a55321",' + b'"component":"gateway","desired_revision":7,"operation_id":' + b'"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","pid":4321,"proc_start_fingerprint":' + b'"linux:/proc-starttime:123","readiness_probe":{"expected_platform":' + b'"hermes","expected_status":"ok","interval_seconds":0.25,' + b'"timeout_seconds":10,"url":"http://127.0.0.1:4312/health"},' + b'"resolved_hermes_bin":"/opt/hermes/bin/hermes","schema":3,' + b'"started_at":1780000000}\n', + marker_bytes, + ) + self.assertNotIn(b"private", marker_bytes) + + boundary_first: Any = json.loads(json.dumps(root_payload)) + boundary_first["profile_path"] = "/opt/hermes/profiles/other" + boundary_first["marker_path"] = "/opt/hermes/profiles/other/logs/zeus-gateway.pid.json" + boundary_first["marker"]["schema"] = 2 + with self.assertRaisesRegex( + LaunchPayloadError, "^profile_path is outside the bot profile boundary$" + ): + _validate_payload(boundary_first) + + mismatch_first: Any = json.loads(json.dumps(root_payload)) + mismatch_first["marker"]["argv"][-1] = "stop" + with self.assertRaisesRegex(LaunchPayloadError, "^marker argv does not match exec argv$"): + _validate_payload(mismatch_first) + + outer_argv_first: Any = json.loads(json.dumps(root_payload)) + outer_argv_first["argv"] = "not-a-list" + outer_argv_first["marker"]["argv"][-1] = "stop" + with self.assertRaisesRegex(LaunchPayloadError, "^argv must be a bounded non-empty list$"): + _validate_payload(outer_argv_first) + + def test_supervisor_keeps_generation_compatibility_and_loose_readiness_adapter(self) -> None: + from zeus.supervisor import ( + Supervisor, + _GatewayGeneration, + _readiness_probe_from_marker, + _readiness_probe_marker_payload, + ) + + self.assertIs(GatewayGeneration, _GatewayGeneration) + self.assertTrue(Supervisor._is_compat_runtime_marker({"schema": 2})) + self.assertTrue(Supervisor._is_compat_runtime_marker({})) + compatible_probe = dict(_probe_payload(), legacy_extra="preserved compatibility") + parsed_probe = _readiness_probe_from_marker(compatible_probe) + self.assertIsInstance(parsed_probe, ReadinessProbe) + self.assertEqual(_probe_payload(), _readiness_probe_marker_payload(parsed_probe)) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/gateway_launcher.py b/zeus/gateway_launcher.py index 1b0d0eb..6fe7c7c 100644 --- a/zeus/gateway_launcher.py +++ b/zeus/gateway_launcher.py @@ -1,12 +1,10 @@ from __future__ import annotations import contextlib -import hashlib import json import math import os import platform -import re import stat import subprocess # nosec B404 import sys @@ -15,9 +13,22 @@ from pathlib import Path from types import TracebackType from typing import NoReturn -from urllib.parse import urlparse -from zeus.models import ID_RE +from zeus.gateway_marker import ( + MarkerValidationError, + _finish_launch_marker, + _parse_launch_marker_argv, + _parse_launch_marker_correlation, + _parse_launch_marker_identity, +) +from zeus.gateway_marker import ( + command_fingerprint as _command_fingerprint, +) +from zeus.gateway_marker import ( + is_owned_runtime_marker as _marker_is_owned_runtime_marker, +) + +command_fingerprint = _command_fingerprint if os.name == "posix": import fcntl @@ -31,9 +42,6 @@ MARKER_PUBLICATION_LOCK_NAME = ".zeus-gateway-marker.lock" MARKER_PUBLICATION_LOCK_TIMEOUT_SECONDS = 30.0 -_OPERATION_ID_RE = re.compile(r"^[0-9a-f]{32}$") -_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{64}$") - class _UnspecifiedProcessStart: pass @@ -41,25 +49,6 @@ class _UnspecifiedProcessStart: _UNSPECIFIED_PROCESS_START = _UnspecifiedProcessStart() _ROOT_KEYS = frozenset({"profile_path", "marker_path", "marker", "argv", "env"}) -_MARKER_KEYS = frozenset( - { - "schema", - "bot_id", - "component", - "action", - "operation_id", - "desired_revision", - "argv", - "resolved_hermes_bin", - "command_fingerprint", - "readiness_probe", - } -) -_RUNTIME_MARKER_KEYS = _MARKER_KEYS | frozenset({"pid", "started_at"}) -_RUNTIME_MARKER_FINGERPRINT_KEYS = _RUNTIME_MARKER_KEYS | frozenset({"proc_start_fingerprint"}) -_PROBE_KEYS = frozenset( - {"url", "expected_status", "expected_platform", "timeout_seconds", "interval_seconds"} -) class LaunchPayloadError(ValueError): @@ -70,11 +59,6 @@ class _ConfirmedMissing(LaunchPayloadError): """A missing path component whose absence was rechecked on retained descriptors.""" -def command_fingerprint(argv: list[str]) -> str: - encoded = json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - def _exact_dict(value: object, keys: frozenset[str], name: str) -> dict[str, object]: if type(value) is not dict: raise LaunchPayloadError(f"{name} must be an object") @@ -131,77 +115,44 @@ def _validate_path(value: object, name: str) -> Path: return path -def _valid_probe_number(value: object) -> bool: - if isinstance(value, bool) or not isinstance(value, int | float): - return False - return math.isfinite(float(value)) and 0 < float(value) <= 3600 - - -def _validate_readiness_probe(value: object) -> object: - if value is None: - return None - probe = _exact_dict(value, _PROBE_KEYS, "readiness_probe") - url = _exact_string(probe["url"], "readiness URL", max_length=2048) - parsed = urlparse(url) - if ( - parsed.scheme != "http" - or parsed.hostname not in {"127.0.0.1", "localhost", "::1"} - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - ): - raise LaunchPayloadError("readiness URL must be loopback HTTP") - _exact_string(probe["expected_status"], "expected status", max_length=128) - _exact_string(probe["expected_platform"], "expected platform", max_length=128) - if not _valid_probe_number(probe["timeout_seconds"]) or not _valid_probe_number( - probe["interval_seconds"] - ): - raise LaunchPayloadError("readiness timing is invalid") - return probe - - def _validate_payload(value: object) -> tuple[Path, dict[str, object], list[str], dict[str, str]]: payload = _exact_dict(value, _ROOT_KEYS, "payload") profile_path = _validate_path(payload["profile_path"], "profile_path") marker_path = _validate_path(payload["marker_path"], "marker_path") - marker = _exact_dict(payload["marker"], _MARKER_KEYS, "marker") - bot_id = _exact_string(marker["bot_id"], "bot_id", max_length=63) - if ID_RE.fullmatch(bot_id) is None: - raise LaunchPayloadError("bot_id is invalid") + try: + marker, bot_id = _parse_launch_marker_identity(payload["marker"]) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc if profile_path.name != bot_id or profile_path.parent.name != "profiles": raise LaunchPayloadError("profile_path is outside the bot profile boundary") if marker_path != profile_path / "logs" / MARKER_NAME: raise LaunchPayloadError("marker_path is outside the bot profile boundary") - if marker["schema"] != 3 or type(marker["schema"]) is not int: - raise LaunchPayloadError("marker schema is invalid") - if marker["component"] != "gateway" or marker["action"] != "run": - raise LaunchPayloadError("marker command intent is invalid") - operation_id = _exact_string(marker["operation_id"], "operation_id", max_length=32) - if _OPERATION_ID_RE.fullmatch(operation_id) is None: - raise LaunchPayloadError("operation_id is invalid") - revision = marker["desired_revision"] - if type(revision) is not int or not 1 <= revision <= 2**63 - 1: - raise LaunchPayloadError("desired_revision is invalid") - + try: + operation_id, revision = _parse_launch_marker_correlation(marker) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc argv = _validate_argv(payload["argv"]) - marker_argv = _validate_argv(marker["argv"]) - if argv != marker_argv: + try: + marker_argv = _parse_launch_marker_argv(marker) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc + if argv != list(marker_argv): raise LaunchPayloadError("marker argv does not match exec argv") - if len(argv) != 5 or argv[1:] != ["-p", bot_id, "gateway", "run"]: - raise LaunchPayloadError("argv is not a Hermes gateway command") - resolved_hermes = _validate_path(marker["resolved_hermes_bin"], "resolved_hermes_bin") - if argv[0] != str(resolved_hermes): - raise LaunchPayloadError("exec argv does not use the resolved Hermes binary") - fingerprint = _exact_string(marker["command_fingerprint"], "command_fingerprint", max_length=64) - if _FINGERPRINT_RE.fullmatch(fingerprint) is None or fingerprint != command_fingerprint(argv): - raise LaunchPayloadError("command fingerprint is invalid") - _validate_readiness_probe(marker["readiness_probe"]) + try: + parsed_marker = _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=marker_argv, + ) + except MarkerValidationError as exc: + raise LaunchPayloadError(str(exc)) from exc env = _validate_env(payload["env"]) if env.get("HERMES_HOME") != str(profile_path.parent.parent): raise LaunchPayloadError("HERMES_HOME does not match the bot profile root") - return profile_path, marker, argv, env + return profile_path, parsed_marker.to_payload(), argv, env def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: @@ -622,50 +573,13 @@ def _is_owned_runtime_marker( pid: int, expected_fingerprint: str, ) -> bool: - if type(value) is not dict or frozenset(value) not in { - _RUNTIME_MARKER_KEYS, - _RUNTIME_MARKER_FINGERPRINT_KEYS, - }: - return False - marker = value - if ( - type(marker["schema"]) is not int - or marker["schema"] != 3 - or marker["bot_id"] != bot_id - or marker["component"] != "gateway" - or marker["action"] != "run" - or marker["operation_id"] != operation_id - or type(marker["desired_revision"]) is not int - or marker["desired_revision"] != desired_revision - or type(marker["pid"]) is not int - or marker["pid"] != pid - or marker["command_fingerprint"] != expected_fingerprint - ): - return False - started_at = marker["started_at"] - if ( - isinstance(started_at, bool) - or not isinstance(started_at, int | float) - or not math.isfinite(float(started_at)) - or float(started_at) <= 0 - ): - return False - if "proc_start_fingerprint" in marker: - fingerprint = marker["proc_start_fingerprint"] - if type(fingerprint) is not str or not fingerprint or len(fingerprint) > 512: - return False - try: - argv = _validate_argv(marker["argv"]) - resolved_hermes = _validate_path(marker["resolved_hermes_bin"], "resolved_hermes_bin") - _validate_readiness_probe(marker["readiness_probe"]) - except LaunchPayloadError: - return False - return ( - len(argv) == 5 - and argv[1:] == ["-p", bot_id, "gateway", "run"] - and argv[0] == str(resolved_hermes) - and _FINGERPRINT_RE.fullmatch(expected_fingerprint) is not None - and command_fingerprint(argv) == expected_fingerprint + return _marker_is_owned_runtime_marker( + value, + bot_id=bot_id, + operation_id=operation_id, + desired_revision=desired_revision, + pid=pid, + expected_fingerprint=expected_fingerprint, ) diff --git a/zeus/gateway_marker.py b/zeus/gateway_marker.py new file mode 100644 index 0000000..b1f61d3 --- /dev/null +++ b/zeus/gateway_marker.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TypeGuard +from urllib.parse import urlparse + +from zeus.models import ID_RE +from zeus.readiness import ReadinessProbe + +__all__ = [ + "GatewayGeneration", + "GatewayLaunchMarker", + "GatewayRuntimeMarker", + "MarkerValidationError", + "command_fingerprint", + "is_compat_runtime_marker", + "is_owned_runtime_marker", + "parse_launch_marker", + "parse_runtime_marker", + "readiness_probe_from_payload", + "readiness_probe_to_payload", +] + +_MAX_ARGV_PARTS = 64 +_MAX_ARG_BYTES = 64 * 1024 +_LAUNCH_MARKER_KEYS = frozenset( + { + "schema", + "bot_id", + "component", + "action", + "operation_id", + "desired_revision", + "argv", + "resolved_hermes_bin", + "command_fingerprint", + "readiness_probe", + } +) +_RUNTIME_MARKER_KEYS = _LAUNCH_MARKER_KEYS | frozenset({"pid", "started_at"}) +_RUNTIME_MARKER_FINGERPRINT_KEYS = _RUNTIME_MARKER_KEYS | frozenset({"proc_start_fingerprint"}) +_READINESS_PROBE_KEYS = frozenset( + {"url", "expected_status", "expected_platform", "timeout_seconds", "interval_seconds"} +) +_OPERATION_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{64}$") + + +class MarkerValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class GatewayGeneration: + operation_id: str + desired_revision: int + pid: int + command_fingerprint: str + proc_start_fingerprint: str | None + + +@dataclass(frozen=True) +class GatewayLaunchMarker: + bot_id: str + operation_id: str + desired_revision: int + argv: tuple[str, ...] + resolved_hermes_bin: str + command_fingerprint: str + readiness_probe: ReadinessProbe | None + + def to_payload(self) -> dict[str, object]: + return { + "schema": 3, + "bot_id": self.bot_id, + "component": "gateway", + "action": "run", + "operation_id": self.operation_id, + "desired_revision": self.desired_revision, + "argv": list(self.argv), + "resolved_hermes_bin": self.resolved_hermes_bin, + "command_fingerprint": self.command_fingerprint, + "readiness_probe": readiness_probe_to_payload(self.readiness_probe), + } + + +@dataclass(frozen=True) +class GatewayRuntimeMarker(GatewayLaunchMarker): + pid: int + started_at: int | float + proc_start_fingerprint: str | None = None + + def to_payload(self) -> dict[str, object]: + payload = super().to_payload() + payload["pid"] = self.pid + payload["started_at"] = self.started_at + if self.proc_start_fingerprint is not None: + payload["proc_start_fingerprint"] = self.proc_start_fingerprint + return payload + + def generation(self) -> GatewayGeneration: + return GatewayGeneration( + operation_id=self.operation_id, + desired_revision=self.desired_revision, + pid=self.pid, + command_fingerprint=self.command_fingerprint, + proc_start_fingerprint=self.proc_start_fingerprint, + ) + + +def command_fingerprint(argv: list[str]) -> str: + encoded = json.dumps(argv, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def parse_launch_marker(value: object) -> GatewayLaunchMarker: + marker, bot_id = _parse_launch_marker_identity(value) + operation_id, revision = _parse_launch_marker_correlation(marker) + argv = _parse_launch_marker_argv(marker) + return _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=argv, + ) + + +def parse_runtime_marker(value: object) -> GatewayRuntimeMarker: + if type(value) is not dict: + raise MarkerValidationError("runtime marker must be an object") + marker = value + if frozenset(marker) not in { + _RUNTIME_MARKER_KEYS, + _RUNTIME_MARKER_FINGERPRINT_KEYS, + } or not all(type(key) is str for key in marker): + raise MarkerValidationError("runtime marker has invalid keys") + + bot_id = _parse_bot_id(marker) + operation_id, revision = _parse_launch_marker_correlation(marker) + argv = _parse_launch_marker_argv(marker) + launch = _finish_launch_marker( + marker, + bot_id=bot_id, + operation_id=operation_id, + revision=revision, + argv=argv, + ) + pid = marker["pid"] + if type(pid) is not int or pid <= 0: + raise MarkerValidationError("marker PID is invalid") + started_at = marker["started_at"] + if not _is_finite_positive_number(started_at): + raise MarkerValidationError("marker started_at is invalid") + process_start: str | None = None + if "proc_start_fingerprint" in marker: + raw_process_start = marker["proc_start_fingerprint"] + if ( + type(raw_process_start) is not str + or not raw_process_start + or len(raw_process_start) > 512 + ): + raise MarkerValidationError("process start fingerprint is invalid") + process_start = raw_process_start + + return GatewayRuntimeMarker( + bot_id=launch.bot_id, + operation_id=launch.operation_id, + desired_revision=launch.desired_revision, + argv=launch.argv, + resolved_hermes_bin=launch.resolved_hermes_bin, + command_fingerprint=launch.command_fingerprint, + readiness_probe=launch.readiness_probe, + pid=pid, + started_at=started_at, + proc_start_fingerprint=process_start, + ) + + +def is_owned_runtime_marker( + value: object, + *, + bot_id: str, + operation_id: str, + desired_revision: int, + pid: int, + expected_fingerprint: str, +) -> bool: + try: + marker = parse_runtime_marker(value) + except MarkerValidationError: + return False + return ( + marker.bot_id == bot_id + and marker.operation_id == operation_id + and marker.desired_revision == desired_revision + and marker.pid == pid + and marker.command_fingerprint == expected_fingerprint + ) + + +def is_compat_runtime_marker(value: object) -> bool: + if type(value) is not dict: + return False + schema = value.get("schema") + return (type(schema) is int and schema == 2) or schema is None + + +def readiness_probe_to_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: + if probe is None: + return None + return { + "url": probe.url, + "expected_status": probe.expected_status, + "expected_platform": probe.expected_platform, + "timeout_seconds": probe.timeout_seconds, + "interval_seconds": probe.interval_seconds, + } + + +def readiness_probe_from_payload(value: object) -> ReadinessProbe | None: + if value is None: + return None + if not isinstance(value, dict): + raise MarkerValidationError("readiness_probe must be an object or null") + url = value.get("url") + expected_status = value.get("expected_status") + expected_platform = value.get("expected_platform") + timeout_seconds = value.get("timeout_seconds") + interval_seconds = value.get("interval_seconds") + if not isinstance(url, str): + raise MarkerValidationError("url must be a string") + try: + parsed = urlparse(url) + valid_url = ( + parsed.scheme == "http" + and parsed.hostname in {"127.0.0.1", "localhost", "::1"} + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + ) + except ValueError: + valid_url = False + if not valid_url: + raise MarkerValidationError("url must be loopback HTTP without credentials or query data") + if not isinstance(expected_status, str) or not expected_status: + raise MarkerValidationError("expected_status must be a non-empty string") + if not isinstance(expected_platform, str) or not expected_platform: + raise MarkerValidationError("expected_platform must be a non-empty string") + if not _is_valid_probe_number(timeout_seconds) or not _is_valid_probe_number(interval_seconds): + raise MarkerValidationError("probe timing values must be finite positive numbers") + return ReadinessProbe( + url=url, + expected_status=expected_status, + expected_platform=expected_platform, + timeout_seconds=float(timeout_seconds), + interval_seconds=float(interval_seconds), + ) + + +def _parse_launch_marker_identity(value: object) -> tuple[dict[str, object], str]: + marker = _exact_dict(value, _LAUNCH_MARKER_KEYS, "marker") + return marker, _parse_bot_id(marker) + + +def _parse_bot_id(marker: dict[str, object]) -> str: + bot_id = _exact_string(marker["bot_id"], "bot_id", max_length=63) + if ID_RE.fullmatch(bot_id) is None: + raise MarkerValidationError("bot_id is invalid") + return bot_id + + +def _parse_launch_marker_correlation(marker: dict[str, object]) -> tuple[str, int]: + if marker["schema"] != 3 or type(marker["schema"]) is not int: + raise MarkerValidationError("marker schema is invalid") + if marker["component"] != "gateway" or marker["action"] != "run": + raise MarkerValidationError("marker command intent is invalid") + operation_id = _exact_string(marker["operation_id"], "operation_id", max_length=32) + if _OPERATION_ID_RE.fullmatch(operation_id) is None: + raise MarkerValidationError("operation_id is invalid") + revision = marker["desired_revision"] + if type(revision) is not int or not 1 <= revision <= 2**63 - 1: + raise MarkerValidationError("desired_revision is invalid") + return operation_id, revision + + +def _parse_launch_marker_argv(marker: dict[str, object]) -> tuple[str, ...]: + return _parse_argv(marker["argv"]) + + +def _finish_launch_marker( + marker: dict[str, object], + *, + bot_id: str, + operation_id: str, + revision: int, + argv: tuple[str, ...], +) -> GatewayLaunchMarker: + if len(argv) != 5 or list(argv[1:]) != ["-p", bot_id, "gateway", "run"]: + raise MarkerValidationError("argv is not a Hermes gateway command") + resolved_hermes, canonical_hermes = _parse_absolute_path( + marker["resolved_hermes_bin"], "resolved_hermes_bin" + ) + if argv[0] != canonical_hermes: + raise MarkerValidationError("exec argv does not use the resolved Hermes binary") + fingerprint = _exact_string(marker["command_fingerprint"], "command_fingerprint", max_length=64) + if _FINGERPRINT_RE.fullmatch(fingerprint) is None or fingerprint != command_fingerprint( + list(argv) + ): + raise MarkerValidationError("command fingerprint is invalid") + readiness = _strict_readiness_probe_from_payload(marker["readiness_probe"]) + return GatewayLaunchMarker( + bot_id=bot_id, + operation_id=operation_id, + desired_revision=revision, + argv=argv, + resolved_hermes_bin=resolved_hermes, + command_fingerprint=fingerprint, + readiness_probe=readiness, + ) + + +def _strict_readiness_probe_from_payload(value: object) -> ReadinessProbe | None: + if value is None: + return None + probe = _exact_dict(value, _READINESS_PROBE_KEYS, "readiness_probe") + url = _exact_string(probe["url"], "readiness URL", max_length=2048) + try: + parsed = urlparse(url) + valid_url = ( + parsed.scheme == "http" + and parsed.hostname in {"127.0.0.1", "localhost", "::1"} + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + ) + except ValueError: + valid_url = False + if not valid_url: + raise MarkerValidationError("readiness URL must be loopback HTTP") + expected_status = _exact_string(probe["expected_status"], "expected status", max_length=128) + expected_platform = _exact_string( + probe["expected_platform"], "expected platform", max_length=128 + ) + timeout_seconds = probe["timeout_seconds"] + interval_seconds = probe["interval_seconds"] + if not _is_valid_probe_number(timeout_seconds) or not _is_valid_probe_number(interval_seconds): + raise MarkerValidationError("readiness timing is invalid") + return ReadinessProbe( + url=url, + expected_status=expected_status, + expected_platform=expected_platform, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + + +def _exact_dict(value: object, keys: frozenset[str], name: str) -> dict[str, object]: + if type(value) is not dict: + raise MarkerValidationError(f"{name} must be an object") + result = value + if set(result) != keys or not all(type(key) is str for key in result): + raise MarkerValidationError(f"{name} has invalid keys") + return result + + +def _exact_string(value: object, name: str, *, max_length: int) -> str: + if type(value) is not str or not value or len(value) > max_length or "\0" in value: + raise MarkerValidationError(f"{name} must be a bounded non-empty string") + return value + + +def _parse_argv(value: object) -> tuple[str, ...]: + if type(value) is not list or not value or len(value) > _MAX_ARGV_PARTS: + raise MarkerValidationError("argv must be a bounded non-empty list") + argv: list[str] = [] + total = 0 + for item in value: + part = _exact_string(item, "argv item", max_length=16 * 1024) + total += len(part.encode("utf-8")) + if total > _MAX_ARG_BYTES: + raise MarkerValidationError("argv is too large") + argv.append(part) + return tuple(argv) + + +def _parse_absolute_path(value: object, name: str) -> tuple[str, str]: + raw = _exact_string(value, name, max_length=16 * 1024) + path = Path(raw) + if not path.is_absolute() or any(part in {".", ".."} for part in path.parts): + raise MarkerValidationError(f"{name} must be an absolute path without traversal") + return raw, str(path) + + +def _is_valid_probe_number(value: object) -> TypeGuard[int | float]: + if isinstance(value, bool) or not isinstance(value, int | float): + return False + try: + number = float(value) + except OverflowError: + return False + return math.isfinite(number) and 0 < number <= 3600 + + +def _is_finite_positive_number(value: object) -> bool: + if isinstance(value, bool) or not isinstance(value, int | float): + return False + try: + number = float(value) + except OverflowError: + return False + return math.isfinite(number) and number > 0 diff --git a/zeus/hermes_adapter.py b/zeus/hermes_adapter.py index a074b8d..61cbcc3 100644 --- a/zeus/hermes_adapter.py +++ b/zeus/hermes_adapter.py @@ -7,7 +7,7 @@ from pathlib import Path from zeus.envfile import parse_env_text -from zeus.gateway_launcher import command_fingerprint +from zeus.gateway_marker import command_fingerprint, readiness_probe_to_payload from zeus.models import ID_RE from zeus.readiness import ReadinessProbe @@ -88,15 +88,6 @@ def launcher_payload( exec_argv = [resolved_hermes, *argv[1:]] profile_path = self.hermes_root / "profiles" / bot_id marker_path = profile_path / "logs" / "zeus-gateway.pid.json" - probe_payload: dict[str, object] | None = None - if readiness_probe is not None: - probe_payload = { - "url": readiness_probe.url, - "expected_status": readiness_probe.expected_status, - "expected_platform": readiness_probe.expected_platform, - "timeout_seconds": readiness_probe.timeout_seconds, - "interval_seconds": readiness_probe.interval_seconds, - } marker: dict[str, object] = { "schema": 3, "bot_id": bot_id, @@ -107,7 +98,7 @@ def launcher_payload( "argv": exec_argv, "resolved_hermes_bin": resolved_hermes, "command_fingerprint": command_fingerprint(exec_argv), - "readiness_probe": probe_payload, + "readiness_probe": readiness_probe_to_payload(readiness_probe), } return { "profile_path": str(profile_path), diff --git a/zeus/supervisor.py b/zeus/supervisor.py index a4a46c0..76dce41 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -20,8 +20,7 @@ from datetime import UTC, datetime, timedelta from enum import Enum from pathlib import Path -from typing import Protocol, TypeGuard -from urllib.parse import urlparse +from typing import Protocol from zeus import process_identity as _process_identity from zeus.errors import ( @@ -37,7 +36,6 @@ LaunchPayloadError, _confirm_marker_missing, _ConfirmedMissing, - _is_owned_runtime_marker, _open_logs, _open_profile_chain, _open_regular_marker, @@ -48,6 +46,15 @@ marker_publication_lock, remove_marker_if_owned, ) +from zeus.gateway_marker import ( + GatewayGeneration, + is_compat_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.gateway_marker import ( + is_owned_runtime_marker as _is_owned_runtime_marker, +) from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEvent, LifecycleEventInput from zeus.logging_utils import tail_file @@ -138,13 +145,7 @@ class _MarkerObservation: reason: str = "" -@dataclass(frozen=True) -class _GatewayGeneration: - operation_id: str - desired_revision: int - pid: int - command_fingerprint: str - proc_start_fingerprint: str | None +_GatewayGeneration = GatewayGeneration @dataclass(frozen=True) @@ -2364,8 +2365,7 @@ def _prepare_reconcile_dead_record_locked( @staticmethod def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: - schema = payload.get("schema") - return (type(schema) is int and schema == 2) or schema is None + return is_compat_runtime_marker(payload) def _recover_pending_intent( self, @@ -3885,61 +3885,11 @@ def _read_process_cmdline(pid: int) -> list[str] | None: def _readiness_probe_marker_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: - if probe is None: - return None - return { - "url": probe.url, - "expected_status": probe.expected_status, - "expected_platform": probe.expected_platform, - "timeout_seconds": probe.timeout_seconds, - "interval_seconds": probe.interval_seconds, - } + return readiness_probe_to_payload(probe) def _readiness_probe_from_marker(value: object) -> ReadinessProbe | None: - if value is None: - return None - if not isinstance(value, dict): - raise ValueError("readiness_probe must be an object or null") - url = value.get("url") - expected_status = value.get("expected_status") - expected_platform = value.get("expected_platform") - timeout_seconds = value.get("timeout_seconds") - interval_seconds = value.get("interval_seconds") - if not isinstance(url, str): - raise ValueError("url must be a string") - parsed = urlparse(url) - if ( - parsed.scheme != "http" - or parsed.hostname not in {"127.0.0.1", "localhost", "::1"} - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - ): - raise ValueError("url must be loopback HTTP without credentials or query data") - if not isinstance(expected_status, str) or not expected_status: - raise ValueError("expected_status must be a non-empty string") - if not isinstance(expected_platform, str) or not expected_platform: - raise ValueError("expected_platform must be a non-empty string") - if not _valid_probe_number(timeout_seconds) or not _valid_probe_number(interval_seconds): - raise ValueError("probe timing values must be finite positive numbers") - return ReadinessProbe( - url=url, - expected_status=expected_status, - expected_platform=expected_platform, - timeout_seconds=float(timeout_seconds), - interval_seconds=float(interval_seconds), - ) - - -def _valid_probe_number(value: object) -> TypeGuard[int | float]: - return ( - isinstance(value, int | float) - and not isinstance(value, bool) - and math.isfinite(float(value)) - and 0 < float(value) <= 3600 - ) + return readiness_probe_from_payload(value) def _read_darwin_cmdline(pid: int) -> list[str] | None: From bd70e72af84913cfda6debe959b532f99689d8f1 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 02:02:53 +0200 Subject: [PATCH 38/49] refactor(supervisor): delegate profile transactions --- tests/test_profile_manager.py | 627 ++++++++++++++++++++++++++++++++++ zeus/profile_manager.py | 136 ++++++++ zeus/supervisor.py | 113 +++--- 3 files changed, 807 insertions(+), 69 deletions(-) create mode 100644 tests/test_profile_manager.py create mode 100644 zeus/profile_manager.py diff --git a/tests/test_profile_manager.py b/tests/test_profile_manager.py new file mode 100644 index 0000000..38fb4cf --- /dev/null +++ b/tests/test_profile_manager.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import ast +import inspect +import json +import os +import tempfile +import unittest +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from unittest.mock import patch + +import zeus.profile_manager as profile_manager_module +from zeus.errors import BotArchiveError, BotDeleteError +from zeus.models import BotCreateRequest, BotRecord +from zeus.profile_manager import ProfileArchive, ProfileDeletion, ProfileManager +from zeus.state import StateStore +from zeus.supervisor import Supervisor +from zeus.templates import TemplateStore + + +class ProfileManagerTests(unittest.TestCase): + def _manager(self, root: Path) -> ProfileManager: + return ProfileManager(root / "hermes", root / "archive") + + def _profile(self, root: Path) -> Path: + return root / "hermes" / "profiles" / "coder" + + def _write_profile(self, root: Path) -> Path: + profile = self._profile(root) + profile.mkdir(parents=True) + (profile / "sentinel.txt").write_text("original\n", encoding="utf-8") + return profile + + def _supervisor_with_bot(self, root: Path) -> tuple[StateStore, Supervisor, Path]: + profile = self._write_profile(root) + store = StateStore(root / "zeus.db") + store.init() + store.upsert_bot( + BotRecord( + bot_id="coder", + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + ) + ) + return store, Supervisor(store, "hermes", root / "hermes"), profile + + def test_preflight_delegates_without_mutating_the_filesystem(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = TemplateStore().get("coding-bot") + + rendered = manager.preflight(request, template) + + self.assertEqual(template.soul.rstrip() + "\n", rendered["SOUL.md"]) + self.assertFalse((root / "hermes").exists()) + + def test_install_transaction_removes_a_new_profile_on_caller_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._profile(root) + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = TemplateStore().get("coding-bot") + + with ( + self.assertRaisesRegex(RuntimeError, "database unavailable"), + manager.install_transaction(request, template) as record, + ): + self.assertEqual("coder", record.bot_id) + self.assertTrue(profile.is_dir()) + raise RuntimeError("database unavailable") + + self.assertFalse(os.path.lexists(profile)) + self.assertEqual([], list(profile.parent.iterdir())) + + def test_install_transaction_restores_an_exact_replaced_profile_on_caller_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + (profile / "SOUL.md").write_text("original profile\n", encoding="utf-8") + original_inode = profile.stat().st_ino + before = { + path.relative_to(profile): path.read_bytes() + for path in profile.rglob("*") + if path.is_file() + } + request = BotCreateRequest(bot_id="coder", template_id="coding-bot") + template = replace(TemplateStore().get("coding-bot"), soul="replacement profile") + + with ( + self.assertRaisesRegex(RuntimeError, "database unavailable"), + manager.install_transaction(request, template), + ): + self.assertEqual( + "replacement profile\n", + (profile / "SOUL.md").read_text(encoding="utf-8"), + ) + raise RuntimeError("database unavailable") + + after = { + path.relative_to(profile): path.read_bytes() + for path in profile.rglob("*") + if path.is_file() + } + self.assertEqual(original_inode, profile.stat().st_ino) + self.assertEqual(before, after) + self.assertEqual(["coder"], sorted(path.name for path in profile.parent.iterdir())) + + def test_delete_stage_rollback_and_finish_use_a_pinned_token(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + + deletion = manager.stage_delete("coder", str(profile)) + + self.assertIsInstance(deletion, ProfileDeletion) + assert deletion is not None + self.assertEqual(profile.resolve(), deletion.profile_path) + self.assertTrue(deletion.tombstone_path.is_dir()) + self.assertFalse(os.path.lexists(profile)) + + manager.rollback_delete(deletion) + + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + self.assertFalse(os.path.lexists(deletion.tombstone_path)) + + second_deletion = manager.stage_delete("coder", str(profile)) + assert second_deletion is not None + self.assertIsNone(manager.finish_delete(second_deletion)) + self.assertFalse(os.path.lexists(profile)) + self.assertFalse(os.path.lexists(second_deletion.tombstone_path)) + + def test_finish_delete_reports_cleanup_failure_and_retains_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + cleanup_error = OSError("cleanup unavailable") + + with patch("zeus.profile_manager.shutil.rmtree", side_effect=cleanup_error): + reported = manager.finish_delete(deletion) + + self.assertIs(cleanup_error, reported) + self.assertTrue(deletion.tombstone_path.is_dir()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_stage_and_rollback_use_a_pinned_token(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + + archive = manager.stage_archive("coder", str(profile)) + + self.assertIsInstance(archive, ProfileArchive) + assert archive is not None + self.assertEqual(profile.resolve(), archive.profile_path) + self.assertTrue(archive.archive_path.is_dir()) + self.assertFalse(os.path.lexists(profile)) + + manager.rollback_archive(archive) + + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + self.assertFalse(os.path.lexists(archive.archive_path)) + + def test_validate_profile_path_accepts_only_the_exact_bot_profile(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + hermes_root = root / "hermes" + profiles_root = hermes_root / "profiles" + exact_profile = profiles_root / "coder" + exact_profile.mkdir(parents=True) + + self.assertEqual( + exact_profile.resolve(), + manager.validate_profile_path("coder", str(exact_profile)), + ) + + rejected = { + "Hermes root": hermes_root, + "profiles root": profiles_root, + "sibling profiles tree": hermes_root.parent / "other-profiles" / "coder", + "other bot": profiles_root / "other-bot", + "nested path": exact_profile / "nested", + "outside path": root / "outside", + } + for label, candidate in rejected.items(): + with self.subTest(label=label), self.assertRaises(BotDeleteError): + manager.validate_profile_path("coder", str(candidate)) + + def test_delete_rollback_refuses_an_occupied_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + profile.mkdir() + replacement = profile / "replacement.txt" + replacement.write_text("replacement\n", encoding="utf-8") + + with ( + patch.object(manager, "finish_delete") as finish_delete, + self.assertRaises(BotDeleteError), + ): + manager.rollback_delete(deletion) + + finish_delete.assert_not_called() + self.assertEqual("replacement\n", replacement.read_text(encoding="utf-8")) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_delete_rollback_refuses_a_dangling_symlink_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + deletion = manager.stage_delete("coder", str(profile)) + assert deletion is not None + target = root / "untouched-delete-target" + profile.symlink_to(target, target_is_directory=True) + + with ( + patch.object(manager, "finish_delete") as finish_delete, + self.assertRaises(BotDeleteError), + ): + manager.rollback_delete(deletion) + + finish_delete.assert_not_called() + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_rollback_refuses_an_occupied_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + archive = manager.stage_archive("coder", str(profile)) + assert archive is not None + profile.mkdir() + replacement = profile / "replacement.txt" + replacement.write_text("replacement\n", encoding="utf-8") + + with ( + patch("zeus.profile_manager.shutil.rmtree") as cleanup, + self.assertRaises(BotArchiveError), + ): + manager.rollback_archive(archive) + + cleanup.assert_not_called() + self.assertEqual("replacement\n", replacement.read_text(encoding="utf-8")) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_archive_rollback_refuses_a_dangling_symlink_destination(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manager = self._manager(root) + profile = self._write_profile(root) + archive = manager.stage_archive("coder", str(profile)) + assert archive is not None + target = root / "untouched-archive-target" + profile.symlink_to(target, target_is_directory=True) + + with ( + patch("zeus.profile_manager.shutil.rmtree") as cleanup, + self.assertRaises(BotArchiveError), + ): + manager.rollback_archive(archive) + + cleanup.assert_not_called() + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_supervisor_delete_carries_the_exact_staged_token_into_finish(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + real_finish_delete = manager.finish_delete + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def capture_finish(deletion: ProfileDeletion) -> OSError | None: + self.assertIs(staged[0], deletion) + return real_finish_delete(deletion) + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete", side_effect=capture_finish) as finish, + ): + response = supervisor.delete_bot("coder", remove_profile=True) + + self.assertEqual("deleted", response.message) + self.assertEqual(1, finish.call_count) + self.assertIsNone(store.get_bot("coder")) + self.assertFalse(os.path.lexists(profile)) + + def test_supervisor_delete_cleanup_failure_stays_post_commit_and_is_audited(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + cleanup_error = PermissionError("cleanup unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete", return_value=cleanup_error), + ): + response = supervisor.delete_bot("coder", remove_profile=True) + + self.assertEqual("deleted; profile cleanup is pending", response.message) + self.assertIsNone(store.get_bot("coder")) + self.assertFalse(os.path.lexists(profile)) + self.assertEqual( + "original\n", + (staged[0].tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + events = [ + json.loads(line) + for line in store.audit_log_path().read_text(encoding="utf-8").splitlines() + ] + self.assertTrue( + any( + event.get("event") == "bot.delete_cleanup_pending" + and event.get("error") == "PermissionError" + for event in events + ) + ) + self.assertTrue( + any( + event.get("event") == "bot.delete" + and event.get("profile_removed") is True + and event.get("cleanup_pending") is True + for event in events + ) + ) + + def test_supervisor_delete_carries_the_exact_staged_token_into_rollback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + real_rollback_delete = manager.rollback_delete + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def capture_rollback(deletion: ProfileDeletion) -> None: + self.assertIs(staged[0], deletion) + real_rollback_delete(deletion) + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "rollback_delete", side_effect=capture_rollback) as rollback, + patch.object(manager, "finish_delete") as finish, + patch.object(store, "delete_bot_with_event", side_effect=database_error), + self.assertRaisesRegex(RuntimeError, "database unavailable") as raised, + ): + supervisor.delete_bot("coder", remove_profile=True) + + self.assertIs(database_error, raised.exception) + self.assertEqual(1, rollback.call_count) + finish.assert_not_called() + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + + def test_supervisor_archive_carries_the_exact_staged_token_into_rollback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileArchive] = [] + real_stage_archive = manager.stage_archive + real_rollback_archive = manager.rollback_archive + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileArchive | None: + archive = real_stage_archive(bot_id, profile_path) + assert archive is not None + staged.append(archive) + return archive + + def capture_rollback(archive: ProfileArchive) -> None: + self.assertIs(staged[0], archive) + real_rollback_archive(archive) + + with ( + patch.object(manager, "stage_archive", side_effect=capture_stage), + patch.object( + manager, + "rollback_archive", + side_effect=capture_rollback, + ) as rollback, + patch.object(store, "delete_bot_with_event", side_effect=database_error), + self.assertRaisesRegex(RuntimeError, "database unavailable") as raised, + ): + supervisor.archive_bot("coder") + + self.assertIs(database_error, raised.exception) + self.assertEqual(1, rollback.call_count) + self.assertEqual("original\n", (profile / "sentinel.txt").read_text(encoding="utf-8")) + + def test_supervisor_delete_chains_refused_compensation_from_database_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileDeletion] = [] + real_stage_delete = manager.stage_delete + database_error = RuntimeError("database unavailable") + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileDeletion | None: + deletion = real_stage_delete(bot_id, profile_path) + assert deletion is not None + staged.append(deletion) + return deletion + + def occupy_destination(*args: object, **kwargs: object) -> bool: + profile.mkdir() + (profile / "replacement.txt").write_text("replacement\n", encoding="utf-8") + raise database_error + + with ( + patch.object(manager, "stage_delete", side_effect=capture_stage), + patch.object(manager, "finish_delete") as finish, + patch.object(store, "delete_bot_with_event", side_effect=occupy_destination), + self.assertRaises(BotDeleteError) as raised, + ): + supervisor.delete_bot("coder", remove_profile=True) + + self.assertIs(database_error, raised.exception.__cause__) + finish.assert_not_called() + self.assertEqual( + "replacement\n", + (profile / "replacement.txt").read_text(encoding="utf-8"), + ) + self.assertEqual( + "original\n", + (staged[0].tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_supervisor_archive_chains_refused_compensation_from_database_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + manager = supervisor._profile_manager + staged: list[ProfileArchive] = [] + real_stage_archive = manager.stage_archive + database_error = RuntimeError("database unavailable") + target = root / "untouched-supervisor-archive-target" + + def capture_stage(bot_id: str, profile_path: Path | str) -> ProfileArchive | None: + archive = real_stage_archive(bot_id, profile_path) + assert archive is not None + staged.append(archive) + return archive + + def occupy_destination(*args: object, **kwargs: object) -> bool: + profile.symlink_to(target, target_is_directory=True) + raise database_error + + with ( + patch.object(manager, "stage_archive", side_effect=capture_stage), + patch.object(store, "delete_bot_with_event", side_effect=occupy_destination), + self.assertRaises(BotArchiveError) as raised, + ): + supervisor.archive_bot("coder") + + self.assertIs(database_error, raised.exception.__cause__) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (staged[0].archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_restore_wrappers_reject_unvalidated_destination_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = StateStore(root / "zeus.db") + store.init() + supervisor = Supervisor(store, "hermes", root / "hermes") + outside = root / "outside" / "coder" + tombstone = root / "delete-evidence" + archive_path = root / "archive-evidence" + for evidence in (tombstone, archive_path): + evidence.mkdir() + (evidence / "sentinel.txt").write_text("retain\n", encoding="utf-8") + + with self.assertRaises(BotDeleteError): + supervisor._restore_tombstoned_profile("coder", str(outside), tombstone) + with self.assertRaises(BotDeleteError): + supervisor._restore_archived_profile("coder", str(outside), archive_path) + + self.assertFalse(os.path.lexists(outside)) + self.assertEqual( + "retain\n", + (tombstone / "sentinel.txt").read_text(encoding="utf-8"), + ) + self.assertEqual( + "retain\n", + (archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_delete_restore_treats_an_exact_dangling_symlink_as_occupied(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + deletion = supervisor._profile_manager.stage_delete("coder", str(profile)) + assert deletion is not None + target = root / "untouched-legacy-delete-target" + profile.symlink_to(target, target_is_directory=True) + + with self.assertRaisesRegex(BotDeleteError, "original path is occupied"): + supervisor._restore_tombstoned_profile( + "coder", + str(profile), + deletion.tombstone_path, + ) + + self.assertIsNotNone(store.get_bot("coder")) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (deletion.tombstone_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_legacy_archive_restore_treats_an_exact_dangling_symlink_as_occupied(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, supervisor, profile = self._supervisor_with_bot(root) + archive = supervisor._profile_manager.stage_archive("coder", str(profile)) + assert archive is not None + target = root / "untouched-legacy-archive-target" + profile.symlink_to(target, target_is_directory=True) + + with self.assertRaisesRegex(BotArchiveError, "original path is occupied"): + supervisor._restore_archived_profile( + "coder", + str(profile), + archive.archive_path, + ) + + self.assertIsNotNone(store.get_bot("coder")) + self.assertTrue(profile.is_symlink()) + self.assertEqual(str(target), os.readlink(profile)) + self.assertFalse(target.exists()) + self.assertEqual( + "original\n", + (archive.archive_path / "sentinel.txt").read_text(encoding="utf-8"), + ) + + def test_transaction_tokens_are_frozen(self) -> None: + deletion = ProfileDeletion(Path("profile"), Path("tombstone")) + archive = ProfileArchive(Path("profile"), Path("archive")) + + with self.assertRaises(FrozenInstanceError): + deletion.profile_path = Path("replacement") + with self.assertRaises(FrozenInstanceError): + archive.profile_path = Path("replacement") + + def test_module_has_no_state_runtime_or_process_dependency(self) -> None: + source = inspect.getsource(profile_manager_module) + tree = ast.parse(source) + imported_roots: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imported_roots.add(node.module.split(".", 1)[0]) + + self.assertTrue({"sqlite3", "signal", "subprocess"}.isdisjoint(imported_roots)) + self.assertNotIn("StateStore", source) + self.assertNotIn("GatewayRuntime", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/profile_manager.py b/zeus/profile_manager.py new file mode 100644 index 0000000..d2b5ac4 --- /dev/null +++ b/zeus/profile_manager.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import os +import shutil +import time +from contextlib import AbstractContextManager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from zeus.errors import BotArchiveError, BotDeleteError +from zeus.models import BotCreateRequest, BotRecord, HermesTemplate, validate_id +from zeus.renderer import ProfileRenderer + + +@dataclass(frozen=True) +class ProfileDeletion: + profile_path: Path + tombstone_path: Path + + +@dataclass(frozen=True) +class ProfileArchive: + profile_path: Path + archive_path: Path + + +class ProfileManager: + def __init__(self, hermes_root: Path | str, archive_root: Path | str) -> None: + self.hermes_root = Path(hermes_root) + self.archive_root = Path(archive_root) + self._renderer = ProfileRenderer(self.hermes_root) + + def preflight( + self, + request: BotCreateRequest, + template: HermesTemplate, + ) -> dict[str, str]: + return self._renderer.preflight(request, template) + + def install_transaction( + self, + request: BotCreateRequest, + template: HermesTemplate, + ) -> AbstractContextManager[BotRecord]: + return self._renderer.transaction(request, template) + + def validate_profile_path(self, bot_id: str, profile_path: Path | str) -> Path: + safe_bot_id = validate_id(bot_id, "bot_id") + profile = Path(profile_path).resolve() + profiles_root = (self.hermes_root / "profiles").resolve() + return self._validate_exact_profile_path(safe_bot_id, profile, profiles_root) + + def _pin_profile_path(self, bot_id: str, profile_path: Path | str) -> Path: + safe_bot_id = validate_id(bot_id, "bot_id") + absolute_profile = Path(os.path.abspath(profile_path)) + profile = absolute_profile.parent.resolve() / absolute_profile.name + profiles_root = (self.hermes_root / "profiles").resolve() + return self._validate_exact_profile_path(safe_bot_id, profile, profiles_root) + + @staticmethod + def _validate_exact_profile_path( + safe_bot_id: str, + profile: Path, + profiles_root: Path, + ) -> Path: + try: + relative = profile.relative_to(profiles_root) + except ValueError as exc: + raise BotDeleteError("bot profile path is outside the Hermes profiles root") from exc + if len(relative.parts) != 1 or relative.parts[0] != safe_bot_id: + raise BotDeleteError("bot profile path does not match bot id") + return profile + + def stage_delete( + self, + bot_id: str, + profile_path: Path | str, + ) -> ProfileDeletion | None: + profile = self.validate_profile_path(bot_id, profile_path) + if not profile.exists(): + return None + tombstone = profile.with_name(f".{profile.name}.deleting-{os.getpid()}-{time.time_ns()}") + try: + os.replace(profile, tombstone) + except OSError as exc: + raise BotDeleteError("could not stage the bot profile for deletion") from exc + return ProfileDeletion(profile_path=profile, tombstone_path=tombstone) + + def rollback_delete(self, deletion: ProfileDeletion) -> None: + if os.path.lexists(deletion.profile_path): + raise BotDeleteError( + "bot state deletion failed and the profile could not be restored because " + "its original path is occupied" + ) + try: + os.replace(deletion.tombstone_path, deletion.profile_path) + except OSError as exc: + raise BotDeleteError( + "bot state deletion failed and the profile could not be restored" + ) from exc + + def finish_delete(self, deletion: ProfileDeletion) -> OSError | None: + try: + shutil.rmtree(deletion.tombstone_path) + except OSError as exc: + return exc + return None + + def stage_archive( + self, + bot_id: str, + profile_path: Path | str, + ) -> ProfileArchive | None: + profile = self.validate_profile_path(bot_id, profile_path) + if not profile.exists(): + return None + self.archive_root.mkdir(parents=True, exist_ok=True) + archive_path = self.archive_root / ( + f"{validate_id(bot_id, 'bot_id')}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" + ) + shutil.move(str(profile), str(archive_path)) + return ProfileArchive(profile_path=profile, archive_path=archive_path) + + def rollback_archive(self, archive: ProfileArchive) -> None: + if os.path.lexists(archive.profile_path): + raise BotArchiveError( + "bot state deletion failed and the archived profile could not be restored " + "because its original path is occupied" + ) + try: + shutil.move(str(archive.archive_path), str(archive.profile_path)) + except OSError as exc: + raise BotArchiveError( + "bot state deletion failed and the archived profile could not be restored" + ) from exc diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 76dce41..6a3623a 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -8,7 +8,6 @@ import platform import re import select -import shutil import signal import stat import subprocess # nosec B404 @@ -71,6 +70,7 @@ ) from zeus.private_io import UnsafeFileError, nofollow_absolute_path, open_private_append from zeus.process_lock import BotProcessLock, LockTimeoutError +from zeus.profile_manager import ProfileArchive, ProfileDeletion, ProfileManager from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env from zeus.reconciliation import ( BotReconcileResult, @@ -81,7 +81,6 @@ ReconcileRunSummary, ReconcileSnapshotDriftError, ) -from zeus.renderer import ProfileRenderer from zeus.state import StateStore @@ -206,6 +205,10 @@ def __init__( hermes_bin=hermes_bin, hermes_root=configured_hermes_root.resolve(), ) + self._profile_manager = ProfileManager( + self.adapter.hermes_root, + self.store.database_path.parent / "archive", + ) self._marker_profiles_root = configured_hermes_root / "profiles" self.popen_factory = popen_factory self.kill_fn = kill_fn @@ -411,8 +414,7 @@ def create_bot( ) self._assert_unregistered_profile_inactive(bot_id, profile_path) - renderer = ProfileRenderer(self.adapter.hermes_root) - renderer.preflight(request, template) + self._profile_manager.preflight(request, template) stopped_record: BotRecord | None = None if existing is not None: active = self._record_may_be_active(existing) @@ -423,7 +425,7 @@ def create_bot( stopped_record = existing try: - with renderer.transaction(request, template) as record: + with self._profile_manager.install_transaction(request, template) as record: self._remove_pid_marker(record.profile_path) self.store.upsert_bot_with_event( record, @@ -475,10 +477,10 @@ def delete_bot( stopped = self._stop_locked(safe_bot_id, context=context) if stopped.status != BotStatus.stopped: raise BotDeleteError(f"could not stop bot before delete: {stopped.message}") - profile_tombstone: Path | None = None + profile_deletion: ProfileDeletion | None = None try: if remove_profile: - profile_tombstone = self._stage_profile_deletion( + profile_deletion = self._profile_manager.stage_delete( safe_bot_id, record.profile_path ) else: @@ -495,11 +497,12 @@ def delete_bot( ) if not deleted: raise KeyError(f"unknown bot: {safe_bot_id}") - except BaseException: - if profile_tombstone is not None: - self._restore_tombstoned_profile( - safe_bot_id, record.profile_path, profile_tombstone - ) + except BaseException as operation_error: + if profile_deletion is not None: + try: + self._profile_manager.rollback_delete(profile_deletion) + except BotDeleteError as rollback_error: + raise rollback_error from operation_error if was_active: try: self._recover_previously_active_bot(record, "deletion", context=context) @@ -509,15 +512,14 @@ def delete_bot( ) from recovery_error raise cleanup_pending = False - if profile_tombstone is not None: - try: - shutil.rmtree(profile_tombstone) - except OSError as exc: + if profile_deletion is not None: + cleanup_error = self._profile_manager.finish_delete(profile_deletion) + if cleanup_error is not None: cleanup_pending = True self.store.append_audit_event( "bot.delete_cleanup_pending", bot_id=safe_bot_id, - error=type(exc).__name__, + error=type(cleanup_error).__name__, ) self.store.append_audit_event( "bot.delete", @@ -557,16 +559,9 @@ def archive_bot( if stopped.status != BotStatus.stopped: raise BotArchiveError(f"could not stop bot before archive: {stopped.message}") - archive_path: Path | None = None + profile_archive: ProfileArchive | None = None try: - if profile_path.exists(): - archive_root = self.store.database_path.parent / "archive" - archive_root.mkdir(parents=True, exist_ok=True) - candidate = archive_root / ( - f"{safe_bot_id}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" - ) - shutil.move(str(profile_path), str(candidate)) - archive_path = candidate + profile_archive = self._profile_manager.stage_archive(safe_bot_id, profile_path) deleted = self.store.delete_bot_with_event( safe_bot_id, event=self._event( @@ -578,9 +573,12 @@ def archive_bot( ) if not deleted: raise KeyError(f"unknown bot: {safe_bot_id}") - except BaseException: - if archive_path is not None: - self._restore_archived_profile(safe_bot_id, record.profile_path, archive_path) + except BaseException as operation_error: + if profile_archive is not None: + try: + self._profile_manager.rollback_archive(profile_archive) + except BotArchiveError as rollback_error: + raise rollback_error from operation_error if was_active: try: self._recover_previously_active_bot(record, "archive", context=context) @@ -589,6 +587,7 @@ def archive_bot( "bot archive failed and the previous bot could not be restarted" ) from recovery_error raise + archive_path = profile_archive.archive_path if profile_archive is not None else None self.store.append_audit_event( "bot.archive", bot_id=safe_bot_id, @@ -3156,27 +3155,13 @@ def _assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) ) def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: - safe_bot_id = validate_id(bot_id, "bot_id") - profile = Path(profile_path).resolve() - profiles_root = (Path(self.adapter.hermes_root) / "profiles").resolve() - try: - relative = profile.relative_to(profiles_root) - except ValueError as exc: - raise BotDeleteError("bot profile path is outside the Hermes profiles root") from exc - if len(relative.parts) != 1 or relative.parts[0] != safe_bot_id: - raise BotDeleteError("bot profile path does not match bot id") - return profile + return self._profile_manager.validate_profile_path(bot_id, profile_path) def _stage_profile_deletion(self, bot_id: str, profile_path: str) -> Path | None: - profile = self._safe_profile_path(bot_id, profile_path) - if not profile.exists(): + deletion = self._profile_manager.stage_delete(bot_id, profile_path) + if deletion is None: return None - tombstone = profile.with_name(f".{profile.name}.deleting-{os.getpid()}-{time.time_ns()}") - try: - os.replace(profile, tombstone) - except OSError as exc: - raise BotDeleteError("could not stage the bot profile for deletion") from exc - return tombstone + return deletion.tombstone_path def _restore_tombstoned_profile( self, @@ -3184,18 +3169,13 @@ def _restore_tombstoned_profile( profile_path: str, tombstone: Path, ) -> None: - profile = self._safe_profile_path(bot_id, profile_path) - if profile.exists() or profile.is_symlink(): - raise BotDeleteError( - "bot state deletion failed and the profile could not be restored because " - "its original path is occupied" + profile = self._profile_manager._pin_profile_path(bot_id, profile_path) + self._profile_manager.rollback_delete( + ProfileDeletion( + profile_path=profile, + tombstone_path=tombstone, ) - try: - os.replace(tombstone, profile) - except OSError as exc: - raise BotDeleteError( - "bot state deletion failed and the profile could not be restored" - ) from exc + ) def _restore_archived_profile( self, @@ -3203,18 +3183,13 @@ def _restore_archived_profile( profile_path: str, archive_path: Path, ) -> None: - profile = self._safe_profile_path(bot_id, profile_path) - if profile.exists() or profile.is_symlink(): - raise BotArchiveError( - "bot state deletion failed and the archived profile could not be restored " - "because its original path is occupied" + profile = self._profile_manager._pin_profile_path(bot_id, profile_path) + self._profile_manager.rollback_archive( + ProfileArchive( + profile_path=profile, + archive_path=archive_path, ) - try: - shutil.move(str(archive_path), str(profile)) - except OSError as exc: - raise BotArchiveError( - "bot state deletion failed and the archived profile could not be restored" - ) from exc + ) def _readiness_probe_for_bot( self, bot_id: str, *, timeout_seconds: float | None = None From 9fdbbd573222dc4891f271d73701d7abaeab72e2 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 02:34:53 +0200 Subject: [PATCH 39/49] refactor(supervisor): delegate gateway runtime effects --- tests/test_crash_recovery.py | 9 +- tests/test_gateway_runtime.py | 536 +++++++++++ zeus/gateway_runtime.py | 1641 +++++++++++++++++++++++++++++++ zeus/supervisor.py | 1701 ++++++++++----------------------- 4 files changed, 2666 insertions(+), 1221 deletions(-) create mode 100644 tests/test_gateway_runtime.py create mode 100644 zeus/gateway_runtime.py diff --git a/tests/test_crash_recovery.py b/tests/test_crash_recovery.py index 9a3cb90..fe4410d 100644 --- a/tests/test_crash_recovery.py +++ b/tests/test_crash_recovery.py @@ -3433,7 +3433,14 @@ def wait(self, timeout: float) -> None: raise subprocess.TimeoutExpired("hermes", timeout) with tempfile.TemporaryDirectory() as tmp: - fingerprints = iter(["test-start:4321", "test-start:4321", "test-start:reused"]) + fingerprints = iter( + [ + "test-start:4321", + "test-start:4321", + "test-start:4321", + "test-start:reused", + ] + ) store, supervisor = self._fixture( Path(tmp), desired=DesiredState.running, diff --git a/tests/test_gateway_runtime.py b/tests/test_gateway_runtime.py new file mode 100644 index 0000000..0f566a1 --- /dev/null +++ b/tests/test_gateway_runtime.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +import ast +import dataclasses +import inspect +import json +import os +import signal +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import ( + GatewayRuntime, + KillFn, + LaunchEffect, + MarkerObservation, + OwnershipCheck, + PopenFactory, + PopenLike, + SignalResult, + StopEffect, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.models import BotRecord, BotStatus, DesiredState +from zeus.profile_manager import ProfileManager +from zeus.state import StateStore +from zeus.supervisor import Supervisor + + +class _NeverExits: + def __init__(self, pid: int) -> None: + self.pid = pid + + def poll(self) -> int | None: + return None + + def wait(self, *, timeout: float) -> None: + del timeout + raise TimeoutError + + +class GatewayRuntimeTests(unittest.TestCase): + bot_id = "coder" + pid = 4321 + operation_id = "a" * 32 + + def _fake_hermes(self, root: Path) -> str: + hermes = root / "bin" / "hermes" + hermes.parent.mkdir(parents=True, exist_ok=True) + hermes.write_text("#!/bin/sh\n", encoding="utf-8") + hermes.chmod(0o755) + return str(hermes.resolve()) + + def _fixture( + self, + root: Path, + *, + fingerprints: list[str | None] | None = None, + signals: list[tuple[int, signal.Signals]] | None = None, + popen_factory=None, + stop_grace_seconds: float = 0.0, + kill_after_timeout: bool = False, + ) -> tuple[GatewayRuntime, BotRecord, Path, str]: + root = root.resolve() + hermes_root = root / "hermes" + profile = hermes_root / "profiles" / self.bot_id + profile.mkdir(parents=True) + (profile / ".env").write_text("", encoding="utf-8") + hermes = self._fake_hermes(root) + adapter = HermesAdapter(hermes, hermes_root) + manager = ProfileManager(hermes_root, root / "archive") + fingerprint_values = list(fingerprints or ["same"]) + + def read_fingerprint(pid: int) -> str | None: + self.assertEqual(self.pid, pid) + if len(fingerprint_values) > 1: + return fingerprint_values.pop(0) + return fingerprint_values[0] + + observed_signals = signals if signals is not None else [] + runtime = GatewayRuntime( + adapter, + manager, + hermes_root / "profiles", + popen_factory=popen_factory or (lambda *args, **kwargs: _NeverExits(self.pid)), + kill_fn=lambda pid, sig: observed_signals.append((pid, sig)), + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [hermes, "-p", self.bot_id, "gateway", "run"], + proc_start_fingerprint_reader=read_fingerprint, + startup_grace_seconds=0, + stop_grace_seconds=stop_grace_seconds, + kill_after_timeout=kill_after_timeout, + cleanup_process_group=False, + ) + record = BotRecord( + bot_id=self.bot_id, + template_id="coding-bot", + display_name="Coder", + profile_path=str(profile), + status=BotStatus.running, + pid=self.pid, + desired_state=DesiredState.stopped, + desired_revision=2, + pending_operation_id="b" * 32, + pending_action="stop", + ) + return runtime, record, profile, hermes + + def _schema3_marker( + self, + runtime: GatewayRuntime, + profile: Path, + *, + operation_id: str | None = None, + desired_revision: int = 1, + pid: int | None = None, + process_start: str = "same", + ) -> tuple[dict[str, object], GatewayGeneration]: + payload = runtime.adapter.launcher_payload( + self.bot_id, + operation_id=operation_id or self.operation_id, + desired_revision=desired_revision, + readiness_probe=None, + ) + marker = dict(payload["marker"]) + marker.update( + { + "pid": pid or self.pid, + "started_at": 1_780_000_000.0, + "proc_start_fingerprint": process_start, + } + ) + marker_path = runtime.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(marker, sort_keys=True) + "\n", encoding="utf-8") + generation = GatewayGeneration( + operation_id=str(marker["operation_id"]), + desired_revision=int(marker["desired_revision"]), + pid=int(marker["pid"]), + command_fingerprint=str(marker["command_fingerprint"]), + proc_start_fingerprint=process_start, + ) + return marker, generation + + def test_effects_are_frozen_bounded_and_do_not_retain_secret_fields(self) -> None: + launch = LaunchEffect("failed", reason="x" * 2_000, readiness_message="z" * 2_000) + stop = StopEffect("pending", reason="y" * 2_000) + + self.assertLessEqual(len(launch.reason), 512) + self.assertLessEqual(len(launch.readiness_message or ""), 512) + self.assertLessEqual(len(stop.reason), 512) + self.assertNotIn("env", vars(launch)) + self.assertNotIn("payload", vars(launch)) + self.assertNotIn("argv", vars(launch)) + with self.assertRaises(dataclasses.FrozenInstanceError): + launch.reason = "changed" # type: ignore[misc] + with self.assertRaises(dataclasses.FrozenInstanceError): + stop.reason = "changed" # type: ignore[misc] + + def test_launch_secret_is_confined_to_private_descriptor_payload(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + captured: dict[str, object] = {} + secret = "descriptor-only-launch-secret" + + class CapturingPopen: + pid = self.pid + + def __init__(self, argv, env, stdout, stderr, **kwargs): + del stdout, stderr, kwargs + captured["argv"] = list(argv) + captured["env"] = dict(env) + payload_fd = os.dup(int(argv[-2])) + ack_fd = os.dup(int(argv[-1])) + + def publish() -> None: + try: + chunks: list[bytes] = [] + while chunk := os.read(payload_fd, 65_536): + chunks.append(chunk) + payload = json.loads(b"".join(chunks)) + captured["payload"] = payload + marker = dict(payload["marker"]) + marker.update( + { + "pid": self.pid, + "started_at": time.time(), + "proc_start_fingerprint": "same", + } + ) + marker_path = Path(payload["marker_path"]) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(marker), encoding="utf-8") + os.write(ack_fd, b"1") + finally: + os.close(payload_fd) + os.close(ack_fd) + + threading.Thread(target=publish, daemon=True).start() + + def poll(self) -> int | None: + return None + + runtime, record, profile, _hermes = self._fixture( + root, + popen_factory=CapturingPopen, + ) + (profile / ".env").write_text( + f"OPENROUTER_API_KEY={secret}\n", + encoding="utf-8", + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + + with patch.dict(os.environ, {"PATH": os.environ.get("PATH", "")}, clear=True): + effect = runtime.launch(launching, probe=None, wait=False) + + private_payload = captured["payload"] + assert isinstance(private_payload, dict) + marker_text = runtime.pid_marker_path(str(profile)).read_text(encoding="utf-8") + self.assertEqual(secret, private_payload["env"]["OPENROUTER_API_KEY"]) + self.assertNotIn(secret, "\0".join(captured["argv"])) + self.assertNotIn(secret, json.dumps(captured["env"], sort_keys=True)) + self.assertNotIn(secret, marker_text) + self.assertNotIn(secret, repr(effect)) + self.assertNotIn(secret, json.dumps(vars(effect), default=str, sort_keys=True)) + + def test_exact_generation_rejects_changed_correlation_and_process_identity(self) -> None: + mutations = { + "operation": lambda marker: marker.__setitem__("operation_id", "c" * 32), + "revision": lambda marker: marker.__setitem__("desired_revision", 9), + "pid": lambda marker: marker.__setitem__("pid", 9999), + "command": lambda marker: marker.__setitem__("command_fingerprint", "0" * 64), + "start": lambda marker: marker.__setitem__("proc_start_fingerprint", "reused"), + } + for name, mutate in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture(Path(tmp)) + marker, generation = self._schema3_marker(runtime, profile) + baseline = runtime.classify_exact_gateway_generation(record, generation) + self.assertEqual("live", baseline.kind) + mutate(marker) + runtime.pid_marker_path(str(profile)).write_text( + json.dumps(marker, sort_keys=True) + "\n", + encoding="utf-8", + ) + + changed = runtime.classify_exact_gateway_generation(record, generation) + + self.assertEqual("untrusted", changed.kind) + + def test_final_pre_sigterm_fingerprint_drift_sends_no_signal_or_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + fingerprints=["same", "same", "reused"], + signals=signals, + ) + _marker, _generation = self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("term_reauthorization_failed", effect.outcome) + self.assertEqual([], signals) + self.assertTrue(marker_path.exists()) + + def test_pre_sigkill_fingerprint_drift_sends_term_only(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + fingerprints=["same", "same", "same", "reused"], + signals=signals, + kill_after_timeout=True, + ) + self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("kill_reauthorization_failed", effect.outcome) + self.assertEqual([(self.pid, signal.SIGTERM)], signals) + self.assertTrue(marker_path.exists()) + + def test_legacy_and_schema2_markers_never_signal_start_or_remove(self) -> None: + payloads: list[dict[str, object]] = [ + {"pid": self.pid, "argv": ["hermes", "gateway", "run"]}, + { + "schema": 2, + "pid": self.pid, + "bot_id": self.bot_id, + "component": "gateway", + "action": "run", + "argv": ["hermes", "-p", self.bot_id, "gateway", "run"], + }, + ] + for payload in payloads: + with self.subTest(schema=payload.get("schema")), tempfile.TemporaryDirectory() as tmp: + signals: list[tuple[int, signal.Signals]] = [] + launches: list[object] = [] + + def forbidden_popen(*args, _launches=launches, **kwargs): + _launches.append((args, kwargs)) + raise AssertionError("stop must not start a process") + + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + signals=signals, + popen_factory=forbidden_popen, + ) + marker_path = runtime.pid_marker_path(str(profile)) + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(json.dumps(payload), encoding="utf-8") + + with runtime.marker_publication_lock(record): + effect = runtime.stop_locked(record, kill_after_timeout=True) + + self.assertEqual("compat_untrusted", effect.outcome) + self.assertEqual([], signals) + self.assertEqual([], launches) + self.assertTrue(marker_path.exists()) + + def test_exact_cleanup_preserves_hardlink_symlink_and_replacement_evidence(self) -> None: + for case in ("hardlink", "symlink", "replacement"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture(Path(tmp)) + marker, generation = self._schema3_marker(runtime, profile) + marker_path = runtime.pid_marker_path(str(profile)) + evidence = Path(tmp) / "evidence.json" + if case == "hardlink": + os.link(marker_path, evidence) + elif case == "symlink": + evidence.write_text(marker_path.read_text(encoding="utf-8"), encoding="utf-8") + marker_path.unlink() + marker_path.symlink_to(evidence) + else: + marker["operation_id"] = "c" * 32 + marker_path.write_text(json.dumps(marker), encoding="utf-8") + + removed = runtime.remove_gateway_generation_marker_locked(record, generation) + + if case == "hardlink": + self.assertTrue(removed) + self.assertFalse(marker_path.exists()) + else: + self.assertFalse(removed) + self.assertTrue(os.path.lexists(marker_path)) + if case != "replacement": + self.assertTrue(evidence.exists()) + + def test_gateway_runtime_has_no_state_store_or_sqlite_dependency(self) -> None: + import zeus.gateway_runtime as gateway_runtime + + source = Path(gateway_runtime.__file__).read_text(encoding="utf-8") + tree = ast.parse(source) + imports = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } | {node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom)} + self.assertNotIn("sqlite3", imports) + self.assertNotIn("zeus.state", imports) + self.assertNotIn("StateStore", source) + + def test_supervisor_reexports_boundary_types_and_preserves_signatures(self) -> None: + import zeus.supervisor as supervisor_module + + self.assertIs(PopenLike, supervisor_module.PopenLike) + self.assertIs(PopenFactory, supervisor_module.PopenFactory) + self.assertIs(KillFn, supervisor_module.KillFn) + self.assertIs(OwnershipCheck, supervisor_module.OwnershipCheck) + self.assertIs(MarkerObservation, supervisor_module._MarkerObservation) + self.assertIs(SignalResult, supervisor_module._SignalResult) + self.assertEqual( + [ + "self", + "store", + "hermes_bin", + "hermes_root", + "popen_factory", + "kill_fn", + "pid_alive_fn", + "cmdline_reader", + "startup_grace_seconds", + "stop_grace_seconds", + "kill_after_timeout", + "lock_timeout_seconds", + "readiness_timeout_seconds", + "readiness_interval_seconds", + "allow_legacy_pid_markers", + "restart_backoff_cap_seconds", + "proc_start_fingerprint_reader", + ], + list(inspect.signature(Supervisor.__init__).parameters), + ) + self.assertEqual( + ["self", "bot_id", "wait", "timeout_seconds", "source", "request_id"], + list(inspect.signature(Supervisor.start).parameters), + ) + self.assertEqual( + ["self", "bot_id", "kill_after_timeout", "source", "request_id"], + list(inspect.signature(Supervisor.stop).parameters), + ) + + def test_supervisor_runtime_proxies_remain_live_after_construction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + hermes_root = root / "hermes" + (hermes_root / "profiles").mkdir(parents=True) + store = StateStore(root / "zeus.db") + store.init() + supervisor = Supervisor(store, "hermes", hermes_root) + + def popen(*args, **kwargs): + return _NeverExits(77) + + def killer(pid, sig): + return None + + def alive(pid): + return True + + def cmdline(pid): + return ["hermes"] + + def fingerprint(pid): + return "changed" + + processes = {"coder": _NeverExits(77)} + + supervisor.popen_factory = popen + supervisor.kill_fn = killer + supervisor.pid_alive_fn = alive + supervisor.cmdline_reader = cmdline + supervisor.proc_start_fingerprint_reader = fingerprint + supervisor._processes = processes + supervisor.stop_grace_seconds = 9.5 + supervisor.kill_after_timeout = True + supervisor.lock_timeout_seconds = 4.25 + + self.assertIs(popen, supervisor._runtime.popen_factory) + self.assertIs(killer, supervisor._runtime.kill_fn) + self.assertIs(alive, supervisor._runtime.pid_alive_fn) + self.assertIs(cmdline, supervisor._runtime.cmdline_reader) + self.assertIs(fingerprint, supervisor._runtime.proc_start_fingerprint_reader) + self.assertIs(processes, supervisor._runtime._processes) + self.assertEqual(9.5, supervisor._runtime.stop_grace_seconds) + self.assertTrue(supervisor._runtime.kill_after_timeout) + self.assertEqual(4.25, supervisor._runtime.lock_timeout_seconds) + + def test_supervisor_normal_stop_uses_shared_final_reauthorization_primitive(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runtime, record, profile, hermes = self._fixture(root) + self._schema3_marker(runtime, profile) + store = StateStore(root / "zeus.db") + store.init() + fingerprints = iter(("same", "same", "reused")) + supervisor = Supervisor( + store, + hermes, + root / "hermes", + pid_alive_fn=lambda pid: True, + cmdline_reader=lambda pid: [hermes, "-p", self.bot_id, "gateway", "run"], + proc_start_fingerprint_reader=lambda pid: next(fingerprints), + kill_fn=lambda pid, sig: None, + ) + supervisor._runtime.stop_grace_seconds = 0 + + with patch.object( + supervisor._runtime, + "reauthorize_and_signal", + wraps=supervisor._runtime.reauthorize_and_signal, + ) as reauthorize: + supervisor._stop_record_effect_locked( + record, + kill_after_timeout=False, + context=supervisor._lifecycle_context("system", None), + complete_stop=False, + ) + + self.assertGreaterEqual(reauthorize.call_count, 1) + + def test_public_runtime_result_types_have_expected_frozen_surface(self) -> None: + observation = MarkerObservation("missing", reason="gone") + self.assertEqual("missing", observation.kind) + self.assertEqual("sent", SignalResult.sent.value) + with self.assertRaises(dataclasses.FrozenInstanceError): + observation.kind = "present" # type: ignore[misc] + + def test_marker_observation_snapshots_payload_and_hides_hostile_values(self) -> None: + hostile = "hostile-marker-value-must-not-escape-repr" + source: dict[str, object] = {"nested": {"value": hostile}} + observation = MarkerObservation("present", payload=source) + source_nested = source["nested"] + assert isinstance(source_nested, dict) + source_nested["value"] = "source-mutated" + + first = observation.payload + assert first is not None + first_nested = first["nested"] + assert isinstance(first_nested, dict) + self.assertEqual(hostile, first_nested["value"]) + first_nested["value"] = "returned-copy-mutated" + + second = observation.payload + assert second is not None + second_nested = second["nested"] + assert isinstance(second_nested, dict) + self.assertEqual(hostile, second_nested["value"]) + self.assertNotIn(hostile, repr(observation)) + + rejected = MarkerObservation("present", payload={"hostile": object()}) + self.assertEqual("untrusted", rejected.kind) + self.assertIsNone(rejected.payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/gateway_runtime.py b/zeus/gateway_runtime.py new file mode 100644 index 0000000..deb989a --- /dev/null +++ b/zeus/gateway_runtime.py @@ -0,0 +1,1641 @@ +from __future__ import annotations + +import contextlib +import errno +import json +import math +import os +import platform +import select +import signal +import stat +import subprocess # nosec B404 +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Protocol + +from zeus import process_identity +from zeus.errors import BotRunningError +from zeus.fs_utils import atomic_write_json +from zeus.gateway_launcher import ( + MAX_PAYLOAD_BYTES, + LaunchPayloadError, + _confirm_marker_missing, + _ConfirmedMissing, + _open_logs, + _open_profile_chain, + _open_regular_marker, + _read_bounded_file, + _reject_duplicate_keys, + _remove_marker_if_owned_locked, + _validate_marker_bindings, + marker_publication_lock, + remove_marker_if_owned, +) +from zeus.gateway_marker import ( + GatewayGeneration, + is_compat_runtime_marker, + is_owned_runtime_marker, + readiness_probe_from_payload, + readiness_probe_to_payload, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.models import BotRecord, TemplateError +from zeus.private_io import UnsafeFileError, nofollow_absolute_path, open_private_append +from zeus.profile_manager import ProfileManager +from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env + + +class PopenLike(Protocol): + pid: int + + def poll(self) -> int | None: ... + + +PopenFactory = Callable[..., PopenLike] +KillFn = Callable[[int, signal.Signals], None] +PidAliveFn = process_identity.PidAliveFn +CmdlineReader = process_identity.CmdlineReader +ProcStartFingerprintReader = process_identity.ProcStartFingerprintReader + + +class SignalResult(Enum): + sent = "sent" + missing = "missing" + denied = "denied" + + +_MAX_EFFECT_TEXT = 512 + + +def _bounded_text(value: str) -> str: + return value[:_MAX_EFFECT_TEXT] + + +@dataclass(frozen=True) +class OwnershipCheck: + verified: bool + reason: str + classification: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + if self.classification is not None: + object.__setattr__( + self, + "classification", + _bounded_text(self.classification), + ) + + +@dataclass(frozen=True, init=False) +class MarkerObservation: + kind: str + reason: str + _payload_json: bytes | None = field(repr=False) + + def __init__( + self, + kind: str, + payload: dict[str, object] | None = None, + reason: str = "", + ) -> None: + snapshot: bytes | None = None + if payload is not None: + try: + snapshot = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError, UnicodeEncodeError): + kind = "untrusted" + reason = "marker payload could not be snapshotted safely" + else: + if len(snapshot) > MAX_PAYLOAD_BYTES: + kind = "untrusted" + reason = "marker payload snapshot is too large" + snapshot = None + object.__setattr__(self, "kind", kind) + object.__setattr__(self, "reason", _bounded_text(reason)) + object.__setattr__(self, "_payload_json", snapshot) + + @property + def payload(self) -> dict[str, object] | None: + if self._payload_json is None: + return None + value = json.loads(self._payload_json) + if type(value) is not dict: + return None + return value + + +@dataclass(frozen=True) +class LaunchEffect: + outcome: str + pid: int | None = None + generation: GatewayGeneration | None = None + reason: str = "" + returncode: int | None = None + error_type: str | None = None + readiness_message: str | None = None + cleanup_complete: bool | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + if self.error_type is not None: + object.__setattr__(self, "error_type", _bounded_text(self.error_type)) + if self.readiness_message is not None: + object.__setattr__( + self, + "readiness_message", + _bounded_text(self.readiness_message), + ) + + +@dataclass(frozen=True) +class StopEffect: + outcome: str + pid: int | None = None + generation: GatewayGeneration | None = None + reason: str = "" + term_result: SignalResult | None = None + kill_result: SignalResult | None = None + marker_removed: bool = False + kill_attempted: bool = False + kill_succeeded: bool | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "reason", _bounded_text(self.reason)) + + +PipeFn = Callable[[], tuple[int, int]] +CloseFn = Callable[[int], None] +ReadBoundedFileFn = Callable[[int], bytes] +RemoveMarkerLockedFn = Callable[..., bool] +ProbeOnceFn = Callable[..., ReadinessResult] + + +@dataclass(frozen=True) +class RuntimeHooks: + pipe: PipeFn + close: CloseFn + read_bounded_file: ReadBoundedFileFn + remove_marker_if_owned_locked: RemoveMarkerLockedFn + probe_once: ProbeOnceFn + + +def default_runtime_hooks() -> RuntimeHooks: + return RuntimeHooks( + pipe=os.pipe, + close=os.close, + read_bounded_file=_read_bounded_file, + remove_marker_if_owned_locked=_remove_marker_if_owned_locked, + probe_once=probe_once, + ) + + +def gateway_process_launch_kwargs() -> dict[str, object]: + if os.name == "posix": + return {"start_new_session": True} + creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + return {"creationflags": creationflags} if creationflags else {} + + +def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + +class GatewayRuntime: + def __init__( + self, + adapter: HermesAdapter, + profile_manager: ProfileManager, + marker_profiles_root: Path, + *, + popen_factory: PopenFactory = subprocess.Popen, + kill_fn: KillFn = os.kill, + pid_alive_fn: PidAliveFn | None = None, + cmdline_reader: CmdlineReader, + proc_start_fingerprint_reader: ProcStartFingerprintReader, + startup_grace_seconds: float = 0.25, + stop_grace_seconds: float = 15.0, + kill_after_timeout: bool = False, + lock_timeout_seconds: float = 30.0, + readiness_timeout_seconds: float = 30.0, + readiness_interval_seconds: float = 0.5, + allow_legacy_pid_markers: bool = True, + cleanup_process_group: bool = False, + hooks_provider: Callable[[], RuntimeHooks] = default_runtime_hooks, + ) -> None: + self.adapter = adapter + self.profile_manager = profile_manager + self.marker_profiles_root = marker_profiles_root + self.popen_factory = popen_factory + self.kill_fn = kill_fn + self.pid_alive_fn = pid_alive_fn + self.cmdline_reader = cmdline_reader + self.proc_start_fingerprint_reader = proc_start_fingerprint_reader + self.startup_grace_seconds = startup_grace_seconds + self.stop_grace_seconds = stop_grace_seconds + self.kill_after_timeout = kill_after_timeout + self.lock_timeout_seconds = lock_timeout_seconds + self.readiness_timeout_seconds = readiness_timeout_seconds + self.readiness_interval_seconds = readiness_interval_seconds + self.allow_legacy_pid_markers = allow_legacy_pid_markers + self.cleanup_process_group = cleanup_process_group + self._hooks_provider = hooks_provider + self._processes: dict[str, PopenLike] = {} + + def _hooks(self) -> RuntimeHooks: + return self._hooks_provider() + + def safe_profile_path(self, bot_id: str, profile_path: str) -> Path: + return self.profile_manager.validate_profile_path(bot_id, profile_path) + + def marker_publication_lock( + self, + record: BotRecord, + ) -> contextlib.AbstractContextManager[object]: + profile_path = self.safe_profile_path(record.bot_id, record.profile_path) + if not os.path.lexists(profile_path): + return contextlib.nullcontext() + return marker_publication_lock( + profile_path, + timeout_seconds=self.lock_timeout_seconds, + ) + + def log_path(self, profile_path: str) -> Path: + return nofollow_absolute_path(Path(profile_path) / "logs" / "zeus-gateway.log") + + def pid_marker_path(self, profile_path: str) -> Path: + return Path(profile_path) / "logs" / "zeus-gateway.pid.json" + + def preflight_start( + self, + record: BotRecord, + *, + timeout_seconds: float | None, + ) -> ReadinessProbe | None: + expected_profile = (Path(self.adapter.hermes_root) / "profiles" / record.bot_id).resolve() + if Path(record.profile_path).resolve() != expected_profile: + raise TemplateError("registered bot profile does not match the Hermes profile path") + safe_profile = self.safe_profile_path(record.bot_id, record.profile_path) + if not safe_profile.is_dir() or safe_profile.is_symlink(): + raise TemplateError("registered bot profile is not a safe directory") + _argv, env = self.adapter.command(record.bot_id, "gateway", "run") + readiness = self.readiness_probe(env, timeout_seconds=timeout_seconds) + self.adapter.launcher_payload( + record.bot_id, + operation_id="0" * 32, + desired_revision=max(1, record.desired_revision + 1), + readiness_probe=readiness, + ) + return readiness + + @staticmethod + def write_pipe_payload(fd: int, payload: bytes) -> None: + offset = 0 + while offset < len(payload): + written = os.write(fd, payload[offset:]) + if written <= 0: + raise OSError("short launcher payload write") + offset += written + + @staticmethod + def read_launcher_ack(fd: int) -> bytes: + deadline = time.monotonic() + 5.0 + acknowledgment = bytearray() + while len(acknowledgment) <= 1: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("gateway launcher acknowledgment timed out") + readable, _writable, _exceptional = select.select([fd], [], [], remaining) + if not readable: + raise TimeoutError("gateway launcher acknowledgment timed out") + chunk = os.read(fd, 2 - len(acknowledgment)) + if not chunk: + return bytes(acknowledgment) + acknowledgment.extend(chunk) + return bytes(acknowledgment) + + def launch( + self, + record: BotRecord, + *, + probe: ReadinessProbe | None, + wait: bool, + marker_lock: Callable[[BotRecord], contextlib.AbstractContextManager[object]] | None = None, + marker_matcher: Callable[..., MarkerObservation] | None = None, + ack_reader: Callable[[int], bytes] | None = None, + pipe_writer: Callable[[int, bytes], None] | None = None, + ) -> LaunchEffect: + operation_id = record.pending_operation_id + if record.pending_action not in {"start", "restart"} or operation_id is None: + raise RuntimeError("gateway launch requires a pending start or restart intent") + payload = self.adapter.launcher_payload( + record.bot_id, + operation_id=operation_id, + desired_revision=record.desired_revision, + readiness_probe=probe, + ) + marker_data = payload["marker"] + if type(marker_data) is not dict: + raise RuntimeError("launcher produced an invalid marker payload") + expected_fingerprint = str(marker_data["command_fingerprint"]) + process: PopenLike | None = None + generation: GatewayGeneration | None = None + payload_read = payload_write = ack_read = ack_write = -1 + hooks = self._hooks() + marker_lock = marker_lock or self.marker_publication_lock + marker_matcher = marker_matcher or self.matching_runtime_marker + ack_reader = ack_reader or self.read_launcher_ack + pipe_writer = pipe_writer or self.write_pipe_payload + try: + encoded_payload = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + if not encoded_payload or len(encoded_payload) > MAX_PAYLOAD_BYTES: + raise ValueError("launcher payload is too large") + with open_private_append(self.log_path(record.profile_path)) as log_file: + payload_read, payload_write = hooks.pipe() + ack_read, ack_write = hooks.pipe() + launcher_argv = self.adapter.launcher_command(payload_read, ack_write) + process = self.popen_factory( + launcher_argv, + env=dict(os.environ), + stdout=log_file, + stderr=log_file, + pass_fds=(payload_read, ack_write), + close_fds=True, + **gateway_process_launch_kwargs(), + ) + hooks.close(payload_read) + payload_read = -1 + hooks.close(ack_write) + ack_write = -1 + pipe_writer(payload_write, encoded_payload) + hooks.close(payload_write) + payload_write = -1 + acknowledgment = ack_reader(ack_read) + hooks.close(ack_read) + ack_read = -1 + if acknowledgment != b"1": + raise RuntimeError("gateway launcher did not acknowledge marker publication") + with marker_lock(record): + marker = marker_matcher( + record, + expected_fingerprint=expected_fingerprint, + expected_pid=process.pid, + require_live_command=True, + ) + generation = self.gateway_generation(marker) + if marker.kind != "live" or generation is None: + raise RuntimeError( + "gateway launcher ownership marker could not be verified: " + marker.reason + ) + self._processes[record.bot_id] = process + except BaseException as exc: + for fd in (payload_read, payload_write, ack_read, ack_write): + if fd >= 0: + with contextlib.suppress(OSError): + hooks.close(fd) + if process is None: + if not isinstance(exc, (OSError, ValueError)): + raise + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "launch_failed", + reason=str(exc), + error_type=type(exc).__name__, + ) + cleanup_complete = self.cleanup_interrupted_launch( + record, + process, + expected_fingerprint=expected_fingerprint, + ) + if not isinstance(exc, Exception): + raise + return LaunchEffect( + "registration_failed_clean" if cleanup_complete else "registration_failed_unknown", + pid=None if cleanup_complete else process.pid, + reason=str(exc), + error_type=type(exc).__name__, + cleanup_complete=cleanup_complete, + ) + if process is None or generation is None: + raise RuntimeError("gateway process factory returned no process") + returncode = self.poll_startup(process) + if returncode is not None: + self.remove_gateway_generation_marker(record, generation) + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "startup_exited", + generation=generation, + reason="gateway exited during startup grace period", + returncode=returncode, + ) + if probe is not None: + if wait: + readiness = self.wait_for_readiness(process, probe) + if process.poll() is not None: + returncode = process.poll() + self.remove_gateway_generation_marker(record, generation) + self._processes.pop(record.bot_id, None) + return LaunchEffect( + "readiness_exited", + generation=generation, + reason="gateway process exited during readiness check", + returncode=returncode, + ) + if readiness.ready: + return LaunchEffect("ready", process.pid, generation) + return LaunchEffect( + "readiness_timeout", + process.pid, + generation, + reason="readiness probe timed out", + readiness_message=readiness.message, + ) + return LaunchEffect("readiness_pending", process.pid, generation) + return LaunchEffect("running", process.pid, generation) + + def cleanup_interrupted_launch( + self, + record: BotRecord, + process: PopenLike, + *, + expected_fingerprint: str, + ) -> bool: + cleanup_errors: list[str] = [] + if not self.terminate_spawned_process(process, cleanup_errors): + return False + self._processes.pop(record.bot_id, None) + operation_id = record.pending_operation_id + if operation_id is None: + return False + remove_marker_if_owned( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=operation_id, + desired_revision=record.desired_revision, + pid=process.pid, + command_fingerprint=expected_fingerprint, + ) + return self.read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + + def cleanup_registered_launch( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + process = self._processes.get(record.bot_id) + if process is None or process.pid != generation.pid: + return False + cleanup_errors: list[str] = [] + if not self.terminate_spawned_process(process, cleanup_errors): + return False + self._processes.pop(record.bot_id, None) + self.remove_gateway_generation_marker(record, generation) + return self.read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + + def read_strict_runtime_marker( + self, + bot_id: str, + registered_profile_path: str, + ) -> MarkerObservation: + profile_path = nofollow_absolute_path(Path(registered_profile_path)) + expected_profile = self.marker_profiles_root / bot_id + if not profile_path.is_absolute() or profile_path != expected_profile: + return MarkerObservation( + "untrusted", + reason="registered profile path does not match the trusted Hermes profile", + ) + profile = None + logs_fd = marker_fd = -1 + hooks = self._hooks() + try: + profile = _open_profile_chain(profile_path) + except _ConfirmedMissing: + return MarkerObservation("missing", reason="marker is missing") + except (OSError, ValueError) as exc: + return MarkerObservation( + "untrusted", reason=f"registered profile cannot be opened safely: {exc}" + ) + try: + try: + logs_fd = _open_logs(profile.fd, create=False) + except ValueError as exc: + if isinstance(exc.__cause__, FileNotFoundError): + try: + profile.confirm_missing("logs") + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", + reason=f"marker directory absence is untrusted: {confirm_error}", + ) + return MarkerObservation("missing", reason="marker is missing") + return MarkerObservation( + "untrusted", reason=f"marker directory cannot be opened safely: {exc}" + ) + try: + marker_fd, marker_stat = _open_regular_marker(logs_fd) + raw = hooks.read_bounded_file(marker_fd) + marker_stat = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except FileNotFoundError: + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", reason=f"marker absence is untrusted: {confirm_error}" + ) + return MarkerObservation("missing", reason="marker is missing") + except ValueError as exc: + if isinstance(exc.__cause__, FileNotFoundError): + try: + _confirm_marker_missing(profile, logs_fd) + except (OSError, ValueError) as confirm_error: + return MarkerObservation( + "untrusted", reason=f"marker absence is untrusted: {confirm_error}" + ) + return MarkerObservation("missing", reason="marker is missing") + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + except FileNotFoundError as exc: + return MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") + finally: + for fd in (marker_fd, logs_fd): + if fd >= 0: + with contextlib.suppress(OSError): + hooks.close(fd) + if profile is not None: + profile.close() + if type(value) is not dict: + return MarkerObservation("untrusted", reason="marker is not an object") + if marker_stat.st_nlink != 1: + return MarkerObservation("untrusted", reason="marker has unexpected hard links") + return MarkerObservation("present", payload=value) + + def matching_runtime_marker( + self, + record: BotRecord, + *, + expected_fingerprint: str, + expected_pid: int | None = None, + require_live_command: bool, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return observed + operation_id = record.pending_operation_id + if operation_id is None: + return MarkerObservation("untrusted", reason="pending operation is missing") + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=expected_pid, + expected_operation_id=operation_id, + expected_revision=record.desired_revision, + expected_fingerprint=expected_fingerprint, + require_live_command=require_live_command, + ) + + def classify_schema3_runtime_marker( + self, + record: BotRecord, + payload: dict[str, object], + *, + expected_pid: int | None = None, + expected_operation_id: str | None = None, + expected_revision: int | None = None, + expected_fingerprint: str | None = None, + require_live_command: bool, + ) -> MarkerObservation: + pid_value = payload.get("pid") + if type(pid_value) is not int or pid_value <= 0: + return MarkerObservation("untrusted", reason="marker PID is invalid") + pid = pid_value + if expected_pid is not None and pid != expected_pid: + return MarkerObservation("untrusted", reason="marker PID does not match") + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + fingerprint = payload.get("command_fingerprint") + if ( + type(operation_id) is not str + or type(revision) is not int + or type(fingerprint) is not str + ): + return MarkerObservation("untrusted", reason="marker correlation is invalid") + if expected_operation_id is not None and operation_id != expected_operation_id: + return MarkerObservation("untrusted", reason="marker operation does not match") + if expected_revision is not None and revision != expected_revision: + return MarkerObservation("untrusted", reason="marker revision does not match") + if expected_fingerprint is not None and fingerprint != expected_fingerprint: + return MarkerObservation("untrusted", reason="marker command does not match") + if not is_owned_runtime_marker( + payload, + bot_id=record.bot_id, + operation_id=operation_id, + desired_revision=revision, + pid=pid, + expected_fingerprint=fingerprint, + ): + return MarkerObservation("untrusted", reason="marker schema or command does not match") + expected_hermes = self.resolved_hermes_bin() + if expected_hermes is None or payload.get("resolved_hermes_bin") != expected_hermes: + return MarkerObservation("untrusted", reason="marker executable is not trusted") + if not require_live_command: + start_identity_error = self.process_start_identity_error(payload, pid) + if start_identity_error is not None: + return MarkerObservation("untrusted", reason=start_identity_error) + return MarkerObservation("live", payload=payload) + pid_state = self.pid_state(pid) + if pid_state is process_identity.PidState.unknown: + return MarkerObservation("untrusted", reason="marker PID liveness is unknown") + if pid_state is process_identity.PidState.dead: + if self.process_start_fingerprint_required() and not self.valid_marker_start( + payload.get("proc_start_fingerprint") + ): + return MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ) + return MarkerObservation("dead", payload=payload, reason="marker PID is dead") + start_identity_error = self.process_start_identity_error(payload, pid) + if start_identity_error is not None: + return MarkerObservation("untrusted", reason=start_identity_error) + live_argv = self.cmdline_reader(pid) + if not live_argv: + return MarkerObservation("untrusted", reason="live gateway command is unavailable") + command_check = process_identity.verify_gateway_command( + live_argv, + record.bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + if not command_check.verified: + return MarkerObservation("untrusted", reason="live gateway command does not match") + return MarkerObservation("live", payload=payload) + + def process_start_identity_error(self, payload: dict[str, object], pid: int) -> str | None: + return process_identity.process_start_identity_error( + payload.get("proc_start_fingerprint"), + self.proc_start_fingerprint_reader(pid), + fingerprint_required=self.process_start_fingerprint_required(), + ) + + @staticmethod + def valid_marker_start(value: object) -> bool: + return process_identity.valid_process_start_fingerprint(value) + + @staticmethod + def process_start_fingerprint_required() -> bool: + return process_identity.process_start_fingerprint_required(platform.system()) + + def classify_existing_runtime_marker( + self, + record: BotRecord, + *, + expected_pid: int | None = None, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return observed + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=expected_pid, + require_live_command=True, + ) + + @staticmethod + def gateway_generation(marker: MarkerObservation) -> GatewayGeneration | None: + payload = marker.payload + if marker.kind not in {"live", "dead"} or payload is None: + return None + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + pid = payload.get("pid") + fingerprint = payload.get("command_fingerprint") + process_start = payload.get("proc_start_fingerprint") + if ( + type(operation_id) is not str + or type(revision) is not int + or type(pid) is not int + or type(fingerprint) is not str + or (process_start is not None and type(process_start) is not str) + ): + return None + return GatewayGeneration( + operation_id=operation_id, + desired_revision=revision, + pid=pid, + command_fingerprint=fingerprint, + proc_start_fingerprint=process_start, + ) + + def classify_exact_gateway_generation( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + ) -> MarkerObservation: + read_marker = read_marker or self.read_strict_runtime_marker + observed = read_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return MarkerObservation("untrusted", reason="previous gateway marker changed") + if observed.payload.get("proc_start_fingerprint") != generation.proc_start_fingerprint: + return MarkerObservation( + "untrusted", reason="previous gateway process identity changed" + ) + return self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=generation.pid, + expected_operation_id=generation.operation_id, + expected_revision=generation.desired_revision, + expected_fingerprint=generation.command_fingerprint, + require_live_command=True, + ) + + def remove_exact_schema3_marker( + self, + record: BotRecord, + marker: MarkerObservation, + ) -> bool: + generation = self.gateway_generation(marker) + return bool( + marker.kind == "dead" + and generation is not None + and self.remove_gateway_generation_marker(record, generation) + ) + + def remove_gateway_generation_marker( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + return remove_marker_if_owned( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=generation.operation_id, + desired_revision=generation.desired_revision, + pid=generation.pid, + command_fingerprint=generation.command_fingerprint, + expected_proc_start_fingerprint=generation.proc_start_fingerprint, + lock_timeout_seconds=self.lock_timeout_seconds, + ) + + def remove_gateway_generation_marker_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: + return self._hooks().remove_marker_if_owned_locked( + self.safe_profile_path(record.bot_id, record.profile_path), + operation_id=generation.operation_id, + desired_revision=generation.desired_revision, + pid=generation.pid, + command_fingerprint=generation.command_fingerprint, + expected_proc_start_fingerprint=generation.proc_start_fingerprint, + ) + + def remove_owned_launch_marker_locked( + self, + record: BotRecord, + *, + observed: MarkerObservation | None = None, + ) -> bool: + if observed is None: + observed = self.read_strict_runtime_marker(record.bot_id, record.profile_path) + if observed.kind == "missing": + return True + if observed.kind != "present" or observed.payload is None or record.pid is None: + return False + if record.pending_action not in {"stop", "restart"}: + return False + marker = self.classify_schema3_runtime_marker( + record, + observed.payload, + expected_pid=record.pid, + expected_revision=record.desired_revision - 1, + require_live_command=True, + ) + generation = self.gateway_generation(marker) + return bool( + marker.kind == "dead" + and generation is not None + and self.remove_gateway_generation_marker_locked(record, generation) + ) + + def reauthorize_and_signal( + self, + record: BotRecord, + generation: GatewayGeneration, + sig: signal.Signals, + *, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + ) -> tuple[MarkerObservation, SignalResult | None]: + classify_exact = classify_exact or self.classify_exact_gateway_generation + current = classify_exact(record, generation) + if current.kind != "live": + return current, None + return current, self.send_signal(generation.pid, sig) + + def stop_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + read_marker: Callable[[str, str], MarkerObservation] | None = None, + classify_existing: Callable[..., MarkerObservation] | None = None, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + remove_owned: Callable[..., bool] | None = None, + remove_generation: Callable[[BotRecord, GatewayGeneration], bool] | None = None, + ) -> StopEffect: + read_marker = read_marker or self.read_strict_runtime_marker + classify_existing = classify_existing or self.classify_existing_runtime_marker + classify_exact = classify_exact or self.classify_exact_gateway_generation + remove_owned = remove_owned or self.remove_owned_launch_marker_locked + remove_generation = remove_generation or self.remove_gateway_generation_marker_locked + observed = read_marker(record.bot_id, record.profile_path) + if ( + observed.kind == "present" + and observed.payload is not None + and is_compat_runtime_marker(observed.payload) + ): + return StopEffect( + "compat_untrusted", + record.pid, + reason="schema-v2 or legacy gateway stop requires manual process resolution", + ) + if not record.pid: + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + marker = classify_existing(record, expected_pid=record.pid) + generation = self.gateway_generation(marker) + if marker.kind == "live" and generation is not None: + return self.stop_generation_locked( + record, + generation, + kill_after_timeout=kill_after_timeout, + classify_exact=classify_exact, + remove_generation=remove_generation, + ) + if marker.kind == "dead": + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + pid_state = self.pid_state(record.pid) + if pid_state is process_identity.PidState.unknown: + return StopEffect("pid_unknown", record.pid, reason="gateway PID liveness is unknown") + if pid_state is process_identity.PidState.dead: + if not remove_owned(record, observed=observed): + return StopEffect( + "cleanup_unverified", + record.pid, + reason="stale gateway marker ownership could not be verified", + ) + return StopEffect("not_running", record.pid, marker_removed=True) + if marker.kind != "live" or generation is None: + return StopEffect( + "ownership_unverified", + record.pid, + reason="refusing to stop process because PID ownership could not be verified", + ) + raise AssertionError("unreachable gateway marker state") + + def stop_generation_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + kill_after_timeout: bool | None, + classify_exact: Callable[[BotRecord, GatewayGeneration], MarkerObservation] | None = None, + remove_generation: Callable[[BotRecord, GatewayGeneration], bool] | None = None, + ) -> StopEffect: + classify_exact = classify_exact or self.classify_exact_gateway_generation + remove_generation = remove_generation or self.remove_gateway_generation_marker_locked + current = classify_exact(record, generation) + term_result: SignalResult | None = None + kill_result: SignalResult | None = None + if current.kind == "dead": + stopped = True + elif current.kind == "live": + current, term_result = self.reauthorize_and_signal( + record, + generation, + signal.SIGTERM, + classify_exact=classify_exact, + ) + if current.kind != "live" or term_result is None: + return StopEffect( + "term_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGTERM", + ) + if term_result is SignalResult.denied: + return StopEffect( + "term_denied", + generation.pid, + generation, + "could not send SIGTERM to the gateway", + term_result=term_result, + ) + stopped = term_result is SignalResult.missing + if not stopped: + stopped = self.wait_for_exit(record.bot_id, generation.pid) + else: + return StopEffect( + "term_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGTERM", + ) + should_kill = self.kill_after_timeout if kill_after_timeout is None else kill_after_timeout + kill_attempted = False + kill_succeeded: bool | None = None + if not stopped and should_kill: + kill_attempted = True + current, kill_result = self.reauthorize_and_signal( + record, + generation, + signal.SIGKILL, + classify_exact=classify_exact, + ) + if current.kind == "dead": + stopped = True + kill_succeeded = True + elif current.kind != "live" or kill_result is None: + return StopEffect( + "kill_reauthorization_failed", + generation.pid, + generation, + current.reason or "gateway ownership changed before SIGKILL", + term_result=term_result, + kill_attempted=True, + ) + elif kill_result is SignalResult.denied: + return StopEffect( + "kill_denied", + generation.pid, + generation, + "could not send SIGKILL to the gateway", + term_result=term_result, + kill_result=kill_result, + kill_attempted=True, + kill_succeeded=False, + ) + else: + stopped = kill_result is SignalResult.missing + if not stopped: + stopped = self.wait_for_exit(record.bot_id, generation.pid) + kill_succeeded = stopped + if not stopped: + return StopEffect( + "grace_expired", + generation.pid, + generation, + "gateway did not stop before grace period expired", + term_result=term_result, + kill_result=kill_result, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + if not remove_generation(record, generation): + return StopEffect( + "cleanup_unverified", + generation.pid, + generation, + "stopped gateway marker cleanup could not be verified", + term_result=term_result, + kill_result=kill_result, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + self._processes.pop(record.bot_id, None) + return StopEffect( + "stopped", + generation.pid, + generation, + term_result=term_result, + kill_result=kill_result, + marker_removed=True, + kill_attempted=kill_attempted, + kill_succeeded=kill_succeeded, + ) + + def readiness_probe_for_bot( + self, + bot_id: str, + *, + timeout_seconds: float | None = None, + ) -> ReadinessProbe | None: + _argv, env = self.adapter.command(bot_id, "gateway", "run") + return self.readiness_probe(env, timeout_seconds=timeout_seconds) + + def readiness_probe_for_live_record( + self, + record: BotRecord, + ) -> tuple[ReadinessProbe | None, str | None]: + try: + payload = json.loads( + self.pid_marker_path(record.profile_path).read_text(encoding="utf-8") + ) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return None, "readiness provenance is unavailable from the PID marker" + if not isinstance(payload, dict): + return None, "readiness provenance in the PID marker is invalid" + if payload.get("schema") not in {2, 3} or "readiness_probe" not in payload: + return self.readiness_probe_for_bot(record.bot_id), None + try: + return readiness_probe_from_payload(payload["readiness_probe"]), None + except ValueError as exc: + return None, f"readiness provenance in the PID marker is invalid: {exc}" + + def readiness_probe( + self, + env: dict[str, str], + *, + timeout_seconds: float | None = None, + ) -> ReadinessProbe | None: + resolved_timeout = ( + self.readiness_timeout_seconds if timeout_seconds is None else timeout_seconds + ) + if ( + isinstance(resolved_timeout, bool) + or not isinstance(resolved_timeout, (int, float)) + or not math.isfinite(float(resolved_timeout)) + or not 0.1 <= float(resolved_timeout) <= 300 + ): + raise TemplateError("readiness timeout must be a finite number between 0.1 and 300") + return readiness_probe_from_env( + env, + timeout_seconds=float(resolved_timeout), + interval_seconds=self.readiness_interval_seconds, + ) + + def wait_for_readiness( + self, + process: PopenLike, + readiness: ReadinessProbe, + ) -> ReadinessResult: + deadline = time.monotonic() + readiness.timeout_seconds + last = ReadinessResult(False, "not probed yet") + probe = self._hooks().probe_once + while time.monotonic() < deadline: + if process.poll() is not None: + return ReadinessResult(False, "gateway process exited during readiness check") + last = probe( + readiness.url, + timeout_seconds=min(5.0, max(0.2, readiness.interval_seconds)), + expected_status=readiness.expected_status, + expected_platform=readiness.expected_platform, + ) + if last.ready: + return last + time.sleep(readiness.interval_seconds) + return ReadinessResult(False, f"readiness timeout: {last.message}", last.payload) + + def pid_state(self, pid: int) -> process_identity.PidState: + if self.pid_alive_fn is not None: + return process_identity.pid_state(pid, pid_alive_fn=self.pid_alive_fn) + + def probe_with_current_kill(probe_pid: int) -> bool: + os.kill(probe_pid, 0) + return True + + return process_identity.pid_state(pid, pid_alive_fn=probe_with_current_kill) + + def send_signal(self, pid: int, sig: signal.Signals) -> SignalResult: + try: + self.kill_fn(pid, sig) + except ProcessLookupError: + return SignalResult.missing + except PermissionError: + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + return SignalResult.denied + raise + return SignalResult.sent + + def assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) -> None: + marker_path = self.pid_marker_path(str(profile_path)) + if not marker_path.exists(): + return + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BotRunningError( + "unregistered bot profile has an unreadable PID marker; refusing replacement" + ) from exc + pid = payload.get("pid") if isinstance(payload, dict) else None + if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: + raise BotRunningError( + "unregistered bot profile has an invalid PID marker; refusing replacement" + ) + if self.pid_state(pid) is not process_identity.PidState.dead: + raise BotRunningError( + f"unregistered bot profile may still own gateway PID {pid}; refusing replacement" + ) + + def write_pid_marker( + self, + profile_path: str, + pid: int, + bot_id: str, + argv: list[str], + *, + readiness_probe: ReadinessProbe | None, + include_readiness_probe: bool, + ) -> None: + path = self.pid_marker_path(profile_path) + path.parent.mkdir(parents=True, exist_ok=True) + resolved_hermes_bin = self.resolved_hermes_bin() + marker_argv = list(argv) + if resolved_hermes_bin: + marker_argv[0] = resolved_hermes_bin + fingerprint = self.proc_start_fingerprint_reader(pid) + payload: dict[str, object] = { + "schema": 2, + "pid": pid, + "bot_id": bot_id, + "component": "gateway", + "action": "run", + "argv": marker_argv, + "resolved_hermes_bin": resolved_hermes_bin, + "started_at": time.time(), + } + if include_readiness_probe: + payload["readiness_probe"] = readiness_probe_to_payload(readiness_probe) + if fingerprint: + payload["proc_start_fingerprint"] = fingerprint + atomic_write_json(path, payload, mode=0o600) + + def remove_pid_marker(self, profile_path: str) -> None: + try: + self.pid_marker_path(profile_path).unlink() + except FileNotFoundError: + return + + def read_pid_marker(self, profile_path: str) -> dict[str, object]: + safe_profile_path = nofollow_absolute_path(Path(profile_path)) + profile = None + logs_fd = marker_fd = -1 + hooks = self._hooks() + try: + try: + profile = _open_profile_chain(safe_profile_path) + logs_fd = _open_logs(profile.fd, create=False) + marker_fd, marker_stat = _open_regular_marker(logs_fd) + except _ConfirmedMissing: + return {"exists": False} + except (LaunchPayloadError, OSError, ValueError) as exc: + if _caused_by_missing_path(exc): + try: + if profile is not None and logs_fd >= 0: + _confirm_marker_missing(profile, logs_fd) + elif profile is not None: + profile.confirm_missing("logs") + else: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from exc + except (LaunchPayloadError, OSError, ValueError) as confirm_error: + raise UnsafeFileError( + "PID marker absence cannot be confirmed safely" + ) from confirm_error + return {"exists": False} + raise UnsafeFileError("PID marker cannot be opened safely") from exc + if marker_stat.st_uid != os.geteuid() or marker_stat.st_nlink != 1: + raise UnsafeFileError("PID marker is not a private regular file") + marker_mode = f"{stat.S_IMODE(marker_stat.st_mode):04o}" + try: + raw = hooks.read_bounded_file(marker_fd) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + try: + _validate_marker_bindings(profile, logs_fd, marker_fd, marker_stat) + except (LaunchPayloadError, OSError, TypeError, ValueError) as binding_error: + raise UnsafeFileError( + "PID marker changed while it was inspected" + ) from binding_error + return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} + try: + current_marker = _validate_marker_bindings( + profile, + logs_fd, + marker_fd, + marker_stat, + ) + except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: + raise UnsafeFileError("PID marker changed while it was inspected") from exc + if ( + not stat.S_ISREG(current_marker.st_mode) + or current_marker.st_uid != os.geteuid() + or current_marker.st_nlink != 1 + or not _same_identity(marker_stat, current_marker) + ): + raise UnsafeFileError("PID marker changed while it was inspected") + finally: + for descriptor in (marker_fd, logs_fd): + if descriptor >= 0: + with contextlib.suppress(OSError): + hooks.close(descriptor) + if profile is not None: + profile.close() + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} + if not isinstance(payload, dict): + return { + "exists": True, + "valid": False, + "mode": marker_mode, + "error": "pid marker must be a JSON object", + } + deprecated = payload.get("schema") is None + safe_payload: dict[str, object] = { + "exists": True, + "valid": True, + "mode": marker_mode, + "deprecated": deprecated, + } + for key in ( + "schema", + "pid", + "bot_id", + "component", + "action", + "started_at", + "proc_start_fingerprint", + ): + if key in payload: + safe_payload[key] = payload[key] + if "readiness_probe" in payload: + try: + readiness = readiness_probe_from_payload(payload["readiness_probe"]) + except ValueError: + safe_payload["readiness_probe"] = "invalid" + else: + safe_payload["readiness_probe"] = readiness_probe_to_payload(readiness) + argv_value = payload.get("argv") + if isinstance(argv_value, list) and all(isinstance(part, str) for part in argv_value): + safe_payload["argv_shape"] = process_identity.safe_command_shape(argv_value) + return safe_payload + + def verify_gateway_pid_ownership( + self, + profile_path: str, + pid: int, + bot_id: str, + *, + expected_record: BotRecord | None, + ) -> OwnershipCheck: + if expected_record is not None and expected_record.profile_path != profile_path: + return OwnershipCheck(False, "marker-mismatch") + observed = self.read_strict_runtime_marker(bot_id, profile_path) + if observed.kind == "missing": + return OwnershipCheck(False, "marker-missing") + if observed.kind != "present" or observed.payload is None: + return OwnershipCheck(False, "marker-mismatch") + payload = observed.payload + if payload.get("schema") == 3: + if expected_record is None: + return OwnershipCheck(False, "marker-mismatch") + marker = self.classify_schema3_runtime_marker( + expected_record, + payload, + expected_pid=pid, + require_live_command=True, + ) + if marker.kind != "live": + return OwnershipCheck(False, marker.reason or "marker-mismatch") + live_argv = self.cmdline_reader(pid) + if not live_argv: + return OwnershipCheck(False, "live-cmdline-missing") + live_check = process_identity.verify_gateway_command( + live_argv, + bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + return OwnershipCheck( + live_check.verified, + live_check.reason, + live_check.classification, + ) + if payload.get("pid") != pid: + return OwnershipCheck(False, "marker-mismatch") + argv_value = payload.get("argv") + if not isinstance(argv_value, list) or not all( + isinstance(part, str) for part in argv_value + ): + return OwnershipCheck(False, "marker-mismatch") + trusted_hermes = self.resolved_hermes_bin() + if trusted_hermes is None: + return OwnershipCheck(False, "untrusted-executable") + marker_check = self.verify_marker_payload(payload, list(argv_value), bot_id) + if not marker_check.verified: + return OwnershipCheck(False, marker_check.reason, marker_check.classification) + live_argv = self.cmdline_reader(pid) + if not live_argv: + return OwnershipCheck(False, "live-cmdline-missing") + live_check = process_identity.verify_gateway_command( + live_argv, + bot_id, + self.trusted_hermes_bins(), + require_trusted_path=True, + ) + if not live_check.verified: + return OwnershipCheck(False, live_check.reason, live_check.classification) + marker_schema = payload.get("schema") + fingerprint = payload.get("proc_start_fingerprint") + if marker_schema == 2: + live_fingerprint = self.proc_start_fingerprint_reader(pid) + if live_fingerprint and not (isinstance(fingerprint, str) and fingerprint): + return OwnershipCheck(False, "pid-start-time-missing", live_check.classification) + if isinstance(fingerprint, str) and fingerprint and live_fingerprint != fingerprint: + return OwnershipCheck(False, "pid-start-time-mismatch", live_check.classification) + elif isinstance(fingerprint, str) and fingerprint: + live_fingerprint = self.proc_start_fingerprint_reader(pid) + if live_fingerprint != fingerprint: + return OwnershipCheck(False, "pid-start-time-mismatch", live_check.classification) + classification = ( + "legacy-marker-valid" + if marker_check.classification == "legacy-marker-valid" + else live_check.classification + ) + return OwnershipCheck(True, "ok", classification) + + def verify_marker_payload( + self, + payload: dict[str, object], + argv: list[str], + bot_id: str, + ) -> OwnershipCheck: + schema = payload.get("schema") + if schema == 3: + pid = payload.get("pid") + operation_id = payload.get("operation_id") + revision = payload.get("desired_revision") + fingerprint = payload.get("command_fingerprint") + if ( + type(pid) is not int + or pid <= 0 + or type(operation_id) is not str + or len(operation_id) != 32 + or any(character not in "0123456789abcdef" for character in operation_id) + or type(revision) is not int + or revision <= 0 + or type(fingerprint) is not str + ): + return OwnershipCheck(False, "marker-mismatch") + if not is_owned_runtime_marker( + payload, + bot_id=bot_id, + operation_id=operation_id, + desired_revision=revision, + pid=pid, + expected_fingerprint=fingerprint, + ): + return OwnershipCheck(False, "marker-mismatch") + resolved_hermes_bin = self.resolved_hermes_bin() + marker_hermes = payload.get("resolved_hermes_bin") + if ( + resolved_hermes_bin is None + or type(marker_hermes) is not str + or process_identity.resolve_executable(marker_hermes) != resolved_hermes_bin + ): + return OwnershipCheck(False, "untrusted-executable") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + resolved_hermes_bin, + require_trusted_path=True, + ) + return OwnershipCheck( + marker_check.verified, + marker_check.reason, + marker_check.classification, + ) + if schema == 2: + if payload.get("bot_id") != bot_id: + return OwnershipCheck(False, "wrong-bot-id") + if payload.get("component") != "gateway" or payload.get("action") != "run": + return OwnershipCheck(False, "wrong-command-intent") + resolved_hermes_bin = self.resolved_hermes_bin() + if not isinstance(payload.get("resolved_hermes_bin"), str): + return OwnershipCheck(False, "untrusted-executable") + marker_hermes = process_identity.resolve_executable(str(payload["resolved_hermes_bin"])) + if marker_hermes != resolved_hermes_bin: + return OwnershipCheck(False, "untrusted-executable") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + resolved_hermes_bin, + require_trusted_path=True, + ) + return OwnershipCheck( + marker_check.verified, + marker_check.reason, + marker_check.classification, + ) + if schema is not None: + return OwnershipCheck(False, "marker-mismatch") + if not self.allow_legacy_pid_markers: + return OwnershipCheck(False, "legacy-marker-disabled") + marker_check = process_identity.verify_gateway_command( + argv, + bot_id, + None, + require_trusted_path=False, + ) + if not marker_check.verified: + return OwnershipCheck(False, marker_check.reason, marker_check.classification) + return OwnershipCheck(True, "ok", "legacy-marker-valid") + + def resolved_hermes_bin(self) -> str | None: + return process_identity.resolve_executable(self.adapter.hermes_bin) + + def trusted_hermes_bins(self) -> set[str]: + return process_identity.trusted_hermes_paths(self.adapter.hermes_bin) + + def terminate_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + ) -> bool: + if process.poll() is not None: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + if self.spawned_tree_stopped(process, timeout=0): + return True + term_result = self.signal_spawned_process(process, signal.SIGTERM, cleanup_errors) + if term_result is SignalResult.missing: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + return self.spawned_tree_stopped(process, timeout=0) + if term_result is SignalResult.denied: + return False + self.reap_spawned_process(process, cleanup_errors, timeout=self.stop_grace_seconds) + if self.spawned_tree_stopped(process, timeout=0): + return True + kill_result = self.signal_spawned_process(process, signal.SIGKILL, cleanup_errors) + if kill_result is SignalResult.missing: + self.reap_spawned_process(process, cleanup_errors, timeout=0) + return self.spawned_tree_stopped(process, timeout=0) + if kill_result is SignalResult.denied: + return False + self.reap_spawned_process(process, cleanup_errors, timeout=self.stop_grace_seconds) + return self.spawned_tree_stopped(process, timeout=self.stop_grace_seconds) + + def signal_spawned_process( + self, + process: PopenLike, + sig: signal.Signals, + cleanup_errors: list[str], + ) -> SignalResult: + if self.cleanup_process_group: + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + return SignalResult.missing + except PermissionError as exc: + cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") + return SignalResult.denied + raise + return SignalResult.sent + method_name = "terminate" if sig == signal.SIGTERM else "kill" + method = getattr(process, method_name, None) + if not callable(method): + return self.send_signal(process.pid, sig) + try: + method() + except ProcessLookupError: + return SignalResult.missing + except PermissionError as exc: + cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") + return SignalResult.denied + except OSError as exc: + if exc.errno == errno.ESRCH: + return SignalResult.missing + if exc.errno == errno.EPERM: + cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") + return SignalResult.denied + raise + return SignalResult.sent + + def reap_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + *, + timeout: float, + ) -> bool: + wait = getattr(process, "wait", None) + if callable(wait): + try: + wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + except Exception as exc: + cleanup_errors.append(f"wait: {type(exc).__name__}: {exc}") + return ( + process.poll() is not None + or self.pid_state(process.pid) is process_identity.PidState.dead + ) + + def spawned_tree_stopped(self, process: PopenLike, *, timeout: float) -> bool: + if not self.cleanup_process_group: + return ( + process.poll() is not None + or self.pid_state(process.pid) is process_identity.PidState.dead + ) + deadline = time.monotonic() + timeout + while True: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return True + except PermissionError: + return False + except OSError as exc: + return exc.errno == errno.ESRCH + if time.monotonic() >= deadline: + return False + time.sleep(0.05) + + def wait_for_exit(self, bot_id: str, pid: int) -> bool: + process = self._processes.get(bot_id) + if process is not None and hasattr(process, "wait"): + try: + process.wait(timeout=self.stop_grace_seconds) + return True + except subprocess.TimeoutExpired: + return False + except Exception: + return False + deadline = time.monotonic() + self.stop_grace_seconds + while ( + self.pid_state(pid) is not process_identity.PidState.dead + and time.monotonic() < deadline + ): + time.sleep(0.1) + return self.pid_state(pid) is process_identity.PidState.dead + + def poll_startup(self, process: PopenLike) -> int | None: + returncode = process.poll() + if returncode is not None or self.startup_grace_seconds <= 0: + return returncode + deadline = time.monotonic() + self.startup_grace_seconds + while time.monotonic() < deadline: + time.sleep(min(0.01, max(deadline - time.monotonic(), 0))) + returncode = process.poll() + if returncode is not None: + return returncode + return process.poll() diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 6a3623a..7f07556 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -1,25 +1,17 @@ from __future__ import annotations import contextlib -import errno -import json -import math import os import platform import re -import select import signal -import stat import subprocess # nosec B404 import threading -import time import uuid -from collections.abc import Callable, Sequence +from collections.abc import Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta -from enum import Enum from pathlib import Path -from typing import Protocol from zeus import process_identity as _process_identity from zeus.errors import ( @@ -29,21 +21,10 @@ BotReplaceError, BotRunningError, ) -from zeus.fs_utils import atomic_write_json from zeus.gateway_launcher import ( - MAX_PAYLOAD_BYTES, LaunchPayloadError, - _confirm_marker_missing, - _ConfirmedMissing, - _open_logs, - _open_profile_chain, - _open_regular_marker, _read_bounded_file, - _reject_duplicate_keys, _remove_marker_if_owned_locked, - _validate_marker_bindings, - marker_publication_lock, - remove_marker_if_owned, ) from zeus.gateway_marker import ( GatewayGeneration, @@ -51,8 +32,16 @@ readiness_probe_from_payload, readiness_probe_to_payload, ) -from zeus.gateway_marker import ( - is_owned_runtime_marker as _is_owned_runtime_marker, +from zeus.gateway_runtime import ( + GatewayRuntime, + KillFn, + MarkerObservation, + OwnershipCheck, + PopenFactory, + PopenLike, + RuntimeHooks, + SignalResult, + gateway_process_launch_kwargs, ) from zeus.hermes_adapter import HermesAdapter from zeus.lifecycle import LifecycleEvent, LifecycleEventInput @@ -68,10 +57,10 @@ TemplateError, validate_id, ) -from zeus.private_io import UnsafeFileError, nofollow_absolute_path, open_private_append +from zeus.private_io import nofollow_absolute_path from zeus.process_lock import BotProcessLock, LockTimeoutError from zeus.profile_manager import ProfileArchive, ProfileDeletion, ProfileManager -from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once, readiness_probe_from_env +from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once from zeus.reconciliation import ( BotReconcileResult, FleetReconciler, @@ -83,15 +72,6 @@ ) from zeus.state import StateStore - -class PopenLike(Protocol): - pid: int - - def poll(self) -> int | None: ... - - -PopenFactory = Callable[..., PopenLike] -KillFn = Callable[[int, signal.Signals], None] PidAliveFn = _process_identity.PidAliveFn CmdlineReader = _process_identity.CmdlineReader ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader @@ -108,17 +88,7 @@ def poll(self) -> int | None: ... _verify_gateway_command = _process_identity.verify_gateway_command -@dataclass(frozen=True) -class OwnershipCheck: - verified: bool - reason: str - classification: str | None = None - - -class _SignalResult(Enum): - sent = "sent" - missing = "missing" - denied = "denied" +_SignalResult = SignalResult class _ReadinessProbeUnset: @@ -137,11 +107,7 @@ class _LifecycleContext: request_id: str | None -@dataclass(frozen=True) -class _MarkerObservation: - kind: str - payload: dict[str, object] | None = None - reason: str = "" +_MarkerObservation = MarkerObservation _GatewayGeneration = GatewayGeneration @@ -156,10 +122,7 @@ class _ReconcileLaunch: def _gateway_process_launch_kwargs() -> dict[str, object]: - if os.name == "posix": - return {"start_new_session": True} - creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) - return {"creationflags": creationflags} if creationflags else {} + return gateway_process_launch_kwargs() def _nofollow_absolute_path(path: Path) -> Path: @@ -210,27 +173,188 @@ def __init__( self.store.database_path.parent / "archive", ) self._marker_profiles_root = configured_hermes_root / "profiles" - self.popen_factory = popen_factory - self.kill_fn = kill_fn - self.pid_alive_fn = pid_alive_fn - self.cmdline_reader = cmdline_reader or _read_process_cmdline self.startup_grace_seconds = startup_grace_seconds - self.stop_grace_seconds = stop_grace_seconds - self.kill_after_timeout = kill_after_timeout self.lock_dir = self.store.database_path.parent / "locks" / "bots" - self.lock_timeout_seconds = lock_timeout_seconds self.readiness_timeout_seconds = readiness_timeout_seconds self.readiness_interval_seconds = readiness_interval_seconds self.allow_legacy_pid_markers = allow_legacy_pid_markers self.restart_backoff_cap_seconds = restart_backoff_cap_seconds - self.proc_start_fingerprint_reader = ( - proc_start_fingerprint_reader or _read_process_start_fingerprint - ) self._cleanup_process_group = os.name == "posix" and popen_factory is subprocess.Popen - self._processes: dict[str, PopenLike] = {} + self._runtime = GatewayRuntime( + self.adapter, + self._profile_manager, + self._marker_profiles_root, + popen_factory=popen_factory, + kill_fn=kill_fn, + pid_alive_fn=pid_alive_fn, + cmdline_reader=cmdline_reader or _read_process_cmdline, + proc_start_fingerprint_reader=( + proc_start_fingerprint_reader or _read_process_start_fingerprint + ), + startup_grace_seconds=startup_grace_seconds, + stop_grace_seconds=stop_grace_seconds, + kill_after_timeout=kill_after_timeout, + lock_timeout_seconds=lock_timeout_seconds, + readiness_timeout_seconds=readiness_timeout_seconds, + readiness_interval_seconds=readiness_interval_seconds, + allow_legacy_pid_markers=allow_legacy_pid_markers, + cleanup_process_group=self._cleanup_process_group, + hooks_provider=self._runtime_hooks, + ) self._locks_guard = threading.Lock() self._bot_locks: dict[str, threading.RLock] = {} + def _runtime_hooks(self) -> RuntimeHooks: + return RuntimeHooks( + pipe=os.pipe, + close=os.close, + read_bounded_file=_read_bounded_file, + remove_marker_if_owned_locked=_remove_marker_if_owned_locked, + probe_once=probe_once, + ) + + def _get_runtime_proxy(self, name: str) -> object: + runtime = self.__dict__.get("_runtime") + if runtime is not None: + return getattr(runtime, name) + return self.__dict__.get(f"_runtime_proxy_{name}") + + def _set_runtime_proxy(self, name: str, value: object) -> None: + runtime = self.__dict__.get("_runtime") + history = self.__dict__.setdefault(f"_runtime_proxy_history_{name}", []) + if isinstance(history, list): + if len(history) >= 32: + del history[0] + history.append( + getattr(runtime, name) + if runtime is not None + else self.__dict__.get(f"_runtime_proxy_{name}") + ) + if runtime is not None: + setattr(runtime, name, value) + else: + self.__dict__[f"_runtime_proxy_{name}"] = value + + def _delete_runtime_proxy(self, name: str) -> None: + history = self.__dict__.get(f"_runtime_proxy_history_{name}") + if not isinstance(history, list) or not history: + self.__dict__.pop(f"_runtime_proxy_{name}", None) + return + previous = history.pop() + runtime = self.__dict__.get("_runtime") + if runtime is not None: + setattr(runtime, name, previous) + else: + self.__dict__[f"_runtime_proxy_{name}"] = previous + + @property + def popen_factory(self) -> PopenFactory: + return self._get_runtime_proxy("popen_factory") # type: ignore[return-value] + + @popen_factory.setter + def popen_factory(self, value: PopenFactory) -> None: + self._set_runtime_proxy("popen_factory", value) + + @popen_factory.deleter + def popen_factory(self) -> None: + self._delete_runtime_proxy("popen_factory") + + @property + def kill_fn(self) -> KillFn: + return self._get_runtime_proxy("kill_fn") # type: ignore[return-value] + + @kill_fn.setter + def kill_fn(self, value: KillFn) -> None: + self._set_runtime_proxy("kill_fn", value) + + @kill_fn.deleter + def kill_fn(self) -> None: + self._delete_runtime_proxy("kill_fn") + + @property + def pid_alive_fn(self) -> PidAliveFn | None: + return self._get_runtime_proxy("pid_alive_fn") # type: ignore[return-value] + + @pid_alive_fn.setter + def pid_alive_fn(self, value: PidAliveFn | None) -> None: + self._set_runtime_proxy("pid_alive_fn", value) + + @pid_alive_fn.deleter + def pid_alive_fn(self) -> None: + self._delete_runtime_proxy("pid_alive_fn") + + @property + def cmdline_reader(self) -> CmdlineReader: + return self._get_runtime_proxy("cmdline_reader") # type: ignore[return-value] + + @cmdline_reader.setter + def cmdline_reader(self, value: CmdlineReader) -> None: + self._set_runtime_proxy("cmdline_reader", value) + + @cmdline_reader.deleter + def cmdline_reader(self) -> None: + self._delete_runtime_proxy("cmdline_reader") + + @property + def proc_start_fingerprint_reader(self) -> ProcStartFingerprintReader: + return self._get_runtime_proxy("proc_start_fingerprint_reader") # type: ignore[return-value] + + @proc_start_fingerprint_reader.setter + def proc_start_fingerprint_reader(self, value: ProcStartFingerprintReader) -> None: + self._set_runtime_proxy("proc_start_fingerprint_reader", value) + + @proc_start_fingerprint_reader.deleter + def proc_start_fingerprint_reader(self) -> None: + self._delete_runtime_proxy("proc_start_fingerprint_reader") + + @property + def _processes(self) -> dict[str, PopenLike]: + return self._get_runtime_proxy("_processes") # type: ignore[return-value] + + @_processes.setter + def _processes(self, value: dict[str, PopenLike]) -> None: + self._set_runtime_proxy("_processes", value) + + @_processes.deleter + def _processes(self) -> None: + self._delete_runtime_proxy("_processes") + + @property + def stop_grace_seconds(self) -> float: + return self._get_runtime_proxy("stop_grace_seconds") # type: ignore[return-value] + + @stop_grace_seconds.setter + def stop_grace_seconds(self, value: float) -> None: + self._set_runtime_proxy("stop_grace_seconds", value) + + @stop_grace_seconds.deleter + def stop_grace_seconds(self) -> None: + self._delete_runtime_proxy("stop_grace_seconds") + + @property + def kill_after_timeout(self) -> bool: + return self._get_runtime_proxy("kill_after_timeout") # type: ignore[return-value] + + @kill_after_timeout.setter + def kill_after_timeout(self, value: bool) -> None: + self._set_runtime_proxy("kill_after_timeout", value) + + @kill_after_timeout.deleter + def kill_after_timeout(self) -> None: + self._delete_runtime_proxy("kill_after_timeout") + + @property + def lock_timeout_seconds(self) -> float: + return self._get_runtime_proxy("lock_timeout_seconds") # type: ignore[return-value] + + @lock_timeout_seconds.setter + def lock_timeout_seconds(self, value: float) -> None: + self._set_runtime_proxy("lock_timeout_seconds", value) + + @lock_timeout_seconds.deleter + def lock_timeout_seconds(self) -> None: + self._delete_runtime_proxy("lock_timeout_seconds") + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: if source not in _LIFECYCLE_SOURCES: raise ValueError("invalid lifecycle event source") @@ -367,15 +491,7 @@ def _marker_publication_lock( self, record: BotRecord, ) -> contextlib.AbstractContextManager[object]: - profile_path = self._safe_profile_path(record.bot_id, record.profile_path) - if not os.path.lexists(profile_path): - # A schema-v3 launcher cannot publish beneath an absent profile, so - # marker absence is already stable without creating profile state. - return contextlib.nullcontext() - return marker_publication_lock( - profile_path, - timeout_seconds=self.lock_timeout_seconds, - ) + return self._runtime.marker_publication_lock(record) def create_bot( self, @@ -743,139 +859,71 @@ def _start_record( record.profile_path, f"restart aborted: launch preflight failed: {exc}", ) - payload = self.adapter.launcher_payload( - bot_id, - operation_id=operation_id, - desired_revision=record.desired_revision, - readiness_probe=probe, + effect = self._runtime.launch( + record, + probe=probe, + wait=wait, + marker_lock=self._marker_publication_lock, + marker_matcher=self._matching_runtime_marker, + ack_reader=self._read_launcher_ack, + pipe_writer=self._write_pipe_payload, ) - marker_data = payload["marker"] - if type(marker_data) is not dict: - raise RuntimeError("launcher produced an invalid marker payload") - expected_fingerprint = str(marker_data["command_fingerprint"]) - log_path = self.log_path(record.profile_path) - process: PopenLike | None = None - payload_read = payload_write = ack_read = ack_write = -1 - try: - encoded_payload = json.dumps( - payload, - ensure_ascii=False, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - if not encoded_payload or len(encoded_payload) > MAX_PAYLOAD_BYTES: - raise ValueError("launcher payload is too large") - with open_private_append(log_path) as log_file: - payload_read, payload_write = os.pipe() - ack_read, ack_write = os.pipe() - launcher_argv = self.adapter.launcher_command(payload_read, ack_write) - process = self.popen_factory( - launcher_argv, - env=dict(os.environ), - stdout=log_file, - stderr=log_file, - pass_fds=(payload_read, ack_write), - close_fds=True, - **_gateway_process_launch_kwargs(), - ) - os.close(payload_read) - payload_read = -1 - os.close(ack_write) - ack_write = -1 - self._write_pipe_payload(payload_write, encoded_payload) - os.close(payload_write) - payload_write = -1 - acknowledgment = self._read_launcher_ack(ack_read) - os.close(ack_read) - ack_read = -1 - if acknowledgment != b"1": - raise RuntimeError("gateway launcher did not acknowledge marker publication") - with self._marker_publication_lock(record): - marker = self._matching_runtime_marker( + if effect.outcome == "launch_failed": + failure_message = f"failed to start gateway: {effect.reason}" + try: + self._complete_failed_intent( record, - expected_fingerprint=expected_fingerprint, - expected_pid=process.pid, - require_live_command=True, - ) - if marker.kind != "live": - raise RuntimeError( - "gateway launcher ownership marker could not be verified: " + marker.reason - ) - self._processes[bot_id] = process - except BaseException as exc: - for fd in (payload_read, payload_write, ack_read, ack_write): - if fd >= 0: - with contextlib.suppress(OSError): - os.close(fd) - if process is None: - if not isinstance(exc, (OSError, ValueError)): - raise - self._processes.pop(bot_id, None) - failure_message = f"failed to start gateway: {exc}" - try: - self._complete_failed_intent( - record, - context=context, - pid=None, - message=failure_message, - reason="gateway process launch failed", - ) - except Exception: - return self._pending_action_required( - record, "launch failure could not be persisted" - ) - self.store.append_audit_event( - "bot.start_failed", - bot_id=bot_id, - error=type(exc).__name__, - message=str(exc), - ) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, + context=context, pid=None, - profile_path=record.profile_path, message=failure_message, + reason="gateway process launch failed", ) - cleanup_complete = self._cleanup_interrupted_intent_launch( - record, - process, - expected_fingerprint=expected_fingerprint, - ) - if not isinstance(exc, Exception): - raise - if cleanup_complete: - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, - pid=None, - profile_path=record.profile_path, - message="gateway start registration failed; spawned process was stopped", + except Exception: + return self._pending_action_required( + record, "launch failure could not be persisted" ) + self.store.append_audit_event( + "bot.start_failed", + bot_id=bot_id, + error=effect.error_type, + message=effect.reason, + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message=failure_message, + ) + if effect.outcome == "registration_failed_clean": + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message="gateway start registration failed; spawned process was stopped", + ) + if effect.outcome == "registration_failed_unknown": return BotStatusResponse( bot_id=bot_id, status=BotStatus.unknown, - pid=process.pid, + pid=effect.pid, profile_path=record.profile_path, message=( "gateway start registration failed and spawned process cleanup " "could not be confirmed" ), ) - if process is None: # Defensive guard for custom process factories. - raise RuntimeError("gateway process factory returned no process") - returncode = self._poll_startup(process) - if returncode is not None: - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - self._processes.pop(bot_id, None) - message = f"gateway exited during startup grace period with return code {returncode}" - terminal = replace(record, pid=process.pid) + generation = effect.generation + if generation is None: + raise RuntimeError("gateway runtime returned no launch generation") + pid = effect.pid if effect.pid is not None else generation.pid + if effect.outcome == "startup_exited": + returncode = effect.returncode + failure_message = ( + f"gateway exited during startup grace period with return code {returncode}" + ) + terminal = replace(record, pid=pid) try: self._complete_failed_intent( terminal, @@ -883,17 +931,15 @@ def _start_record( pid=None, stopped_at=datetime.now(UTC), last_exit_code=returncode, - message=message, + message=failure_message, reason="gateway exited during startup grace period", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) + return self._launch_completion_failure_response(record, generation) self.store.append_audit_event( "bot.start_failed", bot_id=bot_id, - pid=process.pid, + pid=pid, returncode=returncode, ) return BotStatusResponse( @@ -901,143 +947,125 @@ def _start_record( status=BotStatus.failed, pid=None, profile_path=record.profile_path, - message=message, - ) - if probe is not None: - if wait: - readiness = self._wait_for_readiness(process, probe) - if process.poll() is not None: - returncode = process.poll() - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - self._processes.pop(bot_id, None) - try: - self._complete_failed_intent( - record, - context=context, - pid=None, - stopped_at=datetime.now(UTC), - last_exit_code=returncode, - message="gateway process exited during readiness check", - reason="readiness process exited", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event( - "bot.start_failed", - bot_id=bot_id, - pid=process.pid, - returncode=returncode, - reason="readiness_process_exited", - ) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.failed, - pid=None, - profile_path=record.profile_path, - message="gateway process exited during readiness check", - ) - if readiness.ready: - try: - self._complete_started_intent( - record, - context=context, - status=BotStatus.running, - pid=process.pid, - ready_at=datetime.now(UTC), - reset_restart=reset_restart, - reason="gateway readiness probe passed", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=process.pid) - return BotStatusResponse( - bot_id=bot_id, - status=BotStatus.running, - pid=process.pid, - profile_path=record.profile_path, - message="gateway ready", - ) - try: - self._complete_started_intent( - record, - context=context, - status=BotStatus.starting, - pid=process.pid, - last_error=readiness.message, - reason="readiness probe timed out", - ) - except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event( - "bot.start_readiness_pending", - bot_id=bot_id, - pid=process.pid, - url=probe.url, - message=readiness.message, + message=failure_message, + ) + if effect.outcome == "readiness_exited": + try: + self._complete_failed_intent( + record, + context=context, + pid=None, + stopped_at=datetime.now(UTC), + last_exit_code=effect.returncode, + message="gateway process exited during readiness check", + reason="readiness process exited", ) - return BotStatusResponse( - bot_id=bot_id, + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event( + "bot.start_failed", + bot_id=bot_id, + pid=pid, + returncode=effect.returncode, + reason="readiness_process_exited", + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.failed, + pid=None, + profile_path=record.profile_path, + message="gateway process exited during readiness check", + ) + if effect.outcome == "ready": + try: + self._complete_started_intent( + record, + context=context, + status=BotStatus.running, + pid=pid, + ready_at=datetime.now(UTC), + reset_restart=reset_restart, + reason="gateway readiness probe passed", + ) + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.running, + pid=pid, + profile_path=record.profile_path, + message="gateway ready", + ) + if effect.outcome == "readiness_timeout": + try: + self._complete_started_intent( + record, + context=context, status=BotStatus.starting, - pid=process.pid, - profile_path=record.profile_path, - message="readiness timeout; gateway process still alive", + pid=pid, + last_error=effect.readiness_message, + reason="readiness probe timed out", ) + except Exception: + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event( + "bot.start_readiness_pending", + bot_id=bot_id, + pid=pid, + url=probe.url if probe is not None else None, + message=effect.readiness_message, + ) + return BotStatusResponse( + bot_id=bot_id, + status=BotStatus.starting, + pid=pid, + profile_path=record.profile_path, + message="readiness timeout; gateway process still alive", + ) + if effect.outcome == "readiness_pending": try: self._complete_started_intent( record, context=context, status=BotStatus.starting, - pid=process.pid, + pid=pid, reason="gateway process started; readiness probe pending", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) + return self._launch_completion_failure_response(record, generation) self.store.append_audit_event( "bot.start_readiness_pending", bot_id=bot_id, - pid=process.pid, - url=probe.url, + pid=pid, + url=probe.url if probe is not None else None, ) return BotStatusResponse( bot_id=bot_id, status=BotStatus.starting, - pid=process.pid, + pid=pid, profile_path=record.profile_path, message="started; readiness probe pending", ) + if effect.outcome != "running": + raise RuntimeError(f"unknown gateway launch outcome: {effect.outcome}") try: self._complete_started_intent( record, context=context, status=BotStatus.running, - pid=process.pid, + pid=pid, ready_at=datetime.now(UTC), reset_restart=reset_restart, reason="gateway process started without readiness probe", ) except Exception: - return self._launch_completion_failure_response( - record, process, expected_fingerprint=expected_fingerprint - ) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=process.pid) + return self._launch_completion_failure_response(record, generation) + self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) return BotStatusResponse( bot_id=bot_id, status=BotStatus.running, - pid=process.pid, + pid=pid, profile_path=record.profile_path, message=message, ) @@ -1045,45 +1073,13 @@ def _start_record( def _preflight_start( self, record: BotRecord, *, timeout_seconds: float | None ) -> ReadinessProbe | None: - expected_profile = (Path(self.adapter.hermes_root) / "profiles" / record.bot_id).resolve() - if Path(record.profile_path).resolve() != expected_profile: - raise TemplateError("registered bot profile does not match the Hermes profile path") - safe_profile = self._safe_profile_path(record.bot_id, record.profile_path) - if not safe_profile.is_dir() or safe_profile.is_symlink(): - raise TemplateError("registered bot profile is not a safe directory") - _argv, env = self.adapter.command(record.bot_id, "gateway", "run") - probe = self._readiness_probe(env, timeout_seconds=timeout_seconds) - self.adapter.launcher_payload( - record.bot_id, - operation_id="0" * 32, - desired_revision=max(1, record.desired_revision + 1), - readiness_probe=probe, - ) - return probe + return self._runtime.preflight_start(record, timeout_seconds=timeout_seconds) def _write_pipe_payload(self, fd: int, payload: bytes) -> None: - offset = 0 - while offset < len(payload): - written = os.write(fd, payload[offset:]) - if written <= 0: - raise OSError("short launcher payload write") - offset += written + self._runtime.write_pipe_payload(fd, payload) def _read_launcher_ack(self, fd: int) -> bytes: - deadline = time.monotonic() + 5.0 - acknowledgment = bytearray() - while len(acknowledgment) <= 1: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("gateway launcher acknowledgment timed out") - readable, _writable, _exceptional = select.select([fd], [], [], remaining) - if not readable: - raise TimeoutError("gateway launcher acknowledgment timed out") - chunk = os.read(fd, 2 - len(acknowledgment)) - if not chunk: - return bytes(acknowledgment) - acknowledgment.extend(chunk) - return bytes(acknowledgment) + return self._runtime.read_launcher_ack(fd) def _complete_started_intent( self, @@ -1162,37 +1158,18 @@ def _cleanup_interrupted_intent_launch( *, expected_fingerprint: str, ) -> bool: - cleanup_errors: list[str] = [] - stopped = self._terminate_spawned_process(process, cleanup_errors) - if not stopped: - return False - self._processes.pop(record.bot_id, None) - operation_id = record.pending_operation_id - if operation_id is None: - return False - remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=operation_id, - desired_revision=record.desired_revision, - pid=process.pid, - command_fingerprint=expected_fingerprint, - ) - return ( - self._read_strict_runtime_marker(record.bot_id, record.profile_path).kind == "missing" + return self._runtime.cleanup_interrupted_launch( + record, + process, + expected_fingerprint=expected_fingerprint, ) def _launch_completion_failure_response( self, record: BotRecord, - process: PopenLike, - *, - expected_fingerprint: str, + generation: _GatewayGeneration, ) -> BotStatusResponse: - cleaned = self._cleanup_interrupted_intent_launch( - record, - process, - expected_fingerprint=expected_fingerprint, - ) + cleaned = self._runtime.cleanup_registered_launch(record, generation) if cleaned: return BotStatusResponse( record.bot_id, @@ -1204,7 +1181,7 @@ def _launch_completion_failure_response( return BotStatusResponse( record.bot_id, BotStatus.unknown, - process.pid, + generation.pid, record.profile_path, "gateway start completion is unknown and cleanup could not be confirmed", ) @@ -1212,85 +1189,7 @@ def _launch_completion_failure_response( def _read_strict_runtime_marker( self, bot_id: str, registered_profile_path: str ) -> _MarkerObservation: - profile_path = _nofollow_absolute_path(Path(registered_profile_path)) - expected_profile = self._marker_profiles_root / bot_id - if not profile_path.is_absolute() or profile_path != expected_profile: - return _MarkerObservation( - "untrusted", - reason="registered profile path does not match the trusted Hermes profile", - ) - profile = None - logs_fd = marker_fd = -1 - try: - profile = _open_profile_chain(profile_path) - except _ConfirmedMissing: - return _MarkerObservation("missing", reason="marker is missing") - except (OSError, ValueError) as exc: - return _MarkerObservation( - "untrusted", reason=f"registered profile cannot be opened safely: {exc}" - ) - try: - try: - logs_fd = _open_logs(profile.fd, create=False) - except ValueError as exc: - if isinstance(exc.__cause__, FileNotFoundError): - try: - profile.confirm_missing("logs") - except (OSError, ValueError) as confirm_error: - return _MarkerObservation( - "untrusted", - reason=f"marker directory absence is untrusted: {confirm_error}", - ) - return _MarkerObservation("missing", reason="marker is missing") - return _MarkerObservation( - "untrusted", reason=f"marker directory cannot be opened safely: {exc}" - ) - try: - marker_fd, marker_stat = _open_regular_marker(logs_fd) - raw = _read_bounded_file(marker_fd) - marker_stat = _validate_marker_bindings( - profile, - logs_fd, - marker_fd, - marker_stat, - ) - value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) - except FileNotFoundError: - try: - _confirm_marker_missing(profile, logs_fd) - except (OSError, ValueError) as confirm_error: - return _MarkerObservation( - "untrusted", - reason=f"marker absence is untrusted: {confirm_error}", - ) - return _MarkerObservation("missing", reason="marker is missing") - except ValueError as exc: - if isinstance(exc.__cause__, FileNotFoundError): - try: - _confirm_marker_missing(profile, logs_fd) - except (OSError, ValueError) as confirm_error: - return _MarkerObservation( - "untrusted", - reason=f"marker absence is untrusted: {confirm_error}", - ) - return _MarkerObservation("missing", reason="marker is missing") - return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - except FileNotFoundError as exc: - return _MarkerObservation("untrusted", reason=f"marker is invalid: {exc}") - finally: - for fd in (marker_fd, logs_fd): - if fd >= 0: - with contextlib.suppress(OSError): - os.close(fd) - if profile is not None: - profile.close() - if type(value) is not dict: - return _MarkerObservation("untrusted", reason="marker is not an object") - if marker_stat.st_nlink != 1: - return _MarkerObservation("untrusted", reason="marker has unexpected hard links") - return _MarkerObservation("present", payload=value) + return self._runtime.read_strict_runtime_marker(bot_id, registered_profile_path) def _matching_runtime_marker( self, @@ -1300,20 +1199,12 @@ def _matching_runtime_marker( expected_pid: int | None = None, require_live_command: bool, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return observed - operation_id = record.pending_operation_id - if operation_id is None: - return _MarkerObservation("untrusted", reason="pending operation is missing") - return self._classify_schema3_runtime_marker( + return self._runtime.matching_runtime_marker( record, - observed.payload, - expected_pid=expected_pid, - expected_operation_id=operation_id, - expected_revision=record.desired_revision, expected_fingerprint=expected_fingerprint, + expected_pid=expected_pid, require_live_command=require_live_command, + read_marker=self._read_strict_runtime_marker, ) def _classify_schema3_runtime_marker( @@ -1327,72 +1218,19 @@ def _classify_schema3_runtime_marker( expected_fingerprint: str | None = None, require_live_command: bool, ) -> _MarkerObservation: - pid_value = payload.get("pid") - if type(pid_value) is not int or pid_value <= 0: - return _MarkerObservation("untrusted", reason="marker PID is invalid") - pid = pid_value - if expected_pid is not None and pid != expected_pid: - return _MarkerObservation("untrusted", reason="marker PID does not match") - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - fingerprint = payload.get("command_fingerprint") - if ( - type(operation_id) is not str - or type(revision) is not int - or type(fingerprint) is not str - ): - return _MarkerObservation("untrusted", reason="marker correlation is invalid") - if expected_operation_id is not None and operation_id != expected_operation_id: - return _MarkerObservation("untrusted", reason="marker operation does not match") - if expected_revision is not None and revision != expected_revision: - return _MarkerObservation("untrusted", reason="marker revision does not match") - if expected_fingerprint is not None and fingerprint != expected_fingerprint: - return _MarkerObservation("untrusted", reason="marker command does not match") - if not _is_owned_runtime_marker( + return self._runtime.classify_schema3_runtime_marker( + record, payload, - bot_id=record.bot_id, - operation_id=operation_id, - desired_revision=revision, - pid=pid, - expected_fingerprint=fingerprint, - ): - return _MarkerObservation("untrusted", reason="marker schema or command does not match") - expected_hermes = self._resolved_hermes_bin() - if expected_hermes is None or payload.get("resolved_hermes_bin") != expected_hermes: - return _MarkerObservation("untrusted", reason="marker executable is not trusted") - if not require_live_command: - start_identity_error = self._process_start_identity_error(payload, pid) - if start_identity_error is not None: - return _MarkerObservation("untrusted", reason=start_identity_error) - return _MarkerObservation("live", payload=payload) - pid_state = self._pid_state(pid) - if pid_state is _PidState.unknown: - return _MarkerObservation("untrusted", reason="marker PID liveness is unknown") - if pid_state is _PidState.dead: - if self._process_start_fingerprint_required() and not self._valid_marker_start( - payload.get("proc_start_fingerprint") - ): - return _MarkerObservation( - "untrusted", reason="process start fingerprint is unavailable" - ) - return _MarkerObservation("dead", payload=payload, reason="marker PID is dead") - start_identity_error = self._process_start_identity_error(payload, pid) - if start_identity_error is not None: - return _MarkerObservation("untrusted", reason=start_identity_error) - live_argv = self.cmdline_reader(pid) - if not live_argv: - return _MarkerObservation("untrusted", reason="live gateway command is unavailable") - command_check = _verify_gateway_command( - live_argv, - record.bot_id, - self._trusted_hermes_bins(), - require_trusted_path=True, + expected_pid=expected_pid, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_fingerprint=expected_fingerprint, + require_live_command=require_live_command, ) - if not command_check.verified: - return _MarkerObservation("untrusted", reason="live gateway command does not match") - return _MarkerObservation("live", payload=payload) def _process_start_identity_error(self, payload: dict[str, object], pid: int) -> str | None: + if "_runtime" in self.__dict__: + return self._runtime.process_start_identity_error(payload, pid) return _process_identity.process_start_identity_error( payload.get("proc_start_fingerprint"), self.proc_start_fingerprint_reader(pid), @@ -1413,14 +1251,10 @@ def _classify_existing_runtime_marker( *, expected_pid: int | None = None, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return observed - return self._classify_schema3_runtime_marker( + return self._runtime.classify_existing_runtime_marker( record, - observed.payload, expected_pid=expected_pid, - require_live_command=True, + read_marker=self._read_strict_runtime_marker, ) def _remove_exact_schema3_marker( @@ -1428,61 +1262,23 @@ def _remove_exact_schema3_marker( record: BotRecord, marker: _MarkerObservation, ) -> bool: - if marker.kind != "dead": - return False - generation = self._gateway_generation(marker) - if generation is None: - return False - return self._remove_gateway_generation_marker(record, generation) + return self._runtime.remove_exact_schema3_marker(record, marker) def _gateway_generation( self, marker: _MarkerObservation, ) -> _GatewayGeneration | None: - payload = marker.payload - if marker.kind not in {"live", "dead"} or payload is None: - return None - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - pid = payload.get("pid") - fingerprint = payload.get("command_fingerprint") - process_start = payload.get("proc_start_fingerprint") - if ( - type(operation_id) is not str - or type(revision) is not int - or type(pid) is not int - or type(fingerprint) is not str - or (process_start is not None and type(process_start) is not str) - ): - return None - return _GatewayGeneration( - operation_id=operation_id, - desired_revision=revision, - pid=pid, - command_fingerprint=fingerprint, - proc_start_fingerprint=process_start, - ) + return self._runtime.gateway_generation(marker) def _classify_exact_gateway_generation( self, record: BotRecord, generation: _GatewayGeneration, ) -> _MarkerObservation: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return _MarkerObservation("untrusted", reason="previous gateway marker changed") - if observed.payload.get("proc_start_fingerprint") != generation.proc_start_fingerprint: - return _MarkerObservation( - "untrusted", reason="previous gateway process identity changed" - ) - return self._classify_schema3_runtime_marker( + return self._runtime.classify_exact_gateway_generation( record, - observed.payload, - expected_pid=generation.pid, - expected_operation_id=generation.operation_id, - expected_revision=generation.desired_revision, - expected_fingerprint=generation.command_fingerprint, - require_live_command=True, + generation, + read_marker=self._read_strict_runtime_marker, ) def _remove_gateway_generation_marker( @@ -1490,29 +1286,14 @@ def _remove_gateway_generation_marker( record: BotRecord, generation: _GatewayGeneration, ) -> bool: - return remove_marker_if_owned( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=generation.operation_id, - desired_revision=generation.desired_revision, - pid=generation.pid, - command_fingerprint=generation.command_fingerprint, - expected_proc_start_fingerprint=generation.proc_start_fingerprint, - lock_timeout_seconds=self.lock_timeout_seconds, - ) + return self._runtime.remove_gateway_generation_marker(record, generation) def _remove_gateway_generation_marker_locked( self, record: BotRecord, generation: _GatewayGeneration, ) -> bool: - return _remove_marker_if_owned_locked( - self._safe_profile_path(record.bot_id, record.profile_path), - operation_id=generation.operation_id, - desired_revision=generation.desired_revision, - pid=generation.pid, - command_fingerprint=generation.command_fingerprint, - expected_proc_start_fingerprint=generation.proc_start_fingerprint, - ) + return self._runtime.remove_gateway_generation_marker_locked(record, generation) def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: return BotStatusResponse( @@ -1591,25 +1372,32 @@ def _stop_record_effect_locked( context: _LifecycleContext, complete_stop: bool, ) -> BotStatusResponse: - bot_id = record.bot_id - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if ( - observed.kind == "present" - and observed.payload is not None - and self._is_compat_runtime_marker(observed.payload) - ): - return self._pending_action_required( - record, - "schema-v2 or legacy gateway stop requires manual process resolution", - ) - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead - if record.pid and pid_state == _PidState.unknown: - return self._pending_action_required(record, "gateway PID liveness is unknown") - if not record.pid or pid_state == _PidState.dead: - if not self._remove_owned_launch_marker_locked(record, observed=observed): - return self._pending_action_required( - record, "stale gateway marker ownership could not be verified" + effect = self._runtime.stop_locked( + record, + kill_after_timeout=kill_after_timeout, + read_marker=self._read_strict_runtime_marker, + classify_existing=self._classify_existing_runtime_marker, + classify_exact=self._classify_exact_gateway_generation, + remove_owned=self._remove_owned_launch_marker_locked, + remove_generation=self._remove_gateway_generation_marker_locked, + ) + if effect.outcome not in {"not_running", "stopped"}: + if effect.kill_result is not None: + self.store.append_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=effect.pid, + succeeded=bool(effect.kill_succeeded), ) + if effect.outcome == "grace_expired": + reason = ( + "gateway did not stop before grace period expired; " + "Hermes async delegations may still be running" + ) + else: + reason = effect.reason + return self._pending_action_required(record, reason) + if effect.outcome == "not_running": if complete_stop: try: self._complete_stopped_intent( @@ -1621,80 +1409,26 @@ def _stop_record_effect_locked( return self._pending_action_required( record, "stopped state could not be persisted" ) - self.store.append_audit_event("bot.stop", bot_id=bot_id, pid=record.pid) + self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( - bot_id=bot_id, + bot_id=record.bot_id, status=BotStatus.stopped, pid=None, profile_path=record.profile_path, message="not running", ) - - marker = self._classify_existing_runtime_marker(record, expected_pid=record.pid) - generation = self._gateway_generation(marker) - if marker.kind != "live" or generation is None: - return self._pending_action_required( - record, - "refusing to stop process because PID ownership could not be verified", - ) - term_marker = self._classify_exact_gateway_generation(record, generation) - if term_marker.kind != "live": - return self._pending_action_required( - record, - term_marker.reason or "gateway ownership changed before SIGTERM", - ) - - term_result = self._send_signal(record.pid, signal.SIGTERM) - if term_result == _SignalResult.denied: - return self._pending_action_required(record, "could not send SIGTERM to the gateway") - stopped = term_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(bot_id, record.pid) - should_kill = self.kill_after_timeout if kill_after_timeout is None else kill_after_timeout - if not stopped and should_kill: - kill_marker = self._classify_exact_gateway_generation(record, generation) - if kill_marker.kind != "live": - return self._pending_action_required( - record, - kill_marker.reason or "gateway ownership changed before SIGKILL", - ) - kill_result = self._send_signal(record.pid, signal.SIGKILL) - if kill_result == _SignalResult.denied: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=bot_id, - pid=record.pid, - succeeded=False, - ) - return self._pending_action_required( - record, "could not send SIGKILL to the gateway" - ) - stopped = kill_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(bot_id, record.pid) + if effect.kill_result is not None: self.store.append_audit_event( "bot.stop_kill", - bot_id=bot_id, - pid=record.pid, - succeeded=stopped, - ) - if not stopped: - message = ( - "gateway did not stop before grace period expired; " - "Hermes async delegations may still be running" - ) - return self._pending_action_required(record, message) - - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "stopped gateway marker cleanup could not be verified" + bot_id=record.bot_id, + pid=effect.pid, + succeeded=bool(effect.kill_succeeded), ) - self._processes.pop(bot_id, None) if not complete_stop: try: self._update_lifecycle( context, - bot_id, + record.bot_id, BotStatus.stopped, pid=None, action="bot.restart.old_process_stopped", @@ -1715,9 +1449,9 @@ def _stop_record_effect_locked( ) except Exception: return self._pending_action_required(record, "stopped state could not be persisted") - self.store.append_audit_event("bot.stop", bot_id=bot_id, pid=record.pid) + self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( - bot_id=bot_id, + bot_id=record.bot_id, status=BotStatus.stopped, pid=None, profile_path=record.profile_path, @@ -1756,27 +1490,7 @@ def _remove_owned_launch_marker_locked( *, observed: _MarkerObservation | None = None, ) -> bool: - if observed is None: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind == "missing": - return True - if observed.kind != "present" or observed.payload is None or record.pid is None: - return False - if record.pending_action not in {"stop", "restart"}: - return False - marker = self._classify_schema3_runtime_marker( - record, - observed.payload, - expected_pid=record.pid, - expected_revision=record.desired_revision - 1, - require_live_command=True, - ) - generation = self._gateway_generation(marker) - return ( - marker.kind == "dead" - and generation is not None - and self._remove_gateway_generation_marker_locked(record, generation) - ) + return self._runtime.remove_owned_launch_marker_locked(record, observed=observed) def restart( self, @@ -2706,64 +2420,38 @@ def _stop_pending_restart_old_gateway( *, context: _LifecycleContext, ) -> BotStatusResponse: - current = self._classify_exact_gateway_generation(record, generation) - if current.kind == "dead": - stopped = True - elif current.kind == "live": - term_result = self._send_signal(generation.pid, signal.SIGTERM) - if term_result == _SignalResult.denied: - return self._pending_action_required( - record, "could not send SIGTERM to the previous gateway" - ) - stopped = term_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(record.bot_id, generation.pid) - else: - return self._pending_action_required( - record, current.reason or "previous gateway marker changed" - ) - - if not stopped and self.kill_after_timeout: - current = self._classify_exact_gateway_generation(record, generation) - if current.kind == "dead": - stopped = True - elif current.kind != "live": - return self._pending_action_required( - record, - current.reason or "previous gateway ownership changed before SIGKILL", - ) - else: - kill_result = self._send_signal(generation.pid, signal.SIGKILL) - if kill_result == _SignalResult.denied: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=record.bot_id, - pid=generation.pid, - succeeded=False, - ) - return self._pending_action_required( - record, "could not send SIGKILL to the previous gateway" - ) - stopped = kill_result == _SignalResult.missing - if not stopped: - stopped = self._wait_for_exit(record.bot_id, generation.pid) + effect = self._runtime.stop_generation_locked( + record, + generation, + kill_after_timeout=None, + classify_exact=self._classify_exact_gateway_generation, + remove_generation=self._remove_gateway_generation_marker_locked, + ) + if effect.outcome != "stopped": + if effect.kill_result is not None: self.store.append_audit_event( "bot.stop_kill", bot_id=record.bot_id, pid=generation.pid, - succeeded=stopped, + succeeded=bool(effect.kill_succeeded), ) - - if not stopped: + reasons = { + "term_denied": "could not send SIGTERM to the previous gateway", + "kill_denied": "could not send SIGKILL to the previous gateway", + "grace_expired": ("previous gateway did not stop before the grace period expired"), + "cleanup_unverified": ("previous gateway marker cleanup could not be verified"), + } return self._pending_action_required( record, - "previous gateway did not stop before the grace period expired", + reasons.get(effect.outcome, effect.reason), ) - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "previous gateway marker cleanup could not be verified" + if effect.kill_result is not None: + self.store.append_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=generation.pid, + succeeded=bool(effect.kill_succeeded), ) - self._processes.pop(record.bot_id, None) try: self._update_lifecycle( context, @@ -3134,25 +2822,12 @@ def _recover_previously_active_bot( if result.status not in {BotStatus.starting, BotStatus.running}: raise RuntimeError(f"previous bot restart failed after {operation}: {result.message}") - def _assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) -> None: - marker_path = self.pid_marker_path(str(profile_path)) - if not marker_path.exists(): - return - try: - payload = json.loads(marker_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise BotRunningError( - "unregistered bot profile has an unreadable PID marker; refusing replacement" - ) from exc - pid = payload.get("pid") if isinstance(payload, dict) else None - if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: - raise BotRunningError( - "unregistered bot profile has an invalid PID marker; refusing replacement" - ) - if self._pid_state(pid) != _PidState.dead: - raise BotRunningError( - f"unregistered bot profile may still own gateway PID {pid}; refusing replacement" - ) + def _assert_unregistered_profile_inactive( + self, + bot_id: str, + profile_path: Path, + ) -> None: + self._runtime.assert_unregistered_profile_inactive(bot_id, profile_path) def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: return self._profile_manager.validate_profile_path(bot_id, profile_path) @@ -3194,69 +2869,33 @@ def _restore_archived_profile( def _readiness_probe_for_bot( self, bot_id: str, *, timeout_seconds: float | None = None ) -> ReadinessProbe | None: - _argv, env = self.adapter.command(bot_id, "gateway", "run") - return self._readiness_probe(env, timeout_seconds=timeout_seconds) + return self._runtime.readiness_probe_for_bot( + bot_id, + timeout_seconds=timeout_seconds, + ) def _readiness_probe_for_live_record( self, record: BotRecord ) -> tuple[ReadinessProbe | None, str | None]: - try: - payload = json.loads( - self.pid_marker_path(record.profile_path).read_text(encoding="utf-8") - ) - except (FileNotFoundError, json.JSONDecodeError, OSError): - return None, "readiness provenance is unavailable from the PID marker" - if not isinstance(payload, dict): - return None, "readiness provenance in the PID marker is invalid" - if payload.get("schema") not in {2, 3} or "readiness_probe" not in payload: - # Markers written before readiness provenance was added remain supported. - return self._readiness_probe_for_bot(record.bot_id), None - try: - return _readiness_probe_from_marker(payload["readiness_probe"]), None - except ValueError as exc: - return None, f"readiness provenance in the PID marker is invalid: {exc}" + return self._runtime.readiness_probe_for_live_record(record) def _readiness_probe( self, env: dict[str, str], *, timeout_seconds: float | None = None ) -> ReadinessProbe | None: - resolved_timeout = ( - self.readiness_timeout_seconds if timeout_seconds is None else timeout_seconds - ) - if ( - isinstance(resolved_timeout, bool) - or not isinstance(resolved_timeout, (int, float)) - or not math.isfinite(float(resolved_timeout)) - or not 0.1 <= float(resolved_timeout) <= 300 - ): - raise TemplateError("readiness timeout must be a finite number between 0.1 and 300") - return readiness_probe_from_env( - env, - timeout_seconds=float(resolved_timeout), - interval_seconds=self.readiness_interval_seconds, - ) + return self._runtime.readiness_probe(env, timeout_seconds=timeout_seconds) - def _wait_for_readiness(self, process: PopenLike, probe: ReadinessProbe) -> ReadinessResult: - deadline = time.monotonic() + probe.timeout_seconds - last = ReadinessResult(False, "not probed yet") - while time.monotonic() < deadline: - if process.poll() is not None: - return ReadinessResult(False, "gateway process exited during readiness check") - last = probe_once( - probe.url, - timeout_seconds=min(5.0, max(0.2, probe.interval_seconds)), - expected_status=probe.expected_status, - expected_platform=probe.expected_platform, - ) - if last.ready: - return last - time.sleep(probe.interval_seconds) - return ReadinessResult(False, f"readiness timeout: {last.message}", last.payload) + def _wait_for_readiness( + self, + process: PopenLike, + probe: ReadinessProbe, + ) -> ReadinessResult: + return self._runtime.wait_for_readiness(process, probe) def log_path(self, profile_path: str) -> Path: - return _nofollow_absolute_path(Path(profile_path) / "logs" / "zeus-gateway.log") + return self._runtime.log_path(profile_path) def pid_marker_path(self, profile_path: str) -> Path: - return Path(profile_path) / "logs" / "zeus-gateway.pid.json" + return self._runtime.pid_marker_path(profile_path) def _require_bot(self, bot_id: str) -> BotRecord: record = self.store.get_bot(bot_id) @@ -3265,6 +2904,8 @@ def _require_bot(self, bot_id: str) -> BotRecord: return record def _pid_state(self, pid: int) -> _PidState: + if "_runtime" in self.__dict__: + return self._runtime.pid_state(pid) if self.pid_alive_fn is not None: return _process_identity.pid_state(pid, pid_alive_fn=self.pid_alive_fn) @@ -3299,19 +2940,7 @@ def _unknown_pid_response( ) def _send_signal(self, pid: int, sig: signal.Signals) -> _SignalResult: - try: - self.kill_fn(pid, sig) - except ProcessLookupError: - return _SignalResult.missing - except PermissionError: - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - return _SignalResult.denied - raise - return _SignalResult.sent + return self._runtime.send_signal(pid, sig) def _write_pid_marker( self, @@ -3322,149 +2951,24 @@ def _write_pid_marker( *, readiness_probe: ReadinessProbe | None | _ReadinessProbeUnset = _READINESS_PROBE_UNSET, ) -> None: - path = self.pid_marker_path(profile_path) - path.parent.mkdir(parents=True, exist_ok=True) - resolved_hermes_bin = self._resolved_hermes_bin() - marker_argv = list(argv) - if resolved_hermes_bin: - marker_argv[0] = resolved_hermes_bin - fingerprint = self.proc_start_fingerprint_reader(pid) - payload = { - "schema": 2, - "pid": pid, - "bot_id": bot_id, - "component": "gateway", - "action": "run", - "argv": marker_argv, - "resolved_hermes_bin": resolved_hermes_bin, - "started_at": time.time(), - } - if not isinstance(readiness_probe, _ReadinessProbeUnset): - payload["readiness_probe"] = _readiness_probe_marker_payload(readiness_probe) - if fingerprint: - payload["proc_start_fingerprint"] = fingerprint - atomic_write_json(path, payload, mode=0o600) + include_readiness_probe = not isinstance(readiness_probe, _ReadinessProbeUnset) + runtime_probe = ( + None if isinstance(readiness_probe, _ReadinessProbeUnset) else readiness_probe + ) + self._runtime.write_pid_marker( + profile_path, + pid, + bot_id, + argv, + readiness_probe=runtime_probe, + include_readiness_probe=include_readiness_probe, + ) def _remove_pid_marker(self, profile_path: str) -> None: - try: - self.pid_marker_path(profile_path).unlink() - except FileNotFoundError: - return + self._runtime.remove_pid_marker(profile_path) def _read_pid_marker(self, profile_path: str) -> dict[str, object]: - safe_profile_path = _nofollow_absolute_path(Path(profile_path)) - profile = None - logs_fd = marker_fd = -1 - try: - try: - profile = _open_profile_chain(safe_profile_path) - logs_fd = _open_logs(profile.fd, create=False) - marker_fd, marker_stat = _open_regular_marker(logs_fd) - except _ConfirmedMissing: - return {"exists": False} - except (LaunchPayloadError, OSError, ValueError) as exc: - if _caused_by_missing_path(exc): - try: - if profile is not None and logs_fd >= 0: - _confirm_marker_missing(profile, logs_fd) - elif profile is not None: - profile.confirm_missing("logs") - else: - raise UnsafeFileError( - "PID marker absence cannot be confirmed safely" - ) from exc - except (LaunchPayloadError, OSError, ValueError) as confirm_error: - raise UnsafeFileError( - "PID marker absence cannot be confirmed safely" - ) from confirm_error - return {"exists": False} - raise UnsafeFileError("PID marker cannot be opened safely") from exc - if marker_stat.st_uid != os.geteuid() or marker_stat.st_nlink != 1: - raise UnsafeFileError("PID marker is not a private regular file") - marker_mode = f"{stat.S_IMODE(marker_stat.st_mode):04o}" - try: - raw = _read_bounded_file(marker_fd) - except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: - try: - _validate_marker_bindings( - profile, - logs_fd, - marker_fd, - marker_stat, - ) - except (LaunchPayloadError, OSError, TypeError, ValueError) as binding_error: - raise UnsafeFileError( - "PID marker changed while it was inspected" - ) from binding_error - return { - "exists": True, - "valid": False, - "mode": marker_mode, - "error": str(exc), - } - try: - current_marker = _validate_marker_bindings( - profile, - logs_fd, - marker_fd, - marker_stat, - ) - except (LaunchPayloadError, OSError, TypeError, ValueError) as exc: - raise UnsafeFileError("PID marker changed while it was inspected") from exc - if ( - not stat.S_ISREG(current_marker.st_mode) - or current_marker.st_uid != os.geteuid() - or current_marker.st_nlink != 1 - or not _same_identity(marker_stat, current_marker) - ): - raise UnsafeFileError("PID marker changed while it was inspected") - finally: - for descriptor in (marker_fd, logs_fd): - if descriptor >= 0: - with contextlib.suppress(OSError): - os.close(descriptor) - if profile is not None: - profile.close() - try: - payload = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - return {"exists": True, "valid": False, "mode": marker_mode, "error": str(exc)} - if not isinstance(payload, dict): - return { - "exists": True, - "valid": False, - "mode": marker_mode, - "error": "pid marker must be a JSON object", - } - deprecated = payload.get("schema") is None - safe_payload: dict[str, object] = { - "exists": True, - "valid": True, - "mode": marker_mode, - "deprecated": deprecated, - } - for key in ( - "schema", - "pid", - "bot_id", - "component", - "action", - "started_at", - "proc_start_fingerprint", - ): - if key in payload: - safe_payload[key] = payload[key] - if "readiness_probe" in payload: - try: - probe = _readiness_probe_from_marker(payload["readiness_probe"]) - except ValueError: - safe_payload["readiness_probe"] = "invalid" - else: - safe_payload["readiness_probe"] = _readiness_probe_marker_payload(probe) - argv_value = payload.get("argv") - if isinstance(argv_value, list) and all(isinstance(part, str) for part in argv_value): - safe_payload["argv_shape"] = _safe_command_shape(argv_value) - return safe_payload + return self._runtime.read_pid_marker(profile_path) def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: return self._verify_gateway_pid_ownership(profile_path, pid, bot_id).verified @@ -3473,176 +2977,33 @@ def _verify_gateway_pid_ownership( self, profile_path: str, pid: int, bot_id: str ) -> OwnershipCheck: record = self.store.get_bot(bot_id) - if record is not None and record.profile_path != profile_path: - return OwnershipCheck(False, "marker-mismatch") - observed = self._read_strict_runtime_marker(bot_id, profile_path) - if observed.kind == "missing": - return OwnershipCheck(False, "marker-missing") - if observed.kind != "present" or observed.payload is None: - return OwnershipCheck(False, "marker-mismatch") - payload = observed.payload - if payload.get("schema") == 3: - if record is None: - return OwnershipCheck(False, "marker-mismatch") - marker = self._classify_schema3_runtime_marker( - record, - payload, - expected_pid=pid, - require_live_command=True, - ) - if marker.kind != "live": - return OwnershipCheck(False, marker.reason or "marker-mismatch") - live_argv = self.cmdline_reader(pid) - if not live_argv: - return OwnershipCheck(False, "live-cmdline-missing") - live_check = _verify_gateway_command( - live_argv, - bot_id, - self._trusted_hermes_bins(), - require_trusted_path=True, - ) - return OwnershipCheck( - live_check.verified, - live_check.reason, - live_check.classification, - ) - if payload.get("pid") != pid: - return OwnershipCheck(False, "marker-mismatch") - argv_value = payload.get("argv") - if not isinstance(argv_value, list) or not all( - isinstance(part, str) for part in argv_value - ): - return OwnershipCheck(False, "marker-mismatch") - trusted_hermes = self._resolved_hermes_bin() - if trusted_hermes is None: - return OwnershipCheck(False, "untrusted-executable") - marker_check = self._verify_marker_payload(payload, list(argv_value), bot_id) - if not marker_check.verified: - return OwnershipCheck(False, marker_check.reason, marker_check.classification) - live_argv = self.cmdline_reader(pid) - if not live_argv: - return OwnershipCheck(False, "live-cmdline-missing") - live_check = _verify_gateway_command( - live_argv, bot_id, self._trusted_hermes_bins(), require_trusted_path=True - ) - if not live_check.verified: - return OwnershipCheck(False, live_check.reason, live_check.classification) - marker_schema = payload.get("schema") - fingerprint = payload.get("proc_start_fingerprint") - if marker_schema == 2: - live_fingerprint = self.proc_start_fingerprint_reader(pid) - if live_fingerprint and not (isinstance(fingerprint, str) and fingerprint): - return OwnershipCheck( - False, - "pid-start-time-missing", - live_check.classification, - ) - if isinstance(fingerprint, str) and fingerprint and live_fingerprint != fingerprint: - return OwnershipCheck( - False, - "pid-start-time-mismatch", - live_check.classification, - ) - elif isinstance(fingerprint, str) and fingerprint: - live_fingerprint = self.proc_start_fingerprint_reader(pid) - if live_fingerprint != fingerprint: - return OwnershipCheck( - False, - "pid-start-time-mismatch", - live_check.classification, - ) - classification = ( - "legacy-marker-valid" - if marker_check.classification == "legacy-marker-valid" - else live_check.classification + ownership = self._runtime.verify_gateway_pid_ownership( + profile_path, + pid, + bot_id, + expected_record=record, ) - if classification == "legacy-marker-valid": + if ownership.classification == "legacy-marker-valid": self.store.append_audit_event( "bot.pid_marker_legacy_accepted", bot_id=bot_id, pid=pid, ) - return OwnershipCheck(True, "ok", classification) + return ownership def _verify_marker_payload( self, payload: dict[str, object], argv: list[str], bot_id: str ) -> OwnershipCheck: - schema = payload.get("schema") - if schema == 3: - pid = payload.get("pid") - operation_id = payload.get("operation_id") - revision = payload.get("desired_revision") - fingerprint = payload.get("command_fingerprint") - if ( - type(pid) is not int - or pid <= 0 - or type(operation_id) is not str - or _REQUEST_ID_RE.fullmatch(operation_id) is None - or type(revision) is not int - or revision <= 0 - or type(fingerprint) is not str - ): - return OwnershipCheck(False, "marker-mismatch") - if not _is_owned_runtime_marker( - payload, - bot_id=bot_id, - operation_id=operation_id, - desired_revision=revision, - pid=pid, - expected_fingerprint=fingerprint, - ): - return OwnershipCheck(False, "marker-mismatch") - resolved_hermes_bin = self._resolved_hermes_bin() - marker_hermes = payload.get("resolved_hermes_bin") - if ( - resolved_hermes_bin is None - or type(marker_hermes) is not str - or _resolve_executable(marker_hermes) != resolved_hermes_bin - ): - return OwnershipCheck(False, "untrusted-executable") - marker_check = _verify_gateway_command( - argv, - bot_id, - resolved_hermes_bin, - require_trusted_path=True, - ) - return OwnershipCheck( - marker_check.verified, - marker_check.reason, - marker_check.classification, - ) - if schema == 2: - if payload.get("bot_id") != bot_id: - return OwnershipCheck(False, "wrong-bot-id") - if payload.get("component") != "gateway" or payload.get("action") != "run": - return OwnershipCheck(False, "wrong-command-intent") - resolved_hermes_bin = self._resolved_hermes_bin() - if not isinstance(payload.get("resolved_hermes_bin"), str): - return OwnershipCheck(False, "untrusted-executable") - marker_hermes = _resolve_executable(str(payload["resolved_hermes_bin"])) - if marker_hermes != resolved_hermes_bin: - return OwnershipCheck(False, "untrusted-executable") - marker_check = _verify_gateway_command( - argv, bot_id, resolved_hermes_bin, require_trusted_path=True - ) - return OwnershipCheck( - marker_check.verified, - marker_check.reason, - marker_check.classification, - ) - if schema is not None: - return OwnershipCheck(False, "marker-mismatch") - if not self.allow_legacy_pid_markers: - return OwnershipCheck(False, "legacy-marker-disabled") - marker_check = _verify_gateway_command(argv, bot_id, None, require_trusted_path=False) - if not marker_check.verified: - return OwnershipCheck(False, marker_check.reason, marker_check.classification) - return OwnershipCheck(True, "ok", "legacy-marker-valid") + return self._runtime.verify_marker_payload(payload, argv, bot_id) def _resolved_hermes_bin(self) -> str | None: + if "_runtime" in self.__dict__: + return self._runtime.resolved_hermes_bin() return _resolve_executable(self.adapter.hermes_bin) def _trusted_hermes_bins(self) -> set[str]: + if "_runtime" in self.__dict__: + return self._runtime.trusted_hermes_bins() return _trusted_hermes_paths(self.adapter.hermes_bin) def _cleanup_failed_start_registration( @@ -3713,36 +3074,12 @@ def _cleanup_failed_start_registration( ) return stopped - def _terminate_spawned_process(self, process: PopenLike, cleanup_errors: list[str]) -> bool: - if process.poll() is not None: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - if self._spawned_tree_stopped(process, timeout=0): - return True - term_result = self._signal_spawned_process(process, signal.SIGTERM, cleanup_errors) - if term_result == _SignalResult.missing: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - return self._spawned_tree_stopped(process, timeout=0) - if term_result == _SignalResult.denied: - return False - self._reap_spawned_process( - process, - cleanup_errors, - timeout=self.stop_grace_seconds, - ) - if self._spawned_tree_stopped(process, timeout=0): - return True - kill_result = self._signal_spawned_process(process, signal.SIGKILL, cleanup_errors) - if kill_result == _SignalResult.missing: - self._reap_spawned_process(process, cleanup_errors, timeout=0) - return self._spawned_tree_stopped(process, timeout=0) - if kill_result == _SignalResult.denied: - return False - self._reap_spawned_process( - process, - cleanup_errors, - timeout=self.stop_grace_seconds, - ) - return self._spawned_tree_stopped(process, timeout=self.stop_grace_seconds) + def _terminate_spawned_process( + self, + process: PopenLike, + cleanup_errors: list[str], + ) -> bool: + return self._runtime.terminate_spawned_process(process, cleanup_errors) def _signal_spawned_process( self, @@ -3750,41 +3087,7 @@ def _signal_spawned_process( sig: signal.Signals, cleanup_errors: list[str], ) -> _SignalResult: - if self._cleanup_process_group: - try: - os.killpg(process.pid, sig) - except ProcessLookupError: - return _SignalResult.missing - except PermissionError as exc: - cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - cleanup_errors.append(f"killpg: {type(exc).__name__}: {exc}") - return _SignalResult.denied - raise - return _SignalResult.sent - method_name = "terminate" if sig == signal.SIGTERM else "kill" - method = getattr(process, method_name, None) - if not callable(method): - return self._send_signal(process.pid, sig) - try: - method() - except ProcessLookupError: - return _SignalResult.missing - except PermissionError as exc: - cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") - return _SignalResult.denied - except OSError as exc: - if exc.errno == errno.ESRCH: - return _SignalResult.missing - if exc.errno == errno.EPERM: - cleanup_errors.append(f"{method_name}: {type(exc).__name__}: {exc}") - return _SignalResult.denied - raise - return _SignalResult.sent + return self._runtime.signal_spawned_process(process, sig, cleanup_errors) def _reap_spawned_process( self, @@ -3793,62 +3096,20 @@ def _reap_spawned_process( *, timeout: float, ) -> bool: - wait = getattr(process, "wait", None) - if callable(wait): - try: - wait(timeout=timeout) - return True - except subprocess.TimeoutExpired: - return False - except Exception as exc: - cleanup_errors.append(f"wait: {type(exc).__name__}: {exc}") - return process.poll() is not None or self._pid_state(process.pid) == _PidState.dead + return self._runtime.reap_spawned_process( + process, + cleanup_errors, + timeout=timeout, + ) def _spawned_tree_stopped(self, process: PopenLike, *, timeout: float) -> bool: - if not self._cleanup_process_group: - return process.poll() is not None or self._pid_state(process.pid) == _PidState.dead - deadline = time.monotonic() + timeout - while True: - try: - os.killpg(process.pid, 0) - except ProcessLookupError: - return True - except PermissionError: - return False - except OSError as exc: - return exc.errno == errno.ESRCH - if time.monotonic() >= deadline: - return False - time.sleep(0.05) + return self._runtime.spawned_tree_stopped(process, timeout=timeout) def _wait_for_exit(self, bot_id: str, pid: int) -> bool: - process = self._processes.get(bot_id) - if process is not None and hasattr(process, "wait"): - try: - process.wait(timeout=self.stop_grace_seconds) - return True - except subprocess.TimeoutExpired: - return False - except Exception: - return False - - deadline = time.monotonic() + self.stop_grace_seconds - while self._pid_state(pid) != _PidState.dead and time.monotonic() < deadline: - time.sleep(0.1) - return self._pid_state(pid) == _PidState.dead + return self._runtime.wait_for_exit(bot_id, pid) def _poll_startup(self, process: PopenLike) -> int | None: - returncode = process.poll() - if returncode is not None or self.startup_grace_seconds <= 0: - return returncode - - deadline = time.monotonic() + self.startup_grace_seconds - while time.monotonic() < deadline: - time.sleep(min(0.01, max(deadline - time.monotonic(), 0))) - returncode = process.poll() - if returncode is not None: - return returncode - return process.poll() + return self._runtime.poll_startup(process) def _read_process_cmdline(pid: int) -> list[str] | None: From 963418fbe2ab186185d25b07b122d44b27c71c3f Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 02:47:05 +0200 Subject: [PATCH 40/49] refactor(supervisor): isolate pending intent recovery --- docs/ARCHITECTURE.md | 21 +- tests/test_intent_recovery.py | 497 +++++++++++++++++++++++++++ zeus/intent_recovery.py | 630 ++++++++++++++++++++++++++++++++++ zeus/supervisor.py | 433 ++++++----------------- 4 files changed, 1255 insertions(+), 326 deletions(-) create mode 100644 tests/test_intent_recovery.py create mode 100644 zeus/intent_recovery.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f3d70e1..ececd37 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -45,9 +45,14 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.api_server`: Bounded HTTP concurrency and graceful server lifecycle. - `zeus.api_logging`: Locked, fail-open, secret-safe API JSONL output. - `zeus.idempotency`: Key validation and canonical request hashing. +- `zeus.process_identity`: Standard-library process observation and trusted-command checks. +- `zeus.gateway_marker`: Effect-free schema-v3 marker parsing and exact-generation values. - `zeus.hermes_adapter`: Subprocess command construction for Hermes. - `zeus.gateway_launcher`: Descriptor-only marker-before-exec helper. -- `zeus.supervisor`: Gateway lifecycle, PID ownership markers, logs, and status. +- `zeus.profile_manager`: Atomic profile installation, delete, archive, and rollback transactions. +- `zeus.gateway_runtime`: Process, marker, readiness, signal, and cleanup effects. +- `zeus.intent_recovery`: Store-free pending-intent recovery decisions through a structural host. +- `zeus.supervisor`: Public lifecycle facade, locks, durable intent transitions, and audit ordering. - `zeus.api`: Local HTTP routes and compatibility facade. - `zeus.cli`: Operator CLI. - `zeus.doctor`: Readiness diagnostics. @@ -90,6 +95,20 @@ state, and neither mode replaces backup and restore procedures. 7. Stop commits its stopped intent before verifying ownership and signaling; SIGTERM/SIGKILL authorization is rechecked against the exact process identity. +`Supervisor` owns the per-bot locks, lifecycle correlation context, `StateStore` +calls, and authoritative transition/audit ordering. `ProfileManager` changes only +profile and archive paths. `GatewayRuntime` owns the mutable in-process gateway +registry and host effects but never reads or writes SQLite. `PendingIntentRecovery` +contains the pending start, stop, and restart decision flow and resolves facade +methods dynamically through its structural host; it does not acquire locks, mint +operation IDs, or import the concrete facade or persistence layer. + +Status never spawns or signals; it may adopt exact evidence and repair the durable +projection. Reconcile may recover one effect per pass: a pending restart first +stops or cleans its exact schema-v3 predecessor and persists that observation, +then a later pass adopts or launches the replacement with the same pending +operation ID. + Hermes child processes receive a minimal host environment plus profile `.env` values. Operators can allow specific host variables with `ZEUS_ENV_PASSTHROUGH`. diff --git a/tests/test_intent_recovery.py b/tests/test_intent_recovery.py new file mode 100644 index 0000000..d9e80f2 --- /dev/null +++ b/tests/test_intent_recovery.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import ast +import inspect +import unittest +from dataclasses import dataclass, replace +from pathlib import Path + +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import MarkerObservation, StopEffect +from zeus.intent_recovery import PendingIntentRecovery +from zeus.models import BotRecord, BotStatus, BotStatusResponse, DesiredState +from zeus.process_identity import PidState +from zeus.readiness import ReadinessProbe, ReadinessResult + + +@dataclass(frozen=True) +class _Context: + operation_id: str + source: str = "reconcile" + request_id: str | None = "request" + + +def _pending_record( + action: str, + *, + pid: int | None = None, + revision: int = 1, +) -> BotRecord: + return BotRecord( + "coder", + "coding-bot", + "Coder", + "/profiles/coder", + status=BotStatus.starting, + pid=pid, + desired_state=(DesiredState.stopped if action == "stop" else DesiredState.running), + desired_revision=revision, + pending_operation_id="1" * 32, + pending_action=action, + ) + + +class _StrictRecoveryHost: + def __init__(self, recovery: PendingIntentRecovery) -> None: + self.recovery = recovery + self.probe: ReadinessProbe | None = None + self.fingerprint = "f" * 64 + self.strict_marker = MarkerObservation("missing") + self.matching_marker = MarkerObservation("missing") + self.old_marker: MarkerObservation | None = None + self.pid_state = PidState.dead + self.pid_owned = False + self.start_calls = 0 + self.stop_effect_calls = 0 + self.signal_calls = 0 + self.marker_removals = 0 + self.state_updates = 0 + self.started_completions: list[tuple[str, int]] = [] + self.stopped_completions: list[str] = [] + self.audit_events: list[str] = [] + self.generation = GatewayGeneration( + operation_id="0" * 32, + desired_revision=1, + pid=41, + command_fingerprint="a" * 64, + proc_start_fingerprint="start", + ) + + def _recovery_lifecycle_context( + self, + operation_id: str, + context: _Context, + ) -> _Context: + return _Context(operation_id, context.source, context.request_id) + + @staticmethod + def _pending_action_required(record: BotRecord, reason: str) -> BotStatusResponse: + return BotStatusResponse( + record.bot_id, + record.status, + record.pid, + record.profile_path, + f"action required: {reason}", + ) + + def _pending_launch_preflight( + self, + _record: BotRecord, + _operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: + return self.probe, self.fingerprint + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse: + return self.recovery.recover_pending_stop_intent_locked( + self, + record, + context=context, + allow_stop=allow_stop, + ) + + def _recover_pending_restart_predecessor( + self, + record: BotRecord, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse | None: + return self.recovery.recover_pending_restart_predecessor_locked( + self, + record, + context=context, + allow_stop=allow_stop, + ) + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _Context, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: + return self.recovery.recover_pending_launch_locked( + self, + record, + context=context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, + ) + + def _matching_runtime_marker( + self, + _record: BotRecord, + *, + expected_fingerprint: str, + require_live_command: bool, + ) -> MarkerObservation: + if expected_fingerprint != self.fingerprint or not require_live_command: + raise AssertionError("recovery weakened exact marker classification") + return self.matching_marker + + @staticmethod + def _probe_once(_probe: ReadinessProbe) -> ReadinessResult: + return ReadinessResult(False, "not ready") + + def _complete_started_intent( + self, + _record: BotRecord, + *, + context: _Context, + status: BotStatus, + pid: int, + ready_at: object, + reset_restart: bool, + reason: str, + ) -> None: + del status, ready_at, reset_restart, reason + self.started_completions.append((context.operation_id, pid)) + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + context: _Context, + probe: ReadinessProbe | None, + ) -> BotStatusResponse: + del reset_restart, message, context, probe + self.start_calls += 1 + return BotStatusResponse( + record.bot_id, + BotStatus.running, + 99, + record.profile_path, + "started", + ) + + def _read_strict_runtime_marker( + self, + _bot_id: str, + _profile_path: str, + ) -> MarkerObservation: + return self.strict_marker + + def _remove_owned_launch_marker_locked( + self, + _record: BotRecord, + *, + observed: MarkerObservation, + ) -> bool: + del observed + self.marker_removals += 1 + return True + + def _complete_stopped_intent( + self, + _record: BotRecord, + *, + context: _Context, + reason: str, + ) -> None: + del reason + self.stopped_completions.append(context.operation_id) + + def _pid_state(self, _pid: int) -> PidState: + return self.pid_state + + def _pid_owned(self, _profile_path: str, _pid: int, _bot_id: str) -> bool: + return self.pid_owned + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _Context, + complete_stop: bool, + ) -> BotStatusResponse: + del kill_after_timeout, context, complete_stop + self.stop_effect_calls += 1 + return self._pending_action_required( + record, + "gateway marker ownership could not be verified", + ) + + @staticmethod + def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: + return payload.get("schema") in {None, 2} + + def _pending_restart_old_marker( + self, + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: + if self.old_marker is not None: + return self.old_marker + return self.recovery.pending_restart_old_marker(self, record, observed) + + def _classify_schema3_runtime_marker( + self, + _record: BotRecord, + _payload: dict[str, object], + *, + expected_pid: int | None, + expected_revision: int, + require_live_command: bool, + ) -> MarkerObservation: + del expected_pid, expected_revision, require_live_command + return MarkerObservation("untrusted", reason="not configured") + + def _recover_pending_restart_old_gateway( + self, + record: BotRecord, + marker: MarkerObservation, + *, + context: _Context, + allow_stop: bool, + ) -> BotStatusResponse: + return self.recovery.recover_pending_restart_old_gateway( + self, + record, + marker, + context=context, + allow_stop=allow_stop, + ) + + def _gateway_generation( + self, + _marker: MarkerObservation, + ) -> GatewayGeneration | None: + return self.generation + + def _remove_gateway_generation_marker_locked( + self, + _record: BotRecord, + _generation: GatewayGeneration, + ) -> bool: + self.marker_removals += 1 + return True + + def _update_lifecycle( + self, + _context: _Context, + _bot_id: str, + _status: BotStatus, + **_values: object, + ) -> None: + self.state_updates += 1 + + def _stop_pending_restart_old_gateway( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + context: _Context, + ) -> BotStatusResponse: + return self.recovery.stop_pending_restart_old_gateway( + self, + record, + generation, + context=context, + ) + + def _stop_gateway_generation_locked( + self, + _record: BotRecord, + _generation: GatewayGeneration, + ) -> StopEffect: + self.signal_calls += 1 + return StopEffect("stopped", pid=self.generation.pid, generation=self.generation) + + def _append_recovery_audit_event(self, action: str, **_values: object) -> None: + self.audit_events.append(action) + + +class PendingIntentRecoveryTests(unittest.TestCase): + def test_original_publication_is_adopted_once_without_duplicate_launch(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("start") + host.matching_marker = MarkerObservation("live", {"pid": 73}) + + response = recovery.recover( + host, + record, + context=_Context("unrelated"), + allow_launch=True, + ) + + self.assertEqual(response.status, BotStatus.running) + self.assertEqual(response.pid, 73) + self.assertEqual(host.started_completions, [(record.pending_operation_id, 73)]) + self.assertEqual(host.start_calls, 0) + + def test_exact_live_restart_marker_is_adopted_without_popen_factory(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("restart", revision=2) + host.strict_marker = MarkerObservation("missing") + host.matching_marker = MarkerObservation("live", {"pid": 81}) + + response = recovery.recover( + host, + record, + context=_Context("different"), + allow_launch=True, + ) + + self.assertEqual(response.pid, 81) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.started_completions, [(record.pending_operation_id, 81)]) + + def test_compat_restart_and_stop_have_no_signal_delete_start_or_update(self) -> None: + recovery = PendingIntentRecovery() + restart_host = _StrictRecoveryHost(recovery) + restart = _pending_record("restart", pid=41, revision=2) + restart_host.strict_marker = MarkerObservation("present", {"schema": 2, "pid": 41}) + + restart_response = recovery.recover( + restart_host, + restart, + context=_Context("different"), + allow_launch=True, + ) + + stop_host = _StrictRecoveryHost(recovery) + stop = _pending_record("stop", pid=41) + stop_host.pid_state = PidState.alive + stop_response = recovery.recover( + stop_host, + stop, + context=_Context("different"), + allow_launch=False, + ) + + self.assertTrue(restart_response.message.startswith("action required:")) + self.assertTrue(stop_response.message.startswith("action required:")) + for host in (restart_host, stop_host): + self.assertEqual(host.signal_calls, 0) + self.assertEqual(host.marker_removals, 0) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.state_updates, 0) + + def test_restart_stop_and_launch_use_separate_passes_with_one_launch(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + first = _pending_record("restart", pid=41, revision=2) + host.strict_marker = MarkerObservation("present", {"schema": 3}) + host.old_marker = MarkerObservation("live", {"pid": 41}) + + first_response = recovery.recover( + host, + first, + context=_Context("different"), + allow_launch=True, + ) + + self.assertEqual(first_response.status, BotStatus.starting) + self.assertIn("launch on next reconcile", first_response.message) + self.assertEqual(host.signal_calls, 1) + self.assertEqual(host.start_calls, 0) + + second = replace(first, status=BotStatus.stopped, pid=None) + host.strict_marker = MarkerObservation("missing") + host.old_marker = None + host.matching_marker = MarkerObservation("missing") + second_response = recovery.recover( + host, + second, + context=_Context("new-context"), + allow_launch=True, + ) + + self.assertEqual(second_response.status, BotStatus.running) + self.assertEqual(host.start_calls, 1) + self.assertEqual(host.started_completions, []) + + def test_status_with_pending_running_intent_never_launches(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("start") + host.matching_marker = MarkerObservation("missing") + + response = recovery.recover( + host, + record, + context=_Context("status-context", source="cli"), + allow_launch=False, + ) + + self.assertTrue(response.message.startswith("action required:")) + self.assertEqual(host.start_calls, 0) + self.assertEqual(host.signal_calls, 0) + + def test_host_methods_are_resolved_after_recovery_construction(self) -> None: + recovery = PendingIntentRecovery() + host = _StrictRecoveryHost(recovery) + record = _pending_record("stop") + observed: list[str] = [] + + def patched_reader(bot_id: str, _profile_path: str) -> MarkerObservation: + observed.append(bot_id) + return MarkerObservation("missing") + + host._read_strict_runtime_marker = patched_reader # type: ignore[method-assign] + + response = recovery.recover_pending_stop_intent_locked( + host, + record, + context=_Context(record.pending_operation_id or ""), + allow_stop=True, + ) + + self.assertEqual(response.status, BotStatus.stopped) + self.assertEqual(observed, [record.bot_id]) + self.assertEqual(host.stopped_completions, [record.pending_operation_id]) + + def test_module_has_structural_host_and_no_store_lock_or_id_dependency(self) -> None: + import zeus.intent_recovery as module + + source = Path(inspect.getsourcefile(module) or "").read_text(encoding="utf-8") + tree = ast.parse(source) + imports = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + from_imports = { + node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + } + + self.assertTrue(hasattr(module, "_RecoveryHost")) + self.assertFalse( + any(isinstance(node, (ast.With, ast.AsyncWith)) for node in ast.walk(tree)) + ) + self.assertNotIn("sqlite3", imports | from_imports) + self.assertNotIn("uuid", imports | from_imports) + self.assertNotIn("zeus.state", from_imports) + self.assertNotIn("zeus.supervisor", from_imports) + self.assertNotIn("StateStore", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/zeus/intent_recovery.py b/zeus/intent_recovery.py new file mode 100644 index 0000000..9792c05 --- /dev/null +++ b/zeus/intent_recovery.py @@ -0,0 +1,630 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Protocol, TypeVar + +from zeus.errors import BotDeleteError +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import MarkerObservation, StopEffect +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, + TemplateError, +) +from zeus.process_identity import PidState +from zeus.readiness import ReadinessProbe, ReadinessResult + +_ContextT = TypeVar("_ContextT") + + +class _RecoveryHost(Protocol[_ContextT]): + def _recovery_lifecycle_context( + self, + operation_id: str, + context: _ContextT, + ) -> _ContextT: ... + + def _pending_action_required( + self, + record: BotRecord, + reason: str, + ) -> BotStatusResponse: ... + + def _pending_launch_preflight( + self, + record: BotRecord, + operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: ... + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: ... + + def _recover_pending_restart_predecessor( + self, + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse | None: ... + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _ContextT, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: ... + + def _matching_runtime_marker( + self, + record: BotRecord, + *, + expected_fingerprint: str, + require_live_command: bool, + ) -> MarkerObservation: ... + + def _probe_once(self, probe: ReadinessProbe) -> ReadinessResult: ... + + def _complete_started_intent( + self, + record: BotRecord, + *, + context: _ContextT, + status: BotStatus, + pid: int, + reason: str, + ready_at: datetime | None = ..., + last_error: str | None = ..., + reset_restart: bool = ..., + ) -> BotRecord: ... + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + context: _ContextT, + probe: ReadinessProbe | None, + ) -> BotStatusResponse: ... + + def _pid_state(self, pid: int) -> PidState: ... + + def _read_strict_runtime_marker( + self, + bot_id: str, + profile_path: str, + ) -> MarkerObservation: ... + + def _remove_owned_launch_marker_locked( + self, + record: BotRecord, + *, + observed: MarkerObservation, + ) -> bool: ... + + def _complete_stopped_intent( + self, + record: BotRecord, + *, + context: _ContextT, + reason: str, + ) -> BotRecord: ... + + def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: ... + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _ContextT, + complete_stop: bool, + ) -> BotStatusResponse: ... + + def _is_compat_runtime_marker(self, payload: dict[str, object]) -> bool: ... + + def _pending_restart_old_marker( + self, + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: ... + + def _classify_schema3_runtime_marker( + self, + record: BotRecord, + payload: dict[str, object], + *, + expected_pid: int | None, + expected_revision: int, + require_live_command: bool, + ) -> MarkerObservation: ... + + def _recover_pending_restart_old_gateway( + self, + record: BotRecord, + marker: MarkerObservation, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: ... + + def _gateway_generation( + self, + marker: MarkerObservation, + ) -> GatewayGeneration | None: ... + + def _stop_pending_restart_old_gateway( + self, + record: BotRecord, + generation: GatewayGeneration, + *, + context: _ContextT, + ) -> BotStatusResponse: ... + + def _remove_gateway_generation_marker_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> bool: ... + + def _update_lifecycle( + self, + context: _ContextT, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def _stop_gateway_generation_locked( + self, + record: BotRecord, + generation: GatewayGeneration, + ) -> StopEffect: ... + + def _append_recovery_audit_event(self, action: str, **values: object) -> None: ... + + +class PendingIntentRecovery: + def recover( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_launch: bool, + ) -> BotStatusResponse: + action = record.pending_action + operation_id = record.pending_operation_id + if action is None or operation_id is None: + return host._pending_action_required( + record, + "pending lifecycle correlation is invalid", + ) + recovery_context = host._recovery_lifecycle_context(operation_id, context) + if action == "stop": + return host._recover_pending_stop_intent( + record, + context=recovery_context, + allow_stop=allow_launch, + ) + if action not in {"start", "restart"}: + return host._pending_action_required( + record, + "pending lifecycle action is invalid", + ) + try: + probe, fingerprint = host._pending_launch_preflight(record, operation_id) + except (OSError, ValueError, TemplateError, BotDeleteError) as exc: + return host._pending_action_required(record, f"launch preflight failed: {exc}") + if action == "restart": + predecessor_result = host._recover_pending_restart_predecessor( + record, + context=recovery_context, + allow_stop=allow_launch, + ) + if predecessor_result is not None: + return predecessor_result + pending_result = host._recover_pending_launch( + record, + context=recovery_context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, + ) + if pending_result is not None: + return pending_result + return host._start_record( + record, + reset_restart=action == "restart", + message="recovered interrupted gateway launch", + context=recovery_context, + probe=probe, + ) + + def recover_pending_launch_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: + marker = host._matching_runtime_marker( + record, + expected_fingerprint=fingerprint, + require_live_command=True, + ) + if marker.kind == "live" and marker.payload is not None: + marker_pid = marker.payload["pid"] + if isinstance(marker_pid, bool) or not isinstance(marker_pid, int): + return host._pending_action_required(record, "live marker PID is invalid") + status = BotStatus.starting if probe is not None else BotStatus.running + ready_at = None if probe is not None else datetime.now(UTC) + if probe is not None: + readiness = host._probe_once(probe) + if readiness.ready: + status = BotStatus.running + ready_at = datetime.now(UTC) + try: + host._complete_started_intent( + record, + context=context, + status=status, + pid=marker_pid, + ready_at=ready_at, + reset_restart=True, + reason="recovery adopted registered gateway", + ) + except Exception: + return host._pending_action_required( + record, + "gateway adoption could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + status, + marker_pid, + record.profile_path, + "recovered registered gateway", + ) + if marker.kind == "untrusted": + return host._pending_action_required(record, marker.reason) + if not allow_launch: + return host._pending_action_required( + record, + "desired running gateway is missing; reconcile is required", + ) + if marker.kind == "dead": + generation = host._gateway_generation(marker) + if generation is None or not host._remove_gateway_generation_marker_locked( + record, + generation, + ): + return host._pending_action_required( + record, + "dead gateway marker cleanup failed", + ) + if action == "restart": + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: removed dead gateway marker; launch on next reconcile", + ) + return None + + def recover_pending_stop_intent_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: + pid_state = host._pid_state(record.pid) if record.pid else PidState.dead + if pid_state is PidState.unknown: + return host._pending_action_required(record, "gateway PID liveness is unknown") + if not record.pid or pid_state is PidState.dead: + observed = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if not host._remove_owned_launch_marker_locked(record, observed=observed): + return host._pending_action_required( + record, + "stale gateway marker ownership could not be verified", + ) + try: + host._complete_stopped_intent( + record, + context=context, + reason="recovery confirmed gateway stopped", + ) + except Exception: + return host._pending_action_required( + record, + "stopped state could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.stopped, + None, + record.profile_path, + "recovered interrupted stop", + ) + if not allow_stop: + if not host._pid_owned(record.profile_path, record.pid, record.bot_id): + return host._pending_action_required( + record, + "gateway ownership could not be verified", + ) + return host._pending_action_required( + record, + "stop intent is pending reconciliation", + ) + return host._stop_record_effect_locked( + record, + kill_after_timeout=None, + context=context, + complete_stop=True, + ) + + def pending_restart_old_marker( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + observed: MarkerObservation | None = None, + ) -> MarkerObservation | None: + if observed is None: + observed = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if observed.kind != "present" or observed.payload is None: + return None + payload = observed.payload + marker_operation = payload.get("operation_id") + marker_revision = payload.get("desired_revision") + if ( + type(marker_operation) is not str + or marker_operation == record.pending_operation_id + or type(marker_revision) is not int + or marker_revision != record.desired_revision - 1 + ): + return None + return host._classify_schema3_runtime_marker( + record, + payload, + expected_pid=record.pid, + expected_revision=record.desired_revision - 1, + require_live_command=True, + ) + + def recover_pending_restart_predecessor_locked( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse | None: + predecessor = host._read_strict_runtime_marker(record.bot_id, record.profile_path) + if ( + record.pid is not None + and predecessor.kind == "present" + and predecessor.payload is not None + and host._is_compat_runtime_marker(predecessor.payload) + ): + return host._pending_action_required( + record, + "schema-v2 or legacy gateway restart requires manual process resolution", + ) + old_marker = host._pending_restart_old_marker(record, predecessor) + if old_marker is not None: + return host._recover_pending_restart_old_gateway( + record, + old_marker, + context=context, + allow_stop=allow_stop, + ) + if record.pid is None or predecessor.kind != "missing": + return None + pid_state = host._pid_state(record.pid) + if pid_state is PidState.unknown: + return host._pending_action_required( + record, + "previous gateway PID liveness is unknown", + ) + if pid_state is PidState.alive: + return host._pending_action_required(record, "previous gateway marker is missing") + if not allow_stop: + return host._pending_action_required( + record, + "restart intent is pending reconciliation", + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_recovered", + stopped_at=datetime.now(UTC), + last_transition_reason="recovery confirmed the previous gateway stopped", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: recovered stopped gateway; launch on next reconcile", + ) + + def recover_pending_restart_old_gateway( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + marker: MarkerObservation, + *, + context: _ContextT, + allow_stop: bool, + ) -> BotStatusResponse: + if marker.kind == "untrusted": + return host._pending_action_required( + record, + marker.reason or "previous gateway ownership could not be verified", + ) + if not allow_stop: + return host._pending_action_required( + record, + "restart intent is pending reconciliation", + ) + generation = host._gateway_generation(marker) + if generation is None: + return host._pending_action_required( + record, + "previous gateway marker correlation is invalid", + ) + if marker.kind == "live": + if record.pid is None: + return host._pending_action_required( + record, + "previous gateway PID is not recorded", + ) + stopped = host._stop_pending_restart_old_gateway( + record, + generation, + context=context, + ) + if stopped.status is not BotStatus.stopped: + return stopped + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: previous gateway stopped; launch on next reconcile", + ) + if marker.kind != "dead" or marker.payload is None: + return host._pending_action_required( + record, + "previous gateway marker ownership could not be verified", + ) + if not host._remove_gateway_generation_marker_locked(record, generation): + return host._pending_action_required( + record, + "previous gateway marker cleanup could not be verified", + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_recovered", + stopped_at=datetime.now(UTC), + last_transition_reason="recovery confirmed the previous gateway stopped", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + return BotStatusResponse( + record.bot_id, + BotStatus.starting, + None, + record.profile_path, + "restart pending: recovered stopped gateway; launch on next reconcile", + ) + + def stop_pending_restart_old_gateway( + self, + host: _RecoveryHost[_ContextT], + record: BotRecord, + generation: GatewayGeneration, + *, + context: _ContextT, + ) -> BotStatusResponse: + effect = host._stop_gateway_generation_locked(record, generation) + if effect.outcome != "stopped": + if effect.kill_result is not None: + host._append_recovery_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=generation.pid, + succeeded=bool(effect.kill_succeeded), + ) + reasons = { + "term_denied": "could not send SIGTERM to the previous gateway", + "kill_denied": "could not send SIGKILL to the previous gateway", + "grace_expired": "previous gateway did not stop before the grace period expired", + "cleanup_unverified": ("previous gateway marker cleanup could not be verified"), + } + return host._pending_action_required( + record, + reasons.get(effect.outcome, effect.reason), + ) + if effect.kill_result is not None: + host._append_recovery_audit_event( + "bot.stop_kill", + bot_id=record.bot_id, + pid=generation.pid, + succeeded=bool(effect.kill_succeeded), + ) + try: + host._update_lifecycle( + context, + record.bot_id, + BotStatus.stopped, + pid=None, + action="bot.restart.old_process_stopped", + stopped_at=datetime.now(UTC), + last_transition_reason="restart stopped the previous gateway", + clear_ready_at=True, + ) + except Exception: + return host._pending_action_required( + record, + "previous gateway stop could not be persisted", + ) + host._append_recovery_audit_event( + "bot.stop", + bot_id=record.bot_id, + pid=generation.pid, + ) + return BotStatusResponse( + record.bot_id, + BotStatus.stopped, + None, + record.profile_path, + "gateway shutdown completed", + ) diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 7f07556..54038a9 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -41,9 +41,11 @@ PopenLike, RuntimeHooks, SignalResult, + StopEffect, gateway_process_launch_kwargs, ) from zeus.hermes_adapter import HermesAdapter +from zeus.intent_recovery import PendingIntentRecovery from zeus.lifecycle import LifecycleEvent, LifecycleEventInput from zeus.logging_utils import tail_file from zeus.models import ( @@ -54,7 +56,6 @@ DesiredState, HermesTemplate, RestartPolicy, - TemplateError, validate_id, ) from zeus.private_io import nofollow_absolute_path @@ -201,6 +202,7 @@ def __init__( cleanup_process_group=self._cleanup_process_group, hooks_provider=self._runtime_hooks, ) + self._intent_recovery = PendingIntentRecovery() self._locks_guard = threading.Lock() self._bot_locks: dict[str, threading.RLock] = {} @@ -2087,123 +2089,86 @@ def _recover_pending_intent( context: _LifecycleContext, allow_launch: bool, ) -> BotStatusResponse: - action = record.pending_action - operation_id = record.pending_operation_id - if action is None or operation_id is None: - return self._pending_action_required(record, "pending lifecycle correlation is invalid") - recovery_context = _LifecycleContext(operation_id, context.source, context.request_id) - if action == "stop": - try: - with self._marker_publication_lock(record): - return self._recover_pending_stop_intent_locked( - record, - context=recovery_context, - allow_stop=allow_launch, - ) - except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return self._intent_recovery.recover( + self, + record, + context=context, + allow_launch=allow_launch, + ) + + @staticmethod + def _recovery_lifecycle_context( + operation_id: str, + context: _LifecycleContext, + ) -> _LifecycleContext: + return _LifecycleContext(operation_id, context.source, context.request_id) - if action not in {"start", "restart"}: - return self._pending_action_required(record, "pending lifecycle action is invalid") + def _pending_launch_preflight( + self, + record: BotRecord, + operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: + probe = self._preflight_start(record, timeout_seconds=None) + expected = self.adapter.launcher_payload( + record.bot_id, + operation_id=operation_id, + desired_revision=record.desired_revision, + readiness_probe=probe, + ) + marker_template = expected["marker"] + if type(marker_template) is not dict: + raise ValueError("invalid expected marker") + return probe, str(marker_template["command_fingerprint"]) + + @staticmethod + def _probe_once(probe: ReadinessProbe) -> ReadinessResult: + return probe_once( + probe.url, + timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), + expected_status=probe.expected_status, + expected_platform=probe.expected_platform, + ) + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse: try: - probe = self._preflight_start(record, timeout_seconds=None) - expected = self.adapter.launcher_payload( - record.bot_id, - operation_id=operation_id, - desired_revision=record.desired_revision, - readiness_probe=probe, - ) - marker_template = expected["marker"] - if type(marker_template) is not dict: - raise ValueError("invalid expected marker") - fingerprint = str(marker_template["command_fingerprint"]) - except (OSError, ValueError, TemplateError, BotDeleteError) as exc: - return self._pending_action_required(record, f"launch preflight failed: {exc}") - if action == "restart": - predecessor_result = self._recover_pending_restart_predecessor( - record, - context=recovery_context, - allow_stop=allow_launch, - ) - if predecessor_result is not None: - return predecessor_result + with self._marker_publication_lock(record): + return self._recover_pending_stop_intent_locked( + record, + context=context, + allow_stop=allow_stop, + ) + except (BotDeleteError, LaunchPayloadError) as exc: + return self._pending_action_required(record, str(exc)) + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _LifecycleContext, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: try: with self._marker_publication_lock(record): - marker = self._matching_runtime_marker( + return self._intent_recovery.recover_pending_launch_locked( + self, record, - expected_fingerprint=fingerprint, - require_live_command=True, + context=context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, ) - if marker.kind == "live" and marker.payload is not None: - marker_pid = marker.payload["pid"] - if isinstance(marker_pid, bool) or not isinstance(marker_pid, int): - return self._pending_action_required(record, "live marker PID is invalid") - pid = marker_pid - status = BotStatus.starting if probe is not None else BotStatus.running - ready_at = None if probe is not None else datetime.now(UTC) - if probe is not None: - readiness = probe_once( - probe.url, - timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), - expected_status=probe.expected_status, - expected_platform=probe.expected_platform, - ) - if readiness.ready: - status = BotStatus.running - ready_at = datetime.now(UTC) - try: - self._complete_started_intent( - record, - context=recovery_context, - status=status, - pid=pid, - ready_at=ready_at, - reset_restart=True, - reason="recovery adopted registered gateway", - ) - except Exception: - return self._pending_action_required( - record, "gateway adoption could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - status, - pid, - record.profile_path, - "recovered registered gateway", - ) - if marker.kind == "untrusted": - return self._pending_action_required(record, marker.reason) - if not allow_launch: - return self._pending_action_required( - record, "desired running gateway is missing; reconcile is required" - ) - if marker.kind == "dead": - generation = self._gateway_generation(marker) - if generation is None or not self._remove_gateway_generation_marker_locked( - record, generation - ): - return self._pending_action_required( - record, "dead gateway marker cleanup failed" - ) - if action == "restart": - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: removed dead gateway marker; " - "launch on next reconcile", - ) except (BotDeleteError, LaunchPayloadError) as exc: return self._pending_action_required(record, str(exc)) - return self._start_record( - record, - reset_restart=action == "restart", - message="recovered interrupted gateway launch", - context=recovery_context, - probe=probe, - ) def _recover_pending_stop_intent_locked( self, @@ -2212,41 +2177,11 @@ def _recover_pending_stop_intent_locked( context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead - if pid_state is _PidState.unknown: - return self._pending_action_required(record, "gateway PID liveness is unknown") - if not record.pid or pid_state is _PidState.dead: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if not self._remove_owned_launch_marker_locked(record, observed=observed): - return self._pending_action_required( - record, "stale gateway marker ownership could not be verified" - ) - try: - self._complete_stopped_intent( - record, - context=context, - reason="recovery confirmed gateway stopped", - ) - except Exception: - return self._pending_action_required(record, "stopped state could not be persisted") - return BotStatusResponse( - record.bot_id, - BotStatus.stopped, - None, - record.profile_path, - "recovered interrupted stop", - ) - if not allow_stop: - if not self._pid_owned(record.profile_path, record.pid, record.bot_id): - return self._pending_action_required( - record, "gateway ownership could not be verified" - ) - return self._pending_action_required(record, "stop intent is pending reconciliation") - return self._stop_record_effect_locked( + return self._intent_recovery.recover_pending_stop_intent_locked( + self, record, - kill_after_timeout=None, context=context, - complete_stop=True, + allow_stop=allow_stop, ) def _pending_restart_old_marker( @@ -2254,27 +2189,10 @@ def _pending_restart_old_marker( record: BotRecord, observed: _MarkerObservation | None = None, ) -> _MarkerObservation | None: - """Return a strictly verified marker from the generation before a restart intent.""" - if observed is None: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if observed.kind != "present" or observed.payload is None: - return None - payload = observed.payload - marker_operation = payload.get("operation_id") - marker_revision = payload.get("desired_revision") - if ( - type(marker_operation) is not str - or marker_operation == record.pending_operation_id - or type(marker_revision) is not int - or marker_revision != record.desired_revision - 1 - ): - return None - return self._classify_schema3_runtime_marker( + return self._intent_recovery.pending_restart_old_marker( + self, record, - payload, - expected_pid=record.pid, - expected_revision=record.desired_revision - 1, - require_live_command=True, + observed, ) def _recover_pending_restart_predecessor( @@ -2286,61 +2204,11 @@ def _recover_pending_restart_predecessor( ) -> BotStatusResponse | None: try: with self._marker_publication_lock(record): - predecessor = self._read_strict_runtime_marker(record.bot_id, record.profile_path) - if ( - record.pid is not None - and predecessor.kind == "present" - and predecessor.payload is not None - and self._is_compat_runtime_marker(predecessor.payload) - ): - return self._pending_action_required( - record, - "schema-v2 or legacy gateway restart requires manual process resolution", - ) - old_marker = self._pending_restart_old_marker(record, predecessor) - if old_marker is not None: - return self._recover_pending_restart_old_gateway( - record, - old_marker, - context=context, - allow_stop=allow_stop, - ) - if record.pid is None or predecessor.kind != "missing": - return None - pid_state = self._pid_state(record.pid) - if pid_state is _PidState.unknown: - return self._pending_action_required( - record, "previous gateway PID liveness is unknown" - ) - if pid_state is _PidState.alive: - return self._pending_action_required( - record, "previous gateway marker is missing" - ) - if not allow_stop: - return self._pending_action_required( - record, "restart intent is pending reconciliation" - ) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_recovered", - stopped_at=datetime.now(UTC), - last_transition_reason=("recovery confirmed the previous gateway stopped"), - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: recovered stopped gateway; launch on next reconcile", + return self._intent_recovery.recover_pending_restart_predecessor_locked( + self, + record, + context=context, + allow_stop=allow_stop, ) except (BotDeleteError, LaunchPayloadError) as exc: return self._pending_action_required(record, str(exc)) @@ -2353,64 +2221,12 @@ def _recover_pending_restart_old_gateway( context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - if marker.kind == "untrusted": - return self._pending_action_required( - record, - marker.reason or "previous gateway ownership could not be verified", - ) - if not allow_stop: - return self._pending_action_required(record, "restart intent is pending reconciliation") - generation = self._gateway_generation(marker) - if generation is None: - return self._pending_action_required( - record, "previous gateway marker correlation is invalid" - ) - if marker.kind == "live": - if record.pid is None: - return self._pending_action_required(record, "previous gateway PID is not recorded") - stopped = self._stop_pending_restart_old_gateway( - record, - generation, - context=context, - ) - if stopped.status is not BotStatus.stopped: - return stopped - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: previous gateway stopped; launch on next reconcile", - ) - if marker.kind != "dead" or marker.payload is None: - return self._pending_action_required( - record, "previous gateway marker ownership could not be verified" - ) - if not self._remove_gateway_generation_marker_locked(record, generation): - return self._pending_action_required( - record, "previous gateway marker cleanup could not be verified" - ) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_recovered", - stopped_at=datetime.now(UTC), - last_transition_reason="recovery confirmed the previous gateway stopped", - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - return BotStatusResponse( - record.bot_id, - BotStatus.starting, - None, - record.profile_path, - "restart pending: recovered stopped gateway; launch on next reconcile", + return self._intent_recovery.recover_pending_restart_old_gateway( + self, + record, + marker, + context=context, + allow_stop=allow_stop, ) def _stop_pending_restart_old_gateway( @@ -2420,61 +2236,28 @@ def _stop_pending_restart_old_gateway( *, context: _LifecycleContext, ) -> BotStatusResponse: - effect = self._runtime.stop_generation_locked( + return self._intent_recovery.stop_pending_restart_old_gateway( + self, + record, + generation, + context=context, + ) + + def _stop_gateway_generation_locked( + self, + record: BotRecord, + generation: _GatewayGeneration, + ) -> StopEffect: + return self._runtime.stop_generation_locked( record, generation, kill_after_timeout=None, classify_exact=self._classify_exact_gateway_generation, remove_generation=self._remove_gateway_generation_marker_locked, ) - if effect.outcome != "stopped": - if effect.kill_result is not None: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=record.bot_id, - pid=generation.pid, - succeeded=bool(effect.kill_succeeded), - ) - reasons = { - "term_denied": "could not send SIGTERM to the previous gateway", - "kill_denied": "could not send SIGKILL to the previous gateway", - "grace_expired": ("previous gateway did not stop before the grace period expired"), - "cleanup_unverified": ("previous gateway marker cleanup could not be verified"), - } - return self._pending_action_required( - record, - reasons.get(effect.outcome, effect.reason), - ) - if effect.kill_result is not None: - self.store.append_audit_event( - "bot.stop_kill", - bot_id=record.bot_id, - pid=generation.pid, - succeeded=bool(effect.kill_succeeded), - ) - try: - self._update_lifecycle( - context, - record.bot_id, - BotStatus.stopped, - pid=None, - action="bot.restart.old_process_stopped", - stopped_at=datetime.now(UTC), - last_transition_reason="restart stopped the previous gateway", - clear_ready_at=True, - ) - except Exception: - return self._pending_action_required( - record, "previous gateway stop could not be persisted" - ) - self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=generation.pid) - return BotStatusResponse( - record.bot_id, - BotStatus.stopped, - None, - record.profile_path, - "gateway shutdown completed", - ) + + def _append_recovery_audit_event(self, action: str, **values: object) -> None: + self.store.append_audit_event(action, **values) def _restart_delay(self, record: BotRecord) -> float: delay = record.restart_backoff_seconds * (2**record.restart_attempts) From 2946d5a1d7591e07b82ba957710fbede513eb40e Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:00:54 +0200 Subject: [PATCH 41/49] feat(api): add authenticated readiness endpoint --- docs/API.md | 27 +++++- docs/ARCHITECTURE.md | 3 +- docs/OPERATIONS.md | 15 ++++ docs/openapi.json | 92 +++++++++++++++++++++ tests/test_api.py | 154 ++++++++++++++++++++++++++++++++++- tests/test_api_logging.py | 2 + tests/test_repo_contracts.py | 37 +++++++++ tests/test_sqlite_schema.py | 117 +++++++++++++++++++++++++- zeus/api.py | 18 +++- zeus/request_context.py | 1 + zeus/schema.py | 32 ++++++-- zeus/sqlite_db.py | 23 +++++- zeus/state.py | 4 + 13 files changed, 510 insertions(+), 15 deletions(-) diff --git a/docs/API.md b/docs/API.md index 1460a85..e0e7d9f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -38,7 +38,7 @@ Known error codes are `invalid_request`, `invalid_bot_id`, `unknown_bot`, `idempotency_key_conflict`, `idempotency_in_progress`, `idempotency_indeterminate`, `idempotency_response_too_large`, `idempotency_store_unavailable`, -`server_busy`, `server_draining`, and `internal_error`. +`server_busy`, `server_draining`, `not_ready`, and `internal_error`. JSON responses include `cache-control: no-store`. Mutating endpoints that accept request bodies require an `application/json` content type and reject missing or invalid media @@ -159,12 +159,37 @@ permission, or serialization failures never change the HTTP response. Setting ### `GET /health` +Public process-liveness check. It does not access SQLite or authenticate the +caller, so a successful response does not mean the state store is ready. + Returns: ```json {"status":"ok"} ``` +### `GET /ready` + +Authenticated state-store readiness check, also available as `GET /v1/ready`. +It opens the existing SQLite database in read-only mode, requires schema version +6, and executes `SELECT 1`; it does not inspect or start bots. A stopped bot does +not make Zeus unready. + +The route uses the normal read-endpoint authentication policy. It requires +`x-zeus-api-key` unless loopback-only development explicitly enables +`ZEUS_ALLOW_UNAUTH_READS=1`. Query parameters are rejected before the database +probe. + +Success returns: + +```json +{"schema_version":6,"status":"ready"} +``` + +An unavailable, missing, malformed, older, or newer state database returns +`503` with `error.code=not_ready`. Failure to initialize the state store before +the API binds remains a startup failure rather than an HTTP readiness response. + ### `GET /doctor` Returns the same readiness report as `zeus doctor --json`. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ececd37..9d1b5fe 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -62,7 +62,8 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. Zeus uses persistent SQLite WAL mode. `SQLiteDatabase` installs the selected `ZEUS_SQLITE_SYNCHRONOUS` policy on every returned operational connection after both newer-schema guards, foreign-key enforcement, and WAL setup. The raw -read-only schema preflight cannot commit and is intentionally not configured. +read-only schema preflight and readiness probe cannot commit and are +intentionally not configured with durability PRAGMAs. Committed transactions survive an application or Zeus process crash under both NORMAL and FULL. With NORMAL, SQLite omits a WAL sync on most commits, so a host diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 41c12be..84bae68 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -35,6 +35,21 @@ markers, locks, or the best-effort audit JSONL atomic with the database, replace backups, or overcome storage that lies about flushes. Continue the backup and restore procedures below under either mode. +## API Liveness and Readiness + +Use `GET /health` for public process liveness. It does not authenticate or touch +SQLite, so it can remain healthy while persistent state is unavailable. + +Use `GET /ready` (or `GET /v1/ready`) for load-balancer and service readiness. +This ordinary read endpoint requires `x-zeus-api-key`, except on a loopback bind +when `ZEUS_ALLOW_UNAUTH_READS=1` is explicitly enabled. It opens the existing +database read-only, requires the current schema version, and runs `SELECT 1`. +It never creates or migrates a database and does not require bots to be running. + +A ready service returns `{"schema_version":6,"status":"ready"}`. State-store +failures return `503` with `error.code=not_ready`. If state initialization fails +before the API binds, the process exits instead of serving `/ready`. + ## Backup Back up `ZEUS_STATE_DIR` regularly. For the sample systemd deployment, that is diff --git a/docs/openapi.json b/docs/openapi.json index 6c28129..4ed730f 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -83,6 +83,7 @@ "idempotency_response_too_large", "server_busy", "server_draining", + "not_ready", "internal_error" ] }, @@ -96,6 +97,20 @@ } } }, + "ReadinessResponse": { + "type": "object", + "required": ["schema_version", "status"], + "properties": { + "schema_version": { + "type": "integer", + "const": 6 + }, + "status": { + "type": "string", + "const": "ready" + } + } + }, "Bot": { "type": "object", "required": [ @@ -650,6 +665,83 @@ } } }, + "/ready": { + "get": { + "summary": "Check API state-store readiness", + "description": "Authenticates the request, then opens the Zeus database read-only, requires the current schema version, and executes SELECT 1. The /v1/ready alias is provided by the versioned server URL.", + "security": [ + { + "ZeusApiKey": [] + } + ], + "responses": { + "200": { + "description": "The state store is readable at the current schema version", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReadinessResponse" + } + } + } + }, + "400": { + "description": "Invalid query parameters", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "401": { + "description": "Missing API key, invalid API key, or API key not configured", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "429": { + "description": "Authentication rate limit exceeded", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"}, + "Retry-After": { + "description": "Seconds to wait before retrying authentication.", + "schema": { + "type": "integer", + "minimum": 1 + } + } + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + }, + "503": { + "description": "The API key is unconfigured or the state store is not ready", + "headers": { + "X-Request-ID": {"$ref": "#/components/headers/XRequestID"} + }, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Error"} + } + } + } + } + } + }, "/doctor": { "get": { "responses": { diff --git a/tests/test_api.py b/tests/test_api.py index 7943e7e..a0b0a85 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -33,7 +33,7 @@ ReconcileOutcome, ReconcileRunSummary, ) -from zeus.state import StateStore +from zeus.state import SCHEMA_VERSION, StateReadinessError, StateStore JsonPayload = dict[str, Any] | list[Any] @@ -154,6 +154,28 @@ def raw_http_response( conn.close() +def raw_http_exchange( + port: int, + method: str, + path: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> tuple[int, dict[str, str], bytes]: + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + response_body = response.read() + return ( + response.status, + {name.lower(): value for name, value in response.getheaders()}, + response_body, + ) + finally: + conn.close() + + def json_request_body(payload: JsonPayload) -> bytes: return json.dumps(payload).encode("utf-8") @@ -237,6 +259,136 @@ def raw_post_without_content_length(port: int, body: bytes) -> tuple[int, dict[s class ApiBehaviorTests(unittest.TestCase): + def test_health_exact_bytes_are_public_and_never_probe_readiness(self) -> None: + with ( + patch.object( + StateStore, + "check_readiness", + side_effect=AssertionError("health must not probe state"), + ) as readiness, + api_server_with_state() as (port, state_dir), + ): + exchanges = [raw_http_exchange(port, "GET", path) for path in ("/health", "/v1/health")] + rows = wait_for_access_rows(state_dir, 2) + + for status, headers, body in exchanges: + self.assertEqual(200, status) + self.assertEqual(b'{"status": "ok"}', body) + self.assertEqual("application/json", headers["content-type"]) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + readiness.assert_not_called() + self.assertEqual(["/health", "/health"], [row["route"] for row in rows]) + self.assertEqual(["not_required", "not_required"], [row["auth_outcome"] for row in rows]) + + def test_readiness_auth_query_alias_and_override_contract(self) -> None: + with patch.object(StateStore, "check_readiness", return_value=SCHEMA_VERSION) as readiness: + with api_server() as port: + missing_config_status, _ = request_json(port, "GET", "/ready") + self.assertEqual(503, missing_config_status) + readiness.assert_not_called() + + configured = { + "ZEUS_API_KEY": "secret", + "ZEUS_API_AUTH_FAILURE_RATE_PER_MINUTE": "1", + "ZEUS_API_AUTH_FAILURE_BURST": "2", + } + with api_server(configured) as port: + missing_status, _ = request_json(port, "GET", "/ready") + wrong_status, _ = request_json( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "wrong"}, + ) + limited_status, limited_headers, limited_body = request_json_with_headers( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "wrong"}, + ) + query_status, query_body = request_json( + port, + "GET", + "/ready?debug=1", + headers={"x-zeus-api-key": "secret"}, + ) + ready_status, ready_headers, ready_body = raw_http_exchange( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "secret"}, + ) + alias_status, _alias_headers, alias_body = raw_http_exchange( + port, + "GET", + "/v1/ready", + headers={"x-zeus-api-key": "secret"}, + ) + + self.assertEqual((401, 401, 429), (missing_status, wrong_status, limited_status)) + self.assertEqual("auth_rate_limited", limited_body["error"]["code"]) + self.assertGreaterEqual(int(limited_headers["retry-after"]), 1) + self.assertEqual(400, query_status) + self.assertEqual("invalid_request", query_body["error"]["code"]) + expected = b'{"schema_version": 6, "status": "ready"}' + self.assertEqual((200, expected), (ready_status, ready_body)) + self.assertEqual((200, expected), (alias_status, alias_body)) + self.assertRegex(ready_headers["x-request-id"], r"^[0-9a-f]{32}$") + self.assertEqual(2, readiness.call_count) + + with api_server({"ZEUS_ALLOW_UNAUTH_READS": "1"}) as port: + override_status, override_body = request_json(port, "GET", "/ready") + self.assertEqual(200, override_status) + self.assertEqual({"schema_version": SCHEMA_VERSION, "status": "ready"}, override_body) + self.assertEqual(3, readiness.call_count) + + def test_readiness_expected_failure_is_sanitized_and_not_logged_as_exception(self) -> None: + with ( + patch.object( + StateStore, + "check_readiness", + side_effect=StateReadinessError("private sqlite detail"), + ), + api_server_with_state({"ZEUS_API_KEY": "secret"}) as (port, state_dir), + ): + status, headers, body = raw_http_exchange( + port, + "GET", + "/ready", + headers={"x-zeus-api-key": "secret"}, + ) + rows = wait_for_access_rows(state_dir, 1) + + self.assertEqual(503, status) + self.assertEqual( + b'{"error": {"code": "not_ready", ' + b'"message": "state store is not ready", "status": 503}}', + body, + ) + self.assertNotIn("retry-after", headers) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + self.assertEqual(["api.access"], [row["event"] for row in rows]) + self.assertEqual("/ready", rows[0]["route"]) + self.assertEqual("not_ready", rows[0]["error_code"]) + self.assertNotIn("private sqlite detail", json.dumps(rows)) + + def test_readiness_aliases_probe_the_real_current_store(self) -> None: + with api_server({"ZEUS_API_KEY": "secret"}) as port: + responses = [ + raw_http_exchange( + port, + "GET", + path, + headers={"x-zeus-api-key": "secret"}, + ) + for path in ("/ready", "/v1/ready") + ] + + for status, headers, body in responses: + self.assertEqual(200, status) + self.assertEqual(b'{"schema_version": 6, "status": "ready"}', body) + self.assertRegex(headers["x-request-id"], r"^[0-9a-f]{32}$") + def test_exception_status_payload_and_header_contracts(self) -> None: cases = ( ( diff --git a/tests/test_api_logging.py b/tests/test_api_logging.py index 29dd2ef..73072b5 100644 --- a/tests/test_api_logging.py +++ b/tests/test_api_logging.py @@ -108,6 +108,8 @@ def test_idempotency_outcomes_are_a_closed_secret_free_vocabulary(self) -> None: def test_route_templates_hide_bot_ids_and_normalize_v1(self) -> None: self.assertEqual("/bots/{bot_id}/start", route_template("/v1/bots/secret-bot/start")) self.assertEqual("/bots/{bot_id}/history", route_template("/v1/bots/secret-bot/history")) + self.assertEqual("/ready", route_template("/ready")) + self.assertEqual("/ready", route_template("/v1/ready")) self.assertIsNone(route_template("/not-a-route")) def test_api_log_writer_writes_parseable_redacted_lines(self) -> None: diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 06dd9f0..7d20ef6 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -1058,6 +1058,43 @@ def test_openapi_contract_loads_and_documents_required_paths(self) -> None: history["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], ) + def test_readiness_openapi_and_operator_documentation_contract(self) -> None: + spec = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) + api_docs = Path("docs/API.md").read_text(encoding="utf-8") + operations = Path("docs/OPERATIONS.md").read_text(encoding="utf-8") + + self.assertIn("/ready", spec["paths"]) + self.assertNotIn("/v1/ready", spec["paths"]) + self.assertEqual([], spec["paths"]["/health"]["get"]["security"]) + readiness = spec["paths"]["/ready"]["get"] + self.assertEqual([{"ZeusApiKey": []}], readiness["security"]) + self.assertEqual({"200", "400", "401", "429", "503"}, set(readiness["responses"])) + self.assertEqual( + "#/components/schemas/ReadinessResponse", + readiness["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], + ) + retry_after = readiness["responses"]["429"]["headers"]["Retry-After"] + self.assertNotIn("$ref", retry_after) + self.assertEqual({"type": "integer", "minimum": 1}, retry_after["schema"]) + for response in readiness["responses"].values(): + self.assertEqual( + {"$ref": "#/components/headers/XRequestID"}, + response["headers"]["X-Request-ID"], + ) + + schema = spec["components"]["schemas"]["ReadinessResponse"] + self.assertEqual(["schema_version", "status"], schema["required"]) + self.assertEqual(SCHEMA_VERSION, schema["properties"]["schema_version"]["const"]) + self.assertEqual("ready", schema["properties"]["status"]["const"]) + error_codes = spec["components"]["schemas"]["Error"]["properties"]["error"]["properties"][ + "code" + ]["enum"] + self.assertIn("not_ready", error_codes) + for text in (api_docs, operations): + self.assertIn("/ready", text) + self.assertIn("/health", text) + self.assertIn("ZEUS_ALLOW_UNAUTH_READS", text) + def test_wave_two_replay_and_recovery_contracts_are_documented(self) -> None: spec = json.loads(Path("docs/openapi.json").read_text(encoding="utf-8")) api_docs = Path("docs/API.md").read_text(encoding="utf-8") diff --git a/tests/test_sqlite_schema.py b/tests/test_sqlite_schema.py index 5435d41..af9616b 100644 --- a/tests/test_sqlite_schema.py +++ b/tests/test_sqlite_schema.py @@ -11,7 +11,8 @@ from zeus.config import SQLiteSynchronous from zeus.schema import SCHEMA_VERSION, SchemaManager, _preflight_schema_compatibility -from zeus.sqlite_db import SQLiteDatabase +from zeus.sqlite_db import SQLiteDatabase, StateReadinessError +from zeus.state import StateReadinessError as FacadeStateReadinessError from zeus.state import StateStore _ORIGINAL_SQLITE_CONNECT = sqlite3.connect @@ -577,6 +578,120 @@ def connect(self) -> sqlite3.Connection: class SQLiteSchemaTests(unittest.TestCase): + def test_readiness_uses_one_read_only_connection_and_exact_current_schema(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + database_path = Path(tmp) / "zeus.db" + StateStore(database_path).init() + database = SQLiteDatabase(database_path, synchronous=SQLiteSynchronous.FULL) + expected_uri = f"{database_path.resolve().as_uri()}?mode=ro" + calls: list[tuple[object, bool]] = [] + traces: list[str] = [] + real_connect = sqlite3.connect + + def connect_spy(*args: object, **kwargs: object) -> sqlite3.Connection: + calls.append((args[0], bool(kwargs.get("uri", False)))) + conn = real_connect(*args, **kwargs) + conn.set_trace_callback(traces.append) + return conn + + with patch("zeus.sqlite_db.sqlite3.connect", side_effect=connect_spy): + version = database.check_readiness() + + self.assertEqual(SCHEMA_VERSION, version) + self.assertEqual([(expected_uri, True)], calls) + self.assertEqual( + [ + statement + for statement in traces + if "sqlite_master" in statement + or "schema_version" in statement + or statement == "SELECT 1" + ], + traces, + ) + self.assertEqual("SELECT 1", traces[-1]) + self.assertFalse( + any(statement.lstrip().upper().startswith("PRAGMA") for statement in traces) + ) + + def test_readiness_rejects_missing_malformed_old_and_new_state_without_writes(self) -> None: + cases = ( + "missing", + "missing_table", + "empty", + "malformed_text", + "malformed_real", + "old", + "new", + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for case in cases: + with self.subTest(case=case): + database_path = root / case / "zeus.db" + if case != "missing": + database_path.parent.mkdir(parents=True) + with closing(sqlite3.connect(database_path)) as conn: + if case != "missing_table": + conn.execute("CREATE TABLE schema_version (version)") + if case == "malformed_text": + conn.execute("INSERT INTO schema_version VALUES ('six')") + elif case == "malformed_real": + conn.execute("INSERT INTO schema_version VALUES (6.5)") + elif case == "old": + conn.execute( + "INSERT INTO schema_version VALUES (?)", (SCHEMA_VERSION - 1,) + ) + elif case == "new": + conn.execute( + "INSERT INTO schema_version VALUES (?)", (SCHEMA_VERSION + 1,) + ) + conn.commit() + parent_existed = database_path.parent.exists() + before = _database_snapshot(database_path) if database_path.exists() else None + + with self.assertRaisesRegex(StateReadinessError, "state store is not ready"): + SQLiteDatabase(database_path).check_readiness() + + self.assertEqual(parent_existed, database_path.parent.exists()) + self.assertEqual( + before, + _database_snapshot(database_path) if database_path.exists() else None, + ) + if case == "missing": + self.assertFalse(database_path.exists()) + + def test_readiness_normalizes_operational_error_closes_connection_and_chains_cause( + self, + ) -> None: + connection = MagicMock(spec=sqlite3.Connection) + connection.row_factory = None + connection.execute.side_effect = [ + MagicMock(fetchone=MagicMock(return_value=(1,))), + MagicMock(fetchone=MagicMock(return_value={"version": SCHEMA_VERSION})), + sqlite3.OperationalError("injected readiness failure"), + ] + with ( + patch("zeus.sqlite_db.sqlite3.connect", return_value=connection), + self.assertRaises(StateReadinessError) as raised, + ): + SQLiteDatabase("state/zeus.db").check_readiness() + + self.assertIsInstance(raised.exception.__cause__, sqlite3.OperationalError) + self.assertEqual("state store is not ready", str(raised.exception)) + connection.close.assert_called_once_with() + + def test_state_store_readiness_wrapper_and_error_reexport(self) -> None: + database = MagicMock(spec=SQLiteDatabase) + database.check_readiness.return_value = SCHEMA_VERSION + with patch("zeus.state.SQLiteDatabase", return_value=database): + store = StateStore("state/zeus.db") + result = store.check_readiness() + + self.assertIs(StateReadinessError, FacadeStateReadinessError) + self.assertEqual(SCHEMA_VERSION, result) + database.check_readiness.assert_called_once_with() + def test_new_schema_components_construct_without_opening_the_database(self) -> None: with tempfile.TemporaryDirectory() as tmp: database_path = Path(tmp) / "nested" / "zeus.db" diff --git a/zeus/api.py b/zeus/api.py index 980c188..6839c56 100644 --- a/zeus/api.py +++ b/zeus/api.py @@ -43,7 +43,7 @@ ReconcileOutcome, ) from zeus.request_context import RequestContext, new_request_id, route_template -from zeus.state import MAX_IDEMPOTENCY_RESPONSE_BYTES, StateStore +from zeus.state import MAX_IDEMPOTENCY_RESPONSE_BYTES, StateReadinessError, StateStore from zeus.supervisor import Supervisor from zeus.templates import TemplateStore @@ -289,7 +289,21 @@ def _dispatch_get(self) -> None: else: self._validate_query_parameters(set()) self._require_key(read=not self._get_requires_strict_auth(path)) - if path == "/doctor": + if path == "/ready": + try: + schema_version = store.check_readiness() + except StateReadinessError: + self._json_error_response( + HTTPStatus.SERVICE_UNAVAILABLE, + "not_ready", + "state store is not ready", + ) + return + self._json( + HTTPStatus.OK, + {"schema_version": schema_version, "status": "ready"}, + ) + elif path == "/doctor": self._json(HTTPStatus.OK, run_doctor(settings).to_dict()) elif path == "/templates": self._json(HTTPStatus.OK, [template_to_dict(t) for t in TemplateStore().list()]) diff --git a/zeus/request_context.py b/zeus/request_context.py index 3ee9363..0e25607 100644 --- a/zeus/request_context.py +++ b/zeus/request_context.py @@ -49,6 +49,7 @@ _STATIC_ROUTE_TEMPLATES = { "/health": "/health", + "/ready": "/ready", "/doctor": "/doctor", "/templates": "/templates", "/bots": "/bots", diff --git a/zeus/schema.py b/zeus/schema.py index 83be196..96b1926 100644 --- a/zeus/schema.py +++ b/zeus/schema.py @@ -16,7 +16,7 @@ def connect(self) -> sqlite3.Connection: ... def immediate(self) -> AbstractContextManager[sqlite3.Connection]: ... -def _assert_schema_compatible(conn: sqlite3.Connection) -> None: +def _read_schema_version(conn: sqlite3.Connection) -> int | None: table = conn.execute( """ SELECT 1 FROM sqlite_master @@ -24,15 +24,31 @@ def _assert_schema_compatible(conn: sqlite3.Connection) -> None: """ ).fetchone() if table is None: - return + return None row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - if row is not None and int(row["version"]) > SCHEMA_VERSION: + if row is None: + return None + version = row["version"] + if type(version) is not int: + raise TypeError("database schema version must be an integer") + return version + + +def _assert_schema_compatible(conn: sqlite3.Connection) -> None: + version = _read_schema_version(conn) + if version is not None and version > SCHEMA_VERSION: raise RuntimeError( - f"database schema version {int(row['version'])} is newer than supported " - f"version {SCHEMA_VERSION}" + f"database schema version {version} is newer than supported version {SCHEMA_VERSION}" ) +def _assert_schema_current(conn: sqlite3.Connection) -> int: + version = _read_schema_version(conn) + if version != SCHEMA_VERSION: + raise RuntimeError("database schema is not current") + return version + + def _preflight_schema_compatibility(database_path: Path) -> None: if not database_path.exists(): return @@ -108,9 +124,9 @@ def _migrate(self, conn: sqlite3.Connection) -> None: ) """ ) - row = conn.execute("SELECT version FROM schema_version ORDER BY rowid LIMIT 1").fetchone() - current_version = int(row["version"]) if row else 0 - if row is None: + stored_version = _read_schema_version(conn) + current_version = stored_version or 0 + if stored_version is None: conn.execute("INSERT INTO schema_version (version) VALUES (0)") if current_version > SCHEMA_VERSION: raise RuntimeError( diff --git a/zeus/sqlite_db.py b/zeus/sqlite_db.py index 1402d9a..212c29b 100644 --- a/zeus/sqlite_db.py +++ b/zeus/sqlite_db.py @@ -6,7 +6,11 @@ from pathlib import Path from zeus.config import SQLiteSynchronous -from zeus.schema import _assert_schema_compatible, _preflight_schema_compatibility +from zeus.schema import ( + _assert_schema_compatible, + _assert_schema_current, + _preflight_schema_compatibility, +) _SYNCHRONOUS_PRAGMA = { SQLiteSynchronous.NORMAL: "PRAGMA synchronous=NORMAL", @@ -14,6 +18,10 @@ } +class StateReadinessError(RuntimeError): + pass + + class SQLiteDatabase: def __init__( self, @@ -40,6 +48,19 @@ def connect(self) -> sqlite3.Connection: conn.close() raise + def check_readiness(self) -> int: + uri = f"{self.database_path.resolve().as_uri()}?mode=ro" + try: + with closing(sqlite3.connect(uri, uri=True)) as conn: + conn.row_factory = sqlite3.Row + version = _assert_schema_current(conn) + row = conn.execute("SELECT 1").fetchone() + if row is None or type(row[0]) is not int or row[0] != 1: + raise RuntimeError("state store probe returned an invalid result") + return version + except (sqlite3.Error, OSError, RuntimeError, TypeError, ValueError) as exc: + raise StateReadinessError("state store is not ready") from exc + @contextmanager def immediate(self) -> Iterator[sqlite3.Connection]: with closing(self.connect()) as conn: diff --git a/zeus/state.py b/zeus/state.py index 94f5d40..1dba788 100644 --- a/zeus/state.py +++ b/zeus/state.py @@ -32,6 +32,7 @@ from zeus.schema import SCHEMA_VERSION as SCHEMA_VERSION from zeus.schema import SchemaManager from zeus.sqlite_db import SQLiteDatabase +from zeus.sqlite_db import StateReadinessError as StateReadinessError class StateStore: @@ -54,6 +55,9 @@ def database_path(self) -> Path: def connect(self) -> sqlite3.Connection: return self._database.connect() + def check_readiness(self) -> int: + return self._database.check_readiness() + def init(self) -> None: self._schema.init() From 12d3515e81b9ab74a7f8a81bc4674a52b4ea5b20 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:17:11 +0200 Subject: [PATCH 42/49] fix(private-io): recognize Linux dir-fd lstat support --- tests/test_private_io.py | 10 ++++++++++ zeus/private_io.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_private_io.py b/tests/test_private_io.py index cb7f73c..2148a98 100644 --- a/tests/test_private_io.py +++ b/tests/test_private_io.py @@ -223,6 +223,16 @@ def test_missing_descriptor_relative_primitive_fails_before_creation(self) -> No self.assertFalse(path.parent.exists()) + def test_linux_stat_capability_advertises_descriptor_relative_lstat(self) -> None: + path = self.root / "private" + linux_capabilities = frozenset({os.open, os.mkdir, os.stat}) + + with patch.object(private_io.os, "supports_dir_fd", linux_capabilities): + ensure_private_directory(path) + + self.assertTrue(path.is_dir()) + self.assertEqual(0o700, stat.S_IMODE(path.stat().st_mode)) + def test_validate_private_directory_requires_existing_directory(self) -> None: path = self.root / "missing" diff --git a/zeus/private_io.py b/zeus/private_io.py index 8ec43d5..86629fb 100644 --- a/zeus/private_io.py +++ b/zeus/private_io.py @@ -29,7 +29,9 @@ ) _OPEN_DIR_FD_PROBE = os.open _MKDIR_DIR_FD_PROBE = os.mkdir -_LSTAT_DIR_FD_PROBE = os.lstat +# CPython advertises descriptor-relative lstat through os.stat on Linux even +# though os.lstat itself accepts dir_fd. Actual inspections still use lstat. +_LSTAT_DIR_FD_PROBE = os.stat class UnsafeFileError(OSError): From 8decb0e17189b9e56c9bb51252e2e08a8cba2aef Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:22:55 +0200 Subject: [PATCH 43/49] ci: add pinned real Hermes compatibility gate --- .github/workflows/ci.yml | 33 ++++++++ CONTRIBUTING.md | 4 +- README.md | 5 +- docs/COMPATIBILITY.md | 38 +++++---- docs/REAL_HERMES_VERIFICATION.md | 24 +++++- docs/ROADMAP.md | 3 +- requirements-hermes-ci.txt | 138 +++++++++++++++++++++++++++++++ scripts/fresh_vps_verify.sh | 1 - scripts/verify_real_hermes.sh | 74 +++++++++++++++-- tests/test_repo_contracts.py | 70 +++++++++++++++- 10 files changed, 357 insertions(+), 33 deletions(-) create mode 100644 requirements-hermes-ci.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5306bc8..9133712 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,39 @@ jobs: tests.test_fake_hermes_integration \ tests.test_crash_recovery.GatewayLauncherTests + real-hermes: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + - name: Prepare sanitized failure evidence + run: | + mkdir -p .tmp/real-hermes-evidence + printf '%s\n' 'result=failed' 'failure_stage=ci_setup' > .tmp/real-hermes-evidence/summary.txt + - name: Install pinned Hermes compatibility environment + run: python -m pip install --require-hashes --only-binary=:all: -r requirements-hermes-ci.txt + - name: Install Zeus + run: python -m pip install -e . + - name: Check installed dependencies + run: python -m pip check + - name: Verify real Hermes without provider credentials + run: | + ZEUS_VERIFY_START_GATEWAY=1 \ + ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 \ + ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence \ + sh scripts/verify_real_hermes.sh + - name: Upload sanitized failure evidence + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: real-hermes-failure-evidence + path: .tmp/real-hermes-evidence/summary.txt + if-no-files-found: error + retention-days: 7 + package: runs-on: ubuntu-24.04 timeout-minutes: 15 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e68619c..3a9986b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,8 +13,8 @@ make check See the [compatibility policy](docs/COMPATIBILITY.md) for the operating systems, Python versions, and Hermes boundary covered by committed automation. The -real-Hermes check below is a separate release gate because it depends on the -operator's installed Hermes version. +committed Ubuntu/Python 3.11 gate uses the hash-locked Hermes Agent 0.19.0 +environment; the manual check below covers the operator's installed version. ## Quality Bar diff --git a/README.md b/README.md index 7f7fd51..a0878ae 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,10 @@ ZEUS_VERIFY_START_GATEWAY=1 sh scripts/verify_real_hermes.sh ``` The gateway check enables Hermes' local `api_server` platform on loopback, -passes a random per-run API key, probes `/health`, and then stops the bot. +passes an isolated local API key, starts with readiness waiting, verifies process +ownership, probes `/health`, and then stops the bot. Committed CI runs this flow +without provider credentials against the fully hash-locked Hermes Agent 0.19.0 +environment documented in the compatibility policy. For a clean Debian/Ubuntu host, use the fresh VPS harness: diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index ae427ad..14a5aa1 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -9,9 +9,10 @@ an untested platform or external Hermes release into a support claim. | Gate | Committed runner | Python | Scope | | --- | --- | --- | --- | | Main CI matrix | Linux `ubuntu-24.04` | Python 3.11, 3.12, and 3.13 | Unit and integration tests, repository contracts, source-and-branch coverage, formatting, lint, typing, Bandit, and ShellCheck | -| Provisional Python compatibility | Linux `ubuntu-24.04` | Python 3.14 | Full Zeus test suite; non-required and Zeus-only because no Hermes baseline is pinned | +| Provisional Python compatibility | Linux `ubuntu-24.04` | Python 3.14 | Full Zeus test suite; non-required and Zeus-only because the pinned Hermes baseline requires Python below 3.14 | | Subprocess lifecycle | Linux `ubuntu-24.04` | Python 3.11 | Focused multi-process lifecycle and locking behavior | | macOS process lifecycle | macOS `macos-26` | Python 3.13 | Focused process, fake-Hermes integration, and gateway-launcher recovery tests | +| Real Hermes compatibility | Linux `ubuntu-24.04` | Python 3.11 | Hash-locked Hermes Agent 0.19.0 profile rendering, strict diagnostics, loopback gateway readiness, process ownership, and clean shutdown without a model-provider credential | | Package build | Linux `ubuntu-24.04` | Python 3.11 | Wheel and source build, installed-wheel smoke test, dependency consistency, and metadata checks | | Tagged release build | Linux `ubuntu-latest` | Python 3.11 | Full release gate, artifact checksums, and GitHub release artifacts | @@ -25,7 +26,8 @@ platform guarantee. Python 3.14 is a provisional Zeus-only lane with `continue-on-error` behavior. It does not promote Python 3.14 to required Hermes compatibility: the repository -has not yet pinned and passed a Hermes baseline on that interpreter. +pins Hermes Agent 0.19.0, whose package metadata requires Python 3.11 through +3.13, and runs that compatibility gate only on Python 3.11. The package metadata declares `requires-python = ">=3.11"`, while committed CI currently tests the versions listed above. A version absent from that matrix is @@ -49,7 +51,7 @@ clean-host runbook for Debian and Ubuntu. It can bootstrap OS packages, install Zeus into a virtual environment, run local gates, render multiple profiles, and exercise the loopback API. Optional Hermes installation and live probes cross an external network and credential boundary, so their logs are evidence for that -specific host and invocation rather than deterministic CI. +specific host and invocation rather than the committed CI environment. Local development checks such as `make check` and `sh scripts/wheel_smoke.sh` remain useful evidence, but they do not add the developer's operating system to @@ -57,18 +59,24 @@ the automated matrix. ## Hermes boundary -No Hermes version is pinned by this repository. There is no deterministic -real-Hermes CI gate today. The manual -[`scripts/verify_real_hermes.sh`](../scripts/verify_real_hermes.sh) check uses -whichever `hermes` executable is installed on `PATH`: it runs strict diagnostics, -renders a profile, invokes Hermes doctor, and can optionally start a loopback -gateway and probe its health. - -Record `hermes version` with manual verification evidence. Passing against one -installed version does not establish compatibility with every Hermes release. -Before a Hermes baseline becomes required automation, the repository must name -the exact verified version or immutable source, install it reproducibly, and run -the real-Hermes gate without provider secrets in logs or command arguments. +The deterministic CI baseline is Hermes Agent 0.19.0 on Ubuntu 24.04 with Python +3.11. [`requirements-hermes-ci.txt`](../requirements-hermes-ci.txt) pins the +complete 60-package wheel closure and its selected SHA-256 hashes. CI installs it +with pip hash checking and binary-only resolution; it never runs the remote +Hermes installer. The lock also carries Linux arm64 hashes for native local +container verification without changing the CI package versions. + +The gate uses no model-provider credential or paid request. It renders a profile, +runs strict Zeus and Hermes diagnostics, starts the loopback gateway with +`--wait`, checks Zeus process ownership and Hermes `/health`, then stops the bot +and removes runtime state. On failure it uploads only a two-line sanitized stage +summary, never the rendered profile, environment, logs, or process arguments. + +The manual [`scripts/verify_real_hermes.sh`](../scripts/verify_real_hermes.sh) +check still uses whichever `hermes` executable is installed on `PATH` unless +`ZEUS_VERIFY_EXPECTED_HERMES_VERSION` is set. Record `hermes version` with manual +evidence. Passing the pinned baseline does not establish compatibility with every +Hermes release or optional integration. ## Updating this policy diff --git a/docs/REAL_HERMES_VERIFICATION.md b/docs/REAL_HERMES_VERIFICATION.md index 54788ff..664c5ed 100644 --- a/docs/REAL_HERMES_VERIFICATION.md +++ b/docs/REAL_HERMES_VERIFICATION.md @@ -1,6 +1,11 @@ # Real Hermes Verification -The normal test suite uses a fake Hermes executable so the repository can be tested without external credentials or a Hermes install. +The normal test suite uses a fake Hermes executable so the repository can be +tested without external credentials or a Hermes install. CI separately installs +the fully hash-locked Hermes Agent 0.19.0 environment from +`requirements-hermes-ci.txt` on Ubuntu/Python 3.11. The lock contains Linux +x86_64 and arm64 wheel hashes; the gate does not run the remote installer or make +a model-provider request. Before a release, verify against a real Hermes install: @@ -27,19 +32,26 @@ before stopping the bot: ZEUS_VERIFY_START_GATEWAY=1 sh scripts/verify_real_hermes.sh ``` -When gateway startup verification is enabled, the script waits for Zeus to report -the bot as running, then polls Hermes `/health` until the local `api_server` +When gateway startup verification is enabled, the script starts Zeus with +`--wait`, confirms Zeus reports the bot as running, then polls Hermes `/health` +until the local `api_server` reports `{"status":"ok","platform":"hermes-agent"}` or the health timeout expires. This avoids false negatives when Hermes binds the loopback API shortly after the process becomes visible to Zeus. +Successful and failed runs stop the bot and remove the isolated runtime tree. +Failures retain only a sanitized `summary.txt` containing the fixed result and +failure-stage labels. Raw logs, rendered profiles, environments, and command +arguments are never copied into that evidence directory. + Useful overrides: ```bash ZEUS_VERIFY_BOT_ID=my-check-bot ZEUS_VERIFY_TEMPLATE=research-bot ZEUS_VERIFY_STATE_DIR=.zeus-real-hermes-check -ZEUS_VERIFY_GATEWAY_SECONDS=5 +ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence +ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 ZEUS_VERIFY_API_KEY=real-hermes-local-check ZEUS_VERIFY_API_SERVER_HOST=127.0.0.1 ZEUS_VERIFY_API_SERVER_PORT=4312 @@ -47,6 +59,10 @@ ZEUS_VERIFY_HEALTH_TIMEOUT_SECONDS=30 ZEUS_VERIFY_HEALTH_INTERVAL_SECONDS=0.5 ``` +Leave `ZEUS_VERIFY_EXPECTED_HERMES_VERSION` unset for an intentional manual +compatibility check against another installed version. Such a run is local +evidence, not an update to the committed baseline. + Expected failure when Hermes is not installed: ```text diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index eb5105e..fc26d45 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -11,7 +11,8 @@ profiles, processes, lifecycle safety, and reconciliation evidence on one host. - Bundled and custom Hermes template validation and rendering. - PID ownership checks, lifecycle locking, crash recovery, and restart policy. - SQLite lifecycle and reconciliation evidence. -- Credential-free fake-Hermes demo and manual real-Hermes verification scripts. +- Credential-free fake-Hermes demo, hash-locked real-Hermes CI, and manual + real-Hermes verification scripts. - Wheel builds, installed-wheel smoke checks, and GitHub release artifacts. ## Near term diff --git a/requirements-hermes-ci.txt b/requirements-hermes-ci.txt new file mode 100644 index 0000000..44fc467 --- /dev/null +++ b/requirements-hermes-ci.txt @@ -0,0 +1,138 @@ +# Hermes Agent 0.19.0 compatibility lock for CPython 3.11 on Ubuntu x86_64 +# and arm64. +# Resolved from PyPI on 2026-07-22. CI installs this file with --require-hashes +# and --only-binary=:all:. Re-resolve the complete closure when updating Hermes. +annotated-doc==0.0.4 \ + --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 +cffi==2.1.0 \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 +charset-normalizer==3.4.9 \ + --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ + --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 +click==8.4.2 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 +croniter==6.0.0 \ + --hash=sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368 +cryptography==46.0.7 \ + --hash=sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308 \ + --hash=sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325 +distro==1.9.0 \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 +fastapi==0.139.2 \ + --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c +fire==0.7.1 \ + --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 +h11==0.16.0 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +hermes-agent==0.19.0 \ + --hash=sha256:bd0bac012aee38a60894781f4597dc29ee7bedb3448540249921f10d3bef327f +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 +httptools==0.8.0 \ + --hash=sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376 \ + --hash=sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d +httpx==0.28.1 \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +jiter==0.16.0 \ + --hash=sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037 \ + --hash=sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee +markdown==3.10.2 \ + --hash=sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36 +markdown-it-py==4.2.0 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +markupsafe==3.0.3 \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +openai==2.24.0 \ + --hash=sha256:fed30480d7d6c884303287bde864980a4b137b60553ffbcf9ab4a233b7a73d94 +packaging==26.0 \ + --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 +pathspec==1.1.1 \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 +pillow==12.2.0 \ + --hash=sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e \ + --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 +prompt-toolkit==3.0.52 \ + --hash=sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955 +psutil==7.2.2 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e +ptyprocess==0.7.0 \ + --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 +pycparser==3.0 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba +pydantic-core==2.46.4 \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 +pygments==2.20.0 \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 +pyjwt==2.13.0 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 +python-dateutil==2.9.0.post0 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +python-dotenv==1.2.2 \ + --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a +python-multipart==0.0.32 \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 +pytz==2026.2 \ + --hash=sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 +pyyaml==6.0.3 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b +rich==14.3.3 \ + --hash=sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d +ruamel.yaml==0.18.17 \ + --hash=sha256:9c8ba9eb3e793efdf924b60d521820869d5bf0cb9c6f1b82d82de8295e290b9d +ruamel.yaml.clib==0.2.15 \ + --hash=sha256:617d35dc765715fa86f8c3ccdae1e4229055832c452d4ec20856136acc75053f \ + --hash=sha256:ef71831bd61fbdb7aa0399d5c4da06bea37107ab5c79ff884cc07f2450910262 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 +socksio==1.0.0 \ + --hash=sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3 +starlette==1.3.1 \ + --hash=sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6 +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 +termcolor==3.3.0 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 +tqdm==4.69.0 \ + --hash=sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 +urllib3==2.7.0 \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 +uvicorn==0.51.0 \ + --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b +uvloop==0.22.1 \ + --hash=sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21 \ + --hash=sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702 +watchfiles==1.2.0 \ + --hash=sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a \ + --hash=sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e +wcwidth==0.8.2 \ + --hash=sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85 +websockets==15.0.1 \ + --hash=sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413 \ + --hash=sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562 diff --git a/scripts/fresh_vps_verify.sh b/scripts/fresh_vps_verify.sh index 2ee1104..7551aad 100755 --- a/scripts/fresh_vps_verify.sh +++ b/scripts/fresh_vps_verify.sh @@ -375,7 +375,6 @@ run hermes doctor section "Real Hermes Compatibility" run env \ ZEUS_VERIFY_START_GATEWAY="${ZEUS_VPS_START_GATEWAY:-0}" \ - ZEUS_VERIFY_GATEWAY_SECONDS="${ZEUS_VPS_GATEWAY_SECONDS:-5}" \ sh scripts/verify_real_hermes.sh section "Multi Template Profile Check" diff --git a/scripts/verify_real_hermes.sh b/scripts/verify_real_hermes.sh index b89ae6d..8ae4e1e 100755 --- a/scripts/verify_real_hermes.sh +++ b/scripts/verify_real_hermes.sh @@ -6,18 +6,40 @@ set -eu bot_id="${ZEUS_VERIFY_BOT_ID:-real-hermes-check}" template_id="${ZEUS_VERIFY_TEMPLATE:-coding-bot}" state_dir="${ZEUS_VERIFY_STATE_DIR:-.zeus-real-hermes-check}" +evidence_dir="${ZEUS_VERIFY_EVIDENCE_DIR:-.tmp/real-hermes-evidence}" +expected_hermes_version="${ZEUS_VERIFY_EXPECTED_HERMES_VERSION:-}" api_server_host="${ZEUS_VERIFY_API_SERVER_HOST:-127.0.0.1}" api_server_port="${ZEUS_VERIFY_API_SERVER_PORT:-4312}" health_timeout_seconds="${ZEUS_VERIFY_HEALTH_TIMEOUT_SECONDS:-30}" health_interval_seconds="${ZEUS_VERIFY_HEALTH_INTERVAL_SECONDS:-0.5}" case "$state_dir" in - "" | "/" | "." | ".." | /* | ../* | */../*) + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) echo "unsafe ZEUS_VERIFY_STATE_DIR: use a workspace-relative scratch directory" >&2 exit 2 ;; esac +case "$evidence_dir" in + "" | "/" | "." | ".." | /* | ../* | */.. | */../*) + echo "unsafe ZEUS_VERIFY_EVIDENCE_DIR: use a workspace-relative scratch directory" >&2 + exit 2 + ;; +esac + +case "$state_dir/" in + "$evidence_dir/"*) + echo "unsafe verification directories: state and evidence paths must not overlap" >&2 + exit 2 + ;; +esac +case "$evidence_dir/" in + "$state_dir/"*) + echo "unsafe verification directories: state and evidence paths must not overlap" >&2 + exit 2 + ;; +esac + case "$api_server_host" in 127.0.0.1 | localhost) ;; *) @@ -43,21 +65,55 @@ if ! command -v hermes >/dev/null 2>&1; then exit 2 fi +failure_stage="environment" cleanup() { + status=$? + trap - EXIT INT TERM if [ -d "$state_dir" ]; then ZEUS_STATE_DIR="$state_dir" \ ZEUS_HERMES_BIN="$(command -v hermes 2>/dev/null || printf '%s' hermes)" \ python3 -B -m zeus.cli bot stop "$bot_id" >/dev/null 2>&1 || true fi + rm -rf -- "$state_dir" + if [ "$status" -eq 0 ]; then + rm -rf -- "$evidence_dir" + else + rm -rf -- "$evidence_dir" + mkdir -p "$evidence_dir" + chmod 0700 "$evidence_dir" + ( + umask 077 + printf 'result=failed\nfailure_stage=%s\n' "$failure_stage" \ + >"$evidence_dir/summary.txt" + ) + fi + exit "$status" } -trap cleanup EXIT INT TERM - -rm -rf -- "$state_dir" +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +rm -rf -- "$state_dir" "$evidence_dir" + +failure_stage="hermes_version" +if [ -n "$expected_hermes_version" ]; then + actual_hermes_version="$( + python3 -c 'from importlib.metadata import version; print(version("hermes-agent"))' + )" + if [ "$actual_hermes_version" != "$expected_hermes_version" ]; then + echo "Hermes version mismatch: expected $expected_hermes_version, got $actual_hermes_version" >&2 + exit 1 + fi +fi +failure_stage="zeus_doctor" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" ZEUS_API_KEY="${ZEUS_VERIFY_API_KEY:-real-hermes-local-check}" python3 -B -m zeus.cli doctor --strict --json +failure_stage="profile_create" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" python3 -B -m zeus.cli bot create "$bot_id" --template "$template_id" +failure_stage="bot_doctor" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" python3 -B -m zeus.cli bot doctor "$bot_id" +failure_stage="rendered_profile" config_path="$state_dir/hermes/profiles/$bot_id/config.yaml" test -f "$config_path" grep -q "max_async_children" "$config_path" @@ -72,6 +128,7 @@ if [ "${ZEUS_VERIFY_START_GATEWAY:-0}" = "1" ]; then api_server_passthrough="$ZEUS_ENV_PASSTHROUGH,$api_server_passthrough" fi + failure_stage="gateway_start" API_SERVER_ENABLED=1 \ API_SERVER_HOST="$api_server_host" \ API_SERVER_PORT="$api_server_port" \ @@ -79,11 +136,12 @@ if [ "${ZEUS_VERIFY_START_GATEWAY:-0}" = "1" ]; then ZEUS_ENV_PASSTHROUGH="$api_server_passthrough" \ ZEUS_STATE_DIR="$state_dir" \ ZEUS_HERMES_BIN="$(command -v hermes)" \ - python3 -B -m zeus.cli bot start "$bot_id" - sleep "${ZEUS_VERIFY_GATEWAY_SECONDS:-3}" + python3 -B -m zeus.cli bot start "$bot_id" --wait --timeout "$health_timeout_seconds" + failure_stage="gateway_status" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot status "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "running"' + failure_stage="process_ownership" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot inspect "$bot_id" --json \ | python3 -c ' @@ -100,6 +158,7 @@ assert ownership["classification"] in { "legacy-marker-valid", }, ownership ' + failure_stage="loopback_health" python3 - "$api_server_host" "$api_server_port" "$health_timeout_seconds" "$health_interval_seconds" <<'PY' import json import sys @@ -136,9 +195,11 @@ while True: ) time.sleep(interval_seconds) PY + failure_stage="gateway_stop" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot stop "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "stopped"' + failure_stage="stopped_status" ZEUS_STATE_DIR="$state_dir" ZEUS_HERMES_BIN="$(command -v hermes)" \ python3 -B -m zeus.cli bot status "$bot_id" \ | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "stopped"' @@ -146,4 +207,5 @@ else echo "Skipping gateway start. Set ZEUS_VERIFY_START_GATEWAY=1 to exercise hermes gateway run." fi +failure_stage="complete" echo "Real Hermes verification completed for $bot_id using $state_dir" diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 7d20ef6..134aa10 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -130,6 +130,7 @@ def test_publishable_repository_files_exist(self) -> None: "docs/assets/demo.cast", "docs/assets/zeus-hero.png", ".coveragerc", + "requirements-hermes-ci.txt", ".github/workflows/release.yml", ".github/ISSUE_TEMPLATE/bug_report.yml", ".github/ISSUE_TEMPLATE/feature_request.yml", @@ -171,6 +172,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "python-3-14": "ubuntu-24.04", "lifecycle-subprocess": "ubuntu-24.04", "macos-process-lifecycle": "macos-26", + "real-hermes": "ubuntu-24.04", "package": "ubuntu-24.04", } expected_python_versions = { @@ -178,6 +180,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "python-3-14": ("3.14",), "lifecycle-subprocess": ("3.11",), "macos-process-lifecycle": ("3.13",), + "real-hermes": ("3.11",), "package": ("3.11",), } expected_setup_python = { @@ -185,6 +188,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "python-3-14": '"3.14"', "lifecycle-subprocess": '"3.11"', "macos-process-lifecycle": '"3.13"', + "real-hermes": '"3.11"', "package": '"3.11"', } expected_commands = { @@ -215,6 +219,19 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: " tests.test_fake_hermes_integration \\\n" " tests.test_crash_recovery.GatewayLauncherTests", ), + "real-hermes": ( + "mkdir -p .tmp/real-hermes-evidence\n" + "printf '%s\\n' 'result=failed' 'failure_stage=ci_setup' > " + ".tmp/real-hermes-evidence/summary.txt", + "python -m pip install --require-hashes --only-binary=:all: " + "-r requirements-hermes-ci.txt", + "python -m pip install -e .", + "python -m pip check", + "ZEUS_VERIFY_START_GATEWAY=1 \\\n" + "ZEUS_VERIFY_EXPECTED_HERMES_VERSION=0.19.0 \\\n" + "ZEUS_VERIFY_EVIDENCE_DIR=.tmp/real-hermes-evidence \\\n" + "sh scripts/verify_real_hermes.sh", + ), "package": ( 'python -m pip install -e ".[dev]"', "python -m pip check", @@ -244,6 +261,14 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: job_level_continue_on_error[job_name] = values[0] self.assertEqual({"python-3-14": "true"}, job_level_continue_on_error) + real_hermes = jobs["real-hermes"] + self.assertEqual("15", _job_level_scalar(real_hermes, "timeout-minutes")) + self.assertNotIn("secrets.", real_hermes) + self.assertNotRegex(real_hermes, r"(?i)(openai|anthropic|openrouter).*api[_-]?key") + self.assertRegex(real_hermes, r"(?m)^ if: failure\(\)\s*$") + self.assertIn(".tmp/real-hermes-evidence/summary.txt", real_hermes) + self.assertIn("if-no-files-found: error", real_hermes) + python_314 = jobs["python-3-14"].lower() self.assertNotIn("hermes", python_314) self.assertNotIn("verify_real_hermes", python_314) @@ -625,8 +650,8 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No ( "Linux `ubuntu-24.04`", "Python 3.14", - "Full Zeus test suite; non-required and Zeus-only because no Hermes " - "baseline is pinned", + "Full Zeus test suite; non-required and Zeus-only because the pinned " + "Hermes baseline requires Python below 3.14", ), ), "lifecycle-subprocess": ( @@ -651,6 +676,19 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No "Focused process, fake-Hermes integration, and gateway-launcher recovery tests", ), ), + "real-hermes": ( + ci_jobs["real-hermes"], + "Real Hermes compatibility", + "ubuntu-24.04", + ("3.11",), + ( + "Linux `ubuntu-24.04`", + "Python 3.11", + "Hash-locked Hermes Agent 0.19.0 profile rendering, strict diagnostics, " + "loopback gateway readiness, process ownership, and clean shutdown " + "without a model-provider credential", + ), + ), "package": ( ci_jobs["package"], "Package build", @@ -694,7 +732,9 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No ) self.assertIn("lifecycle and package jobs use Python 3.11", compatibility_text) self.assertIn("Debian and Ubuntu", compatibility_text) - self.assertIn("No Hermes version is pinned", compatibility_text) + self.assertIn("Hermes Agent 0.19.0", compatibility_text) + self.assertIn("complete 60-package wheel closure", compatibility_text) + self.assertIn("no model-provider credential", compatibility_text) self.assertIn("whichever `hermes` executable is installed", compatibility_text) self.assertIn("Python 3.14", compatibility_text) self.assertIn("provisional Zeus-only", compatibility_text) @@ -808,6 +848,30 @@ def test_real_hermes_verifier_is_isolated_and_checks_async_cap(self) -> None: self.assertIn("/health", script) self.assertIn("time.monotonic", script) self.assertIn("Hermes /health did not become ready", script) + self.assertIn("ZEUS_VERIFY_EXPECTED_HERMES_VERSION", script) + self.assertIn('bot start "$bot_id" --wait', script) + self.assertIn("ZEUS_VERIFY_EVIDENCE_DIR", script) + self.assertIn("failure_stage=%s", script) + self.assertIn('rm -rf -- "$state_dir"', script) + + def test_real_hermes_ci_lock_is_complete_and_hash_pinned(self) -> None: + requirements = Path("requirements-hermes-ci.txt").read_text(encoding="utf-8") + entries = re.findall( + r"(?m)^([a-z0-9][a-z0-9._-]*)==([^\\\s]+) \\$", + requirements, + ) + hashes = re.findall(r"(?m)^ --hash=sha256:([0-9a-f]{64})(?: \\)?$", requirements) + + self.assertEqual(60, len(entries)) + self.assertEqual(60, len({name for name, _version in entries})) + self.assertEqual(74, len(hashes)) + self.assertIn(("hermes-agent", "0.19.0"), entries) + self.assertIn( + "bd0bac012aee38a60894781f4597dc29ee7bedb3448540249921f10d3bef327f", + hashes, + ) + self.assertNotIn("--index-url", requirements) + self.assertNotIn("git+", requirements) def test_fresh_vps_verifier_bootstraps_and_captures_evidence(self) -> None: script = Path("scripts/fresh_vps_verify.sh").read_text(encoding="utf-8") From 23cbbdae7329972a14ca8baf8d2250a0e86618b8 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:26:26 +0200 Subject: [PATCH 44/49] security: pin built-in container images --- templates/coding-bot.toml | 2 +- templates/deepseek-coding-bot.toml | 2 +- templates/docs-writer-bot.toml | 2 +- templates/gateway-operator.toml | 2 +- templates/log-triage-bot.toml | 2 +- templates/research-bot.toml | 2 +- templates/support-gateway.toml | 2 +- tests/test_repo_contracts.py | 30 +++++++++++++++++++ zeus/bundled_templates/coding-bot.toml | 2 +- .../deepseek-coding-bot.toml | 2 +- zeus/bundled_templates/docs-writer-bot.toml | 2 +- zeus/bundled_templates/gateway-operator.toml | 2 +- zeus/bundled_templates/log-triage-bot.toml | 2 +- zeus/bundled_templates/research-bot.toml | 2 +- zeus/bundled_templates/support-gateway.toml | 2 +- 15 files changed, 44 insertions(+), 14 deletions(-) diff --git a/templates/coding-bot.toml b/templates/coding-bot.toml index a7777ca..f4cdc25 100644 --- a/templates/coding-bot.toml +++ b/templates/coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/deepseek-coding-bot.toml b/templates/deepseek-coding-bot.toml index c4fb3e7..6e66599 100644 --- a/templates/deepseek-coding-bot.toml +++ b/templates/deepseek-coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/docs-writer-bot.toml b/templates/docs-writer-bot.toml index d1a1e64..72199ff 100644 --- a/templates/docs-writer-bot.toml +++ b/templates/docs-writer-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/gateway-operator.toml b/templates/gateway-operator.toml index c9da28c..3256701 100644 --- a/templates/gateway-operator.toml +++ b/templates/gateway-operator.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/log-triage-bot.toml b/templates/log-triage-bot.toml index c8e5ab9..b97a2ee 100644 --- a/templates/log-triage-bot.toml +++ b/templates/log-triage-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/research-bot.toml b/templates/research-bot.toml index 0a41f2c..f5fd247 100644 --- a/templates/research-bot.toml +++ b/templates/research-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/templates/support-gateway.toml b/templates/support-gateway.toml index 91b618f..52a682a 100644 --- a/templates/support-gateway.toml +++ b/templates/support-gateway.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 180 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 134aa10..eb6cf00 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -7,6 +7,7 @@ import subprocess import tempfile import textwrap +import tomllib import unittest from pathlib import Path @@ -106,6 +107,35 @@ def _markdown_table_rows(markdown: str) -> dict[str, tuple[str, ...]]: class RepoContractTests(unittest.TestCase): + def test_builtin_template_copies_are_identical_and_images_are_digest_pinned( + self, + ) -> None: + source_root = Path("templates") + bundled_root = Path("zeus/bundled_templates") + expected_image = ( + "nikolaik/python-nodejs:python3.11-nodejs20@sha256:" + "8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" + ) + source_names = sorted(path.name for path in source_root.glob("*.toml")) + bundled_names = sorted(path.name for path in bundled_root.glob("*.toml")) + + self.assertEqual(source_names, bundled_names) + self.assertGreaterEqual(len(source_names), 7) + for name in source_names: + with self.subTest(template=name): + source_text = (source_root / name).read_text(encoding="utf-8") + bundled_text = (bundled_root / name).read_text(encoding="utf-8") + self.assertEqual(source_text, bundled_text) + + data = tomllib.loads(source_text) + docker_image = data["hermes"]["terminal"]["docker_image"] + self.assertIsInstance(docker_image, str) + self.assertEqual(expected_image, docker_image) + self.assertRegex( + docker_image, + r"\A[a-z0-9./_-]+:[a-zA-Z0-9._-]+@sha256:[0-9a-f]{64}\Z", + ) + def test_publishable_repository_files_exist(self) -> None: required = [ "README.md", diff --git a/zeus/bundled_templates/coding-bot.toml b/zeus/bundled_templates/coding-bot.toml index a7777ca..f4cdc25 100644 --- a/zeus/bundled_templates/coding-bot.toml +++ b/zeus/bundled_templates/coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/deepseek-coding-bot.toml b/zeus/bundled_templates/deepseek-coding-bot.toml index c4fb3e7..6e66599 100644 --- a/zeus/bundled_templates/deepseek-coding-bot.toml +++ b/zeus/bundled_templates/deepseek-coding-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/docs-writer-bot.toml b/zeus/bundled_templates/docs-writer-bot.toml index d1a1e64..72199ff 100644 --- a/zeus/bundled_templates/docs-writer-bot.toml +++ b/zeus/bundled_templates/docs-writer-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/gateway-operator.toml b/zeus/bundled_templates/gateway-operator.toml index c9da28c..3256701 100644 --- a/zeus/bundled_templates/gateway-operator.toml +++ b/zeus/bundled_templates/gateway-operator.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/log-triage-bot.toml b/zeus/bundled_templates/log-triage-bot.toml index c8e5ab9..b97a2ee 100644 --- a/zeus/bundled_templates/log-triage-bot.toml +++ b/zeus/bundled_templates/log-triage-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 240 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/research-bot.toml b/zeus/bundled_templates/research-bot.toml index 0a41f2c..f5fd247 100644 --- a/zeus/bundled_templates/research-bot.toml +++ b/zeus/bundled_templates/research-bot.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 300 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] diff --git a/zeus/bundled_templates/support-gateway.toml b/zeus/bundled_templates/support-gateway.toml index 91b618f..52a682a 100644 --- a/zeus/bundled_templates/support-gateway.toml +++ b/zeus/bundled_templates/support-gateway.toml @@ -18,7 +18,7 @@ backend = "docker" cwd = "." home_mode = "profile" timeout = 180 -docker_image = "nikolaik/python-nodejs:python3.11-nodejs20" +docker_image = "nikolaik/python-nodejs:python3.11-nodejs20@sha256:8f958bdc1b4a422bfafd97cab4f69836401f616ae985d4b57a53d254f5bcb038" docker_mount_cwd_to_workspace = false [hermes.gateway] From 13de3b7e9e2e5633d0d9fec649a3699a8baeabcd Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:35:29 +0200 Subject: [PATCH 45/49] ci(release): require verified release refs --- .github/workflows/release.yml | 4 + CHANGELOG.md | 2 + docs/RELEASE.md | 35 ++-- scripts/check_verified_release_ref.py | 218 +++++++++++++++++++++++ scripts/repo_check.sh | 1 + tests/test_repo_contracts.py | 7 +- tests/test_verified_release_ref.py | 240 ++++++++++++++++++++++++++ 7 files changed, 492 insertions(+), 15 deletions(-) create mode 100644 scripts/check_verified_release_ref.py create mode 100644 tests/test_verified_release_ref.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a8de793..b01dc7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,10 @@ jobs: exit 1 } python scripts/check_version_tag.py "${GITHUB_REF_NAME}" --require-changelog + - name: Require GitHub-verified release ref + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/check_verified_release_ref.py - name: Install shellcheck run: | sudo apt-get update diff --git a/CHANGELOG.md b/CHANGELOG.md index b08a42f..f27aa20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Added descriptive CLI help and a state-free `zeus --version` command backed by the package version. +- Required future release tags and their referenced commits to be signed and GitHub-verified + before the privileged publishing job can run. ## 0.3.0 diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 324667a..f062d92 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -2,8 +2,9 @@ Zeus releases are built from annotated version tags. Keep releases small, verify the repository locally first, and publish GitHub artifacts before considering -package-index distribution. Maintainers are encouraged to sign tags, but the -workflow does not cryptographically verify signer identity. +package-index distribution. Future releases require both the annotated tag and +its referenced commit to carry signatures that GitHub marks verified. The +historical v0.3.0 release predates this policy and remains unchanged. 1. Ensure CI is green on the commit to release. 2. Run the full local release gate: @@ -45,16 +46,18 @@ workflow does not cryptographically verify signer identity. python scripts/check_version_tag.py vX.Y.Z --require-changelog ``` -6. Create and push an annotated tag: +6. Confirm the release commit is signed and GitHub-verified, then create and push + a signed annotated tag: ```bash - git tag -a vX.Y.Z -m "Zeus vX.Y.Z" + git log --show-signature -1 + git tag -s vX.Y.Z -m "Zeus vX.Y.Z" git push origin vX.Y.Z ``` - If the maintainer has a configured signing identity, `git tag -s` may be - used instead. The release gate enforces an annotated tag but does not verify - the signature or signer identity. + A merely annotated or locally valid signature is insufficient: GitHub must + report both the tag and commit verification objects as `verified` with reason + `valid`. Configure the signing identity with GitHub before pushing the tag. 7. Confirm the GitHub release workflow completed and attached the generated `dist/*` artifacts plus `dist/SHA256SUMS.txt` to the GitHub Release. @@ -63,13 +66,17 @@ workflow does not cryptographically verify signer identity. `.github/workflows/release.yml` builds and checks distribution artifacts for `v*.*.*` tags, rejects lightweight tags and tags that do not match -`zeus.__version__`, and requires a matching changelog section. Its read-only -build job runs `make release-check`, including tests, source-and-branch coverage, -repository contracts, formatting, lint, type checks, Bandit, ShellCheck, package -build, wheel smoke verification, metadata checks, and checksum generation. Only -after that job succeeds does a separate privileged job download the checked -artifacts, verify their checksums, create GitHub artifact attestations, and attach -the assets to the GitHub Release. It intentionally does not publish to PyPI. +`zeus.__version__`, requires a matching changelog section, and calls GitHub's API +to require a GitHub-verified annotated tag and referenced commit. The checker +binds the tag target to the workflow's `GITHUB_SHA`, reads `GITHUB_TOKEN` only +from the environment, rejects redirects and malformed responses, and never logs +the token or raw response bodies. Its read-only build job runs +`make release-check`, including tests, source-and-branch coverage, repository +contracts, formatting, lint, type checks, Bandit, ShellCheck, package build, +wheel smoke verification, metadata checks, and checksum generation. Only after +that job succeeds does a separate privileged job download the checked artifacts, +verify their checksums, create GitHub artifact attestations, and attach the assets +to the GitHub Release. It intentionally does not publish to PyPI. Coverage configuration lives in `.coveragerc`, measures only the `zeus` package, and includes branch coverage. The threshold records the honest current baseline; diff --git a/scripts/check_verified_release_ref.py b/scripts/check_verified_release_ref.py new file mode 100644 index 0000000..9fa613e --- /dev/null +++ b/scripts/check_verified_release_ref.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Fail closed unless a release tag and its commit are GitHub-verified.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from collections.abc import Callable, Mapping +from typing import Any, TextIO +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import HTTPRedirectHandler, Request, build_opener + +API_ROOT = "https://api.github.com" +API_VERSION = "2026-03-10" +MAX_RESPONSE_BYTES = 128 * 1024 +REQUEST_TIMEOUT_SECONDS = 15 +REPOSITORY_RE = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9_.-]{0,99}/[A-Za-z0-9][A-Za-z0-9_.-]{0,99}\Z") +TAG_RE = re.compile(r"\Av[0-9][A-Za-z0-9._+-]{0,127}\Z") +SHA_RE = re.compile(r"\A[0-9a-f]{40}\Z") +VERIFIED_AT_RE = re.compile(r"\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\Z") + +JsonObject = dict[str, Any] +FetchJson = Callable[[str, str], JsonObject] + + +class ReleaseVerificationError(RuntimeError): + """A release reference did not satisfy the verification policy.""" + + +class _NoRedirects(HTTPRedirectHandler): + def redirect_request( + self, + req: Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +def _reject_constant(_value: str) -> None: + raise ValueError("non-standard JSON constant") + + +def _strict_object(pairs: list[tuple[str, Any]]) -> JsonObject: + result: JsonObject = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON field") + result[key] = value + return result + + +def _decode_json(body: bytes) -> JsonObject: + try: + value = json.loads( + body.decode("utf-8"), + object_pairs_hook=_strict_object, + parse_constant=_reject_constant, + ) + except (UnicodeError, ValueError): + raise ReleaseVerificationError("GitHub API returned malformed JSON") from None + if not isinstance(value, dict): + raise ReleaseVerificationError("GitHub API returned an invalid response shape") + return value + + +def _fetch_github_json(path: str, token: str, *, opener: Any = None) -> JsonObject: + request = Request( + f"{API_ROOT}{path}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "User-Agent": "zeus-release-ref-verifier", + "X-GitHub-Api-Version": API_VERSION, + }, + method="GET", + ) + active_opener = opener if opener is not None else build_opener(_NoRedirects()) + try: + with active_opener.open(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + if getattr(response, "status", None) != 200: + raise ReleaseVerificationError("GitHub API returned an unexpected status") + body = response.read(MAX_RESPONSE_BYTES + 1) + except ReleaseVerificationError: + raise + except (HTTPError, URLError, TimeoutError, OSError): + raise ReleaseVerificationError("GitHub API request failed") from None + + if len(body) > MAX_RESPONSE_BYTES: + raise ReleaseVerificationError("GitHub API response exceeded the size limit") + return _decode_json(body) + + +def _required_environment(environ: Mapping[str, str], name: str) -> str: + value = environ.get(name) + if not value: + raise ReleaseVerificationError(f"required workflow environment is missing: {name}") + return value + + +def _object(value: Any, label: str) -> JsonObject: + if not isinstance(value, dict): + raise ReleaseVerificationError(f"GitHub API returned malformed {label} metadata") + return value + + +def _sha(value: Any, label: str) -> str: + if not isinstance(value, str) or SHA_RE.fullmatch(value) is None: + raise ReleaseVerificationError(f"GitHub API returned malformed {label} SHA") + return value + + +def _require_verified(value: Any, label: str) -> None: + verification = _object(value, f"{label} verification") + verified_at = verification.get("verified_at") + if ( + verification.get("verified") is not True + or verification.get("reason") != "valid" + or not isinstance(verified_at, str) + or VERIFIED_AT_RE.fullmatch(verified_at) is None + ): + raise ReleaseVerificationError(f"{label} is not GitHub-verified") + + +def verify_release_ref( + environ: Mapping[str, str], + *, + fetch_json: FetchJson | None = None, +) -> tuple[str, str]: + token = _required_environment(environ, "GITHUB_TOKEN") + if token.strip() != token or any( + ord(character) < 33 or ord(character) > 126 for character in token + ): + raise ReleaseVerificationError("GITHUB_TOKEN is malformed") + + repository = _required_environment(environ, "GITHUB_REPOSITORY") + if REPOSITORY_RE.fullmatch(repository) is None: + raise ReleaseVerificationError("GITHUB_REPOSITORY is malformed") + + if _required_environment(environ, "GITHUB_EVENT_NAME") != "push": + raise ReleaseVerificationError("release verification requires a push event") + if _required_environment(environ, "GITHUB_REF_TYPE") != "tag": + raise ReleaseVerificationError("release verification requires a tag ref") + + tag_name = _required_environment(environ, "GITHUB_REF_NAME") + if TAG_RE.fullmatch(tag_name) is None: + raise ReleaseVerificationError("GITHUB_REF_NAME is not a safe release tag") + full_ref = _required_environment(environ, "GITHUB_REF") + if full_ref != f"refs/tags/{tag_name}": + raise ReleaseVerificationError("GITHUB_REF does not match GITHUB_REF_NAME") + + event_sha = _required_environment(environ, "GITHUB_SHA") + if SHA_RE.fullmatch(event_sha) is None: + raise ReleaseVerificationError("GITHUB_SHA is malformed") + + fetch = fetch_json if fetch_json is not None else _fetch_github_json + encoded_tag = quote(tag_name, safe="") + ref_path = f"/repos/{repository}/git/ref/tags/{encoded_tag}" + ref_data = _object(fetch(ref_path, token), "tag ref") + if ref_data.get("ref") != full_ref: + raise ReleaseVerificationError("GitHub tag ref does not match the workflow ref") + ref_object = _object(ref_data.get("object"), "tag ref") + if ref_object.get("type") != "tag": + raise ReleaseVerificationError("release tag is not annotated") + tag_object_sha = _sha(ref_object.get("sha"), "tag object") + + tag_path = f"/repos/{repository}/git/tags/{tag_object_sha}" + tag_data = _object(fetch(tag_path, token), "tag object") + if tag_data.get("tag") != tag_name or tag_data.get("sha") != tag_object_sha: + raise ReleaseVerificationError("GitHub tag object does not match the workflow ref") + _require_verified(tag_data.get("verification"), "release tag") + tag_target = _object(tag_data.get("object"), "tag target") + if tag_target.get("type") != "commit": + raise ReleaseVerificationError("release tag does not reference a commit") + commit_sha = _sha(tag_target.get("sha"), "tag target") + if commit_sha != event_sha: + raise ReleaseVerificationError("release tag commit does not match GITHUB_SHA") + + commit_path = f"/repos/{repository}/git/commits/{commit_sha}" + commit_data = _object(fetch(commit_path, token), "commit") + if commit_data.get("sha") != commit_sha: + raise ReleaseVerificationError("GitHub commit does not match the release tag") + _require_verified(commit_data.get("verification"), "release commit") + return tag_name, commit_sha + + +def main( + *, + environ: Mapping[str, str] | None = None, + fetch_json: FetchJson | None = None, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, +) -> int: + active_environment = os.environ if environ is None else environ + try: + tag_name, _commit_sha = verify_release_ref( + active_environment, + fetch_json=fetch_json, + ) + except ReleaseVerificationError as error: + print(f"release ref verification failed: {error}", file=stderr) + return 1 + + print( + f"Verified GitHub release ref {tag_name}: annotated tag and commit signatures are valid.", + file=stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/repo_check.sh b/scripts/repo_check.sh index f910bad..2224d4a 100755 --- a/scripts/repo_check.sh +++ b/scripts/repo_check.sh @@ -46,6 +46,7 @@ systemd/zeus-reconcile.service systemd/zeus-reconcile.timer scripts/test.sh scripts/check_version_tag.py +scripts/check_verified_release_ref.py scripts/wheel_smoke.sh scripts/generate_checksums.sh scripts/verify_real_hermes.sh diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index eb6cf00..6cf57fc 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -170,6 +170,7 @@ def test_publishable_repository_files_exist(self) -> None: "systemd/zeus-reconcile.service", "systemd/zeus-reconcile.timer", "scripts/repo_check.sh", + "scripts/check_verified_release_ref.py", "scripts/wheel_smoke.sh", "scripts/fresh_vps_verify.sh", "zeus/bundled_templates/__init__.py", @@ -1067,6 +1068,9 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: self.assertIn("attestations: write", workflow) self.assertIn("git cat-file -t", workflow) self.assertIn("Release tags must be annotated tags.", workflow) + self.assertIn("Require GitHub-verified release ref", workflow) + self.assertIn("python scripts/check_verified_release_ref.py", workflow) + self.assertIn("GITHUB_TOKEN: ${{ github.token }}", workflow) self.assertIn("actions/attest-build-provenance@", workflow) self.assertIn("dist/*.tar.gz", workflow) self.assertIn("dist/*.whl", workflow) @@ -1075,7 +1079,8 @@ def test_release_workflow_builds_tag_artifacts(self) -> None: self.assertIn("actions/download-artifact@", workflow) self.assertIn("cd dist && sha256sum -c SHA256SUMS.txt", workflow) self.assertIn("annotated version tags", release_docs) - self.assertIn("does not cryptographically verify signer identity", release_docs) + self.assertIn("GitHub-verified annotated tag", release_docs) + self.assertIn("v0.3.0 release predates", release_docs) self.assertIn("gh attestation verify", release_docs) self.assertIn('build_artifacts="${ZEUS_WHEEL_SMOKE_BUILD:-1}"', wheel_smoke) self.assertIn('fail "ZEUS_WHEEL_SMOKE_BUILD must be 0 or 1"', wheel_smoke) diff --git a/tests/test_verified_release_ref.py b/tests/test_verified_release_ref.py new file mode 100644 index 0000000..6283f40 --- /dev/null +++ b/tests/test_verified_release_ref.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import io +import runpy +import unittest +from pathlib import Path +from typing import Any +from urllib.error import URLError + +SCRIPT = runpy.run_path(str(Path("scripts/check_verified_release_ref.py"))) +ReleaseVerificationError = SCRIPT["ReleaseVerificationError"] +decode_json = SCRIPT["_decode_json"] +fetch_github_json = SCRIPT["_fetch_github_json"] +main = SCRIPT["main"] +verify_release_ref = SCRIPT["verify_release_ref"] + +TAG_OBJECT_SHA = "a" * 40 +COMMIT_SHA = "b" * 40 +TOKEN = "github-token-secret-sentinel" + + +def _environment() -> dict[str, str]: + return { + "GITHUB_TOKEN": TOKEN, + "GITHUB_REPOSITORY": "brainx/zeus", + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/tags/v0.4.0", + "GITHUB_REF_NAME": "v0.4.0", + "GITHUB_REF_TYPE": "tag", + "GITHUB_SHA": COMMIT_SHA, + } + + +def _responses() -> dict[str, dict[str, Any]]: + verified = { + "verified": True, + "reason": "valid", + "verified_at": "2026-07-22T01:23:45Z", + } + return { + "/repos/brainx/zeus/git/ref/tags/v0.4.0": { + "ref": "refs/tags/v0.4.0", + "object": {"type": "tag", "sha": TAG_OBJECT_SHA}, + }, + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}": { + "tag": "v0.4.0", + "sha": TAG_OBJECT_SHA, + "object": {"type": "commit", "sha": COMMIT_SHA}, + "verification": verified, + }, + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}": { + "sha": COMMIT_SHA, + "verification": verified, + }, + } + + +def _fetch_from(responses: dict[str, Any]) -> Any: + def fetch(path: str, _token: str) -> Any: + return responses[path] + + return fetch + + +class VerifiedReleaseRefTests(unittest.TestCase): + def test_accepts_verified_annotated_tag_bound_to_verified_event_commit(self) -> None: + responses = _responses() + calls: list[tuple[str, str]] = [] + + def fetch(path: str, token: str) -> dict[str, Any]: + calls.append((path, token)) + return responses[path] + + self.assertEqual( + ("v0.4.0", COMMIT_SHA), + verify_release_ref(_environment(), fetch_json=fetch), + ) + self.assertEqual(list(responses), [path for path, _token in calls]) + self.assertTrue(all(token == TOKEN for _path, token in calls)) + + def test_rejects_missing_or_malformed_workflow_environment_before_network(self) -> None: + cases = { + "missing token": ("GITHUB_TOKEN", ""), + "non-ascii token": ("GITHUB_TOKEN", "token-\N{SNOWMAN}"), + "wrong event": ("GITHUB_EVENT_NAME", "workflow_dispatch"), + "wrong ref type": ("GITHUB_REF_TYPE", "branch"), + "mismatched ref": ("GITHUB_REF", "refs/tags/v9.9.9"), + "unsafe repository": ("GITHUB_REPOSITORY", "brainx/zeus/extra"), + "unsafe tag": ("GITHUB_REF_NAME", "release/candidate"), + "malformed sha": ("GITHUB_SHA", "not-a-sha"), + } + for label, (name, value) in cases.items(): + with self.subTest(case=label): + environ = _environment() + environ[name] = value + + def unexpected_fetch(_path: str, _token: str) -> dict[str, Any]: + raise AssertionError("network fetch must not run") + + with self.assertRaises(ReleaseVerificationError): + verify_release_ref(environ, fetch_json=unexpected_fetch) + + def test_rejects_lightweight_or_mismatched_tag_ref(self) -> None: + for label, replacement in { + "lightweight": {"type": "commit", "sha": COMMIT_SHA}, + "malformed tag object": {"type": "tag", "sha": "not-a-sha"}, + }.items(): + with self.subTest(case=label): + responses = _responses() + responses["/repos/brainx/zeus/git/ref/tags/v0.4.0"]["object"] = replacement + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_unverified_tag_or_commit(self) -> None: + for endpoint in ( + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}", + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}", + ): + with self.subTest(endpoint=endpoint): + responses = _responses() + responses[endpoint]["verification"] = { + "verified": False, + "reason": "unsigned", + "verified_at": None, + } + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_tag_that_does_not_reference_event_commit(self) -> None: + responses = _responses() + responses[f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}"]["object"] = { + "type": "commit", + "sha": "d" * 40, + } + + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_rejects_malformed_api_shapes(self) -> None: + endpoints_and_payloads: tuple[tuple[str, Any], ...] = ( + ("/repos/brainx/zeus/git/ref/tags/v0.4.0", []), + ("/repos/brainx/zeus/git/ref/tags/v0.4.0", {"ref": "refs/tags/v0.4.0"}), + ( + f"/repos/brainx/zeus/git/tags/{TAG_OBJECT_SHA}", + {"tag": "v0.4.0", "sha": TAG_OBJECT_SHA}, + ), + ( + f"/repos/brainx/zeus/git/commits/{COMMIT_SHA}", + {"sha": COMMIT_SHA, "verification": {"verified": "true"}}, + ), + ) + for endpoint, payload in endpoints_and_payloads: + with self.subTest(endpoint=endpoint, payload=payload): + responses = _responses() + responses[endpoint] = payload + with self.assertRaises(ReleaseVerificationError): + verify_release_ref( + _environment(), + fetch_json=_fetch_from(responses), + ) + + def test_json_decoder_rejects_duplicates_constants_and_raw_body_disclosure(self) -> None: + secret_body = b'{"token":"raw-response-secret-sentinel",' + for body in ( + secret_body, + b'{"verification":NaN}', + b'{"sha":"one","sha":"two"}', + b"[]", + ): + with self.subTest(body=body): + with self.assertRaises(ReleaseVerificationError) as raised: + decode_json(body) + self.assertNotIn("raw-response-secret-sentinel", str(raised.exception)) + self.assertNotIn(body.decode("utf-8", errors="ignore"), str(raised.exception)) + + def test_network_failure_is_redacted_and_token_is_header_only(self) -> None: + class FailingOpener: + request: Any = None + assert_timeout: int | None = None + + def open(self, request: Any, *, timeout: int) -> Any: + self.request = request + self.assert_timeout = timeout + raise URLError(f"network failure containing {TOKEN}") + + opener = FailingOpener() + with self.assertRaises(ReleaseVerificationError) as raised: + fetch_github_json("/repos/brainx/zeus/git/ref/tags/v0.4.0", TOKEN, opener=opener) + + self.assertNotIn(TOKEN, str(raised.exception)) + self.assertNotIn(TOKEN, opener.request.full_url) + self.assertEqual(f"Bearer {TOKEN}", opener.request.get_header("Authorization")) + self.assertEqual(15, opener.assert_timeout) + + def test_main_emits_only_fixed_success_or_redacted_failure_messages(self) -> None: + responses = _responses() + stdout = io.StringIO() + stderr = io.StringIO() + status = main( + environ=_environment(), + fetch_json=lambda path, _token: responses[path], + stdout=stdout, + stderr=stderr, + ) + self.assertEqual(0, status) + self.assertEqual("", stderr.getvalue()) + self.assertIn("v0.4.0", stdout.getvalue()) + self.assertNotIn(TOKEN, stdout.getvalue()) + + stdout = io.StringIO() + stderr = io.StringIO() + + def fail_fetch(_path: str, _token: str) -> dict[str, Any]: + raise ReleaseVerificationError("GitHub API request failed") + + status = main( + environ=_environment(), + fetch_json=fail_fetch, + stdout=stdout, + stderr=stderr, + ) + self.assertEqual(1, status) + self.assertEqual("", stdout.getvalue()) + self.assertEqual( + "release ref verification failed: GitHub API request failed\n", + stderr.getvalue(), + ) + + +if __name__ == "__main__": + unittest.main() From 6b385ec66223fd61cec096d0a090a567614bb929 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:53:03 +0200 Subject: [PATCH 46/49] ci: fix Hermes workflow YAML parsing --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9133712..94c9b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,7 +108,8 @@ jobs: mkdir -p .tmp/real-hermes-evidence printf '%s\n' 'result=failed' 'failure_stage=ci_setup' > .tmp/real-hermes-evidence/summary.txt - name: Install pinned Hermes compatibility environment - run: python -m pip install --require-hashes --only-binary=:all: -r requirements-hermes-ci.txt + run: |- + python -m pip install --require-hashes --only-binary=:all: -r requirements-hermes-ci.txt - name: Install Zeus run: python -m pip install -e . - name: Check installed dependencies From 297457f37fc3a26dcaca2d7f9af1366d149d5000 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 04:04:54 +0200 Subject: [PATCH 47/49] fix(runtime): stabilize post-exec identity checks --- tests/test_gateway_runtime.py | 142 +++++++++++++++++++++++++++++++++- zeus/gateway_runtime.py | 55 ++++++++++--- 2 files changed, 186 insertions(+), 11 deletions(-) diff --git a/tests/test_gateway_runtime.py b/tests/test_gateway_runtime.py index 0f566a1..397fa15 100644 --- a/tests/test_gateway_runtime.py +++ b/tests/test_gateway_runtime.py @@ -63,6 +63,7 @@ def _fixture( fingerprints: list[str | None] | None = None, signals: list[tuple[int, signal.Signals]] | None = None, popen_factory=None, + startup_grace_seconds: float = 0.0, stop_grace_seconds: float = 0.0, kill_after_timeout: bool = False, ) -> tuple[GatewayRuntime, BotRecord, Path, str]: @@ -92,7 +93,7 @@ def read_fingerprint(pid: int) -> str | None: pid_alive_fn=lambda pid: True, cmdline_reader=lambda pid: [hermes, "-p", self.bot_id, "gateway", "run"], proc_start_fingerprint_reader=read_fingerprint, - startup_grace_seconds=0, + startup_grace_seconds=startup_grace_seconds, stop_grace_seconds=stop_grace_seconds, kill_after_timeout=kill_after_timeout, cleanup_process_group=False, @@ -237,6 +238,145 @@ def poll(self) -> int | None: self.assertNotIn(secret, repr(effect)) self.assertNotIn(secret, json.dumps(vars(effect), default=str, sort_keys=True)) + def test_launch_reports_exit_when_acknowledged_marker_process_is_already_dead(self) -> None: + class ExitedAfterAcknowledgment: + pid = self.pid + + def poll(self) -> int: + return 42 + + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + popen_factory=lambda *args, **kwargs: ExitedAfterAcknowledgment(), + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + marker, generation = self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=lambda *args, **kwargs: MarkerObservation( + "dead", payload=marker, reason="marker PID is dead" + ), + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("startup_exited", effect.outcome) + self.assertEqual(self.pid, effect.pid) + self.assertEqual(generation, effect.generation) + self.assertEqual(42, effect.returncode) + self.assertFalse(runtime.pid_marker_path(str(profile)).exists()) + + def test_launch_reports_exit_when_acknowledged_process_identity_disappears(self) -> None: + class ExitedAfterAcknowledgment: + pid = self.pid + + def poll(self) -> int: + return 42 + + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + popen_factory=lambda *args, **kwargs: ExitedAfterAcknowledgment(), + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=lambda *args, **kwargs: MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ), + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("startup_exited", effect.outcome) + self.assertEqual(self.pid, effect.pid) + self.assertIsNotNone(effect.generation) + self.assertEqual(42, effect.returncode) + self.assertFalse(runtime.pid_marker_path(str(profile)).exists()) + + def test_launch_retries_transient_post_exec_identity_before_registration(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + runtime, record, profile, _hermes = self._fixture( + Path(tmp), + startup_grace_seconds=0.1, + ) + launching = dataclasses.replace( + record, + status=BotStatus.starting, + pid=None, + desired_state=DesiredState.running, + desired_revision=1, + pending_operation_id=self.operation_id, + pending_action="start", + ) + marker, generation = self._schema3_marker( + runtime, + profile, + operation_id=self.operation_id, + desired_revision=launching.desired_revision, + ) + observations = iter( + ( + MarkerObservation( + "untrusted", reason="process start fingerprint is unavailable" + ), + MarkerObservation("live", payload=marker), + ) + ) + calls = 0 + + def marker_matcher(*args, **kwargs): + nonlocal calls + calls += 1 + return next(observations) + + effect = runtime.launch( + launching, + probe=None, + wait=False, + marker_matcher=marker_matcher, + ack_reader=lambda fd: b"1", + pipe_writer=lambda fd, payload: None, + ) + + self.assertEqual("running", effect.outcome) + self.assertEqual(generation, effect.generation) + self.assertEqual(2, calls) + def test_exact_generation_rejects_changed_correlation_and_process_identity(self) -> None: mutations = { "operation": lambda marker: marker.__setitem__("operation_id", "c" * 32), diff --git a/zeus/gateway_runtime.py b/zeus/gateway_runtime.py index deb989a..f828d27 100644 --- a/zeus/gateway_runtime.py +++ b/zeus/gateway_runtime.py @@ -69,6 +69,12 @@ class SignalResult(Enum): _MAX_EFFECT_TEXT = 512 +_TRANSIENT_POST_EXEC_MARKER_REASONS = frozenset( + { + "live gateway command is unavailable", + "process start fingerprint is unavailable", + } +) def _bounded_text(value: str) -> str: @@ -359,6 +365,7 @@ def launch( expected_fingerprint = str(marker_data["command_fingerprint"]) process: PopenLike | None = None generation: GatewayGeneration | None = None + marker_acknowledged = False payload_read = payload_write = ack_read = ack_write = -1 hooks = self._hooks() marker_lock = marker_lock or self.marker_publication_lock @@ -399,18 +406,28 @@ def launch( ack_read = -1 if acknowledgment != b"1": raise RuntimeError("gateway launcher did not acknowledge marker publication") + marker_acknowledged = True with marker_lock(record): - marker = marker_matcher( - record, - expected_fingerprint=expected_fingerprint, - expected_pid=process.pid, - require_live_command=True, - ) - generation = self.gateway_generation(marker) - if marker.kind != "live" or generation is None: - raise RuntimeError( - "gateway launcher ownership marker could not be verified: " + marker.reason + registration_deadline = time.monotonic() + max(self.startup_grace_seconds, 0) + while True: + marker = marker_matcher( + record, + expected_fingerprint=expected_fingerprint, + expected_pid=process.pid, + require_live_command=True, ) + generation = self.gateway_generation(marker) + if marker.kind == "live" and generation is not None: + break + if process.poll() is not None: + raise RuntimeError("gateway process exited before marker registration") + remaining = registration_deadline - time.monotonic() + if marker.reason not in _TRANSIENT_POST_EXEC_MARKER_REASONS or remaining <= 0: + raise RuntimeError( + "gateway launcher ownership marker could not be verified: " + + marker.reason + ) + time.sleep(min(0.01, remaining)) self._processes[record.bot_id] = process except BaseException as exc: for fd in (payload_read, payload_write, ack_read, ack_write): @@ -426,6 +443,7 @@ def launch( reason=str(exc), error_type=type(exc).__name__, ) + returncode = process.poll() cleanup_complete = self.cleanup_interrupted_launch( record, process, @@ -433,6 +451,23 @@ def launch( ) if not isinstance(exc, Exception): raise + if marker_acknowledged and returncode is not None and cleanup_complete: + if generation is None: + generation = GatewayGeneration( + operation_id=operation_id, + desired_revision=record.desired_revision, + pid=process.pid, + command_fingerprint=expected_fingerprint, + proc_start_fingerprint=None, + ) + return LaunchEffect( + "startup_exited", + pid=process.pid, + generation=generation, + reason="gateway exited during startup grace period", + returncode=returncode, + cleanup_complete=True, + ) return LaunchEffect( "registration_failed_clean" if cleanup_complete else "registration_failed_unknown", pid=None if cleanup_complete else process.pid, From a03ee99d19909db8b15217df09f504eb001ce7f7 Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 04:05:17 +0200 Subject: [PATCH 48/49] test(vps): isolate Python bootstrap fixtures --- tests/test_fresh_vps_script.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/test_fresh_vps_script.py b/tests/test_fresh_vps_script.py index 9165368..47db09c 100644 --- a/tests/test_fresh_vps_script.py +++ b/tests/test_fresh_vps_script.py @@ -147,8 +147,25 @@ def _pythonless_bootstrap_path(self) -> str: bootstrap_bin.mkdir() for name in ("curl", "sh"): (bootstrap_bin / name).symlink_to(self.stub_bin / name) - for name in ("dirname", "env", "seq", "tee", "uname"): - (bootstrap_bin / name).symlink_to(Path("/usr/bin") / name) + for name in ( + "bash", + "cat", + "chmod", + "date", + "dirname", + "env", + "ln", + "mkdir", + "rm", + "seq", + "sleep", + "tee", + "uname", + ): + executable = shutil.which(name) + if executable is None: + self.fail(f"required fixture executable is unavailable: {name}") + (bootstrap_bin / name).symlink_to(executable) sudo = bootstrap_bin / "sudo" sudo.write_text('#!/bin/bash\nset -eu\nexec "$@"\n', encoding="utf-8") @@ -170,7 +187,7 @@ def _pythonless_bootstrap_path(self) -> str: encoding="utf-8", ) apt_get.chmod(0o700) - return f"{bootstrap_bin}:/bin" + return str(bootstrap_bin) def _run( self, @@ -219,6 +236,12 @@ def test_missing_installer_digest_fails_before_download(self) -> None: self.assertFalse(self.installer_executed.exists()) self.assertFalse(any(INSTALLER_URL in record["argv"] for record in self._curl_records())) + def test_pythonless_bootstrap_path_contains_only_explicit_stubs(self) -> None: + bootstrap_path = self._pythonless_bootstrap_path() + + self.assertEqual([str(self.root / "bootstrap-bin")], bootstrap_path.split(os.pathsep)) + self.assertIsNone(shutil.which("python3", path=bootstrap_path)) + def test_missing_python_is_bootstrapped_before_private_evidence(self) -> None: bootstrap_path = self._pythonless_bootstrap_path() result = self._run( From 297470ea70127312e6512167e6afe57d8e1d52ed Mon Sep 17 00:00:00 2001 From: brainx <12695242+brainx@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:47:15 +0200 Subject: [PATCH 49/49] fix(readiness): preserve fixed-length response framing --- tests/test_readiness.py | 46 ++++++++++++++++++++++++++++++++++------- zeus/readiness.py | 10 ++++----- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index aa42d77..240c244 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -146,6 +146,36 @@ def log_message(self, format: str, *args: Any) -> None: self.assertEqual("ready", result.message) self.assertEqual(_HealthHandler.payload, result.payload) + def test_probe_once_accepts_fixed_length_http_11_response_without_eof(self) -> None: + health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") + release_connection = threading.Event() + + class KeepAliveFixedLengthHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + self.request.recv(4096) + self.request.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: application/json\r\n" + + f"Content-Length: {len(health_payload)}\r\n".encode("ascii") + + b"Connection: keep-alive\r\n\r\n" + + health_payload + ) + release_connection.wait(1) + + server, _thread = self._run_raw_server(KeepAliveFixedLengthHandler) + + try: + result = probe_once( + f"http://127.0.0.1:{server.server_address[1]}/health", + timeout_seconds=0.2, + ) + finally: + release_connection.set() + + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) + def test_probe_once_accepts_chunked_health_payload(self) -> None: class ChunkedHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -283,28 +313,28 @@ def log_message(self, format: str, *args: Any) -> None: self.assertEqual("readiness response exceeds size limit", result.message) self.assertIsNone(result.payload) - def test_probe_once_rejects_trailing_data_after_false_low_content_length(self) -> None: + def test_probe_once_does_not_read_past_fixed_content_length(self) -> None: health_payload = json.dumps(_HealthHandler.payload).encode("utf-8") - oversized_body = health_payload + b" " * MAX_READINESS_RESPONSE_BYTES + connection_data = health_payload + b" " * MAX_READINESS_RESPONSE_BYTES - class FalseLowContentLengthHandler(BaseHTTPRequestHandler): + class FixedLengthWithTrailingConnectionDataHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: self.send_response(200) self.send_header("content-length", str(len(health_payload))) self.end_headers() with contextlib.suppress(BrokenPipeError, ConnectionResetError): - self.wfile.write(oversized_body) + self.wfile.write(connection_data) def log_message(self, format: str, *args: Any) -> None: return - server, _thread = self._run_server(FalseLowContentLengthHandler) + server, _thread = self._run_server(FixedLengthWithTrailingConnectionDataHandler) result = probe_once(f"http://127.0.0.1:{server.server_port}/health") - self.assertFalse(result.ready) - self.assertEqual("readiness response exceeds size limit", result.message) - self.assertIsNone(result.payload) + self.assertTrue(result.ready) + self.assertEqual("ready", result.message) + self.assertEqual(_HealthHandler.payload, result.payload) def test_probe_once_requests_exactly_one_max_plus_one_bounded_read(self) -> None: class TrackingResponse: diff --git a/zeus/readiness.py b/zeus/readiness.py index a635da7..e87999b 100644 --- a/zeus/readiness.py +++ b/zeus/readiness.py @@ -99,12 +99,10 @@ def probe_once( return ReadinessResult(False, "readiness response has invalid content length") if any(length > MAX_READINESS_RESPONSE_BYTES for length in declared_lengths): return ReadinessResult(False, "readiness response exceeds size limit") - # HTTPResponse.read(amt) clamps amt to its internal Content-Length. The - # connection-close request lets an honest fixed-length response reach EOF, - # while clearing this CPython parser field exposes any understated trailing - # data to the same bounded MAX + 1 read. Chunk framing remains handled by - # HTTPResponse because chunked responses already have length=None. - response.length = None + # HTTPResponse.read(amt) clamps fixed-length responses to the declared + # Content-Length, so valid HTTP/1.1 keep-alive responses return without + # waiting for EOF. Responses without a fixed length remain bounded by amt; + # chunk framing is handled by HTTPResponse. body = response.read(MAX_READINESS_RESPONSE_BYTES + 1) if len(body) > MAX_READINESS_RESPONSE_BYTES: return ReadinessResult(False, "readiness response exceeds size limit")