From 3b5bfba0becf9356c9feec33b18e516e0293cfa7 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 19:03:19 +0200 Subject: [PATCH 1/3] feat: add product-owned operational reset --- .../operational-reset-v1-2.7.22.json | 16 + README.md | 17 + .../FORGE_OPERATIONAL_RESET_V1.md | 136 ++ ...ER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md | 8 + .../FORGE_OPERATIONAL_RESET_RUNBOOK.md | 124 ++ ...FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md | 29 + docs/roadmap/FORGE_OPERATIONS_CONSOLE_V1.md | 9 + docs/roadmap/forge-operations-console-v1.json | 15 + forge/__main__.py | 84 + forge/provider_security.py | 6 + forge/runtime/__init__.py | 3 +- forge/runtime/database.py | 214 ++- forge/runtime/operational_reset.py | 1460 +++++++++++++++++ knowledge/bootstrap/10_ROADMAP.md | 9 + product-version.json | 2 +- tests/test_operational_reset.py | 517 ++++++ 16 files changed, 2640 insertions(+), 9 deletions(-) create mode 100644 .github/product-version-operations/operational-reset-v1-2.7.22.json create mode 100644 docs/architecture/FORGE_OPERATIONAL_RESET_V1.md create mode 100644 docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md create mode 100644 docs/reports/FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md create mode 100644 forge/runtime/operational_reset.py create mode 100644 tests/test_operational_reset.py diff --git a/.github/product-version-operations/operational-reset-v1-2.7.22.json b/.github/product-version-operations/operational-reset-v1-2.7.22.json new file mode 100644 index 0000000..e05b67d --- /dev/null +++ b/.github/product-version-operations/operational-reset-v1-2.7.22.json @@ -0,0 +1,16 @@ +{ + "baseline_version": "2.7.21", + "classification_rationale": "user-authorized-operational-reset-v1", + "component": "product", + "event_lineage": "user-authorized-operational-reset-v1", + "expected_head": "03f9514e0f1e66a79ccb684c9fdbe51b28b88090", + "operation_id": "operational-reset-v1-2.7.22", + "policy_revision": "forge-bootstrap-release-cadence-v2", + "product": "forge", + "projection_paths": "product-version.json", + "release_class": "PATCH", + "requested_bump": "patch", + "requested_version": null, + "schema_version": "1", + "target_version": "2.7.22" +} diff --git a/README.md b/README.md index 2afbc07..d908f43 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,23 @@ The exact instance-ID comparison is consistency evidence, not a newly invented cryptographic peer identity; preflight reports the latter as `NOT_ASSERTED`. `CONFIGURED` is therefore never presented as `LIVE_READY`. +## Operational-history reset + +Forge schema 38 provides a bounded, product-owned maintenance service under +`forge server reset`. Start with the strictly read-only installed preview: + +```text +forge --data-root "/path/to/Forge Server" server reset preview +forge --data-root "/path/to/Forge Server" server reset status +``` + +The mutating `prepare`, `apply`, `resume`, `verify`, and `finish` commands are +documented in the [operator runbook](docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md). +They preserve instance/peer/provider/security/allocator bindings, use a verified +SQLite-consistent private backup, fence owning writers durably and reject +unknown tables/files/symlinks. They do not reset EP, create a Mission, rotate or +print credentials, or provide arbitrary SQL/table deletion. + ## Deterministic runtime health evaluation Forge now provides a library-only, read-only health evaluator in diff --git a/docs/architecture/FORGE_OPERATIONAL_RESET_V1.md b/docs/architecture/FORGE_OPERATIONAL_RESET_V1.md new file mode 100644 index 0000000..7a33f77 --- /dev/null +++ b/docs/architecture/FORGE_OPERATIONAL_RESET_V1.md @@ -0,0 +1,136 @@ +# Forge operational-history reset V1 + +**Status:** implemented product service and local administration CLI; source +qualification is recorded separately from protected delivery, release and +installed use. **Scope:** Forge-owned state only. This is neither a factory +reset nor a general SQL/database administration surface. + +## Contract and boundary + +The profile `forge-operational-history-v1` creates a new dataset generation +while retaining the Runtime Instance, project/EP binding, operator/security +authority, provider configuration, credential references, identifier +high-watermarks and anti-replay evidence. It never reads or writes Engineering +Platform CENTRAL and never invokes EP's CLI. + +The same immutable operation identity binds: + +- exact Runtime Instance ID, canonical database path/binding, marker and schema; +- plan, request, effect-set and meaningful-source-revision digests; +- real locally bound operator and current `SECURITY_APPROVAL` plus + `OWNER_PROGRAMME_AUTHORIZATION` authority; +- installed implementation digest and Git source revision when genuinely + available (`UNAVAILABLE_IN_INSTALLED_PACKAGE` is explicit otherwise); +- exact acknowledged referential findings wholly inside the purge set; +- one verified recovery backup and its manifest digest; +- dataset generation before/after, artifact steps and final verification. + +There is no `--force`, arbitrary table list, peer SQL or automatic rollback. +The stable machine envelope is `contract_version=operational-reset-v1` and +contains no bearer values or raw credential verifiers. + +## Schema-owned data classification + +Schema version 38 owns this complete table mapping. Any additional application +table blocks reset until this contract is updated. SQLite indexes, triggers and +system objects are validated separately and are not purge candidates. + +| Category | Tables | Reset treatment | +| --- | --- | --- | +| `INSTALLATION_AND_CONFIGURATION` | `runtime_metadata`, `execution_host_peer_configuration`, `planning_provider_security_config`, `planning_provider_external_session_config` | Preserve exactly. Credential references stay references; no Keychain read/dump/rotation. | +| `SECURITY_AND_AUTHORITY_LEDGER` | `installation_operator_binding`, `installation_operator_audit`, `governance_authority`, `governance_capability_grants`, `governance_decisions`, both provider-security/session audit tables, `planning_provider_generation_permits`, `mission_id_allocations` | Preserve. Pending/transport-committed provider permits become non-executable `INVALIDATED_BY_OPERATIONAL_RESET`; allocator history is unchanged. | +| `OPERATIONAL_HISTORY` | Mission state/lifecycle/recommendation/review/intake/amendment/evidence tables; Action derivation/result/reattempt/closure tables; execution context/receipt/integration/delegation/scheduler/correlation/exchange tables; token-preflight tables; bootstrap portfolio snapshot; Forge operational logs | Tombstone replay-sensitive identities, archive eligible external evidence, then delete in the schema-owned child-first transaction. | +| `DERIVED_CACHE_OR_PROJECTION` | `mission_runtime_projections`, `planning_state`, `dispatcher_state` | Delete from the active generation. | +| `MAINTENANCE_AUDIT` | `operational_reset_state`, `operational_reset_operations`, `operational_reset_audit`, `operational_reset_tombstones`, `operational_reset_artifact_steps` | Preserve; excluded from Mission/run/usage views. | +| `UNKNOWN_OR_UNSUPPORTED` | Anything else | Destructive progression blocked. | + +Record-level rules matter. `planning_provider_generation_permits` remain in the +security ledger but lose executable state. Mission-bound token and reattempt +records are removed only after their receipt/request/authorization identities +become durable reset tombstones. Old correlation, submission, Action and receipt +identities cannot be reintroduced through owning writers. + +## External data classification + +The database, WAL/SHM and owning lock are SQLite/runtime control, not domain +history. `instance/runtime-instance.json`, prior `backups/`, locks and strict +installation/qualification artifacts are preserved. In particular +`artifacts/controlled-installation-*` and entries under +`artifacts/{installation,qualification}/` retain installation/artifact/product +qualification provenance. + +Only product-shaped operational paths are eligible: + +- `artifacts/{operational,runtime,missions}/...`: archive in the operation's + recovery backup, verify digest, then remove from active ingest; +- recognized `.json`, `.jsonl` and `.journal` entries under `journals/`, and + `.log`/`.jsonl` entries under `logs/`: archive, verify and remove; +- regular files under `cache/`: remove as derived cache, without claiming them + as recovery evidence. + +Unknown artifacts, instance files, top-level files, special files, unreadable +entries and every symlink block. The service never follows a symlink and never +touches repositories, source, documents, other workspaces or other instances. + +## State machine, fencing and crash recovery + +```text +read-only PREVIEW + -> PREPARED (durable writer fence + exact authority/request binding) + -> BACKUP_VERIFIED + -> DATABASE_APPLIED (one domain transaction) + -> APPLIED (every filesystem step reconciled) + -> VERIFIED + -> COMPLETED (writer fence released) +``` + +Normal owning `RuntimeDatabase` opens fail while maintenance is active. Every +already-open connection is fenced by schema triggers that directly consult the +durable state before a write, without requiring a connection-local UDF during +normal operation. The reset transaction temporarily removes only its exact +delete/update fence triggers under `BEGIN IMMEDIATE` and restores them before +commit. A process-local OS lock is therefore not the crash boundary. The +same-root mutation lock serializes runtime service ticks and maintenance +commands. A raw/peer SQL writer is unsupported and does not become a product +interface. + +`resume` rereads the same operation. A failure before the SQLite commit leaves +the whole operational population. A failure after commit continues forward. +Each external item has a durable step; if removal happened before its step write, +resume accepts absence only when the verified archive copy still matches. +Neither resume nor uncertainty creates another reset. + +## Backup and recovery + +Prepare uses SQLite's online backup API after the durable writer fence, then +converts the isolated image to a self-contained non-WAL snapshot. The private +`0700` operation directory contains a `0600` database, marker, archived external +artifacts and manifest. The manifest binds source instance/schema/version, +operation/plan/request, source counts, exact known FK findings, inclusions, +exclusions and per-file SHA-256. It excludes Keychain, provider login/session, +venv/tooling, cache, locks and earlier backups. + +Verification copies the database into a temporary non-active root, performs +`quick_check`, reconstructs the meaningful source revision and compares external +digests. A known operational orphan remains honestly listed in the backup; the +backup is byte/semantic recovery evidence, not falsely labelled FK-clean. + +Forward reconciliation is the normal recovery. Do not copy the old database +back over a running instance and do not use it to undo later revocations, +budgets or external effects. A physical restore requires a separately authorized +stopped-root recovery decision; it must first reconcile security/external state. + +## Postconditions + +Verify requires `integrity_check=ok`, `quick_check=ok`, zero foreign-key errors, +zero rows in every operational/projection table, no active pre-reset provider +permit, no active classified operational file, exact preserved-table digests, +same Runtime Instance/marker/peer binding, the expected dataset generation and +a still-verifiable recovery backup. Only its exact verification digest releases +maintenance. “Clean” therefore means no old active/executable operational +population; security tombstones, allocator history and maintenance audit remain. + +Repository document `missions/MISSION-0003.md` is reported as a historical +repository-document namespace. It is not an active Runtime Mission. The reset +does not rename it, fabricate a Mission, reset the allocator or promise that the +next runtime identity will display `MISSION-0003`. diff --git a/docs/architecture/FORGE_SERVER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md b/docs/architecture/FORGE_SERVER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md index 50fff83..9b4765a 100644 --- a/docs/architecture/FORGE_SERVER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md +++ b/docs/architecture/FORGE_SERVER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md @@ -8,6 +8,14 @@ explicit, guarded data operations; it does not authorize their execution now. The [scoped roadmap](../roadmap/FORGE_OPERATIONS_CONSOLE_V1.md) and [documentary DAG](../roadmap/forge-operations-console-v1.json) track this work. +> **Bounded delivered slice:** the separate +> [Forge operational-history reset V1](FORGE_OPERATIONAL_RESET_V1.md) now +> implements and qualifies the owning local application service/CLI needed for +> clean-state preparation. This does not implement FC-EXPORT, FC-IMPORT, +> FC-RELOCATE, FC-VACUUM, the Console/API/daemon or full FC-STATE/FOC-Q. Its +> protected delivery, release, installation and live preview remain evidence +> dimensions separate from source implementation. + ## 1. Locality and ownership ```text diff --git a/docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md b/docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md new file mode 100644 index 0000000..bcfa65e --- /dev/null +++ b/docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md @@ -0,0 +1,124 @@ +# Forge operational reset operator runbook + +This runbook uses only the installed Forge-owned CLI. Replace the placeholders +with values returned by the immediately preceding JSON receipt. Keep the Forge +data root explicit for every command. Exit code `0` means the command produced +its declared readback; exit code `1` means no success may be inferred. + +> **Production safety:** in the current preparation assignment run `preview` +> only against the selected live root. Every command from `prepare` through +> `finish` changes maintenance/backup/reset state and is destructive or enables +> destructive continuation. Use those commands only during the later explicitly +> authorized clean-CENTRAL preflight. + +## 1. Read-only preview + +```text +forge --data-root "/absolute/Forge Server" server reset preview +``` + +Record `target.instance_id`, `target.database_identity`, `schema_version`, +`dataset_generation`, `plan_digest`, `relevant_revision_digest`, counts, +blockers, exact `required_fk_acknowledgements` (under `details`) and the +preserved-bindings digest. A schema-37 installation may be inventoried, but +reports `SCHEMA_MIGRATION_REQUIRED`; preview never migrates it. + +## 2. Enter maintenance and create/verify backup — MUTATING + +```text +forge --data-root "/absolute/Forge Server" server reset prepare \ + --operation-id "forge-reset-" \ + --plan-digest "sha256:" +``` + +For a specifically reviewed operational-only FK finding, repeat its exact ID: + +```text + --acknowledge-operational-fk "sha256:" +``` + +There is no broad ignore option. Findings reaching preserved or unknown data +block. Prepare authenticates the installed local operator and current Security/ +programme capabilities, activates the durable writer fence, produces the +consistent backup and returns `request_digest` plus verified `backup.digest`. +If backup creation fails, leave the instance in maintenance and use `status`; +do not start another operation. + +## 3. Apply the reset — DESTRUCTIVE + +```text +forge --data-root "/absolute/Forge Server" server reset apply \ + --operation-id "forge-reset-" \ + --plan-digest "sha256:" \ + --request-digest "sha256:" \ + --backup-digest "sha256:" +``` + +## 4. Verify — MAINTENANCE REMAINS ACTIVE + +```text +forge --data-root "/absolute/Forge Server" server reset verify \ + --operation-id "forge-reset-" \ + --plan-digest "sha256:" \ + --request-digest "sha256:" \ + --backup-digest "sha256:" +``` + +Require `state=VERIFIED`, all operational counts zero, all integrity checks +green, unchanged target/bindings and a non-empty `verification_digest`. + +## 5. Interrupted operation + +```text +forge --data-root "/absolute/Forge Server" server reset status \ + --operation-id "forge-reset-" + +forge --data-root "/absolute/Forge Server" server reset resume \ + --operation-id "forge-reset-" \ + --plan-digest "sha256:" \ + --request-digest "sha256:" \ + --backup-digest "sha256:" +``` + +Resume reconciles the same operation through verification. If failure happened +before a backup digest exists, omit `--backup-digest`; the same PREPARED operation +finishes its backup first. Do not automatically restore or create a replacement +operation. + +## 6. Release maintenance — MUTATING + +```text +forge --data-root "/absolute/Forge Server" server reset finish \ + --operation-id "forge-reset-" \ + --verification-digest "sha256:" +``` + +An unapplied PREPARED/BACKUP_VERIFIED operation may instead be safely cancelled: + +```text +forge --data-root "/absolute/Forge Server" server reset finish \ + --operation-id "forge-reset-" \ + --cancel-before-apply +``` + +Cancellation is forbidden after database apply. After successful coordinated +Forge and EP verification, perform the separately authorized read-only local +HTTP authentication/preflight; it must not create a Mission or submission. + +## Coordinated later clean-CENTRAL preflight + +1. Run installed read-only preview for Forge and EP; bind both target and plan + digests to one external maintenance reference. +2. Resolve the historical document/runtime Mission namespace decision before T0 + if the normal allocator cannot produce the desired display label. +3. Prepare both owning operations; confirm both durable maintenance states and + both verified backups. +4. Recheck both exact plans under their writer fences. +5. Apply each owning reset sequentially. If either fails, keep both products in + maintenance and resume the same owning operation; never auto-resume the first. +6. Verify both empty operational generations, preserved identities/peer binding, + security/allocator state and backups. +7. Finish both only after the joint coordinator has recorded both verification + digests. Then run the non-generating HTTP authentication check. + +This recipe is intentionally not executed by the current implementation task. diff --git a/docs/reports/FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md b/docs/reports/FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md new file mode 100644 index 0000000..57f8395 --- /dev/null +++ b/docs/reports/FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md @@ -0,0 +1,29 @@ +# Forge operational reset delivery status + +The bounded Forge-owned operational-history reset is implemented in source with +schema 38 and the `forge server reset` command family. It is independent from +the planned Operations Console, full export/import/relocation product and Forge +Server daemon. Those broader nodes remain planned. + +Status dimensions are deliberately separate: + +| Dimension | State | +| --- | --- | +| Architecture/contract | IMPLEMENTED in `FORGE_OPERATIONAL_RESET_V1.md` | +| Application service and CLI | IMPLEMENTED | +| Isolated synthetic positive/negative/crash qualification | PASSED locally: 23 reset regressions within `scripts/validate.sh` | +| Full repository source gate | PASSED locally: 694 tests, product-version and offline-projection validation | +| Independent protected-candidate review/checks | NOT YET EVIDENCED | +| Protected review/merge | NOT YET EVIDENCED in this source branch | +| Stable release publication | NOT YET EVIDENCED | +| Installed artifact | NOT YET EVIDENCED | +| Live selected-root preview | NOT PERFORMED by this source-delivery record | +| Production reset | NOT AUTHORIZED / NOT PERFORMED | +| Mission 3 | NOT STARTED | + +The intended stable patch release is `2.7.22` under +`forge-bootstrap-release-cadence-v2`: protected exact-head review/merge, one +clean wheel/sdist build, full qualification bound to their SHA-256 values, +draft GitHub release, PyPI publication, registry digest readback, installed-wheel +smoke, durable publication/cleanup receipt and completion. No branch-local test +or wheel build is presented as published or installed evidence. diff --git a/docs/roadmap/FORGE_OPERATIONS_CONSOLE_V1.md b/docs/roadmap/FORGE_OPERATIONS_CONSOLE_V1.md index bb6dde3..e9ab10c 100644 --- a/docs/roadmap/FORGE_OPERATIONS_CONSOLE_V1.md +++ b/docs/roadmap/FORGE_OPERATIONS_CONSOLE_V1.md @@ -118,6 +118,15 @@ FOC-Q must qualify all packages, including restart and interrupted data operatio stale snapshots, non-portable credentials, unchanged cumulative budgets, EP-absent/upgraded two-host layouts, refresh load and truthful timeout enforcement. +### Bounded operational-reset slice + +`FORGE_OPERATIONAL_RESET_V1` is implemented as an owning application service +and thin `forge server reset` CLI, with schema-owned preserve/purge rules, +durable writer fencing, verified recovery backup and interrupted-operation +reconciliation. It advances only the reset-specific clean-state preparation +slice. FC-STATE, FOC-5, FOC-Q and every Console node remain PLANNED until their +broader evidence is delivered; the reset adds no Console/API prerequisite. + ## EP parity, logging, five languages and discovery/pairing The [admin-parity contract](../architecture/FORGE_CONSOLE_EP_PARITY_AND_PAIRING_V1.md) diff --git a/docs/roadmap/forge-operations-console-v1.json b/docs/roadmap/forge-operations-console-v1.json index c0062ae..ca4d341 100644 --- a/docs/roadmap/forge-operations-console-v1.json +++ b/docs/roadmap/forge-operations-console-v1.json @@ -272,6 +272,21 @@ } ], "configuration_contract": "docs/architecture/FORGE_SERVER_CONFIGURATION_AND_DATA_OPERATIONS_V1.md", + "delivered_bounded_slices": [ + { + "id": "FORGE_OPERATIONAL_RESET_V1", + "status": "IMPLEMENTED_LOCAL_SOURCE_QUALIFIED_PROTECTED_REVIEW_PENDING", + "contract": "docs/architecture/FORGE_OPERATIONAL_RESET_V1.md", + "runbook": "docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md", + "refines": [ + "FC-STATE", + "FOC-5" + ], + "completes_parent_node": false, + "requires_console_or_server_daemon": false, + "production_reset_authorized": false + } + ], "configuration_work_packages": [ { "id": "FC-CODEX", diff --git a/forge/__main__.py b/forge/__main__.py index eb20d40..8481723 100644 --- a/forge/__main__.py +++ b/forge/__main__.py @@ -93,6 +93,33 @@ def main(argv: list[str] | None = None) -> int: server_commands = server.add_subparsers(dest="server_command", required=True) server_commands.add_parser("init", help="create and validate the configured Forge data root") server_commands.add_parser("status", help="print read-only runtime status as JSON") + reset = server_commands.add_parser("reset", help="operate the Forge-owned operational-history reset") + reset_commands = reset.add_subparsers(dest="reset_command", required=True) + reset_commands.add_parser("preview", help="inspect a read-only reset plan") + prepare_reset = reset_commands.add_parser("prepare", help="authorize maintenance and create a verified backup") + prepare_reset.add_argument("--operation-id", required=True) + prepare_reset.add_argument("--plan-digest", required=True) + prepare_reset.add_argument("--acknowledge-operational-fk", action="append", default=[]) + for name, help_text in ( + ("apply", "apply the authorized destructive reset"), + ("verify", "verify reset integrity and preserved bindings"), + ): + reset_action = reset_commands.add_parser(name, help=help_text) + reset_action.add_argument("--operation-id", required=True) + reset_action.add_argument("--plan-digest", required=True) + reset_action.add_argument("--request-digest", required=True) + reset_action.add_argument("--backup-digest", required=True) + reset_status = reset_commands.add_parser("status", help="read durable reset status") + reset_status.add_argument("--operation-id") + resume_reset = reset_commands.add_parser("resume", help="reconcile the same interrupted reset operation") + resume_reset.add_argument("--operation-id", required=True) + resume_reset.add_argument("--plan-digest", required=True) + resume_reset.add_argument("--request-digest", required=True) + resume_reset.add_argument("--backup-digest") + finish_reset = reset_commands.add_parser("finish", help="safely leave durable maintenance") + finish_reset.add_argument("--operation-id", required=True) + finish_reset.add_argument("--verification-digest") + finish_reset.add_argument("--cancel-before-apply", action="store_true") subparsers.add_parser("status", help="print read-only runtime status as JSON") execution_host = subparsers.add_parser("execution-host", help="manage the selected Execution Host peer") execution_host_commands = execution_host.add_subparsers(dest="execution_host_command", required=True) @@ -140,6 +167,63 @@ def main(argv: list[str] | None = None) -> int: database.close() elif args.command == "status" or (args.command == "server" and args.server_command == "status"): print(json.dumps(_status(args.data_root), sort_keys=True)) + elif args.command == "server" and args.server_command == "reset": + from .runtime.operational_reset import ForgeOperationalResetService, OperationalResetError + service = ForgeOperationalResetService(args.data_root) + try: + if args.reset_command == "preview": + result = service.preview() + elif args.reset_command == "prepare": + result = service.prepare( + operation_id=args.operation_id, + expected_plan_digest=args.plan_digest, + acknowledge_operational_fk=args.acknowledge_operational_fk, + ) + elif args.reset_command in {"apply", "verify"}: + result = getattr(service, args.reset_command)( + operation_id=args.operation_id, plan_digest=args.plan_digest, + request_digest=args.request_digest, backup_digest=args.backup_digest, + ) + elif args.reset_command == "status": + result = service.status(operation_id=args.operation_id) + elif args.reset_command == "resume": + result = service.resume( + operation_id=args.operation_id, plan_digest=args.plan_digest, + request_digest=args.request_digest, backup_digest=args.backup_digest, + ) + else: + result = service.finish( + operation_id=args.operation_id, verification_digest=args.verification_digest, + cancel_before_apply=args.cancel_before_apply, + ) + print(json.dumps(service.operator_envelope(args.reset_command, result), sort_keys=True)) + except (OperationalResetError, OSError, sqlite3.Error, PermissionError, ValueError) as error: + finding = {"code": type(error).__name__, "message": str(error)} + failure = { + "operation_id": getattr(args, "operation_id", None), + "state": "ERROR", "allowed": False, "blockers": [finding], + "error": str(error), + } + try: + envelope = service.operator_envelope(args.reset_command, failure) + except Exception: + # Even an unreadable target retains the coordinator's stable + # top-level contract; no success or target identity is guessed. + envelope = { + "contract_version": "operational-reset-v1", "product": "forge", + "command": args.reset_command, "operation_id": failure["operation_id"], + "state": "ERROR", "allowed": False, + "target": { + "instance_id": None, "database_path": str(service.database_path), + "database_identity": None, "schema_version": None, + }, + "profile": "forge-operational-history-v1", "dataset_generation": None, + "plan_digest": None, "relevant_revision_digest": None, + "backup": None, "counts": {}, "blockers": [finding], "integrity": {}, + "preserved_bindings_digest": None, "details": failure, + } + print(json.dumps(envelope, sort_keys=True)) + return 1 elif args.command == "execution-host": try: if args.execution_host_command == "credential-access": diff --git a/forge/provider_security.py b/forge/provider_security.py index ce2618c..3eb2884 100644 --- a/forge/provider_security.py +++ b/forge/provider_security.py @@ -241,6 +241,12 @@ def _acquire_generation_permit(self, expected_policy, policy_digest, request_dig try: actual=self.invocation_policy(expected_policy.provider_id) if not self._same_policy(actual,expected_policy): raise PermissionError('canonical G011 policy changed before generation permit') + if connection.execute( + "SELECT 1 FROM operational_reset_tombstones " + "WHERE record_id=? AND record_kind IN ('generation_request_digest','token_request_digest')", + (request_digest,), + ).fetchone() is not None: + raise PermissionError('generation request identity was retired by an operational reset') connection.execute("INSERT INTO planning_provider_generation_permits VALUES (?,?,?,?,?,?,?,?)",(permit_id,actual.provider_id,actual.version,policy_digest,request_digest,'PENDING',now,now)) self.db._append_operational_event( component='forge_planning_provider', level='INFO', diff --git a/forge/runtime/__init__.py b/forge/runtime/__init__.py index b18c3f4..c06c039 100644 --- a/forge/runtime/__init__.py +++ b/forge/runtime/__init__.py @@ -9,6 +9,7 @@ RuntimeDatabase, RuntimeDatabaseError, RuntimeIntegrityError, + RuntimeMaintenanceActive, ) from .bootstrap import ( RUNTIME_INSTANCE_VERSION, @@ -49,7 +50,7 @@ __all__ = [ "BootstrapMissionRunner", "MissionRunnerError", "RuntimePromptFactory", "ForgeRuntimeService", "RuntimeServiceTick", - "RUNTIME_SCHEMA_VERSION", "RuntimeDatabase", "RuntimeDatabaseError", "RuntimeIntegrityError", "RuntimeDecisionEvidenceReference", "RuntimeEvidence", + "RUNTIME_SCHEMA_VERSION", "RuntimeDatabase", "RuntimeDatabaseError", "RuntimeIntegrityError", "RuntimeMaintenanceActive", "RuntimeDecisionEvidenceReference", "RuntimeEvidence", "RUNTIME_INSTANCE_VERSION", "RUNTIME_INITIALIZATION_VERSION", "RuntimeBootstrap", "RuntimeIdentity", "RuntimeInstance", "RuntimeLocation", "RuntimePlacement", "RuntimeRecovery", "RuntimeResolutionError", "RuntimeResolver", "repository_identity", "repository_uuid", "DataRootError", "DataRootResolver", "RUNTIME_DIRECTORIES", "HEALTH_SCHEMA_REVISION", "CapabilityReadiness", "CheckApplicability", "CheckPurpose", "CheckState", "HealthCheckDefinition", "HealthEvaluation", "HealthIdentity", "HealthObservation", "HealthState", "LivenessEvaluation", "LivenessState", "ObservationFreshness", "ObservationState", "ReadinessState", "evaluate_health", ] diff --git a/forge/runtime/database.py b/forge/runtime/database.py index b14b533..2f25b71 100644 --- a/forge/runtime/database.py +++ b/forge/runtime/database.py @@ -22,7 +22,7 @@ canonical_repository_root, repository_identity, repository_uuid) -RUNTIME_SCHEMA_VERSION = 37 +RUNTIME_SCHEMA_VERSION = 38 _REQUIRED_METADATA = frozenset(( "schema_version", "migration_version", "forge_version", "created_at", "last_migration", "integrity_status", @@ -40,6 +40,12 @@ "action_derivation_evidence_sets", "mission_amendments", "action_derivation_canary_closures", "execution_host_bindings", "execution_host_exchange_audit", "execution_host_peer_configuration", "forge_operational_logs", + "operational_reset_state", "operational_reset_operations", "operational_reset_audit", + "operational_reset_tombstones", "operational_reset_artifact_steps", +)) +_OPERATIONAL_RESET_TABLES = frozenset(( + "operational_reset_state", "operational_reset_operations", "operational_reset_audit", + "operational_reset_tombstones", "operational_reset_artifact_steps", )) _TOKEN_PREFLIGHT_FAILURE_FIELDS = frozenset(( "failure_id", "mission_id", "provider_id", "occurred_at", "main_head", "policy_digest", @@ -105,6 +111,10 @@ class RuntimeIntegrityError(RuntimeDatabaseError): """The runtime database is inconsistent and must not be used.""" +class RuntimeMaintenanceActive(RuntimeDatabaseError): + """Normal runtime mutation is fenced by a durable maintenance operation.""" + + def _json_value(value: Any) -> Any: if isinstance(value, Enum): return value.value @@ -238,10 +248,22 @@ def __init__(self, workspace_root: Path | str = ".", *, path: Path | str | None self._action_derivation_write_state = {"permitted": False} self._action_derivation_reattempt_write_state = {"permitted": False} self._action_derivation_canary_closure_write_state = {"permitted": False} + self._maintenance_write_state = {"permitted": False} try: self._configure() - while self._connection.execute("PRAGMA user_version").fetchone()[0] != RUNTIME_SCHEMA_VERSION: - self._migrate(forge_version) + self._maintenance_write_state["permitted"] = True + try: + while self._connection.execute("PRAGMA user_version").fetchone()[0] != RUNTIME_SCHEMA_VERSION: + self._migrate(forge_version) + finally: + self._maintenance_write_state["permitted"] = False + maintenance = self._connection.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone() + if maintenance is not None and maintenance[0] is not None: + raise RuntimeMaintenanceActive( + "Forge runtime is fenced by operational reset maintenance: " + str(maintenance[0]) + ) self._initialize_runtime_identity() self.validate_integrity() except Exception: @@ -278,6 +300,7 @@ def _configure(self) -> None: self._connection.create_function("forge_action_derivation_write_permitted", 0, lambda: int(self._action_derivation_write_state["permitted"])) self._connection.create_function("forge_action_derivation_reattempt_write_permitted", 0, lambda: int(self._action_derivation_reattempt_write_state["permitted"])) self._connection.create_function("forge_action_derivation_canary_closure_write_permitted", 0, lambda: int(self._action_derivation_canary_closure_write_state["permitted"])) + self._connection.create_function("forge_maintenance_write_permitted", 0, lambda: int(self._maintenance_write_state["permitted"])) def _insert_governance_grant(self, grant_id: str, installation_id: str, operator_id: str, capability: str, provenance: str, digest: str, occurred_at: str) -> None: @@ -480,6 +503,54 @@ def _require_action_derivation_result_structure(self) -> None: or f"before {operation}" not in sql or fragment not in sql): raise RuntimeIntegrityError(f"durable action-derivation result migration found incompatible {name} trigger") + def _require_operational_reset_structure(self) -> None: + required_columns = { + "operational_reset_state": { + "singleton", "dataset_generation", "active_operation_id", "state", "updated_at", + }, + "operational_reset_operations": { + "operation_id", "runtime_id", "profile", "state", "actor_reference", "authority_digest", + "plan_digest", "request_digest", "backup_digest", "backup_reference", "relevant_revision", + "source_revision", "implementation_digest", "schema_version", "policy_version", + "generation_before", "generation_after", "created_at", "updated_at", "document", + }, + "operational_reset_audit": {"audit_id", "operation_id", "event", "occurred_at", "document"}, + "operational_reset_tombstones": { + "record_kind", "record_id", "source_digest", "operation_id", "retired_at", + }, + "operational_reset_artifact_steps": { + "operation_id", "relative_path", "classification", "source_digest", "size_bytes", "state", + "backup_relative_path", "updated_at", + }, + } + for table, expected in required_columns.items(): + actual = {str(row["name"]) for row in self._connection.execute(f'PRAGMA table_info("{table}")')} + if actual != expected: + raise RuntimeIntegrityError(f"operational reset migration found incompatible {table} table") + state_rows = self._connection.execute( + "SELECT singleton,dataset_generation FROM operational_reset_state" + ).fetchall() + if len(state_rows) != 1 or int(state_rows[0]["singleton"]) != 1 or int(state_rows[0]["dataset_generation"]) < 0: + raise RuntimeIntegrityError("operational reset durable state is inconsistent") + for table in sorted(_TABLES): + prefix = "operational_reset_authorize" if table in _OPERATIONAL_RESET_TABLES else "operational_reset_block" + for operation in ("insert", "update", "delete"): + name = f"{prefix}_{table}_{operation}" + row = self._connection.execute( + "SELECT tbl_name,sql FROM sqlite_master WHERE type='trigger' AND name=?", (name,) + ).fetchone() + sql = "" if row is None or row["sql"] is None else " ".join(str(row["sql"]).lower().split()) + expected_guard = ( + "forge_maintenance_write_permitted() != 1" + if table in _OPERATIONAL_RESET_TABLES else + "active_operation_id from operational_reset_state" + ) + if ( + row is None or row["tbl_name"] != table or f"before {operation}" not in sql + or expected_guard not in sql + ): + raise RuntimeIntegrityError(f"operational reset migration found incompatible {name} trigger") + def _migrate_governance_19_to_20(self, forge_version: str) -> None: """Create and verify all governance objects before advancing schema metadata. @@ -907,14 +978,14 @@ def _migrate(self, forge_version: str) -> None: BEGIN SELECT RAISE(ABORT, 'mission intake evidence is immutable'); END; """) self._set_metadata({ - "schema_version": str(RUNTIME_SCHEMA_VERSION), - "migration_version": str(RUNTIME_SCHEMA_VERSION), + "schema_version": "37", + "migration_version": "37", "forge_version": forge_version, "created_at": _timestamp(), - "last_migration": str(RUNTIME_SCHEMA_VERSION), + "last_migration": "37", "integrity_status": "valid", }) - self._connection.execute(f"PRAGMA user_version={RUNTIME_SCHEMA_VERSION}") + self._connection.execute("PRAGMA user_version=37") elif version == 1: with self._connection: self._connection.executescript(""" @@ -1637,6 +1708,122 @@ def _migrate(self, forge_version: str) -> None: except Exception: self._connection.rollback() raise + elif version == 37: + # Operational reset is deliberately schema-owned. The durable + # fence is visible to every already-open connection through pure + # schema triggers that read persisted maintenance state. Raw and + # read-only tooling therefore needs no connection-local UDF while + # idle; the UDF below authorizes only reset-table mutations. + self._connection.execute("BEGIN IMMEDIATE") + try: + self._connection.executescript(""" + CREATE TABLE IF NOT EXISTS operational_reset_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + dataset_generation INTEGER NOT NULL CHECK (dataset_generation >= 0), + active_operation_id TEXT, + state TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT OR IGNORE INTO operational_reset_state VALUES (1, 0, NULL, 'IDLE', '1970-01-01T00:00:00Z'); + CREATE TABLE IF NOT EXISTS operational_reset_operations ( + operation_id TEXT PRIMARY KEY, + runtime_id TEXT NOT NULL, + profile TEXT NOT NULL, + state TEXT NOT NULL, + actor_reference TEXT NOT NULL, + authority_digest TEXT NOT NULL, + plan_digest TEXT NOT NULL, + request_digest TEXT NOT NULL, + backup_digest TEXT, + backup_reference TEXT, + relevant_revision TEXT NOT NULL, + source_revision TEXT NOT NULL, + implementation_digest TEXT NOT NULL, + schema_version INTEGER NOT NULL, + policy_version TEXT NOT NULL, + generation_before INTEGER NOT NULL, + generation_after INTEGER, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + document TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS operational_reset_audit ( + audit_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL, + event TEXT NOT NULL, + occurred_at TEXT NOT NULL, + document TEXT NOT NULL, + FOREIGN KEY(operation_id) REFERENCES operational_reset_operations(operation_id) + ); + CREATE TABLE IF NOT EXISTS operational_reset_tombstones ( + record_kind TEXT NOT NULL, + record_id TEXT NOT NULL, + source_digest TEXT NOT NULL, + operation_id TEXT NOT NULL, + retired_at TEXT NOT NULL, + PRIMARY KEY(record_kind, record_id), + FOREIGN KEY(operation_id) REFERENCES operational_reset_operations(operation_id) + ); + CREATE TABLE IF NOT EXISTS operational_reset_artifact_steps ( + operation_id TEXT NOT NULL, + relative_path TEXT NOT NULL, + classification TEXT NOT NULL, + source_digest TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + state TEXT NOT NULL, + backup_relative_path TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY(operation_id, relative_path), + FOREIGN KEY(operation_id) REFERENCES operational_reset_operations(operation_id) + ); + CREATE TRIGGER IF NOT EXISTS operational_reset_audit_immutable_update BEFORE UPDATE ON operational_reset_audit + BEGIN SELECT RAISE(ABORT, 'operational reset audit is immutable'); END; + CREATE TRIGGER IF NOT EXISTS operational_reset_audit_immutable_delete BEFORE DELETE ON operational_reset_audit + BEGIN SELECT RAISE(ABORT, 'operational reset audit is immutable'); END; + CREATE TRIGGER IF NOT EXISTS operational_reset_tombstones_immutable_update BEFORE UPDATE ON operational_reset_tombstones + BEGIN SELECT RAISE(ABORT, 'operational reset tombstones are immutable'); END; + CREATE TRIGGER IF NOT EXISTS operational_reset_tombstones_immutable_delete BEFORE DELETE ON operational_reset_tombstones + BEGIN SELECT RAISE(ABORT, 'operational reset tombstones are immutable'); END; + """) + reset_tables = { + "operational_reset_state", "operational_reset_operations", "operational_reset_audit", + "operational_reset_tombstones", "operational_reset_artifact_steps", + } + existing_tables = { + row[0] for row in self._connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + } + for table in sorted(existing_tables): + if table in reset_tables: + for operation in ("INSERT", "UPDATE", "DELETE"): + self._connection.execute( + f"CREATE TRIGGER IF NOT EXISTS operational_reset_authorize_{table}_{operation.lower()} " + f"BEFORE {operation} ON {table} " + "WHEN forge_maintenance_write_permitted() != 1 " + "BEGIN SELECT RAISE(ABORT, 'operational reset service authority required'); END" + ) + else: + for operation in ("INSERT", "UPDATE", "DELETE"): + self._connection.execute( + f"CREATE TRIGGER IF NOT EXISTS operational_reset_block_{table}_{operation.lower()} " + f"BEFORE {operation} ON {table} " + "WHEN (SELECT active_operation_id FROM operational_reset_state WHERE singleton=1) IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'Forge operational reset maintenance is active'); END" + ) + self._maintenance_write_state["permitted"] = True + try: + self._set_metadata({ + "schema_version": "38", "migration_version": "38", + "last_migration": "38", "forge_version": forge_version, + }) + self._connection.execute("PRAGMA user_version=38") + finally: + self._maintenance_write_state["permitted"] = False + self._connection.commit() + except Exception: + self._connection.rollback() + raise elif version != RUNTIME_SCHEMA_VERSION: raise RuntimeIntegrityError("runtime database migration path is unavailable") @@ -1726,6 +1913,7 @@ def validate_integrity(self, *, record_status: bool = True) -> None: self._require_peer_configuration_structure() self._require_execution_host_exchange_audit_structure() self._require_operational_log_structure() + self._require_operational_reset_structure() identity = self.runtime_identity expected_identity = "forge-installation" if self._installation_scoped else repository_identity(self.repository_root) if identity.repository_identity != expected_identity or not identity.runtime_id or identity.status != "active": @@ -2002,8 +2190,16 @@ def execution_host_binding(self, correlation_id: str) -> dict[str, Any] | None: row=self._connection.execute("SELECT document FROM execution_host_bindings WHERE correlation_id=?",(correlation_id,)).fetchone() return None if row is None else json.loads(row["document"]) + def _reject_reset_retired_identity(self, record_kind: str, record_id: str) -> None: + if self._connection.execute( + "SELECT 1 FROM operational_reset_tombstones WHERE record_kind=? AND record_id=?", + (record_kind, record_id), + ).fetchone() is not None: + raise RuntimeIntegrityError(f"{record_kind} identity was retired by an operational reset") + def save_execution_host_binding(self, correlation_id: str, document: Mapping[str, Any]) -> dict[str, Any]: if not correlation_id or document.get("correlation_id") != correlation_id: raise RuntimeDatabaseError("execution host binding correlation is invalid") + self._reject_reset_retired_identity("correlation_id", correlation_id) existing=self.execution_host_binding(correlation_id) if existing is not None and any(existing.get(key) not in (None,value) for key,value in document.items()): raise RuntimeIntegrityError("execution host binding is immutable") merged={**(existing or {}),**document} @@ -2238,6 +2434,8 @@ def create_scheduler_submission(self, submission: Any) -> dict[str, Any]: raise RuntimeDatabaseError("scheduler submission requires a complete CREATED envelope") if not isinstance(document["iteration"], int) or document["iteration"] < 1: raise RuntimeDatabaseError("scheduler submission iteration must be positive") + self._reject_reset_retired_identity("submission_id", str(document["submission_id"])) + self._reject_reset_retired_identity("action_id", str(document["action_id"])) existing = self.scheduler_submission(str(document["submission_id"])) if existing is not None: if existing.get("envelope") != document.get("envelope"): @@ -2361,6 +2559,8 @@ def record_execution_receipt(self, *, receipt_id: str, mission_id: str, executio correlation_identity: str, executed_at: str, outcome: str) -> None: if not all((receipt_id, mission_id, execution_host, execution_run_id, engineering_report_id, correlation_identity, executed_at, outcome)): raise RuntimeDatabaseError("execution receipt requires complete identity, report, correlation, and outcome") + self._reject_reset_retired_identity("execution_receipt_id", receipt_id) + self._reject_reset_retired_identity("correlation_id", correlation_identity) with self._connection: self._connection.execute("INSERT INTO execution_receipts VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (receipt_id, mission_id, execution_host, execution_run_id, engineering_report_id, correlation_identity, executed_at, outcome)) diff --git a/forge/runtime/operational_reset.py b/forge/runtime/operational_reset.py new file mode 100644 index 0000000..dc32144 --- /dev/null +++ b/forge/runtime/operational_reset.py @@ -0,0 +1,1460 @@ +"""Forge-owned, fail-closed operational-history reset service. + +The service owns one deliberately narrow profile. It never accepts a caller +supplied table list and never reaches Engineering Platform storage. Preview is +strictly read-only; every mutation after it is bound to the same target, plan, +operator authority, backup, and durable operation identity. +""" +from __future__ import annotations + +from contextlib import contextmanager +from datetime import UTC, datetime +from hashlib import sha256 +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import stat +import subprocess +import tempfile +import uuid +from typing import Any, Callable, Iterator, Mapping, Sequence + +from forge._version import canonical_version +from forge.operator_identity import MacOSGeneratedUIDIdentityAdapter, NamedOperatorIdentity + +from .data_root import DataRootResolver +from .database import RUNTIME_SCHEMA_VERSION +from .service import RuntimeServiceLock + + +RESET_PROFILE = "forge-operational-history-v1" +RESET_POLICY_VERSION = "1" +_OPERATION_ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{7,127}\Z") +_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z") + + +class OperationalResetError(RuntimeError): + """The requested maintenance operation cannot preserve the reset contract.""" + + +# Explicit schema-owned classification. Adding a product table without adding +# it here makes preview fail closed as UNKNOWN_OR_UNSUPPORTED. +TABLE_CLASSIFICATION: Mapping[str, str] = { + # A — durable installation, binding, and provider configuration. + "runtime_metadata": "INSTALLATION_AND_CONFIGURATION", + "execution_host_peer_configuration": "INSTALLATION_AND_CONFIGURATION", + "planning_provider_security_config": "INSTALLATION_AND_CONFIGURATION", + "planning_provider_external_session_config": "INSTALLATION_AND_CONFIGURATION", + # B — authority, anti-replay, allocation, and consumed-budget lineage. + "installation_operator_binding": "SECURITY_AND_AUTHORITY_LEDGER", + "installation_operator_audit": "SECURITY_AND_AUTHORITY_LEDGER", + "governance_authority": "SECURITY_AND_AUTHORITY_LEDGER", + "governance_capability_grants": "SECURITY_AND_AUTHORITY_LEDGER", + "governance_decisions": "SECURITY_AND_AUTHORITY_LEDGER", + "planning_provider_security_audit": "SECURITY_AND_AUTHORITY_LEDGER", + "planning_provider_external_session_audit": "SECURITY_AND_AUTHORITY_LEDGER", + "planning_provider_generation_permits": "SECURITY_AND_AUTHORITY_LEDGER", + "mission_id_allocations": "SECURITY_AND_AUTHORITY_LEDGER", + # C — old planning/execution history removed from the active generation. + "mission_state": "OPERATIONAL_HISTORY", + "execution_context_snapshots": "OPERATIONAL_HISTORY", + "architecture_reviews": "OPERATIONAL_HISTORY", + "mission_recommendations": "OPERATIONAL_HISTORY", + "decision_evidence": "OPERATIONAL_HISTORY", + "execution_receipts": "OPERATIONAL_HISTORY", + "bootstrap_portfolio_state": "OPERATIONAL_HISTORY", + "mission_lifecycle_events": "OPERATIONAL_HISTORY", + "delegation_requests": "OPERATIONAL_HISTORY", + "integration_evidence": "OPERATIONAL_HISTORY", + "mission_intake_evidence": "OPERATIONAL_HISTORY", + "scheduler_submissions": "OPERATIONAL_HISTORY", + "token_preflight_receipts": "OPERATIONAL_HISTORY", + "token_preflight_receipt_consumptions": "OPERATIONAL_HISTORY", + "token_preflight_failures": "OPERATIONAL_HISTORY", + "action_derivations": "OPERATIONAL_HISTORY", + "action_derivation_results": "OPERATIONAL_HISTORY", + "action_derivation_reattempt_authorizations": "OPERATIONAL_HISTORY", + "action_derivation_reattempt_consumptions": "OPERATIONAL_HISTORY", + "action_derivation_evidence_sets": "OPERATIONAL_HISTORY", + "action_derivation_canary_closures": "OPERATIONAL_HISTORY", + "mission_amendments": "OPERATIONAL_HISTORY", + "execution_host_bindings": "OPERATIONAL_HISTORY", + "execution_host_exchange_audit": "OPERATIONAL_HISTORY", + "forge_operational_logs": "OPERATIONAL_HISTORY", + # D — mutable projections/queues, never durable authority. + "mission_runtime_projections": "DERIVED_CACHE_OR_PROJECTION", + "planning_state": "DERIVED_CACHE_OR_PROJECTION", + "dispatcher_state": "DERIVED_CACHE_OR_PROJECTION", + # E — the reset's own durable reconciliation and anti-replay evidence. + "operational_reset_state": "MAINTENANCE_AUDIT", + "operational_reset_operations": "MAINTENANCE_AUDIT", + "operational_reset_audit": "MAINTENANCE_AUDIT", + "operational_reset_tombstones": "MAINTENANCE_AUDIT", + "operational_reset_artifact_steps": "MAINTENANCE_AUDIT", +} + +PRESERVE_TABLES = tuple( + table for table, category in TABLE_CLASSIFICATION.items() + if category in {"INSTALLATION_AND_CONFIGURATION", "SECURITY_AND_AUTHORITY_LEDGER"} +) +PURGE_TABLES = tuple( + table for table, category in TABLE_CLASSIFICATION.items() + if category in {"OPERATIONAL_HISTORY", "DERIVED_CACHE_OR_PROJECTION"} +) +MAINTENANCE_TABLES = tuple( + table for table, category in TABLE_CLASSIFICATION.items() if category == "MAINTENANCE_AUDIT" +) + +# Child-first order is part of the product contract and mirrors schema FKs. +PURGE_ORDER = ( + "action_derivation_canary_closures", + "action_derivation_results", + "action_derivation_reattempt_consumptions", + "action_derivation_reattempt_authorizations", + "action_derivation_evidence_sets", + "mission_amendments", + "token_preflight_receipt_consumptions", + "token_preflight_receipts", + "token_preflight_failures", + "decision_evidence", + "integration_evidence", + "mission_intake_evidence", + "mission_runtime_projections", + "execution_context_snapshots", + "action_derivations", + "scheduler_submissions", + "delegation_requests", + "execution_receipts", + "mission_recommendations", + "architecture_reviews", + "mission_lifecycle_events", + "execution_host_exchange_audit", + "execution_host_bindings", + "forge_operational_logs", + "planning_state", + "dispatcher_state", + "bootstrap_portfolio_state", + "mission_state", +) + +EXTERNAL_CLASSIFICATION: Mapping[str, str] = { + "instance": "INSTALLATION_AND_CONFIGURATION", + "artifacts": "OPERATIONAL_HISTORY", + "journals": "OPERATIONAL_HISTORY", + "logs": "OPERATIONAL_HISTORY", + "backups": "INSTALLATION_AND_CONFIGURATION", + "cache": "DERIVED_CACHE_OR_PROJECTION", + "locks": "SYSTEM_RUNTIME_CONTROL", +} + +_IMMUTABLE_DELETE_TRIGGERS: Mapping[str, tuple[str, ...]] = { + "execution_context_snapshots": ("execution_context_snapshots_immutable_delete",), + "action_derivation_canary_closures": ("action_derivation_canary_closures_immutable_delete",), + "action_derivation_results": ("action_derivation_results_immutable_delete",), + "action_derivation_reattempt_authorizations": ("action_derivation_reattempt_authorizations_immutable_delete",), + "action_derivation_reattempt_consumptions": ("action_derivation_reattempt_consumptions_immutable_delete",), + "action_derivations": ("action_derivations_failed_immutable_delete",), + "architecture_reviews": ("architecture_reviews_immutable_delete",), + "decision_evidence": ("decision_evidence_immutable_delete",), + "execution_host_exchange_audit": ("execution_host_exchange_audit_immutable_delete",), + "execution_receipts": ("execution_receipts_immutable_delete",), + "forge_operational_logs": ("forge_operational_logs_immutable_delete",), + "integration_evidence": ("integration_evidence_immutable_delete",), + "mission_intake_evidence": ("mission_intake_evidence_immutable_delete",), + "mission_lifecycle_events": ("mission_lifecycle_events_immutable_delete",), + "mission_recommendations": ("mission_recommendations_immutable_delete",), + "token_preflight_failures": ("token_preflight_failures_immutable_delete",), + "token_preflight_receipts": ("token_preflight_receipts_immutable_delete",), + "token_preflight_receipt_consumptions": ("token_preflight_receipt_consumptions_immutable_delete",), +} + + +def _now() -> str: + return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _digest(value: Any) -> str: + encoded = value if isinstance(value, bytes) else _json(value).encode("utf-8") + return "sha256:" + sha256(encoded).hexdigest() + + +def _file_digest(path: Path) -> str: + hasher = sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(block) + return "sha256:" + hasher.hexdigest() + + +def _safe_operation_id(value: str) -> str: + if not isinstance(value, str) or _OPERATION_ID.fullmatch(value) is None: + raise OperationalResetError("operation ID must be 8-128 safe identifier characters") + return value + + +def _has_symlink_component(path: Path) -> bool: + """Inspect the lexical path before ``resolve`` can hide a symlink. + + Missing components are safe to inspect and return ``False``. Each + existing parent is still checked, so ``alias/nonexistent-child`` cannot + escape this guard merely because the leaf does not exist yet. + """ + lexical = path if path.is_absolute() else Path.cwd() / path + current = Path(lexical.anchor) + for part in lexical.parts[1:]: + current /= part + if current.is_symlink(): + # macOS exposes root-owned compatibility aliases such as + # /var -> /private/var. They are immutable to an unprivileged + # operator and precede the selected data hierarchy; accepting + # them avoids rejecting the OS temporary-directory topology. + if ( + current.parent == Path(current.anchor) + and getattr(current.lstat(), "st_uid", -1) == 0 + ): + continue + return True + return False + + +class ForgeOperationalResetService: + """Application service for one local Forge operational-history reset.""" + + def __init__( + self, + data_root: Path | str | None, + *, + identity_resolver: Callable[[], NamedOperatorIdentity] | None = None, + disk_usage: Callable[[Path], Any] = shutil.disk_usage, + fault_hook: Callable[[str, str | None], None] | None = None, + ) -> None: + self._requested_root = None if data_root is None else Path(data_root).expanduser() + self._requested_root_is_symlink = bool( + self._requested_root is not None and _has_symlink_component(self._requested_root) + ) + candidate = DataRootResolver(cli_data_root=data_root).resolve() + self.data_root = candidate + self.database_path = candidate / "forge.db" + self.marker_path = candidate / "instance" / "runtime-instance.json" + self._identity_resolver = identity_resolver or MacOSGeneratedUIDIdentityAdapter().resolve + self._disk_usage = disk_usage + self._fault_hook = fault_hook or (lambda _event, _path=None: None) + self._lock = RuntimeServiceLock(self.database_path) + + def _implementation_digest(self) -> str: + service_bytes = Path(__file__).read_bytes() + return _digest(service_bytes + canonical_version().encode("utf-8")) + + def _source_revision(self) -> str: + """Return a real Git revision when this module is in tracked source. + + Installed wheels do not currently embed the protected-main SHA. That + absence is reported honestly; the independent implementation digest + still binds the exact maintenance code used by this operation. + """ + try: + root = Path(subprocess.check_output( + ("git", "-C", str(Path(__file__).parent), "rev-parse", "--show-toplevel"), + text=True, stderr=subprocess.DEVNULL, + ).strip()).resolve() + relative = Path(__file__).resolve().relative_to(root).as_posix() + subprocess.run( + ("git", "-C", str(root), "ls-files", "--error-unmatch", relative), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + revision = subprocess.check_output( + ("git", "-C", str(root), "rev-parse", "HEAD"), text=True, stderr=subprocess.DEVNULL, + ).strip() + if re.fullmatch(r"[0-9a-f]{40}", revision): + return revision + except (OSError, ValueError, subprocess.CalledProcessError): + pass + return "UNAVAILABLE_IN_INSTALLED_PACKAGE" + + def _require_target_paths(self) -> None: + if self._requested_root_is_symlink or not self.data_root.is_dir() or self.data_root.is_symlink(): + raise OperationalResetError("Forge data root must be a real directory, not a symlink") + if not self.database_path.is_file() or self.database_path.is_symlink(): + raise OperationalResetError("Forge database must be a real file, not a symlink") + if not self.marker_path.is_file() or self.marker_path.is_symlink(): + raise OperationalResetError("Forge runtime marker must be a real file, not a symlink") + + @contextmanager + def _connect(self, *, read_only: bool) -> Iterator[sqlite3.Connection]: + self._require_target_paths() + if read_only: + connection = sqlite3.connect(self.database_path.resolve().as_uri() + "?mode=ro", uri=True) + else: + connection = sqlite3.connect(self.database_path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys=ON") + connection.execute("PRAGMA busy_timeout=5000") + if read_only: + connection.execute("PRAGMA query_only=ON") + else: + connection.execute("PRAGMA synchronous=FULL") + connection.create_function("forge_maintenance_write_permitted", 0, lambda: 1) + # Existing product triggers reference these owning writer functions. + for name in ( + "forge_governance_write_permitted", "forge_token_preflight_write_permitted", + "forge_action_derivation_write_permitted", "forge_action_derivation_reattempt_write_permitted", + "forge_action_derivation_canary_closure_write_permitted", + ): + connection.create_function(name, 0, lambda: 0) + try: + yield connection + finally: + connection.close() + + @staticmethod + def _application_tables(connection: sqlite3.Connection) -> set[str]: + return { + str(row[0]) for row in connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + } + + @staticmethod + def _row_digest(connection: sqlite3.Connection, table: str, *, post_reset: bool = False) -> str: + columns = [str(row[1]) for row in connection.execute(f'PRAGMA table_info("{table}")')] + rows: list[list[Any]] = [] + for row in connection.execute(f'SELECT * FROM "{table}" ORDER BY rowid'): + values = [row[column] for column in columns] + if post_reset and table == "planning_provider_generation_permits": + state_index = columns.index("state") + updated_index = columns.index("updated_at") + if values[state_index] in { + "PENDING", "TRANSPORT_COMMITTED", "INVALIDATED_BY_OPERATIONAL_RESET", + }: + values[state_index] = "INVALIDATED_BY_OPERATIONAL_RESET" + # The timestamp is intentionally normalized out of the + # preservation proof; it is separately bound in audit. + values[updated_index] = "" + rows.append(values) + return _digest({"columns": columns, "rows": rows}) + + def _relevant_revision(self, connection: sqlite3.Connection) -> str: + values: dict[str, str] = {} + for table in sorted(set(TABLE_CLASSIFICATION) - set(MAINTENANCE_TABLES)): + if table == "runtime_metadata": + rows = [ + [row[0], row[1]] for row in connection.execute( + "SELECT key,value FROM runtime_metadata " + "WHERE key NOT IN ('last_access_at','integrity_status') ORDER BY key" + ) + ] + values[table] = _digest(rows) + else: + values[table] = self._row_digest(connection, table) + return _digest(values) + + def _external_inventory(self) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + entries: list[dict[str, Any]] = [] + unknown: list[dict[str, Any]] = [] + recognized_root_files = {"forge.db", "forge.db-wal", "forge.db-shm"} + recognized_root_controls = {"forge-runtime-mutation.lock"} + for child in sorted(self.data_root.iterdir(), key=lambda item: item.name): + relative = child.relative_to(self.data_root).as_posix() + if child.is_symlink(): + unknown.append({"path": relative, "reason": "SYMLINK"}) + continue + if child.is_file(): + if child.name in recognized_root_controls: + entries.append({ + "path": relative, "category": "SYSTEM_RUNTIME_CONTROL", + "effect": "PRESERVE", "size_bytes": child.stat().st_size, + }) + elif child.name not in recognized_root_files: + unknown.append({"path": relative, "reason": "UNKNOWN_ROOT_FILE"}) + continue + if not child.is_dir() or child.name not in EXTERNAL_CLASSIFICATION: + unknown.append({"path": relative, "reason": "UNKNOWN_ROOT_ENTRY"}) + continue + category = EXTERNAL_CLASSIFICATION[child.name] + for current_root, directory_names, file_names in os.walk(child, followlinks=False): + current = Path(current_root) + for name in tuple(directory_names): + candidate = current / name + if candidate.is_symlink(): + unknown.append({ + "path": candidate.relative_to(self.data_root).as_posix(), "reason": "SYMLINK", + }) + directory_names.remove(name) + for name in sorted(file_names): + candidate = current / name + rel = candidate.relative_to(self.data_root).as_posix() + try: + mode = candidate.lstat().st_mode + except OSError as error: + unknown.append({"path": rel, "reason": "UNREADABLE", "error": type(error).__name__}) + continue + if stat.S_ISLNK(mode): + unknown.append({"path": rel, "reason": "SYMLINK"}) + elif not stat.S_ISREG(mode): + unknown.append({"path": rel, "reason": "SPECIAL_FILE"}) + elif child.name == "instance" and rel != "instance/runtime-instance.json": + unknown.append({"path": rel, "reason": "UNKNOWN_INSTANCE_FILE"}) + elif child.name in {"backups", "locks"}: + # Existing recovery material and lock files are outside + # the effect set but remain inventoried by count only. + entries.append({"path": rel, "category": category, "effect": "PRESERVE", "size_bytes": candidate.stat().st_size}) + elif category == "INSTALLATION_AND_CONFIGURATION": + entries.append({ + "path": rel, "category": category, "effect": "PRESERVE", + "size_bytes": candidate.stat().st_size, "digest": _file_digest(candidate), + }) + elif child.name == "artifacts" and ( + Path(rel).name.startswith("controlled-installation-") + or len(Path(rel).parts) > 1 and Path(rel).parts[1] in {"installation", "qualification"} + ): + entries.append({ + "path": rel, "category": "INSTALLATION_AND_CONFIGURATION", + "effect": "PRESERVE", "size_bytes": candidate.stat().st_size, + "digest": _file_digest(candidate), + }) + elif child.name == "artifacts" and not ( + len(Path(rel).parts) > 1 + and Path(rel).parts[1] in {"operational", "runtime", "missions"} + ): + unknown.append({"path": rel, "reason": "UNCLASSIFIED_ARTIFACT"}) + elif child.name == "journals" and candidate.suffix not in {".json", ".jsonl", ".journal"}: + unknown.append({"path": rel, "reason": "UNCLASSIFIED_JOURNAL"}) + elif child.name == "logs" and candidate.suffix not in {".log", ".jsonl"}: + unknown.append({"path": rel, "reason": "UNCLASSIFIED_LOG"}) + else: + entries.append({ + "path": rel, + "category": category, + "effect": "ARCHIVE_AND_REMOVE" if category == "OPERATIONAL_HISTORY" else "REMOVE_CACHE", + "size_bytes": candidate.stat().st_size, + "digest": _file_digest(candidate), + }) + return entries, unknown + + @staticmethod + def _foreign_key_issues(connection: sqlite3.Connection) -> list[dict[str, Any]]: + issues: list[dict[str, Any]] = [] + for row in connection.execute("PRAGMA foreign_key_check"): + table, rowid, parent, fkid = str(row[0]), row[1], str(row[2]), int(row[3]) + if table in PURGE_TABLES and parent in PURGE_TABLES: + scope = "OPERATIONAL_PURGE_SET" + elif table in TABLE_CLASSIFICATION and parent in TABLE_CLASSIFICATION: + scope = "PRESERVATION_SET" + else: + scope = "UNKNOWN_OR_UNSUPPORTED" + issue = {"table": table, "rowid": rowid, "parent": parent, "foreign_key_id": fkid, "scope": scope} + issue["issue_id"] = _digest(issue) + issues.append(issue) + return issues + + @staticmethod + def _writer_fence_issues(connection: sqlite3.Connection) -> list[str]: + issues: list[str] = [] + for table in sorted(TABLE_CLASSIFICATION): + prefix = "operational_reset_authorize" if table in MAINTENANCE_TABLES else "operational_reset_block" + for operation in ("insert", "update", "delete"): + name = f"{prefix}_{table}_{operation}" + row = connection.execute( + "SELECT tbl_name,sql FROM sqlite_master WHERE type='trigger' AND name=?", (name,) + ).fetchone() + sql = "" if row is None or row[1] is None else " ".join(str(row[1]).lower().split()) + guard = ( + "forge_maintenance_write_permitted() != 1" + if table in MAINTENANCE_TABLES else + "active_operation_id from operational_reset_state" + ) + if row is None or row[0] != table or f"before {operation}" not in sql or guard not in sql: + issues.append(name) + return issues + + def _inventory(self, connection: sqlite3.Connection) -> dict[str, Any]: + schema = int(connection.execute("PRAGMA user_version").fetchone()[0]) + tables = self._application_tables(connection) + known_for_schema = set(TABLE_CLASSIFICATION) + if schema == 37: + known_for_schema -= set(MAINTENANCE_TABLES) + unknown_tables = sorted(tables - known_for_schema) + missing_tables = sorted(known_for_schema - tables) + metadata = dict(connection.execute("SELECT key,value FROM runtime_metadata")) if "runtime_metadata" in tables else {} + marker = self.marker_path.read_text(encoding="utf-8").strip() + runtime_id = str(metadata.get("runtime_id", "")) + integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) + quick = str(connection.execute("PRAGMA quick_check").fetchone()[0]) + foreign_keys = self._foreign_key_issues(connection) + table_counts = { + table: int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + for table in sorted(tables & known_for_schema) + } + classifications = { + category: [ + {"table": table, "rows": table_counts.get(table, 0)} + for table, mapped in sorted(TABLE_CLASSIFICATION.items()) if mapped == category and table in tables + ] + for category in ( + "INSTALLATION_AND_CONFIGURATION", "SECURITY_AND_AUTHORITY_LEDGER", "OPERATIONAL_HISTORY", + "DERIVED_CACHE_OR_PROJECTION", "MAINTENANCE_AUDIT", + ) + } + external, unknown_external = self._external_inventory() + generation = 0 + active_operation = None + if "operational_reset_state" in tables: + state = connection.execute( + "SELECT dataset_generation,active_operation_id,state FROM operational_reset_state WHERE singleton=1" + ).fetchone() + if state is not None: + generation, active_operation = int(state[0]), state[1] + target = { + "product": "forge", "runtime_id": runtime_id, + "data_root": str(self.data_root), "database": str(self.database_path), + "database_binding_digest": _digest(str(self.database_path.resolve())), + "marker_digest": _file_digest(self.marker_path), "schema_version": schema, + "dataset_generation": generation, + } + blockers: list[dict[str, Any]] = [] + if schema != RUNTIME_SCHEMA_VERSION: + blockers.append({"code": "SCHEMA_MIGRATION_REQUIRED", "found": schema, "required": RUNTIME_SCHEMA_VERSION}) + if marker != runtime_id or not runtime_id: + blockers.append({"code": "TARGET_IDENTITY_MISMATCH"}) + if integrity != "ok" or quick != "ok": + blockers.append({"code": "PHYSICAL_DATABASE_CORRUPTION", "integrity_check": integrity, "quick_check": quick}) + if unknown_tables or missing_tables: + blockers.append({"code": "UNKNOWN_OR_INCOMPLETE_SCHEMA", "unknown_tables": unknown_tables, "missing_tables": missing_tables}) + fence_issues = self._writer_fence_issues(connection) if schema == RUNTIME_SCHEMA_VERSION else [] + if fence_issues: + blockers.append({"code": "MAINTENANCE_FENCE_INCOMPLETE", "triggers": fence_issues}) + if unknown_external: + blockers.append({"code": "UNKNOWN_OR_UNSAFE_EXTERNAL_DATA", "entries": unknown_external}) + non_operational_fk = [item for item in foreign_keys if item["scope"] != "OPERATIONAL_PURGE_SET"] + if non_operational_fk: + blockers.append({"code": "FOREIGN_KEY_DAMAGE_OUTSIDE_PURGE_SET", "issues": non_operational_fk}) + operational_fk = [item for item in foreign_keys if item["scope"] == "OPERATIONAL_PURGE_SET"] + if active_operation: + blockers.append({"code": "MAINTENANCE_ALREADY_ACTIVE", "operation_id": active_operation}) + relevant_revision = self._relevant_revision(connection) if not unknown_tables and not missing_tables and schema == RUNTIME_SCHEMA_VERSION else None + preserved = {} + if schema == RUNTIME_SCHEMA_VERSION and not unknown_tables and not missing_tables: + preserved = { + table: { + "rows": table_counts[table], + "current_digest": self._row_digest(connection, table), + "expected_post_reset_digest": self._row_digest(connection, table, post_reset=True), + } + for table in PRESERVE_TABLES + } + effect_entries = [item for item in external if item.get("effect") in {"ARCHIVE_AND_REMOVE", "REMOVE_CACHE"}] + external_controls = [ + item for item in external + if item.get("category") == "SYSTEM_RUNTIME_CONTROL" + or str(item.get("path", "")).startswith(("locks/", "backups/")) + ] + meaningful_external = [item for item in external if item not in external_controls] + preserved_external = [ + item for item in external + if item.get("effect") == "PRESERVE" + and item.get("category") == "INSTALLATION_AND_CONFIGURATION" + and not str(item.get("path", "")).startswith("backups/") + ] + effect_set = { + "purge_tables": [{"table": table, "rows": table_counts.get(table, 0)} for table in PURGE_TABLES if table in tables], + "invalidate_active_generation_permits": table_counts.get("planning_provider_generation_permits", 0), + "external": effect_entries, + } + active_permits = ( + int(connection.execute( + "SELECT COUNT(*) FROM planning_provider_generation_permits " + "WHERE state IN ('PENDING','TRANSPORT_COMMITTED')" + ).fetchone()[0]) if "planning_provider_generation_permits" in tables else 0 + ) + effect_set["invalidate_active_generation_permits"] = active_permits + no_op = ( + sum(item["rows"] for item in effect_set["purge_tables"]) == 0 + and not effect_entries + and active_permits == 0 + ) + source_revision = self._source_revision() + plan_core = { + "contract": "forge-operational-reset-plan-1.0", "profile": RESET_PROFILE, + "policy_version": RESET_POLICY_VERSION, "product_version": canonical_version(), + "source_revision": source_revision, + "source_revision_kind": ( + "GIT_HEAD" if source_revision != "UNAVAILABLE_IN_INSTALLED_PACKAGE" + else "UNAVAILABLE_IN_INSTALLED_PACKAGE" + ), + "implementation_digest": self._implementation_digest(), + "target": target, + "relevant_revision": relevant_revision, "classifications": classifications, + "preserved": preserved, "effect_set": effect_set, + "effect_set_digest": _digest(effect_set), "foreign_key_issues": foreign_keys, + "required_fk_acknowledgements": [item["issue_id"] for item in operational_fk], + "integrity": {"integrity_check": integrity, "quick_check": quick}, + "external_inventory": meaningful_external, + "external_inventory_digest": _digest(effect_entries), + "preserved_external": preserved_external, + "preserved_external_digest": _digest(preserved_external), + "unknown_external": unknown_external, + "unknown_tables": unknown_tables, "missing_tables": missing_tables, + "blockers": blockers, "no_op": no_op, + "namespace": { + "repository_document": "missions/MISSION-0003.md", + "repository_document_kind": "HISTORICAL_REPOSITORY_DOCUMENT", + "runtime_identity_namespace": "FORGE_RUNTIME_MISSION", + "allocator_is_reset": False, + }, + } + return { + **plan_core, "plan_digest": _digest(plan_core), + "external_control_entries": external_controls, + } + + def preview(self) -> dict[str, Any]: + """Inspect the exact target without schema migration or domain writes.""" + with self._connect(read_only=True) as connection: + plan = self._inventory(connection) + return { + **plan, + "status": "BLOCKED" if plan["blockers"] or plan["required_fk_acknowledgements"] else "READY", + "execution_allowed": not plan["blockers"] and not plan["required_fk_acknowledgements"], + "summary": "read-only operational reset preview; no Forge operational data changed", + } + + def _operator_authority(self, connection: sqlite3.Connection) -> tuple[str, str]: + identity = self._identity_resolver() + if not isinstance(identity, NamedOperatorIdentity) or not identity.generated_uid: + raise OperationalResetError("trusted local operator identity is unavailable") + row = connection.execute( + "SELECT installation_id,generated_uid,uid,version,status FROM installation_operator_binding" + ).fetchone() + if row is None or row[4] != "ACTIVE" or row[1] != identity.generated_uid or int(row[2]) != identity.uid: + raise OperationalResetError("current operator is not the active installation operator") + reference = sha256(identity.generated_uid.encode("utf-8")).hexdigest()[:16] + capabilities = { + str(item[0]) for item in connection.execute( + "SELECT capability FROM governance_authority WHERE installation_id=? AND operator_id=?", + (row[0], reference), + ) + } + required = {"SECURITY_APPROVAL", "OWNER_PROGRAMME_AUTHORIZATION"} + if not required <= capabilities: + raise OperationalResetError("operational reset requires current Security and programme authority") + authority = { + "installation_id": str(row[0]), "operator_reference": reference, + "binding_version": int(row[3]), "capabilities": sorted(required), + } + return reference, _digest(authority) + + @staticmethod + def _request_digest(operation_id: str, plan: Mapping[str, Any], acknowledgements: Sequence[str]) -> str: + return _digest({ + "operation_id": operation_id, "profile": RESET_PROFILE, + "runtime_id": plan["target"]["runtime_id"], "plan_digest": plan["plan_digest"], + "effect_set_digest": plan["effect_set_digest"], + "acknowledged_operational_fk_issues": sorted(acknowledgements), + }) + + def _audit(self, connection: sqlite3.Connection, operation_id: str, event: str, details: Mapping[str, Any]) -> None: + occurred_at = _now() + document = {"event": event, "operation_id": operation_id, "occurred_at": occurred_at, **details} + connection.execute( + "INSERT INTO operational_reset_audit VALUES (?,?,?,?,?)", + ("forge-reset-audit-" + str(uuid.uuid4()), operation_id, event, occurred_at, _json(document)), + ) + + def prepare( + self, + *, + operation_id: str, + expected_plan_digest: str, + acknowledge_operational_fk: Sequence[str] = (), + ) -> dict[str, Any]: + """Authorize the exact plan, enter maintenance, and verify its backup.""" + operation_id = _safe_operation_id(operation_id) + if _DIGEST.fullmatch(expected_plan_digest or "") is None: + raise OperationalResetError("an exact preview plan digest is required") + acknowledgements = tuple(sorted(set(acknowledge_operational_fk))) + with self._lock.acquire(): + with self._connect(read_only=False) as connection: + if int(connection.execute("PRAGMA user_version").fetchone()[0]) != RUNTIME_SCHEMA_VERSION: + raise OperationalResetError("installed schema migration 38 is required before prepare") + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", (operation_id,) + ).fetchone() + plan = self._inventory(connection) + # Ignore only this operation's durable maintenance-active + # blocker when an idempotent prepare is being resumed. + active = connection.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0] + if existing is not None: + document = json.loads(existing["document"]) + request_digest = self._request_digest(operation_id, document["plan"], acknowledgements) + if ( + existing["plan_digest"] != expected_plan_digest + or existing["request_digest"] != request_digest + ): + raise OperationalResetError("operation ID is already bound to a different request") + connection.rollback() + existing_request = True + else: + existing_request = False + if existing_request: + pass + elif active is not None: + raise OperationalResetError("another operational reset already owns maintenance") + elif plan["plan_digest"] != expected_plan_digest: + raise OperationalResetError("preview plan changed before maintenance authorization") + elif plan["blockers"]: + raise OperationalResetError("preview contains blocking findings") + required = tuple(sorted(plan["required_fk_acknowledgements"])) + if not existing_request and acknowledgements != required: + raise OperationalResetError("exact operational foreign-key findings must be acknowledged") + if existing_request: + continue_prepare = False + else: + continue_prepare = True + if not continue_prepare: + # Leave both the SQLite transaction and process lock + # before the backup continuation reacquires them. + pass + else: + actor_reference, authority_digest = self._operator_authority(connection) + request_digest = self._request_digest(operation_id, plan, acknowledgements) + now = _now() + document = { + "operation_id": operation_id, "state": "PREPARED", "plan": plan, + "acknowledged_operational_fk_issues": list(acknowledgements), + "request_digest": request_digest, "actor_reference": actor_reference, + "authority_digest": authority_digest, "backup": None, "verification": None, + } + connection.execute( + "INSERT INTO operational_reset_operations VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + operation_id, plan["target"]["runtime_id"], RESET_PROFILE, "PREPARED", + actor_reference, authority_digest, expected_plan_digest, request_digest, + None, None, plan["relevant_revision"], plan["source_revision"], + plan["implementation_digest"], + RUNTIME_SCHEMA_VERSION, RESET_POLICY_VERSION, + plan["target"]["dataset_generation"], None, now, now, _json(document), + ), + ) + connection.execute( + "UPDATE operational_reset_state SET active_operation_id=?,state='PREPARED',updated_at=? WHERE singleton=1", + (operation_id, now), + ) + for item in plan["effect_set"]["external"]: + connection.execute( + "INSERT INTO operational_reset_artifact_steps VALUES (?,?,?,?,?,?,?,?)", + ( + operation_id, item["path"], item["category"], item["digest"], + int(item["size_bytes"]), "PENDING", None, now, + ), + ) + self._audit(connection, operation_id, "PREPARED", { + "plan_digest": expected_plan_digest, "request_digest": request_digest, + "effect_set_digest": plan["effect_set_digest"], "actor_reference": actor_reference, + }) + connection.commit() + except Exception: + connection.rollback() + raise + self._fault_hook("after_prepare", None) + return self._ensure_backup(operation_id) + + def _backup_directory(self, operation_id: str) -> Path: + return self.data_root / "backups" / operation_id + + def _ensure_backup(self, operation_id: str) -> dict[str, Any]: + with self._lock.acquire(): + with self._connect(read_only=False) as source: + operation = source.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", (operation_id,) + ).fetchone() + if operation is None: + raise OperationalResetError("unknown operational reset operation") + if operation["backup_digest"]: + return self._operation_receipt(operation_id) + state = source.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0] + if state != operation_id: + raise OperationalResetError("operation does not own durable maintenance") + document = json.loads(operation["document"]) + plan = document["plan"] + backup = self._backup_directory(operation_id) + backup_parent = backup.parent + if backup_parent.is_symlink(): + raise OperationalResetError("backup directory must not be a symlink") + backup_parent.mkdir(mode=0o700, parents=True, exist_ok=True) + estimated = self.database_path.stat().st_size + sum( + int(item["size_bytes"]) for item in plan["effect_set"]["external"] + if item["category"] == "OPERATIONAL_HISTORY" + ) + if int(self._disk_usage(backup_parent).free) < estimated * 2 + 1024 * 1024: + raise OperationalResetError("insufficient free space for verified recovery backup") + if backup.exists() and backup.is_symlink(): + raise OperationalResetError("operation backup path must not be a symlink") + backup.mkdir(mode=0o700, exist_ok=True) + os.chmod(backup, 0o700) + backup_db = backup / "forge.db" + if _has_symlink_component(backup_db): + raise OperationalResetError("backup database path must not be a symlink") + destination = sqlite3.connect(backup_db) + try: + source.backup(destination) + destination.commit() + # Make the isolated recovery image self-contained. The + # active source remains WAL-backed; the snapshot must not + # require a sibling WAL/SHM pair to verify or restore. + destination.execute("PRAGMA journal_mode=DELETE") + finally: + destination.close() + os.chmod(backup_db, 0o600) + backup_marker = backup / "runtime-instance.json" + if _has_symlink_component(backup_marker): + raise OperationalResetError("backup marker path must not be a symlink") + shutil.copyfile(self.marker_path, backup_marker) + os.chmod(backup_marker, 0o600) + external_manifest: list[dict[str, Any]] = [] + for item in plan["effect_set"]["external"]: + if item["category"] != "OPERATIONAL_HISTORY": + continue + source_path = self.data_root / item["path"] + self._require_effect_path(source_path, item["path"], item["digest"]) + destination_path = backup / "external" / item["path"] + if _has_symlink_component(destination_path): + raise OperationalResetError("backup external path contains a symlink") + destination_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source_path, destination_path) + os.chmod(destination_path, 0o600) + if _file_digest(destination_path) != item["digest"]: + raise OperationalResetError("external recovery copy failed digest verification") + external_manifest.append({ + "path": item["path"], "digest": item["digest"], "size_bytes": item["size_bytes"], + }) + backup_manifest = { + "contract": "forge-operational-reset-backup-1.0", "operation_id": operation_id, + "runtime_id": operation["runtime_id"], "plan_digest": operation["plan_digest"], + "request_digest": operation["request_digest"], "source_revision": operation["source_revision"], + "source_revision_kind": plan["source_revision_kind"], + "implementation_digest": operation["implementation_digest"], + "product_version": canonical_version(), "schema_version": RUNTIME_SCHEMA_VERSION, + "snapshot_relevant_revision": operation["relevant_revision"], + "database": {"path": "forge.db", "digest": _file_digest(backup_db), "size_bytes": backup_db.stat().st_size}, + "instance_marker": {"path": "runtime-instance.json", "digest": _file_digest(backup / "runtime-instance.json")}, + "external": external_manifest, + "excluded": ["Keychain", "provider login", "venv", "cache", "locks", "prior backups"], + "source_counts": { + item["table"]: item["rows"] for item in plan["effect_set"]["purge_tables"] + }, + "known_source_foreign_key_issues": plan["foreign_key_issues"], + } + backup_digest = _digest(backup_manifest) + backup_manifest["backup_digest"] = backup_digest + manifest_path = backup / "manifest.json" + temporary = backup / "manifest.json.tmp" + if _has_symlink_component(manifest_path) or _has_symlink_component(temporary): + raise OperationalResetError("backup manifest path must not be a symlink") + temporary.write_text(_json(backup_manifest) + "\n", encoding="utf-8") + os.chmod(temporary, 0o600) + os.replace(temporary, manifest_path) + self._verify_backup_files(backup, backup_manifest, expected_digest=backup_digest) + now = _now() + document["state"] = "BACKUP_VERIFIED" + document["backup"] = {"reference": f"backups/{operation_id}", "digest": backup_digest} + source.execute("BEGIN IMMEDIATE") + try: + source.execute( + "UPDATE operational_reset_operations SET state='BACKUP_VERIFIED',backup_digest=?,backup_reference=?,updated_at=?,document=? WHERE operation_id=?", + (backup_digest, f"backups/{operation_id}", now, _json(document), operation_id), + ) + source.execute( + "UPDATE operational_reset_state SET state='BACKUP_VERIFIED',updated_at=? WHERE singleton=1 AND active_operation_id=?", + (now, operation_id), + ) + for item in external_manifest: + source.execute( + "UPDATE operational_reset_artifact_steps SET state='BACKED_UP',backup_relative_path=?,updated_at=? WHERE operation_id=? AND relative_path=?", + ("external/" + item["path"], now, operation_id, item["path"]), + ) + self._audit(source, operation_id, "BACKUP_VERIFIED", { + "backup_reference": f"backups/{operation_id}", "backup_digest": backup_digest, + }) + source.commit() + except Exception: + source.rollback() + raise + self._fault_hook("after_backup", None) + return self._operation_receipt(operation_id) + + def _verify_backup_files( + self, backup: Path, manifest: Mapping[str, Any], *, expected_digest: str | None = None, + ) -> None: + if _has_symlink_component(backup) or not backup.is_dir(): + raise OperationalResetError("recovery backup directory is invalid") + manifest_core = dict(manifest) + declared_digest = manifest_core.pop("backup_digest", None) + if ( + not isinstance(declared_digest, str) + or _digest(manifest_core) != declared_digest + or expected_digest is not None and declared_digest != expected_digest + ): + raise OperationalResetError("recovery backup manifest binding is invalid") + database = backup / str(manifest["database"]["path"]) + marker = backup / str(manifest["instance_marker"]["path"]) + if _has_symlink_component(database) or _has_symlink_component(marker): + raise OperationalResetError("recovery backup contains an unsafe path") + if _file_digest(database) != manifest["database"]["digest"] or _file_digest(marker) != manifest["instance_marker"]["digest"]: + raise OperationalResetError("recovery backup digest verification failed") + # Restore into an isolated, non-active root and read it there. No + # scheduler, provider, intake, or second RuntimeDatabase is started. + with tempfile.TemporaryDirectory(prefix="forge-reset-restore-check-") as directory: + restored = Path(directory) / "forge.db" + shutil.copyfile(database, restored) + check = sqlite3.connect(restored.resolve().as_uri() + "?mode=ro", uri=True) + check.row_factory = sqlite3.Row + try: + if check.execute("PRAGMA quick_check").fetchone()[0] != "ok": + raise OperationalResetError("isolated recovery backup quick_check failed") + if self._relevant_revision(check) != manifest["snapshot_relevant_revision"]: + raise OperationalResetError("isolated recovery backup does not match the approved source revision") + finally: + check.close() + for item in manifest["external"]: + archived = backup / "external" / str(item["path"]) + try: + archived.resolve(strict=True).relative_to(backup.resolve(strict=True)) + except (OSError, ValueError) as error: + raise OperationalResetError("isolated external recovery path is unsafe") from error + if _has_symlink_component(archived) or not archived.is_file() or _file_digest(archived) != item["digest"]: + raise OperationalResetError("isolated external recovery artifact verification failed") + + def _require_effect_path(self, path: Path, relative: str, expected_digest: str) -> None: + try: + resolved = path.resolve(strict=True) + except OSError as error: + raise OperationalResetError(f"planned external path is unavailable: {relative}") from error + try: + resolved.relative_to(self.data_root.resolve()) + except ValueError as error: + raise OperationalResetError("external effect path escapes the Forge data root") from error + if path.is_symlink() or not path.is_file() or _file_digest(path) != expected_digest: + raise OperationalResetError(f"planned external artifact changed: {relative}") + + def _bound_operation( + self, connection: sqlite3.Connection, operation_id: str, *, + plan_digest: str, request_digest: str, backup_digest: str, + ) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", (_safe_operation_id(operation_id),) + ).fetchone() + if row is None: + raise OperationalResetError("unknown operational reset operation") + if ( + row["plan_digest"] != plan_digest or row["request_digest"] != request_digest + or row["backup_digest"] != backup_digest + ): + raise OperationalResetError("operation identity, plan, request, or backup binding differs") + active = connection.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0] + if active != operation_id: + raise OperationalResetError("operation does not own durable maintenance") + if row["source_revision"] != self._source_revision(): + raise OperationalResetError("source revision changed after authorization") + if row["implementation_digest"] != self._implementation_digest(): + raise OperationalResetError("maintenance service revision changed after authorization") + return row + + @staticmethod + def _tombstone_values(connection: sqlite3.Connection) -> list[tuple[str, str]]: + queries: tuple[tuple[str, str], ...] = ( + ("mission_id", "SELECT mission_id FROM mission_state"), + ("action_id", "SELECT action_id FROM scheduler_submissions"), + ("submission_id", "SELECT submission_id FROM scheduler_submissions"), + ("correlation_id", "SELECT correlation_id FROM execution_host_bindings"), + ("correlation_id", "SELECT correlation_id FROM execution_host_exchange_audit"), + ("execution_receipt_id", "SELECT receipt_id FROM execution_receipts"), + ("action_derivation_id", "SELECT derivation_id FROM action_derivations"), + ("generation_request_digest", "SELECT generation_request_digest FROM action_derivations WHERE generation_request_digest IS NOT NULL"), + ("token_preflight_receipt_id", "SELECT receipt_id FROM token_preflight_receipts"), + ("token_request_digest", "SELECT request_digest FROM token_preflight_receipts"), + ("reattempt_authorization_id", "SELECT authorization_id FROM action_derivation_reattempt_authorizations"), + ) + values: set[tuple[str, str]] = set() + for kind, query in queries: + for row in connection.execute(query): + if row[0] is not None and str(row[0]): + values.add((kind, str(row[0]))) + return sorted(values) + + def _apply_database(self, row: sqlite3.Row) -> None: + operation_id = str(row["operation_id"]) + document = json.loads(row["document"]) + plan = document["plan"] + with self._connect(read_only=False) as connection: + connection.execute("BEGIN IMMEDIATE") + trigger_sql: list[str] = [] + try: + current = self._relevant_revision(connection) + if current != row["relevant_revision"]: + raise OperationalResetError("meaningful source data changed after the approved preview") + external, unknown = self._external_inventory() + effects = [item for item in external if item.get("effect") in {"ARCHIVE_AND_REMOVE", "REMOVE_CACHE"}] + preserved_external = [ + item for item in external + if item.get("effect") == "PRESERVE" + and item.get("category") == "INSTALLATION_AND_CONFIGURATION" + and not str(item.get("path", "")).startswith("backups/") + ] + if ( + unknown or _digest(effects) != plan["external_inventory_digest"] + or _digest(preserved_external) != plan["preserved_external_digest"] + ): + raise OperationalResetError("external operational data changed after the approved preview") + if plan["source_revision"] != self._source_revision(): + raise OperationalResetError("source revision changed after preview") + if plan["implementation_digest"] != self._implementation_digest(): + raise OperationalResetError("maintenance implementation changed after preview") + maintenance_trigger_operations = { + **{table: "delete" for table in PURGE_ORDER}, + "planning_provider_generation_permits": "update", + } + for table, operation in sorted(maintenance_trigger_operations.items()): + name = f"operational_reset_block_{table}_{operation}" + trigger = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='trigger' AND name=? AND tbl_name=?", + (name, table), + ).fetchone() + if trigger is None or not trigger[0]: + raise OperationalResetError(f"required durable writer-fence trigger is missing: {name}") + trigger_sql.append(str(trigger[0])) + connection.execute(f'DROP TRIGGER "{name}"') + for table, names in _IMMUTABLE_DELETE_TRIGGERS.items(): + for name in names: + trigger = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='trigger' AND name=? AND tbl_name=?", + (name, table), + ).fetchone() + if trigger is None or not trigger[0]: + raise OperationalResetError(f"required immutable trigger is missing: {name}") + trigger_sql.append(str(trigger[0])) + connection.execute(f'DROP TRIGGER "{name}"') + retired_at = _now() + retired_values = self._tombstone_values(connection) + for kind, identifier in retired_values: + connection.execute( + "INSERT INTO operational_reset_tombstones VALUES (?,?,?,?,?) " + "ON CONFLICT(record_kind,record_id) DO NOTHING", + (kind, identifier, _digest({"kind": kind, "id": identifier}), operation_id, retired_at), + ) + connection.execute( + "UPDATE planning_provider_generation_permits " + "SET state='INVALIDATED_BY_OPERATIONAL_RESET',updated_at=? " + "WHERE state IN ('PENDING','TRANSPORT_COMMITTED')", + (retired_at,), + ) + for table in PURGE_ORDER: + connection.execute(f'DELETE FROM "{table}"') + for sql in trigger_sql: + connection.execute(sql) + generation_before = int(row["generation_before"]) + generation_after = generation_before if plan["no_op"] else generation_before + 1 + document["state"] = "DATABASE_APPLIED" + document["generation_after"] = generation_after + connection.execute( + "UPDATE operational_reset_operations SET state='DATABASE_APPLIED',generation_after=?,updated_at=?,document=? WHERE operation_id=?", + (generation_after, retired_at, _json(document), operation_id), + ) + connection.execute( + "UPDATE operational_reset_state SET dataset_generation=?,state='DATABASE_APPLIED',updated_at=? " + "WHERE singleton=1 AND active_operation_id=?", + (generation_after, retired_at, operation_id), + ) + self._audit(connection, operation_id, "DATABASE_APPLIED", { + "generation_before": generation_before, "generation_after": generation_after, + "no_op": plan["no_op"], "tombstone_count": len(retired_values), + }) + self._fault_hook("before_database_commit", None) + connection.commit() + except Exception: + connection.rollback() + raise + self._fault_hook("after_database_commit", None) + + def _apply_artifacts(self, operation_id: str) -> None: + with self._connect(read_only=False) as connection: + steps = connection.execute( + "SELECT * FROM operational_reset_artifact_steps WHERE operation_id=? ORDER BY relative_path", + (operation_id,), + ).fetchall() + backup = self._backup_directory(operation_id) + for step in steps: + if step["state"] == "REMOVED": + continue + relative = str(step["relative_path"]) + source = self.data_root / relative + expected = str(step["source_digest"]) + if source.exists(): + self._require_effect_path(source, relative, expected) + if step["classification"] == "OPERATIONAL_HISTORY": + archived = backup / str(step["backup_relative_path"]) + if not archived.is_file() or _file_digest(archived) != expected: + raise OperationalResetError("recovery artifact is unavailable before active removal") + self._fault_hook("before_artifact_remove", relative) + source.unlink() + self._fault_hook("after_artifact_remove", relative) + elif step["classification"] == "OPERATIONAL_HISTORY": + archived = backup / str(step["backup_relative_path"]) + if not archived.is_file() or _file_digest(archived) != expected: + raise OperationalResetError("missing active artifact has no verified recovery copy") + with self._connect(read_only=False) as connection: + now = _now() + with connection: + connection.execute( + "UPDATE operational_reset_artifact_steps SET state='REMOVED',updated_at=? WHERE operation_id=? AND relative_path=?", + (now, operation_id, relative), + ) + self._audit(connection, operation_id, "ARTIFACT_REMOVED", { + "path_digest": _digest(relative), "classification": step["classification"], + }) + with self._connect(read_only=False) as connection: + row = connection.execute( + "SELECT document FROM operational_reset_operations WHERE operation_id=?", (operation_id,) + ).fetchone() + document = json.loads(row[0]) + document["state"] = "APPLIED" + now = _now() + with connection: + connection.execute( + "UPDATE operational_reset_operations SET state='APPLIED',updated_at=?,document=? WHERE operation_id=?", + (now, _json(document), operation_id), + ) + connection.execute( + "UPDATE operational_reset_state SET state='APPLIED',updated_at=? WHERE singleton=1 AND active_operation_id=?", + (now, operation_id), + ) + self._audit(connection, operation_id, "APPLIED", {"database_and_artifacts": "complete"}) + + def apply( + self, *, operation_id: str, plan_digest: str, request_digest: str, backup_digest: str, + ) -> dict[str, Any]: + """Apply or idempotently reconcile the already-authorized operation.""" + with self._lock.acquire(): + with self._connect(read_only=False) as connection: + row = self._bound_operation( + connection, operation_id, plan_digest=plan_digest, + request_digest=request_digest, backup_digest=backup_digest, + ) + state = str(row["state"]) + if state == "BACKUP_VERIFIED": + self._apply_database(row) + state = "DATABASE_APPLIED" + if state in {"DATABASE_APPLIED", "APPLIED"}: + self._apply_artifacts(operation_id) + elif state in {"VERIFIED", "COMPLETED"}: + return self._operation_receipt(operation_id) + else: + raise OperationalResetError(f"operation is not ready to apply from state {state}") + return self._operation_receipt(operation_id) + + def verify( + self, *, operation_id: str, plan_digest: str, request_digest: str, backup_digest: str, + ) -> dict[str, Any]: + """Prove clean operational state and exact preserved bindings.""" + with self._lock.acquire(): + with self._connect(read_only=False) as connection: + row = self._bound_operation( + connection, operation_id, plan_digest=plan_digest, + request_digest=request_digest, backup_digest=backup_digest, + ) + if row["state"] == "VERIFIED": + return self._operation_receipt(operation_id) + if row["state"] != "APPLIED": + raise OperationalResetError("verification requires a fully applied operation") + document = json.loads(row["document"]) + plan = document["plan"] + integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] + quick = connection.execute("PRAGMA quick_check").fetchone()[0] + foreign_keys = list(connection.execute("PRAGMA foreign_key_check")) + counts = { + table: int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + for table in PURGE_TABLES + } + if integrity != "ok" or quick != "ok" or foreign_keys: + raise OperationalResetError("post-reset SQLite integrity verification failed") + if any(counts.values()): + raise OperationalResetError("post-reset operational tables are not empty") + if connection.execute( + "SELECT 1 FROM planning_provider_generation_permits " + "WHERE state IN ('PENDING','TRANSPORT_COMMITTED') LIMIT 1" + ).fetchone(): + raise OperationalResetError("a pre-reset generation permit remains executable") + preserved: dict[str, dict[str, Any]] = {} + for table in PRESERVE_TABLES: + actual = self._row_digest(connection, table, post_reset=(table == "planning_provider_generation_permits")) + expected = plan["preserved"][table]["expected_post_reset_digest"] + if actual != expected: + raise OperationalResetError(f"preserved table changed unexpectedly: {table}") + preserved[table] = { + "rows": int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]), + "digest": actual, + } + external, unknown = self._external_inventory() + active_effects = [ + item for item in external if item.get("effect") in {"ARCHIVE_AND_REMOVE", "REMOVE_CACHE"} + ] + if unknown or active_effects: + raise OperationalResetError("post-reset active external operational data is not empty") + preserved_external = [ + item for item in external + if item.get("effect") == "PRESERVE" + and item.get("category") == "INSTALLATION_AND_CONFIGURATION" + and not str(item.get("path", "")).startswith("backups/") + ] + if _digest(preserved_external) != plan["preserved_external_digest"]: + raise OperationalResetError("preserved external installation artifacts changed") + manifest = json.loads((self._backup_directory(operation_id) / "manifest.json").read_text(encoding="utf-8")) + self._verify_backup_files( + self._backup_directory(operation_id), manifest, expected_digest=backup_digest, + ) + marker = self.marker_path.read_text(encoding="utf-8").strip() + if marker != row["runtime_id"]: + raise OperationalResetError("runtime identity changed during reset") + generation = int(connection.execute( + "SELECT dataset_generation FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0]) + if generation != int(row["generation_after"]): + raise OperationalResetError("dataset generation does not match the applied operation") + verification = { + "integrity_check": "ok", "quick_check": "ok", "foreign_key_errors": 0, + "operational_rows": counts, "preserved": preserved, + "runtime_id": marker, "dataset_generation": generation, + "backup_digest": backup_digest, "active_external_effect_entries": 0, + } + verification_digest = _digest(verification) + verification["verification_digest"] = verification_digest + document["state"] = "VERIFIED" + document["verification"] = verification + now = _now() + with connection: + connection.execute( + "UPDATE operational_reset_operations SET state='VERIFIED',updated_at=?,document=? WHERE operation_id=?", + (now, _json(document), operation_id), + ) + connection.execute( + "UPDATE operational_reset_state SET state='VERIFIED',updated_at=? WHERE singleton=1 AND active_operation_id=?", + (now, operation_id), + ) + self._audit(connection, operation_id, "VERIFIED", { + "verification_digest": verification_digest, "dataset_generation": generation, + }) + return self._operation_receipt(operation_id) + + def resume( + self, *, operation_id: str, plan_digest: str, request_digest: str, backup_digest: str | None, + ) -> dict[str, Any]: + """Reconcile the same operation; never create a replacement reset.""" + status = self.status(operation_id=operation_id) + state = status["state"] + if state == "PREPARED": + receipt = self._ensure_backup(operation_id) + backup_digest = str(receipt["backup_digest"]) + state = str(receipt["state"]) + if not backup_digest: + raise OperationalResetError("resume requires the operation's verified backup digest") + if state in {"BACKUP_VERIFIED", "DATABASE_APPLIED", "APPLIED"}: + receipt = self.apply( + operation_id=operation_id, plan_digest=plan_digest, + request_digest=request_digest, backup_digest=backup_digest, + ) + state = str(receipt["state"]) + if state == "APPLIED": + return self.verify( + operation_id=operation_id, plan_digest=plan_digest, + request_digest=request_digest, backup_digest=backup_digest, + ) + if state in {"VERIFIED", "COMPLETED"}: + return self._operation_receipt(operation_id) + raise OperationalResetError(f"operation cannot be resumed from state {state}") + + def finish( + self, *, operation_id: str, verification_digest: str | None = None, + cancel_before_apply: bool = False, + ) -> dict[str, Any]: + """Leave maintenance only after proof, or cancel before any purge.""" + with self._lock.acquire(): + with self._connect(read_only=False) as connection: + row = connection.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", (_safe_operation_id(operation_id),) + ).fetchone() + if row is None: + raise OperationalResetError("unknown operational reset operation") + active = connection.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0] + if active != operation_id: + if row["state"] in {"COMPLETED", "CANCELLED"}: + return self._operation_receipt(operation_id) + raise OperationalResetError("operation does not own durable maintenance") + document = json.loads(row["document"]) + if cancel_before_apply: + if row["state"] not in {"PREPARED", "BACKUP_VERIFIED"}: + raise OperationalResetError("only an unapplied operation can be cancelled") + final_state = "CANCELLED" + else: + if row["state"] != "VERIFIED": + raise OperationalResetError("maintenance can finish only after successful verification") + actual = document.get("verification", {}).get("verification_digest") + if not verification_digest or verification_digest != actual: + raise OperationalResetError("exact verification digest is required to finish") + final_state = "COMPLETED" + document["state"] = final_state + now = _now() + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + "UPDATE operational_reset_operations SET state=?,updated_at=?,document=? WHERE operation_id=?", + (final_state, now, _json(document), operation_id), + ) + self._audit(connection, operation_id, final_state, { + "verification_digest": verification_digest if final_state == "COMPLETED" else None, + }) + connection.execute( + "UPDATE operational_reset_state SET active_operation_id=NULL,state='IDLE',updated_at=? " + "WHERE singleton=1 AND active_operation_id=?", + (now, operation_id), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return self._operation_receipt(operation_id) + + def status(self, *, operation_id: str | None = None) -> dict[str, Any]: + """Read durable maintenance state without opening a mutating runtime.""" + with self._connect(read_only=True) as connection: + if int(connection.execute("PRAGMA user_version").fetchone()[0]) < 38: + return { + "status": "UNAVAILABLE", "state": "SCHEMA_MIGRATION_REQUIRED", + "schema_version": int(connection.execute("PRAGMA user_version").fetchone()[0]), + } + if operation_id is None: + state = connection.execute("SELECT * FROM operational_reset_state WHERE singleton=1").fetchone() + metadata = dict(connection.execute( + "SELECT key,value FROM runtime_metadata WHERE key IN ('runtime_id','schema_version')" + )) + return { + "status": "OK", "state": str(state["state"]), + "active_operation_id": state["active_operation_id"], + "dataset_generation": int(state["dataset_generation"]), + "runtime_id": metadata.get("runtime_id"), + "database_identity": _digest(str(self.database_path.resolve())), + "schema_version": int(metadata.get("schema_version", RUNTIME_SCHEMA_VERSION)), + } + return self._operation_receipt(_safe_operation_id(operation_id)) + + def _operation_receipt(self, operation_id: str) -> dict[str, Any]: + with self._connect(read_only=True) as connection: + row = connection.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", (operation_id,) + ).fetchone() + if row is None: + raise OperationalResetError("unknown operational reset operation") + document = json.loads(row["document"]) + audit_count = int(connection.execute( + "SELECT COUNT(*) FROM operational_reset_audit WHERE operation_id=?", (operation_id,) + ).fetchone()[0]) + pending_artifacts = int(connection.execute( + "SELECT COUNT(*) FROM operational_reset_artifact_steps WHERE operation_id=? AND state!='REMOVED'", + (operation_id,), + ).fetchone()[0]) + return { + "contract": "forge-operational-reset-receipt-1.0", + "operation_id": operation_id, "state": str(row["state"]), + "runtime_id": str(row["runtime_id"]), "profile": str(row["profile"]), + "plan_digest": str(row["plan_digest"]), "request_digest": str(row["request_digest"]), + "effect_set_digest": document["plan"]["effect_set_digest"], + "backup_reference": row["backup_reference"], "backup_digest": row["backup_digest"], + "source_revision": str(row["source_revision"]), + "source_revision_kind": document["plan"]["source_revision_kind"], + "implementation_digest": str(row["implementation_digest"]), + "schema_version": int(row["schema_version"]), + "policy_version": str(row["policy_version"]), + "generation_before": int(row["generation_before"]), "generation_after": row["generation_after"], + "verification_digest": ( + None if document.get("verification") is None + else document["verification"].get("verification_digest") + ), + "audit_events": audit_count, "pending_artifacts": pending_artifacts, + "summary": "secret-free Forge operational reset maintenance receipt", + } + + def operator_envelope(self, command: str, payload: Mapping[str, Any]) -> dict[str, Any]: + """Project one stable, secret-free cross-product operator envelope.""" + plan: Mapping[str, Any] | None = payload if "target" in payload else None + operation_id = payload.get("operation_id") + if plan is None and isinstance(operation_id, str): + with self._connect(read_only=True) as connection: + if int(connection.execute("PRAGMA user_version").fetchone()[0]) >= 38: + row = connection.execute( + "SELECT document FROM operational_reset_operations WHERE operation_id=?", (operation_id,) + ).fetchone() + if row is not None: + plan = json.loads(row[0])["plan"] + target = {} if plan is None else dict(plan["target"]) + counts = {} if plan is None else { + item["table"]: item["rows"] for item in plan["effect_set"]["purge_tables"] + } + preserved = {} if plan is None else plan.get("preserved", {}) + state = str(payload.get("state") or payload.get("status") or "UNKNOWN") + backup_digest = payload.get("backup_digest") + backup_reference = payload.get("backup_reference") + return { + "contract_version": "operational-reset-v1", + "product": "forge", + "command": command, + "operation_id": operation_id, + "state": state, + "allowed": bool( + payload.get("execution_allowed", state not in {"BLOCKED", "ERROR", "UNAVAILABLE"}) + ), + "target": { + "instance_id": target.get("runtime_id") or payload.get("runtime_id"), + "database_path": target.get("database", str(self.database_path)), + "database_identity": target.get("database_binding_digest") or payload.get("database_identity"), + "schema_version": target.get("schema_version", payload.get("schema_version")), + }, + "profile": payload.get("profile", RESET_PROFILE), + "dataset_generation": target.get( + "dataset_generation", payload.get("generation_after", payload.get("dataset_generation")) + ), + "plan_digest": payload.get("plan_digest"), + "relevant_revision_digest": ( + None if plan is None else plan.get("relevant_revision") + ), + "backup": ( + None if not backup_digest else { + "manifest": backup_reference, "digest": backup_digest, + "verified": state in {"BACKUP_VERIFIED", "DATABASE_APPLIED", "APPLIED", "VERIFIED", "COMPLETED"}, + } + ), + "counts": counts, + "blockers": list(payload.get("blockers", ())), + "integrity": {} if plan is None else dict(plan.get("integrity", {})), + "preserved_bindings_digest": _digest({ + "tables": { + table: value.get("expected_post_reset_digest") for table, value in sorted(preserved.items()) + }, + "external": None if plan is None else plan.get("preserved_external_digest"), + }) if preserved else None, + "details": dict(payload), + } diff --git a/knowledge/bootstrap/10_ROADMAP.md b/knowledge/bootstrap/10_ROADMAP.md index 610f1f5..cff61a3 100644 --- a/knowledge/bootstrap/10_ROADMAP.md +++ b/knowledge/bootstrap/10_ROADMAP.md @@ -48,6 +48,15 @@ guarded controls after Mission views; FOC-Q qualifies the integrated installed console. The JSON records the exact dependencies. This feature is not a new first-E2E prerequisite and changes no executable bootstrap graph or live Mission. +The bounded product-owned operational-history reset is an implemented +clean-state preparation slice beneath FC-STATE/FOC-5, with its own +[contract](../../docs/architecture/FORGE_OPERATIONAL_RESET_V1.md), +[runbook](../../docs/operations/FORGE_OPERATIONAL_RESET_RUNBOOK.md) and +[delivery status](../../docs/reports/FORGE_OPERATIONAL_RESET_DELIVERY_STATUS.md). +It does not complete any FOC node, full FC-STATE, Console, Server daemon, +export/import/relocation/VACUUM or universal installer. A source implementation, +protected release, installed command and live read-only preview remain distinct. + This is instance administration, not a revival of Forge Studio or a replacement for Workspace's project, Mission-authoring, approval, chat and policy UX. EP retains execution/assurance/Git mutation; Forge Platform retains installation diff --git a/product-version.json b/product-version.json index d93825a..c200171 100644 --- a/product-version.json +++ b/product-version.json @@ -1,5 +1,5 @@ { "product": "forge", "schema_version": 1, - "version": "2.7.21" + "version": "2.7.22" } diff --git a/tests/test_operational_reset.py b/tests/test_operational_reset.py new file mode 100644 index 0000000..81b1eb4 --- /dev/null +++ b/tests/test_operational_reset.py @@ -0,0 +1,517 @@ +from __future__ import annotations + +from collections import namedtuple +import json +from pathlib import Path +import sqlite3 +import subprocess +import sys +from tempfile import TemporaryDirectory +import unittest + +from forge.operator_identity import InstallationOperatorService, NamedOperatorIdentity +from forge.runtime import RuntimeBootstrap, RuntimeMaintenanceActive +from forge.runtime.operational_reset import ( + MAINTENANCE_TABLES, + PURGE_TABLES, + RESET_PROFILE, + TABLE_CLASSIFICATION, + ForgeOperationalResetService, + OperationalResetError, +) + + +IDENTITY = NamedOperatorIdentity("123e4567-e89b-42d3-a456-426614174000", 501) +Disk = namedtuple("Disk", "total used free") + + +class OneShotFault: + def __init__(self, event: str) -> None: + self.event = event + self.fired = False + + def __call__(self, event: str, _path: str | None) -> None: + if event == self.event and not self.fired: + self.fired = True + raise RuntimeError("synthetic crash: " + event) + + +class OperationalResetTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = TemporaryDirectory() + self.root = Path(self.temporary.name) + self.database = RuntimeBootstrap(data_root=self.root, forge_version="2.7.21").open() + InstallationOperatorService(self.database, lambda: IDENTITY).first_bind() + + def tearDown(self) -> None: + if getattr(self, "database", None) is not None: + self.database.close() + self.temporary.cleanup() + + def service(self, **values: object) -> ForgeOperationalResetService: + return ForgeOperationalResetService(self.root, identity_resolver=lambda: IDENTITY, **values) + + def seed_operational_state(self, *, external: bool = True) -> None: + document = { + "id": "MISSION-0042", "mission_id": "MISSION-0042", "lifecycle": "ACTIVE", + "status": "ACTIVE", "progress": {}, "resume_point": {}, "execution_policy": {}, + } + self.database._connection.execute( + "INSERT INTO mission_state VALUES (?,?,?,?,?,?,?,?,?)", + ("MISSION-0042", "ACTIVE", "ACTIVE", None, None, "{}", "{}", "{}", json.dumps(document)), + ) + self.database._connection.execute( + "INSERT INTO mission_id_allocations VALUES (?,?,?)", ("MISSION-0042", "now", "source-42") + ) + submission = { + "submission_id": "submission-old", "mission_id": "MISSION-0042", "intent_id": "intent-old", + "action_id": "action-old", "iteration": 1, "state": "CREATED", "envelope": {"old": True}, + } + self.database._connection.execute( + "INSERT INTO scheduler_submissions VALUES (?,?,?,?,?,?,?,?,?)", + ("submission-old", "MISSION-0042", "intent-old", "action-old", 1, "CREATED", None, None, json.dumps(submission)), + ) + self.database._connection.execute( + "INSERT INTO execution_host_bindings VALUES (?,?)", + ("correlation-old", json.dumps({"correlation_id": "correlation-old"})), + ) + self.database._connection.execute( + "INSERT INTO planning_provider_security_config VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + ("config", "provider", "keychain://forge/account", 1, "operator", 1, "now", "now", "model", 5, 10, 20, 5), + ) + self.database._connection.execute( + "INSERT INTO planning_provider_generation_permits VALUES (?,?,?,?,?,?,?,?)", + ("permit-old", "provider", 1, "sha256:" + "1" * 64, "sha256:" + "2" * 64, "PENDING", "now", "now"), + ) + self.database._connection.execute( + "INSERT INTO execution_host_peer_configuration VALUES (?,?,?,?,?)", + (1, "binding", 1, "sha256:" + "3" * 64, json.dumps({ + "binding_id": "binding", "credential_reference": "keychain://forge/ep-token", + })), + ) + self.database._connection.commit() + if external: + operational = self.root / "artifacts" / "operational" + operational.mkdir() + (operational / "evidence.json").write_text('{"evidence":true}\n', encoding="utf-8") + (self.root / "artifacts" / "controlled-installation-e2e-starter-exit-2026-09-14.md").write_text( + "installation qualification\n", encoding="utf-8", + ) + + def prepare(self, service: ForgeOperationalResetService, operation: str = "forge-reset-test-0001") -> dict[str, object]: + plan = service.preview() + self.assertEqual(plan["status"], "READY", plan["blockers"]) + return service.prepare(operation_id=operation, expected_plan_digest=str(plan["plan_digest"])) + + def complete(self, service: ForgeOperationalResetService, operation: str = "forge-reset-test-0001") -> dict[str, object]: + receipt = self.prepare(service, operation) + receipt = service.apply( + operation_id=operation, plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + receipt = service.verify( + operation_id=operation, plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + return service.finish(operation_id=operation, verification_digest=str(receipt["verification_digest"])) + + def test_mapping_covers_every_application_table_and_preview_is_read_only(self) -> None: + runtime_lock = self.root / "forge-runtime-mutation.lock" + runtime_lock.touch() + before = { + table: self.database._connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + for table in TABLE_CLASSIFICATION + } + plan = self.service().preview() + after = { + table: self.database._connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + for table in TABLE_CLASSIFICATION + } + actual = { + row[0] for row in self.database._connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + } + self.assertEqual(set(TABLE_CLASSIFICATION), actual) + self.assertEqual(before, after) + self.assertEqual(plan["profile"], RESET_PROFILE) + self.assertIn( + {"path": "forge-runtime-mutation.lock", "category": "SYSTEM_RUNTIME_CONTROL", "effect": "PRESERVE", "size_bytes": 0}, + plan["external_control_entries"], + ) + self.assertIsNone(self.database._connection.execute( + "SELECT active_operation_id FROM operational_reset_state" + ).fetchone()[0]) + + def test_unknown_table_root_file_and_symlink_fail_closed_without_deleting_user_data(self) -> None: + self.database._connection.execute("CREATE TABLE future_product_data(id TEXT)") + self.database._connection.commit() + user_file = self.root / "notes.txt" + user_file.write_text("keep me", encoding="utf-8") + (self.root / "artifacts" / "escape").symlink_to(user_file) + plan = self.service().preview() + codes = {item["code"] for item in plan["blockers"]} + self.assertIn("UNKNOWN_OR_INCOMPLETE_SCHEMA", codes) + self.assertIn("UNKNOWN_OR_UNSAFE_EXTERNAL_DATA", codes) + self.assertEqual(user_file.read_text(encoding="utf-8"), "keep me") + + def test_missing_durable_writer_fence_blocks_prepare(self) -> None: + self.database._connection.execute("DROP TRIGGER operational_reset_block_mission_state_insert") + self.database._connection.commit() + plan = self.service().preview() + self.assertIn("MAINTENANCE_FENCE_INCOMPLETE", {item["code"] for item in plan["blockers"]}) + with self.assertRaisesRegex(OperationalResetError, "blocking findings"): + self.service().prepare( + operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"]), + ) + + def test_explicit_data_root_symlink_is_rejected_before_canonicalization(self) -> None: + alias = self.root.parent / (self.root.name + "-alias") + alias.symlink_to(self.root, target_is_directory=True) + self.addCleanup(alias.unlink) + service = ForgeOperationalResetService(alias, identity_resolver=lambda: IDENTITY) + with self.assertRaisesRegex(OperationalResetError, "data root.*symlink"): + service.preview() + + def test_explicit_data_root_with_symlink_parent_is_rejected_before_canonicalization(self) -> None: + alias = self.root.parent / (self.root.name + "-parent-alias") + alias.symlink_to(self.root.parent, target_is_directory=True) + self.addCleanup(alias.unlink) + service = ForgeOperationalResetService( + alias / self.root.name, identity_resolver=lambda: IDENTITY, + ) + with self.assertRaisesRegex(OperationalResetError, "data root.*symlink"): + service.preview() + + def test_schema37_preview_inventory_is_read_only_and_prepare_requires_installed_migration(self) -> None: + self.database.close() + self.database = None + connection = sqlite3.connect(self.root / "forge.db") + for (trigger,) in connection.execute( + "SELECT name FROM sqlite_master WHERE type='trigger' AND name LIKE 'operational_reset_%'" + ).fetchall(): + connection.execute(f'DROP TRIGGER "{trigger}"') + for table in MAINTENANCE_TABLES: + triggers = connection.execute( + "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=?", (table,) + ).fetchall() + for (trigger,) in triggers: + connection.execute(f'DROP TRIGGER "{trigger}"') + for table in reversed(MAINTENANCE_TABLES): + connection.execute(f'DROP TABLE "{table}"') + connection.execute("UPDATE runtime_metadata SET value='37' WHERE key IN ('schema_version','migration_version','last_migration')") + connection.execute("PRAGMA user_version=37") + connection.commit() + before = (self.root / "forge.db").stat().st_size + connection.close() + plan = self.service().preview() + self.assertEqual(plan["target"]["schema_version"], 37) + self.assertIn("SCHEMA_MIGRATION_REQUIRED", {item["code"] for item in plan["blockers"]}) + self.assertEqual((self.root / "forge.db").stat().st_size, before) + + def test_reset_preserves_identity_binding_credentials_security_allocator_and_qualification_artifact(self) -> None: + self.seed_operational_state() + runtime_id = self.database.runtime_identity.runtime_id + original_binding = self.database._connection.execute( + "SELECT document FROM execution_host_peer_configuration" + ).fetchone()[0] + original_config = self.database._connection.execute( + "SELECT secret_reference FROM planning_provider_security_config" + ).fetchone()[0] + self.database.close() + self.database = None + service = self.service() + receipt = self.complete(service) + self.assertEqual(receipt["state"], "COMPLETED") + reopened = RuntimeBootstrap(data_root=self.root, forge_version="2.7.21").open() + try: + self.assertEqual(reopened.runtime_identity.runtime_id, runtime_id) + for table in PURGE_TABLES: + self.assertEqual(reopened._connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0], 0, table) + self.assertEqual(reopened._connection.execute("SELECT COUNT(*) FROM mission_id_allocations").fetchone()[0], 1) + self.assertEqual(reopened._connection.execute( + "SELECT document FROM execution_host_peer_configuration" + ).fetchone()[0], original_binding) + self.assertEqual(reopened._connection.execute( + "SELECT secret_reference FROM planning_provider_security_config" + ).fetchone()[0], original_config) + self.assertEqual(reopened._connection.execute( + "SELECT state FROM planning_provider_generation_permits" + ).fetchone()[0], "INVALIDATED_BY_OPERATIONAL_RESET") + self.assertGreater(reopened._connection.execute( + "SELECT COUNT(*) FROM operational_reset_tombstones" + ).fetchone()[0], 0) + finally: + reopened.close() + self.assertTrue((self.root / "artifacts" / "controlled-installation-e2e-starter-exit-2026-09-14.md").is_file()) + self.assertFalse((self.root / "artifacts" / "operational" / "evidence.json").exists()) + encoded = json.dumps(receipt) + self.assertNotIn("keychain://", encoded) + + def test_raw_idle_connection_needs_no_udf_and_existing_owning_writer_is_fenced(self) -> None: + raw = sqlite3.connect(self.root / "forge.db") + raw.execute("UPDATE runtime_metadata SET value=value WHERE key='status'") + raw.commit() + existing = self.database + receipt = self.prepare(self.service()) + with self.assertRaises(sqlite3.DatabaseError): + raw.execute("UPDATE runtime_metadata SET value=value WHERE key='status'") + raw.rollback() + raw.close() + with self.assertRaises(sqlite3.DatabaseError): + existing.save_planning_state({ + "planner_version": "blocked", "current_queue": [], "pending_engineering_actions": [], + "blocked_engineering_actions": [], "execution_policy": {}, "planner_runtime_metadata": {}, + }) + with self.assertRaises(RuntimeMaintenanceActive): + RuntimeBootstrap(data_root=self.root, forge_version="2.7.21").open() + self.service().finish(operation_id=str(receipt["operation_id"]), cancel_before_apply=True) + + def test_stale_plan_wrong_request_and_operation_id_reuse_are_rejected(self) -> None: + plan = self.service().preview() + self.database._connection.execute("INSERT INTO mission_id_allocations VALUES (?,?,?)", ("MISSION-0009", "now", "new")) + self.database._connection.commit() + with self.assertRaisesRegex(OperationalResetError, "plan changed"): + self.service().prepare(operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"])) + current = self.prepare(self.service()) + with self.assertRaisesRegex(OperationalResetError, "different request"): + self.service().prepare( + operation_id="forge-reset-test-0001", expected_plan_digest="sha256:" + "0" * 64, + ) + with self.assertRaisesRegex(OperationalResetError, "differs"): + self.service().apply( + operation_id="forge-reset-test-0001", plan_digest=str(current["plan_digest"]), + request_digest="sha256:" + "0" * 64, backup_digest=str(current["backup_digest"]), + ) + self.service().finish(operation_id="forge-reset-test-0001", cancel_before_apply=True) + + def test_insufficient_disk_leaves_durable_safe_maintenance_and_can_cancel(self) -> None: + service = self.service(disk_usage=lambda _path: Disk(1, 1, 0)) + plan = service.preview() + with self.assertRaisesRegex(OperationalResetError, "insufficient"): + service.prepare(operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"])) + self.assertEqual(service.status()["active_operation_id"], "forge-reset-test-0001") + result = service.finish(operation_id="forge-reset-test-0001", cancel_before_apply=True) + self.assertEqual(result["state"], "CANCELLED") + + def test_backup_nested_symlink_is_rejected_without_writing_outside_root(self) -> None: + self.seed_operational_state() + escape = Path(self.temporary.name + "-backup-escape") + escape.mkdir() + self.addCleanup(lambda: escape.rmdir() if escape.exists() else None) + + def inject_after_authorization(event: str, _path: str | None) -> None: + if event == "after_prepare": + nested = self.root / "backups" / "forge-reset-test-0001" / "external" + nested.mkdir(parents=True) + (nested / "artifacts").symlink_to(escape, target_is_directory=True) + + service = self.service(fault_hook=inject_after_authorization) + plan = service.preview() + with self.assertRaisesRegex(OperationalResetError, "backup external path contains a symlink"): + service.prepare( + operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"]), + ) + self.assertEqual(list(escape.iterdir()), []) + service.finish(operation_id="forge-reset-test-0001", cancel_before_apply=True) + + def test_crash_before_commit_resumes_without_half_purge(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + crashing = self.service(fault_hook=OneShotFault("before_database_commit")) + with self.assertRaisesRegex(RuntimeError, "synthetic crash"): + crashing.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + observer = sqlite3.connect(self.root / "forge.db") + self.assertEqual(observer.execute("SELECT COUNT(*) FROM mission_state").fetchone()[0], 1) + observer.close() + resumed = service.resume( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(resumed["state"], "VERIFIED") + + def test_crash_after_commit_resumes_same_generation(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + crashing = self.service(fault_hook=OneShotFault("after_database_commit")) + with self.assertRaisesRegex(RuntimeError, "synthetic crash"): + crashing.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(service.status(operation_id="forge-reset-test-0001")["state"], "DATABASE_APPLIED") + resumed = service.resume( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(resumed["generation_after"], 1) + self.assertEqual(resumed["state"], "VERIFIED") + + def test_crash_during_artifact_removal_reconciles_from_verified_archive(self) -> None: + self.seed_operational_state() + service = self.service() + receipt = self.prepare(service) + crashing = self.service(fault_hook=OneShotFault("after_artifact_remove")) + with self.assertRaisesRegex(RuntimeError, "synthetic crash"): + crashing.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + resumed = service.resume( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(resumed["state"], "VERIFIED") + self.assertEqual(resumed["pending_artifacts"], 0) + + def test_same_operation_is_idempotent_and_new_empty_reset_is_meaningful_noop(self) -> None: + service = self.service() + receipt = self.complete(service) + same = service.finish(operation_id="forge-reset-test-0001", verification_digest=str(receipt["verification_digest"])) + self.assertEqual(same["generation_after"], receipt["generation_after"]) + plan = service.preview() + self.assertTrue(plan["no_op"]) + + def test_corrupt_backup_blocks_verification_and_maintenance_stays_active(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + receipt = service.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + backup = self.root / str(receipt["backup_reference"]) / "runtime-instance.json" + backup.write_text("tampered\n", encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "backup digest"): + service.verify( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(service.status()["active_operation_id"], "forge-reset-test-0001") + + def test_tampered_backup_manifest_is_rejected_even_when_referenced_files_are_unchanged(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + receipt = service.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + manifest = self.root / str(receipt["backup_reference"]) / "manifest.json" + document = json.loads(manifest.read_text(encoding="utf-8")) + document["excluded"].append("tampered") + manifest.write_text(json.dumps(document), encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "manifest binding"): + service.verify( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + + def test_old_callback_and_submission_identities_are_rejected_after_finish(self) -> None: + self.seed_operational_state(external=False) + self.database.close() + self.database = None + self.complete(self.service()) + database = RuntimeBootstrap(data_root=self.root, forge_version="2.7.21").open() + try: + with self.assertRaisesRegex(Exception, "retired"): + database.save_execution_host_binding("correlation-old", {"correlation_id": "correlation-old"}) + with self.assertRaisesRegex(Exception, "retired"): + database.create_scheduler_submission({ + "submission_id": "submission-old", "mission_id": "new-mission", + "intent_id": "new-intent", "action_id": "action-old", "iteration": 1, + "state": "CREATED", "envelope": {}, + }) + finally: + database.close() + + def test_operational_fk_fixture_requires_exact_acknowledgement_and_is_clean_after_reset(self) -> None: + self.database._connection.execute("PRAGMA foreign_keys=OFF") + self.database._connection.execute( + "INSERT INTO mission_runtime_projections VALUES (?,?,?)", ("missing-mission", "digest", "{}") + ) + self.database._connection.commit() + plan = self.service().preview() + self.assertFalse(plan["blockers"]) + self.assertEqual(len(plan["required_fk_acknowledgements"]), 1) + with self.assertRaisesRegex(OperationalResetError, "must be acknowledged"): + self.service().prepare( + operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"]) + ) + receipt = self.service().prepare( + operation_id="forge-reset-test-0001", expected_plan_digest=str(plan["plan_digest"]), + acknowledge_operational_fk=plan["required_fk_acknowledgements"], + ) + receipt = self.service().resume( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertEqual(receipt["state"], "VERIFIED") + + def test_fk_damage_reaching_preserved_configuration_blocks(self) -> None: + self.database._connection.execute("PRAGMA foreign_keys=OFF") + self.database._connection.execute( + "INSERT INTO planning_provider_security_audit VALUES (?,?,?,?,?,?)", + ("orphan", "missing-config", "operator", "test", "now", "{}"), + ) + self.database._connection.commit() + plan = self.service().preview() + self.assertIn("FOREIGN_KEY_DAMAGE_OUTSIDE_PURGE_SET", {item["code"] for item in plan["blockers"]}) + + def test_operator_envelope_has_stable_cross_product_fields(self) -> None: + plan = self.service().preview() + envelope = self.service().operator_envelope("preview", plan) + self.assertEqual(envelope["contract_version"], "operational-reset-v1") + self.assertEqual(envelope["product"], "forge") + self.assertEqual(set(envelope["target"]), {"instance_id", "database_path", "database_identity", "schema_version"}) + self.assertIn("preserved_bindings_digest", envelope) + + def test_cli_error_retains_stable_cross_product_envelope(self) -> None: + result = subprocess.run( + ( + sys.executable, "-m", "forge", "--data-root", str(self.root), "server", "reset", + "prepare", "--operation-id", "short", "--plan-digest", "invalid", + ), + text=True, capture_output=True, check=False, + ) + self.assertEqual(result.returncode, 1) + payload = json.loads(result.stdout) + required = { + "contract_version", "product", "command", "operation_id", "state", "allowed", "target", + "profile", "dataset_generation", "plan_digest", "relevant_revision_digest", "backup", "counts", + "blockers", "integrity", "preserved_bindings_digest", "details", + } + self.assertTrue(required <= set(payload)) + self.assertFalse(payload["allowed"]) + + def test_packaged_cli_exposes_real_read_only_preview_and_command_family(self) -> None: + self.database.close() + self.database = None + result = subprocess.run( + (sys.executable, "-m", "forge", "--data-root", str(self.root), "server", "reset", "preview"), + text=True, capture_output=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["contract_version"], "operational-reset-v1") + self.assertEqual(payload["command"], "preview") + self.assertEqual(payload["target"]["instance_id"], self.marker_id()) + help_result = subprocess.run( + (sys.executable, "-m", "forge", "server", "reset", "--help"), + text=True, capture_output=True, check=False, + ) + self.assertEqual(help_result.returncode, 0) + for command in ("preview", "prepare", "apply", "status", "resume", "verify", "finish"): + self.assertIn(command, help_result.stdout) + + def marker_id(self) -> str: + return (self.root / "instance" / "runtime-instance.json").read_text(encoding="utf-8").strip() + + +if __name__ == "__main__": + unittest.main() From b048b7d91025c5d7454aadcb25b86dee4c4acce6 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 18:49:40 +0200 Subject: [PATCH 2/3] fix: bound PyPI release readback --- .../workflows/forge-production-release.yml | 18 +- scripts/pypi_distribution_readback.py | 316 ++++++++++++++++++ tests/test_production_release_workflow.py | 20 ++ tests/test_pypi_distribution_readback.py | 215 ++++++++++++ 4 files changed, 560 insertions(+), 9 deletions(-) create mode 100644 scripts/pypi_distribution_readback.py create mode 100644 tests/test_pypi_distribution_readback.py diff --git a/.github/workflows/forge-production-release.yml b/.github/workflows/forge-production-release.yml index 779fdca..474abdc 100644 --- a/.github/workflows/forge-production-release.yml +++ b/.github/workflows/forge-production-release.yml @@ -276,19 +276,19 @@ jobs: with: { python-version: '3.11' } - name: Read back, hash, and reinstall both published distributions env: + READBACK_CACHE_TOKEN: ${{ github.run_id }}-${{ github.run_attempt }} VERSION: ${{ needs.release-context.outputs.version }} run: | set -euo pipefail wheel="forge_autonomy-$VERSION-py3-none-any.whl"; sdist="forge_autonomy-$VERSION.tar.gz" - curl --fail --silent --show-error "https://pypi.org/pypi/forge-autonomy/$VERSION/json" -o pypi.json - python3 - "$wheel" "$sdist" release-input/dist/SHA256SUMS pypi.json <<'PY' - import json, sys - expected = {line.split()[1]: line.split()[0] for line in open(sys.argv[3])} - files = {item['filename']: item for item in json.load(open(sys.argv[4]))['urls']} - for name in sys.argv[1:3]: - assert name in files and files[name]['digests']['sha256'] == expected[name], 'remote hash mismatch' - print(files[name]['url']) - PY + python3 scripts/pypi_distribution_readback.py \ + --version "$VERSION" \ + --hashes release-input/dist/SHA256SUMS \ + --output pypi.json \ + --cache-token "$READBACK_CACHE_TOKEN" \ + --attempts 24 \ + --interval-seconds 15 \ + --timeout-seconds 10 mkdir registry-readback for artifact in "$wheel" "$sdist"; do artifact_url="$(python3 - "$artifact" <<'PY' diff --git a/scripts/pypi_distribution_readback.py b/scripts/pypi_distribution_readback.py new file mode 100644 index 0000000..5d34eb4 --- /dev/null +++ b/scripts/pypi_distribution_readback.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Bounded, fail-closed PyPI readback for one Forge distribution release. + +The classifier is pure. Polling performs only cache-busted HTTP GET requests; +it cannot publish, replace, or delete a distribution. A published filename +with an unexpected digest is an immediate identity conflict, while a missing +expected filename may be retried within the caller's explicit bound. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import tempfile +import time +from typing import Callable, Mapping +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + + +_SEMVER = re.compile(r"^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CACHE_TOKEN = re.compile(r"^[A-Za-z0-9._-]{1,128}$") +_MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +class RegistryReadbackError(ValueError): + """The registry readback input or response is not safely usable.""" + + +class RegistryIdentityConflict(RegistryReadbackError): + """A published filename is already bound to different bytes.""" + + +class RegistryReadbackExhausted(RegistryReadbackError): + """The bounded readback did not observe both expected artifacts.""" + + +@dataclass(frozen=True) +class RegistryClassification: + """Pure classification of one PyPI release-document snapshot.""" + + state: str + missing_filenames: tuple[str, ...] + + +@dataclass(frozen=True) +class RegistryReadbackResult: + """Successful result for one bounded registry readback.""" + + state: str + attempts: int + output: Path + + +def distribution_filenames(version: str) -> tuple[str, str]: + """Return the exact two filenames owned by one Forge release.""" + if not isinstance(version, str) or _SEMVER.fullmatch(version) is None: + raise RegistryReadbackError("Forge release version is invalid") + return ( + f"forge_autonomy-{version}-py3-none-any.whl", + f"forge_autonomy-{version}.tar.gz", + ) + + +def expected_digests_from_sha256sums( + path: Path, + expected_filenames: tuple[str, ...], +) -> dict[str, str]: + """Load exact expected digests, normalizing qualified paths to basenames. + + ``sha256sum dist/file`` records ``dist/file``. PyPI exposes only + ``file``. Normalization therefore belongs at this boundary, with duplicate + basenames rejected so two qualified paths cannot ambiguously bind one + published filename. + """ + if not expected_filenames or len(set(expected_filenames)) != len(expected_filenames): + raise RegistryReadbackError("expected distribution filenames are invalid") + if any(not name or Path(name).name != name for name in expected_filenames): + raise RegistryReadbackError("expected distribution filenames must be basenames") + try: + lines = Path(path).read_text(encoding="utf-8").splitlines() + except OSError as error: + raise RegistryReadbackError("qualified SHA256SUMS is unavailable") from error + + wanted = set(expected_filenames) + observed: dict[str, str] = {} + for number, line in enumerate(lines, start=1): + fields = line.split(maxsplit=1) + if len(fields) != 2: + raise RegistryReadbackError(f"qualified SHA256SUMS line {number} is invalid") + digest, qualified_name = fields + qualified_name = qualified_name.lstrip("*") + if _SHA256.fullmatch(digest) is None or not qualified_name: + raise RegistryReadbackError(f"qualified SHA256SUMS line {number} is invalid") + basename = Path(qualified_name).name + if basename not in wanted: + continue + if basename in observed: + if observed[basename] != digest: + raise RegistryIdentityConflict( + f"qualified SHA256SUMS contains conflicting paths for {basename}" + ) + raise RegistryReadbackError( + f"qualified SHA256SUMS contains a duplicate basename for {basename}" + ) + observed[basename] = digest + + missing = sorted(wanted - set(observed)) + if missing: + raise RegistryReadbackError( + "qualified SHA256SUMS omits expected distribution filenames: " + + ", ".join(missing) + ) + return {name: observed[name] for name in expected_filenames} + + +def classify_pypi_document( + document: object, + expected_digests: Mapping[str, str], +) -> RegistryClassification: + """Classify one response as READY or PENDING, or raise on conflict. + + PENDING is reserved for an otherwise valid release document that omits at + least one expected filename. A present filename with a different SHA-256 + is never retried because the immutable publication identity conflicts. + """ + if not isinstance(expected_digests, Mapping) or not expected_digests: + raise RegistryReadbackError("expected distribution digests are invalid") + for filename, digest in expected_digests.items(): + if ( + not isinstance(filename, str) + or not filename + or Path(filename).name != filename + or not isinstance(digest, str) + or _SHA256.fullmatch(digest) is None + ): + raise RegistryReadbackError("expected distribution digests are invalid") + if not isinstance(document, Mapping) or not isinstance(document.get("urls"), list): + raise RegistryReadbackError("PyPI release document is invalid") + + observed: dict[str, str] = {} + for item in document["urls"]: + if not isinstance(item, Mapping): + raise RegistryReadbackError("PyPI release document contains an invalid file entry") + filename = item.get("filename") + if filename not in expected_digests: + continue + if filename in observed: + raise RegistryIdentityConflict( + f"PyPI release document contains duplicate filename {filename}" + ) + digests = item.get("digests") + digest = digests.get("sha256") if isinstance(digests, Mapping) else None + if not isinstance(digest, str) or _SHA256.fullmatch(digest) is None: + raise RegistryReadbackError( + f"PyPI release document omits a valid SHA-256 for {filename}" + ) + if digest != expected_digests[filename]: + raise RegistryIdentityConflict( + f"PyPI publication identity conflicts for {filename}" + ) + artifact_url = item.get("url") + if not isinstance(artifact_url, str) or not artifact_url.startswith("https://"): + raise RegistryReadbackError( + f"PyPI release document omits a secure artifact URL for {filename}" + ) + observed[filename] = digest + + missing = tuple(sorted(set(expected_digests) - set(observed))) + return RegistryClassification("PENDING" if missing else "READY", missing) + + +def _registry_url(version: str, cache_token: str, attempt: int) -> str: + base = f"https://pypi.org/pypi/forge-autonomy/{quote(version, safe='')}/json" + return base + "?" + urlencode({"forge_readback": f"{cache_token}-{attempt}"}) + + +def _fetch_document( + version: str, + cache_token: str, + attempt: int, + timeout_seconds: float, + opener: Callable[..., object], +) -> object: + request = Request( + _registry_url(version, cache_token, attempt), + headers={ + "Accept": "application/json", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + }, + method="GET", + ) + response = opener(request, timeout=timeout_seconds) + with response: # type: ignore[attr-defined] + raw = response.read(_MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined] + if len(raw) > _MAX_RESPONSE_BYTES: + raise RegistryReadbackError("PyPI release document exceeds the readback limit") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise RegistryReadbackError("PyPI release document is not valid JSON") from error + + +def _atomic_json(path: Path, document: object) -> None: + output = Path(path) + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{output.name}.", dir=output.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(document, stream, sort_keys=True, separators=(",", ":"), allow_nan=False) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary_name, 0o600) + os.replace(temporary_name, output) + except BaseException: + Path(temporary_name).unlink(missing_ok=True) + raise + + +def poll_pypi_release( + *, + version: str, + expected_digests: Mapping[str, str], + output: Path, + cache_token: str, + attempts: int, + interval_seconds: float, + timeout_seconds: float, + opener: Callable[..., object] = urlopen, + sleeper: Callable[[float], None] = time.sleep, +) -> RegistryReadbackResult: + """Poll PyPI through bounded GET-only observations until READY.""" + distribution_filenames(version) + if not isinstance(cache_token, str) or _CACHE_TOKEN.fullmatch(cache_token) is None: + raise RegistryReadbackError("registry readback cache token is invalid") + if not isinstance(attempts, int) or not 1 <= attempts <= 60: + raise RegistryReadbackError("registry readback attempts must be between 1 and 60") + if not isinstance(interval_seconds, (int, float)) or not 0 <= interval_seconds <= 60: + raise RegistryReadbackError("registry readback interval must be between 0 and 60 seconds") + if not isinstance(timeout_seconds, (int, float)) or not 0 < timeout_seconds <= 60: + raise RegistryReadbackError("registry readback timeout must be between 0 and 60 seconds") + + last_state = "UNAVAILABLE" + last_detail = "no registry response" + for attempt in range(1, attempts + 1): + try: + document = _fetch_document( + version, cache_token, attempt, float(timeout_seconds), opener, + ) + classification = classify_pypi_document(document, expected_digests) + last_state = classification.state + last_detail = ( + "missing=" + ",".join(classification.missing_filenames) + if classification.missing_filenames else "exact artifacts observed" + ) + if classification.state == "READY": + _atomic_json(output, document) + return RegistryReadbackResult("READY", attempt, Path(output)) + except RegistryIdentityConflict: + raise + except (RegistryReadbackError, HTTPError, URLError, TimeoutError, OSError) as error: + last_state = "UNAVAILABLE" + last_detail = str(error) + if attempt < attempts: + sleeper(float(interval_seconds)) + + raise RegistryReadbackExhausted( + f"PyPI registry readback exhausted after {attempts} attempts " + f"(last_state={last_state}; {last_detail})" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Read back one exact Forge PyPI release without publication authority." + ) + parser.add_argument("--version", required=True) + parser.add_argument("--hashes", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--cache-token", required=True) + parser.add_argument("--attempts", type=int, default=24) + parser.add_argument("--interval-seconds", type=float, default=15.0) + parser.add_argument("--timeout-seconds", type=float, default=10.0) + args = parser.parse_args(argv) + + try: + filenames = distribution_filenames(args.version) + expected = expected_digests_from_sha256sums(args.hashes, filenames) + result = poll_pypi_release( + version=args.version, + expected_digests=expected, + output=args.output, + cache_token=args.cache_token, + attempts=args.attempts, + interval_seconds=args.interval_seconds, + timeout_seconds=args.timeout_seconds, + ) + except RegistryIdentityConflict as error: + parser.exit(2, f"PYPI_PUBLICATION_IDENTITY_CONFLICT: {error}\n") + except RegistryReadbackExhausted as error: + parser.exit(3, f"PYPI_REGISTRY_READBACK_EXHAUSTED: {error}\n") + except RegistryReadbackError as error: + parser.exit(4, f"PYPI_REGISTRY_READBACK_INVALID: {error}\n") + print(json.dumps({"attempts": result.attempts, "state": result.state}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_production_release_workflow.py b/tests/test_production_release_workflow.py index 7cf9055..be33a99 100644 --- a/tests/test_production_release_workflow.py +++ b/tests/test_production_release_workflow.py @@ -60,6 +60,26 @@ def test_prepublication_operation_binds_exact_artifacts_and_terminal_states(self self.assertIn("Unexpected PyPI identity lookup status", workflow) self.assertIn('for artifact in "$wheel" "$sdist"; do', workflow) self.assertIn("registry-readback-digests.json", workflow) + registry_job = workflow[published_job:complete_job] + self.assertIn("python3 scripts/pypi_distribution_readback.py", registry_job) + self.assertIn('READBACK_CACHE_TOKEN: ${{ github.run_id }}-${{ github.run_attempt }}', registry_job) + self.assertIn('--cache-token "$READBACK_CACHE_TOKEN"', registry_job) + self.assertIn("--attempts 24", registry_job) + self.assertIn("--interval-seconds 15", registry_job) + self.assertIn("--timeout-seconds 10", registry_job) + self.assertNotIn( + 'curl --fail --silent --show-error "https://pypi.org/pypi/forge-autonomy/$VERSION/json"', + registry_job, + ) + self.assertLess( + registry_job.index("python3 scripts/pypi_distribution_readback.py"), + registry_job.index('for artifact in "$wheel" "$sdist"; do'), + ) + self.assertLess( + workflow.index("python3 scripts/pypi_distribution_readback.py"), + workflow.index("--mark-published"), + ) + self.assertNotIn("gh-action-pypi-publish", registry_job) self.assertLess( workflow.index('for target in published-readback published-input/dist "$PENDING_READBACK"; do'), workflow.index("--complete"), diff --git a/tests/test_pypi_distribution_readback.py b/tests/test_pypi_distribution_readback.py new file mode 100644 index 0000000..581719f --- /dev/null +++ b/tests/test_pypi_distribution_readback.py @@ -0,0 +1,215 @@ +"""Regression tests for bounded, identity-safe Forge PyPI readback.""" + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + + +MODULE = Path(__file__).parents[1] / "scripts" / "pypi_distribution_readback.py" +SPEC = importlib.util.spec_from_file_location("forge_pypi_distribution_readback", MODULE) +assert SPEC and SPEC.loader +readback = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = readback +SPEC.loader.exec_module(readback) + + +class _Response: + def __init__(self, document: object) -> None: + self._payload = json.dumps(document).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self, limit: int) -> bytes: + return self._payload[:limit] + + +class _Opener: + def __init__(self, documents: list[object]) -> None: + self._documents = iter(documents) + self.requests: list[tuple[object, float]] = [] + + def __call__(self, request: object, *, timeout: float) -> _Response: + self.requests.append((request, timeout)) + return _Response(next(self._documents)) + + +class PyPIDistributionReadbackTests(unittest.TestCase): + version = "2.7.22" + wheel, sdist = readback.distribution_filenames(version) + wheel_digest = "1" * 64 + sdist_digest = "2" * 64 + expected = {wheel: wheel_digest, sdist: sdist_digest} + + @classmethod + def document(cls, *, wheel: str | None = None, sdist: str | None = None) -> dict[str, object]: + urls: list[dict[str, object]] = [] + if wheel is not None: + urls.append({ + "filename": cls.wheel, + "digests": {"sha256": wheel}, + "url": "https://files.pythonhosted.org/forge-wheel", + }) + if sdist is not None: + urls.append({ + "filename": cls.sdist, + "digests": {"sha256": sdist}, + "url": "https://files.pythonhosted.org/forge-sdist", + }) + return {"urls": urls} + + def test_sha256sums_normalizes_qualified_paths_to_pypi_basenames(self) -> None: + with tempfile.TemporaryDirectory() as directory: + sums = Path(directory) / "SHA256SUMS" + sums.write_text( + f"{self.wheel_digest} dist/{self.wheel}\n" + f"{self.sdist_digest} dist/{self.sdist}\n", + encoding="utf-8", + ) + + self.assertEqual( + self.expected, + readback.expected_digests_from_sha256sums( + sums, (self.wheel, self.sdist), + ), + ) + + def test_sha256sums_rejects_conflicting_duplicate_basenames(self) -> None: + with tempfile.TemporaryDirectory() as directory: + sums = Path(directory) / "SHA256SUMS" + sums.write_text( + f"{self.wheel_digest} dist/{self.wheel}\n" + f"{'3' * 64} rebuilt/{self.wheel}\n" + f"{self.sdist_digest} dist/{self.sdist}\n", + encoding="utf-8", + ) + + with self.assertRaisesRegex( + readback.RegistryIdentityConflict, "conflicting paths", + ): + readback.expected_digests_from_sha256sums( + sums, (self.wheel, self.sdist), + ) + + def test_missing_expected_filename_is_pending(self) -> None: + classified = readback.classify_pypi_document( + self.document(sdist=self.sdist_digest), self.expected, + ) + + self.assertEqual("PENDING", classified.state) + self.assertEqual((self.wheel,), classified.missing_filenames) + + def test_present_filename_with_wrong_digest_is_immediate_conflict(self) -> None: + with self.assertRaisesRegex( + readback.RegistryIdentityConflict, self.wheel, + ): + readback.classify_pypi_document( + self.document(wheel="f" * 64, sdist=self.sdist_digest), + self.expected, + ) + + def test_ready_requires_both_exact_digests(self) -> None: + classified = readback.classify_pypi_document( + self.document(wheel=self.wheel_digest, sdist=self.sdist_digest), + self.expected, + ) + + self.assertEqual("READY", classified.state) + self.assertEqual((), classified.missing_filenames) + + def test_bounded_poll_uses_no_cache_gets_until_ready(self) -> None: + opener = _Opener([ + self.document(sdist=self.sdist_digest), + self.document(wheel=self.wheel_digest, sdist=self.sdist_digest), + ]) + sleeps: list[float] = [] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "pypi.json" + result = readback.poll_pypi_release( + version=self.version, + expected_digests=self.expected, + output=output, + cache_token="run-123-attempt-1", + attempts=3, + interval_seconds=0.25, + timeout_seconds=4, + opener=opener, + sleeper=sleeps.append, + ) + + self.assertEqual(readback.RegistryReadbackResult("READY", 2, output), result) + self.assertEqual([0.25], sleeps) + self.assertEqual(2, len(opener.requests)) + first, second = (entry[0] for entry in opener.requests) + self.assertEqual("GET", first.get_method()) + self.assertEqual("no-cache", first.get_header("Cache-control")) + self.assertEqual("no-cache", first.get_header("Pragma")) + self.assertIn("forge_readback=run-123-attempt-1-1", first.full_url) + self.assertIn("forge_readback=run-123-attempt-1-2", second.full_url) + self.assertEqual( + self.document(wheel=self.wheel_digest, sdist=self.sdist_digest), + json.loads(output.read_text(encoding="utf-8")), + ) + + def test_identity_conflict_stops_without_retry_or_output(self) -> None: + opener = _Opener([ + self.document(wheel="f" * 64, sdist=self.sdist_digest), + self.document(wheel=self.wheel_digest, sdist=self.sdist_digest), + ]) + sleeps: list[float] = [] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "pypi.json" + with self.assertRaises(readback.RegistryIdentityConflict): + readback.poll_pypi_release( + version=self.version, + expected_digests=self.expected, + output=output, + cache_token="run-123-attempt-1", + attempts=3, + interval_seconds=0, + timeout_seconds=4, + opener=opener, + sleeper=sleeps.append, + ) + + self.assertEqual(1, len(opener.requests)) + self.assertEqual([], sleeps) + self.assertFalse(output.exists()) + + def test_exhaustion_is_bounded_and_writes_no_readback(self) -> None: + opener = _Opener([ + self.document(sdist=self.sdist_digest), + self.document(sdist=self.sdist_digest), + self.document(sdist=self.sdist_digest), + ]) + sleeps: list[float] = [] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "pypi.json" + with self.assertRaisesRegex( + readback.RegistryReadbackExhausted, "after 3 attempts", + ): + readback.poll_pypi_release( + version=self.version, + expected_digests=self.expected, + output=output, + cache_token="run-123-attempt-1", + attempts=3, + interval_seconds=0.5, + timeout_seconds=4, + opener=opener, + sleeper=sleeps.append, + ) + + self.assertEqual(3, len(opener.requests)) + self.assertEqual([0.5, 0.5], sleeps) + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() From e6e3d729603131c320fc41e985f2b39b65501e6f Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 19:33:01 +0200 Subject: [PATCH 3/3] fix: close operational reset verification races --- forge/runtime/database.py | 21 +++ forge/runtime/operational_reset.py | 279 ++++++++++++++++++----------- tests/test_operational_reset.py | 104 +++++++++++ 3 files changed, 299 insertions(+), 105 deletions(-) diff --git a/forge/runtime/database.py b/forge/runtime/database.py index 2f25b71..033af6b 100644 --- a/forge/runtime/database.py +++ b/forge/runtime/database.py @@ -1954,6 +1954,7 @@ def _validated_mission_state_document(self, state: Any) -> tuple[dict[str, Any], def _write_mission_state_in_transaction(self, document: Mapping[str, Any], mission_id: str, lifecycle: str, context: Mapping[str, str | None]) -> None: + self._reject_reset_retired_identity("mission_id", mission_id) existing = self._connection.execute( "SELECT status FROM mission_state WHERE mission_id = ?", (mission_id,) ).fetchone() @@ -2000,6 +2001,7 @@ def create_mission_state(self, state: Any) -> dict[str, Any]: required = (mission_id, lifecycle, document.get("status"), document.get("progress"), document.get("resume", document.get("resume_point"))) if not isinstance(mission_id, str) or not mission_id or not isinstance(lifecycle, str) or not lifecycle or any(value is None for value in required[2:]): raise RuntimeDatabaseError("mission state requires identity, lifecycle, status, progress, resume point, and execution policy") + self._reject_reset_retired_identity("mission_id", mission_id) try: with self._connection: self._connection.execute("""INSERT INTO mission_state @@ -2224,6 +2226,7 @@ def record_execution_host_exchange_audit( or event_kind not in {"FORGE_SUBMISSION_SENT", "EP_SUBMISSION_RECEIPT_RECEIVED"}): raise RuntimeDatabaseError("execution host exchange audit identity is invalid") value = _document(document, "execution host exchange audit") + self._reject_reset_retired_identity("correlation_id", correlation_id) if _contains_secret_field(value) or _contains_secret_value(value): raise RuntimeDatabaseError("execution host exchange audit must not contain secrets") encoded = self._dump(value) @@ -2436,6 +2439,7 @@ def create_scheduler_submission(self, submission: Any) -> dict[str, Any]: raise RuntimeDatabaseError("scheduler submission iteration must be positive") self._reject_reset_retired_identity("submission_id", str(document["submission_id"])) self._reject_reset_retired_identity("action_id", str(document["action_id"])) + self._reject_reset_retired_identity("mission_id", str(document["mission_id"])) existing = self.scheduler_submission(str(document["submission_id"])) if existing is not None: if existing.get("envelope") != document.get("envelope"): @@ -2561,6 +2565,7 @@ def record_execution_receipt(self, *, receipt_id: str, mission_id: str, executio raise RuntimeDatabaseError("execution receipt requires complete identity, report, correlation, and outcome") self._reject_reset_retired_identity("execution_receipt_id", receipt_id) self._reject_reset_retired_identity("correlation_id", correlation_identity) + self._reject_reset_retired_identity("mission_id", mission_id) with self._connection: self._connection.execute("INSERT INTO execution_receipts VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (receipt_id, mission_id, execution_host, execution_run_id, engineering_report_id, correlation_identity, executed_at, outcome)) @@ -2600,6 +2605,10 @@ def save_action_derivation(self, derivation: Any) -> dict[str, Any]: request_digest = document.get("generation_request_digest") if request_digest is not None and (not isinstance(request_digest, str) or not _SHA256_DIGEST.fullmatch(request_digest)): raise RuntimeDatabaseError("action derivation generation request digest is malformed") + self._reject_reset_retired_identity("action_derivation_id", document["derivation_id"]) + self._reject_reset_retired_identity("mission_id", document["mission_id"]) + if request_digest is not None: + self._reject_reset_retired_identity("generation_request_digest", request_digest) existing = self._connection.execute( "SELECT document FROM action_derivations WHERE derivation_id = ?", (document["derivation_id"],) ).fetchone() @@ -2697,6 +2706,9 @@ def begin_durable_action_derivation_attempt(self, attempt: Any) -> tuple[dict[st raise RuntimeDatabaseError("legacy audit predecessor reference is inconsistent") if _contains_secret_field(document) or _contains_secret_value(document): raise RuntimeDatabaseError("durable action derivation attempt must not contain secret material") + self._reject_reset_retired_identity("action_derivation_id", document["derivation_id"]) + self._reject_reset_retired_identity("mission_id", document["mission_id"]) + self._reject_reset_retired_identity("generation_request_digest", document["generation_request_digest"]) connection = self._connection connection.execute("BEGIN IMMEDIATE") try: @@ -3424,6 +3436,10 @@ def create_action_derivation_reattempt_authorization(self, authorization: Any) - "installation_id", "created_at", "digest") if any(item not in document or document[item] in (None, "") for item in required): raise RuntimeDatabaseError("action derivation reattempt authorization requires complete lineage") + self._reject_reset_retired_identity("reattempt_authorization_id", document["authorization_id"]) + self._reject_reset_retired_identity("action_derivation_id", document["successor_attempt_id"]) + self._reject_reset_retired_identity("action_derivation_id", document["predecessor_attempt_id"]) + self._reject_reset_retired_identity("mission_id", document["mission_id"]) if (document["predecessor_terminal_state"] != "FAILED" or not isinstance(document["attempt_sequence"], int) or document["attempt_sequence"] < 2 or not re.fullmatch(r"[0-9a-f]{40}", document["main_head"]) @@ -3525,6 +3541,9 @@ def create_token_preflight_receipt(self, receipt: Any) -> dict[str, Any]: "context_with_requested_output", "result", "created_at") if any(item not in document for item in required) or document["result"] != "PASS": raise RuntimeDatabaseError("token preflight receipt requires a complete successful boundary") + self._reject_reset_retired_identity("token_preflight_receipt_id", document["receipt_id"]) + self._reject_reset_retired_identity("token_request_digest", document["request_digest"]) + self._reject_reset_retired_identity("mission_id", document["mission_id"]) if any(not isinstance(document[item], str) or not document[item] for item in required[:8]): raise RuntimeDatabaseError("token preflight receipt identity is invalid") if any(not isinstance(document[item], int) or isinstance(document[item], bool) or document[item] < 0 @@ -3618,6 +3637,8 @@ def record_token_preflight_failure(self, failure: Any) -> dict[str, Any]: "request_digest", "evidence_digest", "effective_contract_digest", "layer") if any(not isinstance(document.get(item), str) or not document[item] for item in required): raise RuntimeDatabaseError("token preflight failure requires complete bounded provenance") + self._reject_reset_retired_identity("mission_id", document["mission_id"]) + self._reject_reset_retired_identity("token_request_digest", document["request_digest"]) if (not _TOKEN_PREFLIGHT_FAILURE_ID.fullmatch(document["failure_id"]) or not _TOKEN_PREFLIGHT_FAILURE_PROVIDER_ID.fullmatch(document["provider_id"]) or not _TOKEN_PREFLIGHT_FAILURE_TIMESTAMP.fullmatch(document["occurred_at"]) diff --git a/forge/runtime/operational_reset.py b/forge/runtime/operational_reset.py index dc32144..662c2e0 100644 --- a/forge/runtime/operational_reset.py +++ b/forge/runtime/operational_reset.py @@ -937,6 +937,33 @@ def _verify_backup_files( if _has_symlink_component(archived) or not archived.is_file() or _file_digest(archived) != item["digest"]: raise OperationalResetError("isolated external recovery artifact verification failed") + def _verify_operation_backup(self, operation: sqlite3.Row) -> Mapping[str, Any]: + """Re-read the authorized recovery image instead of trusting its state label.""" + operation_id = str(operation["operation_id"]) + expected_reference = f"backups/{operation_id}" + expected_digest = operation["backup_digest"] + if operation["backup_reference"] != expected_reference or not isinstance(expected_digest, str): + raise OperationalResetError("operation has no exact persisted recovery-backup binding") + backup = self._backup_directory(operation_id) + manifest_path = backup / "manifest.json" + if _has_symlink_component(manifest_path) or not manifest_path.is_file(): + raise OperationalResetError("recovery backup manifest path is invalid") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise OperationalResetError("recovery backup manifest is unreadable") from error + if not isinstance(manifest, dict): + raise OperationalResetError("recovery backup manifest is malformed") + if ( + manifest.get("operation_id") != operation_id + or manifest.get("runtime_id") != operation["runtime_id"] + or manifest.get("plan_digest") != operation["plan_digest"] + or manifest.get("request_digest") != operation["request_digest"] + ): + raise OperationalResetError("recovery backup manifest operation binding is invalid") + self._verify_backup_files(backup, manifest, expected_digest=expected_digest) + return manifest + def _require_effect_path(self, path: Path, relative: str, expected_digest: str) -> None: try: resolved = path.resolve(strict=True) @@ -1004,6 +1031,11 @@ def _apply_database(self, row: sqlite3.Row) -> None: connection.execute("BEGIN IMMEDIATE") trigger_sql: list[str] = [] try: + # This is intentionally inside the write transaction and + # precedes trigger removal, tombstoning, permit invalidation, + # and every DELETE. A stale BACKUP_VERIFIED label can never + # authorize a purge after recovery bytes were changed. + self._verify_operation_backup(row) current = self._relevant_revision(connection) if current != row["relevant_revision"]: raise OperationalResetError("meaningful source data changed after the approved preview") @@ -1159,6 +1191,9 @@ def apply( self._apply_database(row) state = "DATABASE_APPLIED" if state in {"DATABASE_APPLIED", "APPLIED"}: + # Resume after the database commit must still prove the + # same recovery image before removing active artifacts. + self._verify_operation_backup(row) self._apply_artifacts(operation_id) elif state in {"VERIFIED", "COMPLETED"}: return self._operation_receipt(operation_id) @@ -1166,6 +1201,72 @@ def apply( raise OperationalResetError(f"operation is not ready to apply from state {state}") return self._operation_receipt(operation_id) + def _evaluate_post_reset( + self, connection: sqlite3.Connection, row: sqlite3.Row, *, backup_digest: str, + ) -> dict[str, Any]: + """Evaluate postconditions without changing operation or audit state.""" + document = json.loads(row["document"]) + plan = document["plan"] + integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] + quick = connection.execute("PRAGMA quick_check").fetchone()[0] + foreign_keys = list(connection.execute("PRAGMA foreign_key_check")) + counts = { + table: int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + for table in PURGE_TABLES + } + if integrity != "ok" or quick != "ok" or foreign_keys: + raise OperationalResetError("post-reset SQLite integrity verification failed") + if any(counts.values()): + raise OperationalResetError("post-reset operational tables are not empty") + if connection.execute( + "SELECT 1 FROM planning_provider_generation_permits " + "WHERE state IN ('PENDING','TRANSPORT_COMMITTED') LIMIT 1" + ).fetchone(): + raise OperationalResetError("a pre-reset generation permit remains executable") + preserved: dict[str, dict[str, Any]] = {} + for table in PRESERVE_TABLES: + actual = self._row_digest( + connection, table, post_reset=(table == "planning_provider_generation_permits"), + ) + expected = plan["preserved"][table]["expected_post_reset_digest"] + if actual != expected: + raise OperationalResetError(f"preserved table changed unexpectedly: {table}") + preserved[table] = { + "rows": int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]), + "digest": actual, + } + external, unknown = self._external_inventory() + active_effects = [ + item for item in external if item.get("effect") in {"ARCHIVE_AND_REMOVE", "REMOVE_CACHE"} + ] + if unknown or active_effects: + raise OperationalResetError("post-reset active external operational data is not empty") + preserved_external = [ + item for item in external + if item.get("effect") == "PRESERVE" + and item.get("category") == "INSTALLATION_AND_CONFIGURATION" + and not str(item.get("path", "")).startswith("backups/") + ] + if _digest(preserved_external) != plan["preserved_external_digest"]: + raise OperationalResetError("preserved external installation artifacts changed") + self._verify_operation_backup(row) + marker = self.marker_path.read_text(encoding="utf-8").strip() + if marker != row["runtime_id"]: + raise OperationalResetError("runtime identity changed during reset") + generation = int(connection.execute( + "SELECT dataset_generation FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0]) + if row["generation_after"] is None or generation != int(row["generation_after"]): + raise OperationalResetError("dataset generation does not match the applied operation") + verification = { + "integrity_check": "ok", "quick_check": "ok", "foreign_key_errors": 0, + "operational_rows": counts, "preserved": preserved, + "runtime_id": marker, "dataset_generation": generation, + "backup_digest": backup_digest, "active_external_effect_entries": 0, + } + verification["verification_digest"] = _digest(verification) + return verification + def verify( self, *, operation_id: str, plan_digest: str, request_digest: str, backup_digest: str, ) -> dict[str, Any]: @@ -1176,87 +1277,39 @@ def verify( connection, operation_id, plan_digest=plan_digest, request_digest=request_digest, backup_digest=backup_digest, ) - if row["state"] == "VERIFIED": - return self._operation_receipt(operation_id) - if row["state"] != "APPLIED": + if row["state"] not in {"APPLIED", "VERIFIED"}: raise OperationalResetError("verification requires a fully applied operation") - document = json.loads(row["document"]) - plan = document["plan"] - integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] - quick = connection.execute("PRAGMA quick_check").fetchone()[0] - foreign_keys = list(connection.execute("PRAGMA foreign_key_check")) - counts = { - table: int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) - for table in PURGE_TABLES - } - if integrity != "ok" or quick != "ok" or foreign_keys: - raise OperationalResetError("post-reset SQLite integrity verification failed") - if any(counts.values()): - raise OperationalResetError("post-reset operational tables are not empty") - if connection.execute( - "SELECT 1 FROM planning_provider_generation_permits " - "WHERE state IN ('PENDING','TRANSPORT_COMMITTED') LIMIT 1" - ).fetchone(): - raise OperationalResetError("a pre-reset generation permit remains executable") - preserved: dict[str, dict[str, Any]] = {} - for table in PRESERVE_TABLES: - actual = self._row_digest(connection, table, post_reset=(table == "planning_provider_generation_permits")) - expected = plan["preserved"][table]["expected_post_reset_digest"] - if actual != expected: - raise OperationalResetError(f"preserved table changed unexpectedly: {table}") - preserved[table] = { - "rows": int(connection.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]), - "digest": actual, - } - external, unknown = self._external_inventory() - active_effects = [ - item for item in external if item.get("effect") in {"ARCHIVE_AND_REMOVE", "REMOVE_CACHE"} - ] - if unknown or active_effects: - raise OperationalResetError("post-reset active external operational data is not empty") - preserved_external = [ - item for item in external - if item.get("effect") == "PRESERVE" - and item.get("category") == "INSTALLATION_AND_CONFIGURATION" - and not str(item.get("path", "")).startswith("backups/") - ] - if _digest(preserved_external) != plan["preserved_external_digest"]: - raise OperationalResetError("preserved external installation artifacts changed") - manifest = json.loads((self._backup_directory(operation_id) / "manifest.json").read_text(encoding="utf-8")) - self._verify_backup_files( - self._backup_directory(operation_id), manifest, expected_digest=backup_digest, - ) - marker = self.marker_path.read_text(encoding="utf-8").strip() - if marker != row["runtime_id"]: - raise OperationalResetError("runtime identity changed during reset") - generation = int(connection.execute( - "SELECT dataset_generation FROM operational_reset_state WHERE singleton=1" - ).fetchone()[0]) - if generation != int(row["generation_after"]): - raise OperationalResetError("dataset generation does not match the applied operation") - verification = { - "integrity_check": "ok", "quick_check": "ok", "foreign_key_errors": 0, - "operational_rows": counts, "preserved": preserved, - "runtime_id": marker, "dataset_generation": generation, - "backup_digest": backup_digest, "active_external_effect_entries": 0, - } - verification_digest = _digest(verification) - verification["verification_digest"] = verification_digest - document["state"] = "VERIFIED" - document["verification"] = verification - now = _now() - with connection: - connection.execute( - "UPDATE operational_reset_operations SET state='VERIFIED',updated_at=?,document=? WHERE operation_id=?", - (now, _json(document), operation_id), - ) - connection.execute( - "UPDATE operational_reset_state SET state='VERIFIED',updated_at=? WHERE singleton=1 AND active_operation_id=?", - (now, operation_id), + already_verified = row["state"] == "VERIFIED" + connection.execute("BEGIN IMMEDIATE") + try: + verification = self._evaluate_post_reset( + connection, row, backup_digest=backup_digest, ) - self._audit(connection, operation_id, "VERIFIED", { - "verification_digest": verification_digest, "dataset_generation": generation, - }) + document = json.loads(row["document"]) + if already_verified: + if document.get("verification", {}).get("verification_digest") != verification["verification_digest"]: + raise OperationalResetError("persisted verification no longer matches current postconditions") + connection.rollback() + else: + document["state"] = "VERIFIED" + document["verification"] = verification + now = _now() + connection.execute( + "UPDATE operational_reset_operations SET state='VERIFIED',updated_at=?,document=? WHERE operation_id=?", + (now, _json(document), operation_id), + ) + connection.execute( + "UPDATE operational_reset_state SET state='VERIFIED',updated_at=? WHERE singleton=1 AND active_operation_id=?", + (now, operation_id), + ) + self._audit(connection, operation_id, "VERIFIED", { + "verification_digest": verification["verification_digest"], + "dataset_generation": verification["dataset_generation"], + }) + connection.commit() + except Exception: + connection.rollback() + raise return self._operation_receipt(operation_id) def resume( @@ -1293,34 +1346,50 @@ def finish( """Leave maintenance only after proof, or cancel before any purge.""" with self._lock.acquire(): with self._connect(read_only=False) as connection: - row = connection.execute( - "SELECT * FROM operational_reset_operations WHERE operation_id=?", (_safe_operation_id(operation_id),) - ).fetchone() - if row is None: - raise OperationalResetError("unknown operational reset operation") - active = connection.execute( - "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" - ).fetchone()[0] - if active != operation_id: - if row["state"] in {"COMPLETED", "CANCELLED"}: - return self._operation_receipt(operation_id) - raise OperationalResetError("operation does not own durable maintenance") - document = json.loads(row["document"]) - if cancel_before_apply: - if row["state"] not in {"PREPARED", "BACKUP_VERIFIED"}: - raise OperationalResetError("only an unapplied operation can be cancelled") - final_state = "CANCELLED" - else: - if row["state"] != "VERIFIED": - raise OperationalResetError("maintenance can finish only after successful verification") - actual = document.get("verification", {}).get("verification_digest") - if not verification_digest or verification_digest != actual: - raise OperationalResetError("exact verification digest is required to finish") - final_state = "COMPLETED" - document["state"] = final_state - now = _now() connection.execute("BEGIN IMMEDIATE") try: + row = connection.execute( + "SELECT * FROM operational_reset_operations WHERE operation_id=?", + (_safe_operation_id(operation_id),), + ).fetchone() + if row is None: + raise OperationalResetError("unknown operational reset operation") + active = connection.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0] + if active != operation_id: + if row["state"] in {"COMPLETED", "CANCELLED"}: + connection.rollback() + return self._operation_receipt(operation_id) + raise OperationalResetError("operation does not own durable maintenance") + document = json.loads(row["document"]) + if cancel_before_apply: + if row["state"] not in {"PREPARED", "BACKUP_VERIFIED"}: + raise OperationalResetError("only an unapplied operation can be cancelled") + final_state = "CANCELLED" + else: + if row["state"] != "VERIFIED": + raise OperationalResetError("maintenance can finish only after successful verification") + actual = document.get("verification", {}).get("verification_digest") + if not verification_digest or verification_digest != actual: + raise OperationalResetError("exact verification digest is required to finish") + if row["source_revision"] != self._source_revision(): + raise OperationalResetError("source revision changed before maintenance release") + if row["implementation_digest"] != self._implementation_digest(): + raise OperationalResetError("maintenance implementation changed before maintenance release") + # Re-evaluate under the database write boundary and the + # owning process lock. This deliberately does not + # rewrite VERIFIED state or its audit receipt. + current = self._evaluate_post_reset( + connection, row, backup_digest=str(row["backup_digest"]), + ) + if current["verification_digest"] != actual: + raise OperationalResetError( + "current postconditions differ from the authorized verification", + ) + final_state = "COMPLETED" + document["state"] = final_state + now = _now() connection.execute( "UPDATE operational_reset_operations SET state=?,updated_at=?,document=? WHERE operation_id=?", (final_state, now, _json(document), operation_id), diff --git a/tests/test_operational_reset.py b/tests/test_operational_reset.py index 81b1eb4..fbe0f03 100644 --- a/tests/test_operational_reset.py +++ b/tests/test_operational_reset.py @@ -90,6 +90,7 @@ def seed_operational_state(self, *, external: bool = True) -> None: })), ) self.database._connection.commit() + self.database.save_action_derivation(self.old_action_derivation()) if external: operational = self.root / "artifacts" / "operational" operational.mkdir() @@ -98,6 +99,16 @@ def seed_operational_state(self, *, external: bool = True) -> None: "installation qualification\n", encoding="utf-8", ) + @staticmethod + def old_action_derivation() -> dict[str, object]: + return { + "derivation_id": "derivation-old", "mission_id": "MISSION-0042", + "snapshot_digest": "sha256:" + "4" * 64, "contract_version": "1", + "provider_configuration": "sha256:" + "5" * 64, + "lifecycle": "DERIVATION_REQUESTED", + "generation_request_digest": "sha256:" + "6" * 64, + } + def prepare(self, service: ForgeOperationalResetService, operation: str = "forge-reset-test-0001") -> dict[str, object]: plan = service.preview() self.assertEqual(plan["status"], "READY", plan["blockers"]) @@ -369,6 +380,50 @@ def test_crash_during_artifact_removal_reconciles_from_verified_archive(self) -> self.assertEqual(resumed["state"], "VERIFIED") self.assertEqual(resumed["pending_artifacts"], 0) + def test_tampered_backup_before_apply_keeps_database_and_maintenance_intact(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + backup_marker = self.root / str(receipt["backup_reference"]) / "runtime-instance.json" + backup_marker.write_text("tampered-before-apply\n", encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "backup digest"): + service.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + observer = sqlite3.connect(self.root / "forge.db") + try: + self.assertEqual(observer.execute("SELECT COUNT(*) FROM mission_state").fetchone()[0], 1) + self.assertEqual(observer.execute("SELECT COUNT(*) FROM action_derivations").fetchone()[0], 1) + self.assertEqual(observer.execute( + "SELECT state FROM operational_reset_operations WHERE operation_id='forge-reset-test-0001'" + ).fetchone()[0], "BACKUP_VERIFIED") + self.assertEqual(observer.execute( + "SELECT active_operation_id FROM operational_reset_state WHERE singleton=1" + ).fetchone()[0], "forge-reset-test-0001") + finally: + observer.close() + + def test_tampered_backup_during_database_applied_resume_preserves_active_artifact(self) -> None: + self.seed_operational_state() + service = self.service() + receipt = self.prepare(service) + crashing = self.service(fault_hook=OneShotFault("after_database_commit")) + with self.assertRaisesRegex(RuntimeError, "synthetic crash"): + crashing.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + backup_marker = self.root / str(receipt["backup_reference"]) / "runtime-instance.json" + backup_marker.write_text("tampered-during-resume\n", encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "backup digest"): + service.resume( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + self.assertTrue((self.root / "artifacts" / "operational" / "evidence.json").is_file()) + self.assertEqual(service.status()["active_operation_id"], "forge-reset-test-0001") + def test_same_operation_is_idempotent_and_new_empty_reset_is_meaningful_noop(self) -> None: service = self.service() receipt = self.complete(service) @@ -412,6 +467,39 @@ def test_tampered_backup_manifest_is_rejected_even_when_referenced_files_are_unc request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), ) + def test_finish_rechecks_delayed_external_data_and_backup_without_rewriting_verification(self) -> None: + self.seed_operational_state(external=False) + service = self.service() + receipt = self.prepare(service) + receipt = service.apply( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + receipt = service.verify( + operation_id="forge-reset-test-0001", plan_digest=str(receipt["plan_digest"]), + request_digest=str(receipt["request_digest"]), backup_digest=str(receipt["backup_digest"]), + ) + audit_before = int(receipt["audit_events"]) + delayed = self.root / "journals" / "delayed.jsonl" + delayed.write_text('{"old_callback":true}\n', encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "external operational data"): + service.finish( + operation_id="forge-reset-test-0001", verification_digest=str(receipt["verification_digest"]), + ) + status = service.status(operation_id="forge-reset-test-0001") + self.assertEqual(status["state"], "VERIFIED") + self.assertEqual(status["audit_events"], audit_before) + self.assertEqual(service.status()["active_operation_id"], "forge-reset-test-0001") + delayed.unlink() + backup_marker = self.root / str(receipt["backup_reference"]) / "runtime-instance.json" + backup_marker.write_text("tampered-after-verify\n", encoding="utf-8") + with self.assertRaisesRegex(OperationalResetError, "backup digest"): + service.finish( + operation_id="forge-reset-test-0001", verification_digest=str(receipt["verification_digest"]), + ) + self.assertEqual(service.status(operation_id="forge-reset-test-0001")["state"], "VERIFIED") + self.assertEqual(service.status()["active_operation_id"], "forge-reset-test-0001") + def test_old_callback_and_submission_identities_are_rejected_after_finish(self) -> None: self.seed_operational_state(external=False) self.database.close() @@ -427,6 +515,22 @@ def test_old_callback_and_submission_identities_are_rejected_after_finish(self) "intent_id": "new-intent", "action_id": "action-old", "iteration": 1, "state": "CREATED", "envelope": {}, }) + with self.assertRaisesRegex(Exception, "action_derivation_id identity was retired"): + database.save_action_derivation(self.old_action_derivation()) + with self.assertRaisesRegex(Exception, "mission_id identity was retired"): + database.create_mission_state({ + "mission_id": "MISSION-0042", "lifecycle": "ACTIVE", "status": "ACTIVE", + "progress": {}, "resume_point": {}, "execution_policy": {}, + }) + tombstone_kinds = { + row[0] for row in database._connection.execute( + "SELECT DISTINCT record_kind FROM operational_reset_tombstones" + ) + } + self.assertTrue({ + "mission_id", "action_id", "submission_id", "correlation_id", + "action_derivation_id", "generation_request_digest", + } <= tombstone_kinds) finally: database.close()