diff --git a/.env.example b/.env.example index c2d95d86c8..6de1aacb18 100644 --- a/.env.example +++ b/.env.example @@ -81,6 +81,42 @@ POWERCONTEXT_SERVER_DATABASE_KIND=sqlite # The default database is powercontext; set test before upgrading an existing seekdb deployment. # POWERCONTEXT_SERVER_DATABASE_DATABASE=test +# Deployment and durable work ------------------------------------------------- +# Local installs run every role in one process. Distributed mode requires OceanBase and one role per process. +POWERCONTEXT_SERVER_DEPLOYMENT_MODE=single_node +POWERCONTEXT_SERVER_DEPLOYMENT_ROLE=all +POWERCONTEXT_SERVER_DEPLOYMENT_ID=local +POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION=default + +# Scheduler and member leases use database time. Renewal must stay within the validated lease ratios. +POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_LEASE_SECONDS=30 +POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_RENEW_SECONDS=10 +POWERCONTEXT_SERVER_COORDINATION_SCAN_PAGE_SIZE=100 +POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS=30 +POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS=10 +POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION=1 + +POWERCONTEXT_SERVER_WORKER_CONCURRENCY=4 +POWERCONTEXT_SERVER_WORKER_LEASE_SECONDS=120 +POWERCONTEXT_SERVER_WORKER_HEARTBEAT_SECONDS=30 +POWERCONTEXT_SERVER_WORKER_SHUTDOWN_GRACE_SECONDS=90 +POWERCONTEXT_SERVER_WORKER_MAX_ATTEMPTS=5 +POWERCONTEXT_SERVER_WORKER_RETRY_BASE_SECONDS=2 +POWERCONTEXT_SERVER_WORKER_RETRY_MAX_SECONDS=300 +POWERCONTEXT_SERVER_WORKER_POLL_SECONDS=1 + +POWERCONTEXT_SERVER_OPERATIONS_DEFAULT_WAIT_SECONDS=10 +POWERCONTEXT_SERVER_OPERATIONS_MAXIMUM_WAIT_SECONDS=30 +POWERCONTEXT_SERVER_OPERATIONS_POLL_SECONDS=0.2 +POWERCONTEXT_SERVER_OPERATIONS_RETENTION_DAYS=30 +POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_BATCH_SIZE=500 +POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_INTERVAL_SECONDS=3600 + +# Shared OceanBase fixed-window limiting is optional and disabled by default. +POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED=false +POWERCONTEXT_SERVER_RATE_LIMIT_REQUESTS=120 +POWERCONTEXT_SERVER_RATE_LIMIT_WINDOW_SECONDS=60 + # Runtime --------------------------------------------------------------------- POWERCONTEXT_SERVER_RUNTIME_SOURCE_WINDOW_LIMIT=100 # Maximum sum of explicit assembly section limits; per-family limits still apply. diff --git a/.licenserc.yaml b/.licenserc.yaml index de4604d537..4d32394405 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -68,6 +68,8 @@ header: - 'src/powercontext/server/dashboard/static/vendor/**' - '.gitattributes' - '**/*.service.in' + # license-eye cannot determine the comment style of Alembic's Mako template. + - 'src/powercontext/builtin/persistence/migrations/script.py.mako' - '**/*.jsonl' # license-eye cannot determine the comment style of patch files. - '**/*.patch' diff --git a/desktop/VALIDATION.md b/desktop/VALIDATION.md index af7256fbfc..772d379c45 100644 --- a/desktop/VALIDATION.md +++ b/desktop/VALIDATION.md @@ -177,7 +177,7 @@ The historical Server c535d816 protocol qualification below records the specific ## Compatibility scope -[compatibility.json](src-tauri/src/connections/compatibility.json) preserves each recorded Server source, normalized OpenAPI contract digest, qualification-wheel digest and operation set. The `sqlite-atomic-3cefa45a-v1` record matches the bundled API contract and qualifies its eleven recorded SQLite protocol operations against the exact Server source and wheel below. Historical records retain their exact identities and evidence. Ordinary builds require explicitly selecting the record whose tested Server source and wheel match the deployment, then checking the connection before business operations become available. Records with a different contract digest are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. Selecting a profile does not extend its recorded operation set or qualify a different Server source. CI builds have their own artifact identities and do not change product qualification. +[compatibility.json](src-tauri/src/connections/compatibility.json) preserves each recorded Server source, normalized OpenAPI contract digest, qualification-wheel digest and operation set. The `sqlite-atomic-3df12c9e-v1` record matches the bundled API contract and qualifies its eleven recorded SQLite protocol operations against the exact Server source and wheel below. Historical records retain their exact identities and evidence. Ordinary builds require explicitly selecting the record whose tested Server source and wheel match the deployment, then checking the connection before business operations become available. Records with a different contract digest are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. Selecting a profile does not extend its recorded operation set or qualify a different Server source. CI builds have their own artifact identities and do not change product qualification. The real Server harness covers SQLite with anonymous loopback, static Bearer, HTTPS with explicit CA/base path, and injected-provider/enforced access. It exercises save/search/exact reads, identity changes, binding revocation, ambiguous writes without replay and Scope pagination. Installed UI checks cover repeated launches preserving the existing editor, late committed-save responses preserving a new Scope draft, explicit connection/Scope selection, clipboard copies, connection isolation, draft cancellation, byte limits and independent Server survival after Desktop exit. Historical record `sqlite-atomic-f3a90b2f-v1` remains preserved with its original identities and is disabled for this contract. @@ -187,6 +187,152 @@ Selecting a compatibility profile does not attest the remote binary identity. Ex ### Server protocol qualification records +#### Server 3df12c9e protocol qualification + +`sqlite-atomic-3df12c9e-v1` qualifies Server sources at +`3df12c9e7d83818bd35c07c1bbef1f00926b0fc7` for the bundled OpenAPI digest +`b4d241e67b5a00fddd0a5d851cf86485fa8ff999e97ec067ee9b15b46a59d968`, the merged contract that carries +master's 1.5.0 artifact-storage chain, the atomic-memory evidence projection and the Work Ledger +revision, together with the four Scope-scoped durable operation endpoints. The qualification wheel is +`powercontext-1.2.1.dev620+g3df12c9e7-py3-none-any.whl`, SHA-256 +`29b2e69fc451508f16682981acd131be09e7242c3a8eab7ff578b35ba6221db5`, built from that clean commit. +All 718 packaged runtime files in `powercontext/` and `powercontext_service_bootstrap/` match the +source bytes at that commit. The runtime manifest digest is +`5cf1df36b5e1676601b9729e54b15704cfa9e48748d6229560c3c7e61c53075a` +(sorted `wheel/path sha256` lines, with a final newline). + +On 2026-11-04, macOS 27.2 ARM64 with Python 3.14.7 and Rust 1.68.1 ran the existing +`desktop/tests/real_server.py` harness against the extracted wheel and independent SQLite Server +processes. All four modes passed: anonymous loopback, static Bearer, HTTPS with an explicit CA and +base path, and an injected provider with enforced access. The eleven recorded operations cover +Atomic Memory save/search/exact revision reads, Scope selection and pagination, identity changes, +same-identity revocation, committed writes with lost responses without replay, server discovery, +and Server survival after client exit. The report records the source commit above and +`desktopWorkingTreeDirty=false`. Native probe medians (nearest rank, 20 samples): loopback-anonymous +save 30.41 ms / search 15.98 ms / exact read 13.72 ms; loopback-bearer 41.88 / 24.10 / 23.13 ms; +HTTPS bearer with base path 42.52 / 27.67 / 26.52 ms; injected provider 42.73 / 24.82 / 23.74 ms. + +This is protocol qualification of the recorded source, contract and wheel. It does not qualify +Windows/Intel execution, native credential persistence, installed GUI workflows, signing or release +readiness, and it says nothing about the four new operation endpoints: they are Server-side contract +covered by the Python suite and are deliberately absent from the Desktop operation set above. The +Work Ledger revision this wheel carries is additive and does not change the eleven qualified +operations. Reproduce from this clean commit with +`uv build --wheel --out-dir desktop/.artifacts/server-wheel`, +`cargo build --locked --example server_probe --example diagnostic_cli_probe`, +`uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit 3df12c9e7d83818bd35c07c1bbef1f00926b0fc7`, +and `uv run --no-sync python desktop/tests/real_server.py`. +The bundled preview resolves its qualification by digest, so a contract change and a matching record +have to land together. `tests/test_desktop_contract.py` fails the Python gate for a digest without a +record, exactly as `bundled_qualification_allows_explicit_scope_browsing` fails the native one. + +#### Server cf81c1a8 protocol qualification + +`sqlite-atomic-cf81c1a8-v1` qualifies Server sources at +`cf81c1a8a9c40c1fe186944ba6e334b54aa5da70` for the bundled OpenAPI digest +`c7f00e81a12848249c7105ba99c444b6233f4b2a4849d1d4443c419f53ac764f`, the contract that adds the four +Scope-scoped durable operation endpoints. The qualification wheel is +`powercontext-1.2.1.dev414+gcf81c1a8a-py3-none-any.whl`, SHA-256 +`0530ed69045ed45be3048aafd9256710d1e1ef4b234858b6314513556ff2b904`, built from that clean commit. +All 632 packaged runtime files in `powercontext/` and `powercontext_service_bootstrap/` match the +source bytes at that commit. The runtime manifest digest is +`8f591bfb56ed302481297a2b02f06d1cb4f381eb15090a603a1322c6f72d1e93` +(sorted `wheel/path sha256` lines, with a final newline). + +On 2026-10-30, macOS 27.2 ARM64 with Python 3.14.7 and Rust 1.95.0 ran the existing +`desktop/tests/real_server.py` harness against the extracted wheel and independent SQLite Server +processes. All four modes passed: anonymous loopback, static Bearer, HTTPS with an explicit CA and +base path, and an injected provider with enforced access. The eleven recorded operations cover +Atomic Memory save/search/exact revision reads, Scope selection and pagination, identity changes, +same-identity revocation, committed writes with lost responses without replay, server discovery, +and Server survival after client exit. The report records the source commit above and +`desktopWorkingTreeDirty=false`. + +This is protocol qualification of the recorded source, contract and wheel. It does not qualify +Windows/Intel execution, native credential persistence, installed GUI workflows, signing or release +readiness, and it says nothing about the four new operation endpoints: they are Server-side contract +covered by the Python suite and are deliberately absent from the Desktop operation set above. +Reproduce from this clean commit with the existing wheel build, native `server_probe` build, +`fixture_qualification.py --server-commit cf81c1a8a9c40c1fe186944ba6e334b54aa5da70`, +and `real_server.py` commands recorded under [Server 4ffdf4e9 protocol qualification](#server-4ffdf4e9-protocol-qualification). +The bundled preview resolves its qualification by digest, so a contract change and a matching record +have to land together. `tests/test_desktop_contract.py` fails the Python gate for a digest without a +record, exactly as `bundled_qualification_allows_explicit_scope_browsing` fails the native one. + + + +#### Server d9f33be3 protocol qualification + +`sqlite-atomic-d9f33be3-v1` qualifies Server sources at +`d9f33be3fde7a1801b51aa66c51d25af5b883d69` for the bundled OpenAPI digest +`3a89f4131cf822d458a55e9ea4c57e1314aaffbaff764fd4f4935c7fc9a5f3cb`, the merged contract that carries +master's 1.5.0 artifact-storage chain plus the four Scope-scoped durable operation endpoints. The +qualification wheel is `powercontext-1.2.1.dev498+gd9f33be3f-py3-none-any.whl`, SHA-256 +`c23438d67cb79ee6e23d58307917853167d2f4dd1916138b35f2ceb169d48202`, built from that clean commit. +All 685 packaged runtime files in `powercontext/` and `powercontext_service_bootstrap/` match the +source bytes at that commit. The runtime manifest digest is +`d81c6ed801f5d30686b7e7565ccc16a145af90dbf19b6788334a16d305a45cf3` +(sorted `wheel/path sha256` lines, with a final newline). + +On 2026-11-02, macOS 27.2 ARM64 with Python 3.14.7 and Rust 1.95.0 ran the existing +`desktop/tests/real_server.py` harness against the extracted wheel and independent SQLite Server +processes. All four modes passed: anonymous loopback, static Bearer, HTTPS with an explicit CA and +base path, and an injected provider with enforced access. The eleven recorded operations cover +Atomic Memory save/search/exact revision reads, Scope selection and pagination, identity changes, +same-identity revocation, committed writes with lost responses without replay, server discovery, +and Server survival after client exit. The report records the source commit above and +`desktopWorkingTreeDirty=false`. + +This is protocol qualification of the recorded source, contract and wheel. It does not qualify +Windows/Intel execution, native credential persistence, installed GUI workflows, signing or release +readiness, and it says nothing about the four new operation endpoints: they are Server-side contract +covered by the Python suite and are deliberately absent from the Desktop operation set above. +Reproduce from this clean commit with `uv build --wheel --out-dir desktop/.artifacts/server-wheel`, +`cargo build --locked --example server_probe --example diagnostic_cli_probe`, +`uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit d9f33be3fde7a1801b51aa66c51d25af5b883d69`, +and `uv run --no-sync python desktop/tests/real_server.py`. +The bundled preview resolves its qualification by digest, so a contract change and a matching record +have to land together. `tests/test_desktop_contract.py` fails the Python gate for a digest without a +record, exactly as `bundled_qualification_allows_explicit_scope_browsing` fails the native one. + + +#### Server 4e8b30aa protocol qualification + +`sqlite-atomic-4e8b30aa-v1` qualifies Server sources at +`4e8b30aaa5873cf8f7bca636d2f8eb5f76c17a6e` for the bundled OpenAPI digest +`741791d3eddfaf835d8516f955ebe7f6bde49c9894b93d0263e0d0fbf7eb7e12`, the merged contract that carries +master's 1.5.0 artifact-storage chain, the private Unix socket and inference privacy work, and the +four Scope-scoped durable operation endpoints. The qualification wheel is +`powercontext-1.2.1.dev517+g4e8b30aaa-py3-none-any.whl`, SHA-256 +`90ccd36193b3c4b91d0fca151413694385229d720e55f9f34653328b3955ff8f`, built from that clean commit. +All 696 packaged runtime files in `powercontext/` and `powercontext_service_bootstrap/` match the +source bytes at that commit. The runtime manifest digest is +`6a88c0b314c0ea5e1a31fdadb5d0f6a5583a73eb26eff6bd5fd1b6e780dc06f9` +(sorted `wheel/path sha256` lines, with a final newline). + +On 2026-11-04, macOS 27.2 ARM64 with Python 3.14.7 and Rust 1.68.1 ran the existing +`desktop/tests/real_server.py` harness against the extracted wheel and independent SQLite Server +processes. All four modes passed: anonymous loopback, static Bearer, HTTPS with an explicit CA and +base path, and an injected provider with enforced access. The eleven recorded operations cover +Atomic Memory save/search/exact revision reads, Scope selection and pagination, identity changes, +same-identity revocation, committed writes with lost responses without replay, server discovery, +and Server survival after client exit. The report records the source commit above and +`desktopWorkingTreeDirty=false`. Native probe medians (nearest rank, 20 samples): loopback-anonymous +save 19.16 ms / search 15.84 ms / exact read 13.48 ms; loopback-bearer 34.53 / 25.68 / 24.36 ms; +HTTPS bearer with base path 33.62 / 24.77 / 23.65 ms; injected provider 36.75 / 24.77 / 23.71 ms. + +This is protocol qualification of the recorded source, contract and wheel. It does not qualify +Windows/Intel execution, native credential persistence, installed GUI workflows, signing or release +readiness, and it says nothing about the four new operation endpoints: they are Server-side contract +covered by the Python suite and are deliberately absent from the Desktop operation set above. +Reproduce from this clean commit with `uv build --wheel --out-dir desktop/.artifacts/server-wheel`, +`cargo build --locked --example server_probe --example diagnostic_cli_probe`, +`uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit 4e8b30aaa5873cf8f7bca636d2f8eb5f76c17a6e`, +and `uv run --no-sync python desktop/tests/real_server.py`. +The bundled preview resolves its qualification by digest, so a contract change and a matching record +have to land together. `tests/test_desktop_contract.py` fails the Python gate for a digest without a +record, exactly as `bundled_qualification_allows_explicit_scope_browsing` fails the native one. + #### Server 112db11e protocol qualification `sqlite-atomic-112db11e-v1` qualifies Server source diff --git a/desktop/src-tauri/src/connections/compatibility.json b/desktop/src-tauri/src/connections/compatibility.json index 3f6bc30f4b..04870a3df7 100644 --- a/desktop/src-tauri/src/connections/compatibility.json +++ b/desktop/src-tauri/src/connections/compatibility.json @@ -1,4 +1,64 @@ [ + { + "id": "sqlite-atomic-3df12c9e-v1", + "serverCommit": "3df12c9e7d83818bd35c07c1bbef1f00926b0fc7", + "contractSha256": "b4d241e67b5a00fddd0a5d851cf86485fa8ff999e97ec067ee9b15b46a59d968", + "artifactSha256": "29b2e69fc451508f16682981acd131be09e7242c3a8eab7ff578b35ba6221db5", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "get_server_info", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-3df12c9e-protocol-qualification" + }, + { + "id": "sqlite-atomic-4e8b30aa-v1", + "serverCommit": "4e8b30aaa5873cf8f7bca636d2f8eb5f76c17a6e", + "contractSha256": "741791d3eddfaf835d8516f955ebe7f6bde49c9894b93d0263e0d0fbf7eb7e12", + "artifactSha256": "90ccd36193b3c4b91d0fca151413694385229d720e55f9f34653328b3955ff8f", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "get_server_info", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-4e8b30aa-protocol-qualification" + }, + { + "id": "sqlite-atomic-d9f33be3-v1", + "serverCommit": "d9f33be3fde7a1801b51aa66c51d25af5b883d69", + "contractSha256": "3a89f4131cf822d458a55e9ea4c57e1314aaffbaff764fd4f4935c7fc9a5f3cb", + "artifactSha256": "c23438d67cb79ee6e23d58307917853167d2f4dd1916138b35f2ceb169d48202", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "get_server_info", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-d9f33be3-protocol-qualification" + }, { "id": "sqlite-6e237568-v1", "serverCommit": "6e2375688bc38a667840c6b2dc8269d8848027bc", @@ -247,6 +307,26 @@ ], "evidence": "desktop/VALIDATION.md#server-cc5980f0-protocol-qualification" }, + { + "id": "sqlite-atomic-cf81c1a8-v1", + "serverCommit": "cf81c1a8a9c40c1fe186944ba6e334b54aa5da70", + "contractSha256": "c7f00e81a12848249c7105ba99c444b6233f4b2a4849d1d4443c419f53ac764f", + "artifactSha256": "0530ed69045ed45be3048aafd9256710d1e1ef4b234858b6314513556ff2b904", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "get_server_info", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-cf81c1a8-protocol-qualification" + }, { "id": "sqlite-atomic-4ffdf4e9-v1", "serverCommit": "4ffdf4e97ad43c55d8718a1df581596db2e6097d", diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 6141bee260..8ae97a52c4 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "d59da9040814d0f2b25d930ecb36b93d932ff8906440606d259d3b9456364d3d", + "contractSha256": "b4d241e67b5a00fddd0a5d851cf86485fa8ff999e97ec067ee9b15b46a59d968", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index a021e38622..f6b1a59005 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -155,6 +155,86 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/scopes/{scope_id}/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List durable operations in one Scope + * @description List a snapshot-bounded page of public durable work in one Scope, newest first. Only the operation families a Scope owner can act on are listed; internal maintenance work stays invisible and reads as not found. Use the returned next_cursor to continue. + */ + get: operations["list_operations"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/scopes/{scope_id}/operations/{operation_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Read one durable operation + * @description Read the current state of one public durable operation owned by this Scope. An operation belonging to another Scope, or internal maintenance work, reads as not found rather than revealing its existence. + */ + get: operations["get_operation"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/scopes/{scope_id}/operations/{operation_id}/cancel": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Cancel one durable operation + * @description Request cancellation of a queued or running operation. expected_version must be the state_version of the read that decided to cancel, so a concurrent transition fails with 409 instead of silently winning. A running operation moves to cancelling and stops at its next fence; repeating the request with the current version is idempotent. + */ + post: operations["cancel_operation"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/scopes/{scope_id}/operations/{operation_id}/retry": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Retry one failed durable operation + * @description Grant one fresh bounded attempt to a failed operation, without advancing the Cursor or discarding the Sources it was selected from. expected_version must be the state_version of the read that decided to retry. Only a failed operation can be retried; any other state returns 409 and the original state is preserved. + */ + post: operations["retry_operation"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/scopes/{scope_id}/subject-sources": { parameters: { query?: never; @@ -4728,6 +4808,56 @@ export interface components { scope_id: string; block_id: string; }; + /** + * @description Lifecycle state of one durable operation. + * @enum {string} + */ + OperationStatus: "queued" | "running" | "retry_wait" | "cancelling" | "succeeded" | "failed" | "cancelled"; + DurableOperation: { + operation_id: string; + /** @description Durable work family, for example powercontext.memory.source-window. */ + kind: string; + scope_id: string; + status: components["schemas"]["OperationStatus"]; + /** @description Monotonic version to send as expected_version when cancelling or retrying. */ + state_version: number; + attempt_count: number; + max_attempts: number; + /** @description True once cancellation was requested and the operation has not stopped yet. */ + cancel_requested: boolean; + /** Format: date-time */ + available_at: string; + /** Format: date-time */ + created_at: string; + /** Format: date-time */ + updated_at: string; + /** + * Format: date-time + * @description Set once the operation reached a terminal state. + */ + completed_at: string | null; + /** @description Sanitized result classification for a succeeded operation. */ + result_code: string | null; + error_category: string | null; + error_code: string | null; + }; + ListOperationsRequest: { + kind?: string | null; + /** @description One lifecycle state; unknown values are rejected with 422. */ + status?: string | null; + /** @default 50 */ + limit: number; + cursor?: string | null; + }; + DurableOperationPage: { + items: components["schemas"]["DurableOperation"][]; + /** @description Opaque cursor for the next page; null when this page is the last. */ + next_cursor: string | null; + }; + MutateOperationRequest: { + /** @description The state_version read before deciding to cancel or retry. */ + expected_version: number; + }; TopicMemoryProcessingBlock: { block_id: string; attempt_id: string; @@ -6643,6 +6773,142 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + list_operations: { + parameters: { + query?: { + kind?: string; + status?: "queued" | "running" | "retry_wait" | "cancelling" | "succeeded" | "failed" | "cancelled"; + limit?: number; + cursor?: string; + }; + header?: never; + path: { + scope_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description One stable page of public durable operations. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DurableOperationPage"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 410: components["responses"]["CursorExpired"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; + get_operation: { + parameters: { + query?: never; + header?: never; + path: { + scope_id: string; + operation_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The current operation state. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DurableOperation"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; + cancel_operation: { + parameters: { + query?: never; + header?: never; + path: { + scope_id: string; + operation_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MutateOperationRequest"]; + }; + }; + responses: { + /** @description The operation after the cancellation request. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DurableOperation"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; + retry_operation: { + parameters: { + query?: never; + header?: never; + path: { + scope_id: string; + operation_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MutateOperationRequest"]; + }; + }; + responses: { + /** @description The operation after the retry was accepted. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DurableOperation"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; create_subject_source: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index 1e2f36add1..07f238f621 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "d59da9040814d0f2b25d930ecb36b93d932ff8906440606d259d3b9456364d3d"; +export const contractSha256 = "b4d241e67b5a00fddd0a5d851cf86485fa8ff999e97ec067ee9b15b46a59d968"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docker/README.md b/docker/README.md index cbebd87af5..1ffba2a954 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,7 @@ docker build \ . ``` -Run the Server with persistent SQLite and scheduler data: +Run the Server with persistent SQLite application and Work Ledger data: ```bash docker run --rm \ @@ -34,6 +34,11 @@ requests share one refresh. Checks use the Runtime's credentials and never expos another database or inference provider with the same `POWERCONTEXT_SERVER_*` environment variables used by a regular Server installation. +For a role-separated OceanBase topology, use `docker/compose.distributed.yaml` as a deployment template. It runs a +one-shot migrator, two API replicas, two fenced Scheduler replicas, and two lease-based Worker replicas. Replace the +example database credentials, authentication token, and provider configuration before starting it; only Workers need +model credentials. + ## Network exposure PowerContext refuses to start an unauthenticated Server on a non-loopback address unless diff --git a/docker/compose.distributed.yaml b/docker/compose.distributed.yaml new file mode 100644 index 0000000000..29f9f8319e --- /dev/null +++ b/docker/compose.distributed.yaml @@ -0,0 +1,140 @@ +name: powercontext-distributed + +x-runtime: &runtime + image: ${POWERCONTEXT_IMAGE:-powercontext-server:local} + restart: unless-stopped + environment: &shared-environment + POWERCONTEXT_SERVER_DATABASE_KIND: oceanbase + POWERCONTEXT_SERVER_DATABASE_URL: ${POWERCONTEXT_SERVER_DATABASE_URL:?set an OceanBase URL} + POWERCONTEXT_SERVER_DEPLOYMENT_MODE: distributed + POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION: ${POWERCONTEXT_BEHAVIOR_REVISION:-distributed-v1} + POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS: ${POWERCONTEXT_MEMORY_SCHEDULE_SECONDS:-60} + POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS: ${POWERCONTEXT_EXPERIENCE_SCHEDULE_SECONDS:-60} + +services: + migrate: + <<: *runtime + restart: "no" + # The supported migration entry point is the explicit db-migrate plan/apply workflow; there is no + # `server migrate` command. OceanBase migration commands require a durable evidence directory on + # one fixed maintenance host (--lock-coordination single-host binds it). The host directory must + # be writable by the container user (uid 10001). `manual` backup is a declaration only; set + # POWERCONTEXT_MIGRATION_BACKUP_POLICY=skip to explicitly accept a no-backup run. + # Container-side shell variables are escaped as `$$`; an unescaped `$policy` would be + # replaced by Compose with an empty string before the container ever runs the script. + entrypoint: ["/bin/sh", "-ec"] + command: + - | + set -eu + policy="$${POWERCONTEXT_MIGRATION_BACKUP_POLICY:-manual}" + case "$$policy" in + manual) apply_backup="--backup manual --backup-confirmed" ;; + skip) apply_backup="--backup skip --accept-no-backup" ;; + *) echo "POWERCONTEXT_MIGRATION_BACKUP_POLICY must be manual or skip" >&2; exit 2 ;; + esac + plan_id="$$(powercontext server db-migrate plan \ + --evidence-dir /var/lib/powercontext/migration-state \ + --lock-coordination single-host --backup "$$policy" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["plan_id"])')" + exec powercontext server db-migrate apply \ + --evidence-dir /var/lib/powercontext/migration-state \ + --lock-coordination single-host $$apply_backup \ + --plan-id "$$plan_id" --maintenance-confirmed --yes + environment: + <<: *shared-environment + # The migration command does not start a Runtime or register this role; + # a valid distributed role is still required while parsing ServerSettings. + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: migrator + # Compose supplies the host value (default `manual`) to the script above. + POWERCONTEXT_MIGRATION_BACKUP_POLICY: ${POWERCONTEXT_MIGRATION_BACKUP_POLICY:-manual} + volumes: + - ${POWERCONTEXT_MIGRATION_STATE_DIR:?set a durable migration evidence directory on one fixed host}:/var/lib/powercontext/migration-state + + api-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-a + POWERCONTEXT_SERVER_AUTH_ENABLED: "true" + POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token} + POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true" + ports: ["127.0.0.1:8001:8000"] + depends_on: + migrate: + condition: service_completed_successfully + + api-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-b + POWERCONTEXT_SERVER_AUTH_ENABLED: "true" + POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token} + POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true" + ports: ["127.0.0.1:8002:8000"] + depends_on: + migrate: + condition: service_completed_successfully + + scheduler-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler + POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-a + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + depends_on: + migrate: + condition: service_completed_successfully + + scheduler-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler + POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-b + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + depends_on: + migrate: + condition: service_completed_successfully + + worker-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker + POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-a + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + depends_on: + migrate: + condition: service_completed_successfully + + worker-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker + POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-b + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + depends_on: + migrate: + condition: service_completed_successfully diff --git a/docs/en/development/core-protocol.md b/docs/en/development/core-protocol.md index a0a24c3615..fc8f9e0732 100644 --- a/docs/en/development/core-protocol.md +++ b/docs/en/development/core-protocol.md @@ -102,8 +102,9 @@ conflicts and persistence; the family service is responsible for its domain beha A `Trigger` is a policy over a signal and prior state. It returns a `PolicyTransition` containing the next state and zero or more actions. A Trigger should not open storage, schedule itself, or perform the action it selects. -APScheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy. The Trigger decides what -the observed signal means. +The durable Scheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy and records work +in the database ledger. The Trigger decides what the observed signal means; a fenced Worker performs the selected +action. ## Ownership boundaries @@ -114,7 +115,7 @@ the observed signal means. | Environment-backed process configuration | `powercontext.client.settings`, `powercontext.server.settings` | | HTTP lifecycle and optional MCP transport | `powercontext.server` | | Provider-specific generation and embedding | Inference integration | -| Database and scheduler resource lifetime | Application entry point or Builtin runtime instance | +| Database, Scheduler, and Worker resource lifetime | Application entry point or Builtin runtime instance | Core models use Pydantic `BaseModel`. Add a validator when the value has a real domain constraint. Do not add wrapper properties for stored fields, custom JSON value hierarchies, or a second definition object when the model or protocol diff --git a/docs/en/development/remote-access-implementation.md b/docs/en/development/remote-access-implementation.md index 21cca85981..3ae65979da 100644 --- a/docs/en/development/remote-access-implementation.md +++ b/docs/en/development/remote-access-implementation.md @@ -80,9 +80,17 @@ and OceanBase uses HNSW for `vector` and `hybrid` searches. Inference configuration is documented in [Configure Pydantic AI inference](pydantic-ai-inference.md). -Set `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` to process pending Source windows on a persisted interval. Scheduled -jobs use the SQLite sidecar at `POWERCONTEXT_HOME/scheduler.db`. Scheduling works with either application database and -requires a configured generation pipeline. +Set `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` to process pending Source windows on a persisted interval. A +`single_node/all` deployment drives Memory, Experience, and Profile through the in-process artifact-processing +supervisor, which owns the Family and the source window. A distributed OceanBase deployment replaces that scheduling +with the primary database Work Ledger and splits the roles: `scheduler` only discovers work, `worker` only claims it, and +`api` does neither. + +Exactly one path owns a Family. A Family the ledger drives is removed from the supervisor's binding set, so the two never +claim the same window: otherwise the ledger advances the cursor and the supervisor's next tick records a noop instead of +doing the work. Moving `single_node/all` itself onto the ledger is the bridge step RFC 1430 schedules for after the +in-process supervisor retires. Scheduling works with either application database and requires a configured generation +pipeline. ## HTTP surface @@ -96,6 +104,7 @@ The source contract is `openapi/powercontext.yaml`. Generated Pydantic models an | Capabilities | source types, Artifact families, extraction, search modes | | Sources | capture durable content evidence | | Memory | flush pending Sources, remember explicit entries, search | +| Operations | inspect, list, cancel, and retry durable background work | | Memory entries | list, get, revise, retire | | History | list Memory changes | diff --git a/docs/en/docs/develop/http-api.md b/docs/en/docs/develop/http-api.md index 095f136314..b1e5e2b515 100644 --- a/docs/en/docs/develop/http-api.md +++ b/docs/en/docs/develop/http-api.md @@ -93,6 +93,29 @@ curl --fail \ "$POWERCONTEXT_URL/v1/memory/search" ``` +## Inspect, cancel and retry durable operations + +Memory and Experience background processing is a row per bounded window in the Work Ledger. Those rows can be +inspected and controlled across processes, and only for the Scope that owns them: + +| Method and path | Purpose | Access | +| --- | --- | --- | +| `GET /v1/scopes/{scope_id}/operations` | Page the Scope's public operations, optionally filtered by `kind` and `status` | `scope.read` | +| `GET /v1/scopes/{scope_id}/operations/{operation_id}` | Read one operation's current state | `scope.read` | +| `POST /v1/scopes/{scope_id}/operations/{operation_id}/cancel` | Request cancellation | `scope.contribute` | +| `POST /v1/scopes/{scope_id}/operations/{operation_id}/retry` | Grant one fresh bounded attempt to a failed operation | `scope.contribute` | + +Cancel and retry must send the `state_version` that was read as `expected_version`; `409 operation_conflict` means a +concurrent transition won and the original state is preserved. A running operation moves to `cancelling` and stops at +its next fence. After success, failure or cancellation `state_version` stops changing, so repeated reads are safe. + +An operation owned by another Scope, and internal maintenance work, both return `404`: neither is listed, and neither +can be probed by id. + +These operations come from the Work Ledger, so they appear in a `distributed` deployment; a `single_node` install keeps +processing Source windows through the APScheduler supervisor and has no such rows. `POST /v1/memory/flush` still +completes inside the request with `200` and does not return a pollable handle for `Prefer: respond-async`. + ## Scope-owned operational Prompts Operational Prompts use `family=prompt` and the same Scope access boundary as other Artifacts. Under enforced Access, @@ -211,6 +234,7 @@ use the same policy enforcement point; MCP tool visibility is not permission. | External Skills | `/v1/external-skills/*` | Scan configured targets and resolve or import packages | | Handoff Reports | `/v1/handoff-reports/*` | Generate a read-only report for a Scope selection | | Statistics | `/v1/stats` | Read scoped usage statistics | +| Durable operations | `/v1/scopes/{scope_id}/operations*` | Inspect, list, cancel, or recover Memory and Experience background work | Create an Artifact with `POST /v1/scopes/{scope_id}/artifacts`, specifying `family` in the request body. Profile candidate review uses `/v1/candidates/*`. @@ -244,6 +268,7 @@ Common statuses are: | --- | --- | | `401` | The Server requires a valid bearer token | | `403` | The authenticated Principal is not authorized for the requested action and resource | +| `429` | The shared request window is exhausted; wait for `Retry-After` | | `404` | The requested immutable value does not exist | | `409` | The request conflicts with current immutable state or an expected version | | `413` | A selected Handoff Report exceeds its output limit | diff --git a/docs/en/docs/develop/interfaces.md b/docs/en/docs/develop/interfaces.md index fc3ea13a78..c7e929e077 100644 --- a/docs/en/docs/develop/interfaces.md +++ b/docs/en/docs/develop/interfaces.md @@ -236,12 +236,12 @@ selector retains that selector, and Experience incubation uses its independent c ## Scheduled Experience incubation An integration can capture a completed task as a Content Source with metadata `"kind": "task-outcome"`. When the -Experience schedule is configured, APScheduler scans bounded Source windows and asks the configured schema-bound -pipeline for reusable situation, action, outcome, and lesson proposals. Each proposal cites exact Sources and enters -the Review Inbox as a pending Experience Candidate. +Experience schedule is configured, the durable Scheduler scans bounded Source windows and enqueues versioned work. +A fenced Worker asks the configured schema-bound pipeline for reusable situation, action, outcome, and lesson +proposals. Each proposal cites exact Sources and enters the Review Inbox as a pending Experience Candidate. -Experience incubation has its own persisted Source cursor, independent from Memory extraction. Candidate writes and -cursor advancement commit together; a generation or write failure leaves the window available for retry. Ordinary +Experience incubation has its own persisted Source cursor, independent from Memory extraction. Candidate writes, +cursor advancement, and Work success commit together; a generation or write failure leaves the window available for retry. Ordinary prompt Sources are not Task Outcomes and are ignored by this job. Scheduling stops at the review boundary. It never approves an Experience, includes pending content in diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index e3ef2f3df8..c77822d7d5 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -17,8 +17,8 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 The full Server framework uses two normalization checkpoints before its shared migration chain for the PowerContext 1.5.0 target (stored compatibility IDs shown below): -- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`. -- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003` through a separate frozen source bridge. +- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003 → v1_5_0_004`. +- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003 → v1_5_0_004` through a separate frozen source bridge. Application release numbers and schema revisions are distinct. The 1.4.0 release is not the `s1400` checkpoint: its existing Candidate schema is current, but its Topic attempt storage still needs the registered bridge. @@ -103,6 +103,15 @@ Revision `v1_5_0_002` adds the Source evidence bindings, exact Atomic Memory evi Revision `v1_5_0_003` registers `pc_atomic_memory_evidence`, the disposable exact-revision evidence projection. It is derived from retained history, can be discarded and rebuilt, and never becomes the authority for a trust decision. +Revision `v1_5_0_004` registers the durable Work Ledger and its scheduler coordination tables: +`pc_work_lanes`, `pc_work_items`, `pc_work_keys`, `pc_work_attempts`, `pc_scheduler_leases`, `pc_scheduler_scans`, +`pc_runtime_members`, and the shared `pc_rate_limit_windows`. Ledger execution is at-least-once behind a monotonic fence +token while its commit is at-most-once, so a retried, superseded or cancelled attempt cannot publish a duplicate result +or release another worker's logical key. Both installed-release source paths and vector profiles include these tables; +existing managed databases require explicit maintenance and fresh databases create them through the registered +bootstrap. Earlier revision resources, registered layouts and historical checksums remain unchanged. This registration +does not establish installed-release or external-Provider upgrade acceptance. + The runtime bundle stamps `pc_atomic_memory_evidence_schema.schema_version` with `p0005`, its own revision for the same tables; the Server chain registers that shape as `v1_5_0_002`. Operator messages name both so a runtime deployment and a managed Server deployment can each apply the revision they own. @@ -159,12 +168,13 @@ Sequence `000` identifies a registered historical baseline; upgrade steps start | `v1_5_0_001` | 1.5.0 | 1 | `v1_5_0_001` | Artifact plugin storage and search registrations | | `v1_5_0_002` | 1.5.0 | 2 | `v1_5_0_002` | Retained Source evidence and historical validation | | `v1_5_0_003` | 1.5.0 | 3 | `v1_5_0_003` | Disposable exact-revision evidence projection | +| `v1_5_0_004` | 1.5.0 | 4 | `v1_5_0_004` | Durable Work Ledger and scheduler coordination tables | -Both registered source paths reach `v1_5_0_003`; only 1.3.0 is selected for CI upgrade acceptance: +Both registered source paths reach `v1_5_0_004`; only 1.3.0 is selected for CI upgrade acceptance: ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -> v1_5_0_004 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -> v1_5_0_004 ``` `status` and `plan` show `source_migration`, `target_migration` and `pending_migrations` @@ -172,7 +182,7 @@ before the existing machine fields. `apply` and `verify` show `migration`. Each contains `name`, `software_version`, `sequence` and the compatible `revision` ID. For example: ```json -{"revision":"v1_5_0_001","name":"v1_5_0_001","software_version":"1.5.0","sequence":1} +{"revision":"v1_5_0_004","name":"v1_5_0_004","software_version":"1.5.0","sequence":4} ``` The software version describes the registered database state, not the currently running binary @@ -186,7 +196,7 @@ existing plans and original backups usable. Partial prototype bundles keep their For a new Server migration, use the release-based name directly as its Alembic `revision` and register its schema, domain steps and recovery checkpoints as usual. `down_revision` must reference -the actual preceding ID (`v1_5_0_003` when appending directly to the current head), even if that ID has +the actual preceding ID (`v1_5_0_004` when appending directly to the current head), even if that ID has a display alias. The framework follows this registered chain, not lexical name ordering. Already registered names and release assignments stay immutable; a version bump must not rename history. diff --git a/docs/en/docs/operate/deploy-server.md b/docs/en/docs/operate/deploy-server.md index d2662ed5db..f86640d40d 100644 --- a/docs/en/docs/operate/deploy-server.md +++ b/docs/en/docs/operate/deploy-server.md @@ -93,8 +93,9 @@ export POWERCONTEXT_HOME=/srv/powercontext powercontext server run ``` -The process must be able to create and update this directory. The default SQLite database and scheduler state are -stored below it. Supply the same environment variables whenever your service manager restarts the process. +The process must be able to create and update this directory. The default SQLite database also stores durable +Scheduler, Worker lease, and Operation state. Supply the same environment variables whenever your service manager +restarts the process. `server run` loads `.env` from its current directory when present. Managed deployments should export the variables, configure them in the service manager or container platform, or pass one explicit file so startup does not depend on @@ -271,7 +272,46 @@ docker run --rm \ ``` The image listens on `0.0.0.0:8000` inside the container, so the host-side address in `--publish` is important. The -named volume persists the SQLite database and scheduler state after the container stops. +named volume persists the SQLite database and durable work state after the container stops. + +## Run distributed roles + +Distributed mode requires OceanBase and a schema migration before any role starts. The repository includes +`docker/compose.distributed.yaml` as a topology example with two APIs, two Schedulers, and two Workers. One Scheduler is +leader and the other remains ready to take over; both APIs and both Workers are active. + +The distributed Work Ledger schedules Memory, Experience, and Profile work. Topic Memory processing is not supported +in distributed v1; leave its schedule unset. The single-node Artifact Processing Supervisor role must remain `all` +because `POWERCONTEXT_SERVER_DEPLOYMENT_ROLE` owns process separation in this topology. + +Export secrets and deployment choices without writing them into Compose: + +```bash +export POWERCONTEXT_SERVER_DATABASE_URL="$OCEANBASE_URL" +export POWERCONTEXT_SERVER_AUTH_TOKEN="$POWERCONTEXT_DEPLOYMENT_TOKEN" +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL="openai:gpt-4.1-mini" +export OPENAI_API_KEY +docker compose --file docker/compose.distributed.yaml up migrate +docker compose --file docker/compose.distributed.yaml up -d api-a api-b scheduler-a scheduler-b worker-a worker-b +``` + +The two API examples listen on host ports 8001 and 8002. Put a TLS-terminating load balancer in front of them and use +round-robin routing without session affinity. Distributed MCP is stateless. The example intentionally supplies model +credentials only to Workers and no public port to Scheduler or Worker roles. A production deployment should also use +separate least-privilege database users instead of the shared demonstration URL. + +Every replica in one rollout must use the same behavior revision. Upgrade in this order: + +1. run the `migrate` job (`server db-migrate plan` then `apply`, as wired in the Compose file) with a + dedicated DDL account, its durable evidence directory on one fixed maintenance host, and the + documented backup and confirmation options; +2. replace Workers and wait for readiness; +3. replace Schedulers and confirm a leader can scan; +4. replace APIs. + +Keep `POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` on the older supported value until old Workers have drained. +Rollback in the reverse order and never start a distributed role against a schema that has not reached the packaged +revision. ## Enable authentication diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 3fa36ab702..4fbcb8378a 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -269,7 +269,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_work_lanes,pc_work_items,pc_work_keys,pc_work_attempts,pc_source_write_plugins,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` diff --git a/docs/en/rfcs/0011_remote_access_architecture.md b/docs/en/rfcs/0011_remote_access_architecture.md index 14ef5a9e59..257b3b70f6 100644 --- a/docs/en/rfcs/0011_remote_access_architecture.md +++ b/docs/en/rfcs/0011_remote_access_architecture.md @@ -7,6 +7,10 @@ description: "A proposal for exposing Server services through one OpenAPI HTTP c - Start Date: 2026-07-16 - RFC PR: [oceanbase/powercontext#11](https://github.com/oceanbase/powercontext/pull/11) +> **Deployment boundary:** [RFC 1430](1430_distributed_server_workers.md) defines the accepted durable Operation, +> stateless multi-replica API, role separation, and distributed coordination contract. It supersedes this RFC where +> this document leaves execution and deployment semantics open. + # Summary This RFC proposes a remote access boundary for PowerContext. A Server exposes application services through an diff --git a/docs/en/rfcs/0019_local_source_memory_runtime.md b/docs/en/rfcs/0019_local_source_memory_runtime.md index 318d453831..afccbee057 100644 --- a/docs/en/rfcs/0019_local_source_memory_runtime.md +++ b/docs/en/rfcs/0019_local_source_memory_runtime.md @@ -10,6 +10,10 @@ description: A proposal for a SQLite-backed Runtime that captures scoped Sources > **Note:** PowerContext now uses bundled sqlite-vec for SQLite vector search. Statements about Vec1 in this RFC no > longer apply and remain only as a record of the original design. +> **Execution update:** [RFC 1430](1430_distributed_server_workers.md) replaces the APScheduler sidecar and +> single-process execution assumptions with one database-backed Work Ledger for local and distributed modes. This RFC +> remains authoritative for Source window, cursor, Memory, and domain commit semantics. + # Summary This RFC proposes backend-neutral Runtime storage contracts and a built-in SQLite profile. The Runtime uses the diff --git a/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md b/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md index af2fd39824..50172f3c6e 100644 --- a/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md +++ b/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md @@ -11,6 +11,10 @@ description: A proposal for Source capture and Memory operations through a FastA > **Note:** PowerContext now uses bundled sqlite-vec for SQLite vector search. Statements about Vec1 in this RFC no > longer apply and remain only as a record of the original design. +> **Execution update:** [RFC 1430](1430_distributed_server_workers.md) supersedes this RFC's single-process scheduler, +> synchronous-only flush, and non-durable Operation assumptions. This RFC continues to define the HTTP mapping and +> generated-contract boundary. + # Summary This RFC defines the first concrete remote API built from the architecture in RFC 0011 and the local Runtime in diff --git a/docs/en/rfcs/1430_distributed_server_workers.md b/docs/en/rfcs/1430_distributed_server_workers.md new file mode 100644 index 0000000000..547d77c6d6 --- /dev/null +++ b/docs/en/rfcs/1430_distributed_server_workers.md @@ -0,0 +1,233 @@ +- Proposal Name: `distributed_server_workers` +- Start Date: 2026-09-03 +- Tracking Issue: [oceanbase/powercontext#1430](https://github.com/oceanbase/powercontext/issues/1430) +- Related RFCs: [RFC 0011](0011_remote_access_architecture.md), [RFC 0019](0019_local_source_memory_runtime.md), + [RFC 0020](0020_runtime_backed_memory_remote_access.md), and [RFC 0046](0046_observability_foundations.md) + +# Summary + +PowerContext uses one database-backed Work Ledger for both local and distributed background execution. An API replica +can enqueue and inspect work without process affinity. Schedulers elect one fenced leader while retaining durable +keyset scan positions. Workers claim only available capacity, renew database-time leases, and safely recover abandoned +attempts. Model calls remain at-least-once, while one logical Source window can commit at most one database result. + +OceanBase is the only coordination backend in distributed mode. The same ledger also replaces the APScheduler SQLite +sidecar in `single_node/all`, so there is one execution protocol to understand and test. Redis, Kafka, a public queue +SPI, arbitrary user code, DAG scheduling, priorities, multi-region consensus, and connector-specific checkpoints are +outside this RFC. + +# Decisions + +The design deliberately minimizes sources of truth: + +- OceanBase owns work state, leases, cursors, and domain commits in distributed mode. +- Execution is at-least-once. Provider work may repeat after a crash, but fencing and the domain cursor prevent an old + attempt from committing. +- Memory activation and Experience incubation use the ledger first. Future handlers keep their domain checkpoints in + their own modules. +- The default remains `single_node/all`; distributed processes use exactly one of `api`, `scheduler`, or `worker`. +- Work payloads are versioned references and numeric window boundaries, not Source bodies, prompts, model responses, + credentials, or exception text. + +This avoids entropy from parallel database, broker, and local-sidecar protocols. The compensating complexity is made +explicit as database constraints, a small state machine, fencing tokens, startup validation, and compatibility +advertisements rather than tacit assumptions about one process. + +# Architecture + +```text +API / Scheduler + | short transaction: identify window, deduplicate, enqueue + v +OceanBase Work Ledger + | short transaction: claim lane head and issue lease fence + v +Worker ---- provider/model call outside a transaction ----> + | short transaction: validate fence, commit domain state + cursor + work + v +one logical database result +``` + +The ledger consists of: + +| Table | Responsibility | +| --- | --- | +| `pc_work_items` | Versioned work, scope, lane sequence, safe payload, state, lease, attempt budget, and safe result/error | +| `pc_work_attempts` | Append-only owner/fence/timing/outcome audit and non-sensitive trace identity | +| `pc_work_lanes` | Strict sequence and one active head for each consistency domain | +| `pc_work_keys` | Unique reservation for the current logical window | +| `pc_scheduler_leases` | Leader and migrator ownership with monotonic fences and database-time expiry | +| `pc_scheduler_scans` | Next run time and keyset continuation for each discoverer | +| `pc_runtime_members` | Live role and schema/payload/behavior compatibility advertisements | +| `pc_rate_limit_windows` | Optional shared fixed-window counters keyed by an opaque principal digest and policy | + +The logical key for a Memory window is the SHA-256 digest of its handler kind, scope, cursor name, cursor generation, +and previous cursor. The enqueue-time high watermark and `through` boundary are stored in the work payload but are not +part of that key. Manual and scheduled discovery therefore join the same unadvanced cursor window; Sources arriving +later are handled by the next window. + +Memory and Experience use independent lanes (`memory:{scope}` and `experience:{scope}` before hashing). Different lanes +may execute concurrently, while a lane advances strictly in sequence. A terminal failure retains its logical-key and +lane reservation until an operator retries or cancels it, preventing a poison window from being recreated forever. + +# State machine + +```text +queued ----> running ----> succeeded + | |----> retry_wait ----> running + | |----> failed + | `----> cancelling ----> cancelled + `-----------------------------------> cancelled + +failed -- explicit operator retry --> queued +``` + +Cancellation linearizes at the work row. If cancellation wins before the final transaction, the Worker discards the +prepared provider result and completes the attempt as cancelled. If success has already committed, cancellation returns +`409`. Cancelling one operation does not pause future discovery for its scope. + +# Lock and transaction protocol + +OceanBase 4.3.5 with Read Committed isolation is the baseline. Correctness does not depend on an unlocked read, a +single predicate update, `SKIP LOCKED`, or `GET_LOCK()`. Every path follows this order: + +```text +scheduler lease (scheduler paths only) + -> lane + -> logical key + -> work item + -> domain cursor/head +``` + +Stable lease and lane rows are explicitly locked. A missing row is inserted under a unique key and the operation is +retried after a conflict. Claiming starts with a bounded indexed candidate scan, then locks and claims one lane at a +time in a short transaction. Workers never prefetch more work than their free execution slots. + +Scheduler enqueue and scan updates revalidate the exact leader owner, fence, and database expiry. Worker heartbeats, +failures, cancellation convergence, and completion validate the exact `(work_id, owner, fence)`. All expiry decisions +use `CURRENT_TIMESTAMP(6)` from the database; application time is used only for polling and deadlines. + +Provider, connector, network, and filesystem calls never run under a database transaction. The final Worker +transaction validates an unexpired, uncancelled claim and then invokes the handler's domain commit. Memory and +Experience reuse their existing cursor/head compare-and-swap. Domain writes, cursor advancement, attempt completion, +and Work success either commit together or roll back together. + +# Scheduler and Worker behavior + +Each discoverer stores a durable keyset continuation and scans at most 100 scopes per page. Repeating a page after a +crash is safe because logical keys deduplicate it. A former leader cannot enqueue or save a continuation after another +Scheduler acquires a higher fence. + +A Worker claims only its current free slots. Its attempt lease defaults to 120 seconds and is renewed every 30 seconds. +Expired attempts are closed in the audit trail, moved through retry state, and reclaimed with a higher fence. Retryable +failures use full-jitter exponential backoff, beginning at 2 seconds and capped at 5 minutes. Five automatic generation +attempts are allowed by default. Unsupported payload versions stay visible at the lane head and make Worker readiness +`misconfigured`; they are never guessed, silently dropped, or moved aside. + +# Process roles + +Configuration adds `deployment`, `coordination`, `worker`, `operations`, and `rate_limit` groups. + +- `single_node/all` runs API, Scheduler, and Worker together using the same ledger. A database ownership lease rejects a + mistakenly started second instance. +- `distributed/api` exposes HTTP, Dashboard, MCP, authentication, shared rate limiting, enqueue, and operation queries. + It does not run discovery or background Worker handlers. +- `distributed/scheduler` exposes only health and metrics, owns no provider credentials, and performs bounded discovery. +- `distributed/worker` exposes only health and metrics and owns only the provider credentials required by its handlers. + +Distributed mode requires OceanBase and rejects `role=all`, SQLite, seekDB, and explicit host-local External Skill +targets. All roles use one image but should use separate least-privilege database accounts. DDL belongs to a migrator +account and is never performed automatically by a distributed role. + +Every process heartbeats a boot-unique member identity with build version, current schema and payload range, and a +non-sensitive `behavior_revision`. Credentials, secret URLs, authorization data, and hashes that permit offline secret +guessing are not member metadata. + +# HTTP and Client contract + +`openapi/powercontext.yaml` remains authoritative. `POST /v1/memory/flush` behaves as follows: + +- no pending Source, or completion within the wait budget: `200 FlushMemoryResponse`; +- still queued, running, or waiting to retry: `202 OperationAccepted`, a relative `Location`, and `Retry-After: 2`; +- a failed operation already blocks that logical key: `409 operation_blocked` with its operation ID. + +`Prefer: respond-async` selects an immediate handle. `Prefer: wait=N` selects a bounded wait up to 30 seconds; the +default is 10 seconds. Operation endpoints provide authorized get/list, optimistic cancel, and operator retry. Mutation +bodies carry `expected_version`; illegal or stale transitions return `409`. Public status values are `queued`, +`running`, `retry_wait`, `cancelling`, `succeeded`, `failed`, and `cancelled`. Internal maintenance work is not exposed by +the Operation API. + +Implementation status in this change: the Operation API is served under the owning Scope +(`GET|POST /v1/scopes/{scope_id}/operations[...]`), so authorization resolves from the request and a row belonging to +another Scope reads as `404` exactly like maintenance work. Optimistic cancel and operator retry, the seven public +states, and the maintenance whitelist are implemented. Still proposed and not implemented: the `202 OperationAccepted` +handle, `Prefer: respond-async` / `Prefer: wait=N`, `409 operation_blocked` from a blocking failed window, and the +client `submit_memory_flush()` / polling helpers. Until those land, `POST /v1/memory/flush` keeps answering `200` +synchronously. + +`PowerContextClient.flush_memory()` preserves its synchronous-looking result by submitting and polling until its total +deadline. If that deadline expires it raises `OperationPendingError` with the operation ID. Callers that want explicit +control use `submit_memory_flush()`, `get_operation()`, `list_operations()`, `cancel_operation()`, and +`retry_operation()`. + +Operation endpoints are not projected as MCP tools. In distributed mode FastMCP uses stateless HTTP, so consecutive +requests may reach different API replicas. Server-to-client elicitation and sampling are disabled; the Workstream picker +returns a structured `needs_selection` response. The internal ASGI bridge carries the already authenticated principal, +and tool visibility is never treated as authorization. + +# Health, shutdown, retention, and observability + +Liveness means the process can respond. API readiness requires database, schema, authentication policy, membership, +and behavior compatibility. Missing Scheduler or Worker members make API readiness degraded rather than preventing +durable enqueue and reads. A healthy Scheduler standby is ready. A Worker that lacks a required provider or handler +version stops claiming and reports the precise failing check. + +On SIGTERM, API readiness is removed first. Scheduler stops discovery and conditionally releases its lease. Worker +stops claiming, continues heartbeats for in-flight work, and drains for at most 90 seconds; an ungraceful exit recovers +through lease expiry. + +Succeeded and cancelled work and attempts are retained for 30 days. A durable maintenance handler deletes no more than +500 records per batch and also removes expired rate-limit counters. Failed blocking work is retained until operator +action. Future Source retention must treat every non-terminal work window as a retention root. + +Metrics use only bounded labels such as kind, status, outcome, role, and error category. They cover queue depth and age, +claim and attempt latency, lease expiry, retries, throughput, leadership changes, and member counts. Scope, principal, +and work IDs are not metric labels. Enqueue, claim, execute, commit, and retry are separate spans; retry attempts use span +links rather than pretending to be one uninterrupted trace. + +Logs, spans, work rows, and attempt rows must never contain Source content, prompts, model output, credentials, +authorization headers, or complete secret URLs. Persistent errors use bounded category and code values. + +# Schema and rollout + +Alembic owns a forward-only schema chain. `powercontext server migrate` acquires a database lease and upgrades through +the packaged head. A new database starts at the baseline; a known complete legacy schema is validated, receives only +recognized expansions, and is stamped before the ledger migration. Unknown or partially installed schemas are rejected. +Single-node startup runs the same migration path automatically; distributed roles only validate the current revision. + +Schema evolution follows expand, mixed-version deployment, then contract. Release N+1 must read N and N+1 layouts, and +destructive removal waits until N+2. Workers support the current and previous payload during an actual mixed-version +transition. `emit_payload_version` stays on the old format until old Workers drain. + +The deployment sequence is migrate, Workers, Schedulers, then APIs. Rollback reverses that order and drains any newer +payload before an older Worker is restored. The bridge release first puts `single_node/all` on the Work Ledger; only +after the APScheduler execution path is absent may the same database be served by multiple roles. The old sidecar is an +operator backup artifact and is not deleted automatically. + +# Validation + +Acceptance requires round-robin HTTP, Dashboard API, and stateless MCP tests across two API replicas; overlapping work +on different lanes and serialization on one lane; manual/scheduled deduplication; Worker crash points before claim, +during provider execution, during final commit, and after commit; Scheduler takeover fencing; retry, cancellation, +operator recovery, retention, and privacy tests; and real multi-process OceanBase 4.3.5 tests. Golden schema/payload +fixtures cover mixed versions and upgrade/rollback. SQLite and seekDB remain supported only for single-node regression +tests. + +# Alternatives rejected + +- Redis or Kafka would add another durable truth and a cross-system commit problem before throughput requires it. +- A public Queue/Coordinator SPI would freeze multiple coordination semantics before any second implementation exists. +- `GET_LOCK()` is session-scoped and unsafe as a pooled-connection fencing primitive. +- Keeping APScheduler for local mode would preserve two subtly different execution and recovery protocols. +- Exactly-once provider execution is not achievable across external calls; the enforceable contract is at-least-once + execution with at-most-one fenced database commit. diff --git a/docs/zh/development/core-protocol.md b/docs/zh/development/core-protocol.md index f4d6938eff..38df57055f 100644 --- a/docs/zh/development/core-protocol.md +++ b/docs/zh/development/core-protocol.md @@ -98,7 +98,8 @@ service 负责领域行为。 `Trigger` 是基于 signal 和先前 state 的策略。它返回 `PolicyTransition`，其中包含下一状态和零个或多个 action。 Trigger 不应打开存储、调度自身或执行它选择的 action。 -APScheduler 属于 Builtin runtime 的生命周期，负责决定何时评估策略。Trigger 只解释当前 signal 的含义。 +持久化 Scheduler 属于 Builtin runtime 生命周期，负责决定何时评估策略并把任务写入数据库 ledger。Trigger 只解释 +当前 signal 的含义；带 fence 的 Worker 执行被选择的 action。 ## 职责边界 @@ -109,7 +110,7 @@ APScheduler 属于 Builtin runtime 的生命周期，负责决定何时评估策 | 基于环境变量的进程配置 | `powercontext.client.settings`、`powercontext.server.settings` | | HTTP 生命周期与可选 MCP transport | `powercontext.server` | | Provider 相关的生成和 embedding | 推理集成 | -| 数据库和 scheduler 资源生命周期 | 应用入口或 Builtin runtime instance | +| 数据库、Scheduler 和 Worker 资源生命周期 | 应用入口或 Builtin runtime instance | Core model 使用 Pydantic `BaseModel`。只有存在真实领域约束时才增加 validator。对于普通存储字段，不需要包装 property；模型和协议已经能表达边界时，也不需要自定义 JSON value 层级或第二套 definition 对象。 diff --git a/docs/zh/development/remote-access-implementation.md b/docs/zh/development/remote-access-implementation.md index 0a64c51448..0836a5174d 100644 --- a/docs/zh/development/remote-access-implementation.md +++ b/docs/zh/development/remote-access-implementation.md @@ -77,8 +77,14 @@ export POWERCONTEXT_SERVER_DATABASE_URL="mysql+aoceanbase://user:password@host:2 inference 配置见[配置 Pydantic AI 推理](pydantic-ai-inference.md)。 设置 `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` 可以按持久化 interval 处理待消费的 Source window。 -定时 job 使用 `POWERCONTEXT_HOME/scheduler.db` 作为 SQLite sidecar。调度可以配合任一 application database -使用，但必须配置 generation pipeline。 +`single_node/all` 部署由进程内的 artifact processing supervisor 驱动 Memory、Experience 和 Profile，由它持有 +Family 与 Source window。分布式 OceanBase 部署用主数据库 Work Ledger 取代这一调度，并拆分角色：`scheduler` +只做发现，`worker` 只做领取，`api` 两者都不做。 + +每个 Family 只能有一个持有者。账本接管的 Family 会从 supervisor 的 binding 集合中移除，两者永不争抢同一个 +window：否则账本推进游标后，supervisor 的下一次 tick 只会记录 noop 而不再执行实际工作。把 `single_node/all` +本身迁到账本，是 RFC 1430 安排在进程内 supervisor 退役之后的桥接步骤。调度在任一应用数据库上都可用，且需要 +配置 generation pipeline。 ## HTTP 接口 @@ -92,6 +98,7 @@ inference 配置见[配置 Pydantic AI 推理](pydantic-ai-inference.md)。 | Capabilities | source type、Artifact family、extraction、search mode | | Sources | capture 持久化 content evidence | | Memory | flush 待处理 Source、remember 显式 entry、search | +| Operations | 查询、列举、取消和重试持久后台任务 | | Memory entries | list、get、revise、retire | | History | list Memory change | diff --git a/docs/zh/docs/develop/http-api.md b/docs/zh/docs/develop/http-api.md index c3725619df..68c6669e62 100644 --- a/docs/zh/docs/develop/http-api.md +++ b/docs/zh/docs/develop/http-api.md @@ -86,6 +86,29 @@ curl --fail \ "$POWERCONTEXT_URL/v1/memory/search" ``` +## 检查、取消与重试持久 Operation + +Memory 与 Experience 的后台处理是一行行持久化的 Work Ledger operation。它们可以跨进程检查和控制，且只 +暴露给该 Scope 的访问者： + +| 方法与路径 | 作用 | 权限 | +| --- | --- | --- | +| `GET /v1/scopes/{scope_id}/operations` | 分页列出该 Scope 的公开 operation，可用 `kind`、`status` 过滤 | `scope.read` | +| `GET /v1/scopes/{scope_id}/operations/{operation_id}` | 读取一个 operation 的当前状态 | `scope.read` | +| `POST /v1/scopes/{scope_id}/operations/{operation_id}/cancel` | 请求取消 | `scope.contribute` | +| `POST /v1/scopes/{scope_id}/operations/{operation_id}/retry` | 为 failed operation 再给一次有界尝试 | `scope.contribute` | + +取消与重试都必须把读取时的 `state_version` 作为 `expected_version` 传回；`409 operation_conflict` 表示该 +operation 已被并发状态变更抢先，原状态不受影响。running operation 收到取消请求后进入 `cancelling`，在下一次 +fence 停止。成功、失败、取消之后 `state_version` 不再变化，重复读取是安全的。 + +另一个 Scope 的 operation 与内部 maintenance work 一律返回 `404`：两者既不出现在列表里，也不能通过 ID 探测 +是否存在。 + +这些 operation 只由 Work Ledger 产生，因此出现在 `distributed` 部署中；`single_node` 仍由 APScheduler +supervisor 处理 Source 窗口，没有对应的 operation 行。`POST /v1/memory/flush` 目前仍在请求内同步完成并返回 +`200`，不会为 `Prefer: respond-async` 返回可轮询的句柄。 + ## Scope 内的操作提示词 操作提示词使用 `family=prompt`，遵循 Artifact 的 Scope 权限边界。由于 Prompt 会影响整个 Scope 的运行行为，启用 @@ -192,6 +215,7 @@ Principal，并注入 Authorization Provider。HTTP 与 MCP 使用同一个策 | 外部 Skill | `/v1/external-skills/*` | 扫描已配置 target，解析或导入 package | | Handoff Report | `/v1/handoff-reports/*` | 按 Scope selection 生成只读报告 | | 统计 | `/v1/stats` | 读取指定 scope 的使用统计 | +| 持久 Operation | `/v1/scopes/{scope_id}/operations*` | 检查、列出、取消或恢复 Memory/Experience 后台任务 | 创建 Artifact 使用 `POST /v1/scopes/{scope_id}/artifacts`，在请求体中指定 `family`。 Profile 的候选审核复用 `/v1/candidates/*`。 @@ -223,6 +247,7 @@ Atomic Memory 正文最多为 8192 个 UTF-8 字节，超过限制返回 HTTP `4 | --- | --- | | `401` | Server 要求有效的 Bearer token | | `403` | 已认证 Principal 无权对目标资源执行请求的 action | +| `429` | 共享请求窗口已耗尽；按 `Retry-After` 等待 | | `404` | 请求的不可变值不存在 | | `409` | 请求与当前不可变状态或 expected version 冲突 | | `413` | 选中的 Handoff Report 超过输出限制 | diff --git a/docs/zh/docs/develop/interfaces.md b/docs/zh/docs/develop/interfaces.md index f299c65a2a..29f9048e54 100644 --- a/docs/zh/docs/develop/interfaces.md +++ b/docs/zh/docs/develop/interfaces.md @@ -216,12 +216,12 @@ selector；Experience 孵化继续使用独立的游标和定时任务。 ## 后台 Experience 孵化 Integration 可以把已完成任务采集为 metadata 含 `"kind": "task-outcome"` 的 Content Source。启用 -Experience schedule 后，APScheduler 会扫描有上限的 Source window，并让配置好的 schema-bound pipeline -生成可复用的 situation、action、outcome 和 lesson。每条 proposal 都引用精确 Source，并以 pending -Experience Candidate 进入 Review Inbox。 +Experience schedule 后，持久化 Scheduler 扫描有上限的 Source window 并写入带版本任务；带 fence 的 Worker 再让 +配置好的 schema-bound pipeline 生成可复用的 situation、action、outcome 和 lesson。每条 proposal 都引用精确 Source， +并以 pending Experience Candidate 进入 Review Inbox。 -Experience 孵化使用独立于 Memory extraction 的持久化 Source cursor。Candidate 写入和 cursor 推进会在同一 -事务提交；generation 或写入失败时，该 window 保留给下次重试。普通 Prompt Source 不是 Task Outcome， +Experience 孵化使用独立于 Memory extraction 的持久化 Source cursor。Candidate 写入、cursor 推进和 Work success +会在同一事务提交；generation 或写入失败时，该 window 保留给下次重试。普通 Prompt Source 不是 Task Outcome， 不会进入这个 job。 后台流程止于审核边界：它不会批准 Experience、把 pending 内容放入 PreparedContext、派生 managed Skill、 diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index be639972e5..89f4c3f410 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -14,8 +14,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 完整 Server 框架使用两个归一化检查点接入共享迁移链： -- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`。 -- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`。 +- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003 → v1_5_0_004`。 +- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003 → v1_5_0_004`。 应用发布版本与 schema revision 分别编号。1.4.0 发行包不等于 `s1400` 检查点：其 Candidate 结构已经是当前格式， 但 Topic 尝试状态仍需登记的桥接。两条路径保留既有引用和历史，到达相同的受管目标结构；原有资源摘要分别保留， @@ -83,6 +83,13 @@ SQLite 还登记可移植向量和 FTS 辅助结构。两条来源链均通过 修订 `v1_5_0_003` 登记 `pc_atomic_memory_evidence`，即精确 revision 的可丢弃证据投影。它由保留历史派生， 可以丢弃并重建，从不作为信任判定的权威依据。 +修订 `v1_5_0_004` 登记持久 Work Ledger 及其调度协调表：`pc_work_lanes`、`pc_work_items`、`pc_work_keys`、 +`pc_work_attempts`、`pc_scheduler_leases`、`pc_scheduler_scans`、`pc_runtime_members`，以及共享的 +`pc_rate_limit_windows`。Ledger 执行在单调 fence token 之后至少一次，而其提交至多一次，因此重试、被取代或已取消的 +尝试都不能发布重复结果或释放其他 worker 的逻辑键。两条已安装发布来源路径与向量 profile 都包含这些表；已有受管库 +需要显式维护，新库通过已登记的 bootstrap 创建。历史修订资源、已登记结构和历史校验和保持不变。该登记不代表已通过 +真实安装发布版或外部 Provider 的升级验收。 + runtime bundle 在 `pc_atomic_memory_evidence_schema.schema_version` 中写入 `p0005`，即它自身应用同一结构所用的 revision；Server 链将同一结构登记为 `v1_5_0_002`。运维提示会同时给出两者，使 runtime 部署与受管 Server 部署 各自应用所属的 revision。 @@ -131,12 +138,13 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `v1_5_0_001` | 1.5.0 | 第一步 | `v1_5_0_001` | Artifact 插件存储与检索登记 | | `v1_5_0_002` | 1.5.0 | 第二步 | `v1_5_0_002` | 保留 Source 证据及历史验证 | | `v1_5_0_003` | 1.5.0 | 第三步 | `v1_5_0_003` | 可丢弃的精确 revision 证据投影 | +| `v1_5_0_004` | 1.5.0 | 第四步 | `v1_5_0_004` | 持久 Work Ledger 与调度协调表 | -两条已登记来源路径都升级到 `v1_5_0_003`；CI 升级验收只选择 1.3.0： +两条已登记来源路径都升级到 `v1_5_0_004`；CI 升级验收只选择 1.3.0： ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -> v1_5_0_004 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 -> v1_5_0_004 ``` `status` 和 `plan` 在原有机器字段之前显示 `source_migration`、`target_migration`、 @@ -144,7 +152,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 `software_version`、`sequence` 和兼容保留的 `revision` ID。例如： ```json -{"revision":"v1_5_0_001","name":"v1_5_0_001","software_version":"1.5.0","sequence":1} +{"revision":"v1_5_0_004","name":"v1_5_0_004","software_version":"1.5.0","sequence":4} ``` 这里的软件版本描述已登记的数据库状态，不代表当前运行的软件包版本，也不代表中断维护窗口最初的来源。 @@ -155,7 +163,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 这样可以继续使用已有计划和原始备份。局部原型 bundle 保留自己的独立编号。 后续新增 Server 迁移直接使用该命名作为 Alembic `revision`，并正常登记结构、领域步骤和恢复检查点。 -`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_003`，即使它已有展示别名。 +`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_004`，即使它已有展示别名。 框架按照登记的链执行，不按名称字典序推算。已登记的名称和所属发布版本保持不变，版本升级不能重命名历史。 ## 初始化与维护职责 diff --git a/docs/zh/docs/operate/deploy-server.md b/docs/zh/docs/operate/deploy-server.md index fe63f3590c..a5f114d636 100644 --- a/docs/zh/docs/operate/deploy-server.md +++ b/docs/zh/docs/operate/deploy-server.md @@ -86,8 +86,8 @@ export POWERCONTEXT_HOME=/srv/powercontext powercontext server run ``` -运行进程必须能创建和更新该目录。默认 SQLite 数据库和 scheduler 状态都保存在这里。服务管理器每次重启进程时都应 -提供相同的环境变量。 +运行进程必须能创建和更新该目录。默认 SQLite 数据库也保存持久 Scheduler、Worker lease 和 Operation 状态。 +服务管理器每次重启进程时都应提供相同的环境变量。 当前目录存在 `.env` 时，`server run` 会自动加载。托管部署应导出变量、由服务管理器或容器平台提供，或者显式传入文件， 避免启动行为依赖工作目录： @@ -249,7 +249,43 @@ docker run --rm \ ``` 镜像内部监听 `0.0.0.0:8000`，所以 `--publish` 中的宿主机地址非常重要。容器停止后，named volume 仍会保留 -SQLite 数据库和 scheduler 状态。 +SQLite 数据库和持久 work 状态。 + +## 运行分布式角色 + +分布式模式要求 OceanBase，并且任何角色启动前都必须先迁移 schema。仓库中的 +`docker/compose.distributed.yaml` 提供两 API、两 Scheduler、两 Worker 的拓扑示例。一个 Scheduler 成为 leader， +另一个保持 ready 并可接管；两个 API 和两个 Worker 都会同时工作。 + +分布式 Work Ledger 负责调度 Memory、Experience 和 Profile 工作。分布式 v1 不支持 Topic Memory processing，必须 +保持其周期未设置。该拓扑由 `POWERCONTEXT_SERVER_DEPLOYMENT_ROLE` 拆分进程，因此单机 Artifact Processing +Supervisor 的 role 必须保持为 `all`。 + +通过环境传入 secret 和部署选择，不要把它们写进 Compose： + +```bash +export POWERCONTEXT_SERVER_DATABASE_URL="$OCEANBASE_URL" +export POWERCONTEXT_SERVER_AUTH_TOKEN="$POWERCONTEXT_DEPLOYMENT_TOKEN" +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL="openai:gpt-4.1-mini" +export OPENAI_API_KEY +docker compose --file docker/compose.distributed.yaml up migrate +docker compose --file docker/compose.distributed.yaml up -d api-a api-b scheduler-a scheduler-b worker-a worker-b +``` + +两个 API 示例分别监听宿主机 8001 和 8002 端口。应在它们前面配置终止 TLS 的 load balancer，并使用不带 session +affinity 的 round-robin；分布式 MCP 是 stateless。示例有意只把模型 credential 交给 Worker，也不向宿主机发布 +Scheduler 或 Worker 的端口。生产环境还应为各角色使用独立最小权限数据库账号，而不是复用示例 URL。 + +同一轮发布的所有副本必须使用相同 behavior revision。升级顺序如下： + +1. 使用专门的 DDL 账号运行 `migrate` 任务（即 Compose 文件中接线的 `server db-migrate plan` → + `apply` 流程），配备固定维护主机上的持久 evidence 目录以及文档要求的备份与确认选项； +2. 替换 Worker 并等待 readiness； +3. 替换 Scheduler 并确认 leader 可以扫描； +4. 替换 API。 + +旧 Worker 排空前，`POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` 必须保持旧的受支持值。回滚顺序相反， +并且绝不能让分布式角色连接尚未升级到 packaged revision 的 schema。 ## 启用鉴权 diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 18d9617449..ab0b0464dd 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -260,7 +260,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_work_lanes,pc_work_items,pc_work_keys,pc_work_attempts,pc_source_write_plugins,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` diff --git a/docs/zh/rfcs/0011_remote_access_architecture.md b/docs/zh/rfcs/0011_remote_access_architecture.md index 86144b5664..f9fb553c1c 100644 --- a/docs/zh/rfcs/0011_remote_access_architecture.md +++ b/docs/zh/rfcs/0011_remote_access_architecture.md @@ -7,6 +7,9 @@ description: "远程访问设计提案：由 Server 通过统一的 OpenAPI HTTP - Start Date: 2026-07-16 - RFC PR: [oceanbase/powercontext#11](https://github.com/oceanbase/powercontext/pull/11) +> **部署边界：** [RFC 1430](1430_distributed_server_workers.md) 定义已接受的持久 Operation、无状态多副本 API、 +> 角色拆分和分布式协调契约。本 RFC 未定义执行与部署语义之处，以 RFC 1430 为准。 + # Summary 本 RFC 提议为 PowerContext 建立远程访问边界。Server 通过 OpenAPI 定义的 HTTP contract 暴露 application diff --git a/docs/zh/rfcs/0019_local_source_memory_runtime.md b/docs/zh/rfcs/0019_local_source_memory_runtime.md index 905656392a..01a90adee8 100644 --- a/docs/zh/rfcs/0019_local_source_memory_runtime.md +++ b/docs/zh/rfcs/0019_local_source_memory_runtime.md @@ -7,6 +7,9 @@ description: 本地 Runtime 的设计提案，以 SQLite 保存 Scope 内的 Sou - Start Date: 2026-07-24 - RFC PR: [oceanbase/powercontext#19](https://github.com/oceanbase/powercontext/pull/19) +> **执行更新：** [RFC 1430](1430_distributed_server_workers.md) 使用同一套数据库 Work Ledger 替代 APScheduler +> sidecar 和单进程执行假设，并同时服务本地与分布式模式。本 RFC 仍负责 Source window、cursor、Memory 和领域提交语义。 + > **注意：** PowerContext 当前使用随 builtin 依赖捆绑的 sqlite-vec 提供 SQLite 向量检索。本 RFC 中关于 Vec1 的 > 表述已不再适用，仅作为原始设计记录保留。 diff --git a/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md b/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md index 325c6d4e2f..e09844a0ae 100644 --- a/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md +++ b/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md @@ -11,6 +11,9 @@ description: 基于 Runtime 提供 Source 采集和 Memory 远程操作的提案 > **注意：** PowerContext 当前使用随 builtin 依赖捆绑的 sqlite-vec 提供 SQLite 向量检索。本 RFC 中关于 Vec1 的 > 表述已不再适用，仅作为原始设计记录保留。 +> **执行更新：** [RFC 1430](1430_distributed_server_workers.md) 取代本 RFC 中的单进程调度、仅同步 flush 和 +> 非持久 Operation 假设。本 RFC 继续定义 HTTP 映射和生成契约边界。 + # Summary 本 RFC 定义首个基于 RFC 0011 架构和 RFC 0019 本地 Runtime 的具体远程 API。FastAPI Server 暴露 diff --git a/docs/zh/rfcs/1430_distributed_server_workers.md b/docs/zh/rfcs/1430_distributed_server_workers.md new file mode 100644 index 0000000000..e8961bbf85 --- /dev/null +++ b/docs/zh/rfcs/1430_distributed_server_workers.md @@ -0,0 +1,205 @@ +- Proposal Name: `distributed_server_workers` +- Start Date: 2026-09-03 +- Tracking Issue: [oceanbase/powercontext#1430](https://github.com/oceanbase/powercontext/issues/1430) +- Related RFCs: [RFC 0011](0011_remote_access_architecture.md)、[RFC 0019](0019_local_source_memory_runtime.md)、 + [RFC 0020](0020_runtime_backed_memory_remote_access.md) 和 [RFC 0046](0046_observability_foundations.md) + +# Summary + +PowerContext 在本地和分布式后台执行中统一使用数据库持久化的 Work Ledger。API 副本无需进程粘性即可入队和查询； +Scheduler 通过 fence 选出一个 leader，并持久保存 keyset 扫描位置；Worker 只领取当前空闲容量的任务，以数据库时间续租， +并安全恢复遗留 attempt。模型调用保持 at-least-once，但同一逻辑 Source window 最多提交一个数据库结果。 + +分布式模式只使用 OceanBase 作为协调后端。同一套 ledger 也替代 `single_node/all` 中的 APScheduler SQLite sidecar， +从而只保留一套需要理解和测试的执行协议。Redis、Kafka、公开 Queue SPI、任意用户代码、DAG、优先级、多区域共识以及 +Connector 自有 checkpoint 不属于本 RFC。 + +# 决策 + +本设计有意减少真相来源： + +- 分布式模式下，OceanBase 同时持有任务状态、租约、cursor 和领域提交。 +- 执行为 at-least-once。崩溃后 provider 调用可能重复，但 fence 和领域 cursor 会阻止旧 attempt 提交。 +- Memory activation 和 Experience incubation 首先接入 ledger；后续 handler 的领域 checkpoint 仍由各自模块持有。 +- 默认保持 `single_node/all`；分布式进程必须且只能选择 `api`、`scheduler` 或 `worker`。 +- Work payload 只保存带版本的引用和数字窗口边界，不保存 Source 正文、prompt、模型响应、credential 或原始异常。 + +这避免了数据库、broker 和本地 sidecar 多套协议自然积累的熵。新增复杂度被固化为数据库约束、小型状态机、fencing token、 +启动校验和兼容性声明，而不再依赖“系统只有一个进程”之类的隐性经验。 + +# 架构 + +```text +API / Scheduler + | 短事务：确定窗口、去重、入队 + v +OceanBase Work Ledger + | 短事务：领取 lane head 并签发 lease fence + v +Worker ---- 在事务外执行 provider/model 调用 ----> + | 短事务：验证 fence，提交领域状态 + cursor + work + v +一个逻辑数据库结果 +``` + +Ledger 包含： + +| 表 | 职责 | +| --- | --- | +| `pc_work_items` | 带版本任务、scope、lane sequence、安全 payload、状态、租约、attempt 预算及安全结果/错误 | +| `pc_work_attempts` | append-only owner/fence/时间/结果审计和非敏感 trace identity | +| `pc_work_lanes` | 每个一致性域的严格 sequence 和唯一 active head | +| `pc_work_keys` | 当前逻辑窗口的唯一占位 | +| `pc_scheduler_leases` | Scheduler leader 与 migrator 的 owner、单调 fence 和数据库时间 expiry | +| `pc_scheduler_scans` | 各 discoverer 的下次执行时间和 keyset continuation | +| `pc_runtime_members` | 存活角色以及 schema/payload/behavior 兼容性声明 | +| `pc_rate_limit_windows` | 可选共享固定窗口计数，以 principal 摘要和 policy 为键 | + +Memory 窗口的 logical key 是 handler kind、scope、cursor name、cursor generation 和 previous cursor 的 SHA-256。 +入队时的 high watermark 与 `through` 保存在 payload 中，但不进入 key。因此 manual 与 scheduled discovery 面对同一个 +未推进 cursor 时会加入同一任务；之后到达的 Source 由下一窗口处理。 + +Memory 和 Experience 使用独立 lane（散列前分别为 `memory:{scope}` 与 `experience:{scope}`）。不同 lane 可以并行， +同一 lane 严格按 sequence 推进。终态失败会保留 logical-key 和 lane 占位，直到 operator retry 或 cancel，避免 poison +window 被无限重建。 + +# 状态机 + +```text +queued ----> running ----> succeeded + | |----> retry_wait ----> running + | |----> failed + | `----> cancelling ----> cancelled + `-----------------------------------> cancelled + +failed -- 显式 operator retry --> queued +``` + +取消在 work row 上线性化。如果取消先于最终事务，Worker 丢弃已经准备好的 provider 结果，并把 attempt 完成为 +cancelled；如果成功已经提交，取消返回 `409`。取消某个 operation 不会暂停该 scope 后续的 discovery。 + +# 锁与事务协议 + +基线是 OceanBase 4.3.5 默认 Read Committed。正确性不依赖无锁读取、单条谓词 update、`SKIP LOCKED` 或 +`GET_LOCK()`。所有路径采用统一锁顺序： + +```text +scheduler lease（仅 Scheduler 路径） + -> lane + -> logical key + -> work item + -> domain cursor/head +``` + +稳定存在的 lease/lane row 会被显式锁定。缺失 row 通过唯一键插入，冲突后重试。claim 先做带索引且有上限的候选扫描， +再逐 lane 用短事务领取。Worker 永远不会预取超过空闲执行槽的任务。 + +Scheduler 的 enqueue 和 scan 更新都会重新验证精确 leader owner、fence 与数据库 expiry。Worker heartbeat、失败、 +取消收敛和最终提交都会验证 `(work_id, owner, fence)`。所有 expiry 判断使用数据库的 `CURRENT_TIMESTAMP(6)`； +应用时钟只用于 poll 和 deadline。 + +Provider、Connector、网络和文件系统调用绝不处于数据库事务中。Worker 最终事务先验证 claim 未过期且未取消，随后调用 +handler 的领域 commit。Memory 与 Experience 复用现有 cursor/head CAS。领域写入、cursor 推进、attempt 完成和 +Work success 要么一起提交，要么一起回滚。 + +# Scheduler 与 Worker + +每个 discoverer 持久保存 keyset continuation，每页最多扫描 100 个 scope。崩溃后重复扫描是安全的，因为 logical key +会去重。旧 leader 在另一个 Scheduler 获得更高 fence 后，不能继续 enqueue 或保存 continuation。 + +Worker 只 claim 当前空闲槽。attempt lease 默认 120 秒，每 30 秒续租。过期 attempt 会先写完审计并进入 retry，再由 +更高 fence 重新领取。可重试错误使用 full-jitter 指数退避，默认从 2 秒开始，上限 5 分钟，最多 5 次自动 generation +attempt。无法识别的 payload version 保持在 lane head 可见，并使 Worker readiness 成为 `misconfigured`；不得猜测、 +静默丢弃或跳过。 + +# 进程角色 + +配置新增 `deployment`、`coordination`、`worker`、`operations` 和 `rate_limit` 组。 + +- `single_node/all` 在同一进程运行 API、Scheduler 和 Worker，但仍走统一 ledger。数据库 owner lease 会拒绝误启动的 + 第二个实例。 +- `distributed/api` 暴露 HTTP、Dashboard、MCP、鉴权、共享限流、入队与 operation 查询，不运行 discovery 或后台 + Worker handler。 +- `distributed/scheduler` 只暴露 health 和 metrics，不持有 provider credential，只进行有界 discovery。 +- `distributed/worker` 只暴露 health 和 metrics，仅持有所注册 handler 必需的 provider credential。 + +分布式模式要求 OceanBase，并拒绝 `role=all`、SQLite、seekDB 和显式 host-local External Skill target。所有角色可以 +复用同一镜像，但应使用独立的最小权限数据库账号。DDL 只属于 migrator 账号，分布式角色绝不自动执行。 + +每个进程用 boot-unique member identity 心跳上报 build version、当前 schema/payload range 与非敏感 +`behavior_revision`。Member metadata 不保存 credential、secret URL、authorization 数据或允许离线猜测 secret 的 hash。 + +# HTTP 与 Client 契约 + +`openapi/powercontext.yaml` 仍是唯一来源。`POST /v1/memory/flush` 的行为是： + +- 没有待处理 Source，或在等待预算内完成：`200 FlushMemoryResponse`； +- 仍为 queued、running 或 retry_wait：`202 OperationAccepted`，带相对 `Location` 和 `Retry-After: 2`； +- 同一 logical key 已被失败任务阻塞：`409 operation_blocked`，返回 operation ID。 + +`Prefer: respond-async` 立即返回句柄；`Prefer: wait=N` 在最长 30 秒内等待，默认 10 秒。Operation endpoint 提供授权后 +的 get/list、乐观并发 cancel 和 operator retry。Mutation body 必须包含 `expected_version`，非法或过期转换返回 `409`。 +公开状态为 `queued`、`running`、`retry_wait`、`cancelling`、`succeeded`、`failed`、`cancelled`。内部 maintenance work +不会通过 Operation API 暴露。 + +本次变更的实现状态：Operation API 挂在所属 Scope 下（`GET|POST /v1/scopes/{scope_id}/operations[...]`），授权直接 +从请求解析，属于其它 Scope 的 operation 与 maintenance work 一样返回 `404`。乐观并发 cancel、operator retry、七个 +公开状态与 maintenance 白名单已实现。仍处于提案、尚未实现的部分：`202 OperationAccepted` 句柄、`Prefer: +respond-async` / `Prefer: wait=N`、失败窗口导致的 `409 operation_blocked`，以及客户端 +`submit_memory_flush()` 与轮询辅助方法。在这些落地之前，`POST /v1/memory/flush` 仍同步返回 `200`。 + +`PowerContextClient.flush_memory()` 通过 submit + poll 保持原有同步式返回；总 deadline 到期时抛出携带 operation ID 的 +`OperationPendingError`。需要显式控制的调用方使用 `submit_memory_flush()`、`get_operation()`、`list_operations()`、 +`cancel_operation()` 和 `retry_operation()`。 + +Operation endpoint 不投影为 MCP tool。分布式模式强制 FastMCP stateless HTTP，连续请求可以命中不同 API 副本。 +Server-to-client elicitation 和 sampling 关闭；Workstream picker 返回结构化 `needs_selection`。内部 ASGI bridge 会传递 +已经认证的 principal，工具可见性绝不作为授权边界。 + +# 健康、关闭、保留与可观测性 + +Liveness 只表示进程能响应。API readiness 要求 database、schema、authentication policy、membership 和 behavior +兼容；缺失 Scheduler 或 Worker 只让 API degraded，仍允许持久入队和读取。健康的 Scheduler standby 也是 ready。 +缺少必需 provider 或 handler version 的 Worker 停止 claim，并报告精确失败项。 + +SIGTERM 时，API 先摘除 readiness；Scheduler 停止 discovery 并条件释放 lease；Worker 停止 claim，继续为在途任务 +heartbeat，并最多 drain 90 秒。非优雅退出由 lease expiry 恢复。 + +成功与取消的 work/attempt 默认保留 30 天。持久 maintenance handler 每批最多删除 500 条，并清理过期限流窗口。 +阻塞失败任务在 operator 处理前不得删除。未来 Source retention 必须把所有非终态 work window 当作 retention root。 + +Metric 只使用 kind、status、outcome、role 和 error category 等有界标签，覆盖 queue depth/age、claim/attempt latency、 +lease expiry、retry、throughput、leader change 和 member count。Scope、principal 和 work ID 不能作为 metric label。 +Enqueue、claim、execute、commit 和 retry 使用独立 span；retry attempt 通过 span link 关联。 + +日志、span、work row 与 attempt row 均不得包含 Source 正文、prompt、模型输出、credential、authorization header 或完整 +secret URL。持久错误只保存有界 category 与 code。 + +# Schema 与发布 + +Alembic 管理 forward-only schema chain。`powercontext server migrate` 获取数据库 lease 后升级到 packaged head。新库从 +baseline 创建；已知且完整的 legacy schema 先校验，只应用明确识别的扩展，再 stamp 并创建 ledger。未知或部分安装的 +schema 直接拒绝。单机启动自动运行同一迁移链；分布式角色只读校验当前 revision。 + +Schema 遵循 expand、mixed-version deploy、contract。N+1 必须能读取 N/N+1 layout，破坏性删除最早在 N+2。 +实际混部期间 Worker 支持当前和前一版 payload；`emit_payload_version` 在旧 Worker 排空前继续发送旧格式。 + +部署顺序是 migrate、Worker、Scheduler、API；回滚顺序相反，并在恢复旧 Worker 前排空新版 payload。Bridge release +先让 `single_node/all` 全部走 Work Ledger；只有 APScheduler 执行路径完全退出后，才允许同一数据库扩为多角色。 +旧 sidecar 只作为 operator 备份产物保留，系统不自动删除。 + +# 验收 + +验收覆盖：两个 API 副本间 round-robin 的 HTTP、Dashboard API 与 stateless MCP；不同 lane 并行与同 lane 串行； +manual/scheduled 去重；Worker 在 claim 前、provider 执行中、最终事务中和 commit 后的崩溃恢复；Scheduler takeover +fencing；retry、cancel、operator recovery、retention 与隐私；以及 OceanBase 4.3.5 真实多进程测试。Golden +schema/payload fixture 覆盖混合版本与升级回滚。SQLite 和 seekDB 只保留单机回归支持。 + +# 未采用方案 + +- Redis 或 Kafka 会在吞吐尚不需要时引入第二个持久真相和跨系统提交问题。 +- 公开 Queue/Coordinator SPI 会在不存在第二种实现时过早冻结多套协调语义。 +- `GET_LOCK()` 是 session 级锁，不适合作为连接池环境中的 fencing 原语。 +- 单机继续使用 APScheduler 会保留两套细微不同的执行和恢复协议。 +- 外部 provider 不可能实现真正 exactly-once；可执行的契约是 at-least-once execution 加 at-most-one fenced + database commit。 diff --git a/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs b/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs index c146c7633f..a648f80817 100644 --- a/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs +++ b/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs @@ -180,6 +180,55 @@ const OPERATIONS = { successStatuses: [200, 304], emptyStatuses: [304] }, + list_operations: { + method: "GET", + path: "/v1/scopes/{scope_id}/operations", + location: "query", + scopeMode: "none", + pathParameters: ["scope_id"], + queryParams: [ + "kind", + "status", + "limit", + "cursor" + ], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_operation: { + method: "GET", + path: "/v1/scopes/{scope_id}/operations/{operation_id}", + location: null, + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + cancel_operation: { + method: "POST", + path: "/v1/scopes/{scope_id}/operations/{operation_id}/cancel", + location: "body", + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + retry_operation: { + method: "POST", + path: "/v1/scopes/{scope_id}/operations/{operation_id}/retry", + location: "body", + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, create_subject_source: { method: "POST", path: "/v1/scopes/{scope_id}/subject-sources", diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index 146d95f363..4f28628644 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -249,6 +249,55 @@ const OPERATIONS = { successStatuses: [200, 304], emptyStatuses: [304] }, + list_operations: { + method: "GET", + path: "/v1/scopes/{scope_id}/operations", + location: "query", + scopeMode: "none", + pathParameters: ["scope_id"], + queryParams: [ + "kind", + "status", + "limit", + "cursor" + ], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_operation: { + method: "GET", + path: "/v1/scopes/{scope_id}/operations/{operation_id}", + location: null, + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + cancel_operation: { + method: "POST", + path: "/v1/scopes/{scope_id}/operations/{operation_id}/cancel", + location: "body", + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + retry_operation: { + method: "POST", + path: "/v1/scopes/{scope_id}/operations/{operation_id}/retry", + location: "body", + scopeMode: "none", + pathParameters: ["scope_id", "operation_id"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, create_subject_source: { method: "POST", path: "/v1/scopes/{scope_id}/subject-sources", diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 752324ea32..fe12f70e68 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -25,6 +25,10 @@ export const OPERATIONS = { preview_atomic_memory_restoration: { method: 'POST', path: '/v1/atomic-memory/restoration-previews', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, restore_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/restorations', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_atomic_memory_state: { method: 'GET', path: '/v1/scopes/{scope_id}/artifacts/atomic-memory/{artifact_id}/state', location: null, scopeMode: 'current', pathParameters: ['scope_id', 'artifact_id'], queryParams: [], headerParams: ['If-None-Match'], successStatuses: [200,304], emptyStatuses: [304] }, + list_operations: { method: 'GET', path: '/v1/scopes/{scope_id}/operations', location: "query", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: ['kind','status','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_operation: { method: 'GET', path: '/v1/scopes/{scope_id}/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + cancel_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_subject_source: { method: 'POST', path: '/v1/scopes/{scope_id}/subject-sources', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, get_profile_policy: { method: 'GET', path: '/v1/scopes/{scope_id}/profile-policy', location: null, scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, put_profile_policy: { method: 'PUT', path: '/v1/scopes/{scope_id}/profile-policy', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 752324ea32..fe12f70e68 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -25,6 +25,10 @@ export const OPERATIONS = { preview_atomic_memory_restoration: { method: 'POST', path: '/v1/atomic-memory/restoration-previews', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, restore_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/restorations', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_atomic_memory_state: { method: 'GET', path: '/v1/scopes/{scope_id}/artifacts/atomic-memory/{artifact_id}/state', location: null, scopeMode: 'current', pathParameters: ['scope_id', 'artifact_id'], queryParams: [], headerParams: ['If-None-Match'], successStatuses: [200,304], emptyStatuses: [304] }, + list_operations: { method: 'GET', path: '/v1/scopes/{scope_id}/operations', location: "query", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: ['kind','status','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_operation: { method: 'GET', path: '/v1/scopes/{scope_id}/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + cancel_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_subject_source: { method: 'POST', path: '/v1/scopes/{scope_id}/subject-sources', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, get_profile_policy: { method: 'GET', path: '/v1/scopes/{scope_id}/profile-policy', location: null, scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, put_profile_policy: { method: 'PUT', path: '/v1/scopes/{scope_id}/profile-policy', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 752324ea32..fe12f70e68 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -25,6 +25,10 @@ export const OPERATIONS = { preview_atomic_memory_restoration: { method: 'POST', path: '/v1/atomic-memory/restoration-previews', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, restore_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/restorations', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_atomic_memory_state: { method: 'GET', path: '/v1/scopes/{scope_id}/artifacts/atomic-memory/{artifact_id}/state', location: null, scopeMode: 'current', pathParameters: ['scope_id', 'artifact_id'], queryParams: [], headerParams: ['If-None-Match'], successStatuses: [200,304], emptyStatuses: [304] }, + list_operations: { method: 'GET', path: '/v1/scopes/{scope_id}/operations', location: "query", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: ['kind','status','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_operation: { method: 'GET', path: '/v1/scopes/{scope_id}/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + cancel_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_operation: { method: 'POST', path: '/v1/scopes/{scope_id}/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'operation_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_subject_source: { method: 'POST', path: '/v1/scopes/{scope_id}/subject-sources', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, get_profile_policy: { method: 'GET', path: '/v1/scopes/{scope_id}/profile-policy', location: null, scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, put_profile_policy: { method: 'PUT', path: '/v1/scopes/{scope_id}/profile-policy', location: "body", scopeMode: 'none', pathParameters: ['scope_id'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index 8e3dbacd49..5d8027e3cb 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -326,6 +326,183 @@ paths: '422': {$ref: '#/components/responses/InvalidRequest'} '500': {$ref: '#/components/responses/InternalError'} '503': {$ref: '#/components/responses/Unavailable'} + /v1/scopes/{scope_id}/operations: + get: + tags: [operations] + summary: List durable operations in one Scope + description: >- + List a snapshot-bounded page of public durable work in one Scope, newest first. Only the operation + families a Scope owner can act on are listed; internal maintenance work stays invisible and reads as + not found. Use the returned next_cursor to continue. + operationId: list_operations + x-powercontext-access: + action: "scope.read" + resource: + type: "scope" + scope-id-from: "scope_id" + parameters: + - name: "scope_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + - name: "kind" + in: "query" + required: false + schema: {type: string, minLength: 1, maxLength: 128} + - name: "status" + in: "query" + required: false + schema: + type: string + enum: [queued, running, retry_wait, cancelling, succeeded, failed, cancelled] + - name: "limit" + in: "query" + required: false + schema: {type: integer, minimum: 1, maximum: 100, default: 50} + - name: "cursor" + in: "query" + required: false + schema: {type: string, minLength: 1, maxLength: 4096} + responses: + '200': + description: One stable page of public durable operations. + headers: + X-PowerContext-Request-ID: {$ref: '#/components/headers/RequestId'} + content: + application/json: + schema: {$ref: '#/components/schemas/DurableOperationPage'} + '400': {$ref: '#/components/responses/BadRequest'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '410': {$ref: '#/components/responses/CursorExpired'} + '422': {$ref: '#/components/responses/InvalidRequest'} + '500': {$ref: '#/components/responses/InternalError'} + '503': {$ref: '#/components/responses/Unavailable'} + /v1/scopes/{scope_id}/operations/{operation_id}: + get: + tags: [operations] + summary: Read one durable operation + description: >- + Read the current state of one public durable operation owned by this Scope. An operation belonging to + another Scope, or internal maintenance work, reads as not found rather than revealing its existence. + operationId: get_operation + x-powercontext-access: + action: "scope.read" + resource: + type: "scope" + scope-id-from: "scope_id" + parameters: + - name: "scope_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + - name: "operation_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + responses: + '200': + description: The current operation state. + headers: + X-PowerContext-Request-ID: {$ref: '#/components/headers/RequestId'} + content: + application/json: + schema: {$ref: '#/components/schemas/DurableOperation'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '422': {$ref: '#/components/responses/InvalidRequest'} + '500': {$ref: '#/components/responses/InternalError'} + '503': {$ref: '#/components/responses/Unavailable'} + /v1/scopes/{scope_id}/operations/{operation_id}/cancel: + post: + tags: [operations] + summary: Cancel one durable operation + description: >- + Request cancellation of a queued or running operation. expected_version must be the state_version of the + read that decided to cancel, so a concurrent transition fails with 409 instead of silently winning. A + running operation moves to cancelling and stops at its next fence; repeating the request with the + current version is idempotent. + operationId: cancel_operation + x-powercontext-access: + action: "scope.contribute" + resource: + type: "scope" + scope-id-from: "scope_id" + parameters: + - name: "scope_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + - name: "operation_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/MutateOperationRequest'} + responses: + '200': + description: The operation after the cancellation request. + headers: + X-PowerContext-Request-ID: {$ref: '#/components/headers/RequestId'} + content: + application/json: + schema: {$ref: '#/components/schemas/DurableOperation'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '409': {$ref: '#/components/responses/Conflict'} + '422': {$ref: '#/components/responses/InvalidRequest'} + '500': {$ref: '#/components/responses/InternalError'} + '503': {$ref: '#/components/responses/Unavailable'} + /v1/scopes/{scope_id}/operations/{operation_id}/retry: + post: + tags: [operations] + summary: Retry one failed durable operation + description: >- + Grant one fresh bounded attempt to a failed operation, without advancing the Cursor or discarding the + Sources it was selected from. expected_version must be the state_version of the read that decided to + retry. Only a failed operation can be retried; any other state returns 409 and the original state is + preserved. + operationId: retry_operation + x-powercontext-access: + action: "scope.contribute" + resource: + type: "scope" + scope-id-from: "scope_id" + parameters: + - name: "scope_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + - name: "operation_id" + in: "path" + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/MutateOperationRequest'} + responses: + '200': + description: The operation after the retry was accepted. + headers: + X-PowerContext-Request-ID: {$ref: '#/components/headers/RequestId'} + content: + application/json: + schema: {$ref: '#/components/schemas/DurableOperation'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '409': {$ref: '#/components/responses/Conflict'} + '422': {$ref: '#/components/responses/InvalidRequest'} + '500': {$ref: '#/components/responses/InternalError'} + '503': {$ref: '#/components/responses/Unavailable'} /v1/scopes/{scope_id}/subject-sources: post: tags: ["profile"] @@ -11250,6 +11427,111 @@ components: properties: scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} block_id: {type: string, minLength: 1, maxLength: 36} + OperationStatus: + type: string + description: Lifecycle state of one durable operation. + enum: [queued, running, retry_wait, cancelling, succeeded, failed, cancelled] + DurableOperation: + type: object + additionalProperties: false + required: + - operation_id + - kind + - scope_id + - status + - state_version + - attempt_count + - max_attempts + - cancel_requested + - available_at + - created_at + - updated_at + - completed_at + - result_code + - error_category + - error_code + properties: + operation_id: {type: string, minLength: 1, maxLength: 256} + kind: + type: string + minLength: 1 + maxLength: 128 + description: Durable work family, for example powercontext.memory.source-window. + scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + status: {$ref: '#/components/schemas/OperationStatus'} + state_version: + type: integer + minimum: 1 + description: Monotonic version to send as expected_version when cancelling or retrying. + attempt_count: {type: integer, minimum: 0} + max_attempts: {type: integer, minimum: 1} + cancel_requested: + type: boolean + description: True once cancellation was requested and the operation has not stopped yet. + available_at: {type: string, format: date-time} + created_at: {type: string, format: date-time} + updated_at: {type: string, format: date-time} + completed_at: + type: string + format: date-time + nullable: true + description: Set once the operation reached a terminal state. + result_code: + type: string + nullable: true + description: Sanitized result classification for a succeeded operation. + error_category: + type: string + nullable: true + error_code: + type: string + nullable: true + ListOperationsRequest: + type: object + additionalProperties: false + properties: + kind: + type: string + minLength: 1 + maxLength: 128 + nullable: true + status: + type: string + minLength: 1 + maxLength: 32 + nullable: true + description: One lifecycle state; unknown values are rejected with 422. + limit: + type: integer + minimum: 1 + maximum: 100 + default: 50 + cursor: + type: string + minLength: 1 + maxLength: 4096 + nullable: true + DurableOperationPage: + type: object + additionalProperties: false + required: [items, next_cursor] + properties: + items: + type: array + items: {$ref: '#/components/schemas/DurableOperation'} + next_cursor: + type: string + nullable: true + description: Opaque cursor for the next page; null when this page is the last. + MutateOperationRequest: + type: object + additionalProperties: false + required: [expected_version] + properties: + expected_version: + type: integer + minimum: 1 + description: The state_version read before deciding to cancel or retry. TopicMemoryProcessingBlock: type: object additionalProperties: false diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 0e5a90f9cd..7cec6f4e5e 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.5.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "1c3d8a5a79fe56abd1c1c2da9457b1f9dcb73f3cd38c0e82b4264afd99cba701", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes." + "persistence_sha256": "cd75d58c73c3175cc1fcd04faf17f430054ad57de25d61854c85bebb89c5e823", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes. Revision v1_5_0_004 follows v1_5_0_003 and registers the durable Work Ledger tables (pc_work_lanes, pc_work_items, pc_work_keys, pc_work_attempts, pc_scheduler_leases, pc_scheduler_scans, pc_runtime_members and the shared pc_rate_limit_windows) on every registered backend; it only creates new tables and leaves earlier revision scripts, schema resources and checksums unchanged. Ledger execution stays at-least-once behind a monotonic fence token while its commit is at-most-once, so a retried, superseded or cancelled attempt cannot publish a duplicate result or release another worker's logical key. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. The maintenance backup scope (affected_objects) lists the same eight tables, so a backup taken before this revision covers everything it creates." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/artifacts/profile/service.py b/src/powercontext/builtin/artifacts/profile/service.py index 2aac6a888c..bdfc56d610 100644 --- a/src/powercontext/builtin/artifacts/profile/service.py +++ b/src/powercontext/builtin/artifacts/profile/service.py @@ -21,6 +21,7 @@ import logging from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager, nullcontext +from dataclasses import dataclass from datetime import UTC, datetime from typing import TYPE_CHECKING, Any, Protocol from uuid import uuid4 @@ -46,7 +47,7 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.candidates import CandidateRepository -from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.cursors import SourceCursorRepository, StoredSourceCursor from powercontext.builtin.persistence.database import AsyncDatabase, is_transaction_contention from powercontext.builtin.persistence.errors import GenerationConflictError, RepositoryNotFoundError from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository @@ -86,6 +87,30 @@ def _profile_prompt_refs(): return () if selection is None or selection.artifact is None else (selection.artifact,) +@dataclass(frozen=True) +class PreparedProfileFlush: + """One Profile window prepared outside its fenced commit transaction. + + ``result`` carries a terminal outcome that requires no database write. + Otherwise the snapshot must be applied through ``RelationalProfileService. + commit_flush`` on a caller-provided connection, so the Profile, its cursor + and the caller's own coordination state commit or roll back together. + """ + + scope_id: str + after: int + high: int + evidence_count: int + result: ProfileFlushResult | None + policy: ProfilePolicy | None = None + cursor: StoredSourceCursor | None = None + current: Profile | None = None + evidence: tuple[Any, ...] = () + prompt_refs: tuple[Any, ...] = () + markdown: str | None = None + through: int = 0 + + class RelationalProfileService: def __init__( self, @@ -181,6 +206,8 @@ async def flush( processing: ScopeInvocation | None = None, authorize_commit: Callable[[AsyncConnection, Profile | None], Awaitable[None]] | None = None, ) -> ProfileFlushResult: + """Prepare one window, generate outside transactions, then commit fenced.""" + async with self.operation_context(): if self._prompt_service is None: return await self._flush( @@ -201,16 +228,45 @@ async def flush( authorize_commit=authorize_commit, ) - async def _flush( # noqa: C901 + async def prepare_flush( self, scope_id: str, *, high_watermark: int | None = None, authorize_snapshot: Callable[[Profile | None], Awaitable[None]] | None = None, - on_commit=None, processing: ScopeInvocation | None = None, - authorize_commit: Callable[[AsyncConnection, Profile | None], Awaitable[None]] | None = None, - ) -> ProfileFlushResult: + ) -> PreparedProfileFlush: + """Snapshot one window and generate without committing anything. + + The returned snapshot carries either a terminal ``result`` or the + prepared window for ``commit_flush``. Generation runs outside any + database transaction; nothing is written until ``commit_flush``. + """ + + async with self.operation_context(): + if self._prompt_service is None: + return await self._prepare_flush( + scope_id, + high_watermark=high_watermark, + authorize_snapshot=authorize_snapshot, + processing=processing, + ) + async with self._prompt_service.bind(scope_id, "profile.generate"): + return await self._prepare_flush( + scope_id, + high_watermark=high_watermark, + authorize_snapshot=authorize_snapshot, + processing=processing, + ) + + async def _prepare_flush( + self, + scope_id: str, + *, + high_watermark: int | None, + authorize_snapshot: Callable[[Profile | None], Awaitable[None]] | None, + processing: ScopeInvocation | None, + ) -> PreparedProfileFlush: async with self.database.transaction() as connection: if processing is not None: await processing.start(connection) @@ -223,11 +279,25 @@ async def _flush( # noqa: C901 if policy is None or not policy.generation_enabled: if processing is not None: await processing.complete(connection, remaining_work=after < high) - return ProfileFlushResult(status="disabled", **base) + return PreparedProfileFlush( + scope_id=scope_id, + after=after, + high=high, + evidence_count=0, + result=ProfileFlushResult(status="disabled", **base), + ) if policy.pending_candidate_id is not None: if processing is not None: await processing.complete(connection, remaining_work=True) - return ProfileFlushResult(status="review_pending", candidate_id=policy.pending_candidate_id, **base) + return PreparedProfileFlush( + scope_id=scope_id, + after=after, + high=high, + evidence_count=0, + result=ProfileFlushResult( + status="review_pending", candidate_id=policy.pending_candidate_id, **base + ), + ) current = await self.latest(connection, scope_id) prompt_refs = _profile_prompt_refs() reserved_artifacts = (0 if current is None else 1) + len(prompt_refs) @@ -240,121 +310,200 @@ async def _flush( # noqa: C901 if not window and processing is not None: await processing.complete(connection, remaining_work=False) if not window: - return ProfileFlushResult(status="noop", **base) + return PreparedProfileFlush( + scope_id=scope_id, + after=after, + high=high, + evidence_count=0, + result=ProfileFlushResult(status="noop", **base), + ) # Authorize the exact Head used by generation. The commit CAS below also # rejects a Head created or changed after this snapshot was authorized. if authorize_snapshot is not None: await authorize_snapshot(current) through = window[-1].journal_position evidence = tuple(item for item in window if is_generation_eligible(item.value)) - markdown = None - if evidence: - if self.generator is None: - # Import locally to avoid a cycle through the shared Review service. - from powercontext.builtin.review.generation import GenerationCapabilityUnavailableError - - raise GenerationCapabilityUnavailableError("profile") - markdown = await self.generator.generate( - ProfileGenerationInput( - previous_content=None if current is None else current.content.content, - sources=tuple(item.value.model_dump_json() for item in evidence), - ) + markdown = await self._generate_markdown(current, evidence) + return PreparedProfileFlush( + scope_id=scope_id, + after=after, + high=high, + evidence_count=len(evidence), + result=None, + policy=policy, + cursor=cursor, + current=current, + evidence=evidence, + prompt_refs=prompt_refs, + markdown=markdown, + through=through, + ) + + async def _generate_markdown(self, current: Profile | None, evidence: tuple[Any, ...]) -> str | None: + """Run one generation outside any transaction; ``None`` keeps the window.""" + + if not evidence: + return None + if self.generator is None: + # Import locally to avoid a cycle through the shared Review service. + from powercontext.builtin.review.generation import GenerationCapabilityUnavailableError + + raise GenerationCapabilityUnavailableError("profile") + markdown = await self.generator.generate( + ProfileGenerationInput( + previous_content=None if current is None else current.content.content, + sources=tuple(item.value.model_dump_json() for item in evidence), ) - if markdown is not None: - try: - markdown = normalize_profile_markdown(markdown) - except ValueError as error: - from powercontext.builtin.inference import InvalidInferenceOutputError - - raise InvalidInferenceOutputError("profile-generate", "invalid Markdown") from error - if current is not None and current.content.content == markdown: - markdown = None + ) + if markdown is None: + return None try: - async with self.database.transaction() as connection: - if processing is not None: - await processing.guard(connection) - locked = await self.policies.get(connection, scope_id, for_update=True) - actual_cursor = await self.cursors.load( - connection, scope_id, PROFILE_SOURCE_WINDOW_BINDING, for_update=True - ) - actual = await self.latest(connection, scope_id) - if ( - locked != policy - or actual_cursor != cursor - or (None if actual is None else actual.as_ref()) != (None if current is None else current.as_ref()) - ): - raise BaseValueConflictError("profile_processing", (scope_id,)) # noqa: TRY301 - if authorize_commit is not None: - await authorize_commit(connection, current) - # CAS also serializes SQLite, whose SELECT FOR UPDATE is a no-op. - updated = await self.policies.update(connection, policy) - refs = tuple(item.ref for item in evidence) - parents = () if current is None else (current.as_ref(),) - artifacts = (*parents, *prompt_refs) - source_window = SourceWindow(after=after, through=through) - candidate_id = None - candidate = None - artifact = None - if markdown is not None and policy.activation_mode == "review_required": - candidate_id = self.id_factory("candidate") - candidate = await self.candidates.create( - connection, - scope_id, - candidate_id, - "profile", - ProfileCandidateProposal( - content=markdown, - source_window=source_window, + markdown = normalize_profile_markdown(markdown) + except ValueError as error: + from powercontext.builtin.inference import InvalidInferenceOutputError + + raise InvalidInferenceOutputError("profile-generate", "invalid Markdown") from error + if current is not None and current.content.content == markdown: + return None + return markdown + + async def commit_flush( + self, + connection: AsyncConnection, + prepared: PreparedProfileFlush, + /, + *, + authorize_commit: Callable[[AsyncConnection, Profile | None], Awaitable[None]] | None = None, + on_commit=None, + processing: ScopeInvocation | None = None, + ) -> ProfileFlushResult: + """Apply a prepared window on the caller's transaction. + + No transaction is opened here and no conflict is absorbed: the caller + owns the transaction boundary and decides how contention is retried, + which is what lets a Work item share this commit with its ledger write. + """ + + if prepared.result is not None: + raise ValueError("a terminal Profile flush has no commit") # noqa: TRY003 + scope_id = prepared.scope_id + policy = prepared.policy + cursor = prepared.cursor + current = prepared.current + if processing is not None: + await processing.guard(connection) + locked = await self.policies.get(connection, scope_id, for_update=True) + actual_cursor = await self.cursors.load(connection, scope_id, PROFILE_SOURCE_WINDOW_BINDING, for_update=True) + actual = await self.latest(connection, scope_id) + if ( + locked is None + or policy is None + or locked != policy + or actual_cursor != cursor + or (None if actual is None else actual.as_ref()) != (None if current is None else current.as_ref()) + ): + raise BaseValueConflictError("profile_processing", (scope_id,)) + if authorize_commit is not None: + await authorize_commit(connection, current) + # CAS also serializes SQLite, whose SELECT FOR UPDATE is a no-op. + updated = await self.policies.update(connection, policy) + refs = tuple(item.ref for item in prepared.evidence) + parents = () if current is None else (current.as_ref(),) + artifacts = (*parents, *prepared.prompt_refs) + source_window = SourceWindow(after=prepared.after, through=prepared.through) + candidate_id = None + candidate = None + artifact = None + if prepared.markdown is not None and policy.activation_mode == "review_required": + candidate_id = self.id_factory("candidate") + candidate = await self.candidates.create( + connection, + scope_id, + candidate_id, + "profile", + ProfileCandidateProposal( + content=prepared.markdown, + source_window=source_window, + generator_id=PROFILE_SOURCE_WINDOW_BINDING, + generator_version="1", + created_at=datetime.now(UTC), + ), + sources=refs, + artifacts=artifacts, + target=None if current is None else current.as_ref(), + reason="New source evidence", + ) + await self.policies.update(connection, updated, pending_candidate_id=candidate_id) + else: + if prepared.markdown is not None: + draft = ProfileDraft( + content=ProfileContent( + content=prepared.markdown, + generation=ProfileGeneration( + mode="automatic", + created_at=datetime.now(UTC), generator_id=PROFILE_SOURCE_WINDOW_BINDING, generator_version="1", - created_at=datetime.now(UTC), + source_window=source_window, ), - sources=refs, - artifacts=artifacts, - target=None if current is None else current.as_ref(), - reason="New source evidence", - ) - await self.policies.update(connection, updated, pending_candidate_id=candidate_id) - else: - if markdown is not None: - draft = ProfileDraft( - content=ProfileContent( - content=markdown, - generation=ProfileGeneration( - mode="automatic", - created_at=datetime.now(UTC), - generator_id=PROFILE_SOURCE_WINDOW_BINDING, - generator_version="1", - source_window=source_window, - ), - ), - sources=refs, - artifacts=artifacts, - ) - artifact = ( - await self.artifacts.create(connection, scope_id, PROFILE_ARTIFACT_ID, draft) - if current is None - else await self.artifacts.revise(connection, scope_id, current, draft) - ) - await self.cursors.save( - connection, - scope_id, - PROFILE_SOURCE_WINDOW_BINDING, - SourceCursor(sequence=through), - expected_generation=None if cursor is None else cursor.generation, - ) - if on_commit is not None: - await on_commit(connection, artifact, candidate) - if processing is not None: - await processing.complete(connection, remaining_work=candidate_id is not None or through < high) - return ProfileFlushResult( - status="review_pending" if candidate_id else ("updated" if artifact else "noop"), - previous_cursor=after, - current_cursor=after if candidate_id else through, - high_watermark=high, - processed_source_count=len(evidence), - candidate_id=candidate_id, - artifact=None if artifact is None else artifact.as_ref(), + ), + sources=refs, + artifacts=artifacts, + ) + artifact = ( + await self.artifacts.create(connection, scope_id, PROFILE_ARTIFACT_ID, draft) + if current is None + else await self.artifacts.revise(connection, scope_id, current, draft) + ) + await self.cursors.save( + connection, + scope_id, + PROFILE_SOURCE_WINDOW_BINDING, + SourceCursor(sequence=prepared.through), + expected_generation=None if cursor is None else cursor.generation, + ) + if on_commit is not None: + await on_commit(connection, artifact, candidate) + if processing is not None: + await processing.complete( + connection, remaining_work=candidate_id is not None or prepared.through < prepared.high + ) + return ProfileFlushResult( + status="review_pending" if candidate_id else ("updated" if artifact else "noop"), + previous_cursor=prepared.after, + current_cursor=prepared.after if candidate_id else prepared.through, + high_watermark=prepared.high, + processed_source_count=prepared.evidence_count, + candidate_id=candidate_id, + artifact=None if artifact is None else artifact.as_ref(), + ) + + async def _flush( + self, + scope_id: str, + *, + high_watermark: int | None, + authorize_snapshot: Callable[[Profile | None], Awaitable[None]] | None, + on_commit, + processing: ScopeInvocation | None, + authorize_commit: Callable[[AsyncConnection, Profile | None], Awaitable[None]] | None, + ) -> ProfileFlushResult: + prepared = await self._prepare_flush( + scope_id, + high_watermark=high_watermark, + authorize_snapshot=authorize_snapshot, + processing=processing, + ) + if prepared.result is not None: + return prepared.result + try: + async with self.database.transaction() as connection: + return await self.commit_flush( + connection, + prepared, + authorize_commit=authorize_commit, + on_commit=on_commit, + processing=processing, ) except ( BaseValueConflictError, @@ -369,10 +518,10 @@ async def _flush( # noqa: C901 refreshed = await self.cursors.load(connection, scope_id, PROFILE_SOURCE_WINDOW_BINDING) return ProfileFlushResult( status="conflict", - previous_cursor=after, + previous_cursor=prepared.after, current_cursor=0 if refreshed is None else refreshed.cursor.sequence, - high_watermark=high, - processed_source_count=len(evidence), + high_watermark=prepared.high, + processed_source_count=prepared.evidence_count, ) async def scan(self, *, max_concurrency: int = 4) -> None: diff --git a/src/powercontext/builtin/persistence/__init__.py b/src/powercontext/builtin/persistence/__init__.py index cfefdfd7b3..4ac2dfd1f5 100644 --- a/src/powercontext/builtin/persistence/__init__.py +++ b/src/powercontext/builtin/persistence/__init__.py @@ -21,6 +21,15 @@ ) from powercontext.builtin.persistence.candidates import CandidateRepository from powercontext.builtin.persistence.connectors import ConnectorCheckpointRepository +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + RuntimeMember, + RuntimeMemberSpec, + SchedulerScan, + StaleCoordinatorLeaseError, + StaleScanStateError, +) from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.decision_observations import DecisionObservationRepository from powercontext.builtin.persistence.errors import ( @@ -38,6 +47,7 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.external_skills import ExternalSkillRepository +from powercontext.builtin.persistence.rate_limit import RateLimitDecision, RateLimitRepository from powercontext.builtin.persistence.recurrence import RecurrenceRepository from powercontext.builtin.persistence.skill_packages import SkillPackageRepository from powercontext.builtin.persistence.skill_publications import ( @@ -66,6 +76,18 @@ NoTopicMemoryIndex, TopicMemoryIndex, ) +from powercontext.builtin.persistence.work import ( + EnqueueResult, + StaleWorkClaimError, + StoredWork, + WorkClaim, + WorkFailure, + WorkRepository, + WorkResult, + WorkSpec, + WorkStateConflictError, + WorkStatus, +) __all__ = ( "GLOBAL_ARTIFACT_PROCESSING_SUPERVISOR_GROUP", @@ -78,8 +100,11 @@ "CandidateRepository", "CompositeTopicMemoryIndex", "ConnectorCheckpointRepository", + "CoordinationRepository", + "CoordinatorLease", "DatabaseClosedError", "DecisionObservationRepository", + "EnqueueResult", "ExternalSkillRepository", "GenerationConflictError", "IdentityMismatchError", @@ -88,17 +113,25 @@ "InvalidStoredPayloadError", "NoTopicMemoryIndex", "PersistenceError", + "RateLimitDecision", + "RateLimitRepository", "RecurrenceRepository", "RemoteAgentSkillTarget", "RemoteAgentSkillTargetRepository", "RemoteAgentSkillTargetState", "RepositoryError", "RepositoryNotFoundError", + "RuntimeMember", + "RuntimeMemberSpec", + "SchedulerScan", "SkillPackageRepository", "SkillPublication", "SkillPublicationDesiredState", "SkillPublicationRepository", "SourceDefinitionManifestRepository", + "StaleCoordinatorLeaseError", + "StaleScanStateError", + "StaleWorkClaimError", "StatisticsRepository", "StoredArtifactProcessingBindingState", "StoredArtifactProcessingLease", @@ -106,6 +139,14 @@ "StoredModelUsage", "StoredPayloadConflictError", "StoredRecallTokenUsage", + "StoredWork", "TopicMemoryIndex", "TopicMemoryRepository", + "WorkClaim", + "WorkFailure", + "WorkRepository", + "WorkResult", + "WorkSpec", + "WorkStateConflictError", + "WorkStatus", ) diff --git a/src/powercontext/builtin/persistence/coordination.py b/src/powercontext/builtin/persistence/coordination.py new file mode 100644 index 0000000000..f163ec5c22 --- /dev/null +++ b/src/powercontext/builtin/persistence/coordination.py @@ -0,0 +1,416 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Database-time coordination primitives for schedulers and role members.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import ( + InvalidRepositoryArgumentError, + InvalidStoredColumnError, + RepositoryError, +) +from powercontext.builtin.persistence.tables import ( + RUNTIME_MEMBERS_TABLE, + SCHEDULER_LEASES_TABLE, + SCHEDULER_SCANS_TABLE, +) +from powercontext.limits import MAX_SCOPE_ID_LENGTH + + +class CoordinatorLease(BaseModel): + """Monotonic fencing token held until database time reaches its expiry.""" + + model_config = ConfigDict(frozen=True) + + lease_name: str + owner_id: str + fence: int + lease_expires_at: datetime + + +class SchedulerScan(BaseModel): + """Durable keyset continuation for one bounded discoverer scan.""" + + model_config = ConfigDict(frozen=True) + + discoverer: str + next_run_at: datetime + continuation: str | None + state_version: int + updated_at: datetime + + +class RuntimeMember(BaseModel): + """Non-sensitive compatibility advertisement for one live process.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + member_id: str + role: Literal["all", "api", "scheduler", "worker"] + build_version: str + schema_min: int + schema_max: int + payload_min: int + payload_max: int + behavior_revision: str + heartbeat_at: datetime + expires_at: datetime + + +class RuntimeMemberSpec(BaseModel): + """Compatibility data supplied on member registration and heartbeat.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + member_id: str + role: Literal["all", "api", "scheduler", "worker"] + build_version: str + schema_min: int + schema_max: int + payload_min: int + payload_max: int + behavior_revision: str + + +class StaleCoordinatorLeaseError(RepositoryError): + """Raised when an old owner or fence attempts a coordinated write.""" + + def __init__(self, lease_name: str) -> None: + self.lease_name = lease_name + super().__init__(f"coordinator lease {lease_name!r} is no longer current") + + +class StaleScanStateError(RepositoryError): + """Raised when a scheduler scan continuation loses its version race.""" + + def __init__(self, discoverer: str) -> None: + self.discoverer = discoverer + super().__init__(f"scheduler scan {discoverer!r} changed concurrently") + + +class CoordinationRepository: + """Coordinate leases, scans, and role discovery in caller transactions.""" + + async def acquire_lease( + self, + connection: AsyncConnection, + *, + lease_name: str, + owner_id: str, + lease_seconds: int, + ) -> CoordinatorLease | None: + _require_text("lease_name", lease_name, 64) + _require_text("owner_id", owner_id, 128) + _require_positive("lease_seconds", lease_seconds) + now = await database_now(connection) + row = await _lock_or_create_lease(connection, lease_name, now) + current_owner = None if row["owner_id"] is None else str(row["owner_id"]) + current_expiry = _optional_datetime(row["lease_expires_at"], "lease_expires_at") + current_fence = _nonnegative_integer(row["fence"], "fence") + if current_owner == owner_id and current_expiry is not None and current_expiry > now: + fence = current_fence + elif current_expiry is None or current_expiry <= now: + fence = current_fence + 1 + else: + return None + expires_at = now + timedelta(seconds=lease_seconds) + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .values(owner_id=owner_id, fence=fence, lease_expires_at=expires_at, updated_at=now) + ) + return CoordinatorLease( + lease_name=lease_name, + owner_id=owner_id, + fence=fence, + lease_expires_at=expires_at, + ) + + async def assert_lease(self, connection: AsyncConnection, lease: CoordinatorLease, /) -> None: + row = await _lock_lease(connection, lease.lease_name) + now = await database_now(connection) + expires_at = None if row is None else _optional_datetime(row["lease_expires_at"], "lease_expires_at") + if ( + row is None + or row["owner_id"] != lease.owner_id + or row["fence"] != lease.fence + or expires_at is None + or expires_at <= now + ): + raise StaleCoordinatorLeaseError(lease.lease_name) + + async def release_lease(self, connection: AsyncConnection, lease: CoordinatorLease, /) -> bool: + now = await database_now(connection) + result = await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where( + SCHEDULER_LEASES_TABLE.c.lease_name == lease.lease_name, + SCHEDULER_LEASES_TABLE.c.owner_id == lease.owner_id, + SCHEDULER_LEASES_TABLE.c.fence == lease.fence, + ) + .values(owner_id=None, lease_expires_at=now, updated_at=now) + ) + return result.rowcount == 1 + + async def load_scan(self, connection: AsyncConnection, discoverer: str, /) -> SchedulerScan | None: + _require_text("discoverer", discoverer, 128) + row = ( + ( + await connection.execute( + select(SCHEDULER_SCANS_TABLE).where(SCHEDULER_SCANS_TABLE.c.discoverer == discoverer) + ) + ) + .mappings() + .one_or_none() + ) + return None if row is None else _decode_scan(row) + + async def save_scan( + self, + connection: AsyncConnection, + discoverer: str, + /, + *, + next_run_at: datetime, + continuation: str | None, + expected_version: int | None, + ) -> SchedulerScan: + _require_text("discoverer", discoverer, 128) + if continuation is not None: + _require_text("continuation", continuation, MAX_SCOPE_ID_LENGTH) + next_run_at = _normalized_datetime(next_run_at) + now = await database_now(connection) + if expected_version is None: + created = await insert_if_absent( + connection, + SCHEDULER_SCANS_TABLE, + { + "discoverer": discoverer, + "next_run_at": next_run_at, + "continuation": continuation, + "state_version": 1, + "updated_at": now, + }, + ) + if not created: + raise StaleScanStateError(discoverer) from None + else: + _require_positive("expected_version", expected_version) + result = await connection.execute( + update(SCHEDULER_SCANS_TABLE) + .where( + SCHEDULER_SCANS_TABLE.c.discoverer == discoverer, + SCHEDULER_SCANS_TABLE.c.state_version == expected_version, + ) + .values( + next_run_at=next_run_at, + continuation=continuation, + state_version=expected_version + 1, + updated_at=now, + ) + ) + if result.rowcount != 1: + raise StaleScanStateError(discoverer) + result = await self.load_scan(connection, discoverer) + if result is None: + raise InvalidStoredColumnError("discoverer", "a persisted scheduler scan") + return result + + async def heartbeat_member( + self, + connection: AsyncConnection, + spec: RuntimeMemberSpec, + /, + *, + ttl_seconds: int, + ) -> RuntimeMember: + _validate_member(spec) + _require_positive("ttl_seconds", ttl_seconds) + now = await database_now(connection) + expires_at = now + timedelta(seconds=ttl_seconds) + values = { + **spec.model_dump(mode="python"), + "heartbeat_at": now, + "expires_at": expires_at, + } + result = await connection.execute( + update(RUNTIME_MEMBERS_TABLE).where(RUNTIME_MEMBERS_TABLE.c.member_id == spec.member_id).values(**values) + ) + if result.rowcount != 1: + created = await insert_if_absent(connection, RUNTIME_MEMBERS_TABLE, values) + if not created: + result = await connection.execute( + update(RUNTIME_MEMBERS_TABLE) + .where(RUNTIME_MEMBERS_TABLE.c.member_id == spec.member_id) + .values(**values) + ) + if result.rowcount != 1: + raise InvalidStoredColumnError("member_id", "a concurrently persisted runtime member") + return RuntimeMember.model_validate(values) + + async def live_members( + self, + connection: AsyncConnection, + /, + *, + role: Literal["all", "api", "scheduler", "worker"] | None = None, + ) -> tuple[RuntimeMember, ...]: + now = await database_now(connection) + statement = select(RUNTIME_MEMBERS_TABLE).where(RUNTIME_MEMBERS_TABLE.c.expires_at > now) + if role is not None: + statement = statement.where(RUNTIME_MEMBERS_TABLE.c.role == role) + rows = (await connection.execute(statement.order_by(RUNTIME_MEMBERS_TABLE.c.member_id))).mappings().all() + return tuple(_decode_member(row) for row in rows) + + +async def _lock_or_create_lease( + connection: AsyncConnection, + lease_name: str, + now: datetime, +) -> Mapping[Any, Any]: + row = await _lock_lease(connection, lease_name) + if row is not None: + return row + await insert_if_absent( + connection, + SCHEDULER_LEASES_TABLE, + { + "lease_name": lease_name, + "owner_id": None, + "fence": 0, + "lease_expires_at": None, + "updated_at": now, + }, + ) + row = await _lock_lease(connection, lease_name) + if row is None: + raise InvalidStoredColumnError("lease_name", "an initialized coordinator lease") + return row + + +async def _lock_lease(connection: AsyncConnection, lease_name: str) -> Mapping[Any, Any] | None: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .values(fence=SCHEDULER_LEASES_TABLE.c.fence) + ) + return ( + ( + await connection.execute( + select(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +def _decode_scan(row: Mapping[Any, Any]) -> SchedulerScan: + return SchedulerScan( + discoverer=str(row["discoverer"]), + next_run_at=_stored_datetime(row["next_run_at"], "next_run_at"), + continuation=None if row["continuation"] is None else str(row["continuation"]), + state_version=_positive_integer(row["state_version"], "state_version"), + updated_at=_stored_datetime(row["updated_at"], "updated_at"), + ) + + +def _decode_member(row: Mapping[Any, Any]) -> RuntimeMember: + return RuntimeMember.model_validate({ + "member_id": str(row["member_id"]), + "role": str(row["role"]), + "build_version": str(row["build_version"]), + "schema_min": _positive_integer(row["schema_min"], "schema_min"), + "schema_max": _positive_integer(row["schema_max"], "schema_max"), + "payload_min": _positive_integer(row["payload_min"], "payload_min"), + "payload_max": _positive_integer(row["payload_max"], "payload_max"), + "behavior_revision": str(row["behavior_revision"]), + "heartbeat_at": _stored_datetime(row["heartbeat_at"], "heartbeat_at"), + "expires_at": _stored_datetime(row["expires_at"], "expires_at"), + }) + + +def _validate_member(spec: RuntimeMemberSpec) -> None: + _require_text("member_id", spec.member_id, 128) + _require_text("build_version", spec.build_version, 64) + _require_text("behavior_revision", spec.behavior_revision, 128) + for name in ("schema_min", "schema_max", "payload_min", "payload_max"): + _require_positive(name, getattr(spec, name)) + if spec.schema_max < spec.schema_min: + raise InvalidRepositoryArgumentError("schema_max", "must not be less than schema_min") + if spec.payload_max < spec.payload_min: + raise InvalidRepositoryArgumentError("payload_max", "must not be less than payload_min") + + +def _stored_datetime(value: object, column: str) -> datetime: + if not isinstance(value, datetime): + raise InvalidStoredColumnError(column, "a datetime") + return _normalized_datetime(value) + + +def _optional_datetime(value: object, column: str) -> datetime | None: + return None if value is None else _stored_datetime(value, column) + + +def _normalized_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +def _nonnegative_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidStoredColumnError(column, "a non-negative integer") + return value + + +def _require_positive(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _require_text(field: str, value: str, maximum: int) -> None: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise InvalidRepositoryArgumentError(field, "must be a non-empty trimmed string") + if len(value) > maximum: + raise InvalidRepositoryArgumentError(field, f"must not exceed {maximum} characters") + + +__all__ = [ + "CoordinationRepository", + "CoordinatorLease", + "RuntimeMember", + "RuntimeMemberSpec", + "SchedulerScan", + "StaleCoordinatorLeaseError", + "StaleScanStateError", +] diff --git a/src/powercontext/builtin/persistence/cursors.py b/src/powercontext/builtin/persistence/cursors.py index 42212503c7..537d134ccd 100644 --- a/src/powercontext/builtin/persistence/cursors.py +++ b/src/powercontext/builtin/persistence/cursors.py @@ -21,12 +21,10 @@ from pydantic import BaseModel from sqlalchemy import select, update -from sqlalchemy.dialects.mysql import insert as mysql_insert -from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes -from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE +from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE, insert_if_absent from powercontext.builtin.persistence.errors import ( GenerationConflictError, InvalidRepositoryArgumentError, @@ -160,12 +158,15 @@ async def save( payload = dump_model(cursor, kind="source-cursor", name=binding_name) if expected_generation is None: generation = 1 - created = await _insert_if_absent( + created = await insert_if_absent( connection, - scope_id=scope_id, - binding_name=binding_name, - cursor=payload, - generation=generation, + SOURCE_CURSORS_TABLE, + { + "scope_id": scope_id, + "binding_name": binding_name, + "cursor": payload, + "generation": generation, + }, ) if not created: # The unique constraint is the initial-generation CAS. Read only @@ -205,34 +206,6 @@ async def save( ) -async def _insert_if_absent( - connection: AsyncConnection, - *, - scope_id: str, - binding_name: str, - cursor: bytes, - generation: int, -) -> bool: - values = { - "scope_id": scope_id, - "binding_name": binding_name, - "cursor": cursor, - "generation": generation, - } - if connection.dialect.name == "sqlite": - statement = sqlite_insert(SOURCE_CURSORS_TABLE).values(**values).on_conflict_do_nothing() - elif connection.dialect.name == "mysql": - # OceanBase MySQL mode accepts SAVEPOINT but does not retain it for - # RELEASE/ROLLBACK. INSERT IGNORE keeps first creation atomic without - # relying on a nested transaction; all values are validated first, so - # the only ignored error is the table's cursor identity conflict. - statement = mysql_insert(SOURCE_CURSORS_TABLE).values(**values).prefix_with("IGNORE") - else: - raise InvalidRepositoryArgumentError("dialect", f"{connection.dialect.name!r} does not support source cursors") - result = await connection.execute(statement) - return result.rowcount == 1 - - def _decode_row(row: Mapping[Any, Any]) -> StoredSourceCursor: return StoredSourceCursor( scope_id=str(row["scope_id"]), diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index d73c1aca91..aaee7d42b1 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -18,17 +18,23 @@ import asyncio import time -from collections.abc import AsyncIterator, Callable, Collection +from collections.abc import AsyncIterator, Callable, Collection, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext +from datetime import UTC, datetime from sqlite3 import SQLITE_INTERRUPT from typing import Any from aiosqlite import Connection as SQLiteConnection from aiosqlite import Cursor as SQLiteCursor -from sqlalchemy.exc import OperationalError, PendingRollbackError +from sqlalchemy import Table, insert +from sqlalchemy.exc import IntegrityError, OperationalError, PendingRollbackError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine -from powercontext.builtin.persistence.errors import DatabaseClosedError, PersistenceError +from powercontext.builtin.persistence.errors import ( + DatabaseClosedError, + InvalidStoredColumnError, + PersistenceError, +) # Repositories that read a whole Scope selection in one statement chunk it to stay # below the lowest bind-parameter ceiling across the supported backends. @@ -515,3 +521,56 @@ def is_transaction_contention(error: OperationalError) -> bool: if isinstance(sqlite_code, int) and sqlite_code & 0xFF in {5, 6}: return True return bool(original is not None and original.args and original.args[0] in {1205, 1213}) + + +async def database_now(connection: AsyncConnection, /) -> datetime: + """Return the database clock as normalized *naive* UTC. + + Coordination decisions (Lease expiry, fence ordering, ``lease_expires_at`` + comparisons) are made against columns the Work Ledger stores as naive UTC, + so this helper deliberately strips the offset. ``dream.database_now`` + returns an aware datetime for its own records and is not interchangeable + here: mixing the two produces "can't compare offset-naive and + offset-aware datetimes" inside the claim path. + """ + + statement = "SELECT UTC_TIMESTAMP(6)" if connection.dialect.name == "mysql" else "SELECT CURRENT_TIMESTAMP" + value = (await connection.exec_driver_sql(statement)).scalar_one() + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if not isinstance(value, datetime): + raise InvalidStoredColumnError("CURRENT_TIMESTAMP", "a datetime") + if value.tzinfo is not None: + return value.astimezone(UTC).replace(tzinfo=None) + return value + + +async def insert_if_absent( + connection: AsyncConnection, + table: Table, + values: Mapping[str, Any], + /, +) -> bool: + """Insert validated values without aborting the transaction on a unique-key race. + + OceanBase's async dialect deliberately leaves ``do_begin`` empty. A + zero-row locking update therefore may not establish a server transaction, + which makes a SAVEPOINT-based insert race unsafe. The supported SQL + dialects provide a conflict-tolerant insert that starts the real + transaction and reports whether this caller created the row. + """ + + statement = insert(table).values(**values) + if connection.dialect.name == "mysql": + result = await connection.execute(statement.prefix_with("IGNORE")) + return result.rowcount == 1 + if connection.dialect.name == "sqlite": + result = await connection.execute(statement.prefix_with("OR IGNORE")) + return result.rowcount == 1 + + try: + async with connection.begin_nested(): + await connection.execute(statement) + except IntegrityError: + return False + return True diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index aadd20d129..81bfce2c47 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -44,6 +44,10 @@ "v1_5_0_003": [ "schemas/sqlite-v1_5_0_003.json", "schemas/mysql-v1_5_0_003.json" + ], + "v1_5_0_004": [ + "schemas/sqlite-v1_5_0_004.json", + "schemas/mysql-v1_5_0_004.json" ] }, "sqlite_baselines": { @@ -79,6 +83,9 @@ ], "v1_5_0_003": [ "357287e3bae474ca1810dd0949fe56326dff48f5adf0db851b4b4cf3cf01a308" + ], + "v1_5_0_004": [ + "05e19278f66734ed98cc632b5ffe860ea4fc49ae7f5de49c824b9ade1562da46" ] }, "affected_objects": [ @@ -154,6 +161,7 @@ "pc_model_usage_daily", "pc_portable_restore_receipts", "pc_profile_policies", + "pc_rate_limit_windows", "pc_recall_effort_daily", "pc_recall_token_daily", "pc_receipt_migration_review", @@ -161,6 +169,9 @@ "pc_recurrence_observation", "pc_retained_dream_candidate_heads", "pc_retained_dream_candidate_versions", + "pc_runtime_members", + "pc_scheduler_leases", + "pc_scheduler_scans", "pc_scope_bindings", "pc_scope_context_references", "pc_scope_creation_requests", @@ -195,7 +206,11 @@ "pc_topic_memory_topic_fts_docsize", "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", - "pc_topic_memory_work_budgets" + "pc_topic_memory_work_budgets", + "pc_work_attempts", + "pc_work_items", + "pc_work_keys", + "pc_work_lanes" ], "readiness_scope": "server", "retained_objects": [ @@ -323,6 +338,11 @@ "artifact_storage", "portable_storage", "atomic_memory_evidence_projection" + ], + "v1_5_0_004": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -343,7 +363,8 @@ "schemas/mysql-v1_4_1_006.json", "schemas/mysql-v1_5_0_001.json", "schemas/mysql-v1_5_0_002.json", - "schemas/mysql-v1_5_0_003.json" + "schemas/mysql-v1_5_0_003.json", + "schemas/mysql-v1_5_0_004.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_004.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_004.py new file mode 100644 index 0000000000..a3c707be77 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_004.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register the Work Ledger tables through the shared MySQL DDL executor. + +The revision follows master's published ``v1_5_0_003`` head so databases +already migrated by a released binary keep an upgrade path, and both the +SQLite and MySQL histories stay linear with the same release-based name. +""" + +from alembic import context + +revision = "v1_5_0_004" +down_revision = "v1_5_0_003" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_004.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_004.json new file mode 100644 index 0000000000..6fd15cd034 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_004.json @@ -0,0 +1,4244 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "v1_5_0_004": { + "statements": [ + "CREATE TABLE pc_work_lanes (\n\tlane_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\thead_sequence BIGINT, \n\tnext_sequence BIGINT NOT NULL, \n\tPRIMARY KEY (lane_key), \n\tCONSTRAINT ck_pc_work_lanes_head_positive CHECK (head_sequence IS NULL OR head_sequence > 0), \n\tCONSTRAINT ck_pc_work_lanes_next_positive CHECK (next_sequence > 0)\n)", + "CREATE TABLE pc_work_items (\n\twork_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tlogical_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tlane_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tlane_sequence BIGINT NOT NULL, \n\tkind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tpayload_version INTEGER NOT NULL, \n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tpayload MEDIUMBLOB NOT NULL, \n\tstatus VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tavailable_at DATETIME(6) NOT NULL, \n\tlease_owner VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tlease_fence BIGINT NOT NULL, \n\tlease_expires_at DATETIME(6), \n\tattempt_count INTEGER NOT NULL, \n\tgeneration_attempt_count INTEGER NOT NULL, \n\trecovery_generation INTEGER NOT NULL, \n\tmax_attempts INTEGER NOT NULL, \n\tcancel_requested TINYINT(1) NOT NULL, \n\tresult_code VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tresult_payload MEDIUMBLOB, \n\terror_category VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\terror_code VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tstate_version BIGINT NOT NULL, \n\tcreated_at DATETIME(6) NOT NULL, \n\tupdated_at DATETIME(6) NOT NULL, \n\tcompleted_at DATETIME(6), \n\tPRIMARY KEY (work_id), \n\tCONSTRAINT uq_pc_work_items_lane_sequence UNIQUE (lane_key, lane_sequence), \n\tCONSTRAINT ck_pc_work_items_status CHECK (status IN ('queued', 'running', 'retry_wait', 'cancelling', 'succeeded', 'failed', 'cancelled')), \n\tCONSTRAINT ck_pc_work_items_lane_sequence_positive CHECK (lane_sequence > 0), \n\tCONSTRAINT ck_pc_work_items_payload_version_positive CHECK (payload_version > 0), \n\tCONSTRAINT ck_pc_work_items_fence_nonnegative CHECK (lease_fence >= 0), \n\tCONSTRAINT ck_pc_work_items_attempts_nonnegative CHECK (attempt_count >= 0), \n\tCONSTRAINT ck_pc_work_items_generation_attempts_nonnegative CHECK (generation_attempt_count >= 0), \n\tCONSTRAINT ck_pc_work_items_recovery_nonnegative CHECK (recovery_generation >= 0), \n\tCONSTRAINT ck_pc_work_items_max_attempts_positive CHECK (max_attempts > 0), \n\tCONSTRAINT ck_pc_work_items_state_version_positive CHECK (state_version > 0)\n)", + "CREATE INDEX ix_pc_work_items_claim ON pc_work_items (status, available_at)", + "CREATE INDEX ix_pc_work_items_completed ON pc_work_items (completed_at)", + "CREATE INDEX ix_pc_work_items_lane_status ON pc_work_items (lane_key, status)", + "CREATE INDEX ix_pc_work_items_scope_created ON pc_work_items (scope_id, created_at)", + "CREATE TABLE pc_work_keys (\n\tlogical_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\twork_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tlane_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcreated_at DATETIME(6) NOT NULL, \n\tPRIMARY KEY (logical_key), \n\tCONSTRAINT uq_pc_work_keys_work_id UNIQUE (work_id)\n)", + "CREATE TABLE pc_work_attempts (\n\twork_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tattempt_no INTEGER NOT NULL, \n\trecovery_generation INTEGER NOT NULL, \n\towner_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfence BIGINT NOT NULL, \n\tstarted_at DATETIME(6) NOT NULL, \n\theartbeat_at DATETIME(6) NOT NULL, \n\tfinished_at DATETIME(6), \n\toutcome VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\terror_category VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\terror_code VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttrace_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tspan_id VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tPRIMARY KEY (work_id, attempt_no), \n\tCONSTRAINT ck_pc_work_attempts_attempt_positive CHECK (attempt_no > 0), \n\tCONSTRAINT ck_pc_work_attempts_recovery_nonnegative CHECK (recovery_generation >= 0), \n\tCONSTRAINT ck_pc_work_attempts_fence_positive CHECK (fence > 0)\n)", + "CREATE INDEX ix_pc_work_attempts_finished ON pc_work_attempts (finished_at)", + "CREATE TABLE pc_scheduler_leases (\n\tlease_name VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\towner_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tfence BIGINT NOT NULL, \n\tlease_expires_at DATETIME(6), \n\tupdated_at DATETIME(6) NOT NULL, \n\tPRIMARY KEY (lease_name), \n\tCONSTRAINT ck_pc_scheduler_leases_fence_nonnegative CHECK (fence >= 0)\n)", + "CREATE TABLE pc_scheduler_scans (\n\tdiscoverer VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tnext_run_at DATETIME(6) NOT NULL, \n\tcontinuation VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tstate_version BIGINT NOT NULL, \n\tupdated_at DATETIME(6) NOT NULL, \n\tPRIMARY KEY (discoverer), \n\tCONSTRAINT ck_pc_scheduler_scans_version_positive CHECK (state_version > 0)\n)", + "CREATE TABLE pc_runtime_members (\n\tmember_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\t`role` VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbuild_version VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_min INTEGER NOT NULL, \n\tschema_max INTEGER NOT NULL, \n\tpayload_min INTEGER NOT NULL, \n\tpayload_max INTEGER NOT NULL, \n\tbehavior_revision VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\theartbeat_at DATETIME(6) NOT NULL, \n\texpires_at DATETIME(6) NOT NULL, \n\tPRIMARY KEY (member_id), \n\tCONSTRAINT ck_pc_runtime_members_role CHECK (role IN ('all', 'api', 'scheduler', 'worker')), \n\tCONSTRAINT ck_pc_runtime_members_schema_range CHECK (schema_min > 0 AND schema_max >= schema_min), \n\tCONSTRAINT ck_pc_runtime_members_payload_range CHECK (payload_min > 0 AND payload_max >= payload_min)\n)", + "CREATE INDEX ix_pc_runtime_members_role_expiry ON pc_runtime_members (`role`, expires_at)", + "CREATE TABLE pc_rate_limit_windows (\n\tprincipal_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tpolicy_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\twindow_started_at DATETIME(6) NOT NULL, \n\trequest_count BIGINT NOT NULL, \n\texpires_at DATETIME(6) NOT NULL, \n\tPRIMARY KEY (principal_key, policy_id, window_started_at), \n\tCONSTRAINT ck_pc_rate_limit_windows_count_positive CHECK (request_count > 0)\n)", + "CREATE INDEX ix_pc_rate_limit_windows_expiry ON pc_rate_limit_windows (expires_at)" + ], + "changes": [ + { + "pc_work_lanes": { + "columns": { + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "head_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "next_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "lane_key", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_lanes_head_positive": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "head_sequence" + ] + ], + [ + "binary", + ">", + [ + "column", + "head_sequence" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_work_lanes_next_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "next_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_work_items": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "available_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_owner": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lease_fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation_attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "max_attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cancel_requested": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_items_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "queued" + ], + [ + "literal", + "running" + ], + [ + "literal", + "retry_wait" + ], + [ + "literal", + "cancelling" + ], + [ + "literal", + "succeeded" + ], + [ + "literal", + "failed" + ], + [ + "literal", + "cancelled" + ] + ] + ], + "enforced": true + }, + "ck_pc_work_items_lane_sequence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "lane_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_payload_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "payload_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "lease_fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_generation_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation_attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_max_attempts_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "max_attempts" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_state_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_work_items_lane_sequence": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "lane_sequence", + null, + "asc" + ] + ], + "unique": true + } + } + } + }, + { + "pc_work_items": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "available_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_owner": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lease_fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation_attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "max_attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cancel_requested": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_items_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "queued" + ], + [ + "literal", + "running" + ], + [ + "literal", + "retry_wait" + ], + [ + "literal", + "cancelling" + ], + [ + "literal", + "succeeded" + ], + [ + "literal", + "failed" + ], + [ + "literal", + "cancelled" + ] + ] + ], + "enforced": true + }, + "ck_pc_work_items_lane_sequence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "lane_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_payload_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "payload_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "lease_fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_generation_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation_attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_max_attempts_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "max_attempts" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_state_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_work_items_lane_sequence": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "lane_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_work_items_claim": { + "columns": [ + [ + "status", + null, + "asc" + ], + [ + "available_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_work_items": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "available_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_owner": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lease_fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation_attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "max_attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cancel_requested": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_items_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "queued" + ], + [ + "literal", + "running" + ], + [ + "literal", + "retry_wait" + ], + [ + "literal", + "cancelling" + ], + [ + "literal", + "succeeded" + ], + [ + "literal", + "failed" + ], + [ + "literal", + "cancelled" + ] + ] + ], + "enforced": true + }, + "ck_pc_work_items_lane_sequence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "lane_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_payload_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "payload_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "lease_fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_generation_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation_attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_max_attempts_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "max_attempts" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_state_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_work_items_lane_sequence": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "lane_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_work_items_claim": { + "columns": [ + [ + "status", + null, + "asc" + ], + [ + "available_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_completed": { + "columns": [ + [ + "completed_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_work_items": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "available_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_owner": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lease_fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation_attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "max_attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cancel_requested": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_items_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "queued" + ], + [ + "literal", + "running" + ], + [ + "literal", + "retry_wait" + ], + [ + "literal", + "cancelling" + ], + [ + "literal", + "succeeded" + ], + [ + "literal", + "failed" + ], + [ + "literal", + "cancelled" + ] + ] + ], + "enforced": true + }, + "ck_pc_work_items_lane_sequence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "lane_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_payload_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "payload_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "lease_fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_generation_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation_attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_max_attempts_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "max_attempts" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_state_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_work_items_lane_sequence": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "lane_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_work_items_claim": { + "columns": [ + [ + "status", + null, + "asc" + ], + [ + "available_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_completed": { + "columns": [ + [ + "completed_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_lane_status": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "status", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_work_items": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "available_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_owner": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lease_fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation_attempt_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "max_attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cancel_requested": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_items_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "queued" + ], + [ + "literal", + "running" + ], + [ + "literal", + "retry_wait" + ], + [ + "literal", + "cancelling" + ], + [ + "literal", + "succeeded" + ], + [ + "literal", + "failed" + ], + [ + "literal", + "cancelled" + ] + ] + ], + "enforced": true + }, + "ck_pc_work_items_lane_sequence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "lane_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_payload_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "payload_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "lease_fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_generation_attempts_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation_attempt_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_max_attempts_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "max_attempts" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_items_state_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_work_items_lane_sequence": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "lane_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_work_items_claim": { + "columns": [ + [ + "status", + null, + "asc" + ], + [ + "available_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_completed": { + "columns": [ + [ + "completed_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_lane_status": { + "columns": [ + [ + "lane_key", + null, + "asc" + ], + [ + "status", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_work_items_scope_created": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "created_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_work_keys": { + "columns": { + "logical_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "lane_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "logical_key", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_work_keys_work_id": { + "columns": [ + [ + "work_id", + null, + "asc" + ] + ], + "unique": true + } + } + } + }, + { + "pc_work_attempts": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "attempt_no": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "owner_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "started_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "heartbeat_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "finished_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "outcome": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "trace_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "span_id": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ], + [ + "attempt_no", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_attempts_attempt_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "attempt_no" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_attempts_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_attempts_fence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_work_attempts": { + "columns": { + "work_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "attempt_no": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recovery_generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "owner_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "started_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "heartbeat_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "finished_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "outcome": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_category": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "trace_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "span_id": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "work_id", + null, + "asc" + ], + [ + "attempt_no", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_work_attempts_attempt_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "attempt_no" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_attempts_recovery_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recovery_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_work_attempts_fence_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_work_attempts_finished": { + "columns": [ + [ + "finished_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_scheduler_leases": { + "columns": { + "lease_name": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "lease_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_scheduler_leases_fence_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "fence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_scheduler_scans": { + "columns": { + "discoverer": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "next_run_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "continuation": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "discoverer", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_scheduler_scans_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_runtime_members": { + "columns": { + "member_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "schema_min": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_max": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload_min": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload_max": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "behavior_revision": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "heartbeat_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "member_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_runtime_members_role": { + "expression": [ + "in", + [ + "column", + "role" + ], + [ + [ + "literal", + "all" + ], + [ + "literal", + "api" + ], + [ + "literal", + "scheduler" + ], + [ + "literal", + "worker" + ] + ] + ], + "enforced": true + }, + "ck_pc_runtime_members_schema_range": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "schema_min" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "schema_max" + ], + [ + "column", + "schema_min" + ] + ] + ], + "enforced": true + }, + "ck_pc_runtime_members_payload_range": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "payload_min" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "payload_max" + ], + [ + "column", + "payload_min" + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_runtime_members": { + "columns": { + "member_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "schema_min": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_max": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload_min": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload_max": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "behavior_revision": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "heartbeat_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "member_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_runtime_members_role": { + "expression": [ + "in", + [ + "column", + "role" + ], + [ + [ + "literal", + "all" + ], + [ + "literal", + "api" + ], + [ + "literal", + "scheduler" + ], + [ + "literal", + "worker" + ] + ] + ], + "enforced": true + }, + "ck_pc_runtime_members_schema_range": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "schema_min" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "schema_max" + ], + [ + "column", + "schema_min" + ] + ] + ], + "enforced": true + }, + "ck_pc_runtime_members_payload_range": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "payload_min" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "payload_max" + ], + [ + "column", + "payload_min" + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_runtime_members_role_expiry": { + "columns": [ + [ + "role", + null, + "asc" + ], + [ + "expires_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_rate_limit_windows": { + "columns": { + "principal_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "window_started_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "request_count": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "principal_key", + null, + "asc" + ], + [ + "policy_id", + null, + "asc" + ], + [ + "window_started_at", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_rate_limit_windows_count_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "request_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_rate_limit_windows": { + "columns": { + "principal_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "window_started_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "request_count": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expires_at": { + "type": [ + "datetime", + [ + "6" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "principal_key", + null, + "asc" + ], + [ + "policy_id", + null, + "asc" + ], + [ + "window_started_at", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_rate_limit_windows_count_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "request_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_rate_limit_windows_expiry": { + "columns": [ + [ + "expires_at", + null, + "asc" + ] + ], + "unique": false + } + } + } + } + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_004.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_004.json new file mode 100644 index 0000000000..b9a9cf34e0 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_004.json @@ -0,0 +1,19 @@ +{ + "create": [ + "CREATE TABLE pc_work_lanes (\n\tlane_key VARCHAR(64) NOT NULL, \n\thead_sequence BIGINT, \n\tnext_sequence BIGINT NOT NULL, \n\tPRIMARY KEY (lane_key), \n\tCONSTRAINT ck_pc_work_lanes_head_positive CHECK (head_sequence IS NULL OR head_sequence > 0), \n\tCONSTRAINT ck_pc_work_lanes_next_positive CHECK (next_sequence > 0)\n)", + "CREATE TABLE pc_work_items (\n\twork_id VARCHAR(36) NOT NULL, \n\tlogical_key VARCHAR(64) NOT NULL, \n\tlane_key VARCHAR(64) NOT NULL, \n\tlane_sequence BIGINT NOT NULL, \n\tkind VARCHAR(128) NOT NULL, \n\tpayload_version INTEGER NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tpayload BLOB NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\tavailable_at DATETIME NOT NULL, \n\tlease_owner VARCHAR(128), \n\tlease_fence BIGINT NOT NULL, \n\tlease_expires_at DATETIME, \n\tattempt_count INTEGER NOT NULL, \n\tgeneration_attempt_count INTEGER NOT NULL, \n\trecovery_generation INTEGER NOT NULL, \n\tmax_attempts INTEGER NOT NULL, \n\tcancel_requested BOOLEAN NOT NULL, \n\tresult_code VARCHAR(128), \n\tresult_payload BLOB, \n\terror_category VARCHAR(64), \n\terror_code VARCHAR(128), \n\tstate_version BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tcompleted_at DATETIME, \n\tPRIMARY KEY (work_id), \n\tCONSTRAINT uq_pc_work_items_lane_sequence UNIQUE (lane_key, lane_sequence), \n\tCONSTRAINT ck_pc_work_items_status CHECK (status IN ('queued', 'running', 'retry_wait', 'cancelling', 'succeeded', 'failed', 'cancelled')), \n\tCONSTRAINT ck_pc_work_items_lane_sequence_positive CHECK (lane_sequence > 0), \n\tCONSTRAINT ck_pc_work_items_payload_version_positive CHECK (payload_version > 0), \n\tCONSTRAINT ck_pc_work_items_fence_nonnegative CHECK (lease_fence >= 0), \n\tCONSTRAINT ck_pc_work_items_attempts_nonnegative CHECK (attempt_count >= 0), \n\tCONSTRAINT ck_pc_work_items_generation_attempts_nonnegative CHECK (generation_attempt_count >= 0), \n\tCONSTRAINT ck_pc_work_items_recovery_nonnegative CHECK (recovery_generation >= 0), \n\tCONSTRAINT ck_pc_work_items_max_attempts_positive CHECK (max_attempts > 0), \n\tCONSTRAINT ck_pc_work_items_state_version_positive CHECK (state_version > 0)\n)", + "CREATE INDEX ix_pc_work_items_claim ON pc_work_items (status, available_at)", + "CREATE INDEX ix_pc_work_items_completed ON pc_work_items (completed_at)", + "CREATE INDEX ix_pc_work_items_lane_status ON pc_work_items (lane_key, status)", + "CREATE INDEX ix_pc_work_items_scope_created ON pc_work_items (scope_id, created_at)", + "CREATE TABLE pc_work_keys (\n\tlogical_key VARCHAR(64) NOT NULL, \n\twork_id VARCHAR(36) NOT NULL, \n\tlane_key VARCHAR(64) NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tPRIMARY KEY (logical_key), \n\tCONSTRAINT uq_pc_work_keys_work_id UNIQUE (work_id)\n)", + "CREATE TABLE pc_work_attempts (\n\twork_id VARCHAR(36) NOT NULL, \n\tattempt_no INTEGER NOT NULL, \n\trecovery_generation INTEGER NOT NULL, \n\towner_id VARCHAR(128) NOT NULL, \n\tfence BIGINT NOT NULL, \n\tstarted_at DATETIME NOT NULL, \n\theartbeat_at DATETIME NOT NULL, \n\tfinished_at DATETIME, \n\toutcome VARCHAR(32), \n\terror_category VARCHAR(64), \n\terror_code VARCHAR(128), \n\ttrace_id VARCHAR(32), \n\tspan_id VARCHAR(16), \n\tPRIMARY KEY (work_id, attempt_no), \n\tCONSTRAINT ck_pc_work_attempts_attempt_positive CHECK (attempt_no > 0), \n\tCONSTRAINT ck_pc_work_attempts_recovery_nonnegative CHECK (recovery_generation >= 0), \n\tCONSTRAINT ck_pc_work_attempts_fence_positive CHECK (fence > 0)\n)", + "CREATE INDEX ix_pc_work_attempts_finished ON pc_work_attempts (finished_at)", + "CREATE TABLE pc_scheduler_leases (\n\tlease_name VARCHAR(64) NOT NULL, \n\towner_id VARCHAR(128), \n\tfence BIGINT NOT NULL, \n\tlease_expires_at DATETIME, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (lease_name), \n\tCONSTRAINT ck_pc_scheduler_leases_fence_nonnegative CHECK (fence >= 0)\n)", + "CREATE TABLE pc_scheduler_scans (\n\tdiscoverer VARCHAR(128) NOT NULL, \n\tnext_run_at DATETIME NOT NULL, \n\tcontinuation VARCHAR(256), \n\tstate_version BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (discoverer), \n\tCONSTRAINT ck_pc_scheduler_scans_version_positive CHECK (state_version > 0)\n)", + "CREATE TABLE pc_runtime_members (\n\tmember_id VARCHAR(128) NOT NULL, \n\trole VARCHAR(16) NOT NULL, \n\tbuild_version VARCHAR(64) NOT NULL, \n\tschema_min INTEGER NOT NULL, \n\tschema_max INTEGER NOT NULL, \n\tpayload_min INTEGER NOT NULL, \n\tpayload_max INTEGER NOT NULL, \n\tbehavior_revision VARCHAR(128) NOT NULL, \n\theartbeat_at DATETIME NOT NULL, \n\texpires_at DATETIME NOT NULL, \n\tPRIMARY KEY (member_id), \n\tCONSTRAINT ck_pc_runtime_members_role CHECK (role IN ('all', 'api', 'scheduler', 'worker')), \n\tCONSTRAINT ck_pc_runtime_members_schema_range CHECK (schema_min > 0 AND schema_max >= schema_min), \n\tCONSTRAINT ck_pc_runtime_members_payload_range CHECK (payload_min > 0 AND payload_max >= payload_min)\n)", + "CREATE INDEX ix_pc_runtime_members_role_expiry ON pc_runtime_members (role, expires_at)", + "CREATE TABLE pc_rate_limit_windows (\n\tprincipal_key VARCHAR(64) NOT NULL, \n\tpolicy_id VARCHAR(64) NOT NULL, \n\twindow_started_at DATETIME NOT NULL, \n\trequest_count BIGINT NOT NULL, \n\texpires_at DATETIME NOT NULL, \n\tPRIMARY KEY (principal_key, policy_id, window_started_at), \n\tCONSTRAINT ck_pc_rate_limit_windows_count_positive CHECK (request_count > 0)\n)", + "CREATE INDEX ix_pc_rate_limit_windows_expiry ON pc_rate_limit_windows (expires_at)" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_004_work_ledger.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_004_work_ledger.py new file mode 100644 index 0000000000..1fccc447bd --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_004_work_ledger.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add the durable Work Ledger and its scheduler coordination tables. + +The revision is appended after master's published ``v1_5_0_003`` head rather +than being inserted before it: databases already migrated by a released binary +are stamped at that head, and re-parenting a published revision would leave them +with no upgrade path to create the ledger tables. +""" + +import json +from pathlib import Path + +from alembic import op + +revision = "v1_5_0_004" +down_revision = "v1_5_0_003" + + +def upgrade(): + resource = Path(__file__).parents[1] / "schemas/sqlite-v1_5_0_004.json" + connection = op.get_bind() + for statement in json.loads(resource.read_text())["create"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json index 4d8bd0a7bc..a497852357 100644 --- a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json @@ -43,6 +43,10 @@ "v1_5_0_003": [ "schemas/sqlite-v1_5_0_003.json", "schemas/mysql-v1_5_0_003.json" + ], + "v1_5_0_004": [ + "schemas/sqlite-v1_5_0_004.json", + "schemas/mysql-v1_5_0_004.json" ] }, "sqlite_baselines": { @@ -78,6 +82,9 @@ ], "v1_5_0_003": [ "357287e3bae474ca1810dd0949fe56326dff48f5adf0db851b4b4cf3cf01a308" + ], + "v1_5_0_004": [ + "05e19278f66734ed98cc632b5ffe860ea4fc49ae7f5de49c824b9ade1562da46" ] }, "affected_objects": [ @@ -153,6 +160,7 @@ "pc_model_usage_daily", "pc_portable_restore_receipts", "pc_profile_policies", + "pc_rate_limit_windows", "pc_recall_effort_daily", "pc_recall_token_daily", "pc_receipt_migration_review", @@ -160,6 +168,9 @@ "pc_recurrence_observation", "pc_retained_dream_candidate_heads", "pc_retained_dream_candidate_versions", + "pc_runtime_members", + "pc_scheduler_leases", + "pc_scheduler_scans", "pc_scope_bindings", "pc_scope_context_references", "pc_scope_creation_requests", @@ -194,7 +205,11 @@ "pc_topic_memory_topic_fts_docsize", "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", - "pc_topic_memory_work_budgets" + "pc_topic_memory_work_budgets", + "pc_work_attempts", + "pc_work_items", + "pc_work_keys", + "pc_work_lanes" ], "readiness_scope": "server", "retained_objects": [ @@ -327,6 +342,11 @@ "artifact_storage", "portable_storage", "atomic_memory_evidence_projection" + ], + "v1_5_0_004": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -347,7 +367,8 @@ "schemas/mysql-v1_4_1_006.json", "schemas/mysql-v1_5_0_001.json", "schemas/mysql-v1_5_0_002.json", - "schemas/mysql-v1_5_0_003.json" + "schemas/mysql-v1_5_0_003.json", + "schemas/mysql-v1_5_0_004.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/oceanbase/profile.py b/src/powercontext/builtin/persistence/oceanbase/profile.py index f6740a52d1..2303f95c4d 100644 --- a/src/powercontext/builtin/persistence/oceanbase/profile.py +++ b/src/powercontext/builtin/persistence/oceanbase/profile.py @@ -133,11 +133,18 @@ async def open( config: OceanBaseConfig, *, tables: tuple[Table, ...], + create_schema: bool = True, _archive_recovery: bool = False, ) -> AsyncIterator[OceanBaseProfile]: """Create, initialize and exclusively own an official OceanBase engine.""" - async with cls._connect(config, tables=tables, provision=True, _archive_recovery=_archive_recovery) as profile: + async with cls._connect( + config, + tables=tables, + provision=True, + create_schema=create_schema, + _archive_recovery=_archive_recovery, + ) as profile: yield profile @classmethod @@ -161,6 +168,7 @@ async def _connect( *, tables: tuple[Table, ...], provision: bool, + create_schema: bool = True, _archive_recovery: bool = False, ) -> AsyncIterator[OceanBaseProfile]: _register_official_dialect() @@ -179,7 +187,9 @@ async def _connect( else: await require_existing_database(engine, name=name) async with _initialized_profile( - cls(database=database, tables=tables), _archive_recovery=_archive_recovery + cls(database=database, tables=tables), + create_schema=create_schema, + _archive_recovery=_archive_recovery, ) as profile: yield profile finally: @@ -192,6 +202,7 @@ async def attach( engine: AsyncEngine, *, tables: tuple[Table, ...], + create_schema: bool = True, ) -> AsyncIterator[OceanBaseProfile]: """Initialize an explicitly selected caller-owned database without replacing its engine.""" _validate_oceanbase_url(engine.url) @@ -200,7 +211,7 @@ async def attach( "Attached OceanBase engines must select a database; use OceanBaseProfile.open for defaults" ) database = AsyncDatabase.attach(engine) - async with _initialized_profile(cls(database=database, tables=tables)) as profile: + async with _initialized_profile(cls(database=database, tables=tables), create_schema=create_schema) as profile: yield profile @@ -218,7 +229,10 @@ def _create_engine(config: OceanBaseConfig, url: URL) -> AsyncEngine: @asynccontextmanager async def _initialized_profile( - profile: OceanBaseProfile, *, _archive_recovery: bool = False + profile: OceanBaseProfile, + *, + create_schema: bool = True, + _archive_recovery: bool = False, ) -> AsyncIterator[OceanBaseProfile]: try: async with profile.database.transaction() as connection: @@ -230,7 +244,8 @@ async def _initialized_profile( ) async with profile.database.transaction() as connection: await _require_compatible_identity_collations(connection, profile.tables) - await create_tables(connection, profile.tables) + if create_schema: + await create_tables(connection, profile.tables) yield profile finally: await profile.database.close() diff --git a/src/powercontext/builtin/persistence/rate_limit.py b/src/powercontext/builtin/persistence/rate_limit.py new file mode 100644 index 0000000000..a59c5131ba --- /dev/null +++ b/src/powercontext/builtin/persistence/rate_limit.py @@ -0,0 +1,216 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Database-time fixed-window rate limiting for horizontally scaled APIs.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, InvalidStoredColumnError +from powercontext.builtin.persistence.tables import RATE_LIMIT_WINDOWS_TABLE + + +@dataclass(frozen=True, slots=True) +class RateLimitDecision: + """One atomic counter decision.""" + + allowed: bool + remaining: int + retry_after_seconds: int + + +class RateLimitRepository: + """Increment one shared fixed window under a row lock.""" + + async def consume( + self, + connection: AsyncConnection, + *, + principal_key: str, + policy_id: str, + limit: int, + window_seconds: int, + ) -> RateLimitDecision: + _require_digest(principal_key) + _require_text("policy_id", policy_id, 64) + _require_positive("limit", limit) + _require_positive("window_seconds", window_seconds) + now = await database_now(connection) + window_start = _window_start(now, window_seconds) + expires_at = window_start + timedelta(seconds=window_seconds) + row, created = await _lock_or_create_window( + connection, + principal_key=principal_key, + policy_id=policy_id, + window_start=window_start, + expires_at=expires_at, + ) + count = _positive_integer(row["request_count"], "request_count") + retry_after = max(1, math.ceil((expires_at - now).total_seconds())) + if created: + return RateLimitDecision( + allowed=True, + remaining=max(limit - count, 0), + retry_after_seconds=retry_after, + ) + if count >= limit: + return RateLimitDecision(allowed=False, remaining=0, retry_after_seconds=retry_after) + next_count = count + 1 + await connection.execute( + update(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .values(request_count=next_count) + ) + return RateLimitDecision( + allowed=True, + remaining=max(limit - next_count, 0), + retry_after_seconds=retry_after, + ) + + async def purge_expired(self, connection: AsyncConnection, /, *, limit: int = 500) -> int: + """Delete one bounded batch of expired fixed-window counters.""" + + if limit < 1 or limit > 500: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 500") + now = await database_now(connection) + rows = ( + await connection.execute( + select( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at, + ) + .where(RATE_LIMIT_WINDOWS_TABLE.c.expires_at <= now) + .order_by(RATE_LIMIT_WINDOWS_TABLE.c.expires_at) + .limit(limit) + ) + ).all() + deleted = 0 + for principal_key, policy_id, window_started_at in rows: + result = await connection.execute( + delete(RATE_LIMIT_WINDOWS_TABLE).where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_started_at, + RATE_LIMIT_WINDOWS_TABLE.c.expires_at <= now, + ) + ) + deleted += result.rowcount + return deleted + + +async def _lock_or_create_window( + connection: AsyncConnection, + *, + principal_key: str, + policy_id: str, + window_start: datetime, + expires_at: datetime, +) -> tuple[Mapping[Any, Any], bool]: + row = await _lock_window(connection, principal_key, policy_id, window_start) + if row is not None: + return row, False + created = await insert_if_absent( + connection, + RATE_LIMIT_WINDOWS_TABLE, + { + "principal_key": principal_key, + "policy_id": policy_id, + "window_started_at": window_start, + "request_count": 1, + "expires_at": expires_at, + }, + ) + row = await _lock_window(connection, principal_key, policy_id, window_start) + if row is None: + raise InvalidStoredColumnError( # noqa: TRY003 + "rate limit window", + "an initialized fixed window", + ) + return row, created + + +async def _lock_window( + connection: AsyncConnection, + principal_key: str, + policy_id: str, + window_start: datetime, +) -> Mapping[Any, Any] | None: + await connection.execute( + update(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .values(request_count=RATE_LIMIT_WINDOWS_TABLE.c.request_count) + ) + return ( + ( + await connection.execute( + select(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +def _window_start(now: datetime, window_seconds: int) -> datetime: + aware = now.replace(tzinfo=UTC) if now.tzinfo is None else now.astimezone(UTC) + epoch = int(aware.timestamp()) + return datetime.fromtimestamp(epoch - epoch % window_seconds, tz=UTC).replace(tzinfo=None) + + +def _require_digest(value: str) -> None: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise InvalidRepositoryArgumentError("principal_key", "must be a lowercase SHA-256 digest") + + +def _require_text(field: str, value: str, maximum: int) -> None: + if not value.strip() or value != value.strip() or len(value) > maximum: + raise InvalidRepositoryArgumentError(field, f"must be trimmed and contain at most {maximum} characters") + + +def _require_positive(field: str, value: int) -> None: + if isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +__all__ = ["RateLimitDecision", "RateLimitRepository"] diff --git a/src/powercontext/builtin/persistence/seekdb/profile.py b/src/powercontext/builtin/persistence/seekdb/profile.py index 9ad17985b0..620185eab2 100644 --- a/src/powercontext/builtin/persistence/seekdb/profile.py +++ b/src/powercontext/builtin/persistence/seekdb/profile.py @@ -130,11 +130,18 @@ async def open( config: SeekDBConfig, *, tables: tuple[Table, ...], + create_schema: bool = True, _archive_recovery: bool = False, ) -> AsyncIterator[SeekDBProfile]: """Start seekdb locally and connect through its async Unix socket.""" - async with cls._connect(config, tables=tables, provision=True, _archive_recovery=_archive_recovery) as profile: + async with cls._connect( + config, + tables=tables, + provision=True, + create_schema=create_schema, + _archive_recovery=_archive_recovery, + ) as profile: yield profile @classmethod @@ -158,6 +165,7 @@ async def _connect( *, tables: tuple[Table, ...], provision: bool, + create_schema: bool = True, _archive_recovery: bool = False, ) -> AsyncIterator[SeekDBProfile]: from powercontext.builtin.persistence.migrations.locking import assert_no_local_migration @@ -189,8 +197,9 @@ async def _connect( await prepare_topic_memory_recovery_schema( engine, initialize=any(table.name == BUDGET for table in tables) ) - async with database.transaction() as connection: - await create_tables(connection, tables) + if create_schema: + async with database.transaction() as connection: + await create_tables(connection, tables) yield profile finally: close_task = asyncio.create_task(database.close()) diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 8ed39180b0..9b1efe283e 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -97,6 +97,7 @@ async def open( *, tables: tuple[Table, ...], load_vector_extension: bool = False, + create_schema: bool = True, _archive_recovery: bool = False, ) -> AsyncIterator[SQLiteProfile]: """Create, initialize and exclusively own one SQLite engine.""" @@ -121,8 +122,9 @@ async def open( engine, initialize=any(table.name == BUDGET for table in tables) ) await _warm_sqlite(engine, config) - async with database.transaction() as connection: - await create_tables(connection, tables) + if create_schema: + async with database.transaction() as connection: + await create_tables(connection, tables) yield profile finally: await database.close() diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 226e8d9ac1..66c756fb29 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -91,6 +91,12 @@ def _entry_text_type(): return Text().with_variant(MEDIUMTEXT(), "mysql") +def _coordination_timestamp_type(): + """Preserve the microsecond precision used by database-time leases.""" + + return DateTime(timezone=False).with_variant(DATETIME(fsp=6), "mysql") + + SCOPES_TABLE = Table( "pc_scopes", SHARED_METADATA, @@ -1118,6 +1124,166 @@ def _entry_text_type(): ) +WORK_LANES_TABLE = Table( + "pc_work_lanes", + SHARED_METADATA, + Column("lane_key", identity_string(64), primary_key=True), + Column("head_sequence", BigInteger), + Column("next_sequence", BigInteger, nullable=False), + CheckConstraint("head_sequence IS NULL OR head_sequence > 0", name="ck_pc_work_lanes_head_positive"), + CheckConstraint("next_sequence > 0", name="ck_pc_work_lanes_next_positive"), +) + +WORK_ITEMS_TABLE = Table( + "pc_work_items", + SHARED_METADATA, + Column("work_id", identity_string(36), primary_key=True), + Column("logical_key", identity_string(64), nullable=False), + Column("lane_key", identity_string(64), nullable=False), + Column("lane_sequence", BigInteger, nullable=False), + Column("kind", identity_string(128), nullable=False), + Column("payload_version", Integer, nullable=False), + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), nullable=False), + Column("payload", _canonical_payload_type(), nullable=False), + Column("status", identity_string(16), nullable=False), + Column("available_at", _coordination_timestamp_type(), nullable=False), + Column("lease_owner", identity_string(128)), + Column("lease_fence", BigInteger, nullable=False), + Column("lease_expires_at", _coordination_timestamp_type()), + Column("attempt_count", Integer, nullable=False), + Column("generation_attempt_count", Integer, nullable=False), + Column("recovery_generation", Integer, nullable=False), + Column("max_attempts", Integer, nullable=False), + Column("cancel_requested", Boolean, nullable=False), + Column("result_code", identity_string(128)), + Column("result_payload", _canonical_payload_type()), + Column("error_category", identity_string(64)), + Column("error_code", identity_string(128)), + Column("state_version", BigInteger, nullable=False), + Column("created_at", _coordination_timestamp_type(), nullable=False), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + Column("completed_at", _coordination_timestamp_type()), + UniqueConstraint("lane_key", "lane_sequence", name="uq_pc_work_items_lane_sequence"), + CheckConstraint( + "status IN ('queued', 'running', 'retry_wait', 'cancelling', 'succeeded', 'failed', 'cancelled')", + name="ck_pc_work_items_status", + ), + CheckConstraint("lane_sequence > 0", name="ck_pc_work_items_lane_sequence_positive"), + CheckConstraint("payload_version > 0", name="ck_pc_work_items_payload_version_positive"), + CheckConstraint("lease_fence >= 0", name="ck_pc_work_items_fence_nonnegative"), + CheckConstraint("attempt_count >= 0", name="ck_pc_work_items_attempts_nonnegative"), + CheckConstraint("generation_attempt_count >= 0", name="ck_pc_work_items_generation_attempts_nonnegative"), + CheckConstraint("recovery_generation >= 0", name="ck_pc_work_items_recovery_nonnegative"), + CheckConstraint("max_attempts > 0", name="ck_pc_work_items_max_attempts_positive"), + CheckConstraint("state_version > 0", name="ck_pc_work_items_state_version_positive"), +) +Index("ix_pc_work_items_claim", WORK_ITEMS_TABLE.c.status, WORK_ITEMS_TABLE.c.available_at) +Index("ix_pc_work_items_lane_status", WORK_ITEMS_TABLE.c.lane_key, WORK_ITEMS_TABLE.c.status) +Index("ix_pc_work_items_completed", WORK_ITEMS_TABLE.c.completed_at) +Index("ix_pc_work_items_scope_created", WORK_ITEMS_TABLE.c.scope_id, WORK_ITEMS_TABLE.c.created_at) + +WORK_KEYS_TABLE = Table( + "pc_work_keys", + SHARED_METADATA, + Column("logical_key", identity_string(64), primary_key=True), + Column("work_id", identity_string(36), nullable=False), + Column("lane_key", identity_string(64), nullable=False), + Column("created_at", _coordination_timestamp_type(), nullable=False), + # Named so the frozen migration bundle can register the exact constraint: + # an anonymous UNIQUE has no identity to verify on OceanBase. + UniqueConstraint("work_id", name="uq_pc_work_keys_work_id"), +) + +WORK_ATTEMPTS_TABLE = Table( + "pc_work_attempts", + SHARED_METADATA, + Column("work_id", identity_string(36), primary_key=True), + Column("attempt_no", Integer, primary_key=True), + Column("recovery_generation", Integer, nullable=False), + Column("owner_id", identity_string(128), nullable=False), + Column("fence", BigInteger, nullable=False), + Column("started_at", _coordination_timestamp_type(), nullable=False), + Column("heartbeat_at", _coordination_timestamp_type(), nullable=False), + Column("finished_at", _coordination_timestamp_type()), + Column("outcome", identity_string(32)), + Column("error_category", identity_string(64)), + Column("error_code", identity_string(128)), + Column("trace_id", identity_string(32)), + Column("span_id", identity_string(16)), + CheckConstraint("attempt_no > 0", name="ck_pc_work_attempts_attempt_positive"), + CheckConstraint("recovery_generation >= 0", name="ck_pc_work_attempts_recovery_nonnegative"), + CheckConstraint("fence > 0", name="ck_pc_work_attempts_fence_positive"), +) +Index("ix_pc_work_attempts_finished", WORK_ATTEMPTS_TABLE.c.finished_at) + +WORK_TABLES = ( + WORK_LANES_TABLE, + WORK_ITEMS_TABLE, + WORK_KEYS_TABLE, + WORK_ATTEMPTS_TABLE, +) + +SCHEDULER_LEASES_TABLE = Table( + "pc_scheduler_leases", + SHARED_METADATA, + Column("lease_name", identity_string(64), primary_key=True), + Column("owner_id", identity_string(128)), + Column("fence", BigInteger, nullable=False), + Column("lease_expires_at", _coordination_timestamp_type()), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("fence >= 0", name="ck_pc_scheduler_leases_fence_nonnegative"), +) + +SCHEDULER_SCANS_TABLE = Table( + "pc_scheduler_scans", + SHARED_METADATA, + Column("discoverer", identity_string(128), primary_key=True), + Column("next_run_at", _coordination_timestamp_type(), nullable=False), + Column("continuation", identity_string(MAX_SCOPE_ID_LENGTH)), + Column("state_version", BigInteger, nullable=False), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("state_version > 0", name="ck_pc_scheduler_scans_version_positive"), +) + +RUNTIME_MEMBERS_TABLE = Table( + "pc_runtime_members", + SHARED_METADATA, + Column("member_id", identity_string(128), primary_key=True), + Column("role", identity_string(16), nullable=False), + Column("build_version", identity_string(64), nullable=False), + Column("schema_min", Integer, nullable=False), + Column("schema_max", Integer, nullable=False), + Column("payload_min", Integer, nullable=False), + Column("payload_max", Integer, nullable=False), + Column("behavior_revision", identity_string(128), nullable=False), + Column("heartbeat_at", _coordination_timestamp_type(), nullable=False), + Column("expires_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("role IN ('all', 'api', 'scheduler', 'worker')", name="ck_pc_runtime_members_role"), + CheckConstraint("schema_min > 0 AND schema_max >= schema_min", name="ck_pc_runtime_members_schema_range"), + CheckConstraint("payload_min > 0 AND payload_max >= payload_min", name="ck_pc_runtime_members_payload_range"), +) +Index("ix_pc_runtime_members_role_expiry", RUNTIME_MEMBERS_TABLE.c.role, RUNTIME_MEMBERS_TABLE.c.expires_at) + +RATE_LIMIT_WINDOWS_TABLE = Table( + "pc_rate_limit_windows", + SHARED_METADATA, + Column("principal_key", identity_string(64), primary_key=True), + Column("policy_id", identity_string(64), primary_key=True), + Column("window_started_at", _coordination_timestamp_type(), primary_key=True), + Column("request_count", BigInteger, nullable=False), + Column("expires_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("request_count > 0", name="ck_pc_rate_limit_windows_count_positive"), +) +Index("ix_pc_rate_limit_windows_expiry", RATE_LIMIT_WINDOWS_TABLE.c.expires_at) + +COORDINATION_TABLES = ( + SCHEDULER_LEASES_TABLE, + SCHEDULER_SCANS_TABLE, + RUNTIME_MEMBERS_TABLE, + RATE_LIMIT_WINDOWS_TABLE, +) + + MAX_MEMORY_ENTRY_ID_LENGTH = 128 MAX_MEMORY_ENTRY_KIND_LENGTH = 128 MAX_MEMORY_HASH_LENGTH = 64 @@ -1438,6 +1604,8 @@ def _entry_text_type(): + TOPIC_MEMORY_TABLES + MEMORY_TABLES + STATISTICS_TABLES + + WORK_TABLES + + COORDINATION_TABLES + DECISION_OBSERVATION_TABLES + RECURRENCE_TABLES + ( diff --git a/src/powercontext/builtin/persistence/work.py b/src/powercontext/builtin/persistence/work.py new file mode 100644 index 0000000000..36cd4d1fdb --- /dev/null +++ b/src/powercontext/builtin/persistence/work.py @@ -0,0 +1,1375 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Durable work ledger with lane serialization, leases, and fencing.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable, Mapping +from contextlib import suppress +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator +from sqlalchemy import and_, delete, false, func, insert, not_, or_, select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.codec import stored_bytes +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import ( + InvalidRepositoryArgumentError, + InvalidStoredColumnError, + InvalidStoredPayloadError, + RepositoryError, + RepositoryNotFoundError, +) +from powercontext.builtin.persistence.tables import ( + WORK_ATTEMPTS_TABLE, + WORK_ITEMS_TABLE, + WORK_KEYS_TABLE, + WORK_LANES_TABLE, +) +from powercontext.limits import MAX_SCOPE_ID_LENGTH + +if TYPE_CHECKING: + from powercontext.builtin.runtime.work_observability import WorkObserver + +_MAX_PAYLOAD_BYTES = 16 * 1024 +_TERMINAL_STATUSES = frozenset({"succeeded", "cancelled"}) + + +class WorkStatus(StrEnum): + """Persisted lifecycle states for one logical work window.""" + + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + CANCELLING = "cancelling" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class WorkSpec(BaseModel): + """Safe, versioned references needed to execute one logical window.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: str + payload_version: int = Field(ge=1) + scope_id: str + lane_key: str + logical_key: str + payload: dict[str, JsonValue] + max_attempts: int = Field(default=5, ge=1, le=100) + available_at: datetime | None = None + + @field_validator("kind") + @classmethod + def validate_kind(cls, value: str) -> str: + return _validated_text("kind", value, 128) + + @field_validator("scope_id") + @classmethod + def validate_scope_id(cls, value: str) -> str: + return _validated_text("scope_id", value, MAX_SCOPE_ID_LENGTH) + + @field_validator("lane_key", "logical_key") + @classmethod + def validate_digest(cls, value: str, info: Any) -> str: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{info.field_name} must be a lowercase SHA-256 digest") # noqa: TRY003 + return value + + +class WorkResult(BaseModel): + """Sanitized result metadata persisted after a successful commit.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + code: str + payload: dict[str, JsonValue] = Field(default_factory=dict) + + @field_validator("code") + @classmethod + def validate_code(cls, value: str) -> str: + return _validated_text("code", value, 128) + + +WorkCommit = Callable[[AsyncConnection], Awaitable[WorkResult | None]] + + +class WorkFailure(BaseModel): + """Bounded failure classification; raw exceptions are never persisted.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + category: str + code: str + retryable: bool + + @field_validator("category") + @classmethod + def validate_category(cls, value: str) -> str: + return _validated_text("category", value, 64) + + @field_validator("code") + @classmethod + def validate_code(cls, value: str) -> str: + return _validated_text("code", value, 128) + + +class StoredWork(BaseModel): + """Decoded work item returned by repository operations.""" + + model_config = ConfigDict(frozen=True) + + work_id: str + logical_key: str + lane_key: str + lane_sequence: int + kind: str + payload_version: int + scope_id: str + payload: dict[str, JsonValue] + status: WorkStatus + available_at: datetime + lease_owner: str | None + lease_fence: int + lease_expires_at: datetime | None + attempt_count: int + generation_attempt_count: int + recovery_generation: int + max_attempts: int + cancel_requested: bool + result_code: str | None + result_payload: dict[str, JsonValue] | None + error_category: str | None + error_code: str | None + state_version: int + created_at: datetime + updated_at: datetime + completed_at: datetime | None + + +class EnqueueResult(BaseModel): + """Whether enqueue created a row or joined an existing logical window.""" + + model_config = ConfigDict(frozen=True) + + created: bool + work: StoredWork + + +class WorkClaim(BaseModel): + """A fenced lease plus the safe payload needed by a Worker handler.""" + + model_config = ConfigDict(frozen=True) + + work_id: str + logical_key: str + lane_key: str + lane_sequence: int + kind: str + payload_version: int + scope_id: str + payload: dict[str, JsonValue] + owner_id: str + fence: int + attempt_no: int + generation_attempt_no: int + recovery_generation: int + created_at: datetime + claimed_at: datetime + previous_trace_id: str | None = None + previous_span_id: str | None = None + lease_expires_at: datetime + + +class WorkQueueStatistic(BaseModel): + """One bounded queue gauge sample grouped by kind and state.""" + + model_config = ConfigDict(frozen=True) + + kind: str + status: WorkStatus + depth: int + oldest_age_seconds: float + + +class WorkStateConflictError(RepositoryError): + """Raised when an optimistic operation observes a different state.""" + + def __init__(self, work_id: str, expected_version: int, status: WorkStatus, actual_version: int) -> None: + self.work_id = work_id + self.expected_version = expected_version + self.status = status + self.actual_version = actual_version + super().__init__( + f"work {work_id!r} changed: expected version {expected_version}, " + f"found {status.value} at version {actual_version}" + ) + + +class StaleWorkClaimError(RepositoryError): + """Raised when a lease owner or fence can no longer mutate a work item.""" + + def __init__(self, work_id: str) -> None: + self.work_id = work_id + super().__init__(f"work claim for {work_id!r} is no longer current") + + +class WorkRepository: + """Persist and transition work using short caller-owned transactions.""" + + def __init__(self, *, observer: WorkObserver | None = None) -> None: + self._observer = observer + + async def enqueue(self, connection: AsyncConnection, spec: WorkSpec, /) -> EnqueueResult: + payload = _dump_payload(spec.payload, kind="work", name=spec.logical_key) + now = await database_now(connection) + lane = await _lock_or_create_lane(connection, spec.lane_key) + existing = await _work_for_logical_key(connection, spec.logical_key, for_update=True) + if existing is not None: + _verify_logical_identity(existing, spec) + self._observe_enqueue(spec.kind, created=False) + return EnqueueResult(created=False, work=existing) + + work_id = str(uuid4()) + created = await insert_if_absent( + connection, + WORK_KEYS_TABLE, + { + "logical_key": spec.logical_key, + "work_id": work_id, + "lane_key": spec.lane_key, + "created_at": now, + }, + ) + if not created: + existing = await _work_for_logical_key(connection, spec.logical_key, for_update=True) + if existing is None: + raise InvalidStoredColumnError("logical_key", "a concurrently persisted work key") + _verify_logical_identity(existing, spec) + self._observe_enqueue(spec.kind, created=False) + return EnqueueResult(created=False, work=existing) + + sequence = _positive_integer(lane["next_sequence"], "next_sequence") + available_at = _normalized_datetime(spec.available_at) if spec.available_at is not None else now + await connection.execute( + update(WORK_LANES_TABLE) + .where(WORK_LANES_TABLE.c.lane_key == spec.lane_key) + .values( + next_sequence=sequence + 1, + head_sequence=sequence if lane["head_sequence"] is None else lane["head_sequence"], + ) + ) + await connection.execute( + insert(WORK_ITEMS_TABLE).values( + work_id=work_id, + logical_key=spec.logical_key, + lane_key=spec.lane_key, + lane_sequence=sequence, + kind=spec.kind, + payload_version=spec.payload_version, + scope_id=spec.scope_id, + payload=payload, + status=WorkStatus.QUEUED.value, + available_at=available_at, + lease_owner=None, + lease_fence=0, + lease_expires_at=None, + attempt_count=0, + generation_attempt_count=0, + recovery_generation=0, + max_attempts=spec.max_attempts, + cancel_requested=False, + result_code=None, + result_payload=None, + error_category=None, + error_code=None, + state_version=1, + created_at=now, + updated_at=now, + completed_at=None, + ) + ) + self._observe_enqueue(spec.kind, created=True) + return EnqueueResult(created=True, work=await self.get(connection, work_id)) + + async def claim( + self, + connection: AsyncConnection, + *, + worker_id: str, + supported: Mapping[str, frozenset[int]], + lease_seconds: int, + limit: int, + expired_retry_delay_seconds: int = 0, + ) -> tuple[WorkClaim, ...]: + _validated_text("worker_id", worker_id, 128) + _require_positive("lease_seconds", lease_seconds) + _require_positive("limit", limit) + _require_nonnegative("expired_retry_delay_seconds", expired_retry_delay_seconds) + if not supported: + return () + _validate_supported(supported) + + now = await database_now(connection) + supported_pairs = [ + and_(WORK_ITEMS_TABLE.c.kind == kind, WORK_ITEMS_TABLE.c.payload_version.in_(versions)) + for kind, versions in supported.items() + ] + supported_due = and_( + or_(*supported_pairs), + WORK_ITEMS_TABLE.c.status.in_((WorkStatus.QUEUED.value, WorkStatus.RETRY_WAIT.value)), + WORK_ITEMS_TABLE.c.available_at <= now, + ) + expired = and_( + WORK_ITEMS_TABLE.c.status.in_((WorkStatus.RUNNING.value, WorkStatus.CANCELLING.value)), + WORK_ITEMS_TABLE.c.lease_expires_at <= now, + ) + candidate_ids = ( + await connection.scalars( + select(WORK_ITEMS_TABLE.c.work_id) + .join( + WORK_LANES_TABLE, + and_( + WORK_LANES_TABLE.c.lane_key == WORK_ITEMS_TABLE.c.lane_key, + WORK_LANES_TABLE.c.head_sequence == WORK_ITEMS_TABLE.c.lane_sequence, + ), + ) + .where(or_(supported_due, expired)) + .order_by(WORK_ITEMS_TABLE.c.available_at, WORK_ITEMS_TABLE.c.created_at) + .limit(max(32, limit * 4)) + ) + ).all() + + claims: list[WorkClaim] = [] + for work_id_value in candidate_ids: + if len(claims) >= limit: + break + claim = await self._claim_candidate( + connection, + str(work_id_value), + worker_id=worker_id, + supported=supported, + lease_seconds=lease_seconds, + expired_retry_delay_seconds=expired_retry_delay_seconds, + ) + if claim is not None: + claims.append(claim) + return tuple(claims) + + async def heartbeat( + self, + connection: AsyncConnection, + claim: WorkClaim, + /, + *, + lease_seconds: int, + ) -> StoredWork: + _require_positive("lease_seconds", lease_seconds) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + raise StaleWorkClaimError(claim.work_id) + expires_at = now + timedelta(seconds=lease_seconds) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values(lease_expires_at=expires_at, updated_at=now, state_version=work.state_version + 1) + ) + await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + ) + .values(heartbeat_at=now) + ) + return await self.get(connection, claim.work_id) + + async def record_attempt_trace( + self, + connection: AsyncConnection, + claim: WorkClaim, + /, + *, + trace_id: str, + span_id: str, + ) -> None: + """Bind one non-sensitive span identity to the current fenced attempt.""" + + _require_hex("trace_id", trace_id, 32) + _require_hex("span_id", span_id, 16) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + _verify_claim(work, claim, await database_now(connection)) + result = await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + WORK_ATTEMPTS_TABLE.c.finished_at.is_(None), + ) + .values(trace_id=trace_id, span_id=span_id) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(claim.work_id) + + async def complete( + self, + connection: AsyncConnection, + claim: WorkClaim, + result: WorkResult | None, + /, + *, + commit: WorkCommit | None = None, + ) -> StoredWork: + if result is None and commit is None: + raise InvalidRepositoryArgumentError("result", "result or commit is required") + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + raise StaleWorkClaimError(claim.work_id) + if commit is not None: + committed_result = await commit(connection) + if committed_result is not None: + result = committed_result + if result is None: + raise InvalidRepositoryArgumentError("result", "commit did not produce a result") + result_payload = _dump_payload(result.payload, kind="work-result", name=claim.work_id) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values( + status=WorkStatus.SUCCEEDED.value, + lease_owner=None, + lease_expires_at=None, + result_code=result.code, + result_payload=result_payload, + error_category=None, + error_code=None, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await self._finish_attempt(connection, claim, now=now, outcome="succeeded") + self._observe_attempt(claim, now=now, outcome="succeeded") + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, claim.work_id) + + async def fail( + self, + connection: AsyncConnection, + claim: WorkClaim, + failure: WorkFailure, + /, + *, + retry_delay_seconds: int, + ) -> StoredWork: + _require_nonnegative("retry_delay_seconds", retry_delay_seconds) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + return await self._cancel_running(connection, work, claim=claim, now=now) + retry = failure.retryable and work.generation_attempt_count < work.max_attempts + status = WorkStatus.RETRY_WAIT if retry else WorkStatus.FAILED + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values( + status=status.value, + available_at=now + timedelta(seconds=retry_delay_seconds) if retry else work.available_at, + lease_owner=None, + lease_expires_at=None, + error_category=failure.category, + error_code=failure.code, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + await self._finish_attempt( + connection, + claim, + now=now, + outcome="retry_wait" if retry else "failed", + failure=failure, + ) + self._observe_attempt( + claim, + now=now, + outcome="retry_wait" if retry else "failed", + error_category=failure.category, + ) + return await self.get(connection, claim.work_id) + + async def retry(self, connection: AsyncConnection, work_id: str, /, *, expected_version: int) -> StoredWork: + _require_work_id(work_id) + _require_positive("expected_version", expected_version) + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + await _lock_logical_key(connection, unlocked.logical_key, work_id) + work = await _lock_work(connection, work_id) + _verify_version(work, expected_version) + if work.status is not WorkStatus.FAILED: + raise WorkStateConflictError(work_id, expected_version, work.status, work.state_version) + now = await database_now(connection) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.QUEUED.value, + available_at=now, + generation_attempt_count=0, + recovery_generation=work.recovery_generation + 1, + cancel_requested=False, + error_category=None, + error_code=None, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + return await self.get(connection, work_id) + + async def cancel(self, connection: AsyncConnection, work_id: str, /, *, expected_version: int) -> StoredWork: + _require_work_id(work_id) + _require_positive("expected_version", expected_version) + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + logical_key_locked = await _lock_logical_key_if_present(connection, unlocked.logical_key) + work = await _lock_work(connection, work_id) + _verify_version(work, expected_version) + if work.status in {WorkStatus.CANCELLING, WorkStatus.SUCCEEDED, WorkStatus.CANCELLED}: + raise WorkStateConflictError(work_id, expected_version, work.status, work.state_version) + if not logical_key_locked: + raise StaleWorkClaimError(work_id) + now = await database_now(connection) + if work.status is WorkStatus.RUNNING: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.CANCELLING.value, + cancel_requested=True, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + else: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.CANCELLED.value, + cancel_requested=True, + lease_owner=None, + lease_expires_at=None, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, work_id) + + async def get(self, connection: AsyncConnection, work_id: str, /) -> StoredWork: + _require_work_id(work_id) + row = ( + (await connection.execute(select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == work_id))) + .mappings() + .one_or_none() + ) + if row is None: + raise RepositoryNotFoundError("work", work_id) + return _decode_work(row) + + async def list( + self, + connection: AsyncConnection, + /, + *, + scope_id: str | None = None, + kind: str | None = None, + allowed_kinds: frozenset[str] | None = None, + status: WorkStatus | None = None, + limit: int = 100, + before: tuple[datetime, str] | None = None, + ) -> tuple[StoredWork, ...]: + statement = _work_list_statement( + scope_id=scope_id, + kind=kind, + allowed_kinds=allowed_kinds, + status=status, + before=before, + limit=limit, + ) + if statement is None: + return () + rows = ( + ( + await connection.execute( + statement.order_by(WORK_ITEMS_TABLE.c.created_at.desc(), WORK_ITEMS_TABLE.c.work_id.desc()).limit( + limit + ) + ) + ) + .mappings() + .all() + ) + return tuple(_decode_work(row) for row in rows) + + async def purge_terminal( + self, + connection: AsyncConnection, + /, + *, + completed_before: datetime, + limit: int = 500, + ) -> int: + """Delete one bounded batch of successful or cancelled operation history.""" + + if limit < 1 or limit > 500: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 500") + cutoff = _normalized_datetime(completed_before) + work_ids = tuple( + str(value) + for value in ( + await connection.scalars( + select(WORK_ITEMS_TABLE.c.work_id) + .where( + WORK_ITEMS_TABLE.c.status.in_(_TERMINAL_STATUSES), + WORK_ITEMS_TABLE.c.completed_at.is_not(None), + WORK_ITEMS_TABLE.c.completed_at < cutoff, + ) + .order_by(WORK_ITEMS_TABLE.c.completed_at, WORK_ITEMS_TABLE.c.work_id) + .limit(limit) + ) + ).all() + ) + if not work_ids: + return 0 + await connection.execute(delete(WORK_ATTEMPTS_TABLE).where(WORK_ATTEMPTS_TABLE.c.work_id.in_(work_ids))) + result = await connection.execute( + delete(WORK_ITEMS_TABLE).where( + WORK_ITEMS_TABLE.c.work_id.in_(work_ids), + WORK_ITEMS_TABLE.c.status.in_(_TERMINAL_STATUSES), + WORK_ITEMS_TABLE.c.completed_at < cutoff, + ) + ) + return result.rowcount + + async def unsupported_head_count( + self, + connection: AsyncConnection, + supported: Mapping[str, frozenset[int]], + /, + ) -> int: + """Count visible lane heads that no registered handler can decode.""" + + _validate_supported(supported) + supported_pairs = [ + and_(WORK_ITEMS_TABLE.c.kind == kind, WORK_ITEMS_TABLE.c.payload_version.in_(versions)) + for kind, versions in supported.items() + ] + compatible = or_(*supported_pairs) if supported_pairs else false() + value = await connection.scalar( + select(func.count()) + .select_from(WORK_ITEMS_TABLE) + .join( + WORK_LANES_TABLE, + and_( + WORK_LANES_TABLE.c.lane_key == WORK_ITEMS_TABLE.c.lane_key, + WORK_LANES_TABLE.c.head_sequence == WORK_ITEMS_TABLE.c.lane_sequence, + ), + ) + .where( + WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES), + not_(compatible), + ) + ) + return int(value or 0) + + async def queue_statistics(self, connection: AsyncConnection, /) -> tuple[WorkQueueStatistic, ...]: + """Aggregate non-terminal queue state without high-cardinality labels.""" + + now = await database_now(connection) + rows = ( + await connection.execute( + select( + WORK_ITEMS_TABLE.c.kind, + WORK_ITEMS_TABLE.c.status, + func.count().label("depth"), + func.min(WORK_ITEMS_TABLE.c.created_at).label("oldest_created_at"), + ) + .where(WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES)) + .group_by(WORK_ITEMS_TABLE.c.kind, WORK_ITEMS_TABLE.c.status) + ) + ).mappings() + return tuple( + WorkQueueStatistic( + kind=str(row["kind"]), + status=WorkStatus(str(row["status"])), + depth=int(row["depth"]), + oldest_age_seconds=max( + 0.0, + (now - _stored_datetime(row["oldest_created_at"], "oldest_created_at")).total_seconds(), + ), + ) + for row in rows + ) + + async def _claim_locked( + self, + connection: AsyncConnection, + work: StoredWork, + *, + worker_id: str, + lease_seconds: int, + now: datetime, + ) -> WorkClaim: + previous_trace_id, previous_span_id = await _previous_attempt_trace(connection, work.work_id) + fence = work.lease_fence + 1 + attempt_no = work.attempt_count + 1 + expires_at = now + timedelta(seconds=lease_seconds) + result = await connection.execute( + update(WORK_ITEMS_TABLE) + .where( + WORK_ITEMS_TABLE.c.work_id == work.work_id, + WORK_ITEMS_TABLE.c.state_version == work.state_version, + WORK_ITEMS_TABLE.c.status == work.status.value, + ) + .values( + status=WorkStatus.RUNNING.value, + lease_owner=worker_id, + lease_fence=fence, + lease_expires_at=expires_at, + attempt_count=attempt_no, + generation_attempt_count=work.generation_attempt_count + 1, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(work.work_id) + await connection.execute( + insert(WORK_ATTEMPTS_TABLE).values( + work_id=work.work_id, + attempt_no=attempt_no, + recovery_generation=work.recovery_generation, + owner_id=worker_id, + fence=fence, + started_at=now, + heartbeat_at=now, + finished_at=None, + outcome=None, + error_category=None, + error_code=None, + trace_id=None, + span_id=None, + ) + ) + self._observe_claim(work, now=now) + return WorkClaim( + work_id=work.work_id, + logical_key=work.logical_key, + lane_key=work.lane_key, + lane_sequence=work.lane_sequence, + kind=work.kind, + payload_version=work.payload_version, + scope_id=work.scope_id, + payload=work.payload, + owner_id=worker_id, + fence=fence, + attempt_no=attempt_no, + generation_attempt_no=work.generation_attempt_count + 1, + recovery_generation=work.recovery_generation, + created_at=work.created_at, + claimed_at=now, + previous_trace_id=previous_trace_id, + previous_span_id=previous_span_id, + lease_expires_at=expires_at, + ) + + async def _claim_candidate( + self, + connection: AsyncConnection, + work_id: str, + *, + worker_id: str, + supported: Mapping[str, frozenset[int]], + lease_seconds: int, + expired_retry_delay_seconds: int, + ) -> WorkClaim | None: + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + await _lock_logical_key(connection, unlocked.logical_key, work_id) + work = await _lock_work(connection, work_id) + if not await _is_lane_head(connection, work): + return None + now = await database_now(connection) + if work.status in {WorkStatus.RUNNING, WorkStatus.CANCELLING}: + if work.lease_expires_at is None or work.lease_expires_at > now: + return None + await self._expire_claim( + connection, + work, + now=now, + retry_delay_seconds=expired_retry_delay_seconds, + ) + return None + if work.status not in {WorkStatus.QUEUED, WorkStatus.RETRY_WAIT} or work.available_at > now: + return None + if work.payload_version not in supported.get(work.kind, frozenset()): + return None + return await self._claim_locked( + connection, + work, + worker_id=worker_id, + lease_seconds=lease_seconds, + now=now, + ) + + async def _expire_claim( + self, + connection: AsyncConnection, + work: StoredWork, + *, + now: datetime, + retry_delay_seconds: int, + ) -> None: + claim = _claim_from_work(work) + if work.status is WorkStatus.CANCELLING or work.cancel_requested: + await self._cancel_running(connection, work, claim=claim, now=now) + return + retry = work.generation_attempt_count < work.max_attempts + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work.work_id) + .values( + status=WorkStatus.RETRY_WAIT.value if retry else WorkStatus.FAILED.value, + available_at=now + timedelta(seconds=retry_delay_seconds) if retry else work.available_at, + lease_owner=None, + lease_expires_at=None, + error_category="lease", + error_code="lease_expired", + state_version=work.state_version + 1, + updated_at=now, + ) + ) + await self._finish_attempt( + connection, + claim, + now=now, + outcome="lease_expired_retry" if retry else "lease_expired_failed", + failure=WorkFailure(category="lease", code="lease_expired", retryable=retry), + ) + self._observe_attempt( + claim, + now=now, + outcome="lease_expired_retry" if retry else "lease_expired_failed", + error_category="lease", + ) + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_lease_expiry( + work.kind, + outcome="retry_wait" if retry else "failed", + ) + + async def _cancel_running( + self, + connection: AsyncConnection, + work: StoredWork, + *, + claim: WorkClaim, + now: datetime, + ) -> StoredWork: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work.work_id) + .values( + status=WorkStatus.CANCELLED.value, + lease_owner=None, + lease_expires_at=None, + cancel_requested=True, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await self._finish_attempt(connection, claim, now=now, outcome="cancelled") + self._observe_attempt(claim, now=now, outcome="cancelled") + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, work.work_id) + + async def _finish_attempt( + self, + connection: AsyncConnection, + claim: WorkClaim, + *, + now: datetime, + outcome: str, + failure: WorkFailure | None = None, + ) -> None: + result = await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + WORK_ATTEMPTS_TABLE.c.finished_at.is_(None), + ) + .values( + heartbeat_at=now, + finished_at=now, + outcome=outcome, + error_category=None if failure is None else failure.category, + error_code=None if failure is None else failure.code, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(claim.work_id) + + def _observe_enqueue(self, kind: str, *, created: bool) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_enqueue(kind, created=created) + + def _observe_claim(self, work: StoredWork, *, now: datetime) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_claim( + work.kind, + latency_seconds=max(0.0, (now - work.created_at).total_seconds()), + ) + + def _observe_attempt( + self, + claim: WorkClaim, + *, + now: datetime, + outcome: str, + error_category: str = "none", + ) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_attempt( + claim.kind, + outcome=outcome, + error_category=error_category, + duration_seconds=max(0.0, (now - claim.claimed_at).total_seconds()), + ) + + +async def _lock_or_create_lane(connection: AsyncConnection, lane_key: str) -> Mapping[Any, Any]: + await connection.execute( + update(WORK_LANES_TABLE) + .where(WORK_LANES_TABLE.c.lane_key == lane_key) + .values(next_sequence=WORK_LANES_TABLE.c.next_sequence) + ) + row = await _lane_row(connection, lane_key) + if row is not None: + return row + await insert_if_absent( + connection, + WORK_LANES_TABLE, + {"lane_key": lane_key, "head_sequence": None, "next_sequence": 1}, + ) + row = await _lane_row(connection, lane_key) + if row is None: + raise InvalidStoredColumnError("lane_key", "an initialized work lane") + return row + + +async def _previous_attempt_trace(connection: AsyncConnection, work_id: str) -> tuple[str | None, str | None]: + row = ( + await connection.execute( + select(WORK_ATTEMPTS_TABLE.c.trace_id, WORK_ATTEMPTS_TABLE.c.span_id) + .where(WORK_ATTEMPTS_TABLE.c.work_id == work_id) + .order_by(WORK_ATTEMPTS_TABLE.c.attempt_no.desc()) + .limit(1) + ) + ).one_or_none() + if row is None or (row.trace_id is None and row.span_id is None): + return None, None + if row.trace_id is None or row.span_id is None: + raise InvalidStoredColumnError("attempt trace", "a complete trace/span pair") # noqa: TRY003 + trace_id = str(row.trace_id) + span_id = str(row.span_id) + _require_hex("trace_id", trace_id, 32) + _require_hex("span_id", span_id, 16) + return trace_id, span_id + + +async def _lock_lane(connection: AsyncConnection, lane_key: str) -> None: + await _lock_or_create_lane(connection, lane_key) + + +async def _lane_row(connection: AsyncConnection, lane_key: str) -> Mapping[Any, Any] | None: + return ( + ( + await connection.execute( + select(WORK_LANES_TABLE).where(WORK_LANES_TABLE.c.lane_key == lane_key).with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +async def _lock_logical_key(connection: AsyncConnection, logical_key: str, work_id: str) -> None: + if not await _lock_logical_key_if_present(connection, logical_key): + raise StaleWorkClaimError(work_id) + + +async def _lock_logical_key_if_present(connection: AsyncConnection, logical_key: str) -> bool: + row = ( + await connection.execute( + select(WORK_KEYS_TABLE.c.work_id).where(WORK_KEYS_TABLE.c.logical_key == logical_key).with_for_update() + ) + ).one_or_none() + return row is not None + + +async def _lock_work(connection: AsyncConnection, work_id: str) -> StoredWork: + row = ( + ( + await connection.execute( + select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == work_id).with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + if row is None: + raise RepositoryNotFoundError("work", work_id) + return _decode_work(row) + + +async def _work_for_logical_key( + connection: AsyncConnection, + logical_key: str, + *, + for_update: bool, +) -> StoredWork | None: + statement = ( + select(WORK_ITEMS_TABLE) + .join(WORK_KEYS_TABLE, WORK_KEYS_TABLE.c.work_id == WORK_ITEMS_TABLE.c.work_id) + .where(WORK_KEYS_TABLE.c.logical_key == logical_key) + ) + if for_update: + statement = statement.with_for_update() + row = (await connection.execute(statement)).mappings().one_or_none() + return None if row is None else _decode_work(row) + + +async def _is_lane_head(connection: AsyncConnection, work: StoredWork) -> bool: + head = await connection.scalar( + select(WORK_LANES_TABLE.c.head_sequence).where(WORK_LANES_TABLE.c.lane_key == work.lane_key) + ) + return head == work.lane_sequence + + +async def _release_logical_key(connection: AsyncConnection, work: StoredWork) -> None: + result = await connection.execute( + delete(WORK_KEYS_TABLE).where( + WORK_KEYS_TABLE.c.logical_key == work.logical_key, + WORK_KEYS_TABLE.c.work_id == work.work_id, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(work.work_id) + + +async def _advance_lane(connection: AsyncConnection, work: StoredWork) -> None: + next_sequence = await connection.scalar( + select(WORK_ITEMS_TABLE.c.lane_sequence) + .where( + WORK_ITEMS_TABLE.c.lane_key == work.lane_key, + WORK_ITEMS_TABLE.c.lane_sequence > work.lane_sequence, + WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES), + ) + .order_by(WORK_ITEMS_TABLE.c.lane_sequence) + .limit(1) + ) + await connection.execute( + update(WORK_LANES_TABLE) + .where( + WORK_LANES_TABLE.c.lane_key == work.lane_key, + WORK_LANES_TABLE.c.head_sequence == work.lane_sequence, + ) + .values(head_sequence=next_sequence) + ) + + +def _work_list_statement( + *, + scope_id: str | None, + kind: str | None, + allowed_kinds: frozenset[str] | None, + status: WorkStatus | None, + before: tuple[datetime, str] | None, + limit: int, +) -> Any | None: + if not _validate_work_list(scope_id=scope_id, kind=kind, allowed_kinds=allowed_kinds, limit=limit): + return None + + statement = select(WORK_ITEMS_TABLE) + if scope_id is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.scope_id == scope_id) + if kind is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.kind == kind) + if allowed_kinds is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.kind.in_(allowed_kinds)) + if status is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.status == status.value) + if before is not None: + created_at, work_id = before + created_at = _normalized_datetime(created_at) + _require_work_id(work_id) + statement = statement.where( + or_( + WORK_ITEMS_TABLE.c.created_at < created_at, + and_(WORK_ITEMS_TABLE.c.created_at == created_at, WORK_ITEMS_TABLE.c.work_id < work_id), + ) + ) + return statement + + +def _validate_work_list( + *, + scope_id: str | None, + kind: str | None, + allowed_kinds: frozenset[str] | None, + limit: int, +) -> bool: + if scope_id is not None: + _validated_text("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + if kind is not None: + _validated_text("kind", kind, 128) + if allowed_kinds is not None: + if not allowed_kinds: + return False + for allowed_kind in allowed_kinds: + _validated_text("allowed_kind", allowed_kind, 128) + if limit < 1 or limit > 100: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 100") + return True + + +def _decode_work(row: Mapping[Any, Any]) -> StoredWork: + work_id = str(row["work_id"]) + return StoredWork( + work_id=work_id, + logical_key=str(row["logical_key"]), + lane_key=str(row["lane_key"]), + lane_sequence=_positive_integer(row["lane_sequence"], "lane_sequence"), + kind=str(row["kind"]), + payload_version=_positive_integer(row["payload_version"], "payload_version"), + scope_id=str(row["scope_id"]), + payload=_load_payload(row["payload"], kind="work", name=work_id), + status=WorkStatus(str(row["status"])), + available_at=_stored_datetime(row["available_at"], "available_at"), + lease_owner=None if row["lease_owner"] is None else str(row["lease_owner"]), + lease_fence=_nonnegative_integer(row["lease_fence"], "lease_fence"), + lease_expires_at=_optional_datetime(row["lease_expires_at"], "lease_expires_at"), + attempt_count=_nonnegative_integer(row["attempt_count"], "attempt_count"), + generation_attempt_count=_nonnegative_integer(row["generation_attempt_count"], "generation_attempt_count"), + recovery_generation=_nonnegative_integer(row["recovery_generation"], "recovery_generation"), + max_attempts=_positive_integer(row["max_attempts"], "max_attempts"), + cancel_requested=bool(row["cancel_requested"]), + result_code=None if row["result_code"] is None else str(row["result_code"]), + result_payload=( + None + if row["result_payload"] is None + else _load_payload(row["result_payload"], kind="work-result", name=work_id) + ), + error_category=None if row["error_category"] is None else str(row["error_category"]), + error_code=None if row["error_code"] is None else str(row["error_code"]), + state_version=_positive_integer(row["state_version"], "state_version"), + created_at=_stored_datetime(row["created_at"], "created_at"), + updated_at=_stored_datetime(row["updated_at"], "updated_at"), + completed_at=_optional_datetime(row["completed_at"], "completed_at"), + ) + + +def _verify_logical_identity(work: StoredWork, spec: WorkSpec) -> None: + if work.kind != spec.kind or work.scope_id != spec.scope_id or work.lane_key != spec.lane_key: + raise RepositoryError( # noqa: TRY003 + f"logical work key {spec.logical_key!r} is bound to a different work identity" + ) + + +def _verify_claim(work: StoredWork, claim: WorkClaim, now: datetime) -> None: + if ( + work.status not in {WorkStatus.RUNNING, WorkStatus.CANCELLING} + or work.lease_owner != claim.owner_id + or work.lease_fence != claim.fence + or work.attempt_count != claim.attempt_no + or work.lease_expires_at is None + or work.lease_expires_at <= now + ): + raise StaleWorkClaimError(claim.work_id) + + +def _verify_version(work: StoredWork, expected_version: int) -> None: + if work.state_version != expected_version: + raise WorkStateConflictError(work.work_id, expected_version, work.status, work.state_version) + + +def _claim_from_work(work: StoredWork) -> WorkClaim: + if work.lease_owner is None or work.lease_expires_at is None or work.attempt_count < 1: + raise InvalidStoredColumnError("work lease", "a complete active lease") # noqa: TRY003 + return WorkClaim( + work_id=work.work_id, + logical_key=work.logical_key, + lane_key=work.lane_key, + lane_sequence=work.lane_sequence, + kind=work.kind, + payload_version=work.payload_version, + scope_id=work.scope_id, + payload=work.payload, + owner_id=work.lease_owner, + fence=work.lease_fence, + attempt_no=work.attempt_count, + generation_attempt_no=work.generation_attempt_count, + recovery_generation=work.recovery_generation, + created_at=work.created_at, + claimed_at=work.updated_at, + previous_trace_id=None, + previous_span_id=None, + lease_expires_at=work.lease_expires_at, + ) + + +def _dump_payload(payload: dict[str, JsonValue], *, kind: str, name: str) -> bytes: + try: + encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + except (TypeError, ValueError) as error: + raise InvalidStoredPayloadError(kind, name, "value is not JSON serializable") from error + if len(encoded) > _MAX_PAYLOAD_BYTES: + raise InvalidRepositoryArgumentError("payload", f"must not exceed {_MAX_PAYLOAD_BYTES} encoded bytes") + return encoded + + +def _load_payload(payload: object, *, kind: str, name: str) -> dict[str, JsonValue]: + try: + decoded = json.loads(stored_bytes(payload, column="payload")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise InvalidStoredPayloadError(kind, name, "payload is not valid JSON") from error + if not isinstance(decoded, dict) or any(not isinstance(key, str) for key in decoded): + raise InvalidStoredPayloadError(kind, name, "payload is not an object") + return decoded + + +def _stored_datetime(value: object, column: str) -> datetime: + if not isinstance(value, datetime): + raise InvalidStoredColumnError(column, "a datetime") + return _normalized_datetime(value) + + +def _optional_datetime(value: object, column: str) -> datetime | None: + return None if value is None else _stored_datetime(value, column) + + +def _normalized_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +def _nonnegative_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidStoredColumnError(column, "a non-negative integer") + return value + + +def _require_positive(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _require_nonnegative(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidRepositoryArgumentError(field, "must be a non-negative integer") + + +def _require_work_id(value: str) -> None: + _validated_text("work_id", value, 36) + + +def _require_hex(field: str, value: str, length: int) -> None: + if len(value) != length or any(character not in "0123456789abcdef" for character in value): + raise InvalidRepositoryArgumentError(field, f"must be {length} lowercase hexadecimal characters") + + +def _validate_supported(supported: Mapping[str, frozenset[int]]) -> None: + for kind, versions in supported.items(): + _validated_text("supported kind", kind, 128) + if not versions or any(version < 1 for version in versions): + raise InvalidRepositoryArgumentError("supported", "each kind must have positive payload versions") + + +def _validated_text(field: str, value: str, maximum: int) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{field} must be a non-empty trimmed string") # noqa: TRY003 + if len(value) > maximum: + raise ValueError(f"{field} must not exceed {maximum} characters") # noqa: TRY003 + return value + + +__all__ = [ + "EnqueueResult", + "StaleWorkClaimError", + "StoredWork", + "WorkClaim", + "WorkCommit", + "WorkFailure", + "WorkQueueStatistic", + "WorkRepository", + "WorkResult", + "WorkSpec", + "WorkStateConflictError", + "WorkStatus", +] diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index dd31c8272d..b1715d00da 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -4454,6 +4454,9 @@ def __init__( self.archive = archive_service self.handoff_report: HandoffReportApplication | None = None self.artifact_processing_supervisor: ArtifactProcessingSupervisors | None = None + # Composition assigns this after the ledger components start; it stays a + # plain attribute so the runtime does not depend on the composition module. + self.work_background: Any | None = None source_ids = scope_ids if source_scope_ids is None else source_scope_ids self.processor = None if source_ids is None else ScheduledSourceProcessor(self, source_ids) self.experience_processor = ( @@ -4463,6 +4466,12 @@ def __init__( async def __aenter__(self) -> BuiltinRuntime: return self + @property + def operations(self) -> Any | None: + """Return the durable Work Ledger operations composed for this process.""" + + return None if self.work_background is None else self.work_background.operations + @property def primary_database(self) -> AsyncDatabase: """Borrow the primary database while the composed Runtime remains open.""" diff --git a/src/powercontext/builtin/runtime/atomic_memory_processing.py b/src/powercontext/builtin/runtime/atomic_memory_processing.py index a50c538152..623fe77cfb 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_processing.py +++ b/src/powercontext/builtin/runtime/atomic_memory_processing.py @@ -37,7 +37,7 @@ project_atomic_memory_evidence, require_atomic_memory_pipeline, ) -from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryContent, PreparedAtomicMemory from powercontext.builtin.artifacts.atomic_memory.reconciliation import ( ATOMIC_MEMORY_RECONCILIATION_INSTRUCTIONS, AtomicMemoryArtifactEvidence, @@ -73,7 +73,7 @@ RepositoryNotFoundError, ) from powercontext.builtin.persistence.memory_windows import MemorySourceWindowRepository -from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.sources import SourceRepository, StoredSource from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, SOURCE_JOURNAL_HEADS_TABLE, @@ -130,6 +130,170 @@ def from_runtime(cls, runtime: RuntimeConfig) -> AtomicMemoryProcessingConfig: ) +class PreparedMemoryWindow: + """A prepared window whose publication can join the caller's transaction. + + ``publish`` runs the guarded cursor/artifact publication on the caller's + open transaction and is the only place that advances the Source cursor. + The Work Ledger handler runs it inside the Work completion transaction, + so a cancellation or lease takeover accepted before publication discards + the prepared result together with the rolled-back transaction. + """ + + def __init__( + self, + processor: AtomicMemorySourceWindowProcessor, + *, + scope_id: str, + context: ArtifactSearchExecutionContext | None, + cursor, + transition, + action, + window: tuple[StoredSource, ...], + eligible: tuple[StoredSource, ...], + consulted_artifacts: tuple[ArtifactRef, ...], + workset: AtomicMemoryWindowWorkset, + prepared: tuple[PreparedAtomicMemory, ...], + decision, + observation, + high_watermark: int, + ) -> None: + self.processor = processor + self.scope_id = scope_id + self.context = context + self.cursor = cursor + self.transition = transition + self.action = action + self.window = window + self.eligible = eligible + self.consulted_artifacts = consulted_artifacts + self.workset = workset + self.prepared = prepared + self.decision = decision + self.observation = observation + self.high_watermark = high_watermark + self.finalized = None + + def result(self) -> MemoryFlushResult: + decision = self.decision + return _flush_result( + self.action.after, + self.action.through, + self.high_watermark, + len(self.eligible), + held=decision is not None and decision.verdict is MemoryWriteVerdict.HOLD, + hold_codes=() if decision is None or decision.code is None else (decision.code.value,), + ) + + def emit_finalized(self) -> None: + if self.finalized is not None: + emit_decision_observation(self.finalized) + + def emit_failed(self) -> None: + self.processor._emit_failed_observation(self.observation) + + async def publish(self, connection, processing) -> MemoryFlushResult: + processor = self.processor + application = processor.application + scope_id = self.scope_id + context = self.context + action = self.action + cursor = self.cursor + writes = tuple(write for item in self.prepared for write in item.plan.writes) + with processor._stage( + "memory.commit", + { + "powercontext.memory.commit.memory_changed": bool(writes), + "powercontext.memory.commit.artifact_revision_count": sum(write.draft is not None for write in writes), + }, + ): + # Every Atomic write uses journal -> heads. Source capture's journal + # reservation prevents the exact window changing under validation. + await connection.execute( + update(SOURCE_JOURNAL_HEADS_TABLE) + .where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) + .values(position=SOURCE_JOURNAL_HEADS_TABLE.c.position) + ) + if processing is not None: + await processing.guard(connection) + actual = await processor.cursors.load(connection, scope_id, SOURCE_WINDOW_TRIGGER_NAME, for_update=True) + actual_generation = None if actual is None else actual.generation + expected_generation = None if cursor is None else cursor.generation + if ( + 0 if actual is None else actual.cursor.sequence + ) != action.after or actual_generation != expected_generation: + raise GenerationConflictError(SOURCE_WINDOW_TRIGGER_NAME, expected_generation, actual_generation) + current_window = await processor.sources.list_window( + connection, scope_id, after=action.after, through=action.through + ) + processor._require_window(current_window, action.after, action.through) + if current_window != self.window: + _conflict("Source window changed during preparation") + if tuple(item for item in current_window if is_generation_eligible(item.value)) != self.eligible: + _conflict("Source generation eligibility changed during preparation") + local_policy = False + if context is not None and context.access is not None: + from powercontext.server.authz import BuiltinAuthorizationProvider + from powercontext.server.authz.casbin import CasbinAuthorizationProvider + + local_policy = ( + type(context.access.provider) in {BuiltinAuthorizationProvider, CasbinAuthorizationProvider} + and context.access.supports_read_snapshot(connection.engine) + and context.access.supports_write_transaction(connection.engine) + ) + async with processor._prefetch_artifact_policy( + connection, + scope_id, + context, + self.consulted_artifacts if local_policy else (), + include_scope=True, + ): + await application.security.authorize(connection, scope_id, context, "read") + await application.security.authorize(connection, scope_id, context, "create") + await application.security.authorize_sources( + connection, scope_id, context, tuple(item.ref for item in self.eligible) + ) + for ref in self.consulted_artifacts: + await processor._authorize_artifact(connection, scope_id, context, ref) + results = () + if self.decision is None or self.decision.verdict is not MemoryWriteVerdict.HOLD: + source_read_set = await processor.sources.preload( + connection, + scope_id, + tuple(source for write in writes if write.draft is not None for source in write.draft.sources), + known=current_window, + ) + results = await application.service.commit_window( + connection, + scope_id, + self.prepared, + context, + read_set=tuple(self.workset.reads.values()), + source_read_set=source_read_set, + ) + self.finalized = await processor._finalize_observation( + connection, + self.observation, + final_action=( + "memory_write_hold" + if self.decision is not None and self.decision.verdict is MemoryWriteVerdict.HOLD + else "memory_write_applied" + ), + subject_refs=tuple(_artifact_reference(result.primary.ref) for result in results), + ) + await processor.cursors.save( + connection, + scope_id, + SOURCE_WINDOW_TRIGGER_NAME, + self.transition.state, + expected_generation=expected_generation, + ) + await processor.windows.clear_consumed(connection, scope_id, action.through) + if processing is not None: + await processing.complete(connection, remaining_work=action.through < self.high_watermark) + return self.result() + + class AtomicMemorySourceWindowProcessor: """Keep the existing cursor/binding while replacing collection Memory extraction.""" @@ -183,6 +347,40 @@ async def flush( return await self._retry_window(scope_id, limit, processing, selected_context, span) return await self._retry_window(scope_id, limit, processing, selected_context, span) + async def prepare_window( + self, + scope_id: str, + limit: int, + *, + context: ArtifactSearchExecutionContext | None = None, + window: tuple[int, int] | None = None, + ) -> PreparedMemoryWindow | MemoryFlushResult: + """Prepare one Source window's decisions without publishing. + + ``window`` pins the logical input to persisted boundaries ``(after, through)`` + instead of re-reading the live high watermark, so a retried or delayed claim + processes exactly the rows its Work record describes. + + Returns a ``PreparedMemoryWindow`` whose ``publish`` commits the Memory + artifacts and advances the cursor on the caller's open transaction, or a + ``MemoryFlushResult`` when the trigger has no window to consume. The + caller owns conflict recovery: unlike ``flush``, a conflicting + publication raises instead of retrying internally. + """ + selected_context = context + async with deferred_decision_audit(selected_context): + with self._stage("memory.flush", {"powercontext.memory.flush.source_count": 0}) as span: + if self.prompts is not None: + legacy = await self.prompts.read_configuration(scope_id, "memory.extract") + if legacy.mode == "custom": + raise PromptError("legacy_memory_prompt_unsupported", during_inference=True) + async with ( + self.prompts.bind(scope_id, "atomic_memory.extract"), + self.prompts.bind(scope_id, "atomic_memory.reconcile"), + ): + return await self._prepare_window(scope_id, limit, None, selected_context, span, window) + return await self._prepare_window(scope_id, limit, None, selected_context, span, window) + async def _retry_window(self, scope_id, limit, processing, context, span) -> MemoryFlushResult: for attempt in range(3): try: @@ -198,6 +396,30 @@ async def _retry_window(self, scope_id, limit, processing, context, span) -> Mem raise AssertionError("unreachable") async def _flush_once(self, scope_id, limit, processing, context, span) -> MemoryFlushResult: + prepared = await self._prepare_window(scope_id, limit, processing, context, span) + if isinstance(prepared, MemoryFlushResult): + return prepared + try: + async with self.application.database.transaction() as connection: + result = await prepared.publish(connection, processing) + except Exception: + prepared.emit_failed() + raise + prepared.emit_finalized() + return result + + async def _prepare_window(self, scope_id, limit, processing, context, span, window=None): + """Select one Source window and prepare its decisions without publishing. + + ``window`` pins the logical input to the persisted ``(after, through)`` + boundaries. Preparing the live window instead would silently absorb Sources + that arrived after the Work record was written, so the claim would process a + different input than the one it reports. + + Returns a ``PreparedMemoryWindow`` whose ``publish`` commits the Memory + artifacts and advances the cursor on the caller's transaction, or a + ``MemoryFlushResult`` when the trigger has no window to consume. + """ application = self.application trigger = SourceWindowTrigger() async with application.database.transaction() as connection: @@ -207,7 +429,15 @@ async def _flush_once(self, scope_id, limit, processing, context, span) -> Memor await application.security.authorize(connection, scope_id, context, "create") cursor = await self.cursors.load(connection, scope_id, SOURCE_WINDOW_TRIGGER_NAME) state = trigger.initial_state() if cursor is None else cursor.cursor - high_watermark, limit = await self.windows.signal(connection, scope_id, state.sequence, limit) + if window is None: + high_watermark, limit = await self.windows.signal(connection, scope_id, state.sequence, limit) + else: + after, through = window + if state.sequence != after or through <= after: + raise AtomicMemoryConflictError("The Source cursor moved before window preparation") # noqa: TRY003 + # Express the persisted window as the trigger's own bounded signal: + # sequence=through with limit=through-after activates exactly this span. + high_watermark, limit = through, through - after signal = SourceHighWatermark(sequence=high_watermark, limit=limit) transition = trigger.activate(signal, state) if not transition.actions: @@ -244,35 +474,21 @@ async def _flush_once(self, scope_id, limit, processing, context, span) -> Memor if reducible and action.through - action.after > 1: await self._reduce_window(scope_id, context, processing, cursor, state, action, high_watermark) raise - try: - finalized = await self._publish( - scope_id, - context, - processing, - cursor, - transition, - action, - window, - eligible, - consulted_artifacts, - workset, - prepared, - decision, - observation, - high_watermark, - ) - except Exception: - self._emit_failed_observation(observation) - raise - if finalized is not None: - emit_decision_observation(finalized) - return self._result( - action.after, - action.through, - high_watermark, - len(eligible), - held=decision is not None and decision.verdict is MemoryWriteVerdict.HOLD, - hold_codes=() if decision is None or decision.code is None else (decision.code.value,), + return PreparedMemoryWindow( + self, + scope_id=scope_id, + context=context, + cursor=cursor, + transition=transition, + action=action, + window=window, + eligible=eligible, + consulted_artifacts=consulted_artifacts, + workset=workset, + prepared=prepared, + decision=decision, + observation=observation, + high_watermark=high_watermark, ) async def _reduce_window(self, scope_id, context, processing, cursor, state, action, high_watermark): @@ -339,122 +555,6 @@ async def _inspect_decisions(self, scope_id, workset, context): decision, observation = await self._assess_write_gate(scope_id, workset, prepared) return prepared, decision, observation - async def _publish( - self, - scope_id, - context, - processing, - cursor, - transition, - action, - window, - eligible, - consulted_artifacts, - workset, - prepared, - decision, - observation, - high_watermark, - ): - application = self.application - writes = tuple(write for item in prepared for write in item.plan.writes) - finalized = None - with self._stage( - "memory.commit", - { - "powercontext.memory.commit.memory_changed": bool(writes), - "powercontext.memory.commit.artifact_revision_count": sum(write.draft is not None for write in writes), - }, - ): - async with application.database.transaction() as connection: - # Every Atomic write uses journal -> heads. Source capture's journal - # reservation prevents the exact window changing under validation. - await connection.execute( - update(SOURCE_JOURNAL_HEADS_TABLE) - .where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) - .values(position=SOURCE_JOURNAL_HEADS_TABLE.c.position) - ) - if processing is not None: - await processing.guard(connection) - actual = await self.cursors.load(connection, scope_id, SOURCE_WINDOW_TRIGGER_NAME, for_update=True) - actual_generation = None if actual is None else actual.generation - expected_generation = None if cursor is None else cursor.generation - if ( - 0 if actual is None else actual.cursor.sequence - ) != action.after or actual_generation != expected_generation: - raise GenerationConflictError(SOURCE_WINDOW_TRIGGER_NAME, expected_generation, actual_generation) - current_window = await self.sources.list_window( - connection, scope_id, after=action.after, through=action.through - ) - self._require_window(current_window, action.after, action.through) - if current_window != window: - _conflict("Source window changed during preparation") - if tuple(item for item in current_window if is_generation_eligible(item.value)) != eligible: - _conflict("Source generation eligibility changed during preparation") - local_policy = False - if context is not None and context.access is not None: - from powercontext.server.authz import BuiltinAuthorizationProvider - from powercontext.server.authz.casbin import CasbinAuthorizationProvider - - local_policy = ( - type(context.access.provider) in {BuiltinAuthorizationProvider, CasbinAuthorizationProvider} - and context.access.supports_read_snapshot(connection.engine) - and context.access.supports_write_transaction(connection.engine) - ) - async with self._prefetch_artifact_policy( - connection, - scope_id, - context, - consulted_artifacts if local_policy else (), - include_scope=True, - ): - await application.security.authorize(connection, scope_id, context, "read") - await application.security.authorize(connection, scope_id, context, "create") - await application.security.authorize_sources( - connection, scope_id, context, tuple(item.ref for item in eligible) - ) - for ref in consulted_artifacts: - await self._authorize_artifact(connection, scope_id, context, ref) - results = () - if decision is None or decision.verdict is not MemoryWriteVerdict.HOLD: - source_read_set = await self.sources.preload( - connection, - scope_id, - tuple( - source for write in writes if write.draft is not None for source in write.draft.sources - ), - known=current_window, - ) - results = await application.service.commit_window( - connection, - scope_id, - prepared, - context, - read_set=tuple(workset.reads.values()), - source_read_set=source_read_set, - ) - finalized = await self._finalize_observation( - connection, - observation, - final_action=( - "memory_write_hold" - if decision is not None and decision.verdict is MemoryWriteVerdict.HOLD - else "memory_write_applied" - ), - subject_refs=tuple(_artifact_reference(result.primary.ref) for result in results), - ) - await self.cursors.save( - connection, - scope_id, - SOURCE_WINDOW_TRIGGER_NAME, - transition.state, - expected_generation=expected_generation, - ) - await self.windows.clear_consumed(connection, scope_id, action.through) - if processing is not None: - await processing.complete(connection, remaining_work=action.through < high_watermark) - return finalized - async def _prepare(self, scope_id, eligible, pipeline, context): workset = AtomicMemoryWindowWorkset() artifacts: dict[tuple[str, str, int], ArtifactRef] = {} @@ -946,16 +1046,7 @@ def _emit_failed_observation(observation): @staticmethod def _result(after, through, high_watermark, count, *, held=False, hold_codes=()): - return MemoryFlushResult( - previous_cursor=after, - current_cursor=through, - high_watermark=high_watermark, - source_count=count, - memory_ref=None, - remaining_work=through < high_watermark, - held_count=1 if held else 0, - hold_codes=hold_codes, - ) + return _flush_result(after, through, high_watermark, count, held=held, hold_codes=hold_codes) @dataclass(slots=True) @@ -994,6 +1085,19 @@ def _window_operation_id(prepared) -> str: return f"atomic-memory-window:{sha256(identities.encode('utf-8')).hexdigest()}" +def _flush_result(after, through, high_watermark, count, *, held=False, hold_codes=()): + return MemoryFlushResult( + previous_cursor=after, + current_cursor=through, + high_watermark=high_watermark, + source_count=count, + memory_ref=None, + remaining_work=through < high_watermark, + held_count=1 if held else 0, + hold_codes=hold_codes, + ) + + def _conflict(detail: str) -> NoReturn: raise AtomicMemoryConflictError(detail) diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index db9aaa72e9..00ceac0efc 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import json import logging import os @@ -219,6 +220,7 @@ from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService from powercontext.builtin.runtime.topic_memory_scope import topic_memory_processing_block from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher +from powercontext.builtin.runtime.work_observability import WorkObserver from powercontext.builtin.runtime.worker_observability import worker_model_usage_collector from powercontext.builtin.sources import ( BUILTIN_SOURCE_REGISTRY, @@ -548,6 +550,7 @@ async def open_builtin_runtime( artifact_processing_bindings: Sequence[ArtifactProcessingBinding] = (), worker_security: dict[str, Any] | None = None, processing_observer: Callable[[str, str, float], None] | None = None, + work_observer: WorkObserver | None = None, worker_initializer: Callable[[], None] | WorkerInitializerPreparation | None = None, worker_entrypoint_wrapper: Callable[[WorkerEntrypoint], WorkerEntrypoint] | None = None, scheduled_source_runner: ScheduledSourceRunner | None = None, @@ -921,7 +924,7 @@ async def run_profile(scope_id, high): _open_artifact_processing_supervisor( config, contexts, - processing_bindings, + _supervisor_bindings(config, processing_bindings), tracing=tracing, dispatch_processing=dispatch_processing, recovery=_archive_recovery, @@ -929,6 +932,9 @@ async def run_profile(scope_id, high): ) ) contexts.profiles.operation_context = runtime._operation + runtime.work_background = await resources.enter_async_context( + _open_work_background(config, contexts, tracing=tracing, observer=work_observer) + ) if config.handoff_report.enabled: runtime.handoff_report = HandoffReportApplication( contexts.scopes, @@ -1013,7 +1019,10 @@ def _artifact_processing_bindings( # noqa: C901 - validate and assemble one reg "handoff": None, "prompt": None, } - for family, schedule in automatic.items(): + # A distributed deployment never runs the process-local supervisor, so a + # schedule there only paces the Work Ledger discoverer and cannot require a + # local candidate pipeline. + for family, schedule in automatic.items() if config.deployment.mode != "distributed" else (): if schedule is not None and family not in declared | registered: if ( family == "topic-memory" @@ -1161,6 +1170,343 @@ async def _open_artifact_processing_supervisor( yield ArtifactProcessingSupervisors(tuple(supervisors)) +class _RateLimitWindowMaintenance: + """Delete bounded batches of expired fixed-window counters on an interval. + + Single-node installs start no ledger scheduler or Worker, so the + ``OperationMaintenanceHandler`` purge would never run and every active + principal would accumulate one window row per interval indefinitely. This + loop keeps the table bounded independently of distributed ledger execution. + """ + + def __init__( + self, + *, + database: Any, + interval_seconds: float, + batch_size: int, + repository: Any = None, + ) -> None: + from powercontext.builtin.persistence.rate_limit import RateLimitRepository + + self._database = database + self._interval_seconds = interval_seconds + self._batch_size = batch_size + self._repository = RateLimitRepository() if repository is None else repository + self._stop_requested = asyncio.Event() + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + self._task = asyncio.create_task(self._run(), name="powercontext-rate-limit-maintenance") + + async def stop(self) -> None: + self._stop_requested.set() + if self._task is not None: + await asyncio.gather(self._task, return_exceptions=True) + self._task = None + + async def _run(self) -> None: + while not self._stop_requested.is_set(): + try: + await self.purge_once() + except Exception as error: + log_safely( + logger, + logging.WARNING, + f"rate limit window maintenance failed: {error}", + exc_info=error, + extra={"event": "rate-limit.window.maintenance-failed"}, + ) + try: + await asyncio.wait_for(self._stop_requested.wait(), timeout=self._interval_seconds) + except TimeoutError: + continue + + async def purge_once(self) -> int: + """Delete one bounded batch of expired counters and return the row count.""" + + async with self._database.transaction() as connection: + return await self._repository.purge_expired(connection, limit=self._batch_size) + + +class WorkBackground: + """Role-specific durable Work Ledger services attached to one runtime. + + ``operations`` is always present so the HTTP layer can inspect, cancel and + retry history regardless of role; the execution components stay ``None`` when + the role does not run them or no background schedule is configured. + """ + + __slots__ = ("membership", "operations", "rate_limit_maintenance", "scheduler", "worker") + + def __init__( + self, + *, + operations: Any, + membership: Any = None, + scheduler: Any = None, + worker: Any = None, + rate_limit_maintenance: Any = None, + ) -> None: + self.operations = operations + self.membership = membership + self.scheduler = scheduler + self.worker = worker + self.rate_limit_maintenance = rate_limit_maintenance + + +def _work_background_requested(config: BuiltinConfig) -> bool: + """True when this deployment opted into ledger-driven background execution. + + The ledger and the APScheduler-driven artifact supervisors both drive the + same source windows, so exactly one of them may own a Family. RFC 1430 makes + that ordering explicit: a bridge release moves ``single_node/all`` onto the + ledger only once the APScheduler path has fully retired. Until then a + ``single_node`` install keeps the supervisors it already has, and the ledger + starts only for an explicit ``distributed`` deployment, where the roles are + split across processes and no single process can own every window. + """ + + return config.deployment.mode == "distributed" + + +def _ledger_families(config: BuiltinConfig) -> frozenset[str]: + """Families the durable Work Ledger drives, or empty when it is inactive. + + The supervisor must skip these Families, otherwise both paths claim the same + window: the ledger advances the cursor and the supervisor's next tick sees an + empty window and records a noop instead of doing the work. + """ + + if not _work_background_requested(config): + return frozenset() + runtime = config.runtime + families = set() + if runtime.memory_schedule_seconds is not None: + families.add("memory") + if runtime.experience_schedule_seconds is not None: + families.add("experience") + if runtime.profile_schedule_enabled: + families.add("profile") + return frozenset(families) + + +def _work_role_components(role: str) -> tuple[bool, bool]: + """Report which ledger components a deployment role runs: (scheduler, worker). + + ``api`` serves requests only, ``scheduler`` only discovers work, ``worker`` + only claims it, and ``all`` does both. + """ + + return role in ("all", "scheduler"), role in ("all", "worker") + + +def _supervisor_bindings( + config: BuiltinConfig, bindings: Sequence[ArtifactProcessingBinding] +) -> tuple[ArtifactProcessingBinding, ...]: + """Drop the Families the ledger owns so the two paths never share a window.""" + + owned = _ledger_families(config) + if not owned: + return tuple(bindings) + return tuple(binding for binding in bindings if binding.artifact_family not in owned) + + +def _work_discoverers(config: BuiltinConfig, contexts: RelationalContexts) -> tuple[Any, ...]: + from powercontext.builtin.runtime.work_handlers import ( + CURRENT_WORK_PAYLOAD_VERSION, + ExperienceWorkDiscoverer, + MemoryWorkDiscoverer, + OperationMaintenanceDiscoverer, + ProfileWorkDiscoverer, + ) + + runtime = config.runtime + maximum = config.worker.max_attempts + discoverers: list[Any] = [] + if runtime.memory_schedule_seconds is not None: + discoverers.append( + MemoryWorkDiscoverer( + contexts, + interval_seconds=runtime.memory_schedule_seconds, + window_limit=runtime.source_window_limit, + max_attempts=maximum, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + ) + ) + if runtime.experience_schedule_seconds is not None: + discoverers.append( + ExperienceWorkDiscoverer( + contexts, + interval_seconds=runtime.experience_schedule_seconds, + window_limit=runtime.source_window_limit, + max_attempts=maximum, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + ) + ) + if runtime.profile_schedule_enabled: + discoverers.append( + ProfileWorkDiscoverer( + contexts, + cron=runtime.profile_cron, + timezone=runtime.profile_timezone, + max_attempts=maximum, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + ) + ) + discoverers.append( + OperationMaintenanceDiscoverer( + interval_seconds=config.operations.cleanup_interval_seconds, + max_attempts=maximum, + ) + ) + return tuple(discoverers) + + +def _work_handlers(config: BuiltinConfig, contexts: RelationalContexts) -> tuple[Any, ...]: + from powercontext.builtin.runtime.work_handlers import ( + ExperienceWorkHandler, + MemoryWorkHandler, + OperationMaintenanceHandler, + ProfileWorkHandler, + ) + + runtime = config.runtime + return ( + MemoryWorkHandler(contexts, window_limit=runtime.source_window_limit), + ExperienceWorkHandler(contexts), + ProfileWorkHandler( + contexts.profiles, + max_concurrency=runtime.profile_max_concurrency, + ), + OperationMaintenanceHandler( + retention_days=config.operations.retention_days, + batch_size=config.operations.cleanup_batch_size, + ), + ) + + +def _build_version() -> str: + from importlib.metadata import PackageNotFoundError, version + + try: + return version("powercontext") + except PackageNotFoundError: # pragma: no cover - source checkouts without installed metadata + return "unknown" + + +@asynccontextmanager +async def _open_work_background( + config: BuiltinConfig, + contexts: RelationalContexts, + *, + tracing: RuntimeTracing | None = None, + observer: WorkObserver | None = None, +) -> AsyncIterator[WorkBackground]: + """Start the durable Work Ledger components this role is responsible for. + + The ledger drives Families that a supervisor no longer owns, so it is opened + only for an explicit ``distributed`` deployment (see + ``_work_background_requested``). A ``single_node`` install keeps its + APScheduler supervisors and only gets the ``OperationManager`` facade, which + is what an unconfigured install has always exposed. A distributed deployment + splits the roles: ``api`` never claims or scans work, ``scheduler`` only + scans, ``worker`` only claims, and ``all`` does both. + """ + + from powercontext.builtin.runtime.durable_scheduler import DurableScheduler + from powercontext.builtin.runtime.membership import RuntimeMembership + from powercontext.builtin.runtime.operations import OperationManager + from powercontext.builtin.runtime.work_handlers import CURRENT_WORK_PAYLOAD_VERSION + from powercontext.builtin.runtime.worker import DurableWorker + + if not _work_background_requested(config): + maintenance = _RateLimitWindowMaintenance( + database=contexts.database, + interval_seconds=config.operations.cleanup_interval_seconds, + batch_size=config.operations.cleanup_batch_size, + ) + maintenance.start() + try: + yield WorkBackground( + operations=OperationManager( + contexts=contexts, + operations=config.operations, + worker=config.worker, + local_worker=None, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + observer=observer, + ), + rate_limit_maintenance=maintenance, + ) + finally: + await maintenance.stop() + return + + role = config.deployment.role + runs_scheduler, runs_worker = _work_role_components(role) + async with AsyncExitStack() as stack: + membership = RuntimeMembership( + database=contexts.database, + deployment=config.deployment, + coordination=config.coordination, + build_version=_build_version(), + observer=observer, + ) + await membership.start() + stack.push_async_callback(membership.stop) + scheduler = ( + DurableScheduler( + database=contexts.database, + scheduler_id=f"{membership.instance_id}:scheduler", + discoverers=_work_discoverers(config, contexts), + config=config.coordination, + observer=observer, + tracing=tracing, + ) + if runs_scheduler + else None + ) + worker = ( + DurableWorker( + database=contexts.database, + worker_id=f"{membership.instance_id}:worker", + handlers=_work_handlers(config, contexts), + config=config.worker, + observer=observer, + tracing=tracing, + ) + if runs_worker + else None + ) + components = [component for component in (membership, scheduler, worker) if component is not None] + tasks = [ + asyncio.create_task(component.run(), name=f"powercontext-work-{index}") + for index, component in enumerate(components) + ] + try: + yield WorkBackground( + operations=OperationManager( + contexts=contexts, + operations=config.operations, + worker=config.worker, + local_worker=worker, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + observer=observer, + ), + membership=membership, + scheduler=scheduler, + worker=worker, + ) + finally: + # Drain the worker before leadership and membership disappear so an + # in-flight claim still commits under a valid lease. + for component in (worker, scheduler, membership): + if component is not None: + await component.stop() + await asyncio.gather(*tasks, return_exceptions=True) + + def compose_artifact_search(contexts, embedding_model=None, *, observer=None): """Use the same Family-owned Searchers in Server, SDK and generation Workers.""" from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher @@ -2016,9 +2362,21 @@ async def preflight_builtin_runtime( ) if config.inference.embedding_model is not None: await _embedding_models(config.inference, resources, None) - if config.runtime.schedule_seconds is not None and config.inference.generation_model is None: + # A distributed deployment runs no local schedule: the Work Ledger + # paces discovery and the Worker executes elsewhere, so a persisted + # interval there does not require a model in this process. + executes_scheduled_work_locally = config.deployment.mode != "distributed" + if ( + executes_scheduled_work_locally + and config.runtime.schedule_seconds is not None + and config.inference.generation_model is None + ): raise BuiltinConfigurationError("scheduled-pipeline") - if config.runtime.experience_schedule_seconds is not None and config.inference.generation_model is None: + if ( + executes_scheduled_work_locally + and config.runtime.experience_schedule_seconds is not None + and config.inference.generation_model is None + ): raise BuiltinConfigurationError("scheduled-experience-pipeline") if config.runtime.memory_rerank_enabled and ( config.inference.generation_model is None and config.inference.rerank_model is None diff --git a/src/powercontext/builtin/runtime/config.py b/src/powercontext/builtin/runtime/config.py index 5e2dbead92..8c2efeb3d7 100644 --- a/src/powercontext/builtin/runtime/config.py +++ b/src/powercontext/builtin/runtime/config.py @@ -199,15 +199,14 @@ def validate_memory_capacity_order(self) -> RuntimeConfig: @model_validator(mode="after") def validate_profile_schedule(self): - from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + from zoneinfo import ZoneInfoNotFoundError - from apscheduler.triggers.cron import CronTrigger + from powercontext.builtin.runtime.cron import CronSchedule try: - timezone = ZoneInfo(self.profile_timezone) + CronSchedule.parse(self.profile_cron, self.profile_timezone) except ZoneInfoNotFoundError as error: raise ValueError("invalid Profile schedule timezone") from error # noqa: TRY003 - CronTrigger.from_crontab(self.profile_cron, timezone=timezone) return self @model_validator(mode="after") @@ -303,6 +302,95 @@ def validate_topic_memory_history_candidates(self) -> RuntimeConfig: return self +class DeploymentConfig(BaseModel): + """Process topology and non-sensitive compatibility identity.""" + + mode: Literal["single_node", "distributed"] = "single_node" + role: Literal["all", "api", "scheduler", "worker"] = "all" + id: str = Field(default="local", min_length=1, max_length=128) + behavior_revision: str = Field(default="default", min_length=1, max_length=128) + + @field_validator("id", "behavior_revision") + @classmethod + def validate_trimmed_identifier(cls, value: str) -> str: + if value != value.strip(): + raise ValueError("deployment identifiers must be trimmed") # noqa: TRY003 + return value + + +class CoordinationConfig(BaseModel): + """Database-backed scheduler and member lease policy.""" + + scheduler_lease_seconds: int = Field(default=30, ge=3) + scheduler_renew_seconds: int = Field(default=10, ge=1) + scan_page_size: int = Field(default=100, ge=1, le=100) + member_ttl_seconds: int = Field(default=30, ge=3) + member_heartbeat_seconds: int = Field(default=10, ge=1) + emit_payload_version: int = Field(default=1, ge=1, le=1) + + @model_validator(mode="after") + def validate_lease_intervals(self) -> CoordinationConfig: + if self.scheduler_renew_seconds * 3 > self.scheduler_lease_seconds: + raise ValueError( # noqa: TRY003 + "scheduler_renew_seconds must not exceed one third of scheduler_lease_seconds" + ) + if self.member_heartbeat_seconds * 3 > self.member_ttl_seconds: + raise ValueError( # noqa: TRY003 + "member_heartbeat_seconds must not exceed one third of member_ttl_seconds" + ) + return self + + +class WorkerConfig(BaseModel): + """Worker claim, retry, heartbeat, and drain policy.""" + + concurrency: int = Field(default=4, ge=1, le=256) + lease_seconds: int = Field(default=120, ge=3) + heartbeat_seconds: int = Field(default=30, ge=1) + shutdown_grace_seconds: int = Field(default=90, ge=0) + max_attempts: int = Field(default=5, ge=1, le=100) + retry_base_seconds: float = Field(default=2.0, gt=0) + retry_max_seconds: float = Field(default=300.0, gt=0) + poll_seconds: float = Field(default=1.0, gt=0) + + @model_validator(mode="after") + def validate_lease_intervals(self) -> WorkerConfig: + if self.heartbeat_seconds * 3 >= self.lease_seconds: + raise ValueError( # noqa: TRY003 + "heartbeat_seconds must be less than one third of lease_seconds" + ) + if self.shutdown_grace_seconds >= self.lease_seconds: + raise ValueError("shutdown_grace_seconds must be less than lease_seconds") # noqa: TRY003 + if self.retry_max_seconds < self.retry_base_seconds: + raise ValueError("retry_max_seconds must not be less than retry_base_seconds") # noqa: TRY003 + return self + + +class OperationsConfig(BaseModel): + """Synchronous wait facade and durable operation retention policy.""" + + default_wait_seconds: float = Field(default=10.0, ge=0, le=30) + maximum_wait_seconds: float = Field(default=30.0, ge=0, le=30) + poll_seconds: float = Field(default=0.2, gt=0) + retention_days: int = Field(default=30, ge=1) + cleanup_batch_size: int = Field(default=500, ge=1, le=500) + cleanup_interval_seconds: float = Field(default=3600.0, gt=0) + + @model_validator(mode="after") + def validate_wait_policy(self) -> OperationsConfig: + if self.default_wait_seconds > self.maximum_wait_seconds: + raise ValueError("default_wait_seconds must not exceed maximum_wait_seconds") # noqa: TRY003 + return self + + +class RateLimitConfig(BaseModel): + """Optional shared fixed-window request limit.""" + + enabled: bool = False + requests: int = Field(default=120, ge=1) + window_seconds: int = Field(default=60, ge=1) + + class HandoffReportConfig(BaseModel): """Optional Handoff Report feature registration.""" @@ -525,6 +613,11 @@ class BuiltinConfig(BaseModel): handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) inference: InferenceConfig = Field(default_factory=InferenceConfig) external_skills: ExternalSkillsConfig = Field(default_factory=ExternalSkillsConfig) + deployment: DeploymentConfig = Field(default_factory=DeploymentConfig) + coordination: CoordinationConfig = Field(default_factory=CoordinationConfig) + worker: WorkerConfig = Field(default_factory=WorkerConfig) + operations: OperationsConfig = Field(default_factory=OperationsConfig) + rate_limit: RateLimitConfig = Field(default_factory=RateLimitConfig) @model_validator(mode="before") @classmethod @@ -532,7 +625,28 @@ def default_database_to_sqlite(cls, value: Any) -> Any: return normalize_database_discriminator(value) @model_validator(mode="after") - def validate_artifact_processing_role(self) -> BuiltinConfig: + def validate_deployment(self) -> BuiltinConfig: + if self.deployment.mode == "single_node" and self.deployment.role != "all": + raise ValueError("single_node deployment role must be 'all'") # noqa: TRY003 + if self.deployment.mode == "distributed": + if not isinstance(self.database, OceanBaseConfig): + raise ValueError("distributed deployment requires OceanBase") # noqa: TRY003 + if self.deployment.role == "all": + raise ValueError("distributed deployment role must be api, scheduler, or worker") # noqa: TRY003 + if self.runtime.artifact_processing_role != "all": + raise ValueError( # noqa: TRY003 + "distributed deployment does not use the process-local artifact processing role" + ) + if self.runtime.topic_memory_schedule_seconds is not None: + raise ValueError("distributed deployment does not support Topic Memory processing") # noqa: TRY003 + if self.external_skills.agent_targets: + raise ValueError( # noqa: TRY003 + "distributed deployment does not support host-local external Skill targets" + ) + if self.deployment.role == "worker" and self.inference.generation_model is None: + # The worker executes Memory, Experience and Profile work, and + # every one of those paths calls a generation model. + raise ValueError("distributed worker requires a generation model") # noqa: TRY003 if not isinstance(self.database, OceanBaseConfig) and self.runtime.artifact_processing_role != "all": raise ValueError( # noqa: TRY003 "runtime.artifact_processing_role must be 'all' for SQLite and embedded seekdb" @@ -542,9 +656,14 @@ def validate_artifact_processing_role(self) -> BuiltinConfig: __all__ = [ "BuiltinConfig", + "CoordinationConfig", "DatabaseConfig", + "DeploymentConfig", "ExternalSkillsConfig", "HandoffReportConfig", "InferenceConfig", + "OperationsConfig", + "RateLimitConfig", "RuntimeConfig", + "WorkerConfig", ] diff --git a/src/powercontext/builtin/runtime/cron.py b/src/powercontext/builtin/runtime/cron.py new file mode 100644 index 0000000000..ec0fef0784 --- /dev/null +++ b/src/powercontext/builtin/runtime/cron.py @@ -0,0 +1,150 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small five-field cron schedule used by durable work discovery.""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, date, datetime, time, timedelta +from zoneinfo import ZoneInfo + +_MONTH_NAMES = { + name: index + for index, name in enumerate( + ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], 1 + ) +} +_WEEKDAY_NAMES = {name: index for index, name in enumerate(["mon", "tue", "wed", "thu", "fri", "sat", "sun"])} + + +@dataclass(frozen=True, slots=True) +class CronSchedule: + """Validated crontab expression with deterministic UTC next-run calculation.""" + + minutes: frozenset[int] + hours: frozenset[int] + days: frozenset[int] + months: frozenset[int] + weekdays: frozenset[int] + timezone: ZoneInfo + + @classmethod + def parse(cls, expression: str, timezone: str, /) -> CronSchedule: + fields = expression.split() + if len(fields) != 5: + raise ValueError("cron expression must contain five fields") # noqa: TRY003 + return cls( + minutes=_parse_field(fields[0], minimum=0, maximum=59), + hours=_parse_field(fields[1], minimum=0, maximum=23), + days=_parse_field(fields[2], minimum=1, maximum=31), + months=_parse_field(fields[3], minimum=1, maximum=12, names=_MONTH_NAMES), + weekdays=_parse_field(fields[4], minimum=0, maximum=6, names=_WEEKDAY_NAMES), + timezone=ZoneInfo(timezone), + ) + + def next_after(self, value: datetime, /) -> datetime: + """Return the first matching instant after ``value`` as UTC-naive time.""" + + utc = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + # A repeated fall-back hour maps a later wall time to an earlier instant, so + # the earliest future occurrence can belong to the previous local date. + first_date = utc.astimezone(self.timezone).date() - timedelta(days=1) + for offset in range(366 * 8 + 1): + candidate_date = first_date + timedelta(days=offset) + if not self._matches_date(candidate_date): + continue + # Order the day's occurrences by real instant, not by wall clock: on a + # transition day 01:00 fold=1 (06:00Z) precedes 01:16 fold=0 (05:16Z). + occurrences = sorted( + ( + instant + for hour in self.hours + for minute in self.minutes + for instant in _local_occurrences( + datetime.combine(candidate_date, time(hour, minute)), self.timezone + ) + ), + key=lambda instant: instant.astimezone(UTC), + ) + for candidate in occurrences: + if candidate.astimezone(UTC) > utc: + return candidate.astimezone(UTC).replace(tzinfo=None) + raise ValueError("cron expression has no occurrence in the next eight years") # noqa: TRY003 + + def _matches_date(self, value: date) -> bool: + return value.month in self.months and value.day in self.days and value.weekday() in self.weekdays + + +def _parse_field( + value: str, + *, + minimum: int, + maximum: int, + names: dict[str, int] | None = None, +) -> frozenset[int]: + selected: set[int] = set() + for part in value.casefold().split(","): + if not part: + raise ValueError("cron fields cannot contain empty items") # noqa: TRY003 + base, separator, step_text = part.partition("/") + if separator and (not step_text.isdecimal() or int(step_text) < 1): + raise ValueError("cron step must be a positive integer") # noqa: TRY003 + step = int(step_text) if separator else 1 + if base == "*": + first, last = minimum, maximum + elif "-" in base: + first_text, last_text = base.split("-", 1) + first = _field_value(first_text, names) + last = _field_value(last_text, names) + else: + first = _field_value(base, names) + last = maximum if separator else first + if first < minimum or last > maximum or first > last: + raise ValueError("cron field is outside its valid range") # noqa: TRY003 + selected.update(range(first, last + 1, step)) + return frozenset(selected) + + +def _field_value(value: str, names: dict[str, int] | None) -> int: + if names is not None and value in names: + return names[value] + if not value.isdecimal(): + raise ValueError("cron field must be numeric or a supported name") # noqa: TRY003 + return int(value) + + +def _local_occurrences(naive: datetime, timezone: ZoneInfo) -> Iterator[datetime]: + """Yield every real instant of a local wall time, earliest UTC instant first. + + Ambiguous local times (DST fall-back repeated hour) occur twice; nonexistent + local times (spring-forward gap) never occur and yield nothing. + """ + + first = naive.replace(tzinfo=timezone) + if not _is_local_time(first): + return + yield first + second = naive.replace(tzinfo=timezone, fold=1) + if second.astimezone(UTC) != first.astimezone(UTC): + yield second + + +def _is_local_time(value: datetime) -> bool: + round_trip = value.astimezone(UTC).astimezone(value.tzinfo) + return round_trip.replace(tzinfo=None) == value.replace(tzinfo=None) + + +__all__ = ["CronSchedule"] diff --git a/src/powercontext/builtin/runtime/durable_scheduler.py b/src/powercontext/builtin/runtime/durable_scheduler.py new file mode 100644 index 0000000000..87876053a1 --- /dev/null +++ b/src/powercontext/builtin/runtime/durable_scheduler.py @@ -0,0 +1,273 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fenced leader scheduler with durable bounded scan continuations.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Iterable +from contextlib import AbstractContextManager, nullcontext, suppress +from dataclasses import dataclass +from datetime import timedelta +from typing import Protocol + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + StaleCoordinatorLeaseError, +) +from powercontext.builtin.persistence.database import AsyncDatabase, database_now +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime.config import CoordinationConfig +from powercontext.builtin.runtime.protocols import RuntimeSpan, RuntimeTracing +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue + +_SCHEDULER_LEASE_NAME = "work-discovery" +_EMPTY_DISCOVERERS = "at least one work discoverer is required" +_INVALID_DISCOVERER_INTERVAL = "discoverer interval must be positive" +_INVALID_DISCOVERER_NAME = "discoverer name must be a non-empty trimmed string" + + +@dataclass(frozen=True) +class DiscoveryPage: + """One bounded page and its next opaque keyset position.""" + + specs: tuple[WorkSpec, ...] + continuation: str | None + + +class WorkDiscoverer(Protocol): + """Find logical work without executing external side effects.""" + + name: str + interval_seconds: float + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: ... + + +class DurableScheduler: + """Acquire one leader lease and enqueue pages under its exact fence.""" + + def __init__( + self, + *, + database: AsyncDatabase, + scheduler_id: str, + discoverers: Iterable[WorkDiscoverer], + config: CoordinationConfig, + coordination: CoordinationRepository | None = None, + work: WorkRepository | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._database = database + self._scheduler_id = scheduler_id + self._discoverers = _discoverer_map(discoverers) + self._config = config + self._coordination = CoordinationRepository() if coordination is None else coordination + self._work = WorkRepository(observer=observer) if work is None else work + self._observer = observer + self._tracing = tracing + self._poll_seconds = min( + float(config.scheduler_renew_seconds), + *(discoverer.interval_seconds for discoverer in self._discoverers.values()), + ) + self._lease: CoordinatorLease | None = None + self._stop_requested = asyncio.Event() + self._tick_lock = asyncio.Lock() + self._failed = False + + async def tick(self) -> bool: + """Renew or acquire leadership and process at most one page per discoverer.""" + + async with self._tick_lock: + if self._stop_requested.is_set(): + return False + return await self._tick() + + async def _tick(self) -> bool: + """Run one tick while the caller owns the scheduler lifecycle lock.""" + + previous = self._lease + with self._background( + "scheduler.tick", + operation="scheduler.tick", + attributes={"powercontext.scheduler.discoverer_count": len(self._discoverers)}, + ) as span: + async with self._database.transaction() as connection: + lease = await self._coordination.acquire_lease( + connection, + lease_name=_SCHEDULER_LEASE_NAME, + owner_id=self._scheduler_id, + lease_seconds=self._config.scheduler_lease_seconds, + ) + self._lease = lease + self._observe_leadership(previous, lease) + if lease is None: + if span is not None: + span.set_outcome("standby") + return False + for discoverer in self._discoverers.values(): + await self._scan_once(discoverer, lease) + if span is not None: + span.set_outcome("leader") + await refresh_work_queue(self._database, self._work, self._observer) + return True + + async def run(self) -> None: + """Maintain leadership until stopped; standby instances keep polling.""" + + while not self._stop_requested.is_set(): + try: + await self.tick() + self._failed = False + except asyncio.CancelledError: + raise + except StaleCoordinatorLeaseError: + self._observe_leadership(self._lease, None) + self._lease = None + except Exception: + self._observe_leadership(self._lease, None) + self._lease = None + self._failed = True + with suppress(TimeoutError): + await asyncio.wait_for( + self._stop_requested.wait(), + timeout=self._poll_seconds, + ) + + async def readiness(self) -> str: + """Treat both a healthy leader and a healthy standby as ready.""" + + return ReadinessCheckStatus.UNAVAILABLE if self._failed else ReadinessCheckStatus.READY + + async def stop(self) -> None: + """Stop discovery and conditionally release current leadership.""" + + self._stop_requested.set() + async with self._tick_lock: + lease = self._lease + self._lease = None + self._observe_leadership(lease, None) + if lease is None: + return + async with self._database.transaction() as connection: + await self._coordination.release_lease(connection, lease) + + async def _scan_once(self, discoverer: WorkDiscoverer, lease: CoordinatorLease) -> None: + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + scan = await self._coordination.load_scan(connection, discoverer.name) + now = await database_now(connection) + if scan is not None and scan.next_run_at > now: + return + + continuation = None if scan is None else scan.continuation + page = await discoverer.page(continuation, self._config.scan_page_size) + if len(page.specs) > self._config.scan_page_size: + message = f"discoverer {discoverer.name} exceeded the configured scan page" + raise ValueError(message) + for spec in page.specs: + with self._stage( + "work.enqueue", + attributes={ + "powercontext.work.kind": spec.kind, + "powercontext.work.payload_version": spec.payload_version, + }, + ) as span: + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + result = await self._work.enqueue(connection, spec) + if span is not None: + span.set_outcome("created" if result.created else "joined") + + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + now = await database_now(connection) + next_run = getattr(discoverer, "next_run_at", None) + next_run_at = now + timedelta(seconds=discoverer.interval_seconds) if next_run is None else next_run(now) + if next_run_at <= now: + message = f"discoverer {discoverer.name} produced a non-future next run" + raise ValueError(message) + await self._coordination.save_scan( + connection, + discoverer.name, + next_run_at=now if page.continuation is not None else next_run_at, + continuation=page.continuation, + expected_version=None if scan is None else scan.state_version, + ) + + def _observe_leadership( + self, + previous: CoordinatorLease | None, + current: CoordinatorLease | None, + ) -> None: + if self._observer is None: + return + outcome = None + if previous is None and current is not None: + outcome = "acquired" + elif previous is not None and current is None: + outcome = "lost" + elif previous is not None and current is not None and previous.fence != current.fence: + outcome = "reacquired" + if outcome is not None: + with suppress(Exception): + self._observer.observe_scheduler_leadership(outcome=outcome) + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + def _background( + self, + name: str, + *, + operation: str, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.background(name, operation=operation, attributes=attributes) + + +def _discoverer_map(discoverers: Iterable[WorkDiscoverer]) -> dict[str, WorkDiscoverer]: + registered: dict[str, WorkDiscoverer] = {} + for discoverer in discoverers: + if not discoverer.name.strip() or discoverer.name != discoverer.name.strip(): + raise ValueError(_INVALID_DISCOVERER_NAME) + if discoverer.interval_seconds <= 0: + raise ValueError(_INVALID_DISCOVERER_INTERVAL) + if discoverer.name in registered: + message = f"duplicate discoverer name: {discoverer.name}" + raise ValueError(message) + registered[discoverer.name] = discoverer + if not registered: + raise ValueError(_EMPTY_DISCOVERERS) + return registered + + +__all__ = [ + "DiscoveryPage", + "DurableScheduler", + "WorkDiscoverer", +] diff --git a/src/powercontext/builtin/runtime/membership.py b/src/powercontext/builtin/runtime/membership.py new file mode 100644 index 0000000000..3d69880b38 --- /dev/null +++ b/src/powercontext/builtin/runtime/membership.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runtime role membership and single-node ownership leases.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import suppress +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + RuntimeMemberSpec, +) +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.runtime.config import CoordinationConfig, DeploymentConfig +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver + +_SINGLE_NODE_LEASE = "single-node-runtime" +_SCHEMA_VERSION = 2 +_PAYLOAD_VERSION = 1 + + +class DuplicateSingleNodeError(RuntimeError): + """Raised before serving when another single-node runtime owns the database.""" + + def __init__(self) -> None: + super().__init__("another single-node runtime already owns this database") + + +class RuntimeMembership: + """Advertise compatibility and maintain exclusive local ownership.""" + + def __init__( + self, + *, + database: AsyncDatabase, + deployment: DeploymentConfig, + coordination: CoordinationConfig, + build_version: str, + repository: CoordinationRepository | None = None, + observer: WorkObserver | None = None, + ) -> None: + self._database = database + self._deployment = deployment + self._config = coordination + self._build_version = build_version + self._repository = CoordinationRepository() if repository is None else repository + self._observer = observer + boot_id = uuid4().hex + self._member_id = hashlib.sha256(f"{deployment.id}\0{boot_id}".encode()).hexdigest() + self._single_node_lease: CoordinatorLease | None = None + self._stop_requested = asyncio.Event() + self._failed = False + + @property + def instance_id(self) -> str: + """Return the boot-unique, non-sensitive owner used for fencing.""" + + return self._member_id + + async def start(self) -> None: + """Acquire required ownership and publish the first heartbeat.""" + + async with self._database.transaction() as connection: + if self._deployment.mode == "single_node": + self._single_node_lease = await self._repository.acquire_lease( + connection, + lease_name=_SINGLE_NODE_LEASE, + owner_id=self._member_id, + lease_seconds=self._config.member_ttl_seconds, + ) + if self._single_node_lease is None: + raise DuplicateSingleNodeError + await self._heartbeat(connection) + + async def run(self) -> None: + """Renew role and optional owner leases until shutdown.""" + + try: + while not self._stop_requested.is_set(): + with suppress(TimeoutError): + await asyncio.wait_for( + self._stop_requested.wait(), + timeout=self._config.member_heartbeat_seconds, + ) + if self._stop_requested.is_set(): + return + async with self._database.transaction() as connection: + if self._single_node_lease is not None: + renewed = await self._repository.acquire_lease( + connection, + lease_name=_SINGLE_NODE_LEASE, + owner_id=self._member_id, + lease_seconds=self._config.member_ttl_seconds, + ) + self._single_node_lease = _validated_renewal(self._single_node_lease, renewed) + await self._heartbeat(connection) + except asyncio.CancelledError: + raise + except Exception: + self._failed = True + raise + + async def stop(self) -> None: + """Stop heartbeats and conditionally release single-node ownership.""" + + self._stop_requested.set() + lease = self._single_node_lease + self._single_node_lease = None + if lease is not None: + async with self._database.transaction() as connection: + await self._repository.release_lease(connection, lease) + + async def readiness(self) -> str: + """Verify this member is still live and compatible with its peers.""" + + if self._failed: + return ReadinessCheckStatus.UNAVAILABLE + async with self._database.transaction() as connection: + members = await self._repository.live_members(connection) + current = next((member for member in members if member.member_id == self._member_id), None) + if current is None: + return ReadinessCheckStatus.UNAVAILABLE + if any( + member.behavior_revision != self._deployment.behavior_revision + or member.schema_min > _SCHEMA_VERSION + or member.schema_max < _SCHEMA_VERSION + or member.payload_min > _PAYLOAD_VERSION + or member.payload_max < _PAYLOAD_VERSION + for member in members + ): + return ReadinessCheckStatus.MISCONFIGURED + return ReadinessCheckStatus.READY + + async def role_readiness(self, role: str) -> str: + """Report whether at least one compatible live member serves ``role``.""" + + async with self._database.transaction() as connection: + members = await self._repository.live_members(connection) + return ( + ReadinessCheckStatus.READY + if any( + member.role in {role, "all"} and member.behavior_revision == self._deployment.behavior_revision + for member in members + ) + else ReadinessCheckStatus.UNAVAILABLE + ) + + async def _heartbeat(self, connection: AsyncConnection) -> None: + await self._repository.heartbeat_member( + connection, + RuntimeMemberSpec( + member_id=self._member_id, + role=self._deployment.role, + build_version=self._build_version, + schema_min=_SCHEMA_VERSION, + schema_max=_SCHEMA_VERSION, + payload_min=_PAYLOAD_VERSION, + payload_max=_PAYLOAD_VERSION, + behavior_revision=self._deployment.behavior_revision, + ), + ttl_seconds=self._config.member_ttl_seconds, + ) + if self._observer is not None: + members = await self._repository.live_members(connection) + counts = dict.fromkeys(("all", "api", "scheduler", "worker"), 0) + for member in members: + counts[member.role] += 1 + with suppress(Exception): + self._observer.set_runtime_members(counts) + + +def _validated_renewal(current: CoordinatorLease, renewed: CoordinatorLease | None) -> CoordinatorLease: + if renewed is None or renewed.fence != current.fence: + raise DuplicateSingleNodeError + return renewed + + +__all__ = ["DuplicateSingleNodeError", "RuntimeMembership"] diff --git a/src/powercontext/builtin/runtime/operations.py b/src/powercontext/builtin/runtime/operations.py new file mode 100644 index 0000000000..73c553b415 --- /dev/null +++ b/src/powercontext/builtin/runtime/operations.py @@ -0,0 +1,217 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Application service for durable operation submission and control.""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import json +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from datetime import datetime +from time import monotonic + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, RepositoryNotFoundError +from powercontext.builtin.persistence.work import StoredWork, WorkRepository, WorkStatus +from powercontext.builtin.runtime.config import OperationsConfig, WorkerConfig +from powercontext.builtin.runtime.protocols import RuntimeSpan, RuntimeTracing +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.work_handlers import ( + EXPERIENCE_WORK_KIND, + MEMORY_WORK_KIND, + PROFILE_WORK_KIND, +) +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue +from powercontext.builtin.runtime.worker import DurableWorker + +_TERMINAL = frozenset({WorkStatus.SUCCEEDED, WorkStatus.FAILED, WorkStatus.CANCELLED}) +# Scope owners may inspect, cancel and retry the work their own Sources produced. +# Maintenance work stays private: it is neither listed nor reachable by id. +_PUBLIC_KINDS = frozenset({MEMORY_WORK_KIND, PROFILE_WORK_KIND, EXPERIENCE_WORK_KIND}) + + +@dataclass(frozen=True) +class OperationPage: + items: tuple[StoredWork, ...] + next_cursor: str | None + + +class RuntimeOperationError(RuntimeError): + """Base class for durable operation facade errors.""" + + +class RuntimeOperationPendingError(RuntimeOperationError): + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} is still pending") + + +class RuntimeOperationFailedError(RuntimeOperationError): + def __init__(self, operation: StoredWork) -> None: + self.operation = operation + super().__init__(f"operation {operation.work_id} failed with {operation.error_code or 'unknown'}") + + +class RuntimeOperationCancelledError(RuntimeOperationError): + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} was cancelled") + + +class OperationManager: + """Submit, wait for, query, and mutate durable work records.""" + + def __init__( + self, + *, + contexts: RelationalContexts, + operations: OperationsConfig, + worker: WorkerConfig, + local_worker: DurableWorker | None, + payload_version: int, + repository: WorkRepository | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._contexts = contexts + self._database: AsyncDatabase = contexts.database + self._operations = operations + self._worker_config = worker + self._local_worker = local_worker + self._payload_version = payload_version + self._repository = WorkRepository(observer=observer) if repository is None else repository + self._observer = observer + self._tracing = tracing + + @property + def default_wait_seconds(self) -> float: + return self._operations.default_wait_seconds + + @property + def maximum_wait_seconds(self) -> float: + return self._operations.maximum_wait_seconds + + @property + def database(self) -> AsyncDatabase: + """Expose the shared database to process-level coordination adapters.""" + + return self._database + + async def wait(self, operation_id: str, /, *, timeout_seconds: float) -> StoredWork | None: + if timeout_seconds < 0 or timeout_seconds > self._operations.maximum_wait_seconds: + raise InvalidRepositoryArgumentError( + "timeout_seconds", + f"must be between 0 and {self._operations.maximum_wait_seconds}", + ) + deadline = monotonic() + timeout_seconds + while True: + operation = await self.get(operation_id) + if operation.status in _TERMINAL: + return operation + remaining = deadline - monotonic() + if remaining <= 0: + return None + await asyncio.sleep(min(self._operations.poll_seconds, remaining)) + + async def get(self, operation_id: str, /) -> StoredWork: + async with self._database.transaction() as connection: + operation = await self._repository.get(connection, operation_id) + return _require_public_operation(operation) + + async def list( + self, + /, + *, + scope_id: str | None, + kind: str | None, + status: WorkStatus | None, + cursor: str | None, + limit: int, + ) -> OperationPage: + before = None if cursor is None else _decode_cursor(cursor) + async with self._database.transaction() as connection: + items = await self._repository.list( + connection, + scope_id=scope_id, + kind=kind, + allowed_kinds=_PUBLIC_KINDS, + status=status, + limit=limit, + before=before, + ) + next_cursor = None + if len(items) == limit: + last = items[-1] + next_cursor = _encode_cursor(last.created_at, last.work_id) + return OperationPage(items=items, next_cursor=next_cursor) + + async def cancel(self, operation_id: str, /, *, expected_version: int) -> StoredWork: + async with self._database.transaction() as connection: + _require_public_operation(await self._repository.get(connection, operation_id)) + operation = await self._repository.cancel(connection, operation_id, expected_version=expected_version) + await refresh_work_queue(self._database, self._repository, self._observer) + return operation + + async def retry(self, operation_id: str, /, *, expected_version: int) -> StoredWork: + async with self._database.transaction() as connection: + _require_public_operation(await self._repository.get(connection, operation_id)) + operation = await self._repository.retry(connection, operation_id, expected_version=expected_version) + if self._local_worker is not None: + self._local_worker.notify() + await refresh_work_queue(self._database, self._repository, self._observer) + return operation + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + +def _require_public_operation(operation: StoredWork) -> StoredWork: + if operation.kind not in _PUBLIC_KINDS: + raise RepositoryNotFoundError("operation", operation.work_id) + return operation + + +def _encode_cursor(created_at: datetime, work_id: str) -> str: + payload = json.dumps([created_at.isoformat(), work_id], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(payload).decode().rstrip("=") + + +def _decode_cursor(value: str) -> tuple[datetime, str]: + try: + padding = "=" * (-len(value) % 4) + created_at, work_id = json.loads(base64.urlsafe_b64decode(f"{value}{padding}")) + return datetime.fromisoformat(created_at), str(work_id) + except (ValueError, TypeError, json.JSONDecodeError, binascii.Error): + raise InvalidRepositoryArgumentError("cursor", "must be a valid operation cursor") from None + + +__all__ = [ + "OperationManager", + "OperationPage", + "RuntimeOperationCancelledError", + "RuntimeOperationError", + "RuntimeOperationFailedError", + "RuntimeOperationPendingError", +] diff --git a/src/powercontext/builtin/runtime/processing_registry.py b/src/powercontext/builtin/runtime/processing_registry.py index 9ca1a9f303..2207bb0dc5 100644 --- a/src/powercontext/builtin/runtime/processing_registry.py +++ b/src/powercontext/builtin/runtime/processing_registry.py @@ -47,6 +47,11 @@ def processing_capabilities(config: BuiltinConfig) -> tuple[str, ...]: """Use explicit cross-role declarations or infer executable built-ins.""" + # Distributed v1 executes Memory, Experience, and Profile work through the + # database-backed Work Ledger. The process-local Artifact Processing + # Supervisor remains the single-node execution engine. + if config.deployment.mode == "distributed": + return () declared = config.runtime.artifact_processing_families if declared is not None: return tuple(sorted(declared)) diff --git a/src/powercontext/builtin/runtime/protocols.py b/src/powercontext/builtin/runtime/protocols.py index ce44b6107f..0a295031b1 100644 --- a/src/powercontext/builtin/runtime/protocols.py +++ b/src/powercontext/builtin/runtime/protocols.py @@ -16,9 +16,10 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from contextlib import AbstractContextManager -from typing import Protocol, TypeVar +from dataclasses import dataclass +from typing import Any, Protocol, TypeVar from powercontext.builtin.artifacts.handoff import ActivateHandoff, HandoffActivation from powercontext.builtin.runtime.models import ( @@ -38,6 +39,14 @@ TraceAttribute = str | bool | int | float +@dataclass(frozen=True, slots=True) +class RuntimeTraceContext: + """Portable non-sensitive trace identity used to link background attempts.""" + + trace_id: str + span_id: str + + class RuntimeSpan(Protocol): """Record bounded attributes and a deferred outcome for one internal Runtime stage.""" @@ -62,9 +71,17 @@ def background( *, operation: str, attributes: Mapping[str, TraceAttribute], + links: Sequence[RuntimeTraceContext] = (), ) -> AbstractContextManager[RuntimeSpan]: ... +def runtime_trace_context(span: RuntimeSpan | None) -> RuntimeTraceContext | None: + """Read an optional concrete tracing context without expanding the port.""" + + value: Any = getattr(span, "trace_context", None) + return value if isinstance(value, RuntimeTraceContext) else None + + class PowerContextProvider(Protocol[SourcesT, ArtifactsT, TriggersT]): """Resolve an already composed context without transferring lifecycle ownership.""" diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 52fb49ddbd..3d2bb41c3b 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -188,6 +188,7 @@ from powercontext.builtin.runtime.atomic_memory_processing import ( AtomicMemoryProcessingConfig, AtomicMemorySourceWindowProcessor, + PreparedMemoryWindow, ) from powercontext.builtin.runtime.decision_model import DecisionModel from powercontext.builtin.runtime.models import ( @@ -1399,6 +1400,25 @@ async def scope_ids(self) -> tuple[str, ...]: ).scalars() return tuple(str(value) for value in values) + async def scope_ids_page(self, after: str | None, limit: int, /) -> tuple[str, ...]: + """Return one bounded keyset page of scopes that own a Source journal. + + Restored for the Work Ledger discoverers: the scheduler scans scopes in + keyset pages, so a full ``scope_ids()`` materialisation is neither + bounded nor restartable. + """ + + if limit < 1 or limit > 100: + raise ValueError("scope page limit must be between 1 and 100") # noqa: TRY003 + async with self.database.transaction() as connection: + statement = select(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id) + if after is not None: + statement = statement.where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id > validate_scope_id(after)) + values = ( + await connection.execute(statement.order_by(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id).limit(limit)) + ).scalars() + return tuple(str(value) for value in values) + async def unconsumed_source_scope_ids(self) -> tuple[str, ...]: """Return built-in Source windows that still have journal input.""" @@ -1547,6 +1567,32 @@ async def process_memory( atomic_context=atomic_context, ) + async def prepare_memory( + self, + scope_id: str, + limit: int, + /, + *, + atomic_context: ArtifactSearchExecutionContext | None = None, + window: tuple[int, int] | None = None, + ) -> PreparedMemoryWindow | MemoryFlushResult: + """Prepare one Memory window's decisions without publishing it. + + ``window`` pins the logical input to persisted ``(after, through)`` boundaries + so a retried or delayed claim processes the rows its Work record describes. + + The returned window's ``publish`` commits the Memory artifacts and + advances the Source cursor on the caller's open transaction, so the + Work Ledger handler can run it inside the fenced Work completion + transaction. A ``MemoryFlushResult`` return means nothing to publish. + """ + services = self._services_for(scope_id) + return await _RelationalTriggers( + services=services, + lock=self._state_for(services.scope_id).scope_locks.setdefault("activation", asyncio.Lock()), + tracing=self._tracing, + ).prepare(limit=limit, atomic_context=atomic_context, window=window) + async def incubate_experience( self, scope_id: str, @@ -1810,6 +1856,27 @@ async def flush( context=atomic_context, ) + async def prepare( + self, + *, + limit: int, + atomic_context: ArtifactSearchExecutionContext | None = None, + window: tuple[int, int] | None = None, + ) -> PreparedMemoryWindow | MemoryFlushResult: + """Prepare one window's decisions while holding the activation lock. + + The lock is released before the caller publishes, so the fenced Work + completion transaction never runs model calls under it; publication + relies on the cursor row lock and generation check instead. + """ + async with self._lock: + return await self._services.atomic_memory_processor.prepare_window( + self._services.scope_id, + limit, + context=atomic_context, + window=window, + ) + def _stage( self, name: str, diff --git a/src/powercontext/builtin/runtime/work_handlers.py b/src/powercontext/builtin/runtime/work_handlers.py new file mode 100644 index 0000000000..4b8aabddd8 --- /dev/null +++ b/src/powercontext/builtin/runtime/work_handlers.py @@ -0,0 +1,845 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Built-in Memory and Experience work discovery and exact-window handlers.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.artifacts.atomic_memory.errors import AtomicMemoryConflictError +from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME +from powercontext.builtin.artifacts.profile.models import ( + PROFILE_SOURCE_WINDOW_BINDING, + ProfileFlushResult, +) +from powercontext.builtin.artifacts.profile.service import PreparedProfileFlush, RelationalProfileService +from powercontext.builtin.artifacts.prompt.service import current_prompt +from powercontext.builtin.inference.usage import bind_usage_reporter +from powercontext.builtin.persistence.cursors import StoredSourceCursor +from powercontext.builtin.persistence.database import database_now, is_transaction_contention +from powercontext.builtin.persistence.errors import GenerationConflictError +from powercontext.builtin.persistence.rate_limit import RateLimitRepository +from powercontext.builtin.persistence.tables import PROFILE_POLICIES_TABLE +from powercontext.builtin.persistence.work import ( + WorkClaim, + WorkRepository, + WorkResult, + WorkSpec, +) +from powercontext.builtin.records import BaseValueConflictError +from powercontext.builtin.runtime.cron import CronSchedule +from powercontext.builtin.runtime.durable_scheduler import DiscoveryPage +from powercontext.builtin.runtime.models import ExperienceIncubationResult, MemoryFlushResult +from powercontext.builtin.runtime.relational import RelationalContexts, _validate_experience_plans +from powercontext.builtin.runtime.worker import PreparedWork, WorkExecutionError +from powercontext.builtin.sources import SourceCursor, validate_scope_id +from powercontext.builtin.statistics import ModelUsagePurpose +from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME, SourceHighWatermark, SourceWindowTrigger +from powercontext.errors import RevisionConflictError + +MEMORY_WORK_KIND = "powercontext.memory.source-window" +EXPERIENCE_WORK_KIND = "powercontext.experience.incubation" +PROFILE_WORK_KIND = "powercontext.profile.source-window" +MAINTENANCE_WORK_KIND = "powercontext.maintenance.operations" +CURRENT_WORK_PAYLOAD_VERSION = 1 + +ProfileWorkRunner = Callable[[str, int, RelationalProfileService], Awaitable[PreparedProfileFlush]] + + +class WorkRequester(BaseModel): + """Non-sensitive identity reference carried across async execution.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: Literal["user", "service"] + id: str = Field(min_length=1, max_length=255) + + @field_validator("id") + @classmethod + def validate_id(cls, value: str) -> str: + if value != value.strip(): + raise ValueError("requester id must be trimmed") # noqa: TRY003 + return value + + +class SourceWindowPayload(BaseModel): + """Reference-only snapshot of one bounded Source cursor transition.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + cursor_name: str + cursor_generation: int = Field(ge=0) + after: int = Field(ge=0) + through: int = Field(ge=1) + high_watermark: int = Field(ge=1) + requester: WorkRequester | None = None + + +class ProfileWorkPayload(BaseModel): + """Reference-only boundary for one scheduled Profile catch-up.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + cursor_generation: int = Field(ge=0) + after: int = Field(ge=0) + high_watermark: int = Field(ge=1) + + +class MemoryWorkDiscoverer: + """Scan Source scopes and describe pending Memory windows.""" + + name = MEMORY_WORK_KIND + + def __init__( + self, + contexts: RelationalContexts, + *, + interval_seconds: float, + window_limit: int, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + ) -> None: + self._contexts = contexts + self.interval_seconds = interval_seconds + self._window_limit = window_limit + self._max_attempts = max_attempts + self._payload_version = payload_version + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: + scopes = await self._contexts.scope_ids_page(continuation, limit) + specs: list[WorkSpec] = [] + for scope_id in scopes: + spec = await memory_work_spec( + self._contexts, + scope_id, + limit=self._window_limit, + max_attempts=self._max_attempts, + payload_version=self._payload_version, + ) + if spec is not None: + specs.append(spec) + return DiscoveryPage( + specs=tuple(specs), + continuation=scopes[-1] if len(scopes) == limit else None, + ) + + +class ExperienceWorkDiscoverer: + """Scan Source scopes and describe pending Experience windows.""" + + name = EXPERIENCE_WORK_KIND + + def __init__( + self, + contexts: RelationalContexts, + *, + interval_seconds: float, + window_limit: int, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + ) -> None: + self._contexts = contexts + self.interval_seconds = interval_seconds + self._window_limit = window_limit + self._max_attempts = max_attempts + self._payload_version = payload_version + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: + scopes = await self._contexts.scope_ids_page(continuation, limit) + specs: list[WorkSpec] = [] + for scope_id in scopes: + spec = await experience_work_spec( + self._contexts, + scope_id, + limit=self._window_limit, + max_attempts=self._max_attempts, + payload_version=self._payload_version, + ) + if spec is not None: + specs.append(spec) + return DiscoveryPage( + specs=tuple(specs), + continuation=scopes[-1] if len(scopes) == limit else None, + ) + + +class ProfileWorkDiscoverer: + """Scan enabled Profile policies on a persistent cron schedule.""" + + name = PROFILE_WORK_KIND + interval_seconds = 60.0 + + def __init__( + self, + contexts: RelationalContexts, + *, + cron: str, + timezone: str, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + ) -> None: + self._contexts = contexts + self._schedule = CronSchedule.parse(cron, timezone) + self._max_attempts = max_attempts + self._payload_version = payload_version + + def next_run_at(self, now: datetime, /) -> datetime: + return self._schedule.next_after(now) + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: + async with self._contexts.database.transaction() as connection: + query = select(PROFILE_POLICIES_TABLE.c.scope_id).where( + PROFILE_POLICIES_TABLE.c.generation_enabled.is_(True), + PROFILE_POLICIES_TABLE.c.pending_candidate_id.is_(None), + ) + if continuation is not None: + query = query.where(PROFILE_POLICIES_TABLE.c.scope_id > validate_scope_id(continuation)) + values = ( + await connection.execute(query.order_by(PROFILE_POLICIES_TABLE.c.scope_id).limit(limit)) + ).scalars() + scopes = tuple(str(value) for value in values) + specs: list[WorkSpec] = [] + for scope_id in scopes: + spec = await profile_work_spec( + self._contexts, + scope_id, + max_attempts=self._max_attempts, + payload_version=self._payload_version, + ) + if spec is not None: + specs.append(spec) + return DiscoveryPage( + specs=tuple(specs), + continuation=scopes[-1] if len(scopes) == limit else None, + ) + + +class OperationMaintenanceDiscoverer: + """Schedule one globally serialized, bounded history-retention batch.""" + + name = MAINTENANCE_WORK_KIND + + def __init__(self, *, interval_seconds: float, max_attempts: int) -> None: + self.interval_seconds = interval_seconds + self._max_attempts = max_attempts + + async def page(self, continuation: str | None, _limit: int, /) -> DiscoveryPage: + if continuation is not None: + return DiscoveryPage(specs=(), continuation=None) + return DiscoveryPage( + specs=( + WorkSpec( + kind=MAINTENANCE_WORK_KIND, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + scope_id="system:operations", + lane_key=_digest("maintenance:operations"), + logical_key=_digest(MAINTENANCE_WORK_KIND), + payload={}, + max_attempts=self._max_attempts, + ), + ), + continuation=None, + ) + + +class MemoryWorkHandler: + """Run one claimed Memory window through the Atomic Memory processor. + + Preparation selects the window and runs the model calls without any + domain write. The publication (Memory artifacts plus cursor advance) is + returned as a ``PreparedWork.commit`` closure and runs inside the fenced + Work completion transaction, after ``WorkRepository.complete`` verified + the live claim and cancellation: a cancel accepted during preparation + discards the prepared result with the rolled-back transaction. A retried + attempt re-reads the cursor, so a window that already committed retires + instead of repeating the domain write. + """ + + kind = MEMORY_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, contexts: RelationalContexts, *, window_limit: int) -> None: + self._contexts = contexts + self._window_limit = window_limit + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + payload = _payload(claim) + services = self._contexts._services_for(claim.scope_id) + async with self._contexts.database.transaction() as connection: + state_row = await services.repositories.cursors.load( + connection, + claim.scope_id, + SOURCE_WINDOW_TRIGGER_NAME, + ) + current_sequence, current_generation = _cursor_position(state_row) + if current_sequence >= payload.through: + return PreparedWork( + result=_memory_result( + payload, + current=current_sequence, + source_count=0, + code="already_committed", + ) + ) + if _is_superseded_window(payload, current_sequence, current_generation): + # A committed racing window consumed this snapshot's prefix; retire + # the key so the lane keeps moving and discovery re-enqueues the rest. + return PreparedWork( + result=_memory_result( + payload, + current=current_sequence, + source_count=0, + code="superseded", + ) + ) + _require_exact_cursor(payload, current_sequence, current_generation) + # Pin preparation to the persisted window: re-reading the live high watermark + # would absorb Sources that arrived after this Work record was written, so the + # claim would publish rows outside the window it reports. + prepared = await self._contexts.prepare_memory( + claim.scope_id, + self._window_limit, + window=(payload.after, payload.through), + ) + if isinstance(prepared, MemoryFlushResult): + # The trigger selected no window; there is nothing to publish and + # nothing the ledger fence could discard. + return PreparedWork( + result=WorkResult( + code="processed" if prepared.processed else "idle", + payload=prepared.model_dump(mode="json", exclude_none=True), + ) + ) + + async def commit(connection: AsyncConnection) -> WorkResult: + try: + # Runs inside the Work completion transaction after + # WorkRepository.complete has verified the live claim and + # cancellation: a cancel accepted before publication rolls this + # domain write back with the fenced completion. + flushed = await prepared.publish(connection, processing=None) + except (AtomicMemoryConflictError, GenerationConflictError, RevisionConflictError) as error: + raise WorkExecutionError(category="conflict", code="memory_conflict", retryable=True) from error + prepared.emit_finalized() + return WorkResult( + code="processed" if flushed.processed else "idle", + payload=flushed.model_dump(mode="json", exclude_none=True), + ) + + return PreparedWork(result=None, commit=commit) + + +class ExperienceWorkHandler: + """Prepare Experience proposals and commit Candidates with cursor and Work.""" + + kind = EXPERIENCE_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, contexts: RelationalContexts) -> None: + self._contexts = contexts + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + payload = _payload(claim) + services = self._contexts._services_for(claim.scope_id) + pipeline = services.experience_pipeline + if pipeline is None: + raise WorkExecutionError(category="configuration", code="experience_pipeline_unavailable", retryable=False) + async with self._contexts.database.transaction() as connection: + state_row = await services.repositories.cursors.load( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + ) + current_sequence, current_generation = _cursor_position(state_row) + if current_sequence >= payload.through: + return PreparedWork( + result=_experience_result( + payload, + source_count=payload.through - payload.after, + candidate_count=0, + code="already_committed", + ) + ) + if _is_superseded_window(payload, current_sequence, current_generation): + # A committed racing window consumed this snapshot's prefix. + # Retire instead of failing so the key is released and the lane + # keeps moving; discovery re-enqueues the remaining Sources. + return PreparedWork( + result=_experience_result( + payload, + source_count=0, + candidate_count=0, + code="superseded", + current_cursor=current_sequence, + ) + ) + _require_exact_cursor(payload, current_sequence, current_generation) + rows = await services.repositories.sources.list_window( + connection, + claim.scope_id, + after=payload.after, + through=payload.through, + ) + eligible_rows = await services.generation_sources().list_window_for_generation( + connection, + claim.scope_id, + after=payload.after, + through=payload.through, + ) + _require_complete_window(rows, payload) + try: + async with services.prompts.bind(claim.scope_id, "experience.incubate"): + with bind_usage_reporter( + self._contexts.model_usage_reporter(claim.scope_id), + generation_purpose=ModelUsagePurpose.EXPERIENCE_GENERATION, + ): + plans = ( + () if not eligible_rows else await pipeline.incubate(tuple(row.value for row in eligible_rows)) + ) + selection = current_prompt("experience.incubate") + prompt_refs = () if selection is None or selection.artifact is None else (selection.artifact,) + finally: + await self._contexts.flush_model_usage() + _validate_experience_plans(plans, eligible_rows) + + async def commit(connection: AsyncConnection) -> WorkResult: + locked = await services.repositories.cursors.load( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + for_update=True, + ) + sequence, generation = _cursor_position(locked) + _require_exact_cursor(payload, sequence, generation) + review = services.review(connection) + candidate_ids: list[str] = [] + for plan in plans: + candidate = await review.propose_experience( + plan.proposal, + sources=plan.sources, + artifacts=prompt_refs, + target=None, + reason=plan.reason, + ) + candidate_ids.append(candidate.candidate_id) + for proposal in await services.recurrence_ledger().record_window(connection, eligible_rows): + target_content = await services.repositories.artifacts.get( + connection, + claim.scope_id, + proposal.target, + ) + candidate = await review.propose_experience( + proposal.proposal, + sources=proposal.sources, + artifacts=(target_content.as_ref(),), + target=proposal.target, + reason=proposal.reason, + ) + candidate_ids.append(candidate.candidate_id) + await services.repositories.cursors.save( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + SourceCursor(sequence=payload.through), + expected_generation=None if payload.cursor_generation == 0 else payload.cursor_generation, + ) + return _experience_result( + payload, + source_count=len(eligible_rows), + candidate_count=len(candidate_ids), + candidate_ids=tuple(candidate_ids), + code="processed", + ) + + return PreparedWork(result=None, commit=commit) + + +class ProfileWorkHandler: + """Prepare one Profile window outside locks and commit it with Work success. + + One attempt prepares and commits exactly one window: the model call runs + outside any transaction during ``prepare``, and the Profile, its cursor + and the Work success commit inside the single fenced completion + transaction. Remaining windows are re-discovered on the next scheduled + scan, so an expired or superseded Worker can never publish a Profile. + """ + + kind = PROFILE_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__( + self, + profiles: RelationalProfileService, + *, + max_concurrency: int, + runner: ProfileWorkRunner | None = None, + ) -> None: + self._profiles = profiles + self._runner = runner + self._semaphore = asyncio.Semaphore(max_concurrency) + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + payload = _profile_payload(claim) + async with self._semaphore: + prepared = ( + await self._runner(claim.scope_id, payload.high_watermark, self._profiles) + if self._runner is not None + else await self._profiles.prepare_flush(claim.scope_id, high_watermark=payload.high_watermark) + ) + if prepared.result is not None: + return PreparedWork(result=_profile_result(prepared.result)) + + async def commit(connection: AsyncConnection) -> WorkResult: + try: + flushed = await self._profiles.commit_flush(connection, prepared) + except ( + BaseValueConflictError, + GenerationConflictError, + RevisionConflictError, + IntegrityError, + OperationalError, + ) as error: + if isinstance(error, OperationalError) and not is_transaction_contention(error): + raise + raise WorkExecutionError(category="conflict", code="profile_cursor_changed", retryable=True) from error + return _profile_result(flushed) + + return PreparedWork(result=None, commit=commit) + + +class OperationMaintenanceHandler: + """Delete one bounded batch of expired successful/cancelled operations.""" + + kind = MAINTENANCE_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, *, retention_days: int, batch_size: int) -> None: + self._retention_days = retention_days + self._batch_size = batch_size + self._repository = WorkRepository() + self._rate_limits = RateLimitRepository() + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + if claim.payload: + raise WorkExecutionError(category="payload", code="invalid_payload", retryable=False) + + async def commit(connection: AsyncConnection) -> WorkResult: + now = await database_now(connection) + deleted = await self._repository.purge_terminal( + connection, + completed_before=now - timedelta(days=self._retention_days), + limit=self._batch_size, + ) + counters_deleted = await self._rate_limits.purge_expired(connection, limit=self._batch_size) + return WorkResult( + code="cleaned", + payload={ + "operations_deleted": deleted, + "rate_limit_windows_deleted": counters_deleted, + }, + ) + + return PreparedWork(result=None, commit=commit) + + +async def memory_work_spec( + contexts: RelationalContexts, + scope_id: str, + /, + *, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + return await _discover_spec( + contexts, + scope_id, + cursor_name=SOURCE_WINDOW_TRIGGER_NAME, + kind=MEMORY_WORK_KIND, + limit=limit, + max_attempts=max_attempts, + payload_version=payload_version, + ) + + +async def experience_work_spec( + contexts: RelationalContexts, + scope_id: str, + /, + *, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + return await _discover_spec( + contexts, + scope_id, + cursor_name=EXPERIENCE_INCUBATION_CURSOR_NAME, + kind=EXPERIENCE_WORK_KIND, + limit=limit, + max_attempts=max_attempts, + payload_version=payload_version, + ) + + +async def profile_work_spec( + contexts: RelationalContexts, + scope_id: str, + /, + *, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + scope = validate_scope_id(scope_id) + async with contexts.database.transaction() as connection: + services = contexts._services_for(scope) + cursor = await services.repositories.cursors.load(connection, scope, PROFILE_SOURCE_WINDOW_BINDING) + high_watermark = await services.repositories.sources.journal_position(connection, scope) + after, generation = _cursor_position(cursor) + if after >= high_watermark: + return None + payload = ProfileWorkPayload( + cursor_generation=generation, + after=after, + high_watermark=high_watermark, + ) + return WorkSpec( + kind=PROFILE_WORK_KIND, + payload_version=payload_version, + scope_id=scope, + lane_key=_digest(f"profile:{scope}"), + logical_key=_digest( + json.dumps( + [PROFILE_WORK_KIND, scope, PROFILE_SOURCE_WINDOW_BINDING, generation, after], + ensure_ascii=False, + separators=(",", ":"), + ) + ), + payload=payload.model_dump(mode="json"), + max_attempts=max_attempts, + ) + + +async def _discover_spec( + contexts: RelationalContexts, + scope_id: str, + *, + cursor_name: str, + kind: str, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + scope = validate_scope_id(scope_id) + async with contexts.database.transaction() as connection: + state_row, high_watermark = await _window_state(contexts, connection, scope, cursor_name) + payload = _window_payload(state_row, high_watermark, cursor_name, limit) + return ( + None + if payload is None + else _work_spec( + kind=kind, + scope_id=scope, + payload=payload, + max_attempts=max_attempts, + payload_version=payload_version, + ) + ) + + +async def _window_state( + contexts: RelationalContexts, + connection: AsyncConnection, + scope_id: str, + cursor_name: str, +) -> tuple[StoredSourceCursor | None, int]: + services = contexts._services_for(scope_id) + state_row = await services.repositories.cursors.load(connection, scope_id, cursor_name) + high_watermark = await services.repositories.sources.journal_position(connection, scope_id) + return state_row, high_watermark + + +def _window_payload( + state_row: StoredSourceCursor | None, + high_watermark: int, + cursor_name: str, + limit: int, + *, + requester: WorkRequester | None = None, +) -> SourceWindowPayload | None: + state = SourceCursor() if state_row is None else state_row.cursor + transition = SourceWindowTrigger().activate( + SourceHighWatermark(sequence=high_watermark, limit=limit), + state, + ) + if not transition.actions: + return None + action = transition.actions[0] + return SourceWindowPayload( + cursor_name=cursor_name, + cursor_generation=0 if state_row is None else state_row.generation, + after=action.after, + through=action.through, + high_watermark=high_watermark, + requester=requester, + ) + + +def _work_spec( + *, + kind: str, + scope_id: str, + payload: SourceWindowPayload, + max_attempts: int, + payload_version: int, +) -> WorkSpec: + lane_key = _digest(f"{kind.split('.')[1]}:{scope_id}") + logical_key = _digest( + json.dumps( + [kind, scope_id, payload.cursor_name, payload.cursor_generation, payload.after], + ensure_ascii=False, + separators=(",", ":"), + ) + ) + return WorkSpec( + kind=kind, + payload_version=payload_version, + scope_id=scope_id, + lane_key=lane_key, + logical_key=logical_key, + payload=payload.model_dump(mode="json", exclude_none=True), + max_attempts=max_attempts, + ) + + +def _payload(claim: WorkClaim) -> SourceWindowPayload: + try: + return SourceWindowPayload.model_validate(claim.payload) + except ValidationError: + raise WorkExecutionError(category="payload", code="invalid_payload", retryable=False) from None + + +def _profile_payload(claim: WorkClaim) -> ProfileWorkPayload: + try: + return ProfileWorkPayload.model_validate(claim.payload) + except ValidationError: + raise WorkExecutionError(category="payload", code="invalid_payload", retryable=False) from None + + +def _cursor_position(state_row: StoredSourceCursor | None) -> tuple[int, int]: + return (0, 0) if state_row is None else (state_row.cursor.sequence, state_row.generation) + + +def _require_exact_cursor(payload: SourceWindowPayload, sequence: int, generation: int) -> None: + if sequence != payload.after or generation != payload.cursor_generation: + raise WorkExecutionError(category="conflict", code="cursor_changed", retryable=False) + + +def _is_superseded_window(payload: SourceWindowPayload, sequence: int, generation: int) -> bool: + """True when a committed racing window already consumed this snapshot's prefix. + + A generation mismatch means the Source cursor was regenerated and the + window identity no longer holds, which stays a terminal failure — except + for snapshots taken before the cursor row existed at all: the racing + window's first commit creates that row at generation 1, which is an + ordinary advance, not a reset. + """ + + if payload.cursor_generation == 0: + return generation == 1 and payload.after < sequence < payload.through + return generation == payload.cursor_generation and payload.after < sequence < payload.through + + +def _require_complete_window(rows, payload: SourceWindowPayload) -> None: + expected = payload.through - payload.after + if len(rows) != expected or any( + row.journal_position != payload.after + offset for offset, row in enumerate(rows, start=1) + ): + raise WorkExecutionError(category="retention", code="source_window_unavailable", retryable=False) + + +def _memory_result( + payload: SourceWindowPayload, + *, + current: int, + source_count: int, + code: str, +) -> WorkResult: + result = MemoryFlushResult( + previous_cursor=payload.after, + current_cursor=current, + high_watermark=payload.high_watermark, + source_count=source_count, + memory_ref=None, + ) + return WorkResult(code=code, payload=result.model_dump(mode="json", exclude_none=True)) + + +def _experience_result( + payload: SourceWindowPayload, + *, + source_count: int, + candidate_count: int, + candidate_ids: tuple[str, ...] = (), + code: str, + current_cursor: int | None = None, +) -> WorkResult: + result = ExperienceIncubationResult( + previous_cursor=payload.after, + current_cursor=payload.through if current_cursor is None else current_cursor, + high_watermark=payload.high_watermark, + source_count=source_count, + candidate_count=candidate_count, + candidate_ids=candidate_ids, + ) + return WorkResult(code=code, payload=result.model_dump(mode="json")) + + +def _profile_result(result: ProfileFlushResult) -> WorkResult: + return WorkResult(code=result.status, payload=result.model_dump(mode="json", exclude_none=True)) + + +def _digest(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +__all__ = [ + "CURRENT_WORK_PAYLOAD_VERSION", + "EXPERIENCE_WORK_KIND", + "MEMORY_WORK_KIND", + "PROFILE_WORK_KIND", + "ExperienceWorkDiscoverer", + "ExperienceWorkHandler", + "MemoryWorkDiscoverer", + "MemoryWorkHandler", + "ProfileWorkDiscoverer", + "ProfileWorkHandler", + "ProfileWorkPayload", + "ProfileWorkRunner", + "SourceWindowPayload", + "WorkRequester", + "experience_work_spec", + "memory_work_spec", +] diff --git a/src/powercontext/builtin/runtime/work_observability.py b/src/powercontext/builtin/runtime/work_observability.py new file mode 100644 index 0000000000..143f835c72 --- /dev/null +++ b/src/powercontext/builtin/runtime/work_observability.py @@ -0,0 +1,69 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded-label observability port for durable background work.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from powercontext.builtin.persistence.database import AsyncDatabase + from powercontext.builtin.persistence.work import WorkQueueStatistic, WorkRepository + + +class WorkObserver(Protocol): + """Receive work events containing only bounded dimensions.""" + + def observe_work_enqueue(self, kind: str, *, created: bool) -> None: ... + + def observe_work_claim(self, kind: str, *, latency_seconds: float) -> None: ... + + def observe_work_attempt( + self, + kind: str, + *, + outcome: str, + error_category: str, + duration_seconds: float, + ) -> None: ... + + def observe_work_lease_expiry(self, kind: str, *, outcome: str) -> None: ... + + def observe_scheduler_leadership(self, *, outcome: str) -> None: ... + + def set_work_queue(self, samples: Sequence[WorkQueueStatistic]) -> None: ... + + def set_runtime_members(self, counts: Mapping[str, int]) -> None: ... + + +async def refresh_work_queue( + database: AsyncDatabase, + repository: WorkRepository, + observer: WorkObserver | None, +) -> None: + """Best-effort refresh aggregate queue gauges from durable state.""" + + if observer is None: + return + try: + async with database.transaction() as connection: + samples = await repository.queue_statistics(connection) + observer.set_work_queue(samples) + except Exception: + return + + +__all__ = ["WorkObserver", "refresh_work_queue"] diff --git a/src/powercontext/builtin/runtime/worker.py b/src/powercontext/builtin/runtime/worker.py new file mode 100644 index 0000000000..c9eb0ffd1c --- /dev/null +++ b/src/powercontext/builtin/runtime/worker.py @@ -0,0 +1,418 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lease-aware Worker loop over the durable Work Ledger.""" + +from __future__ import annotations + +import asyncio +import logging +import random +from collections.abc import Awaitable, Callable, Iterable +from contextlib import AbstractContextManager, nullcontext, suppress +from dataclasses import dataclass +from typing import Protocol + +from powercontext._logging import log_safely +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + StoredWork, + WorkClaim, + WorkCommit, + WorkFailure, + WorkRepository, + WorkResult, +) +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.protocols import ( + RuntimeSpan, + RuntimeTraceContext, + RuntimeTracing, + runtime_trace_context, +) +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue + +ClaimReadiness = Callable[[], Awaitable[str]] +WorkAuthorizer = Callable[[WorkClaim], Awaitable[None]] +WorkSucceededObserver = Callable[[StoredWork], Awaitable[None]] +_EMPTY_HANDLERS = "at least one handler is required" +_INVALID_HANDLER_KIND = "handler kind must be a non-empty trimmed string" +_INVALID_HANDLER_VERSIONS = "handler versions must be positive" + +logger = logging.getLogger(__name__) + + +class WorkHandler(Protocol): + """Prepare one versioned task outside a database transaction.""" + + kind: str + supported_versions: frozenset[int] + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: ... + + +@dataclass(frozen=True) +class PreparedWork: + """Sanitized result and optional domain write for the final transaction.""" + + result: WorkResult | None + commit: WorkCommit | None = None + + +class WorkExecutionError(RuntimeError): + """A handler failure safe to classify in persistent operation metadata.""" + + def __init__(self, *, category: str, code: str, retryable: bool) -> None: + self.failure = WorkFailure(category=category, code=code, retryable=retryable) + super().__init__(f"work handler failed with {category}/{code}") + + +class DurableWorker: + """Claim no more than available slots and recover through lease expiry.""" + + def __init__( + self, + *, + database: AsyncDatabase, + worker_id: str, + handlers: Iterable[WorkHandler], + config: WorkerConfig, + repository: WorkRepository | None = None, + random_source: Callable[[float, float], float] = random.uniform, + claim_readiness: ClaimReadiness | None = None, + authorizer: WorkAuthorizer | None = None, + succeeded_observer: WorkSucceededObserver | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._database = database + self._worker_id = worker_id + self._config = config + self._repository = WorkRepository(observer=observer) if repository is None else repository + self._random_source = random_source + self._claim_readiness = claim_readiness + self._authorizer = authorizer + self._succeeded_observer = succeeded_observer + self._observer = observer + self._tracing = tracing + self._handlers = _handler_map(handlers) + self._stop_requested = asyncio.Event() + self._wake_requested = asyncio.Event() + self._claim_lock = asyncio.Lock() + self._running: set[asyncio.Task[None]] = set() + self._failed = False + + @property + def supported(self) -> dict[str, frozenset[int]]: + """Return bounded handler compatibility advertised by this Worker.""" + + return {kind: handler.supported_versions for kind, handler in self._handlers.items()} + + async def run_once(self) -> int: + """Claim current free capacity and await those attempts.""" + + async with self._claim_lock: + capacity = self._config.concurrency - len(self._running) + if capacity <= 0 or self._stop_requested.is_set(): + return 0 + if self._claim_readiness is not None and await self._claim_readiness() != ReadinessCheckStatus.READY: + return 0 + with self._background( + "work.claim", + operation="work.claim", + attributes={"powercontext.work.claim.capacity": capacity}, + ) as span: + claims = await self._claim_capacity(capacity) + if span is not None: + span.set_attributes({"powercontext.work.claim.count": len(claims)}) + tasks = {asyncio.create_task(self._execute(claim)) for claim in claims} + self._running.update(tasks) + for task in tasks: + task.add_done_callback(self._running.discard) + if tasks: + await asyncio.gather(*tasks) + await refresh_work_queue(self._database, self._repository, self._observer) + return len(claims) + + async def _claim_capacity(self, capacity: int) -> tuple[WorkClaim, ...]: + """Claim at most one lane per short transaction.""" + + claims: list[WorkClaim] = [] + supported = self.supported + for _ in range(capacity): + async with self._database.transaction() as connection: + claimed = await self._repository.claim( + connection, + worker_id=self._worker_id, + supported=supported, + lease_seconds=self._config.lease_seconds, + limit=1, + ) + if claimed: + claims.extend(claimed) + else: + break + return tuple(claims) + + async def run(self) -> None: + """Poll until shutdown, executing each claimed batch concurrently.""" + + while not self._stop_requested.is_set(): + self._wake_requested.clear() + try: + claimed = await self.run_once() + self._failed = False + except asyncio.CancelledError: + raise + except Exception: + self._failed = True + claimed = 0 + if claimed: + continue + with suppress(TimeoutError): + await asyncio.wait_for(self._wake_requested.wait(), timeout=self._config.poll_seconds) + + def notify(self) -> None: + """Wake the polling loop after a producer commits new work.""" + + self._wake_requested.set() + + async def readiness(self) -> str: + """Report database-loop and registered payload compatibility.""" + + if self._failed: + return ReadinessCheckStatus.UNAVAILABLE + if self._claim_readiness is not None: + dependency = await self._claim_readiness() + if dependency != ReadinessCheckStatus.READY: + return dependency + async with self._database.transaction() as connection: + unknown = await self._repository.unsupported_head_count(connection, self.supported) + return ReadinessCheckStatus.MISCONFIGURED if unknown else ReadinessCheckStatus.READY + + async def stop(self) -> None: + """Stop claiming, then give in-flight handlers their configured grace.""" + + self._stop_requested.set() + self._wake_requested.set() + if not self._running: + return + _, pending = await asyncio.wait(self._running, timeout=self._config.shutdown_grace_seconds) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + async def _execute(self, claim: WorkClaim) -> None: + links = ( + () + if claim.previous_trace_id is None or claim.previous_span_id is None + else (RuntimeTraceContext(trace_id=claim.previous_trace_id, span_id=claim.previous_span_id),) + ) + with self._background( + "work.execute", + operation="work.execute", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.payload_version": claim.payload_version, + "powercontext.work.attempt": claim.attempt_no, + "powercontext.work.recovery_generation": claim.recovery_generation, + }, + links=links, + ) as span: + context = runtime_trace_context(span) + if context is not None and not await self._bind_attempt_trace(claim, context): + if span is not None: + span.set_outcome("stale") + return + outcome = await self._execute_claim(claim) + if span is not None: + span.set_outcome(outcome) + + async def _execute_claim(self, claim: WorkClaim) -> str: + handler = self._handlers[claim.kind] + heartbeat_stop = asyncio.Event() + heartbeat = asyncio.create_task(self._heartbeat(claim, heartbeat_stop)) + try: + if self._authorizer is not None: + await self._authorizer(claim) + prepared = await handler.prepare(claim) + if heartbeat.done() and heartbeat.exception() is not None: + await heartbeat + with self._stage( + "work.commit", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.payload_version": claim.payload_version, + }, + ): + async with self._database.transaction() as connection: + completed = await self._repository.complete( + connection, + claim, + prepared.result, + commit=prepared.commit, + ) + except asyncio.CancelledError: + raise + except StaleWorkClaimError: + return await self._finish_cancel_if_owned(claim) + except WorkExecutionError as error: + return await self._record_failure(claim, error.failure) + except Exception: + return await self._record_failure( + claim, + WorkFailure(category="internal", code="unhandled_handler_error", retryable=True), + ) + else: + await self._observe_success(completed) + return "succeeded" + finally: + heartbeat_stop.set() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _observe_success(self, work: StoredWork) -> None: + if self._succeeded_observer is None: + return + try: + await self._succeeded_observer(work) + except asyncio.CancelledError: + raise + except Exception as error: + log_safely( + logger, + logging.ERROR, + "Work success observer failed", + extra={ + "event": "work.success_observer_failed", + "work_kind": work.kind, + "error_type": type(error).__name__, + }, + ) + + async def _finish_cancel_if_owned(self, claim: WorkClaim) -> str: + """Converge a fenced cancelling claim without waiting for lease expiry.""" + + try: + async with self._database.transaction() as connection: + work = await self._repository.fail( + connection, + claim, + WorkFailure(category="cancellation", code="cancel_requested", retryable=False), + retry_delay_seconds=0, + ) + except StaleWorkClaimError: + return "stale" + return work.status.value + + async def _heartbeat(self, claim: WorkClaim, stop: asyncio.Event) -> None: + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=self._config.heartbeat_seconds) + except TimeoutError: + async with self._database.transaction() as connection: + await self._repository.heartbeat( + connection, + claim, + lease_seconds=self._config.lease_seconds, + ) + + async def _bind_attempt_trace(self, claim: WorkClaim, context: RuntimeTraceContext) -> bool: + try: + async with self._database.transaction() as connection: + await self._repository.record_attempt_trace( + connection, + claim, + trace_id=context.trace_id, + span_id=context.span_id, + ) + except StaleWorkClaimError: + return False + return True + + async def _record_failure(self, claim: WorkClaim, failure: WorkFailure) -> str: + ceiling = min( + self._config.retry_max_seconds, + self._config.retry_base_seconds * (2 ** max(claim.generation_attempt_no - 1, 0)), + ) + delay = self._random_source(0.0, ceiling) + try: + with self._stage( + "work.retry", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.error_category": failure.category, + "powercontext.work.retryable": failure.retryable, + }, + ) as span: + async with self._database.transaction() as connection: + work = await self._repository.fail( + connection, + claim, + failure, + retry_delay_seconds=max(0, round(delay)), + ) + if span is not None: + span.set_outcome(work.status.value) + return work.status.value + except StaleWorkClaimError: + return "stale" + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + def _background( + self, + name: str, + *, + operation: str, + attributes: dict[str, str | bool | int | float], + links: tuple[RuntimeTraceContext, ...] = (), + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.background(name, operation=operation, attributes=attributes, links=links) + + +def _handler_map(handlers: Iterable[WorkHandler]) -> dict[str, WorkHandler]: + registered: dict[str, WorkHandler] = {} + for handler in handlers: + if not handler.kind.strip() or handler.kind != handler.kind.strip(): + raise ValueError(_INVALID_HANDLER_KIND) + if not handler.supported_versions or any(version < 1 for version in handler.supported_versions): + raise ValueError(_INVALID_HANDLER_VERSIONS) + if handler.kind in registered: + message = f"duplicate handler kind: {handler.kind}" + raise ValueError(message) + registered[handler.kind] = handler + if not registered: + raise ValueError(_EMPTY_HANDLERS) + return registered + + +__all__ = [ + "DurableWorker", + "PreparedWork", + "WorkExecutionError", + "WorkHandler", +] diff --git a/src/powercontext/cli/config.py b/src/powercontext/cli/config.py index e7dd29ac1e..1de492c170 100644 --- a/src/powercontext/cli/config.py +++ b/src/powercontext/cli/config.py @@ -681,6 +681,11 @@ def _validate_builtin_runtime(values: Mapping[str, str]) -> None: handoff_report=settings.handoff_report, inference=settings.inference, external_skills=settings.external_skills, + deployment=settings.deployment, + coordination=settings.coordination, + worker=settings.worker, + operations=settings.operations, + rate_limit=settings.rate_limit, ) ) ) diff --git a/src/powercontext/client/__init__.py b/src/powercontext/client/__init__.py index 26cb3dcced..ccac48a5f7 100644 --- a/src/powercontext/client/__init__.py +++ b/src/powercontext/client/__init__.py @@ -19,6 +19,8 @@ ClientError, ForbiddenResponseError, InvalidResponseError, + OperationFailedError, + OperationPendingError, ServerResponseError, TransportError, UnauthorizedResponseError, @@ -43,6 +45,8 @@ "ClientError", "ForbiddenResponseError", "InvalidResponseError", + "OperationFailedError", + "OperationPendingError", "PowerContextClient", "ReceiverSyncResult", "RemoteConnectorWorker", diff --git a/src/powercontext/client/errors.py b/src/powercontext/client/errors.py index c261735f3a..9a9216b718 100644 --- a/src/powercontext/client/errors.py +++ b/src/powercontext/client/errors.py @@ -42,6 +42,24 @@ def __init__(self, path: str, *, request_id: str | None) -> None: super().__init__(f"response from {path} violated the API schema") +class OperationPendingError(ClientError): + """Raised when a durable operation outlives the Client deadline.""" + + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} is still pending") + + +class OperationFailedError(ClientError): + """Raised when a durable operation reaches failed or cancelled.""" + + def __init__(self, operation_id: str, *, code: str | None) -> None: + self.operation_id = operation_id + self.code = code + suffix = "" if code is None else f" ({code})" + super().__init__(f"operation {operation_id} did not succeed{suffix}") + + class ServerResponseError(ClientError): """Raised when the Server returns a non-success status.""" @@ -103,6 +121,8 @@ def server_response_error( "ClientError", "ForbiddenResponseError", "InvalidResponseError", + "OperationFailedError", + "OperationPendingError", "ServerResponseError", "TransportError", "UnauthorizedResponseError", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 3a065b3b14..0266e9064f 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -2128,6 +2128,82 @@ class RetryTopicMemoryProcessingRequest(BaseModel): block_id: Annotated[StrictStr, Field(max_length=36, min_length=1)] +class OperationStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + CANCELLING = "cancelling" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class DurableOperation(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + operation_id: Annotated[StrictStr, Field(max_length=256, min_length=1)] + kind: Annotated[ + StrictStr, + Field( + description="Durable work family, for example powercontext.memory.source-window.", + max_length=128, + min_length=1, + ), + ] + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + status: OperationStatus + state_version: Annotated[ + StrictInt, Field(description="Monotonic version to send as expected_version when cancelling or retrying.", ge=1) + ] + attempt_count: Annotated[StrictInt, Field(ge=0)] + max_attempts: Annotated[StrictInt, Field(ge=1)] + cancel_requested: Annotated[ + StrictBool, Field(description="True once cancellation was requested and the operation has not stopped yet.") + ] + available_at: AwareDatetime + created_at: AwareDatetime + updated_at: AwareDatetime + completed_at: Annotated[AwareDatetime | None, Field(description="Set once the operation reached a terminal state.")] + result_code: Annotated[ + StrictStr | None, Field(description="Sanitized result classification for a succeeded operation.") + ] + error_category: Annotated[StrictStr | None, Field(...)] + error_code: Annotated[StrictStr | None, Field(...)] + + +class ListOperationsRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: Annotated[StrictStr | None, Field(max_length=128, min_length=1)] = None + status: Annotated[ + StrictStr | None, + Field(description="One lifecycle state; unknown values are rejected with 422.", max_length=32, min_length=1), + ] = None + limit: Annotated[StrictInt, Field(ge=1, le=100)] = 50 + cursor: Annotated[StrictStr | None, Field(max_length=4096, min_length=1)] = None + + +class DurableOperationPage(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: list[DurableOperation] + next_cursor: Annotated[ + StrictStr | None, Field(description="Opaque cursor for the next page; null when this page is the last.") + ] + + +class MutateOperationRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_version: Annotated[ + StrictInt, Field(description="The state_version read before deciding to cancel or retry.", ge=1) + ] + + class TopicMemoryProcessingBlock(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index 4d8edb8e8f..c16972e466 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -74,6 +74,8 @@ DownloadRemoteSkillPackageRequest, DreamRun, DreamRunPage, + DurableOperation, + DurableOperationPage, EnableAllArtifactPluginsResponse, EnrollRemoteSkillTargetRequest, ExperienceArtifact, @@ -127,6 +129,7 @@ ListMemoryChangesResponse, ListMemoryEntriesRequest, ListMemoryEntriesResponse, + ListOperationsRequest, ListRemoteSkillTargetsRequest, ListRemoteSkillTargetsResponse, ListScopesRequest, @@ -134,6 +137,7 @@ MemoryCapacity, MemoryMutationResponse, MergeAtomicMemoryRequest, + MutateOperationRequest, PlanAccessOwnershipRequest, PrepareContextRequest, PrepareContextResult, @@ -494,6 +498,122 @@ class AccessRequirement(BaseModel): access=AccessRequirement(action=None, resource=None, scope_id_field=None, resolver="atomic_memory_state_access"), ) +LIST_OPERATIONS = Operation[ListOperationsRequest, DurableOperationPage]( + method="GET", + path="/v1/scopes/{scope_id}/operations", + operation_id="list_operations", + request_type=ListOperationsRequest, + request_location="query", + path_parameters=("scope_id",), + response_type=DurableOperationPage, + success_status=200, + summary="List durable operations in one Scope", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "One stable page of public durable operations.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 410: {"$ref": "#/components/responses/CursorExpired"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 500: {"$ref": "#/components/responses/InternalError"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + +GET_OPERATION = Operation[None, DurableOperation]( + method="GET", + path="/v1/scopes/{scope_id}/operations/{operation_id}", + operation_id="get_operation", + request_type=None, + request_location=None, + path_parameters=("scope_id", "operation_id"), + response_type=DurableOperation, + success_status=200, + summary="Read one durable operation", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The current operation state.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 500: {"$ref": "#/components/responses/InternalError"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + +CANCEL_OPERATION = Operation[MutateOperationRequest, DurableOperation]( + method="POST", + path="/v1/scopes/{scope_id}/operations/{operation_id}/cancel", + operation_id="cancel_operation", + request_type=MutateOperationRequest, + request_location="body", + path_parameters=("scope_id", "operation_id"), + response_type=DurableOperation, + success_status=200, + summary="Cancel one durable operation", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The operation after the cancellation request.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 500: {"$ref": "#/components/responses/InternalError"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement( + action="scope.contribute", resource="scope", scope_id_field="scope_id", resolver="request" + ), +) + +RETRY_OPERATION = Operation[MutateOperationRequest, DurableOperation]( + method="POST", + path="/v1/scopes/{scope_id}/operations/{operation_id}/retry", + operation_id="retry_operation", + request_type=MutateOperationRequest, + request_location="body", + path_parameters=("scope_id", "operation_id"), + response_type=DurableOperation, + success_status=200, + summary="Retry one failed durable operation", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The operation after the retry was accepted.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 500: {"$ref": "#/components/responses/InternalError"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement( + action="scope.contribute", resource="scope", scope_id_field="scope_id", resolver="request" + ), +) + CREATE_SUBJECT_SOURCE = Operation[CreateSubjectSourceRequest, CreateSubjectSourceResponse]( method="POST", path="/v1/scopes/{scope_id}/subject-sources", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index a3dd5a25c0..7fc1ca79b9 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -295,6 +295,340 @@ }, } }, + "/v1/scopes/{scope_id}/operations": { + "get": { + "tags": ["operations"], + "summary": "List durable operations in one Scope", + "description": "List a snapshot-bounded " + "page of public durable " + "work in one Scope, newest " + "first. Only the operation " + "families a Scope owner can " + "act on are listed; " + "internal maintenance work " + "stays invisible and reads " + "as not found. Use the " + "returned next_cursor to " + "continue.", + "operationId": "list_operations", + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "parameters": [ + { + "name": "scope_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + { + "name": "kind", + "in": "query", + "required": False, + "schema": {"type": "string", "minLength": 1, "maxLength": 128}, + }, + { + "name": "status", + "in": "query", + "required": False, + "schema": { + "type": "string", + "enum": [ + "queued", + "running", + "retry_wait", + "cancelling", + "succeeded", + "failed", + "cancelled", + ], + }, + }, + { + "name": "limit", + "in": "query", + "required": False, + "schema": {"type": "integer", "minimum": 1, "maximum": 100, "default": 50}, + }, + { + "name": "cursor", + "in": "query", + "required": False, + "schema": {"type": "string", "minLength": 1, "maxLength": 4096}, + }, + ], + "responses": { + "200": { + "description": "One stable page of public durable operations.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/DurableOperationPage"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "410": {"$ref": "#/components/responses/CursorExpired"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/scopes/{scope_id}/operations/{operation_id}": { + "get": { + "tags": ["operations"], + "summary": "Read one durable operation", + "description": "Read the " + "current " + "state of " + "one public " + "durable " + "operation " + "owned by " + "this Scope. " + "An " + "operation " + "belonging " + "to another " + "Scope, or " + "internal " + "maintenance " + "work, reads " + "as not " + "found " + "rather than " + "revealing " + "its " + "existence.", + "operationId": "get_operation", + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "parameters": [ + { + "name": "scope_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + ], + "responses": { + "200": { + "description": "The current operation state.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DurableOperation"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/scopes/{scope_id}/operations/{operation_id}/cancel": { + "post": { + "tags": ["operations"], + "summary": "Cancel one durable operation", + "description": "Request " + "cancellation " + "of " + "a " + "queued " + "or " + "running " + "operation. " + "expected_version " + "must " + "be " + "the " + "state_version " + "of " + "the " + "read " + "that " + "decided " + "to " + "cancel, " + "so " + "a " + "concurrent " + "transition " + "fails " + "with " + "409 " + "instead " + "of " + "silently " + "winning. " + "A " + "running " + "operation " + "moves " + "to " + "cancelling " + "and " + "stops " + "at " + "its " + "next " + "fence; " + "repeating " + "the " + "request " + "with " + "the " + "current " + "version " + "is " + "idempotent.", + "operationId": "cancel_operation", + "x-powercontext-access": { + "action": "scope.contribute", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "parameters": [ + { + "name": "scope_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/MutateOperationRequest"}} + }, + }, + "responses": { + "200": { + "description": "The operation after the cancellation request.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DurableOperation"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/scopes/{scope_id}/operations/{operation_id}/retry": { + "post": { + "tags": ["operations"], + "summary": "Retry one failed durable operation", + "description": "Grant " + "one " + "fresh " + "bounded " + "attempt " + "to a " + "failed " + "operation, " + "without " + "advancing " + "the " + "Cursor " + "or " + "discarding " + "the " + "Sources " + "it " + "was " + "selected " + "from. " + "expected_version " + "must " + "be " + "the " + "state_version " + "of " + "the " + "read " + "that " + "decided " + "to " + "retry. " + "Only " + "a " + "failed " + "operation " + "can " + "be " + "retried; " + "any " + "other " + "state " + "returns " + "409 " + "and " + "the " + "original " + "state " + "is " + "preserved.", + "operationId": "retry_operation", + "x-powercontext-access": { + "action": "scope.contribute", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "parameters": [ + { + "name": "scope_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/MutateOperationRequest"}} + }, + }, + "responses": { + "200": { + "description": "The operation after the retry was accepted.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DurableOperation"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, "/v1/scopes/{scope_id}/subject-sources": { "post": { "tags": ["profile"], @@ -11320,6 +11654,111 @@ "type": "object", "required": ["scope_id", "block_id"], }, + "OperationStatus": { + "type": "string", + "enum": ["queued", "running", "retry_wait", "cancelling", "succeeded", "failed", "cancelled"], + "description": "Lifecycle state of one durable operation.", + }, + "DurableOperation": { + "properties": { + "operation_id": {"type": "string", "maxLength": 256, "minLength": 1}, + "kind": { + "type": "string", + "maxLength": 128, + "minLength": 1, + "description": "Durable work family, for example powercontext.memory.source-window.", + }, + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "status": {"$ref": "#/components/schemas/OperationStatus"}, + "state_version": { + "type": "integer", + "minimum": 1.0, + "description": "Monotonic version to send as expected_version when cancelling or retrying.", + }, + "attempt_count": {"type": "integer", "minimum": 0.0}, + "max_attempts": {"type": "integer", "minimum": 1.0}, + "cancel_requested": { + "type": "boolean", + "description": "True once cancellation was requested and the operation has not stopped yet.", + }, + "available_at": {"type": "string", "format": "date-time"}, + "created_at": {"type": "string", "format": "date-time"}, + "updated_at": {"type": "string", "format": "date-time"}, + "completed_at": { + "type": "string", + "format": "date-time", + "description": "Set once the operation reached a terminal state.", + "nullable": True, + }, + "result_code": { + "type": "string", + "description": "Sanitized result classification for a succeeded operation.", + "nullable": True, + }, + "error_category": {"type": "string", "nullable": True}, + "error_code": {"type": "string", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "operation_id", + "kind", + "scope_id", + "status", + "state_version", + "attempt_count", + "max_attempts", + "cancel_requested", + "available_at", + "created_at", + "updated_at", + "completed_at", + "result_code", + "error_category", + "error_code", + ], + }, + "ListOperationsRequest": { + "properties": { + "kind": {"type": "string", "maxLength": 128, "minLength": 1, "nullable": True}, + "status": { + "type": "string", + "maxLength": 32, + "minLength": 1, + "description": "One lifecycle state; unknown values are rejected with 422.", + "nullable": True, + }, + "limit": {"type": "integer", "maximum": 100.0, "minimum": 1.0, "default": 50}, + "cursor": {"type": "string", "maxLength": 4096, "minLength": 1, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + }, + "DurableOperationPage": { + "properties": { + "items": {"items": {"$ref": "#/components/schemas/DurableOperation"}, "type": "array"}, + "next_cursor": { + "type": "string", + "description": "Opaque cursor for the next page; null when this page is the last.", + "nullable": True, + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["items", "next_cursor"], + }, + "MutateOperationRequest": { + "properties": { + "expected_version": { + "type": "integer", + "minimum": 1.0, + "description": "The state_version read before deciding to cancel or retry.", + } + }, + "additionalProperties": False, + "type": "object", + "required": ["expected_version"], + }, "TopicMemoryProcessingBlock": { "properties": { "block_id": {"type": "string"}, diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index a84b8fcd8c..387e7fdd3a 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -178,6 +178,11 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.skill_publications import SkillPublication +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + WorkStateConflictError, + WorkStatus, +) from powercontext.builtin.publication import ( ArtifactPublicationApplication, ArtifactPublicationConflictError, @@ -689,7 +694,10 @@ ArtifactTagPage, ArtifactTagSet, BackgroundPermissionGap, + DurableOperationPage, GetTopicMemoryProcessingBlockRequest, + ListOperationsRequest, + MutateOperationRequest, OwnershipGapItem, QueryArtifactTagsRequest, ReplaceArtifactTagsRequest, @@ -703,9 +711,9 @@ TopicMemoryProcessingStatusRequest, ) from powercontext.http._generated.models import Allowed as TransportOwnershipAllowed -from powercontext.http._generated.models import ( - ArtifactFamily as TransportArtifactFamily, -) +from powercontext.http._generated.models import ArtifactFamily as TransportArtifactFamily +from powercontext.http._generated.models import DurableOperation as TransportOperation +from powercontext.http._generated.models import OperationStatus as TransportOperationStatus from powercontext.http._generated.models import OwnershipGapKind as TransportOwnershipGapKind from powercontext.http._generated.models import ScopeId1 as TransportScopeId from powercontext.http._generated.models import ( @@ -721,6 +729,7 @@ APPLY_ACCESS_OWNERSHIP, APPROVE_CANDIDATE, BIND_ARTIFACT_HANDLE, + CANCEL_OPERATION, CAPTURE_CONTENT_SOURCE, CAPTURE_WORKING_NOTE, CHECK_ACCESS, @@ -767,6 +776,7 @@ GET_MEMORY_CAPACITY, GET_MEMORY_ENTRY, GET_MEMORY_ENTRY_TAGS, + GET_OPERATION, GET_PROFILE_POLICY, GET_PROMPT_CONFIGURATION, GET_READINESS, @@ -794,6 +804,7 @@ LIST_MANAGED_SKILLS, LIST_MEMORY_CHANGES, LIST_MEMORY_ENTRIES, + LIST_OPERATIONS, LIST_REMOTE_SKILL_TARGETS, LIST_SCOPES, LIST_SOURCES, @@ -830,6 +841,7 @@ RESOLVE_SCOPE_BINDING, RESOLVE_SCOPE_SELECTION, RETIRE_MEMORY_ENTRY, + RETRY_OPERATION, RETRY_TOPIC_MEMORY_PROCESSING, REVISE_CANDIDATE, REVISE_MEMORY_ENTRY, @@ -1400,6 +1412,7 @@ class ServerApplication(Protocol): remote_skills: _RemoteSkillApplication statistics: _StatisticsApplication handoff_report: HandoffReportApplication | None + operations: Any | None class _RuntimeNotReadyError(RuntimeError): @@ -1410,10 +1423,31 @@ class _PreconditionRequiredError(RuntimeError): """Raised when an Artifact mutation omits If-Match.""" +def _mount_boundary_routes( + app: FastAPI, + *, + infrastructure_only: bool, + route_profile: Literal["full", "health"], +) -> bool: + """Mount the health boundary and report whether the data plane stays unmounted. + + ``infrastructure_only`` exposes liveness alone. ``route_profile="health"`` adds readiness for + distributed Scheduler/Worker roles; metrics are mounted by the caller and every business route + stays on the API replicas. + """ + + _add_route(app, GET_LIVENESS, get_liveness) + if infrastructure_only: + return True + _add_route(app, GET_READINESS, get_readiness) + return route_profile == "health" + + def create_app( *, application: ServerApplication | None = None, infrastructure_only: bool = False, + route_profile: Literal["full", "health"] = "full", capability_provider: CapabilityProvider | None = None, readiness_probe: ReadinessProbe | None = None, lifespan: Lifespan[FastAPI] | None = None, @@ -1426,13 +1460,18 @@ def create_app( authentication_provider: AuthenticationProvider | None = None, allow_insecure_remote_http: bool = False, ) -> FastAPI: - """Build the HTTP adapter around an optional Runtime application binding.""" + """Build the HTTP adapter around an optional Runtime application binding. + + ``route_profile="health"`` exposes only the health and metrics boundary + (RFC 1430) for distributed Scheduler/Worker roles; the data plane is + served by the API replicas. + """ app = FastAPI( title=API_TITLE, version=API_VERSION, description=API_DESCRIPTION, - openapi_url=None if infrastructure_only else "/openapi.json", + openapi_url=None if infrastructure_only or route_profile == "health" else "/openapi.json", docs_url=None, redoc_url=None, lifespan=lifespan, @@ -1535,15 +1574,12 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: response.headers[REQUEST_ID_HEADER] = request_id return response - if infrastructure_only: - _add_route(app, GET_LIVENESS, get_liveness) + if _mount_boundary_routes(app, infrastructure_only=infrastructure_only, route_profile=route_profile): return app _add_route(app, CREATE_DREAM_RUN, create_dream_run) _add_route(app, GET_DREAM_RUN, get_dream_run) _add_route(app, LIST_DREAM_RUNS, list_dream_runs) - _add_route(app, GET_LIVENESS, get_liveness) - _add_route(app, GET_READINESS, get_readiness) _add_route(app, GET_SERVER_INFO, get_server_info) _add_route(app, GET_CAPABILITIES, get_capabilities) _add_route(app, LIST_ARTIFACT_PLUGINS, list_artifact_plugins) @@ -1624,6 +1660,10 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, SUBMIT_SOURCE_OBSERVATION, submit_source_observation) _add_route(app, COMMIT_CONNECTOR_CHECKPOINT, commit_connector_checkpoint) _add_route(app, FLUSH_MEMORY, flush_memory) + _add_route(app, LIST_OPERATIONS, list_operations) + _add_route(app, GET_OPERATION, get_operation) + _add_route(app, CANCEL_OPERATION, cancel_operation) + _add_route(app, RETRY_OPERATION, retry_operation) _add_route(app, REMEMBER_MEMORY, remember_memory) _add_route(app, SEARCH_MEMORY, search_memory) _add_route(app, PREPARE_BOOTSTRAP_CONTEXT, prepare_bootstrap_context) @@ -3478,6 +3518,117 @@ async def flush_memory( return mapping.flush_response(result) +def _require_operations(application: ServerApplication) -> Any: + operations = application.operations + if operations is None: + raise _RuntimeNotReadyError + return operations + + +def _operation_response(operation: Any) -> TransportOperation: + """Project one stored work row onto the public operation surface. + + ``payload``, ``logical_key`` and the lease internals stay private: a caller + sees the lifecycle it can act on, never the references the ledger keeps. + """ + + return TransportOperation( + operation_id=operation.work_id, + kind=operation.kind, + scope_id=operation.scope_id, + status=TransportOperationStatus(operation.status.value), + state_version=operation.state_version, + attempt_count=operation.attempt_count, + max_attempts=operation.max_attempts, + cancel_requested=operation.cancel_requested, + available_at=_aware_datetime(operation.available_at), + created_at=_aware_datetime(operation.created_at), + updated_at=_aware_datetime(operation.updated_at), + completed_at=None if operation.completed_at is None else _aware_datetime(operation.completed_at), + result_code=operation.result_code, + error_category=operation.error_category, + error_code=operation.error_code, + ) + + +async def _scoped_operation(manager: Any, scope_id: str, operation_id: str) -> Any: + """Load one operation, hiding anything that does not belong to this Scope. + + The row carries its own Scope, so the path Scope is an ownership assertion. + A mismatch reads as not found for the same reason maintenance work does: a + caller must not be able to probe another Scope's identifiers. + """ + + operation = await manager.get(operation_id) + if operation.scope_id != scope_id: + raise RepositoryNotFoundError("operation", operation_id) + return operation + + +def _list_operations_query( + http_request: Request, + kind: Annotated[str | None, Query(min_length=1, max_length=128)] = None, + status: Annotated[str | None, Query(min_length=1, max_length=32)] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1, max_length=4096)] = None, +) -> ListOperationsRequest: + allowed = {"kind", "status", "limit", "cursor"} + if set(http_request.query_params) - allowed: + raise InvalidBaseAccessRequestError("query", "contains unknown parameters") + if any(len(http_request.query_params.getlist(name)) > 1 for name in allowed): + raise InvalidBaseAccessRequestError("query", "parameters must be provided at most once") + if status is not None and status not in {item.value for item in WorkStatus}: + raise InvalidBaseAccessRequestError("status", "must be a known lifecycle state") + return ListOperationsRequest(kind=kind, status=status, limit=limit, cursor=cursor) + + +async def list_operations( + scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + request: Annotated[ListOperationsRequest, Depends(_list_operations_query)], + application: Annotated[ServerApplication, Depends(_require_application)], +) -> DurableOperationPage: + manager = _require_operations(application) + page = await manager.list( + scope_id=scope_id, + kind=request.kind, + status=None if request.status is None else WorkStatus(request.status), + cursor=request.cursor, + limit=request.limit, + ) + return DurableOperationPage(items=[_operation_response(item) for item in page.items], next_cursor=page.next_cursor) + + +async def get_operation( + scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + operation_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + application: Annotated[ServerApplication, Depends(_require_application)], +) -> TransportOperation: + manager = _require_operations(application) + return _operation_response(await _scoped_operation(manager, scope_id, operation_id)) + + +async def cancel_operation( + scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + operation_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + request: MutateOperationRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> TransportOperation: + manager = _require_operations(application) + await _scoped_operation(manager, scope_id, operation_id) + return _operation_response(await manager.cancel(operation_id, expected_version=request.expected_version)) + + +async def retry_operation( + scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + operation_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], + request: MutateOperationRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> TransportOperation: + manager = _require_operations(application) + await _scoped_operation(manager, scope_id, operation_id) + return _operation_response(await manager.retry(operation_id, expected_version=request.expected_version)) + + async def remember_memory( request: RememberMemoryRequest, application: Annotated[ServerApplication, Depends(_require_application)], @@ -6232,6 +6383,13 @@ def _map_governance_error(error: Exception) -> tuple[int, str, str, dict[str, An return status.HTTP_404_NOT_FOUND, "not_found", "The requested value was not found.", None if isinstance(error, StoredPayloadConflictError): return status.HTTP_409_CONFLICT, "generation_conflict", "The requested state is stale.", None + if isinstance(error, (WorkStateConflictError, StaleWorkClaimError)): + return ( + status.HTTP_409_CONFLICT, + "operation_conflict", + "The operation changed since it was read.", + None, + ) if isinstance(error, InvalidArtifactLifecycleError): return status.HTTP_422_UNPROCESSABLE_CONTENT, "invalid_lifecycle", str(error), None return None diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index b0d24635c7..b317f7adad 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -288,6 +288,11 @@ async def _processing_maintenance( inference=settings.inference, handoff_report=settings.handoff_report, external_skills=settings.external_skills, + deployment=settings.deployment, + coordination=settings.coordination, + worker=settings.worker, + operations=settings.operations, + rate_limit=settings.rate_limit, ) manifest = canonical_processing_manifest(config) database = settings.database @@ -985,6 +990,11 @@ async def _run_background_async(settings: ServerSettings, tracing: Any) -> None: handoff_report=settings.handoff_report, inference=settings.inference, external_skills=settings.external_skills, + deployment=settings.deployment, + coordination=settings.coordination, + worker=settings.worker, + operations=settings.operations, + rate_limit=settings.rate_limit, ) from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage @@ -1026,6 +1036,7 @@ async def _run_background_async(settings: ServerSettings, tracing: Any) -> None: model_usage_observer=None if metrics is None else metrics.observe_model_usage, worker_model_usage_observer=None if metrics is None else metrics.observe_worker_model_usage, processing_observer=None if metrics is None else metrics.observe_processing, + work_observer=metrics, worker_initializer=logging_bootstrap, worker_entrypoint_wrapper=WorkerTracing(settings.tracing).wrap, ) diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index 4fe26093c8..192d535234 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -116,6 +116,7 @@ from powercontext.server.metrics import CONTENT_TYPE_LATEST, HttpMetricsMiddleware, ServerMetrics from powercontext.server.middleware import AuthenticationMiddleware from powercontext.server.processing_security import build_worker_security +from powercontext.server.rate_limit import SharedRateLimitMiddleware from powercontext.server.request_id import HttpRequestIdMiddleware from powercontext.server.settings import MissingAuthenticationProviderError, ServerSettings from powercontext.server.tracing import HttpTracingMiddleware, ServerTracing @@ -198,8 +199,25 @@ def create_server_app( # noqa: C901 handoff_report=resolved.handoff_report, inference=resolved.inference, external_skills=resolved.external_skills, + deployment=resolved.deployment, + coordination=resolved.coordination, + worker=resolved.worker, + operations=resolved.operations, + rate_limit=resolved.rate_limit, ) metrics = ServerMetrics(role=resolved.runtime.artifact_processing_role) if resolved.metrics.enabled else None + # Distributed Scheduler/Worker roles expose only the health/metrics boundary + # (RFC 1430); the authenticated API replicas serve the data plane. + route_profile = ( + "health" + if resolved.deployment.mode == "distributed" and resolved.deployment.role in {"scheduler", "worker"} + else "full" + ) + if route_profile == "health" and (resolved.dashboard.enabled or resolved.mcp.enabled): + raise ValueError( # noqa: TRY003 + "Distributed Scheduler and Worker roles expose only health and metrics endpoints; " + "disable the Dashboard and MCP for these roles" + ) resolved_tracing = ( ServerTracing.context_only(role=resolved.runtime.artifact_processing_role) if tracing is None else tracing ) @@ -290,6 +308,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # noqa: C901 - shared model_usage_observer=None if metrics is None else metrics.observe_model_usage, worker_model_usage_observer=None if metrics is None else metrics.observe_worker_model_usage, processing_observer=None if metrics is None else metrics.observe_processing, + work_observer=metrics, scheduled_source_runner=scheduled_source_runner, scheduled_experience_runner=scheduled_experience_runner, worker_security=worker_security, @@ -433,10 +452,17 @@ def current_capabilities() -> Capabilities: dashboard_enabled=resolved.dashboard.enabled, ), ) + if resolved.rate_limit.enabled: + # Appended last so it runs inside the authentication middleware and sees + # the principal that request was bound to. + configured_middleware.append( + Middleware(SharedRateLimitMiddleware, config=resolved.rate_limit), + ) app = create_app( lifespan=lifespan, readiness_probe=readiness_probe, + route_profile=route_profile, middleware=configured_middleware, metrics=metrics, tracing=resolved_tracing, diff --git a/src/powercontext/server/metrics.py b/src/powercontext/server/metrics.py index cedff16918..b985949797 100644 --- a/src/powercontext/server/metrics.py +++ b/src/powercontext/server/metrics.py @@ -19,7 +19,7 @@ import asyncio import math import re -from collections.abc import Iterator, Mapping +from collections.abc import Iterator, Mapping, Sequence from contextlib import suppress from threading import RLock from time import perf_counter, time @@ -41,6 +41,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from typing_extensions import override +from powercontext.builtin.persistence.work import WorkQueueStatistic from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingModelUsageDelta from powercontext.limits import MAX_ARTIFACT_FAMILY_LENGTH from powercontext.server.context import is_internal_bridge @@ -292,6 +293,61 @@ def __init__(self, *, role: str = "server") -> None: "Failed passive Runtime snapshot refreshes.", registry=self.registry, ) + self.work_enqueues = Counter( + "powercontext_work_enqueues_total", + "Durable logical work enqueue decisions.", + ("kind", "outcome"), + registry=self.registry, + ) + self.work_claim_latency = Histogram( + "powercontext_work_claim_latency_seconds", + "Time from durable work creation to a Worker claim.", + ("kind",), + registry=self.registry, + ) + self.work_attempts = Counter( + "powercontext_work_attempts_total", + "Completed durable work attempts.", + ("kind", "outcome", "error_category"), + registry=self.registry, + ) + self.work_attempt_duration = Histogram( + "powercontext_work_attempt_duration_seconds", + "Durable work attempt execution duration.", + ("kind", "outcome"), + registry=self.registry, + ) + self.work_lease_expirations = Counter( + "powercontext_work_lease_expirations_total", + "Worker leases recovered after database-time expiry.", + ("kind", "outcome"), + registry=self.registry, + ) + self.scheduler_leadership_changes = Counter( + "powercontext_scheduler_leadership_changes_total", + "Scheduler leadership transitions observed by this process.", + ("outcome",), + registry=self.registry, + ) + self.work_queue_depth = Gauge( + "powercontext_work_queue_depth", + "Durable non-terminal work grouped by bounded kind and state.", + ("kind", "status"), + registry=self.registry, + ) + self.work_queue_oldest_age = Gauge( + "powercontext_work_queue_oldest_age_seconds", + "Age of the oldest durable work item in each bounded queue group.", + ("kind", "status"), + registry=self.registry, + ) + self.runtime_role_members = Gauge( + "powercontext_runtime_role_members", + "Compatible live runtime members grouped by role.", + ("role",), + registry=self.registry, + ) + self._work_queue_labels: set[tuple[str, str]] = set() self._processing = _ProcessingMetricsCollector() self.registry.register(self) self.set_runtime_scopes(0, 0) @@ -413,6 +469,48 @@ def refresh_runtime(self, runtime: BuiltinRuntime | None) -> None: self._processing = processing self.operational.publish(operational) + def observe_work_enqueue(self, kind: str, *, created: bool) -> None: + self.work_enqueues.labels(kind=kind, outcome="created" if created else "joined").inc() + + def observe_work_claim(self, kind: str, *, latency_seconds: float) -> None: + self.work_claim_latency.labels(kind=kind).observe(max(0.0, latency_seconds)) + + def observe_work_attempt( + self, + kind: str, + *, + outcome: str, + error_category: str, + duration_seconds: float, + ) -> None: + self.work_attempts.labels( + kind=kind, + outcome=outcome, + error_category=error_category, + ).inc() + self.work_attempt_duration.labels(kind=kind, outcome=outcome).observe(max(0.0, duration_seconds)) + + def observe_work_lease_expiry(self, kind: str, *, outcome: str) -> None: + self.work_lease_expirations.labels(kind=kind, outcome=outcome).inc() + + def observe_scheduler_leadership(self, *, outcome: str) -> None: + self.scheduler_leadership_changes.labels(outcome=outcome).inc() + + def set_work_queue(self, samples: Sequence[WorkQueueStatistic]) -> None: + current = {(sample.kind, str(sample.status)) for sample in samples} + for kind, status in self._work_queue_labels - current: + self.work_queue_depth.labels(kind=kind, status=status).set(0) + self.work_queue_oldest_age.labels(kind=kind, status=status).set(0) + for sample in samples: + status = str(sample.status) + self.work_queue_depth.labels(kind=sample.kind, status=status).set(sample.depth) + self.work_queue_oldest_age.labels(kind=sample.kind, status=status).set(sample.oldest_age_seconds) + self._work_queue_labels = current + + def set_runtime_members(self, counts: Mapping[str, int]) -> None: + for role in ("all", "api", "scheduler", "worker"): + self.runtime_role_members.labels(role=role).set(counts.get(role, 0)) + def collect(self) -> Iterator[Metric]: # Capture the pair before yielding: registry readers in other threads must # never combine processing counts with a different freshness observation. diff --git a/src/powercontext/server/rate_limit.py b/src/powercontext/server/rate_limit.py new file mode 100644 index 0000000000..3219a549a2 --- /dev/null +++ b/src/powercontext/server/rate_limit.py @@ -0,0 +1,122 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared fixed-window admission for external HTTP requests. + +The window lives in the deployment database rather than in process memory, so +every API replica charges the same principal budget. The middleware runs after +principal binding: without a bound principal it falls back to the client host, +which keeps an unauthenticated deployment from sharing one global window. +""" + +from __future__ import annotations + +import hashlib + +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.rate_limit import RateLimitDecision, RateLimitRepository +from powercontext.builtin.runtime.config import RateLimitConfig +from powercontext.http import ErrorDetail, ErrorResponse +from powercontext.server.context import current_principal, is_internal_bridge + +POLICY_ID = "server.http" + +# Liveness, readiness and metrics are scraped on a fixed interval and must not +# consume a caller's budget. +_EXEMPT_PREFIXES = ("/health/live", "/health/ready", "/metrics") + + +class SharedRateLimitMiddleware: + """Charge each external request to its principal's fixed window.""" + + def __init__( + self, + app: ASGIApp, + *, + config: RateLimitConfig, + repository: RateLimitRepository | None = None, + ) -> None: + self.app = app + self._config = config + self._repository = RateLimitRepository() if repository is None else repository + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or is_internal_bridge(): + await self.app(scope, receive, send) + return + if scope.get("path", "").startswith(_EXEMPT_PREFIXES): + await self.app(scope, receive, send) + return + database = _runtime_database(scope) + if database is None: + await self.app(scope, receive, send) + return + decision = await self._consume(database, _principal_key(scope)) + if not decision.allowed: + response = JSONResponse( + ErrorResponse( + error=ErrorDetail( + code="rate_limited", + message="The request budget for this principal is exhausted.", + details=None, + ) + ).model_dump(mode="json"), + status_code=429, + headers={"Retry-After": str(decision.retry_after_seconds)}, + ) + await response(scope, receive, send) + return + await self.app(scope, receive, send) + + async def _consume(self, database: AsyncDatabase, principal_key: str) -> RateLimitDecision: + async with database.transaction() as connection: + return await self._repository.consume( + connection, + principal_key=principal_key, + policy_id=POLICY_ID, + limit=self._config.requests, + window_seconds=self._config.window_seconds, + ) + + +def _runtime_database(scope: Scope) -> AsyncDatabase | None: + """Return the composed runtime's database, or ``None`` outside the data plane.""" + + app = scope.get("app") + application = None if app is None else getattr(app.state, "application", None) + if application is None: + return None + try: + return application.primary_database + except RuntimeError: + # The lifespan has not opened the primary database yet (or already closed it). + return None + + +def _principal_key(scope: Scope) -> str: + """Derive the 64-character digest the repository requires as a window key.""" + + principal = current_principal() + if principal is not None: + identity = f"{principal.type}:{principal.id}" + else: + client = scope.get("client") + identity = f"anonymous:{client[0] if client else 'unknown'}" + return hashlib.sha256(identity.encode()).hexdigest() + + +__all__ = ["POLICY_ID", "SharedRateLimitMiddleware"] diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index 4242fbd50c..7d0bed7f74 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -33,11 +33,16 @@ from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime.config import ( + CoordinationConfig, DatabaseConfig, + DeploymentConfig, ExternalSkillsConfig, HandoffReportConfig, InferenceConfig, + OperationsConfig, + RateLimitConfig, RuntimeConfig, + WorkerConfig, ) from powercontext.paths import default_database_path, default_seekdb_path, powercontext_data_dir, sqlite_url from powercontext.transport import is_loopback_host @@ -270,6 +275,11 @@ class ServerSettings(BaseSettings): handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) inference: InferenceConfig = Field(default_factory=InferenceConfig) external_skills: ExternalSkillsConfig = Field(default_factory=ExternalSkillsConfig) + deployment: DeploymentConfig = Field(default_factory=DeploymentConfig) + coordination: CoordinationConfig = Field(default_factory=CoordinationConfig) + worker: WorkerConfig = Field(default_factory=WorkerConfig) + operations: OperationsConfig = Field(default_factory=OperationsConfig) + rate_limit: RateLimitConfig = Field(default_factory=RateLimitConfig) @classmethod @override @@ -316,7 +326,13 @@ def resolve_workspace(cls, value: Path) -> Path: @model_validator(mode="after") def configure_default_local_skill_targets(self) -> ServerSettings: if "external_skills" not in self.model_fields_set: - self.external_skills = _default_local_external_skills(self.workspace) + if self.deployment.mode == "distributed": + # A distributed deployment rejects host-local Skill targets, so + # injecting the workspace defaults here would make every + # distributed configuration invalid before it is read. + self.external_skills = ExternalSkillsConfig() + else: + self.external_skills = _default_local_external_skills(self.workspace) return self @field_validator("public_url") diff --git a/src/powercontext/server/tracing.py b/src/powercontext/server/tracing.py index 5821c35a01..77b649c85d 100644 --- a/src/powercontext/server/tracing.py +++ b/src/powercontext/server/tracing.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio -from collections.abc import Iterator, Mapping +from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager, suppress from contextvars import ContextVar from importlib import import_module @@ -32,11 +32,22 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.trace.id_generator import RandomIdGenerator from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ParentBased -from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer, set_span_in_context +from opentelemetry.trace import ( + Link, + Span, + SpanContext, + SpanKind, + Status, + StatusCode, + TraceFlags, + Tracer, + set_span_in_context, +) from starlette.types import ASGIApp, Message, Receive, Scope, Send from typing_extensions import override from powercontext._logging import error_category +from powercontext.builtin.runtime.protocols import RuntimeTraceContext from powercontext.server.context import bind_request_id, is_internal_bridge, reset_request_id from powercontext.server.settings import TracingConfig from powercontext.server.telemetry import service_resource @@ -122,6 +133,7 @@ def start_span( kind: SpanKind, attributes: dict[str, Any], context: Context | None = None, + links: Sequence[Link] = (), ) -> _ActiveSpan: return _ActiveSpan.start( self.tracer, @@ -129,6 +141,7 @@ def start_span( kind=kind, attributes=attributes, context=context, + links=links, ) @contextmanager @@ -166,6 +179,7 @@ def background( *, operation: str, attributes: Mapping[str, _TraceAttribute], + links: Sequence[RuntimeTraceContext] = (), ) -> Iterator[_ActiveSpan]: """Trace one scheduled activation as an independent trace root.""" @@ -184,6 +198,7 @@ def background( "powercontext.operation.name": operation, "powercontext.operation.unit": "background", }, + links=_otel_links(links), ) try: yield span @@ -244,6 +259,19 @@ def __init__(self, span: Span | None, token: Token[Context] | None) -> None: def outcome(self) -> str | None: return self._outcome + @property + def trace_context(self) -> RuntimeTraceContext | None: + if self.span is None: + return None + with suppress(Exception): + context = self.span.get_span_context() + if context.is_valid: + return RuntimeTraceContext( + trace_id=f"{context.trace_id:032x}", + span_id=f"{context.span_id:016x}", + ) + return None + def set_outcome(self, outcome: str) -> None: self._outcome = outcome @@ -256,10 +284,11 @@ def start( kind: SpanKind, attributes: dict[str, Any], context: Context | None, + links: Sequence[Link] = (), ) -> _ActiveSpan: span: Span | None = None try: - span = tracer.start_span(name, context=context, kind=kind, attributes=attributes) + span = tracer.start_span(name, context=context, kind=kind, attributes=attributes, links=links) token = otel_context.attach(set_span_in_context(span, context)) except Exception: if span is not None: @@ -415,6 +444,23 @@ async def on_request( return result +def _otel_links(contexts: Sequence[RuntimeTraceContext]) -> tuple[Link, ...]: + links: list[Link] = [] + for context in contexts: + try: + span_context = SpanContext( + trace_id=int(context.trace_id, 16), + span_id=int(context.span_id, 16), + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ) + except (TypeError, ValueError): + continue + if span_context.is_valid: + links.append(Link(span_context)) + return tuple(links) + + def configure_server_tracing(config: TracingConfig, *, role: str = "server") -> ServerTracing: """Build request context and optional OTLP export from standard OTel configuration.""" diff --git a/tests/builtin/artifacts/profile/test_scheduler.py b/tests/builtin/artifacts/profile/test_scheduler.py index 72b4cc241c..10088514ea 100644 --- a/tests/builtin/artifacts/profile/test_scheduler.py +++ b/tests/builtin/artifacts/profile/test_scheduler.py @@ -14,12 +14,10 @@ import asyncio -from datetime import datetime +from datetime import UTC, datetime from functools import partial -from zoneinfo import ZoneInfo import pytest -from apscheduler.triggers.cron import CronTrigger from pydantic import ValidationError from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING @@ -31,6 +29,7 @@ SpawnArtifactProcessingWorkerLauncher, ) from powercontext.builtin.runtime.composition import open_builtin_contexts +from powercontext.builtin.runtime.cron import CronSchedule from powercontext.builtin.runtime.family_processing import process_family_invocation from powercontext.builtin.scope import ScopeDraft @@ -38,9 +37,9 @@ def test_profile_cron_defaults_and_validation(): config = RuntimeConfig() assert not config.profile_schedule_enabled - trigger = CronTrigger.from_crontab(config.profile_cron, timezone=ZoneInfo(config.profile_timezone)) - now = datetime(2026, 9, 7, 1, 0, tzinfo=ZoneInfo("Asia/Shanghai")) - assert trigger.get_next_fire_time(None, now) == now.replace(hour=2) + schedule = CronSchedule.parse(config.profile_cron, config.profile_timezone) + now = datetime(2026, 9, 6, 17, 0, tzinfo=UTC) + assert schedule.next_after(now) == now.replace(hour=18, tzinfo=None) with pytest.raises((ValidationError, ValueError)): RuntimeConfig(profile_cron="invalid") with pytest.raises((ValidationError, KeyError)): diff --git a/tests/builtin/persistence/test_artifact_storage_lifecycle.py b/tests/builtin/persistence/test_artifact_storage_lifecycle.py index 133afa3075..e272191b28 100644 --- a/tests/builtin/persistence/test_artifact_storage_lifecycle.py +++ b/tests/builtin/persistence/test_artifact_storage_lifecycle.py @@ -78,7 +78,7 @@ def test_fixed_artifact_storage_resumes_after_registration_columns(backend): # def _next_fixed_revision(bundle, directory): """A subsequent fixed-table addition leaves the space format untouched.""" shutil.copytree(bundle.directory, directory) - revision = "v1_5_0_002" + revision = "v1_5_0_003" sql = "CREATE TABLE pc_lifecycle_probe (id INTEGER NOT NULL, PRIMARY KEY (id))" name, layout = _parse_table(sql) resource = "schemas/mysql-lifecycle-probe.json" @@ -88,10 +88,10 @@ def _next_fixed_revision(bundle, directory): "revisions": {revision: {"statements": [sql], "changes": [{name: layout}]}}, }) ) - (directory / "versions/v1_5_0_002_probe.py").write_text( + (directory / "versions/v1_5_0_003_probe.py").write_text( f"revision = '{revision}'\ndown_revision = '{bundle.head}'\ndef upgrade():\n pass\n" ) - (directory / "mysql/versions/v1_5_0_002.py").write_text( + (directory / "mysql/versions/v1_5_0_003.py").write_text( "from alembic import context\n" f"revision = '{revision}'\ndown_revision = '{bundle.head}'\n" "def upgrade():\n context.config.attributes['execute_revision'](revision)\n" diff --git a/tests/builtin/persistence/test_atomic_description_vector_layout.py b/tests/builtin/persistence/test_atomic_description_vector_layout.py index 534a838205..7b05f1ddaf 100644 --- a/tests/builtin/persistence/test_atomic_description_vector_layout.py +++ b/tests/builtin/persistence/test_atomic_description_vector_layout.py @@ -43,7 +43,7 @@ def test_managed_sqlite_vector_schema_includes_registered_nullable_description(t database = tmp_path / "vector.db" runner = SQLiteMigrationRunner(database, bundle, configuration=configuration) result = runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) - assert result.revision == "v1_5_0_003" + assert result.revision == "v1_5_0_004" with closing(sqlite3.connect(database)) as connection: columns = {row[1]: row for row in connection.execute("PRAGMA table_info(pc_atomic_memory_current)")} assert columns["description"][2:4] == ("VARCHAR(200)", 0) @@ -156,11 +156,11 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm upgraded = SQLiteMigrationRunner(database, current, configuration=configuration) plan = upgraded.plan() assert plan.source_revision == "v1_4_1_004" - assert plan.target_revision == "v1_5_0_003" - assert plan.revisions == ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") + assert plan.target_revision == "v1_5_0_004" + assert plan.revisions == ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", "v1_5_0_004") assert plan.domain_steps == ("artifact_storage", "atomic_memory_evidence", "atomic_memory_evidence_projection") assert upgraded.evidence_path.read_bytes() == original_evidence_bytes - assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_003" + assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_004" assert upgraded.verify().state == "ready" evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) for revision, checksum in original_evidence["release_checksums"].items(): diff --git a/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py index 6e677d1d05..1e718bb29f 100644 --- a/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py +++ b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py @@ -34,6 +34,17 @@ EVIDENCE_REVISION = "v1_5_0_002" PROJECTION_REVISION = "v1_5_0_003" +LEDGER_REVISION = "v1_5_0_004" +LEDGER_TABLES = { + "pc_work_lanes", + "pc_work_items", + "pc_work_keys", + "pc_work_attempts", + "pc_scheduler_leases", + "pc_scheduler_scans", + "pc_runtime_members", + "pc_rate_limit_windows", +} EVIDENCE_TABLES = { "pc_atomic_memory_evidence_schema", "pc_atomic_memory_evidence_history", @@ -159,8 +170,15 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, plan = upgraded.plan() assert plan.state == "migration_required" assert plan.source_revision == "v1_4_1_004" - assert plan.target_revision == PROJECTION_REVISION - assert plan.revisions == ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", EVIDENCE_REVISION, PROJECTION_REVISION) + assert plan.target_revision == LEDGER_REVISION + assert plan.revisions == ( + "v1_4_1_005", + "v1_4_1_006", + "v1_5_0_001", + EVIDENCE_REVISION, + PROJECTION_REVISION, + LEDGER_REVISION, + ) assert plan.domain_steps == ( "artifact_storage", "atomic_memory_evidence", @@ -169,7 +187,7 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, assert upgraded.evidence_path.read_bytes() == old_evidence_bytes result = upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) assert result.changed - assert result.revision == PROJECTION_REVISION + assert result.revision == LEDGER_REVISION assert result.backup_state == "completed" # Reconstruct both bundle and runner: verification must use retained on-disk @@ -186,11 +204,12 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, "v1_5_0_001", EVIDENCE_REVISION, PROJECTION_REVISION, + LEDGER_REVISION, } ready = reopened.plan() assert ready.state == "ready" assert ready.revisions == () - assert reopened.verify().revision == PROJECTION_REVISION + assert reopened.verify().revision == LEDGER_REVISION assert not apply(reopened).changed assert reopened.evidence_path.read_bytes() == completed_evidence_bytes with closing(sqlite3.connect(database)) as connection: @@ -230,18 +249,23 @@ def test_evidence_projection_upgrade_preserves_registered_evidence_revision(tmp_ shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - removed = manifest["revision_resources"].pop(PROJECTION_REVISION) - removed.append("atomic_memory_evidence_projection.py") - revision = current.script.get_revision(PROJECTION_REVISION) - removed.append(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) - removed.append(f"mysql/versions/{PROJECTION_REVISION}.py") + # Drop every step appended after the evidence revision: the projection and the + # Work Ledger, so this copy ends where the older released bundle ended. + removed: list[str] = [] + for dropped in (PROJECTION_REVISION, LEDGER_REVISION): + removed.extend(manifest["revision_resources"].pop(dropped)) + revision = current.script.get_revision(dropped) + removed.append(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed.append(f"mysql/versions/{dropped}.py") + del manifest["sqlite_fingerprints"][dropped] + del manifest["domain_readiness"][dropped] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != dropped] for resource in removed: (directory / resource).unlink() - del manifest["sqlite_fingerprints"][PROJECTION_REVISION] - del manifest["domain_readiness"][PROJECTION_REVISION] - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != PROJECTION_REVISION] manifest["mysql_state_layouts"] = [name for name in manifest["mysql_state_layouts"] if name not in removed] manifest["affected_objects"].remove("pc_atomic_memory_evidence") + for table in LEDGER_TABLES: + manifest["affected_objects"].remove(table) manifest_path.write_text(json.dumps(manifest), encoding="utf-8") previous = MigrationBundle(directory, embedding_profile=embedding) assert previous.head == EVIDENCE_REVISION @@ -254,12 +278,10 @@ def test_evidence_projection_upgrade_preserves_registered_evidence_revision(tmp_ before = old.evidence_path.read_bytes() upgraded = SQLiteMigrationRunner(database, current, configuration=config) plan = upgraded.plan() - assert plan.revisions == (PROJECTION_REVISION,) + assert plan.revisions == (PROJECTION_REVISION, LEDGER_REVISION) assert plan.domain_steps == ("atomic_memory_evidence_projection",) assert old.evidence_path.read_bytes() == before - assert ( - upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == PROJECTION_REVISION - ) + assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == LEDGER_REVISION assert upgraded.verify().state == "ready" with closing(sqlite3.connect(database)) as connection: assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence").fetchone() == (0,) diff --git a/tests/builtin/persistence/test_coordination.py b/tests/builtin/persistence/test_coordination.py new file mode 100644 index 0000000000..24ac7454b6 --- /dev/null +++ b/tests/builtin/persistence/test_coordination.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + StaleCoordinatorLeaseError, + StaleScanStateError, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, SCHEDULER_LEASES_TABLE + + +def test_scheduler_lease_takeover_increments_fence_and_rejects_the_old_owner() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + repository = CoordinationRepository() + async with profile.database.transaction() as connection: + first = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-a", + lease_seconds=30, + ) + assert first is not None + renewed = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-a", + lease_seconds=30, + ) + assert renewed is not None + assert renewed.fence == first.fence + + async with profile.database.transaction() as connection: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == "scheduler") + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + second = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-b", + lease_seconds=30, + ) + + assert second is not None + assert second.fence > first.fence + with pytest.raises(StaleCoordinatorLeaseError): + async with profile.database.transaction() as connection: + await repository.assert_lease(connection, first) + + asyncio.run(scenario()) + + +def test_scheduler_scan_state_uses_versioned_keyset_continuation() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + repository = CoordinationRepository() + async with profile.database.transaction() as connection: + initial = await repository.load_scan(connection, "memory") + assert initial is None + first = await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 1, tzinfo=UTC), + continuation="project:a", + expected_version=None, + ) + assert first.state_version == 1 + assert first.continuation == "project:a" + + async with profile.database.transaction() as connection: + second = await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 2, tzinfo=UTC), + continuation=None, + expected_version=first.state_version, + ) + assert second.state_version == 2 + + with pytest.raises(StaleScanStateError): + async with profile.database.transaction() as connection: + await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 3, tzinfo=UTC), + continuation=None, + expected_version=first.state_version, + ) + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_database.py b/tests/builtin/persistence/test_database.py new file mode 100644 index 0000000000..784d125f10 --- /dev/null +++ b/tests/builtin/persistence/test_database.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import cast + +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now + + +def test_mysql_coordination_time_is_queried_in_utc() -> None: + statements: list[str] = [] + expected = datetime(2026, 9, 4, 12, 0, tzinfo=UTC).replace(tzinfo=None) + + class Result: + @staticmethod + def scalar_one() -> datetime: + return expected + + class Connection: + dialect = SimpleNamespace(name="mysql") + + @staticmethod + async def exec_driver_sql(statement: str) -> Result: + statements.append(statement) + return Result() + + value = asyncio.run(database_now(cast(AsyncConnection, Connection()))) + + assert value == expected + assert statements == ["SELECT UTC_TIMESTAMP(6)"] diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 6a9abe288a..74af6392ae 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -73,7 +73,7 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_003" + assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_004" assert plan.revisions == ( "s1400", "s1401", @@ -84,6 +84,7 @@ def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", + "v1_5_0_004", ) assert dump(source.database) == before result = apply(source) diff --git a/tests/builtin/persistence/test_mysql_initialization_preflight.py b/tests/builtin/persistence/test_mysql_initialization_preflight.py index 085a49aef0..89cc0162dc 100644 --- a/tests/builtin/persistence/test_mysql_initialization_preflight.py +++ b/tests/builtin/persistence/test_mysql_initialization_preflight.py @@ -253,7 +253,7 @@ def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypa before = database.read_bytes(), journal.path.read_bytes() plan = runner.plan() assert plan.state == "recovery_required" and plan.source_revision == predecessor - assert plan.revisions == ("v1_5_0_002", "v1_5_0_003") + assert plan.revisions == ("v1_5_0_002", "v1_5_0_003", "v1_5_0_004") assert (database.read_bytes(), journal.path.read_bytes()) == before assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [] @@ -278,7 +278,7 @@ def execute_revision(connection, _config, _source, target, guard): monkeypatch.setattr(runner, "_upgrade", execute_revision) result = runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) - assert result.state == "ready" and result.revision == "v1_5_0_003" + assert result.state == "ready" and result.revision == "v1_5_0_004" assert result.backup_state == "not_required" assert runner.verify().state == "ready" assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [ @@ -288,7 +288,7 @@ def execute_revision(connection, _config, _source, target, guard): assert completed is not None and completed.state == "complete" assert completed.maintenance_window_id == "interrupted-evidence-ddl" assert completed.source_revision is None and completed.backup_state == "not_required" - assert completed.last_revision == "v1_5_0_003" and completed.pending_revision is None + assert completed.last_revision == "v1_5_0_004" and completed.pending_revision is None assert completed.release_checksums == runner.checksums assert not runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True).changed diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index b5e27d1ec1..2894087ff8 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -706,7 +706,7 @@ def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( ) assert runner.plan(backup_policy="manual").source_revision == "s1300" result = _apply(runner, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_003" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_004" assert runner.verify().state == "ready" prepare_server_storage(settings) assert not _apply(runner, policy="manual").changed @@ -837,7 +837,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_003", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_004", "dedicated", b'{"history":"unchanged"}') (main_path,) = backend.evidence_directory.glob("*/maintenance.json") evidence = json.loads(main_path.read_text()) @@ -847,7 +847,7 @@ def inspect_history(connection, identity, guard): for inspect in (runner.plan, runner.verify, runner.apply): with pytest.raises(MigrationError, match="checksum_conflict"): inspect() - assert backend.connections().run(inspect_history) == ("v1_5_0_003", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_004", "dedicated", b'{"history":"unchanged"}') assert { path: path.read_bytes() for path in backend.evidence_directory.rglob("maintenance.json") } == original_evidence @@ -914,7 +914,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_003", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_004", b'{"history":"unchanged"}') def test_mysql_registered_projection_repair_preserves_authority_and_revision(backend: _BackendTarget) -> None: @@ -982,7 +982,7 @@ def inspect_content(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_content) == ("v1_5_0_003", content, "Preserve immutable nebula evidence.") + assert backend.connections().run(inspect_content) == ("v1_5_0_004", content, "Preserve immutable nebula evidence.") @pytest.mark.parametrize("origin", ("empty", "released")) @@ -1146,7 +1146,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan _load_full_server_baseline(backend) runner = _full_server_runner(backend) original = runner.plan(backup_policy="manual") - assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_003" + assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_004" _interrupt_after_ddl(runner, interruption) journals = list(backend.evidence_directory.rglob("maintenance.json")) assert len(journals) == 1 @@ -1168,7 +1168,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan result = _apply(resumed, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_003" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_004" assert result.backup_state == "user_confirmed" assert result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" @@ -1248,7 +1248,7 @@ def client_active(connection: Connection | None, _identity: BackendIdentity, _gu # interruption. An arbitrary remote failure cannot be acknowledged blindly. resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_003" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_004" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journals[0].read_text(encoding="utf-8")) diff --git a/tests/builtin/persistence/test_mysql_schema.py b/tests/builtin/persistence/test_mysql_schema.py index 5f0b711062..257e848660 100644 --- a/tests/builtin/persistence/test_mysql_schema.py +++ b/tests/builtin/persistence/test_mysql_schema.py @@ -180,6 +180,12 @@ def _assert_restore_layers_are_parent_first( def test_documented_obloader_restore_layers_are_parent_first() -> None: + transient_tables = { + "pc_rate_limit_windows", + "pc_runtime_members", + "pc_scheduler_leases", + "pc_scheduler_scans", + } restore_guides = ( Path("docs/en/docs/operate/troubleshoot.md"), Path("docs/zh/docs/operate/troubleshoot.md"), @@ -195,9 +201,13 @@ def test_documented_obloader_restore_layers_are_parent_first() -> None: assert len(set(restore_plans)) == 1 restore_layers = restore_plans[0] - _assert_restore_layers_are_parent_first( - restore_layers, tuple(table for table in BUILTIN_TABLES if table.name != "pc_artifact_vectors") + durable_tables = tuple( + table for table in BUILTIN_TABLES if table.name not in transient_tables and table.name != "pc_artifact_vectors" + ) + assert {table.name for table in BUILTIN_TABLES} - {table.name for table in durable_tables} == ( + transient_tables | {"pc_artifact_vectors"} ) + _assert_restore_layers_are_parent_first(restore_layers, durable_tables) def test_every_mysql_utf8mb4_key_stays_below_the_innodb_limit() -> None: diff --git a/tests/builtin/persistence/test_portable_storage_migration.py b/tests/builtin/persistence/test_portable_storage_migration.py index b083790f77..d016bed258 100644 --- a/tests/builtin/persistence/test_portable_storage_migration.py +++ b/tests/builtin/persistence/test_portable_storage_migration.py @@ -126,8 +126,9 @@ def test_registered_s1401_upgrade_preserves_completed_evidence_and_matches_fresh "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", + "v1_5_0_004", ) - assert apply(upgraded).revision == "v1_5_0_003" + assert apply(upgraded).revision == "v1_5_0_004" assert upgraded.verify().state == "ready" assert not apply(upgraded).changed fresh = SQLiteMigrationRunner(tmp_path / "fresh.db", bundle, configuration=config) @@ -186,11 +187,11 @@ def test_source_write_plugin_upgrade_preserves_data_and_registration(tmp_path, v bundle = server_bundle(embedding_profile=embedding) upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) assert upgraded.plan().revisions == ( - ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") + ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", "v1_5_0_004") if include_owner - else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") + else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", "v1_5_0_004") ) - assert apply(upgraded).revision == "v1_5_0_003" + assert apply(upgraded).revision == "v1_5_0_004" definition = b'{"configuration":{"id":"tickets","entry":"tickets"}}' with closing(sqlite3.connect(database)) as connection, connection: assert connection.execute("SELECT title FROM pc_scopes").fetchall() == [("preserved",)] diff --git a/tests/builtin/persistence/test_rate_limit.py b/tests/builtin/persistence/test_rate_limit.py new file mode 100644 index 0000000000..8df8614eec --- /dev/null +++ b/tests/builtin/persistence/test_rate_limit.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +from powercontext.builtin.persistence.rate_limit import RateLimitRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import RATE_LIMIT_WINDOWS_TABLE + + +def test_fixed_window_counter_is_shared_by_repository_callers() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=(RATE_LIMIT_WINDOWS_TABLE,)) as profile: + repository = RateLimitRepository() + decisions = [] + for _ in range(3): + async with profile.database.transaction() as connection: + decisions.append( + await repository.consume( + connection, + principal_key="a" * 64, + policy_id="api.default", + limit=2, + window_seconds=60, + ) + ) + + assert [decision.allowed for decision in decisions] == [True, True, False] + assert [decision.remaining for decision in decisions] == [1, 0, 0] + assert decisions[-1].retry_after_seconds > 0 + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_release140_server_migrations.py b/tests/builtin/persistence/test_release140_server_migrations.py index ae499d8836..c34faf9f72 100644 --- a/tests/builtin/persistence/test_release140_server_migrations.py +++ b/tests/builtin/persistence/test_release140_server_migrations.py @@ -144,6 +144,7 @@ def test_registered_sources_choose_distinct_lineages(tmp_path, release, expected "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", + "v1_5_0_004", ) diff --git a/tests/builtin/persistence/test_work.py b/tests/builtin/persistence/test_work.py new file mode 100644 index 0000000000..833d8e9a9a --- /dev/null +++ b/tests/builtin/persistence/test_work.py @@ -0,0 +1,418 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy import func, select, update + +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import WORK_ATTEMPTS_TABLE, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + WorkFailure, + WorkRepository, + WorkResult, + WorkSpec, + WorkStateConflictError, + WorkStatus, +) + + +def _spec(*, logical_key: str = "a" * 64, lane_key: str = "b" * 64) -> WorkSpec: + return WorkSpec( + kind="powercontext.memory.source-window.v1", + payload_version=1, + scope_id="project:test", + lane_key=lane_key, + logical_key=logical_key, + payload={"after": 0, "through": 2}, + ) + + +def test_enqueue_deduplicates_one_logical_window_and_orders_each_lane() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec()) + duplicate = await repository.enqueue(connection, _spec()) + second = await repository.enqueue(connection, _spec(logical_key="c" * 64)) + + assert first.created is True + assert duplicate.created is False + assert duplicate.work.work_id == first.work.work_id + assert first.work.lane_sequence == 1 + assert second.work.lane_sequence == 2 + + asyncio.run(scenario()) + + +def test_only_the_lane_head_can_be_claimed_and_completion_advances_it() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec()) + second = await repository.enqueue(connection, _spec(logical_key="c" * 64)) + + async with profile.database.transaction() as connection: + claims = await repository.claim( + connection, + worker_id="worker-a", + supported={first.work.kind: frozenset({1})}, + lease_seconds=120, + limit=2, + ) + assert len(claims) == 1 + assert claims[0].work_id == first.work.work_id + + async with profile.database.transaction() as connection: + await repository.complete( + connection, + claims[0], + WorkResult(code="processed", payload={"current_cursor": 2}), + ) + async with profile.database.transaction() as connection: + next_claims = await repository.claim( + connection, + worker_id="worker-b", + supported={second.work.kind: frozenset({1})}, + lease_seconds=120, + limit=2, + ) + + assert len(next_claims) == 1 + assert next_claims[0].work_id == second.work.work_id + + asyncio.run(scenario()) + + +def test_concurrent_workers_cannot_claim_the_same_work(tmp_path: Path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'work.db'}") + async with SQLiteProfile.open(config, tables=WORK_TABLES) as first_profile: + async with SQLiteProfile.open(config, tables=WORK_TABLES) as second_profile: + repository = WorkRepository() + async with first_profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + + async def claim(profile: SQLiteProfile, worker_id: str): + async with profile.database.transaction() as connection: + return await repository.claim( + connection, + worker_id=worker_id, + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + + results = await asyncio.gather( + claim(first_profile, "worker-a"), + claim(second_profile, "worker-b"), + ) + + assert sum(len(result) for result in results) == 1 + + asyncio.run(scenario()) + + +def test_expired_claim_is_fenced_before_another_attempt_runs() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + first = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + async with profile.database.transaction() as connection: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == first.work_id) + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + assert ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + expired_retry_delay_seconds=0, + ) + ) == () + + async with profile.database.transaction() as connection: + second = ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + assert second.fence > first.fence + with pytest.raises(StaleWorkClaimError): + async with profile.database.transaction() as connection: + await repository.complete(connection, first, WorkResult(code="late", payload={})) + + asyncio.run(scenario()) + + +def test_retry_budget_blocks_the_lane_until_an_operator_recovers_or_cancels() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec().model_copy(update={"max_attempts": 1})) + claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + failed = await repository.fail( + connection, + claim, + WorkFailure(category="provider", code="temporarily_unavailable", retryable=True), + retry_delay_seconds=0, + ) + + assert failed.status is WorkStatus.FAILED + async with profile.database.transaction() as connection: + duplicate = await repository.enqueue(connection, _spec()) + assert duplicate.created is False + assert duplicate.work.status is WorkStatus.FAILED + + async with profile.database.transaction() as connection: + recovered = await repository.retry( + connection, + failed.work_id, + expected_version=failed.state_version, + ) + assert recovered.status is WorkStatus.QUEUED + assert recovered.recovery_generation == 1 + + with pytest.raises(WorkStateConflictError): + async with profile.database.transaction() as connection: + await repository.cancel( + connection, + recovered.work_id, + expected_version=failed.state_version, + ) + + asyncio.run(scenario()) + + +def test_running_cancel_linearizes_before_commit_and_recovers_after_expiry() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + running = await repository.get(connection, claim.work_id) + + async with profile.database.transaction() as connection: + cancelling = await repository.cancel( + connection, + claim.work_id, + expected_version=running.state_version, + ) + assert cancelling.status is WorkStatus.CANCELLING + with pytest.raises(WorkStateConflictError): + async with profile.database.transaction() as connection: + await repository.cancel( + connection, + claim.work_id, + expected_version=cancelling.state_version, + ) + + committed = False + + async def domain_commit(_connection): + nonlocal committed + committed = True + return WorkResult(code="late", payload={}) + + with pytest.raises(StaleWorkClaimError): + async with profile.database.transaction() as connection: + await repository.complete(connection, claim, None, commit=domain_commit) + assert committed is False + + async with profile.database.transaction() as connection: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + assert ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + ) == () + cancelled = await repository.get(connection, claim.work_id) + assert cancelled.status is WorkStatus.CANCELLED + + asyncio.run(scenario()) + + +def test_retry_claim_links_to_the_previous_attempt_trace_context() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + first = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.record_attempt_trace( + connection, + first, + trace_id="1" * 32, + span_id="2" * 16, + ) + await repository.fail( + connection, + first, + WorkFailure(category="provider", code="unavailable", retryable=True), + retry_delay_seconds=0, + ) + + async with profile.database.transaction() as connection: + second = ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + assert second.previous_trace_id == "1" * 32 + assert second.previous_span_id == "2" * 16 + + asyncio.run(scenario()) + + +def test_retention_purges_only_old_success_and_cancelled_history() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + succeeded = await repository.enqueue(connection, _spec()) + success_claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={succeeded.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.complete(connection, success_claim, WorkResult(code="done", payload={})) + + cancelled = await repository.enqueue( + connection, + _spec(logical_key="c" * 64, lane_key="d" * 64), + ) + await repository.cancel( + connection, + cancelled.work.work_id, + expected_version=cancelled.work.state_version, + ) + + failed = await repository.enqueue( + connection, + _spec(logical_key="e" * 64, lane_key="f" * 64).model_copy(update={"max_attempts": 1}), + ) + failed_claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={failed.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.fail( + connection, + failed_claim, + WorkFailure(category="provider", code="secret_free_code", retryable=False), + retry_delay_seconds=0, + ) + + old = datetime(2000, 1, 1, tzinfo=UTC) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id.in_((succeeded.work.work_id, cancelled.work.work_id))) + .values(completed_at=old) + ) + assert ( + await repository.purge_terminal( + connection, + completed_before=datetime(2020, 1, 1, tzinfo=UTC), + limit=500, + ) + == 2 + ) + attempts = await connection.scalar(select(func.count()).select_from(WORK_ATTEMPTS_TABLE)) + + assert attempts == 1 + async with profile.database.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, succeeded.work.work_id) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, cancelled.work.work_id) + blocked = await repository.get(connection, failed.work.work_id) + assert blocked.status is WorkStatus.FAILED + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_cron.py b/tests/builtin/runtime/test_cron.py new file mode 100644 index 0000000000..1a89679a61 --- /dev/null +++ b/tests/builtin/runtime/test_cron.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for cron scheduling across DST transitions.""" + +from datetime import datetime + +from powercontext.builtin.runtime.cron import CronSchedule + + +def test_next_after_stays_in_the_future_during_the_repeated_hour() -> None: + """The fall-back repeated hour must never yield a UTC instant before the input.""" + + schedule = CronSchedule.parse("* * * * *", "America/New_York") + result = schedule.next_after(datetime(2026, 11, 1, 6, 15)) # noqa: DTZ001 - naive on purpose + assert result > datetime(2026, 11, 1, 6, 15) # noqa: DTZ001 - naive on purpose + # 06:15 UTC is 01:15 EDT->EST territory; the next minute is 01:16 EST = 06:16 UTC. + assert result == datetime(2026, 11, 1, 6, 16) # noqa: DTZ001 - naive on purpose + + +def test_next_after_returns_the_first_repeated_hour_occurrence() -> None: + """The earlier occurrence inside the repeated hour must not be skipped.""" + + schedule = CronSchedule.parse("* * * * *", "America/New_York") + # 05:15 UTC is 01:15 EDT (fold=0); 01:16 EDT = 05:16 UTC comes first. + assert schedule.next_after(datetime(2026, 11, 1, 5, 15)) == datetime(2026, 11, 1, 5, 16) # noqa: DTZ001 - naive on purpose + # 05:00 UTC is 01:00 EDT; the earliest future minute is 01:01 EDT = 05:01 UTC, + # not 02:00 EST = 07:00 UTC and not 01:01 EST = 06:01 UTC. + assert schedule.next_after(datetime(2026, 11, 1, 5, 0)) == datetime(2026, 11, 1, 5, 1) # noqa: DTZ001 - naive on purpose + + +def test_next_after_walks_every_minute_through_the_repeated_hour() -> None: + """Minute-by-minute scans advance one real minute per step across the transition.""" + + schedule = CronSchedule.parse("* * * * *", "America/New_York") + # 05:59 UTC is 01:59 EDT, the last minute before the fall-back. + previous = schedule.next_after(datetime(2026, 11, 1, 5, 59)) # noqa: DTZ001 - naive on purpose + assert previous == datetime(2026, 11, 1, 6, 0) # noqa: DTZ001 - naive on purpose + for expected in (datetime(2026, 11, 1, 6, minute) for minute in range(1, 60)): # noqa: DTZ001 - naive on purpose + result = schedule.next_after(previous) + assert result == expected + previous = result + # The first minute after the repeated hour is 02:00 EST = 07:00 UTC. + assert schedule.next_after(previous) == datetime(2026, 11, 1, 7, 0) # noqa: DTZ001 - naive on purpose + + +def test_next_after_emits_both_occurrences_of_an_ambiguous_wall_time() -> None: + """01:30 happens twice on the fall-back day and each occurrence is returned once.""" + + schedule = CronSchedule.parse("30 1 * * *", "America/New_York") + first = schedule.next_after(datetime(2026, 11, 1, 5, 0)) # noqa: DTZ001 - naive on purpose + assert first == datetime(2026, 11, 1, 5, 30) # 01:30 EDT # noqa: DTZ001 - naive on purpose + second = schedule.next_after(first) + assert second == datetime(2026, 11, 1, 6, 30) # 01:30 EST, the repeated hour # noqa: DTZ001 - naive on purpose + third = schedule.next_after(second) + assert third == datetime(2026, 11, 2, 6, 30) # next day, 01:30 EST # noqa: DTZ001 - naive on purpose + + +def test_next_after_skips_nonexistent_wall_times_in_the_spring_forward_gap() -> None: + """02:30 does not exist on the spring-forward day; the next day's match is used.""" + + schedule = CronSchedule.parse("30 2 * * *", "America/New_York") + result = schedule.next_after(datetime(2026, 3, 8, 6, 0)) # noqa: DTZ001 - naive on purpose + # 02:30 EST is skipped by the gap on 2026-03-08; the next 02:30 is on 03-09 (EDT, 06:30 UTC). + assert result == datetime(2026, 3, 9, 6, 30) # noqa: DTZ001 - naive on purpose + + +def test_next_after_is_monotonic_under_repeated_feedback() -> None: + """Feeding a returned instant back must always strictly advance.""" + + schedule = CronSchedule.parse("*/15 * * * *", "America/New_York") + current = datetime(2026, 11, 1, 5, 0) # noqa: DTZ001 - naive on purpose + for _ in range(24): + following = schedule.next_after(current) + assert following > current + current = following + + +def test_next_after_ordinary_days_are_unchanged() -> None: + """Non-transition days keep the documented UTC-naive semantics.""" + + schedule = CronSchedule.parse("30 9 * * *", "America/New_York") + # 2026-06-15: 09:30 EDT = 13:30 UTC. + assert schedule.next_after(datetime(2026, 6, 15, 12, 0)) == datetime(2026, 6, 15, 13, 30) # noqa: DTZ001 - naive on purpose + assert schedule.next_after(datetime(2026, 6, 15, 13, 30)) == datetime(2026, 6, 16, 13, 30) # noqa: DTZ001 - naive on purpose diff --git a/tests/builtin/runtime/test_deployment_config.py b/tests/builtin/runtime/test_deployment_config.py new file mode 100644 index 0000000000..672718ad5d --- /dev/null +++ b/tests/builtin/runtime/test_deployment_config.py @@ -0,0 +1,92 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import pytest +from pydantic import SecretStr, ValidationError + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import ( + BuiltinConfig, + CoordinationConfig, + DeploymentConfig, + InferenceConfig, + RuntimeConfig, + WorkerConfig, +) +from powercontext.builtin.runtime.processing_registry import processing_capabilities + +_OCEANBASE = OceanBaseConfig(url=SecretStr("mysql+aoceanbase://root@localhost:2881/powercontext?charset=utf8mb4")) + + +def test_single_node_all_is_the_backwards_compatible_default() -> None: + config = BuiltinConfig(database=SQLiteConfig()) + + assert config.deployment.mode == "single_node" + assert config.deployment.role == "all" + assert config.worker.concurrency == 4 + assert config.worker.lease_seconds == 120 + + +def test_distributed_requires_oceanbase_and_one_process_role() -> None: + with pytest.raises(ValidationError, match="distributed deployment requires OceanBase"): + BuiltinConfig( + database=SQLiteConfig(), + deployment=DeploymentConfig(mode="distributed", role="api", id="api-a"), + ) + + with pytest.raises(ValidationError, match="distributed deployment role must be"): + BuiltinConfig( + database=_OCEANBASE, + deployment=DeploymentConfig(mode="distributed", role="all", id="all-a"), + ) + + +def test_distributed_uses_the_work_ledger_instead_of_process_local_supervisors() -> None: + deployment = DeploymentConfig(mode="distributed", role="worker", id="worker-a") + config = BuiltinConfig( + database=_OCEANBASE, + deployment=deployment, + inference=InferenceConfig(generation_model="test"), + ) + + assert processing_capabilities(config) == () + + with pytest.raises(ValidationError, match="process-local artifact processing role"): + BuiltinConfig( + database=_OCEANBASE, + deployment=deployment, + runtime=RuntimeConfig(artifact_processing_role="background"), + ) + with pytest.raises(ValidationError, match="does not support Topic Memory processing"): + BuiltinConfig( + database=_OCEANBASE, + deployment=deployment, + runtime=RuntimeConfig(topic_memory_schedule_seconds=60), + ) + + +def test_worker_heartbeat_and_shutdown_must_fit_inside_the_lease() -> None: + with pytest.raises(ValidationError, match="heartbeat_seconds must be less than one third"): + WorkerConfig(lease_seconds=120, heartbeat_seconds=40) + + with pytest.raises(ValidationError, match="shutdown_grace_seconds must be less than lease_seconds"): + WorkerConfig(lease_seconds=120, shutdown_grace_seconds=120) + + +def test_scheduler_emits_only_the_supported_payload_version() -> None: + with pytest.raises(ValidationError): + CoordinationConfig.model_validate({"emit_payload_version": 2}) diff --git a/tests/builtin/runtime/test_durable_model_usage.py b/tests/builtin/runtime/test_durable_model_usage.py new file mode 100644 index 0000000000..acb4082887 --- /dev/null +++ b/tests/builtin/runtime/test_durable_model_usage.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest + +from powercontext.builtin.artifacts.experience import ExperienceCandidateInput +from powercontext.builtin.inference import GenerationResult, InferenceUsage +from powercontext.builtin.inference.usage import UsageReportingStructuredGenerator +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.statistics import StatisticsRepository +from powercontext.builtin.persistence.work import WorkRepository, WorkStatus +from powercontext.builtin.runtime import BuiltinConfig, RuntimeConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.work_handlers import ExperienceWorkHandler, experience_work_spec +from powercontext.builtin.runtime.worker import DurableWorker +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import ContentCapture +from powercontext.builtin.statistics import ModelUsageOperation, ModelUsagePurpose +from powercontext.sources import Source + + +class _Model: + async def generate(self, value: str, /) -> GenerationResult[str]: + return GenerationResult(output=value, usage=InferenceUsage(requests=1, input_tokens=7, output_tokens=3)) + + +class _Pipeline: + def __init__(self, *, fail_after_generation: bool) -> None: + self._fail = fail_after_generation + self._generator = UsageReportingStructuredGenerator(_Model()) + + async def _generate(self) -> None: + async with asyncio.timeout(0.02): + await self._generator.generate("bounded model request") + if self._fail: + raise RuntimeError("generation result rejected") # noqa: TRY003 + + async def incubate(self, sources: tuple[Source, ...], /) -> tuple[ExperienceCandidateInput, ...]: + await self._generate() + return () + + +@pytest.mark.parametrize("fail_after_generation", [False, True]) +def test_durable_usage_survives_slow_accounting_and_failed_generation( + tmp_path, monkeypatch: pytest.MonkeyPatch, fail_after_generation: bool +) -> None: + original_record = StatisticsRepository.record + + async def slow_record(repository, connection, *args): + # This delay exceeds the model deadline and must be owned by the recorder. + await asyncio.sleep(0.05) + await original_record(repository, connection, *args) + + monkeypatch.setattr(StatisticsRepository, "record", slow_record) + + async def scenario() -> None: + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'usage.db'}"), + runtime=RuntimeConfig(model_usage_write_timeout_seconds=2, model_usage_flush_timeout_seconds=1), + ) + pipeline = _Pipeline(fail_after_generation=fail_after_generation) + async with open_builtin_contexts(config, experience_pipeline=pipeline) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Usage", summary="Durable usage accounting", idempotency_key="usage") + ) + context = await contexts.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="one", content="One durable source.")) + spec = await experience_work_spec(contexts, scope.scope_id, limit=100, max_attempts=1, payload_version=1) + assert spec is not None + repository = WorkRepository() + async with contexts.database.transaction() as connection: + queued = await repository.enqueue(connection, spec) + worker = DurableWorker( + database=contexts.database, + worker_id="usage-worker", + handlers=(ExperienceWorkHandler(contexts),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + day = datetime.now(UTC).date() + # Read persisted rows directly: the completed attempt must drain its + # accepted usage without relying on a statistics read or shutdown. + async with contexts.database.transaction() as connection: + work = await repository.get(connection, queued.work.work_id) + model_usage, _ = await contexts.repositories.statistics.daily_usage_many( + connection, + (scope.scope_id,), + day, + day, + estimator=None, + ) + usage = model_usage[scope.scope_id] + assert work.status is (WorkStatus.FAILED if fail_after_generation else WorkStatus.SUCCEEDED) + assert len(usage) == 1 + assert usage[0].purpose is ModelUsagePurpose.EXPERIENCE_GENERATION + assert usage[0].operation is ModelUsageOperation.GENERATION + assert (usage[0].requests, usage[0].input_tokens, usage[0].output_tokens) == (1, 7, 3) + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_durable_scheduler.py b/tests/builtin/runtime/test_durable_scheduler.py new file mode 100644 index 0000000000..351a7d4fab --- /dev/null +++ b/tests/builtin/runtime/test_durable_scheduler.py @@ -0,0 +1,191 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + StaleCoordinatorLeaseError, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, SCHEDULER_LEASES_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime.config import CoordinationConfig +from powercontext.builtin.runtime.durable_scheduler import DiscoveryPage, DurableScheduler + + +class _Discoverer: + name = "test-discoverer" + interval_seconds = 60.0 + + def __init__(self) -> None: + self.pages: list[str | None] = [] + + async def page(self, continuation: str | None, limit: int) -> DiscoveryPage: + self.pages.append(continuation) + if continuation is None: + return DiscoveryPage(specs=(_spec(1),), continuation="next") + return DiscoveryPage(specs=(_spec(2),), continuation=None) + + +class _PausingDiscoverer: + name = "pausing-discoverer" + interval_seconds = 60.0 + + def __init__(self) -> None: + self.entered = asyncio.Event() + self.resume = asyncio.Event() + + async def page(self, continuation: str | None, limit: int) -> DiscoveryPage: + del continuation, limit + self.entered.set() + await self.resume.wait() + return DiscoveryPage(specs=(_spec(1),), continuation=None) + + +class _PausingCoordinationRepository(CoordinationRepository): + def __init__(self) -> None: + self.entered = asyncio.Event() + self.resume = asyncio.Event() + + async def acquire_lease( + self, + connection: AsyncConnection, + *, + lease_name: str, + owner_id: str, + lease_seconds: int, + ) -> CoordinatorLease | None: + self.entered.set() + await self.resume.wait() + return await super().acquire_lease( + connection, + lease_name=lease_name, + owner_id=owner_id, + lease_seconds=lease_seconds, + ) + + +def _spec(index: int) -> WorkSpec: + return WorkSpec( + kind="test.handler.v1", + payload_version=1, + scope_id=f"scope:{index}", + lane_key=f"{index + 1:064x}", + logical_key=f"{index + 101:064x}", + payload={}, + ) + + +def test_scheduler_persists_keyset_continuation_and_enqueues_under_its_fence() -> None: + async def scenario() -> None: + tables = WORK_TABLES + COORDINATION_TABLES + async with SQLiteProfile.open(SQLiteConfig(), tables=tables) as profile: + discoverer = _Discoverer() + scheduler = DurableScheduler( + database=profile.database, + scheduler_id="scheduler-a", + discoverers=(discoverer,), + config=CoordinationConfig(), + ) + assert await scheduler.tick() is True + assert await scheduler.tick() is True + + async with profile.database.transaction() as connection: + work = await WorkRepository().list(connection) + assert len(work) == 2 + assert discoverer.pages == [None, "next"] + + await scheduler.stop() + + asyncio.run(scenario()) + + +def test_old_scheduler_cannot_enqueue_after_a_higher_fence_takes_over() -> None: + async def scenario() -> None: + tables = WORK_TABLES + COORDINATION_TABLES + async with SQLiteProfile.open(SQLiteConfig(), tables=tables) as profile: + discoverer = _PausingDiscoverer() + scheduler = DurableScheduler( + database=profile.database, + scheduler_id="scheduler-a", + discoverers=(discoverer,), + config=CoordinationConfig(), + ) + stale_tick = asyncio.create_task(scheduler.tick()) + await asyncio.wait_for(discoverer.entered.wait(), timeout=1) + + coordination = CoordinationRepository() + async with profile.database.transaction() as connection: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == "work-discovery") + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + replacement = await coordination.acquire_lease( + connection, + lease_name="work-discovery", + owner_id="scheduler-b", + lease_seconds=30, + ) + assert replacement is not None + + discoverer.resume.set() + with pytest.raises(StaleCoordinatorLeaseError): + await stale_tick + async with profile.database.transaction() as connection: + assert await WorkRepository().list(connection) == () + + asyncio.run(scenario()) + + +def test_scheduler_stop_releases_lease_acquired_during_shutdown() -> None: + async def scenario() -> None: + tables = WORK_TABLES + COORDINATION_TABLES + async with SQLiteProfile.open(SQLiteConfig(), tables=tables) as profile: + coordination = _PausingCoordinationRepository() + scheduler = DurableScheduler( + database=profile.database, + scheduler_id="scheduler-a", + discoverers=(_Discoverer(),), + config=CoordinationConfig(), + coordination=coordination, + ) + run_task = asyncio.create_task(scheduler.run()) + await asyncio.wait_for(coordination.entered.wait(), timeout=1) + + stop_task = asyncio.create_task(scheduler.stop()) + await asyncio.sleep(0) + coordination.resume.set() + await stop_task + await run_task + + async with profile.database.transaction() as connection: + replacement = await CoordinationRepository().acquire_lease( + connection, + lease_name="work-discovery", + owner_id="scheduler-b", + lease_seconds=30, + ) + assert replacement is not None + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_membership.py b/tests/builtin/runtime/test_membership.py new file mode 100644 index 0000000000..81b256d45d --- /dev/null +++ b/tests/builtin/runtime/test_membership.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +import pytest + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES +from powercontext.builtin.runtime.config import CoordinationConfig, DeploymentConfig +from powercontext.builtin.runtime.membership import DuplicateSingleNodeError, RuntimeMembership + + +def test_single_node_membership_rejects_a_second_live_runtime() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + config = CoordinationConfig(member_ttl_seconds=3, member_heartbeat_seconds=1) + deployment = DeploymentConfig() + first = RuntimeMembership( + database=profile.database, + deployment=deployment, + coordination=config, + build_version="test", + ) + second = RuntimeMembership( + database=profile.database, + deployment=deployment, + coordination=config, + build_version="test", + ) + + await first.start() + with pytest.raises(DuplicateSingleNodeError): + await second.start() + + await first.stop() + await second.start() + assert await second.readiness() == "ready" + await second.stop() + + asyncio.run(scenario()) + + +def test_api_membership_reports_missing_worker_and_scheduler_as_degraded_dependencies() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + membership = RuntimeMembership( + database=profile.database, + deployment=DeploymentConfig(mode="distributed", role="api", id="api-a"), + coordination=CoordinationConfig(), + build_version="test", + ) + await membership.start() + + assert await membership.readiness() == "ready" + assert await membership.role_readiness("scheduler") == "unavailable" + assert await membership.role_readiness("worker") == "unavailable" + await membership.stop() + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_memory_work.py b/tests/builtin/runtime/test_memory_work.py new file mode 100644 index 0000000000..53508adf12 --- /dev/null +++ b/tests/builtin/runtime/test_memory_work.py @@ -0,0 +1,257 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +import pytest + +from powercontext.builtin.artifacts.atomic_memory.errors import AtomicMemoryConflictError +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import StaleWorkClaimError, WorkRepository +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.models import MemoryFlushResult +from powercontext.builtin.runtime.work_handlers import ( + MEMORY_WORK_KIND, + MemoryWorkHandler, + memory_work_spec, +) +from powercontext.builtin.sources import ContentCapture, SourceCursor +from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME + + +def _config(tmp_path): + return BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'memory.db'}")) + + +async def _claim(contexts, spec): + repository = WorkRepository() + async with contexts.database.transaction() as connection: + await repository.enqueue(connection, spec) + claims = await repository.claim( + connection, + worker_id="worker-a", + supported={MEMORY_WORK_KIND: frozenset({1})}, + lease_seconds=30, + limit=1, + ) + return claims[0] + + +class _StubPreparedWindow: + """A prepared window whose publication records its connection.""" + + def __init__(self, contexts, published): + self._contexts = contexts + self._published = published + + async def publish(self, connection, processing): + await self._contexts.repositories.cursors.save( + connection, + "project", + SOURCE_WINDOW_TRIGGER_NAME, + SourceCursor(sequence=1), + expected_generation=None, + ) + self._published.append(connection) + return MemoryFlushResult( + previous_cursor=0, + current_cursor=1, + high_watermark=1, + source_count=1, + memory_ref=None, + ) + + def emit_finalized(self) -> None: + pass + + def emit_failed(self) -> None: + pass + + +def _stub_prepare_memory(contexts, published): + async def fake_prepare_memory(scope_id: str, limit: int, **_: object) -> _StubPreparedWindow: + return _StubPreparedWindow(contexts, published) + + return fake_prepare_memory + + +def test_memory_work_spec_tracks_the_unconsumed_source_window(tmp_path) -> None: + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + assert await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) is None + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + + spec = await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) + assert spec is not None + assert spec.kind == MEMORY_WORK_KIND + assert spec.scope_id == "project" + assert spec.payload["cursor_name"] == SOURCE_WINDOW_TRIGGER_NAME + assert spec.payload["after"] == 0 + assert spec.payload["through"] == 1 + + asyncio.run(scenario()) + + +def test_memory_work_handler_defers_publication_to_the_fenced_commit(tmp_path, monkeypatch) -> None: + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + spec = await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) + assert spec is not None + claim = await _claim(contexts, spec) + + published: list[object] = [] + monkeypatch.setattr(contexts, "prepare_memory", _stub_prepare_memory(contexts, published)) + prepared = await MemoryWorkHandler(contexts, window_limit=10).prepare(claim) + # Preparation publishes nothing; the domain write joins the fenced + # Work completion transaction through the commit closure. + assert prepared.result is None + assert prepared.commit is not None + assert published == [] + + repository = WorkRepository() + async with contexts.database.transaction() as connection: + work = await repository.complete(connection, claim, None, commit=prepared.commit) + assert work.status.value == "succeeded" + assert len(published) == 1 + async with contexts.database.transaction() as connection: + row = await contexts.repositories.cursors.load(connection, "project", SOURCE_WINDOW_TRIGGER_NAME) + assert row is not None and row.cursor.sequence == 1 + + repeated = await MemoryWorkHandler(contexts, window_limit=10).prepare(claim) + assert repeated.result is not None + assert repeated.result.code == "already_committed" + + asyncio.run(scenario()) + + +def test_memory_publication_is_fenced_by_a_cancellation_before_commit(tmp_path, monkeypatch) -> None: + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + spec = await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) + assert spec is not None + claim = await _claim(contexts, spec) + + published: list[object] = [] + monkeypatch.setattr(contexts, "prepare_memory", _stub_prepare_memory(contexts, published)) + prepared = await MemoryWorkHandler(contexts, window_limit=10).prepare(claim) + assert prepared.commit is not None + + # A cancel accepted while preparation is paused must fence the + # later publication: the completion transaction sees cancel_requested + # and rolls the domain write back with it. + repository = WorkRepository() + async with contexts.database.transaction() as connection: + stored = await repository.get(connection, claim.work_id) + await repository.cancel( + connection, + claim.work_id, + expected_version=stored.state_version, + ) + with pytest.raises(StaleWorkClaimError): + async with contexts.database.transaction() as connection: + await repository.complete(connection, claim, None, commit=prepared.commit) + assert published == [] + async with contexts.database.transaction() as connection: + row = await contexts.repositories.cursors.load(connection, "project", SOURCE_WINDOW_TRIGGER_NAME) + assert row is None or row.cursor.sequence == 0 + + asyncio.run(scenario()) + + +def test_memory_work_handler_returns_the_trigger_result_without_a_commit(tmp_path, monkeypatch) -> None: + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + spec = await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) + assert spec is not None + claim = await _claim(contexts, spec) + + async def fake_prepare_memory(scope_id: str, limit: int, **_: object) -> MemoryFlushResult: + # The trigger selected nothing to publish for this attempt. + return MemoryFlushResult( + previous_cursor=1, + current_cursor=1, + high_watermark=1, + source_count=0, + memory_ref=None, + ) + + monkeypatch.setattr(contexts, "prepare_memory", fake_prepare_memory) + prepared = await MemoryWorkHandler(contexts, window_limit=10).prepare(claim) + assert prepared.commit is None + assert prepared.result is not None + assert prepared.result.code == "idle" + assert prepared.result.payload["current_cursor"] == 1 + + asyncio.run(scenario()) + + +def test_memory_work_handler_pins_preparation_to_the_persisted_window(tmp_path, monkeypatch) -> None: + """A Source captured after the Work record must stay outside this attempt's window.""" + + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + spec = await memory_work_spec(contexts, "project", limit=100, max_attempts=3, payload_version=1) + assert spec is not None + assert (spec.payload["after"], spec.payload["through"]) == (0, 1) + claim = await _claim(contexts, spec) + + # The second Source lands between discovery and execution. Preparing the + # live window would absorb it into this claim and report high_watermark=2. + await context.sources.capture(ContentCapture(source_id="two", content="arrived later")) + + requested: dict[str, object] = {} + + async def fake_prepare_memory(scope_id: str, limit: int, *, window=None, **_: object): + requested["window"] = window + return _StubPreparedWindow(contexts, []) + + monkeypatch.setattr(contexts, "prepare_memory", fake_prepare_memory) + prepared = await MemoryWorkHandler(contexts, window_limit=100).prepare(claim) + + assert prepared.commit is not None + # The persisted boundaries, not the live high watermark (0, 2). + assert requested["window"] == (0, 1) + + asyncio.run(scenario()) + + +def test_prepare_memory_refuses_a_window_the_cursor_already_left(tmp_path) -> None: + """A claim whose persisted window no longer matches the cursor must not publish.""" + + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + await contexts.get("project") + async with contexts.database.transaction() as connection: + await contexts.repositories.cursors.save( + connection, + "project", + SOURCE_WINDOW_TRIGGER_NAME, + SourceCursor(sequence=1), + expected_generation=None, + ) + with pytest.raises(AtomicMemoryConflictError): + await contexts.prepare_memory("project", 100, window=(0, 1)) + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_operation_maintenance.py b/tests/builtin/runtime/test_operation_maintenance.py new file mode 100644 index 0000000000..df213c8a41 --- /dev/null +++ b/tests/builtin/runtime/test_operation_maintenance.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkResult, WorkSpec, WorkStatus +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.work_handlers import OperationMaintenanceDiscoverer, OperationMaintenanceHandler +from powercontext.builtin.runtime.worker import DurableWorker + + +def test_operation_retention_runs_as_bounded_durable_work() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES + COORDINATION_TABLES) as profile: + repository = WorkRepository() + old_spec = WorkSpec( + kind="test.completed", + payload_version=1, + scope_id="project:test", + lane_key="a" * 64, + logical_key="b" * 64, + payload={}, + ) + async with profile.database.transaction() as connection: + old = await repository.enqueue(connection, old_spec) + claim = ( + await repository.claim( + connection, + worker_id="setup", + supported={old_spec.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.complete(connection, claim, WorkResult(code="done", payload={})) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == old.work.work_id) + .values(completed_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + + discoverer = OperationMaintenanceDiscoverer(interval_seconds=3600, max_attempts=5) + page = await discoverer.page(None, 100) + async with profile.database.transaction() as connection: + maintenance = await repository.enqueue(connection, page.specs[0]) + + worker = DurableWorker( + database=profile.database, + worker_id="maintenance-worker", + handlers=(OperationMaintenanceHandler(retention_days=1, batch_size=500),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + + async with profile.database.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, old.work.work_id) + completed = await repository.get(connection, maintenance.work.work_id) + assert completed.status is WorkStatus.SUCCEEDED + assert completed.result_payload == { + "operations_deleted": 1, + "rate_limit_windows_deleted": 0, + } + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_profile_work.py b/tests/builtin/runtime/test_profile_work.py new file mode 100644 index 0000000000..40585ca6de --- /dev/null +++ b/tests/builtin/runtime/test_profile_work.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expired Profile workers must not publish outside the fenced completion transaction.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING as BINDING +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES, WORK_ITEMS_TABLE +from powercontext.builtin.persistence.work import StaleWorkClaimError, WorkRepository, WorkStatus +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.work_handlers import ( + CURRENT_WORK_PAYLOAD_VERSION, + PROFILE_WORK_KIND, + ProfileWorkHandler, + profile_work_spec, +) +from powercontext.builtin.scope import ScopeDraft + + +class _Generator: + async def generate(self, value): + return "# Preferences\n\nGenerated profile for the fenced work commit." + + +def test_profile_work_publishes_only_inside_the_fenced_completion_transaction(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'profile-work.db'}")) + assert isinstance(config.database, SQLiteConfig) + async with SQLiteProfile.open(config.database, tables=BUILTIN_TABLES) as profile: + contexts = RelationalContexts(database=profile.database) + scope = (await contexts.scopes.create(ScopeDraft(title="p", summary="p", idempotency_key="p"))).scope_id + await contexts.records.create_source(scope, "content", "Profile evidence for the fenced commit.") + await contexts.profiles.put_policy(scope, generation_enabled=True, expected_version=0) + contexts.profiles.generator = _Generator() + + spec = await profile_work_spec( + contexts, scope, max_attempts=5, payload_version=CURRENT_WORK_PAYLOAD_VERSION + ) + assert spec is not None + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, spec) + (claim_a,) = await repository.claim( + connection, + worker_id="worker-a", + supported={PROFILE_WORK_KIND: frozenset({CURRENT_WORK_PAYLOAD_VERSION})}, + lease_seconds=120, + limit=1, + ) + + handler = ProfileWorkHandler(contexts.profiles, max_concurrency=1) + prepared_a = await handler.prepare(claim_a) + + # prepare() must not publish: nothing is durable before the fence. + async with profile.database.transaction() as connection: + cursor = await SourceCursorRepository().load(connection, scope, BINDING) + artifact = await contexts.profiles.latest(connection, scope) + assert cursor is None + assert artifact is None + + # The lease expires and a superseding Worker takes over with a higher fence. + # The first pass converts the expired attempt to retry_wait; the second + # pass claims it with fence 2. + async with profile.database.transaction() as connection: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == enqueued.work.work_id) + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + assert not await repository.claim( + connection, + worker_id="worker-b", + supported={PROFILE_WORK_KIND: frozenset({CURRENT_WORK_PAYLOAD_VERSION})}, + lease_seconds=120, + limit=1, + expired_retry_delay_seconds=0, + ) + (claim_b,) = await repository.claim( + connection, + worker_id="worker-b", + supported={PROFILE_WORK_KIND: frozenset({CURRENT_WORK_PAYLOAD_VERSION})}, + lease_seconds=120, + limit=1, + expired_retry_delay_seconds=0, + ) + + # Worker A's fenced completion is rejected and still publishes nothing. + with pytest.raises(StaleWorkClaimError): + async with profile.database.transaction() as connection: + await repository.complete(connection, claim_a, prepared_a.result, commit=prepared_a.commit) + async with profile.database.transaction() as connection: + cursor = await SourceCursorRepository().load(connection, scope, BINDING) + artifact = await contexts.profiles.latest(connection, scope) + assert cursor is None + assert artifact is None + + # Worker B commits the Profile, its cursor and the Work success together. + prepared_b = await handler.prepare(claim_b) + async with profile.database.transaction() as connection: + stored = await repository.complete(connection, claim_b, prepared_b.result, commit=prepared_b.commit) + assert stored.status is WorkStatus.SUCCEEDED + async with profile.database.transaction() as connection: + cursor = await SourceCursorRepository().load(connection, scope, BINDING) + artifact = await contexts.profiles.latest(connection, scope) + assert cursor is not None and cursor.cursor.sequence == 1 + assert artifact is not None + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_recurrence_ledger.py b/tests/builtin/runtime/test_recurrence_ledger.py index bc1581444e..c19b94608c 100644 --- a/tests/builtin/runtime/test_recurrence_ledger.py +++ b/tests/builtin/runtime/test_recurrence_ledger.py @@ -27,7 +27,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef -from powercontext.builtin.artifacts.experience import Experience +from powercontext.builtin.artifacts.experience import Experience, ExperienceCandidateInput from powercontext.builtin.artifacts.experience.models import ( ExperienceContent, FailureRecord, @@ -53,7 +53,13 @@ SCOPES_TABLE, SHARED_TABLES, ) +from powercontext.builtin.persistence.work import WorkRepository, WorkStatus +from powercontext.builtin.review.models import CandidateStatus +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import WorkerConfig from powercontext.builtin.runtime.recurrence import RelationalRecurrenceLedger +from powercontext.builtin.runtime.work_handlers import ExperienceWorkHandler, experience_work_spec +from powercontext.builtin.runtime.worker import DurableWorker from powercontext.builtin.sources.content import CONTENT_SOURCE_ADAPTER, ContentSource from powercontext.builtin.work.models import ( HandoffReceipt, @@ -62,7 +68,7 @@ TaskOutcomeStatus, WorkClaim, ) -from powercontext.sources import SourceMaterialization, SourceRef +from powercontext.sources import Source, SourceMaterialization, SourceRef SCOPE = "scope-a" CUE = "openapi contract changed without regenerating the client" @@ -609,3 +615,53 @@ def test_the_ledger_module_performs_no_read_path_instrumentation() -> None: source = Path("src/powercontext/builtin/artifacts/experience/recurrence.py").read_text(encoding="utf-8") for forbidden in ("sqlalchemy", "AsyncConnection", "Repository"): assert forbidden not in source, forbidden + + +class _NoExperienceCandidates: + async def incubate(self, sources: tuple[Source, ...], /) -> tuple[ExperienceCandidateInput, ...]: + return () + + +def test_durable_experience_windows_preserve_recurrence_and_repair_candidates(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'recurrence.db'}")) + async with open_builtin_contexts(config, experience_pipeline=_NoExperienceCandidates()) as contexts: + sources = contexts.repositories.sources + repository = WorkRepository() + async with contexts.database.transaction() as connection: + await _seed_scope(connection) + await _seed_artifacts(connection, sources) + await _add_receipt(sources, connection) + worker = DurableWorker( + database=contexts.database, + worker_id="recurrence-worker", + handlers=(ExperienceWorkHandler(contexts),), + config=WorkerConfig(concurrency=1), + ) + for ordinal in range(1, 4): + async with contexts.database.transaction() as connection: + await _add_source(sources, connection, f"outcome-{ordinal}", _failed_outcome().model_dump_json()) + spec = await experience_work_spec(contexts, SCOPE, limit=100, max_attempts=1, payload_version=1) + assert spec is not None + async with contexts.database.transaction() as connection: + queued = await repository.enqueue(connection, spec) + assert await worker.run_once() == 1 + async with contexts.database.transaction() as connection: + completed = await repository.get(connection, queued.work.work_id) + observations = await contexts.repositories.recurrence.observations(connection, SCOPE) + assert completed.status is WorkStatus.SUCCEEDED + assert completed.result_payload is not None + assert completed.result_payload["candidate_count"] == (1 if ordinal == 3 else 0) + assert sum(observation.event == "recurred" for observation in observations) == ordinal + + candidates = await contexts.review(SCOPE).list_candidates( + status=CandidateStatus.PENDING, family="experience", cursor=None, limit=10 + ) + assert len(candidates.candidates) == 1 + assert candidates.candidates[0].target == EXPERIENCE_REF + assert candidates.candidates[0].reason is not None + assert "recurred 3 times" in candidates.candidates[0].reason + assert await experience_work_spec(contexts, SCOPE, limit=100, max_attempts=1, payload_version=1) is None + assert await worker.run_once() == 0 + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_work_composition.py b/tests/builtin/runtime/test_work_composition.py new file mode 100644 index 0000000000..1509823e9e --- /dev/null +++ b/tests/builtin/runtime/test_work_composition.py @@ -0,0 +1,420 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +import logging +from datetime import UTC, datetime, timedelta +from time import monotonic +from types import SimpleNamespace +from typing import Literal, cast + +from sqlalchemy import func, insert, select + +from powercontext.builtin.persistence.rate_limit import RATE_LIMIT_WINDOWS_TABLE +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import WorkRepository +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingBinding +from powercontext.builtin.runtime.composition import ( + _ledger_families, + _open_work_background, + _supervisor_bindings, + _work_background_requested, + _work_discoverers, + _work_handlers, + _work_role_components, +) +from powercontext.builtin.runtime.config import ( + CoordinationConfig, + DeploymentConfig, + OperationsConfig, + RuntimeConfig, + WorkerConfig, +) +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.work_handlers import ( + EXPERIENCE_WORK_KIND, + MAINTENANCE_WORK_KIND, + MEMORY_WORK_KIND, + PROFILE_WORK_KIND, + memory_work_spec, +) +from powercontext.builtin.sources import ContentCapture + + +def _config(tmp_path, **overrides): + return BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'work.db'}"), + **overrides, + ) + + +def _distributed(runtime: RuntimeConfig, role: Literal["api", "scheduler", "worker"] = "worker") -> BuiltinConfig: + """A distributed config without an OceanBase handle: only top-level helpers read it.""" + + return cast( + "BuiltinConfig", + SimpleNamespace( + deployment=DeploymentConfig(mode="distributed", role=role), + runtime=runtime, + ), + ) + + +def test_ledger_only_starts_for_a_distributed_deployment() -> None: + # A single-node install keeps its APScheduler supervisors: RFC 1430 moves + # single_node/all onto the ledger only once that path has fully retired. + assert _work_background_requested(BuiltinConfig()) is False + assert _work_background_requested(BuiltinConfig(runtime=RuntimeConfig(memory_schedule_seconds=30))) is False + assert _work_background_requested(BuiltinConfig(runtime=RuntimeConfig(profile_schedule_enabled=True))) is False + assert _work_background_requested(_distributed(RuntimeConfig(memory_schedule_seconds=30))) is True + + +def test_ledger_owned_families_follow_the_configured_schedules() -> None: + assert _ledger_families(BuiltinConfig(runtime=RuntimeConfig(memory_schedule_seconds=30))) == frozenset() + + families = _ledger_families( + _distributed( + RuntimeConfig( + memory_schedule_seconds=30, + experience_schedule_seconds=30, + profile_schedule_enabled=True, + ) + ) + ) + assert families == {"memory", "experience", "profile"} + + # An unscheduled Family stays with the supervisor even in a distributed run. + assert _ledger_families(_distributed(RuntimeConfig(memory_schedule_seconds=30))) == {"memory"} + + +def test_supervisor_drops_only_the_families_the_ledger_owns() -> None: + fakes = [SimpleNamespace(artifact_family=family) for family in ("memory", "experience", "skill")] + bindings = cast("list[ArtifactProcessingBinding]", fakes) + config = _distributed(RuntimeConfig(memory_schedule_seconds=30, profile_schedule_enabled=True)) + + kept = _supervisor_bindings(config, bindings) + assert [binding.artifact_family for binding in kept] == ["experience", "skill"] + + # A single-node config owns nothing, so every binding reaches the supervisor. + assert len(_supervisor_bindings(BuiltinConfig(), bindings)) == 3 + + +def test_work_roles_split_scanning_from_claiming() -> None: + assert _work_role_components("all") == (True, True) + assert _work_role_components("api") == (False, False) + assert _work_role_components("scheduler") == (True, False) + assert _work_role_components("worker") == (False, True) + + +def test_composition_registers_every_discoverer_and_handler(tmp_path) -> None: + sentinel = object() + config = _config( + tmp_path, + runtime=RuntimeConfig( + memory_schedule_seconds=30, + experience_schedule_seconds=30, + profile_schedule_enabled=True, + ), + ) + + discoverers = _work_discoverers(config, cast("RelationalContexts", sentinel)) + assert [discoverer.name for discoverer in discoverers] == [ + MEMORY_WORK_KIND, + EXPERIENCE_WORK_KIND, + PROFILE_WORK_KIND, + MAINTENANCE_WORK_KIND, + ] + + class _Profiles: + """Stand-in for the shared Profile service the handler only stores.""" + + fake_contexts = cast("RelationalContexts", type("Contexts", (), {"profiles": _Profiles()})()) + handlers = _work_handlers(config, fake_contexts) + assert [handler.kind for handler in handlers] == [ + MEMORY_WORK_KIND, + EXPERIENCE_WORK_KIND, + PROFILE_WORK_KIND, + MAINTENANCE_WORK_KIND, + ] + + +def test_single_node_runtime_exposes_operations_without_ledger_execution(tmp_path) -> None: + async def scenario() -> None: + config = _config(tmp_path, runtime=RuntimeConfig(memory_schedule_seconds=30)) + async with ( + open_builtin_contexts(config) as contexts, + _open_work_background(config, contexts) as background, + ): + assert background.operations is not None + # The supervisor owns every Family here, so the ledger stays idle. + assert background.scheduler is None + assert background.worker is None + assert background.membership is None + + asyncio.run(scenario()) + + +class _RecordingObserver: + """Record-only WorkObserver standing in for the ServerMetrics registry.""" + + def __init__(self) -> None: + self.enqueues: list[tuple[str, bool]] = [] + self.queue_samples: object = None + + def observe_work_enqueue(self, kind: str, *, created: bool) -> None: + self.enqueues.append((kind, created)) + + def observe_work_claim(self, kind: str, *, latency_seconds: float) -> None: + pass + + def observe_work_attempt( + self, + kind: str, + *, + outcome: str, + error_category: str, + duration_seconds: float, + ) -> None: + pass + + def observe_work_lease_expiry(self, kind: str, *, outcome: str) -> None: + pass + + def observe_scheduler_leadership(self, *, outcome: str) -> None: + pass + + def set_work_queue(self, samples) -> None: + self.queue_samples = list(samples) + + def set_runtime_members(self, counts) -> None: + pass + + +def test_operations_refresh_the_queue_gauges_through_the_wired_observer(tmp_path) -> None: + async def scenario() -> None: + observer = _RecordingObserver() + config = _config(tmp_path, runtime=RuntimeConfig(memory_schedule_seconds=30)) + async with ( + open_builtin_contexts(config) as contexts, + _open_work_background(config, contexts, observer=observer) as background, + ): + assert background.operations is not None + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="remember this")) + spec = await memory_work_spec(contexts, "project", limit=10, max_attempts=3, payload_version=1) + assert spec is not None + + repository = WorkRepository(observer=observer) + async with contexts.database.transaction() as connection: + await repository.enqueue(connection, spec) + assert observer.enqueues == [(MEMORY_WORK_KIND, True)] + + page = await background.operations.list( + scope_id="project", kind=MEMORY_WORK_KIND, status=None, cursor=None, limit=10 + ) + stored = page.items[0] + await background.operations.cancel(stored.work_id, expected_version=stored.state_version) + # The cancel path refreshes the durable queue gauges through the + # same observer the production composition wired in. + assert observer.queue_samples is not None + + asyncio.run(scenario()) + + +def test_distributed_ledger_wires_the_work_observer_into_every_component(monkeypatch) -> None: + """Distributed+SQLite is rejected by BuiltinConfig, so spy the ledger components.""" + + async def scenario() -> None: + observer = _RecordingObserver() + constructed: dict[str, dict[str, object]] = {} + + class _SpyComponent: + def __init__(self, *args, **kwargs) -> None: + constructed[self._role] = kwargs + self._observer = kwargs.get("observer") + self._stopped = asyncio.Event() + + _role = "" + instance_id = "instance-1" + + async def start(self) -> None: + pass + + async def run(self) -> None: + await self._stopped.wait() + + async def stop(self) -> None: + self._stopped.set() + + class _SpyMembership(_SpyComponent): + _role = "membership" + + class _SpyScheduler(_SpyComponent): + _role = "scheduler" + + class _SpyWorker(_SpyComponent): + _role = "worker" + + monkeypatch.setattr("powercontext.builtin.runtime.membership.RuntimeMembership", _SpyMembership) + monkeypatch.setattr("powercontext.builtin.runtime.durable_scheduler.DurableScheduler", _SpyScheduler) + monkeypatch.setattr("powercontext.builtin.runtime.worker.DurableWorker", _SpyWorker) + monkeypatch.setattr("powercontext.builtin.runtime.composition._work_discoverers", lambda config, contexts: ()) + monkeypatch.setattr("powercontext.builtin.runtime.composition._work_handlers", lambda config, contexts: ()) + + config = cast( + "BuiltinConfig", + SimpleNamespace( + deployment=DeploymentConfig(mode="distributed", role="all"), + runtime=RuntimeConfig(memory_schedule_seconds=30), + coordination=CoordinationConfig(), + worker=WorkerConfig(), + operations=OperationsConfig(), + ), + ) + contexts = cast("RelationalContexts", SimpleNamespace(database=None)) + async with _open_work_background(config, contexts, observer=observer) as background: + assert background.membership is not None + assert background.scheduler is not None + assert background.worker is not None + assert background.operations is not None + # Every ledger component feeds the same bounded-label observer so + # /metrics reports real durable work, not only direct test calls. + assert background.membership._observer is observer + assert background.scheduler._observer is observer + assert background.worker._observer is observer + assert background.operations._observer is observer + assert constructed["scheduler"]["observer"] is observer + assert constructed["worker"]["observer"] is observer + assert constructed["membership"]["observer"] is observer + + asyncio.run(scenario()) + + +def test_single_node_maintenance_purges_expired_rate_limit_windows(tmp_path) -> None: + """The single-node background must bound the rate limit window table.""" + + async def scenario() -> None: + config = _config( + tmp_path, + operations=OperationsConfig(cleanup_interval_seconds=0.1, cleanup_batch_size=500), + ) + async with ( + open_builtin_contexts(config) as contexts, + _open_work_background(config, contexts) as background, + ): + assert background.rate_limit_maintenance is not None + expired_at = datetime.now(tz=UTC) - timedelta(hours=1) + async with contexts.database.transaction() as connection: + await connection.execute( + insert(RATE_LIMIT_WINDOWS_TABLE).values( + principal_key="a" * 64, + policy_id="server.http", + window_started_at=expired_at, + expires_at=expired_at + timedelta(seconds=60), + request_count=1, + ) + ) + deadline = monotonic() + 5 + rows = -1 + while monotonic() < deadline: + async with contexts.database.transaction() as connection: + rows = ( + await connection.execute(select(func.count()).select_from(RATE_LIMIT_WINDOWS_TABLE)) + ).scalar() + if rows == 0: + break + await asyncio.sleep(0.05) + # The maintenance loop deleted the expired window row without any + # distributed ledger execution. + assert rows == 0 + + asyncio.run(scenario()) + + +def test_rate_limit_maintenance_reports_a_failed_purge_and_keeps_running(tmp_path, caplog) -> None: + """A failing purge must be logged through the module logger, not dropped, and must not stop the loop.""" + + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + attempts: list[int] = [] + twice = asyncio.Event() + + class _FailingPurge: + async def purge_expired(self, connection, *, limit): + attempts.append(limit) + if len(attempts) >= 2: + twice.set() + raise RuntimeError("purge failed") # noqa: TRY003 + + from powercontext.builtin.runtime.composition import _RateLimitWindowMaintenance + + maintenance = _RateLimitWindowMaintenance( + database=contexts.database, + interval_seconds=0.05, + batch_size=7, + repository=_FailingPurge(), + ) + maintenance.start() + await asyncio.wait_for(twice.wait(), timeout=5) + await maintenance.stop() + + # The loop retried after the failure instead of exiting. + assert len(attempts) >= 2 + assert set(attempts) == {7} + failures = [ + record + for record in caplog.records + if record.name == "powercontext.builtin.runtime.composition" + and getattr(record, "event", None) == "rate-limit.window.maintenance-failed" + ] + assert len(failures) == len(attempts) + assert all( + record.getMessage() == "rate limit window maintenance failed: purge failed" for record in failures + ) + assert all(record.levelno == logging.WARNING for record in failures) + + asyncio.run(scenario()) + + +def test_rate_limit_maintenance_stops_when_requested(tmp_path) -> None: + async def scenario() -> None: + async with open_builtin_contexts(_config(tmp_path)) as contexts: + purges: list[int] = [] + + class _PurgeSpy: + async def purge_expired(self, connection, *, limit): + purges.append(limit) + return 0 + + from powercontext.builtin.runtime.composition import _RateLimitWindowMaintenance + + maintenance = _RateLimitWindowMaintenance( + database=contexts.database, + interval_seconds=0.05, + batch_size=10, + repository=_PurgeSpy(), + ) + maintenance.start() + await asyncio.sleep(0.25) + while_count = len(purges) + assert while_count >= 2 + await maintenance.stop() + after_stop = len(purges) + await asyncio.sleep(0.15) + assert len(purges) == after_stop + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_worker.py b/tests/builtin/runtime/test_worker.py new file mode 100644 index 0000000000..ffc7930715 --- /dev/null +++ b/tests/builtin/runtime/test_worker.py @@ -0,0 +1,339 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import WORK_ATTEMPTS_TABLE, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkResult, WorkSpec, WorkStatus +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.protocols import RuntimeTraceContext +from powercontext.builtin.runtime.worker import DurableWorker, PreparedWork, WorkExecutionError + + +class _Handler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self, executed: list[str]) -> None: + self._executed = executed + + async def prepare(self, claim) -> PreparedWork: + async def commit(_connection: AsyncConnection) -> None: + self._executed.append(claim.work_id) + + return PreparedWork(result=WorkResult(code="done", payload={}), commit=commit) + + +class _OverlapHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.active = 0 + self.maximum_active = 0 + self.both_entered = asyncio.Event() + + async def prepare(self, _claim) -> PreparedWork: + self.active += 1 + self.maximum_active = max(self.maximum_active, self.active) + if self.active == 2: + self.both_entered.set() + await asyncio.wait_for(self.both_entered.wait(), timeout=1) + self.active -= 1 + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _FlakyHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.calls = 0 + + async def prepare(self, _claim) -> PreparedWork: + self.calls += 1 + if self.calls == 1: + raise WorkExecutionError(category="provider", code="unavailable", retryable=True) + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _BlockingHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def prepare(self, _claim) -> PreparedWork: + self.started.set() + await self.release.wait() + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _SensitiveFailureHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self, sensitive_text: str) -> None: + self._sensitive_text = sensitive_text + + async def prepare(self, _claim) -> PreparedWork: + raise RuntimeError(self._sensitive_text) + + +class _TraceSpan: + def __init__(self, trace_context: RuntimeTraceContext | None = None) -> None: + self.trace_context = trace_context + + def set_attributes(self, _attributes) -> None: + return + + def set_outcome(self, _outcome: str) -> None: + return + + +class _Tracing: + def __init__(self) -> None: + self.execute_links: list[tuple[RuntimeTraceContext, ...]] = [] + self.attributes: list[dict[str, str | bool | int | float]] = [] + + @contextmanager + def stage( + self, + name: str, + *, + attributes: Mapping[str, str | bool | int | float], + ) -> Iterator[_TraceSpan]: + del name + self.attributes.append(dict(attributes)) + yield _TraceSpan() + + @contextmanager + def background( + self, + name: str, + *, + operation: str, + attributes: Mapping[str, str | bool | int | float], + links: Sequence[RuntimeTraceContext] = (), + ) -> Iterator[_TraceSpan]: + del operation + self.attributes.append(dict(attributes)) + links = tuple(links) + if name == "work.execute": + self.execute_links.append(links) + index = len(self.execute_links) + yield _TraceSpan(RuntimeTraceContext(trace_id=f"{index:032x}", span_id=f"{index:016x}")) + return + yield _TraceSpan() + + +def _spec(index: int) -> WorkSpec: + return WorkSpec( + kind="test.handler.v1", + payload_version=1, + scope_id=f"scope:{index}", + lane_key=f"{index + 1:064x}", + logical_key=f"{index + 101:064x}", + payload={}, + ) + + +def test_worker_claims_only_free_slots_and_executes_different_lanes_concurrently() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + executed: list[str] = [] + handler = _Handler(executed) + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec(1)) + second = await repository.enqueue(connection, _spec(2)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=2), + ) + assert await worker.run_once() == 2 + + async with profile.database.transaction() as connection: + first_stored = await repository.get(connection, first.work.work_id) + second_stored = await repository.get(connection, second.work.work_id) + assert first_stored.status is WorkStatus.SUCCEEDED + assert second_stored.status is WorkStatus.SUCCEEDED + assert set(executed) == {first.work.work_id, second.work.work_id} + + asyncio.run(scenario()) + + +def test_worker_overlaps_different_lanes_but_never_claims_an_unknown_payload() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + handler = _OverlapHandler() + repository = WorkRepository() + async with profile.database.transaction() as connection: + await repository.enqueue(connection, _spec(1)) + await repository.enqueue(connection, _spec(2)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=2), + ) + assert await worker.run_once() == 2 + assert handler.maximum_active == 2 + + unknown = _spec(3).model_copy(update={"payload_version": 2}) + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, unknown) + assert await worker.readiness() == "misconfigured" + assert await worker.run_once() == 0 + async with profile.database.transaction() as connection: + stored = await repository.get(connection, enqueued.work.work_id) + assert stored.status is WorkStatus.QUEUED + + asyncio.run(scenario()) + + +def test_retried_worker_attempt_links_to_the_persisted_previous_span() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + tracing = _Tracing() + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(_FlakyHandler(),), + config=WorkerConfig(concurrency=1, retry_base_seconds=0.01, retry_max_seconds=0.01), + random_source=lambda _low, _high: 0, + tracing=tracing, + ) + assert await worker.run_once() == 1 + assert await worker.run_once() == 1 + async with profile.database.transaction() as connection: + completed = await repository.get(connection, enqueued.work.work_id) + + assert completed.status is WorkStatus.SUCCEEDED + assert tracing.execute_links == [ + (), + (RuntimeTraceContext(trace_id=f"{1:032x}", span_id=f"{1:016x}"),), + ] + + asyncio.run(scenario()) + + +def test_worker_converges_a_running_cancel_without_waiting_for_lease_expiry() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + handler = _BlockingHandler() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=1), + ) + attempt = asyncio.create_task(worker.run_once()) + await asyncio.wait_for(handler.started.wait(), timeout=1) + async with profile.database.transaction() as connection: + running = await repository.get(connection, enqueued.work.work_id) + cancelling = await repository.cancel( + connection, + running.work_id, + expected_version=running.state_version, + ) + assert cancelling.status is WorkStatus.CANCELLING + + handler.release.set() + assert await asyncio.wait_for(attempt, timeout=1) == 1 + async with profile.database.transaction() as connection: + cancelled = await repository.get(connection, enqueued.work.work_id) + outcome = await connection.scalar( + select(WORK_ATTEMPTS_TABLE.c.outcome).where(WORK_ATTEMPTS_TABLE.c.work_id == enqueued.work.work_id) + ) + + assert cancelled.status is WorkStatus.CANCELLED + assert outcome == "cancelled" + + asyncio.run(scenario()) + + +def test_worker_does_not_persist_or_observe_sensitive_exception_text(caplog) -> None: + sensitive_text = "source-content=private credential=https://user:secret@example.test" + + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + tracing = _Tracing() + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(_SensitiveFailureHandler(sensitive_text),), + config=WorkerConfig(concurrency=1), + random_source=lambda _low, _high: 0, + tracing=tracing, + ) + with caplog.at_level(logging.DEBUG): + assert await worker.run_once() == 1 + + async with profile.database.transaction() as connection: + work = ( + ( + await connection.execute( + select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == enqueued.work.work_id) + ) + ) + .mappings() + .one() + ) + attempt = ( + ( + await connection.execute( + select(WORK_ATTEMPTS_TABLE).where(WORK_ATTEMPTS_TABLE.c.work_id == enqueued.work.work_id) + ) + ) + .mappings() + .one() + ) + + assert work["status"] == WorkStatus.RETRY_WAIT.value + assert work["error_category"] == "internal" + assert work["error_code"] == "unhandled_handler_error" + assert sensitive_text not in repr(dict(work)) + assert sensitive_text not in repr(dict(attempt)) + assert sensitive_text not in caplog.text + assert sensitive_text not in repr(tracing.attributes) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_artifact_processing_persistence.py b/tests/e2e/test_artifact_processing_persistence.py index 43bf5037dc..246eed47c4 100644 --- a/tests/e2e/test_artifact_processing_persistence.py +++ b/tests/e2e/test_artifact_processing_persistence.py @@ -28,7 +28,7 @@ from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.persistence.tables import SHARED_TABLES +from powercontext.builtin.persistence.tables import BUILTIN_TABLES, SHARED_TABLES from powercontext.builtin.runtime import ( ArtifactProcessingBinding, ArtifactProcessingWorkAssignment, @@ -122,6 +122,8 @@ async def scenario() -> None: def test_runtime_capture_rolls_back_source_when_pending_write_fails(tmp_path: Path) -> None: async def scenario() -> None: config = _sqlite_config(tmp_path / "rollback.db") + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES): + pass async with ( SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, profile.database.transaction() as connection, diff --git a/tests/e2e/test_atomic_memory_description_upgrade.py b/tests/e2e/test_atomic_memory_description_upgrade.py index bebf24b2b8..48db983ef8 100644 --- a/tests/e2e/test_atomic_memory_description_upgrade.py +++ b/tests/e2e/test_atomic_memory_description_upgrade.py @@ -85,9 +85,9 @@ def test_nonempty_released_memory_upgrades_without_rewriting_history(tmp_path, r ) result = _apply(runner) - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_003" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_004" verified = runner.verify() - assert verified.state == "ready" and verified.readiness_scope == "server" and verified.revision == "v1_5_0_003" + assert verified.state == "ready" and verified.readiness_scope == "server" and verified.revision == "v1_5_0_004" assert not _apply(runner).changed with closing(sqlite3.connect(runner.database)) as connection: assert ( diff --git a/tests/e2e/test_durable_experience_work.py b/tests/e2e/test_durable_experience_work.py new file mode 100644 index 0000000000..ba72a56035 --- /dev/null +++ b/tests/e2e/test_durable_experience_work.py @@ -0,0 +1,160 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +from powercontext.builtin.artifacts.experience import ( + EXPERIENCE_INCUBATION_CURSOR_NAME, + ExperienceCandidateInput, + ExperienceContent, +) +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import EnqueueResult, WorkRepository, WorkStatus +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.work_handlers import ExperienceWorkHandler, experience_work_spec +from powercontext.builtin.runtime.worker import DurableWorker +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import ContentCapture, ContentSource +from powercontext.sources import Source, SourceRef + + +class _TaskOutcomePipeline: + async def incubate(self, sources: tuple[Source, ...], /) -> tuple[ExperienceCandidateInput, ...]: + return tuple( + ExperienceCandidateInput( + proposal=ExperienceContent( + situation="A durable worker processed one bounded Source window.", + action="Incubate every captured fact into a reviewed proposal.", + outcome=source.content, + lesson="Keep the window bounded so each commit stays fenced.", + ), + sources=(SourceRef(source_type="content", source_id=source.name),), + ) + for source in sources + if isinstance(source, ContentSource) + ) + + +def test_manual_and_scheduled_discovery_join_one_window_and_commit_once(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'durable.db'}"), + ) + async with open_builtin_contexts(config, experience_pipeline=_TaskOutcomePipeline()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Durable work", summary="Joined window commit", idempotency_key="join") + ) + context = await contexts.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="one", content="First durable fact.")) + await context.sources.capture(ContentCapture(source_id="two", content="Second durable fact.")) + + manual_spec = await experience_work_spec( + contexts, scope.scope_id, limit=100, max_attempts=5, payload_version=1 + ) + scheduled_spec = await experience_work_spec( + contexts, scope.scope_id, limit=100, max_attempts=5, payload_version=1 + ) + assert manual_spec is not None and scheduled_spec is not None + assert manual_spec.logical_key == scheduled_spec.logical_key + + repository = WorkRepository() + async with contexts.database.transaction() as connection: + manual = await repository.enqueue(connection, manual_spec) + scheduled = await repository.enqueue(connection, scheduled_spec) + assert isinstance(manual, EnqueueResult) and isinstance(scheduled, EnqueueResult) + assert manual.created is True + assert scheduled.created is False + assert manual.work.work_id == scheduled.work.work_id + + worker = DurableWorker( + database=contexts.database, + worker_id="worker-a", + handlers=(ExperienceWorkHandler(contexts),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + + async with contexts.database.transaction() as connection: + stored = await repository.get(connection, manual.work.work_id) + assert stored.status is WorkStatus.SUCCEEDED + assert stored.result_payload is not None + assert stored.result_payload["current_cursor"] == 2 + assert stored.result_payload["candidate_count"] == 2 + + asyncio.run(scenario()) + + +def test_stale_discovery_snapshot_retires_instead_of_blocking_the_lane(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'stale-window.db'}"), + ) + async with open_builtin_contexts(config, experience_pipeline=_TaskOutcomePipeline()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Durable work", summary="Stale snapshot retirement", idempotency_key="stale") + ) + context = await contexts.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="one", content="First durable fact.")) + await context.sources.capture(ContentCapture(source_id="two", content="Second durable fact.")) + + # A scheduled discovery snapshot taken before the manual window exists. + stale = await experience_work_spec(contexts, scope.scope_id, limit=2, max_attempts=5, payload_version=1) + assert stale is not None + + # The manual window 0 -> 1 wins the race and commits first. + manual = await experience_work_spec(contexts, scope.scope_id, limit=1, max_attempts=5, payload_version=1) + assert manual is not None + repository = WorkRepository() + async with contexts.database.transaction() as connection: + await repository.enqueue(connection, manual) + worker = DurableWorker( + database=contexts.database, + worker_id="worker-a", + handlers=(ExperienceWorkHandler(contexts),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + + # The stale snapshot is enqueued only after that commit released the key. + async with contexts.database.transaction() as connection: + stale_enqueued = await repository.enqueue(connection, stale) + assert stale_enqueued.created is True + + assert await worker.run_once() == 1 + async with contexts.database.transaction() as connection: + stored = await repository.get(connection, stale_enqueued.work.work_id) + assert stored.status is WorkStatus.SUCCEEDED + assert stored.result_code == "superseded" + + # The lane kept moving: discovery covers the remainder and both facts land. + remainder = await experience_work_spec(contexts, scope.scope_id, limit=2, max_attempts=5, payload_version=1) + assert remainder is not None + async with contexts.database.transaction() as connection: + remainder_enqueued = await repository.enqueue(connection, remainder) + assert remainder_enqueued.created is True + assert await worker.run_once() == 1 + async with contexts.database.transaction() as connection: + stored = await repository.get(connection, remainder_enqueued.work.work_id) + cursor_state = await SourceCursorRepository().load( + connection, scope.scope_id, EXPERIENCE_INCUBATION_CURSOR_NAME + ) + assert stored.status is WorkStatus.SUCCEEDED + assert stored.result_code == "processed" + assert cursor_state is not None and cursor_state.cursor.sequence == 2 + + asyncio.run(scenario()) diff --git a/tests/e2e/test_operation_api.py b/tests/e2e/test_operation_api.py new file mode 100644 index 0000000000..4dac8b09e2 --- /dev/null +++ b/tests/e2e/test_operation_api.py @@ -0,0 +1,108 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +import pytest + +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + WorkRepository, + WorkSpec, + WorkStateConflictError, + WorkStatus, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import OperationsConfig, WorkerConfig +from powercontext.builtin.runtime.operations import OperationManager +from powercontext.builtin.runtime.work_handlers import EXPERIENCE_WORK_KIND +from powercontext.builtin.sources import ContentCapture + + +def _public_spec() -> WorkSpec: + return WorkSpec( + kind=EXPERIENCE_WORK_KIND, + payload_version=1, + scope_id="project", + lane_key="a" * 64, + logical_key="b" * 64, + payload={ + "cursor_name": "experience.incubation", + "cursor_generation": 0, + "after": 0, + "through": 1, + "high_watermark": 1, + }, + ) + + +def _internal_spec() -> WorkSpec: + return WorkSpec( + kind="powercontext.maintenance.operations", + payload_version=1, + scope_id="system:operations", + lane_key="c" * 64, + logical_key="d" * 64, + payload={}, + ) + + +def test_operation_manager_queries_public_operations_and_mutates_optimistically(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'operations.db'}")) + async with open_builtin_contexts(config) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="queued work")) + manager = OperationManager( + contexts=contexts, + operations=OperationsConfig(), + worker=WorkerConfig(), + local_worker=None, + payload_version=1, + ) + repository = WorkRepository() + async with contexts.database.transaction() as connection: + public = await repository.enqueue(connection, _public_spec()) + internal = await repository.enqueue(connection, _internal_spec()) + + stored = await manager.get(public.work.work_id) + assert stored.status is WorkStatus.QUEUED + + # Maintenance work stays private: the public facade hides it. + with pytest.raises(RepositoryNotFoundError): + await manager.get(internal.work.work_id) + + page = await manager.list(scope_id="project", kind=None, status=None, cursor=None, limit=10) + assert [item.work_id for item in page.items] == [public.work.work_id] + assert page.next_cursor is None + + # Optimistic mutations: a stale version loses, the current version wins. + with pytest.raises(WorkStateConflictError): + await manager.cancel(public.work.work_id, expected_version=stored.state_version + 1) + cancelled = await manager.cancel(public.work.work_id, expected_version=stored.state_version) + assert cancelled.status is WorkStatus.CANCELLED + + # Cancellation releases the logical key, so the closed operation can + # neither be rejoined nor retried. + with pytest.raises(StaleWorkClaimError): + await manager.retry(public.work.work_id, expected_version=cancelled.state_version) + + observed = await manager.wait(public.work.work_id, timeout_seconds=0) + assert observed is not None and observed.status is WorkStatus.CANCELLED + + asyncio.run(scenario()) diff --git a/tests/native/test_oceanbase_engine_recovery.py b/tests/native/test_oceanbase_engine_recovery.py index 0a11c51009..5e9ae7da8d 100644 --- a/tests/native/test_oceanbase_engine_recovery.py +++ b/tests/native/test_oceanbase_engine_recovery.py @@ -175,7 +175,7 @@ def ping(connection, _identity, _guard): _apply(_full_server_runner(backend), policy="manual") resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.revision == "v1_5_0_003" and result.readiness_scope == "server" + assert result.state == "ready" and result.revision == "v1_5_0_004" and result.readiness_scope == "server" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journal.read_text()) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index ea2e5f69fb..76c00512fe 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -63,6 +63,8 @@ SupportState, ) +_TEST_MEMBER_TTL_SECONDS = 3 + pytestmark = [ pytest.mark.native_service, pytest.mark.skipif( @@ -99,6 +101,9 @@ def test_native_personal_service_lifecycle(tmp_path: Path) -> None: assert adapter.loaded_registration().state is ManagerOwnershipState.OWNED assert adapter.manager_state() is ManagerState.INACTIVE + if isinstance(adapter, WindowsTaskSchedulerAdapter): + # schtasks /End cannot run the Server shutdown hook that releases the lease. + time.sleep(_TEST_MEMBER_TTL_SECONDS + 1) adapter.start(reload_definition=False) restarted = _wait_for_status(controller) assert restarted.ok, f"{restarted}\n\n{_failure_diagnostics(adapter, tmp_path)}" @@ -401,7 +406,7 @@ def test_native_public_migration_switches_an_installed_130_service( if failure in {"startup", "switch"}: assert applied.exit_code == 1, applied.output assert result["error"] == "service_start_failed" - assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_003" + assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_004" assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} phase = "switching" if failure == "switch" else "start_failed" assert json.loads(controller.maintenance_path.read_text())["phase"] == phase @@ -800,6 +805,9 @@ def _environment_file(tmp_path: Path) -> Path: "\n".join(( f"POWERCONTEXT_HOME={data_dir}", f"POWERCONTEXT_SERVER_HTTP_PORT={_unused_loopback_port()}", + "POWERCONTEXT_SERVER_DASHBOARD_ENABLED=false", + f"POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS={_TEST_MEMBER_TTL_SECONDS}", + "POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS=1", "", )), encoding="utf-8", diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py index ca9bf53111..baf4622e81 100644 --- a/tests/test_config_cli.py +++ b/tests/test_config_cli.py @@ -168,6 +168,50 @@ def test_validate_accepts_minimal_server_environment_without_inference_models(tm assert settings.http.port == 8888 +@pytest.mark.parametrize("role", ("api", "scheduler")) +def test_validate_accepts_non_executing_distributed_roles_without_inference_models( + role: str, + tmp_path: Path, +) -> None: + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://root@127.0.0.1:2881/powercontext?charset=utf8mb4", + "POWERCONTEXT_SERVER_DEPLOYMENT_MODE=distributed", + f"POWERCONTEXT_SERVER_DEPLOYMENT_ROLE={role}", + "POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS=60", + "POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS=60", + "", + )), + encoding="utf-8", + ) + + result = CliRunner().invoke(config_cli.app, ["validate", "--env-file", str(environment)]) + + assert result.exit_code == 0 + assert "Configuration is valid" in result.output + + +def test_validate_rejects_distributed_worker_without_inference_model(tmp_path: Path) -> None: + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://root@127.0.0.1:2881/powercontext?charset=utf8mb4", + "POWERCONTEXT_SERVER_DEPLOYMENT_MODE=distributed", + "POWERCONTEXT_SERVER_DEPLOYMENT_ROLE=worker", + "", + )), + encoding="utf-8", + ) + + result = CliRunner().invoke(config_cli.app, ["validate", "--env-file", str(environment)]) + + assert result.exit_code == 2 + assert "built-in runtime cannot be configured" in result.output + + @pytest.mark.parametrize("token", ["literal-${PR1525_ABSENT}", "token'with-apostrophe"]) def test_server_settings_context_preserves_strict_env_file_values( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_desktop_contract.py b/tests/test_desktop_contract.py index f0d031a380..f28c120ea6 100644 --- a/tests/test_desktop_contract.py +++ b/tests/test_desktop_contract.py @@ -48,6 +48,37 @@ def test_desktop_contract_identity_matches_openapi(manifest: str) -> None: assert actual == expected, f"Stale Desktop contract identity in {manifest}; run 'pnpm --dir desktop generate'." +def test_desktop_bundle_qualifies_the_current_contract() -> None: + """The shipped preview must carry a qualification record for its own contract. + ``desktop/src-tauri/tests/session.rs`` asserts this too, but that check only runs in + ``cargo test``. Advancing the contract digest therefore used to look green in the Python + gate and deterministic red on four platforms. Recording the requirement here makes the + two gates agree and names the harness that has to run before the digest moves. + """ + contract = (REPOSITORY_ROOT / "openapi/powercontext.yaml").read_bytes().replace(b"\r\n", b"\n") + expected = hashlib.sha256(contract).hexdigest() + bundled = json.loads( + (REPOSITORY_ROOT / "desktop/src-tauri/src/transport/operations.json").read_text(encoding="utf-8") + ) + profiles = json.loads( + (REPOSITORY_ROOT / "desktop/src-tauri/src/connections/compatibility.json").read_text(encoding="utf-8") + ) + current = [profile for profile in profiles if profile["contractSha256"] == expected] + assert current, ( + f"No compatibility record matches the bundled Desktop contract {expected}; build the wheel " + "from a clean commit, run " + "'uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit ' " + "and 'uv run --no-sync python desktop/tests/real_server.py', then record the profile in " + "desktop/src-tauri/src/connections/compatibility.json and desktop/VALIDATION.md." + ) + for profile in current: + assert sorted(profile["operations"]) == sorted(bundled["operations"]), ( + f"{profile['id']} does not record the bundled Desktop operation set" + ) + assert re.fullmatch(r"[0-9a-f]{40}", profile["serverCommit"]), profile["id"] + assert re.fullmatch(r"[0-9a-f]{64}", profile["artifactSha256"]), profile["id"] + + def test_bundled_desktop_contract_has_a_qualified_business_profile() -> None: native = REPOSITORY_ROOT / "desktop/src-tauri/src" contract = json.loads((native / "transport/operations.json").read_text(encoding="utf-8")) diff --git a/tests/test_docker_contract.py b/tests/test_docker_contract.py index e3c60655c2..ff5f44b722 100644 --- a/tests/test_docker_contract.py +++ b/tests/test_docker_contract.py @@ -25,16 +25,19 @@ import os import re +import subprocess from pathlib import Path from unittest.mock import Mock import pytest +import yaml from typer.testing import CliRunner from powercontext.cli.app import create_cli from powercontext.server.cli import app as server_app _DOCKERFILE = Path(__file__).resolve().parent.parent / "docker" / "Dockerfile" +_DISTRIBUTED_COMPOSE = _DOCKERFILE.with_name("compose.distributed.yaml") _SERVER_ENV_PATTERN = re.compile(r"(POWERCONTEXT_SERVER_[A-Z0-9_]+)=([^\s\\]+)") @@ -69,3 +72,126 @@ def test_documented_container_invocation_starts_out_of_the_box(monkeypatch: pyte assert result.exit_code == 0, result.output run_server.assert_called_once() assert run_server.call_args.kwargs["host"] == "0.0.0.0" # noqa: S104 - the image's documented bind. + + +def test_distributed_compose_separates_roles_and_model_credentials() -> None: + document = yaml.safe_load(_DISTRIBUTED_COMPOSE.read_text(encoding="utf-8")) + services = document["services"] + assert set(services) == {"migrate", "api-a", "api-b", "scheduler-a", "scheduler-b", "worker-a", "worker-b"} + + # `server migrate` is not a registered command; the migrator must use the supported explicit + # db-migrate plan/apply workflow with OceanBase's evidence directory and host coordination. + assert services["migrate"]["entrypoint"] == ["/bin/sh", "-ec"] + (script,) = services["migrate"]["command"] + assert "server db-migrate plan" in script + assert "server db-migrate apply" in script and "--plan-id" in script + assert "--evidence-dir /var/lib/powercontext/migration-state" in script + assert "--lock-coordination single-host" in script + assert "--maintenance-confirmed" in script and "--yes" in script + assert services["migrate"]["environment"]["POWERCONTEXT_SERVER_DEPLOYMENT_ROLE"] == "api" + assert services["migrate"]["environment"]["POWERCONTEXT_SERVER_DEPLOYMENT_ID"] == "migrator" + for name, service in services.items(): + environment = service["environment"] + assert environment["POWERCONTEXT_SERVER_DATABASE_KIND"] == "oceanbase" + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_MODE"] == "distributed" + if name == "migrate": + continue + role = name.split("-", 1)[0] + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_ROLE"] == role + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_ID"] == name + if role in {"scheduler", "worker"}: + assert "ports" not in service + assert environment["POWERCONTEXT_SERVER_DASHBOARD_ENABLED"] == "false" + assert environment["POWERCONTEXT_SERVER_MCP_ENABLED"] == "false" + assert ("POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL" in environment) is (role == "worker") + assert ("OPENAI_API_KEY" in environment) is (role == "worker") + assert ("ANTHROPIC_API_KEY" in environment) is (role == "worker") + + +_COMPOSE_VARIABLE = re.compile(r"\$\$|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::([-?])([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)") + + +def _compose_interpolate(text: str, environment: dict[str, str]) -> str: + """Apply Compose's own interpolation rules to a raw YAML string. + + ``$$`` becomes a literal ``$`` for the container, ``${NAME:-default}`` falls back to + the default, and an unset ``$NAME`` -- bare or braced -- is replaced with an empty + string: the exact behavior that turned ``case "$policy" in`` into ``case "" in``. + """ + + def replace(match: re.Match[str]) -> str: + if match.group(0) == "$$": + return "$" + name = match.group(1) or match.group(4) + operator, argument = match.group(2), match.group(3) + value = environment.get(name) + if operator == "?": + assert value, f"{name} is required by this Compose file" + return value + return value if value else (argument or "") + + return _COMPOSE_VARIABLE.sub(replace, text) + + +def _run_compose_rendered_migrator(tmp_path: Path, environment: dict[str, str]) -> list[str]: + """Render the migrator command the way Compose does, then execute it with a stub CLI.""" + + document = yaml.safe_load(_DISTRIBUTED_COMPOSE.read_text(encoding="utf-8")) + (rendered,) = document["services"]["migrate"]["command"] + # Compose interpolates the YAML string before the container starts; the container's + # shell then sees the resolved script. + rendered = _compose_interpolate(rendered, environment) + + bin_directory = tmp_path / "bin" + bin_directory.mkdir() + recorded = tmp_path / "apply-arguments.txt" + stub = bin_directory / "powercontext" + stub.write_text( + "#!/bin/sh\n" + 'if [ "$1 $2 $3" = "server db-migrate plan" ]; then\n' + ' echo \'{"plan_id": "plan-abc"}\'\n' + " exit 0\n" + "fi\n" + f'echo "$@" > "{recorded}"\n', + encoding="utf-8", + ) + stub.chmod(0o755) + + result = subprocess.run( + ["/bin/sh", "-ec", rendered], + capture_output=True, + text=True, + env={ + **environment, + "PATH": f"{bin_directory}{os.pathsep}{os.environ.get('PATH', '')}", + }, + check=False, + ) + assert result.returncode == 0, result.stderr + return recorded.read_text(encoding="utf-8").split() + + +def test_compose_rendered_migrator_runs_with_the_default_backup_policy(tmp_path: Path) -> None: + """Compose must not blank out the script's own shell variables.""" + + # Every deployment variable is present; the shell bookkeeping variables are not host variables. + arguments = _run_compose_rendered_migrator( + tmp_path, + {"POWERCONTEXT_MIGRATION_STATE_DIR": str(tmp_path / "migration-state")}, + ) + assert arguments[:3] == ["server", "db-migrate", "apply"] + assert arguments[arguments.index("--plan-id") + 1] == "plan-abc" + assert arguments[arguments.index("--backup") + 1] == "manual" + assert "--backup-confirmed" in arguments + + +def test_compose_rendered_migrator_honors_the_skip_backup_policy(tmp_path: Path) -> None: + arguments = _run_compose_rendered_migrator( + tmp_path, + { + "POWERCONTEXT_MIGRATION_STATE_DIR": str(tmp_path / "migration-state"), + "POWERCONTEXT_MIGRATION_BACKUP_POLICY": "skip", + }, + ) + assert arguments[arguments.index("--backup") + 1] == "skip" + assert "--accept-no-backup" in arguments diff --git a/tests/test_managed_atomic_maintenance.py b/tests/test_managed_atomic_maintenance.py index 450c804fba..50992d4929 100644 --- a/tests/test_managed_atomic_maintenance.py +++ b/tests/test_managed_atomic_maintenance.py @@ -76,7 +76,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) assert plan["source_revision"] == "s1300" - assert plan["target_revision"] == "v1_5_0_003" + assert plan["target_revision"] == "v1_5_0_004" assert plan["readiness_scope"] == "server" assert historical.read_bytes() == original @@ -103,7 +103,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b with closing(sqlite3.connect(result["backup_ref"])) as connection: assert tuple(connection.iterdump()) == original_dump with closing(sqlite3.connect(historical)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_004",)] assert connection.execute("SELECT content FROM pc_artifacts").fetchall() == [(b'{"history":"unchanged"}',)] verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) assert verified.exit_code == 0, verified.output diff --git a/tests/test_managed_processing_maintenance.py b/tests/test_managed_processing_maintenance.py index 7377ea3eeb..989ca7e786 100644 --- a/tests/test_managed_processing_maintenance.py +++ b/tests/test_managed_processing_maintenance.py @@ -75,7 +75,7 @@ def test_registered_mode_switch_backs_up_preserves_revision_and_fences_old_worke assert result.backup_state == "completed" assert result.backup_ref and Path(result.backup_ref).is_file() with closing(sqlite3.connect(path)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_004",)] assert ( json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ "mode" diff --git a/tests/test_migration_revision_names.py b/tests/test_migration_revision_names.py index 89f7799b42..3de63bddb7 100644 --- a/tests/test_migration_revision_names.py +++ b/tests/test_migration_revision_names.py @@ -64,9 +64,10 @@ def invoke(action, *options): "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", + "v1_5_0_004", ] assert [item["revision"] for item in plan["pending_migrations"]] == list(reviewed.revisions) - target = {"revision": "v1_5_0_003", "name": "v1_5_0_003", "software_version": "1.5.0", "sequence": 3} + target = {"revision": "v1_5_0_004", "name": "v1_5_0_004", "software_version": "1.5.0", "sequence": 4} assert plan["target_migration"] == target applied = invoke( @@ -81,7 +82,7 @@ def invoke(action, *options): ) assert applied["migration"] == target and applied["server_ready"] with closing(sqlite3.connect(runner.database)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_004",)] evidence = runner.evidence_path.read_bytes() recorded = json.loads(evidence) assert recorded["source_revision"] == expected_source["revision"] @@ -96,6 +97,7 @@ def invoke(action, *options): "v1_5_0_001", "v1_5_0_002", "v1_5_0_003", + "v1_5_0_004", } assert runner.verify().state == "ready" assert invoke("verify")["migration"] == target diff --git a/tests/test_operation_routes.py b/tests/test_operation_routes.py new file mode 100644 index 0000000000..ee72c0715e --- /dev/null +++ b/tests/test_operation_routes.py @@ -0,0 +1,200 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The durable operation endpoints promised by the contract and the docs. + +The runtime facade is covered by ``tests/e2e/test_operation_api.py``. This file +covers the transport in front of it: the four routes, the Scope ownership check +that keeps another Scope's identifiers unprobeable, the maintenance-work +whitelist, and the optimistic-concurrency contract of cancel and retry. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec, WorkStatus +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import OperationsConfig, WorkerConfig +from powercontext.builtin.runtime.operations import OperationManager +from powercontext.builtin.runtime.work_handlers import EXPERIENCE_WORK_KIND, MEMORY_WORK_KIND +from powercontext.server.app import create_app + +SCOPE = "project:operations" + + +def _spec(kind: str, scope_id: str, suffix: str) -> WorkSpec: + return WorkSpec( + kind=kind, + payload_version=1, + scope_id=scope_id, + lane_key=f"{suffix:0<64}"[:64], + logical_key=f"{suffix:1<64}"[:64], + payload={"through": 1}, + ) + + +def test_operation_routes_expose_only_scope_owned_public_work(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'operations.db'}")) + async with open_builtin_contexts(config) as contexts: + manager = OperationManager( + contexts=contexts, + operations=OperationsConfig(), + worker=WorkerConfig(), + local_worker=None, + payload_version=1, + ) + repository = WorkRepository() + async with contexts.database.transaction() as connection: + memory = await repository.enqueue(connection, _spec(MEMORY_WORK_KIND, SCOPE, "a")) + experience = await repository.enqueue(connection, _spec(EXPERIENCE_WORK_KIND, SCOPE, "b")) + foreign = await repository.enqueue(connection, _spec(MEMORY_WORK_KIND, "project:other", "c")) + maintenance = await repository.enqueue( + connection, + _spec("powercontext.maintenance.operations", SCOPE, "d"), + ) + + app = create_app(application=SimpleNamespace(operations=manager)) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://server") as client: + listed = await client.get(f"/v1/scopes/{SCOPE}/operations") + assert listed.status_code == 200, listed.text + body = listed.json() + # Both rows share a creation timestamp at this resolution, so the + # page order is only guaranteed by the (created_at, work_id) key. + assert {item["operation_id"] for item in body["items"]} == { + experience.work.work_id, + memory.work.work_id, + } + assert body["next_cursor"] is None + assert {item["status"] for item in body["items"]} == {"queued"} + assert {item["kind"] for item in body["items"]} == {MEMORY_WORK_KIND, EXPERIENCE_WORK_KIND} + + # A page is bounded by limit and continued by the opaque cursor. + first = await client.get(f"/v1/scopes/{SCOPE}/operations", params={"limit": 1}) + assert first.status_code == 200, first.text + page = first.json() + assert len(page["items"]) == 1 + assert page["next_cursor"] + second = await client.get( + f"/v1/scopes/{SCOPE}/operations", + params={"limit": 1, "cursor": page["next_cursor"]}, + ) + assert second.status_code == 200, second.text + assert len(second.json()["items"]) == 1 + assert second.json()["items"][0]["operation_id"] != page["items"][0]["operation_id"] + + # Kind and status filters narrow the same page. + filtered = await client.get( + f"/v1/scopes/{SCOPE}/operations", + params={"kind": MEMORY_WORK_KIND, "status": "queued"}, + ) + assert filtered.status_code == 200, filtered.text + assert [item["operation_id"] for item in filtered.json()["items"]] == [memory.work.work_id] + assert ( + await client.get(f"/v1/scopes/{SCOPE}/operations", params={"status": "no-such-state"}) + ).status_code == 422 + assert (await client.get(f"/v1/scopes/{SCOPE}/operations", params={"unknown": "1"})).status_code == 422 + + # Retrieval answers for this Scope and hides everything else. + detail = await client.get(f"/v1/scopes/{SCOPE}/operations/{memory.work.work_id}") + assert detail.status_code == 200, detail.text + assert detail.json()["operation_id"] == memory.work.work_id + assert ( + await client.get(f"/v1/scopes/project:other/operations/{memory.work.work_id}") + ).status_code == 404 + assert (await client.get(f"/v1/scopes/{SCOPE}/operations/{foreign.work.work_id}")).status_code == 404 + assert ( + await client.get(f"/v1/scopes/{SCOPE}/operations/{maintenance.work.work_id}") + ).status_code == 404 + + # Cancel is optimistic: the stale version loses and reports 409. + stale = await client.post( + f"/v1/scopes/{SCOPE}/operations/{memory.work.work_id}/cancel", + json={"expected_version": memory.work.state_version + 1}, + ) + assert stale.status_code == 409, stale.text + assert stale.json()["error"]["code"] == "operation_conflict" + cancelled = await client.post( + f"/v1/scopes/{SCOPE}/operations/{memory.work.work_id}/cancel", + json={"expected_version": memory.work.state_version}, + ) + assert cancelled.status_code == 200, cancelled.text + assert cancelled.json()["status"] == "cancelled" + assert cancelled.json()["cancel_requested"] is True + assert cancelled.json()["state_version"] == memory.work.state_version + 1 + + # A cancelled operation cannot be retried; a failed one can. + assert ( + await client.post( + f"/v1/scopes/{SCOPE}/operations/{memory.work.work_id}/retry", + json={"expected_version": cancelled.json()["state_version"]}, + ) + ).status_code == 409 + + async with contexts.database.transaction() as connection: + claims = await repository.claim( + connection, + worker_id="test-worker", + supported={EXPERIENCE_WORK_KIND: frozenset({1})}, + lease_seconds=30, + limit=1, + ) + assert len(claims) == 1 + failed = await repository.fail( + connection, + claims[0], + _failure(), + retry_delay_seconds=0, + ) + assert failed.status is WorkStatus.FAILED + retried = await client.post( + f"/v1/scopes/{SCOPE}/operations/{experience.work.work_id}/retry", + json={"expected_version": failed.state_version}, + ) + assert retried.status_code == 200, retried.text + assert retried.json()["status"] == "queued" + assert retried.json()["error_category"] is None + assert retried.json()["error_code"] is None + + asyncio.run(scenario()) + + +def _failure(): + from powercontext.builtin.persistence.work import WorkFailure + + return WorkFailure(category="generation", code="provider_unavailable", retryable=False) + + +def test_operation_routes_require_a_composed_ledger() -> None: + app = create_app(application=SimpleNamespace(operations=None)) + transport = httpx.ASGITransport(app=app) + + async def scenario() -> None: + async with httpx.AsyncClient(transport=transport, base_url="http://server") as client: + response = await client.get(f"/v1/scopes/{SCOPE}/operations") + assert response.status_code == 503 + assert response.json()["error"]["code"] == "runtime_not_ready" + + asyncio.run(scenario()) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_registered_source_maintenance_cli.py b/tests/test_registered_source_maintenance_cli.py index 5809d3781d..1a1e73264c 100644 --- a/tests/test_registered_source_maintenance_cli.py +++ b/tests/test_registered_source_maintenance_cli.py @@ -66,7 +66,7 @@ def test_release140_compatibility_commands_preserve_source_and_share_full_upgrad planned = CliRunner().invoke(cli, [*arguments, "plan", "--backup", "skip"]) assert planned.exit_code == 0, planned.output plan = json.loads(planned.output) - assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_003" + assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_004" assert plan["readiness_scope"] == "server" assert plan["compatibility_steps"] == [] full = CliRunner().invoke(cli, ["server", "db-migrate", "plan", "--backup", "skip"]) diff --git a/tests/test_server_metrics.py b/tests/test_server_metrics.py index 4efc73c36e..377f3dcf07 100644 --- a/tests/test_server_metrics.py +++ b/tests/test_server_metrics.py @@ -26,6 +26,7 @@ from prometheus_client.parser import text_string_to_metric_families from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import WorkQueueStatistic, WorkStatus from powercontext.builtin.runtime import RuntimeConfig from powercontext.server.app import create_app from powercontext.server.factory import create_server_app @@ -210,3 +211,34 @@ def observe_application(self, operation: str, outcome: str, started_at: float) - response = TestClient(create_app(metrics=metrics)).get("/v1/capabilities") assert response.status_code == 200 + + +def test_work_metrics_use_only_bounded_kind_status_and_outcome_labels() -> None: + metrics = ServerMetrics() + metrics.observe_work_enqueue("powercontext.memory.source-window", created=True) + metrics.observe_work_claim("powercontext.memory.source-window", latency_seconds=0.5) + metrics.observe_work_attempt( + "powercontext.memory.source-window", + outcome="retry_wait", + error_category="provider", + duration_seconds=1.25, + ) + metrics.observe_work_lease_expiry("powercontext.memory.source-window", outcome="retry_wait") + metrics.observe_scheduler_leadership(outcome="acquired") + metrics.set_work_queue(( + WorkQueueStatistic( + kind="powercontext.memory.source-window", + status=WorkStatus.QUEUED, + depth=2, + oldest_age_seconds=3.5, + ), + )) + metrics.set_runtime_members({"api": 2, "worker": 3}) + + rendered = metrics.render().decode() + assert 'powercontext_work_queue_depth{kind="powercontext.memory.source-window",status="queued"} 2.0' in rendered + assert 'powercontext_runtime_role_members{role="api"} 2.0' in rendered + assert 'powercontext_runtime_role_members{role="worker"} 3.0' in rendered + assert "scope_id" not in rendered + assert "work_id" not in rendered + assert "principal" not in rendered diff --git a/tests/test_server_migration_cli.py b/tests/test_server_migration_cli.py new file mode 100644 index 0000000000..82ed9719d9 --- /dev/null +++ b/tests/test_server_migration_cli.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import sqlite3 + +from typer.testing import CliRunner + +from powercontext.cli.app import create_cli +from powercontext.server.cli import app as server_app + + +def test_server_migrate_runs_the_packaged_forward_only_chain(tmp_path) -> None: + database = tmp_path / "runtime.db" + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=sqlite", + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}", + )), + encoding="utf-8", + ) + + common = ["--env-file", str(environment)] + plan = json.loads( + CliRunner() + .invoke( + create_cli((server_app,)), + ["server", "db-migrate", "plan", *common, "--backup", "skip"], + ) + .output + ) + + result = CliRunner().invoke( + create_cli((server_app,)), + [ + "server", + "db-migrate", + "apply", + *common, + "--backup", + "skip", + "--plan-id", + plan["plan_id"], + "--yes", + "--maintenance-confirmed", + "--accept-no-backup", + ], + ) + + assert result.exit_code == 0, result.output + with sqlite3.connect(database) as connection: + revision = connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() + work_table = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'pc_work_items'" + ).fetchone() + assert revision == [("v1_5_0_004",)] + assert work_table == ("pc_work_items",) diff --git a/tests/test_server_rate_limit.py b/tests/test_server_rate_limit.py new file mode 100644 index 0000000000..cce2b93d18 --- /dev/null +++ b/tests/test_server_rate_limit.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import RateLimitConfig +from powercontext.server.factory import create_server_app +from powercontext.server.settings import McpConfig, ServerSettings + + +def _settings(tmp_path, *, enabled: bool): + return ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}"), + rate_limit=RateLimitConfig(enabled=enabled, requests=1, window_seconds=60), + mcp=McpConfig(enabled=False), + ) + + +def test_enabled_rate_limit_charges_a_principal_budget(tmp_path) -> None: + app = create_server_app(settings=_settings(tmp_path, enabled=True)) + with TestClient(app) as client: + first = client.get("/v1/operations") + second = client.get("/v1/operations") + probe = client.get("/health/live") + + assert first.status_code != 429 + assert second.status_code == 429 + assert second.json()["error"]["code"] == "rate_limited" + assert int(second.headers["retry-after"]) > 0 + # Liveness probes never consume the caller's window. + assert probe.status_code == 200 + + +def test_disabled_rate_limit_leaves_requests_unlimited(tmp_path) -> None: + app = create_server_app(settings=_settings(tmp_path, enabled=False)) + with TestClient(app) as client: + statuses = {client.get("/v1/operations").status_code for _ in range(3)} + + assert 429 not in statuses diff --git a/tests/test_server_role_routes.py b/tests/test_server_role_routes.py new file mode 100644 index 0000000000..f606b31edf --- /dev/null +++ b/tests/test_server_role_routes.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The deployment role gates the actual factory route table, not only composition.""" + +from typing import Literal + +import pytest +from pydantic import SecretStr + +from powercontext.builtin.persistence.oceanbase.profile import OceanBaseConfig +from powercontext.builtin.runtime.config import DeploymentConfig, InferenceConfig +from powercontext.server.factory import create_server_app +from powercontext.server.settings import McpConfig, ServerSettings + +Role = Literal["all", "api", "scheduler", "worker"] +DATABASE_URL = SecretStr("mysql+aoceanbase://user:pass@localhost:2881/powercontext?charset=utf8mb4") + + +def _settings(role: Role, *, mcp: McpConfig) -> ServerSettings: + return ServerSettings( + deployment=DeploymentConfig(mode="distributed", role=role, id=f"node-{role}"), + database=OceanBaseConfig(url=DATABASE_URL), + # A distributed Worker validates a generation model at composition time; + # the route table assertions below never open the runtime. + inference=InferenceConfig(generation_model="stub:model"), + mcp=mcp, + ) + + +def _paths(app) -> set[str]: + return {route.path for route in app.routes} + + +@pytest.mark.parametrize("role", ["scheduler", "worker"]) +def test_distributed_roles_expose_only_the_health_metrics_boundary(role): + app = create_server_app(settings=_settings(role, mcp=McpConfig(enabled=False))) + paths = _paths(app) + assert "/health/live" in paths + assert "/health/ready" in paths + assert "/metrics" in paths + data_plane = [path for path in paths if path.startswith("/v1/")] + assert data_plane == [] + assert "/openapi.json" not in paths + assert "/docs" not in paths + + +def test_distributed_api_keeps_the_data_plane(): + app = create_server_app(settings=_settings("api", mcp=McpConfig(enabled=False))) + paths = _paths(app) + assert "/health/live" in paths + assert "/v1/memory/flush" in paths + assert "/v1/scopes/{scope_id}/operations" in paths + + +def test_single_node_keeps_the_data_plane(): + settings = ServerSettings(database=OceanBaseConfig(url=DATABASE_URL), mcp=McpConfig(enabled=False)) + paths = _paths(create_server_app(settings=settings)) + assert "/v1/memory/flush" in paths + assert "/v1/scopes/{scope_id}/operations" in paths + + +def test_health_boundary_roles_reject_dashboard_and_mcp(): + with pytest.raises(ValueError, match="health and metrics"): + create_server_app(settings=_settings("worker", mcp=McpConfig(enabled=True))) diff --git a/tests/test_server_tracing.py b/tests/test_server_tracing.py index 9c1063dcda..be997e6f41 100644 --- a/tests/test_server_tracing.py +++ b/tests/test_server_tracing.py @@ -27,6 +27,7 @@ from opentelemetry.trace import SpanKind, StatusCode from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.protocols import RuntimeTraceContext from powercontext.client import PowerContextClient from powercontext.server.factory import create_server_app from powercontext.server.settings import ( @@ -495,6 +496,27 @@ def test_background_stage_starts_a_fresh_trace_outside_an_ambient_span() -> None assert attributes["powercontext.background.source_count"] == 0 +def test_background_stage_links_a_recovered_attempt_without_parenting_it() -> None: + tracing, exporter = _tracing() + + with tracing.background("work.execute", operation="work.execute", attributes={}) as first: + context = first.trace_context + assert context is not None + with tracing.background( + "work.execute", + operation="work.execute", + attributes={}, + links=(RuntimeTraceContext(trace_id=context.trace_id, span_id=context.span_id),), + ): + pass + + first_span, recovered_span = exporter.get_finished_spans() + assert recovered_span.parent is None + assert len(recovered_span.links) == 1 + assert recovered_span.links[0].context.trace_id == first_span.context.trace_id + assert recovered_span.links[0].context.span_id == first_span.context.span_id + + def test_background_isolates_child_spans_when_root_start_fails(monkeypatch) -> None: tracing, exporter = _tracing() ambient = tracing.start_span("HTTP flush_memory", kind=SpanKind.SERVER, attributes={})