diff --git a/.github/ISSUE_TEMPLATE/bounded-adapter.yml b/.github/ISSUE_TEMPLATE/bounded-adapter.yml new file mode 100644 index 0000000..b0c1c2c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bounded-adapter.yml @@ -0,0 +1,35 @@ +name: Bounded adapter proposal +description: Propose one deterministic job adapter without expanding the control plane. +title: "[adapter] " +labels: ["adapter"] +body: + - type: input + id: owner + attributes: + label: Workflow owner and public contract + description: Link to the independently owned workflow or fixture contract. + validations: + required: true + - type: textarea + id: boundary + attributes: + label: Adapter boundary + description: Define input, output, timeout, retry, cost, and redaction behavior. + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Proposed evidence + description: Describe deterministic fixtures, failure cases, and verification. + validations: + required: true + - type: checkboxes + id: non_goals + attributes: + label: Scope agreement + options: + - label: This proposal does not add a new scheduler or hosted control plane. + required: true + - label: Model output will not control retry, budget, or terminal-state policy. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..26f298c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Private security report + url: https://github.com/ashishki/Agent-Runtime-Grid/security/advisories/new + about: Report vulnerabilities without disclosing sensitive details publicly. diff --git a/.github/ISSUE_TEMPLATE/evidence-integrity.yml b/.github/ISSUE_TEMPLATE/evidence-integrity.yml new file mode 100644 index 0000000..3c6190a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/evidence-integrity.yml @@ -0,0 +1,26 @@ +name: Evidence integrity defect +description: Report a checksum, manifest, report, or claim-verification mismatch. +title: "[evidence] " +labels: ["evidence"] +body: + - type: textarea + id: artifact + attributes: + label: Artifact and revision + description: Identify the artifact, source revision, and command that produced it. + validations: + required: true + - type: textarea + id: mismatch + attributes: + label: Verification mismatch + description: Show expected versus observed hashes or semantics without private data. + validations: + required: true + - type: textarea + id: impact + attributes: + label: Claim impact + description: State exactly which conclusion is no longer supported. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/runtime-defect.yml b/.github/ISSUE_TEMPLATE/runtime-defect.yml new file mode 100644 index 0000000..5660d2f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/runtime-defect.yml @@ -0,0 +1,35 @@ +name: Runtime defect +description: Reproduce a bounded lifecycle, queue, storage, or worker defect. +title: "[defect] " +labels: ["bug"] +body: + - type: textarea + id: invariant + attributes: + label: Violated invariant + description: State the expected deterministic runtime rule. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Minimal reproduction + description: Include revision, command, sanitized inputs, and observed output. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Python, Postgres, Redis, OS, and container versions. + validations: + required: true + - type: checkboxes + id: safety + attributes: + label: Safety checks + options: + - label: The report contains no secrets or private data. + required: true + - label: The reproduction does not target production infrastructure. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..104809a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,14 @@ +## Invariant and scope + +Describe the bounded runtime invariant or documentation contract changed. + +## Reproduction and verification + +List the failing-before/passing-after test and the full local checks run. + +## Evidence and safety + +- [ ] No secrets, private data, generated reports, or absolute local paths are committed. +- [ ] Claims and known limits remain accurate. +- [ ] Database tests target only the local project/test database. +- [ ] License review was updated if dependencies changed. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04e98be..09a141c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: services: postgres: - image: postgres:16 + image: postgres:16.14-alpine3.24@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 env: POSTGRES_USER: testuser POSTGRES_PASSWORD: testpassword @@ -27,7 +27,7 @@ jobs: - 5432:5432 redis: - image: redis:7-alpine + image: redis:7.2.14-alpine3.21@sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8 options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -37,10 +37,10 @@ jobs: - 6379:6379 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" cache: "pip" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ba4b25b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +All notable changes are documented here. The project follows Semantic +Versioning after its first tagged release. + +## Unreleased + +- Narrowed the supported surface to the local CLI/library runtime. +- Removed non-running API/worker services and unconnected dashboards from the + default Compose topology. +- Added persistent finalization-conflict accounting distinct from terminal-event + invariant violations. +- Added explicit, local-only destructive reset controls. +- Added Apache-2.0 licensing and bounded contribution/security surfaces. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0613868 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +Agent Runtime Grid accepts narrow, reproducible changes within its local runtime +boundary. Good contributions fix a demonstrated lifecycle defect, add a bounded +job adapter, strengthen evidence verification, or correct documentation. + +Broad orchestration frameworks, hosted-control-plane features, provider demos, +and claims without reproducible evidence are out of scope. + +## Local setup + +```bash +python3 -m venv .venv +PATH=.venv/bin:$PATH python -m pip install -e . -r requirements-dev.txt +docker-compose up -d postgres redis +PATH=.venv/bin:$PATH ruff check src tests +PATH=.venv/bin:$PATH ruff format --check src tests +PATH=.venv/bin:$PATH python -m pytest -q +``` + +The integration suite creates and drops tables in the local +`agent_runtime_grid` or `agent_runtime_grid_test` database. Never point tests or +proof commands at production data. + +## Change contract + +1. Open a focused issue using the closest template. +2. Add a failing regression test before changing behavior. +3. Keep deterministic control decisions outside model/provider code. +4. Do not commit credentials, customer data, generated reports, or absolute + local paths. +5. Update known limits and evidence semantics when a claim changes. +6. Run the full lint, format, and test commands above. + +Pull requests should describe the invariant, the reproduction, the expected +failure before the change, and the verification after it. A passing test alone +does not justify a new product claim. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index ec796be..f420587 100644 --- a/README.md +++ b/README.md @@ -1,241 +1,153 @@ # Agent Runtime Grid -Agent Runtime Grid is a local-first T1 runtime for executing batches of AI and agent jobs with queue-backed dispatch, durable state, bounded retries, timeouts, cancellation, idempotent finalization, artifacts, logs, metrics, cost telemetry, and reliability reports. - -The project is built around a deterministic control plane. Scheduling, Redis Streams dispatch, Postgres-backed lifecycle state, retry policy, budget boundaries, and terminal state transitions are owned by code. Optional AI behavior belongs inside bounded sample job payloads, not inside runtime control decisions. - -## Why This Exists - -AI and agent workloads often start as one-off scripts. That breaks down when operators need to run many jobs, recover from partial failure, prove what happened, and compare reliability across runs. - -Agent Runtime Grid provides the execution layer for that shape of work: - -- submit many jobs as one run -- dispatch through Redis Streams -- process with bounded workers -- persist lifecycle and event history in Postgres -- retry transient failures -- stop timed-out or cancelled work -- prevent duplicate terminal finalization -- write artifacts and reports -- keep default execution in stub mode with zero model cost - -## What Works Today - -The current implementation proves the core runtime mechanics: - -- job submission and idempotency-key handling -- Postgres job registry and append-only event log -- Redis Streams publish, lease, acknowledge, and DLQ paths -- worker lifecycle state transitions -- transient retry and non-retryable policy failure handling -- timeout and cancellation terminal paths -- idempotent finalization with a database guard -- JSON artifacts with input digests -- sanitized structured job log records -- runtime metrics and in-memory trace spans -- queue and backpressure inspection -- deterministic failure injection plans -- cost telemetry records and rollup output -- enforceable stub/live budget gates and strict cost rollup checks -- batch CLI helpers and benchmark report rendering -- Eval-Ground-Truth-Lab case execution through normal queue/workers -- deterministic gdev-agent webhook evaluation jobs -- full-stack artifact proof command that ingests ready Eval Lab and gdev-agent - artifacts, runs selected cases through Grid workers, and writes cross-linked - runtime evidence -- failure-injection report pack generation -- committed evidence snapshots under `docs/evidence/` for smoke, reliability, - failure-injection, and cross-project artifact proof surfaces -- operator queue inspection, stale recovery, and pending lease renewal primitives -- automated worker heartbeat renewal for active long-running jobs -- bounded autonomous routine contract for manual, cron, webhook, and event - triggers without expanding Grid beyond T1 runtime responsibilities - -Current baseline: `80 passed` with one upstream FastAPI/Starlette deprecation warning. - -## Quickstart - -Create and activate a virtual environment, then install dependencies: +Agent Runtime Grid is an alpha, local-first execution and reliability harness for +batches of AI and agent jobs. Its control path is deterministic: Postgres owns +lifecycle state, Redis Streams owns delivery state, and workers enforce bounded +retry, timeout, cancellation, budget, artifact, and terminal-state rules. -```bash -python3 -m venv .venv -PATH=.venv/bin:$PATH python -m pip install -e . -r requirements-dev.txt -``` +This repository is the optional runtime component in the portfolio. It is not a +hosted service, a production claim, or a dependency of Eval Ground Truth Lab. + +## Start here: a five-minute local proof -Start local services: +Prerequisites: Python 3.12, Docker, and Docker Compose. ```bash +python3 -m venv .venv +PATH=.venv/bin:$PATH python -m pip install -e . -r requirements-dev.txt docker-compose up -d postgres redis +PATH=.venv/bin:$PATH agent-runtime-grid smoke \ + --jobs 20 \ + --workers 4 \ + --mode stub \ + --reset-local-database \ + --report reports/smoke.md ``` -Run the baseline: +`--reset-local-database` is deliberately explicit. The command refuses to drop +remote or unrelated databases. Do not point the proof commands at production +infrastructure. + +Verify the implementation before trusting a report: ```bash +PATH=.venv/bin:$PATH ruff check src tests +PATH=.venv/bin:$PATH ruff format --check src tests PATH=.venv/bin:$PATH python -m pytest -q -PATH=.venv/bin:$PATH ruff check -PATH=.venv/bin:$PATH ruff format --check ``` -Run the current smoke report harness: +## What is implemented -```bash -PATH=.venv/bin:$PATH python -m pytest tests/load/test_benchmark_harness.py::test_smoke_benchmark_writes_report -q -``` +- idempotent batch submission and Postgres lifecycle/event records; +- Redis Streams publish, lease, acknowledge, retry, stale recovery, and DLQ paths; +- bounded workers with timeout and cancellation terminal paths; +- a database terminal-finalization guard; +- separate persistent counts for rejected finalization attempts and actual + duplicate terminal-event invariant violations; +- content-digested JSON artifacts and integrity checks; +- deterministic failure injection and zero-cost stub execution; +- cost records and enforceable budget boundaries; +- queue/backpressure inspection and in-process metrics/traces; +- smoke, reliability, failure-injection, Eval Lab, and gdev artifact proof paths. -Run the real 100-job smoke command: +The default Compose file starts only the dependencies used by these paths: +Postgres and Redis. Earlier placeholder `api`, `worker`, Prometheus, and Grafana +services were removed because they did not run or observe the product. -```bash -PATH=.venv/bin:$PATH python -m agent_runtime_grid.cli smoke \ - --jobs 100 \ - --workers 4 \ - --failure-rate 0 \ - --mode stub \ - --report reports/smoke.md +## Product boundary + +```text +CLI or Python caller + -> Postgres job registry and append-only event history + -> Redis Streams delivery queue + -> bounded in-process worker pool + -> job adapter + -> artifacts + reliability evidence ``` -Run the real 500-job reliability proof: +The FastAPI module remains experimental library code and is not part of the +default runnable surface. There is no long-running worker service contract yet. +Metrics can be rendered in process; this repository does not claim a connected +dashboard deployment. + +## Reliability proof + +The larger deterministic proof injects retry, timeout, and idempotency cases: ```bash -PATH=.venv/bin:$PATH python -m agent_runtime_grid.cli benchmark v1-proof \ +PATH=.venv/bin:$PATH agent-runtime-grid benchmark v1-proof \ --jobs 500 \ --workers 20 \ --failure-rate 0.10 \ --include-timeouts \ --repeat-idempotency-submissions \ + --reset-local-database \ --report reports/v1/reliability_report.md ``` -The smoke and 500-job reliability proof commands now run end to end through local Redis Streams, workers, Postgres state, artifacts, and report validation. +The proof uses stub jobs and reports an estimated model cost of zero. It is local +reliability evidence, not load capacity or production SLO evidence. -Run the cross-project artifact proof once Eval Lab and gdev-agent artifacts exist: +## Relationship to the portfolio -```bash -PATH=.venv/bin:$PATH python -m agent_runtime_grid.cli proof full-stack \ - --eval-lab-dataset ../Eval-Ground-Truth-Lab/datasets/gdev_agent/triage_v1.jsonl \ - --eval-lab-report ../Eval-Ground-Truth-Lab/reports/gdev-agent/baseline_report.md \ - --gdev-artifact ../gdev-agent/eval/results/last_run.json \ - --jobs 20 \ - --workers 4 \ - --report reports/full-stack/runtime_report.md -``` +- [Eval Ground Truth Lab](https://github.com/ashishki/Eval-Ground-Truth-Lab) + owns datasets, comparison, and release gates. Runtime Grid is optional. +- [gdev-agent](https://github.com/ashishki/gdev-agent) is one reference workload; + its own repository owns tenant isolation, application behavior, and quality. +- AI Workflow Playbook is a governance companion, not a runtime dependency. +- The planned `ai-workflow-reliability-lab` umbrella pins compatible component + releases; it does not absorb this repository or its history. -This submits selected Eval Lab/gdev cases as normal Grid jobs, dispatches them -through Redis Streams, completes them with workers, writes artifacts and -Eval-compatible result JSON, then renders one report linking quality evidence -and runtime reliability evidence. This is `full-stack-artifact-proof`: it uses -ready artifacts and deterministic jobs by default. It does not call live -gdev-agent over HTTP. +Cross-project artifact proof commands are documented in +[`docs/INTEGRATIONS.md`](docs/INTEGRATIONS.md). They consume explicitly supplied +local artifacts and do not prove a hosted integration. -Optionally run the live-local proof against a locally running `gdev-agent` in -deterministic demo mode. The webhook secret is read from the named environment -variable; it is not stored in the job payload or report: +## Evidence and interpretation -```bash -PATH=.venv/bin:$PATH python -m agent_runtime_grid.cli proof full-stack-live-local \ - --eval-lab-dataset ../Eval-Ground-Truth-Lab/datasets/gdev_agent/triage_v1.jsonl \ - --eval-lab-report ../Eval-Ground-Truth-Lab/reports/gdev-agent/baseline_report.md \ - --gdev-artifact ../gdev-agent/eval/results/last_run.json \ - --gdev-base-url http://localhost:8000 \ - --gdev-webhook-secret-env GDEV_AGENT_WEBHOOK_SECRET \ - --jobs 20 \ - --workers 4 \ - --report reports/full-stack/live_local_runtime_report.md -``` +Generated reports live under `reports/` and are ignored by default. Curated +snapshots under [`docs/evidence/`](docs/evidence/) show report shape, while the +[`evidence index`](docs/EVIDENCE_INDEX.md) states how to reproduce them. A +committed snapshot is illustrative until its command, source revision, inputs, +and checksums are independently verified. -## Architecture +Important evidence terms: -The runtime path is: +- `idempotency replay` means the same submission key returned the existing job; +- `finalization conflict attempt` means the DB guard rejected a competing + terminal write; +- `duplicate terminal event` means an invariant violation and must remain zero; +- `artifact integrity` validates recorded bytes, not the semantic quality of an + agent response. -```text -API / CLI - -> Postgres job registry and event log - -> Redis Streams queue - -> worker runtime - -> job runner - -> artifact store - -> metrics, traces, and reports -``` +## Known limits + +Runtime Grid is not: + +- a hosted or multi-tenant SaaS; +- a Kubernetes, Temporal, or Ray replacement; +- exactly-once execution; +- a production sandbox for untrusted arbitrary code; +- a general autonomous-agent framework; +- evidence of customer traffic, production scale, or an external user. + +The current supported boundary is a local CLI/library proof with Postgres, +Redis, deterministic adapters, and inspectable evidence. See +[`docs/KNOWN_LIMITS.md`](docs/KNOWN_LIMITS.md) for the longer list. + +## Open-source scope + +The project is licensed under Apache-2.0. The direct dependency and service +license review is recorded in [`docs/LICENSE_REVIEW.md`](docs/LICENSE_REVIEW.md). +Contributions are intentionally limited to reproducible runtime defects, +bounded adapters, evidence verification, and documentation corrections; see +[`CONTRIBUTING.md`](CONTRIBUTING.md) and [`SECURITY.md`](SECURITY.md). + +## Reference documents -Core references: - -- `docs/ARCHITECTURE.md` - canonical architecture and runtime boundaries -- `docs/ARCHITECTURE_DIAGRAM.md` - compact runtime and evidence-flow diagram -- `docs/STACK_OVERVIEW.md` - three-project stack map and live/artifact proof split -- `docs/CASE_STUDY.md` - concise reliability and integration case study -- `docs/EVIDENCE_INDEX.md` - evidence and verification pointers -- `docs/KNOWN_LIMITS.md` - known limits and non-goals -- `docs/SECURITY_BOUNDARIES.md` - API auth and local bind safety rules -- `docs/OBSERVABILITY.md` - queue/backpressure metrics and report definitions -- `docs/INTEGRATIONS.md` - Eval Lab and gdev-agent integration boundaries -- `docs/OPERATIONS.md` - local operator commands for queue inspection and stale recovery -- `docs/AUTONOMOUS_ROUTINE_CONTRACT.md` - bounded routine trigger/runtime contract -- `docs/TRIGGER_SECURITY.md` - cron/webhook/event/manual trigger security boundaries -- `reports/README.md` - report locations and expectations -- `docs/evidence/` - committed evidence snapshots for generated report surfaces - -## Operational Guarantees - -Current local guarantees: - -- durable job lifecycle state is Postgres-authoritative -- Redis Streams is treated as delivery state, not final lifecycle authority -- terminal state changes are guarded by idempotent finalization -- default stub execution makes no paid model calls -- secrets and raw job payloads must not appear in logs, metrics, traces, or committed config -- health is public and secret-free -- non-health API routes require `API_TOKEN` when configured -- no-token API mode is allowed only on localhost bindings -- stub mode blocks provider calls and live dispatch requires explicit budgets -- local mode stays inside T1 Docker Compose boundaries - -Current proof coverage: - -- T16 real 100-job smoke command - implemented -- T17 real 500-job reliability proof - implemented -- T18 worker crash and stale lease recovery - implemented -- T19 backpressure and queue lag metrics - implemented -- T20 API auth and local boundary proof - implemented -- T21 artifact integrity in reports - implemented -- T22 cost budget gates - implemented -- T23 Eval-Ground-Truth-Lab integration - implemented -- T24 gdev-agent batch simulation - implemented -- T25 failure-injection report pack - implemented -- T29 cross-project runtime proof - implemented -- T31 full-stack live-local proof - implemented - -## Reports - -Reports are written under `reports/`. Generated report contents are ignored by -git so local benchmark output does not churn history. Stable placeholders, -report documentation, and curated committed snapshots under `docs/evidence/` -are committed for inspection. - -Evidence index: `docs/EVIDENCE_INDEX.md`. - -Report guide: `reports/README.md`. - -Committed snapshots: - -- `docs/evidence/runtime-smoke-100.md` -- `docs/evidence/runtime-reliability-500.md` -- `docs/evidence/full-stack-artifact-proof.md` -- `docs/evidence/full-stack-live-local.md` -- `docs/evidence/failure-injection-pack-summary.md` - -## Known Limits - -The current known limits are intentional and documented in `docs/KNOWN_LIMITS.md`. - -Short version: - -- not a Temporal replacement -- not a Ray replacement -- not a Kubernetes replacement -- not a production sandbox for arbitrary untrusted code -- not a SaaS or multi-tenant billing system -- not an autonomous swarm -- not exactly-once execution -- not a general scheduler for arbitrary unbounded routines - -The v1 target is narrower: local-first reliability evidence for many AI and agent jobs under queue, worker, retry, timeout, cancellation, artifact, cost, and reporting controls. +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) +- [`docs/ARCHITECTURE_DIAGRAM.md`](docs/ARCHITECTURE_DIAGRAM.md) +- [`docs/EVIDENCE_INDEX.md`](docs/EVIDENCE_INDEX.md) +- [`docs/OPERATIONS.md`](docs/OPERATIONS.md) +- [`docs/SECURITY_BOUNDARIES.md`](docs/SECURITY_BOUNDARIES.md) +- [`docs/OBSERVABILITY.md`](docs/OBSERVABILITY.md) +- [`reports/README.md`](reports/README.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0582948 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,27 @@ +# Security policy + +## Supported scope + +Security fixes are accepted for the current `0.1.x` development line. This is a +local-first alpha and is not supported as an internet-facing hosted service. + +## Reporting a vulnerability + +Use GitHub's private vulnerability-reporting or security-advisory flow for this +repository. Do not publish secrets, exploit payloads, private data, or a working +attack in a public issue. + +Include the affected revision, a minimal reproduction, expected impact, and any +safe mitigation already tested. Reports about third-party services should also +follow that provider's disclosure process. + +## Boundaries + +- Compose binds Postgres and Redis to loopback and uses development credentials. +- Proof commands refuse destructive reset for remote or unrelated databases and + require an explicit local-reset flag. +- Arbitrary untrusted job execution is not a supported use case. +- Stub mode is the default evidence path; live provider use requires explicit + secrets and cost limits outside committed files. + +See `docs/SECURITY_BOUNDARIES.md` and `docs/KNOWN_LIMITS.md` before deployment. diff --git a/docker-compose.yml b/docker-compose.yml index 7cc7c6e..9720f72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,42 +1,6 @@ services: - api: - image: python:3.12-slim - working_dir: /app - command: ["python", "-c", "import agent_runtime_grid; print(agent_runtime_grid.__version__)"] - environment: - PYTHONPATH: /app/src - DATABASE_URL: postgresql+asyncpg://agent_runtime_grid:local-dev-password@postgres:5432/agent_runtime_grid - REDIS_URL: redis://redis:6379/0 - ARTIFACT_ROOT: /app/artifacts - LLM_MODE: stub - volumes: - - .:/app:ro - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - worker: - image: python:3.12-slim - working_dir: /app - command: ["python", "-c", "import agent_runtime_grid; print(agent_runtime_grid.__version__)"] - environment: - PYTHONPATH: /app/src - DATABASE_URL: postgresql+asyncpg://agent_runtime_grid:local-dev-password@postgres:5432/agent_runtime_grid - REDIS_URL: redis://redis:6379/0 - ARTIFACT_ROOT: /app/artifacts - LLM_MODE: stub - volumes: - - .:/app:ro - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - postgres: - image: postgres:16-alpine + image: postgres:16.14-alpine3.24@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 environment: POSTGRES_DB: agent_runtime_grid POSTGRES_USER: agent_runtime_grid @@ -52,7 +16,7 @@ services: retries: 10 redis: - image: redis:7-alpine + image: redis:7.2.14-alpine3.21@sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8 ports: - "127.0.0.1:6379:6379" volumes: @@ -63,30 +27,6 @@ services: timeout: 5s retries: 10 - prometheus: - image: prom/prometheus:v2.55.1 - ports: - - "127.0.0.1:9090:9090" - volumes: - - prometheus-data:/prometheus - - grafana: - image: grafana/grafana:11.3.1 - environment: - GF_SECURITY_ADMIN_USER: local-admin - GF_SECURITY_ADMIN_PASSWORD: local-dev-password - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer - ports: - - "127.0.0.1:3000:3000" - volumes: - - grafana-data:/var/lib/grafana - depends_on: - - prometheus - volumes: postgres-data: redis-data: - prometheus-data: - grafana-data: - diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a7d91e2..b2a37ca 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,284 +1,98 @@ -# Architecture - Agent Runtime Grid +# Architecture -Version: 1.0 -Last updated: 2026-06-11 -Status: Active +Last updated: 2026-07-13. Status: alpha, local-first CLI/library. ---- +## Supported runtime path -## System Overview - -Agent Runtime Grid is a queue-driven runtime for executing many AI or agent jobs with durable state, bounded worker execution, retries, timeouts, idempotent finalization, artifacts, logs, cost tracking, and observability. It serves AI engineers, platform engineers, and operators who need evidence that agent workloads can be operated under concurrency and failure. The control plane is deterministic: scheduling, state transitions, retry policy, budget enforcement, and audit events are code-owned, while optional AI behavior is limited to bounded sample job payloads. - ---- - -## Problem Fit and Adoption Reality - -### Problem-First Entry Gate - -| Question | Answer | -|----------|--------| -| Concrete operational pain | AI and agent workflows are often run as scripts or ad hoc one-off jobs that fail silently, duplicate work, lose intermediate state, and lack retry, timeout, cost, and observability semantics under concurrent load. | -| Current workaround | Operators run scripts manually, schedule simple cron jobs, call LLM APIs synchronously, inspect logs by hand, and rerun failed tasks manually. | -| Why existing process is insufficient | Scripts, ordinary CI, and manual SOPs do not prove dispatch reliability, backpressure handling, worker isolation, lifecycle control, idempotency, cost attribution, or recovery after partial failure. | -| First user / operator who feels the pain | The operator running agent simulations and evaluation batches; secondarily an AI platform or agent infrastructure team evaluating runtime behavior. | -| What would make v1 not worth adopting | A single synchronous FastAPI wrapper, no concurrent worker execution, untestable retry/idempotency semantics, or cosmetic observability that cannot support a failure report. | -| First proof of value | Run 500 synthetic jobs with 20 workers, 10% injected failures, timeout cases, and repeated idempotency-key submissions; report completion rate, duplicate-finalization count, retry count, queue lag, p95 duration, cost, and artifact completeness. | - -### Adoption Reality Gate - -| Boundary | Decision | -|----------|----------| -| Work AI is expected to improve | Optional sample jobs may use AI for repo analysis, ticket classification, summarization, or synthetic agent tasks. The runtime itself does not use AI for scheduling or policy decisions. | -| Work AI will not replace | Human ownership remains required for job definitions, safety policies, budget ceilings, tool permissions, high-risk approval, and interpretation of failure reports. | -| Claims not allowed before evidence | Do not claim this replaces Temporal, Ray, Kubernetes, Airflow, managed batch systems, production sandboxing, or exactly-once execution. Do not claim a fully autonomous agent swarm. | -| Local-to-production evidence required | Before trusted or remote operation, collect load-test reports, failure-injection reports, idempotency evidence, budget telemetry, and approval-boundary review for any external side-effecting job type. | - ---- - -## Solution Shape - -| Decision | Selection | Justification | -|----------|-----------|---------------| -| Primary shape | Hybrid decomposition | The control plane is deterministic and workflow-oriented, while individual sample jobs may contain bounded AI behavior behind explicit job-type contracts. | -| Governance level | Standard | The project is non-trivial infrastructure with persistent state, queues, workers, cost, and evidence needs, but v1 is local-only, non-compliance, and not a privileged production autonomous runtime. | -| Runtime tier | T1 | Local Docker Compose services and bounded worker containers are required for Postgres, Redis, and worker isolation. T2/T3 are not justified because v1 forbids privileged containers, untrusted arbitrary code execution, and persistent mutable autonomous workers. | - -### Rejected Lower-Complexity Options - -| Rejected option | Why it is insufficient | -|-----------------|------------------------| -| Single synchronous FastAPI wrapper | It cannot exercise queue lag, worker concurrency, retry isolation, cancellation, or at-least-once delivery behavior. | -| Manual scripts plus log inspection | It leaves idempotency, retries, artifacts, and cost attribution unverifiable under repeated failure. | -| Cron-only batch runner | It lacks durable per-job lifecycle events, worker leases, backpressure visibility, and controlled requeue after crash. | -| External scheduler as the whole project | It would hide the runtime behavior and implementation evidence that v1 is meant to validate; external systems may be referenced later, not treated as the v1 proof. | - -### Minimum Viable Control Surface - -- Explicit job schema validation, allowed job types, timeout bounds, retry bounds, budget bounds, and artifact size limits. -- Durable job registry, immutable event log, worker lease semantics, and idempotent finalization keyed by job ID and idempotency key. -- Docker Compose service boundary for API, worker, Postgres, Redis, Prometheus, and Grafana. -- Human approval for new external side effects, budget increases, broader tool permissions, new network egress, and remote/cloud runtime promotion. - -### Human Approval Boundaries - -| Boundary | Human approval required? | Why | -|----------|--------------------------|-----| -| Enabling a new job type with external side effects | Yes | The runtime can amplify duplicate external writes under retries. | -| Increasing per-run or per-job budget ceilings | Yes | Live LLM mode can spend money across many jobs and retries. | -| Allowing new network egress domains from workers | Yes | Egress expands data, security, and cost exposure. | -| Enabling shell, package, or toolchain mutation inside job containers | Yes | Mutation moves the runtime toward T2 behavior and requires stronger isolation. | -| Promoting from local mode to remote/cloud operation | Yes | Blast radius, secrets, auth, and recovery expectations change. | -| Dispatching configured local stub jobs within bounds | No | This is the normal deterministic runtime path. | - -### Deterministic vs LLM-Owned Subproblems - -| Subproblem | Owner | Reason | -|------------|-------|--------| -| Job schema validation, allowed job types, budget limits, timeout bounds, retry bounds, artifact size limits | Deterministic | These are policy constraints that must be reproducible and testable. | -| Queue selection, worker leasing, retry eligibility, cancellation, state transitions, DLQ routing | Deterministic | Runtime control behavior must not vary with model output. | -| Cost calculation, token aggregation, queue lag, p50/p95/p99 metrics, retry counters | Deterministic | Measurements must be auditable and comparable across runs. | -| Sample job summarization, repo analysis, or ticket classification | Optional LLM inside job payload | AI behavior is isolated to job work and governed by job-type budget and tool boundaries. | - -### Runtime and Isolation Model - -| Property | Decision | -|----------|----------| -| Isolation boundary | T1 container boundary for API, worker, Postgres, Redis, Prometheus, and Grafana in Docker Compose. | -| Persistence model | Postgres stores job registry and event log; local filesystem or MinIO-compatible storage stores artifacts; Redis Streams stores queue state. | -| Network model | Default local service-to-service traffic only. External LLM or GitHub API egress is disabled unless explicitly enabled by config and human approval. | -| Secrets model | Secrets come from environment variables or local uncommitted `.env` files. Workers receive only job-type-specific allowlisted variables. | -| Runtime mutation boundary | Runtime containers do not install packages or mutate toolchains at job execution time in v1. Sample jobs run from prebuilt images. | -| Rollback / recovery model | Rebuild containers from source, requeue unfinished jobs after stale worker lease, mark timed-out jobs deterministically, preserve event log, and rerun datasets with the same seed. | - -T0 is insufficient because the project needs local service dependencies and bounded worker containers. T2/T3 are out of scope for v1. - ---- - -## Inference / Model Strategy - -| Path / Task | Model class | Why this class | Fallback / escalation | Budget / latency constraint | -|-------------|-------------|----------------|-----------------------|-----------------------------| -| Runtime scheduler, state machine, retries, idempotency, budget enforcement | No model | These paths must be deterministic and testable. | None. Model use here requires an ADR and human approval. | $0. | -| Default stub benchmark jobs | Stub model or deterministic fixture | The v1 reliability proof must run without paid API calls. | None required. | $0 default run. | -| Optional live LLM sample jobs | Small or standard structured-output model | The job may classify, summarize, or analyze text, but quality demands are modest and bounded. | Stronger model only for marked jobs with approval and cost budget update. | Target below $5 per full benchmark run; per-job budget configurable. | - ---- - -## Cost Budget and Attribution - -| Budget Scope | Limit | Approval Trigger | Attribution Fields | Evidence Location | -|--------------|-------|------------------|--------------------|-------------------| -| Default stub benchmark run | $0 | Any model call during stub mode | project, run_id, job_id, job_type, worker_id, model, environment | `docs/COST_BUDGET.md` | -| Optional live LLM benchmark run | Target below $5 per full run | Projected or actual spend above configured run limit | project, run_id, job_id, job_type, worker_id, model, operator, environment | `docs/COST_BUDGET.md`; future telemetry from T12 | -| Per job | Configurable, default disabled in stub mode | Model escalation, retry expansion, tool-call expansion, or job-type budget increase | project, task/workflow, agent/role, model, operator, feature/workload, environment | `docs/COST_BUDGET.md` | - -Until T12 adds project-owned cost telemetry, cost thresholds are manual-review boundaries and runtime config gates, not CI-enforced rollups. - ---- - -## Capability Profiles - -| Profile | Status | Activation note | Justification | -|---------|--------|-----------------|---------------| -| RAG | OFF | Future activation requires an ADR and dedicated eval artifact. | v1 runtime does not need retrieval-backed answers. | -| Tool-Use | OFF | Future activation requires an ADR and dedicated eval artifact. | The runtime records and constrains job tools deterministically. LLM-directed unsafe tool selection is out of scope for v1. | -| Agentic | OFF | Future activation requires an ADR and dedicated eval artifact. | The control plane is not an agent loop. It runs jobs but does not let a model plan scheduling, retries, or safety decisions. | -| Planning | OFF | Future activation requires an ADR and dedicated eval artifact. | The product output is job execution and evidence reports, not structured plans for downstream execution. | -| Compliance | OFF | Future activation requires an ADR and dedicated eval artifact. | No named compliance framework applies to v1 synthetic local data. Standard security and audit controls still apply. | - ---- - -## Component Table - -| Component | File / Directory | Responsibility | -|-----------|------------------|----------------| -| API control plane | `src/agent_runtime_grid/api/` | Job submission, status, cancellation, health, and artifact metadata endpoints. | -| Domain models | `src/agent_runtime_grid/domain/` | Job, run, worker, event, retry, timeout, and artifact types. | -| Persistence | `src/agent_runtime_grid/storage/` | Postgres repositories for job registry, event log, idempotency records, and cost records. | -| Queue adapter | `src/agent_runtime_grid/queue/` | Redis Streams publish, lease, ack, retry, and DLQ operations. | -| Worker runtime | `src/agent_runtime_grid/worker/` | Worker loop, lease renewal, timeout enforcement, job execution, cancellation handling. | -| Job runners | `src/agent_runtime_grid/jobs/` | Stub jobs, failure injection, optional live LLM sample jobs, and artifact writers. | -| Observability | `src/agent_runtime_grid/observability/` | Prometheus metrics, OpenTelemetry tracing, structured logs, shared tracing helper. | -| CLI | `src/agent_runtime_grid/cli/` | Batch submission, benchmark execution, cleanup, and report commands. | -| Tests | `tests/` | Unit, integration, idempotency, failure-injection, and load harness tests. | -| Reports | `reports/` | Load-test, failure-injection, and cost rollup outputs. | - ---- - -## Data Flow - -1. Operator submits a job or batch through the CLI or API with a job type, payload, idempotency key, timeout, retry limit, and optional budget. -2. API validates schema and policy bounds, writes the job registry row and append-only event, then publishes the job to Redis Streams. -3. Worker leases a queued job, writes a running event, starts the bounded runner, emits metrics and trace spans, and renews the lease while active. -4. Runner writes logs and artifacts to local artifact storage and reports usage/cost metadata through the worker. -5. Worker finalizes the job through an idempotent database transaction, ensuring one terminal finalization for the job and idempotency key. -6. Retry-eligible transient failures are requeued within configured bounds; timeout, cancellation, policy failure, and exhausted retry cases go to terminal states or DLQ. -7. Operator inspects API state, artifacts, logs, Prometheus/Grafana dashboards, and generated benchmark reports. - ---- - -## Tech Stack +```text +Typer CLI or Python caller + -> Postgres job registry, events, finalization guard, conflict records + -> Redis Streams delivery and pending state + -> bounded in-process worker pool + -> allowlisted job adapter + -> local artifacts + -> Markdown + versioned JSON evidence + checksum manifest +``` -| Area | Choice | Rationale | -|------|--------|-----------| -| Language | Python 3.12 | Strong FastAPI, async, Pydantic, SQLAlchemy, pytest, and infrastructure tooling ecosystem. | -| API | FastAPI | Suitable for typed request/response APIs, async handlers, and health/status endpoints. | -| Validation | Pydantic v2 | Enforces job schemas and policy bounds before persistence or queue dispatch. | -| Database | Postgres 16 | Durable job registry, idempotency constraints, event log, and reporting queries. | -| ORM / migrations | SQLAlchemy 2.x and Alembic | Explicit async persistence and versioned schema evolution. | -| Queue | Redis Streams | Lightweight local queue with consumer groups, pending entries, and retry/DLQ patterns. | -| Worker runtime | Async Python workers in Docker Compose | Matches T1 runtime and local benchmark goal without Kubernetes complexity. | -| Artifacts | Local filesystem first, MinIO-compatible interface later | Keeps v1 runnable locally while preserving a path to object storage. | -| Observability | Prometheus, Grafana, OpenTelemetry | Supports queue lag, worker utilization, job duration, retries, cost, and traces. | -| CLI | Typer | Batch submission and benchmark workflows are operator-facing commands. | -| Tests | pytest, pytest-asyncio, httpx | Covers async API, worker, queue, persistence, and integration behavior. | -| Load testing | Locust or k6 | Generates 100-500 job benchmark evidence and report artifacts. | +Postgres is authoritative for lifecycle state. Redis is authoritative only for +delivery, leases, consumer lag, and DLQ state. At-least-once delivery is +expected; terminal writes are protected by a unique database guard. ---- +## Runtime components -## Observability +| Component | Responsibility | Current surface | +|---|---|---| +| `storage/` | Jobs, events, finalizations, conflict attempts, costs | Postgres/SQLAlchemy | +| `queue/` | Publish, lease, acknowledge, retry, pending recovery, DLQ | Redis Streams | +| `worker/` | Bounded execution, timeout, cancellation, retry, stale recovery | In-process async workers | +| `jobs/` | Deterministic stub and bounded Eval/gdev adapters | Allowlisted Python runners | +| `artifacts/` | JSON artifact write and byte-integrity validation | Local filesystem | +| `evidence.py` | Portable JSON bundle, SHA-256 manifest, strict verifier | CLI-generated evidence | +| `observability/` | In-process metrics, sanitized logs, trace records | Library surface only | +| `cli/` | Submit, inspect, smoke, reliability, proof, verification | Supported entry point | +| `api/` | Typed experimental endpoints | Not in default Compose/support contract | -- Metrics: queue depth, queue lag, worker utilization, job duration, retry count, timeout count, failure rate, DLQ count, duplicate-finalization count, artifact completeness, token usage, and estimated cost. -- Traces: API submission, queue publish, worker lease, job execution, artifact write, finalization, retry, timeout, and cancellation spans use `src/agent_runtime_grid/observability/tracing.py`. -- Logs: structured logs include job ID, run ID, worker ID, event type, trace ID, and sanitized error class. Secrets and raw external tokens are never logged. -- Reports: benchmark and failure-injection commands write durable reports under `reports/`. +The default Compose topology contains only Postgres and Redis. Workers are +started by proof commands. There is no claimed long-running worker service, +connected Prometheus deployment, or Grafana dashboard. ---- +## Deterministic authority -## Security Boundaries +Code, not model output, owns schema validation, job allowlisting, idempotency, +retry eligibility, timeout, cancellation, budgets, terminal transitions, +artifact hashing, and evidence verification. Optional AI behavior must stay +inside a bounded adapter with explicit inputs, outputs, cost, and side effects. -- `GET /health` is intentionally public and returns only health status. -- Mutation and inspection endpoints use local token auth when `API_TOKEN` is configured; local mode may disable auth only when bound to localhost. -- No real sensitive data is used in v1; datasets are synthetic. -- Secrets are environment-only and `.env` files are ignored. -- Workers receive scoped env allowlists per job type and no broad host credential mount. -- External network calls are disabled by default. Enabling LLM or GitHub API calls requires explicit config and human approval. +## Finalization semantics ---- +The first terminal transaction creates one `job_finalizations` row, updates the +job, and appends one terminal event. A competing attempt is rejected by the +unique key and recorded in `finalization_conflict_attempts`. -## External Integrations +These are deliberately different metrics: -| Integration | Required for v1 default? | Credentials | Boundary | -|-------------|--------------------------|-------------|----------| -| OpenAI or other LLM provider | No | Optional API key via environment variable | Disabled in stub mode; enabled only for marked live LLM jobs and budget-gated. | -| GitHub API | No | Optional token via environment variable | Optional repo-analysis sample job; disabled unless configured. | -| Prometheus / Grafana | Yes, local containers | No external credentials | Local observability stack. | -| MinIO-compatible object store | No | Optional local credentials | Future artifact backend; local filesystem first. | +- finalization conflict attempts may be non-zero under a race; +- duplicate terminal-event invariant violations must be zero. ---- +Reports calculate both from Postgres, so the values persist across process +restarts. Idempotency-key replays are measured separately. -## File Layout +## Evidence model -```text -. -├── .github/workflows/ci.yml -├── docker-compose.yml -├── pyproject.toml -├── requirements.txt -├── requirements-dev.txt -├── src/agent_runtime_grid/ -│ ├── api/ -│ ├── cli/ -│ ├── domain/ -│ ├── jobs/ -│ ├── observability/ -│ ├── queue/ -│ ├── storage/ -│ └── worker/ -├── tests/ -│ ├── integration/ -│ ├── load/ -│ └── unit/ -├── docs/ -│ ├── ARCHITECTURE.md -│ ├── COST_BUDGET.md -│ ├── DECISION_LOG.md -│ ├── EVIDENCE_INDEX.md -│ ├── KNOWN_LIMITS.md -│ ├── README.md -│ ├── STACK_OVERVIEW.md -│ ├── spec.md -│ └── evidence/ -└── reports/ -``` +Each real smoke, reliability, or full-stack proof writes three sibling files: ---- +- `.md` for review; +- `.json` using `agent-runtime-grid.run-evidence.v1`; +- `.manifest.json` with SHA-256 for the first two files. -## Runtime Contract +The JSON records command configuration, deterministic seed where applicable, +source revision/dirty state, portable environment metadata, lifecycle metrics, +and artifact hashes. Absolute local paths are replaced with portable artifact or +input references. `agent-runtime-grid verify-evidence` rejects changed, missing, +duplicated, path-traversing, symlinked, or extra sidecar files. -| Env var | Required | Used by | Purpose | -|---------|----------|---------|---------| -| `DATABASE_URL` | Yes | API, worker, tests | Postgres connection string. | -| `REDIS_URL` | Yes | API, worker, tests | Redis Streams connection string. | -| `ARTIFACT_ROOT` | Yes | Worker, API | Local artifact storage root. | -| `API_TOKEN` | No when bound to localhost, yes outside local-only operation | API | Local token auth for non-health routes. | -| `WORKER_CONCURRENCY` | No | Worker | Number of concurrent jobs per worker process. | -| `JOB_DEFAULT_TIMEOUT_SECONDS` | No | API, worker | Default timeout applied when job type does not specify one. | -| `JOB_MAX_RETRIES` | No | API, worker | Upper bound for retryable failures. | -| `LLM_MODE` | No | Job runners | `stub` by default; `live` requires budget approval. | -| `OPENAI_API_KEY` or provider-specific key | No | Optional live LLM jobs | Required only when `LLM_MODE=live` and a provider is configured. | -| `GITHUB_TOKEN` | No | Optional repo-analysis jobs | Required only for GitHub API sample jobs. | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | No | API, worker | Optional trace export target. | -| `PROMETHEUS_PORT` | No | API, worker | Metrics endpoint port. | +This is tamper-evident local evidence, not immutable storage or an attestation by +an independent party. ---- +## Destructive-operation boundary -## Continuity and Retrieval Model +Proof commands do not reset a database by default. The explicit +`--reset-local-database` option accepts only loopback/Compose hosts and the +`agent_runtime_grid` or `agent_runtime_grid_test` database names. It rejects +remote and unrelated databases even when the flag is present. -- Canonical truth: `docs/ARCHITECTURE.md`, ADRs when created, evidence artifacts, code, tests, benchmark reports, and CI results. -- Retrieval convenience: `docs/DECISION_LOG.md`, `docs/EVIDENCE_INDEX.md`, `docs/STACK_OVERVIEW.md`, `docs/KNOWN_LIMITS.md`, and benchmark reports. -- Scoped retrieval rule: consult the architecture, decision log, and evidence index before broad searching. -- Required lookup triggers: architecture changes, runtime-tier changes, cost-budget changes, external egress/tool permission changes, retry/idempotency semantics changes, open finding resolution, and benchmark baseline changes. -- Generated context packets or semantic indexes are not in use for v1; if added later, they are navigation surfaces only and must cite canonical files. +## Integrations ---- +Eval Ground Truth Lab owns evaluation data and release decisions. gdev-agent owns +application behavior and tenant isolation. Runtime Grid may execute bounded +adapters or consume their supplied artifacts, but it does not import their code +or make either project depend on this runtime. -## Non-Goals +## Non-goals -- Not a replacement for Temporal, Ray, Kubernetes, Airflow, or managed batch systems. -- Not production-ready for arbitrary untrusted code execution. -- Not exactly-once execution; v1 targets at-least-once execution with idempotent finalization. -- Not a full SaaS UI, multi-tenant billing system, or compliance product. -- Not a fully autonomous agent swarm; scheduling and safety decisions remain deterministic. -- Not a T2/T3 privileged runtime unless a future ADR changes mode, runtime tier, and evidence requirements. +- hosted control plane or production SLOs; +- exactly-once execution; +- untrusted arbitrary-code sandboxing; +- general scheduler or autonomous swarm; +- replacement for Temporal, Ray, Kubernetes, or Airflow; +- customer, tenant, or production-traffic claims. diff --git a/docs/ARCHITECTURE_DIAGRAM.md b/docs/ARCHITECTURE_DIAGRAM.md index a956291..7938949 100644 --- a/docs/ARCHITECTURE_DIAGRAM.md +++ b/docs/ARCHITECTURE_DIAGRAM.md @@ -1,102 +1,58 @@ -# Architecture Diagram - -## Runtime Path +# Architecture diagram ```text - +---------------------+ - | API / CLI | - | submit, status, | - | smoke, benchmark, | - | failure reports | - +----------+----------+ - | - v - +---------------------+ - | Postgres | - | jobs, events, | - | finalizations, cost | - +----------+----------+ - | - v - +---------------------+ - | Redis Streams | - | queue, pending, | - | consumer lag, DLQ | - +----------+----------+ - | - v - +---------------------+ - | workers | - | lease, timeout, | - | cancel, retry, | - | stale recovery | - +----------+----------+ - | - +--------------------+--------------------+ - | | | - v v v - +----------------+ +----------------+ +----------------+ - | stub jobs | | eval_lab_case | | gdev_webhook_ | - | deterministic | | Eval-Ground- | | eval | - | local runner | | Truth-Lab | | gdev-agent | - | | | integration | | integration | - +-------+--------+ +-------+--------+ +-------+--------+ - | | | - +---------------------+---------------------+ - | - v - +---------------------+ - | artifacts | - | path, SHA-256, | - | size, input digest, | - | attempt metadata | - +----------+----------+ - | - v - +---------------------+ - | reports | - | smoke, 500-job, | - | failure injection, | - | artifact proof, | - | cost rollup | - +---------------------+ + supported caller + CLI / Python library + | + v + +-------------------+ + | Postgres | + | jobs + events | + | terminal guard | + | conflict attempts | + +---------+---------+ + | + v + +-------------------+ + | Redis Streams | + | delivery + leases | + | lag + DLQ | + +---------+---------+ + | + v + +-------------------+ + | bounded workers | + | timeout + retry | + | cancellation | + +---------+---------+ + | + +-----------+-----------+ + | | | + v v v + stub jobs Eval adapter gdev adapter + | | | + +-----------+-----------+ + | + v + artifacts + SHA-256 + | + v + Markdown + JSON + manifest + | + v + strict local verifier ``` -## Evidence Flow - -```text -Eval-Ground-Truth-Lab cases - -> eval_lab_case jobs - -> Redis Streams - -> workers - -> runtime artifacts - -> Eval Lab result path cross-link - -> reliability reports - -gdev-agent webhook cases - -> gdev_webhook_eval jobs - -> Redis Streams - -> workers - -> sanitized response artifacts - -> Eval Lab-compatible result path - -> reliability reports - -Full-stack artifact proof - -> ready Eval Lab dataset/report paths - -> ready gdev-agent artifact path - -> selected gdev_webhook_eval jobs - -> runtime artifacts and report cross-links - -> optional live-local proof remains separate -``` +Authority boundaries: -## Authority Boundaries +| Concern | Authority | +|---|---| +| Lifecycle and terminal state | Postgres | +| Delivery and pending state | Redis Streams | +| Runtime policy | Deterministic Python code | +| Evaluation release decision | Eval Ground Truth Lab | +| Workload behavior/isolation | Workload repository | +| Evidence bytes | Local bundle plus SHA-256 manifest | -| Area | Authority | -|------|-----------| -| Lifecycle state | Postgres jobs and event log | -| Delivery state | Redis Streams pending and consumer-group state | -| Terminal finalization | Postgres finalization guard | -| Queue/backpressure metrics | Redis Streams plus Postgres event timing | -| Artifacts | Local artifact store with integrity metadata | -| Reports | Generated from runtime state and validated scenario evidence | -| Cost gates | Budget policy plus strict cost rollup | +The API module, remote workers, dashboards, and hosted execution are outside the +supported diagram because they are not default runnable product surfaces. diff --git a/docs/CASE_STUDY.md b/docs/CASE_STUDY.md index 3446af1..419184b 100644 --- a/docs/CASE_STUDY.md +++ b/docs/CASE_STUDY.md @@ -1,130 +1,45 @@ -# Case Study - Agent Runtime Grid +# Case study: separating delivery races from lifecycle defects ## Problem -AI and agent workloads often begin as scripts. Scripts are hard to trust once the workload becomes a batch: operators need durable state, queue-backed dispatch, retry boundaries, timeout behavior, cancellation, artifacts, cost control, and evidence that the run behaved as expected. +At-least-once queues can deliver or process the same work more than once. A +runtime must show both that it observed contention and that contention did not +produce duplicate terminal state. A single counter hardcoded to zero cannot +support either conclusion. -Agent Runtime Grid provides a local-first T1 execution runtime for running many AI, agent, and evaluation jobs with deterministic control-plane behavior. +## Implementation -## Architecture +Runtime Grid uses a unique Postgres row per job as the terminal guard. The first +transaction records the finalization and terminal event. A competing transaction +records a durable `finalization_conflict_attempts` row without updating lifecycle +state. Reports query that table and independently count duplicate terminal events. -The runtime path is: +Idempotency-key replay is a third concept: it returns an existing job before a +worker reaches finalization and is reported separately. -```text -API / CLI - -> Postgres job registry and event log - -> Redis Streams dispatch - -> bounded workers - -> job runners - -> artifacts, metrics, traces, and reports -``` - -Postgres is the lifecycle authority. Redis Streams is delivery state. Workers execute known local job types and write terminal state through an idempotent finalization guard. Artifacts are written to local storage with SHA-256, size, input digest, job ID, run ID, attempt number, and created-at evidence. - -## Reliability - -Implemented reliability mechanics: - -- idempotency-key replay and conflict detection -- Redis Streams publish, lease, acknowledge, retry, and DLQ behavior -- worker state transitions for submitted, running, completed, failed, timed out, and cancelled jobs -- stale lease recovery for workers that exit after lease and before acknowledgement -- duplicate terminal finalization prevention through a database uniqueness guard -- budget-blocked terminal events for dispatch or retry projection failures -- queue/backpressure inspection from Redis and Postgres state - -## Benchmark - -The real smoke command runs 100 jobs through local Redis Streams, workers, Postgres state, artifact generation, validation, and report rendering: - -```bash -python -m agent_runtime_grid.cli smoke \ - --jobs 100 \ - --workers 4 \ - --failure-rate 0 \ - --mode stub \ - --report reports/smoke.md -``` - -The v1 reliability proof runs 500 jobs with worker concurrency, controlled failures, timeout cases, repeated idempotency submissions, artifacts, and runtime-state report generation: +## Reproduction ```bash -python -m agent_runtime_grid.cli benchmark v1-proof \ - --jobs 500 \ - --workers 20 \ - --failure-rate 0.10 \ - --include-timeouts \ - --repeat-idempotency-submissions \ - --report reports/v1/reliability_report.md +docker-compose up -d postgres redis +PATH=.venv/bin:$PATH python -m pytest \ + tests/integration/test_idempotent_finalization.py -q ``` -Current local baseline: 80 passing tests with one upstream FastAPI/Starlette deprecation warning. +The race test requires one successful finalization, one persisted rejected +attempt, and zero duplicate terminal events. Replay and stale-recovery tests +require zero terminal defects while preserving their own event trails. -## Failure - -Failure evidence is packaged by: +Run the local reliability proof and verify its machine-readable evidence: ```bash -python -m agent_runtime_grid.cli failure-reports write-pack \ - --output-dir reports/failure-injection -``` - -The report pack covers: - -- transient retry -- timeout -- cancellation -- stale worker recovery -- duplicate finalization prevention -- DLQ routing - -Each report includes scenario, command, expected behavior, actual lifecycle, event trail, metrics, artifact evidence, and known limits. - -## Integrations - -Eval-Ground-Truth-Lab integration uses `eval_lab_case` jobs. Runtime Grid loads JSONL cases, executes deterministic local evaluation work, writes runtime artifacts, and cross-links Eval Lab result paths without importing or hardcoding a sibling checkout. - -gdev-agent integration uses `gdev_webhook_eval` jobs. Runtime Grid runs deterministic local webhook evaluation cases through the same queue and worker path, stores request hashes and sanitized responses, writes normalized fields, and cross-links Eval Lab-compatible result output. - -The current full-stack artifact proof command connects ready artifacts from both -adjacent projects: - -```bash -python -m agent_runtime_grid.cli proof full-stack \ - --eval-lab-dataset ../Eval-Ground-Truth-Lab/datasets/gdev_agent/triage_v1.jsonl \ - --eval-lab-report ../Eval-Ground-Truth-Lab/reports/gdev-agent/baseline_report.md \ - --gdev-artifact ../gdev-agent/eval/results/last_run.json \ - --jobs 20 \ - --workers 4 \ - --report reports/full-stack/runtime_report.md +PATH=.venv/bin:$PATH agent-runtime-grid benchmark v1-proof \ + --jobs 500 --workers 20 --failure-rate 0.10 \ + --include-timeouts --repeat-idempotency-submissions \ + --reset-local-database \ + --report reports/v1/reliability_report.md +PATH=.venv/bin:$PATH agent-runtime-grid verify-evidence \ + --manifest reports/v1/reliability_report.manifest.json ``` -That command validates the Eval Lab and gdev-agent evidence paths, submits -selected cases as Grid jobs, runs them through Redis Streams and workers, writes -artifacts and Eval-compatible result JSON, and produces one runtime report that -cross-links quality evidence with lifecycle, artifact, idempotency, and -queue/backpressure evidence. - -It is artifact-linked proof by default, not live HTTP end to end. The optional -`proof full-stack-live-local` mode makes workers call a locally running -`gdev-agent` `/webhook` endpoint with operator-supplied localhost config and a -webhook secret read from an environment variable. - -## Trade-offs - -This project is intentionally not a Temporal, Ray, Kubernetes, Airflow, or managed batch platform replacement. It proves local T1 reliability mechanics and evidence paths, not remote production orchestration. - -The runtime uses at-least-once delivery with idempotent finalization, not exactly-once execution. Redis Streams can redeliver; Postgres finalization decides terminal lifecycle truth. - -## Production - -Before remote or trusted production operation, the runtime would need: - -- deployment-grade auth and configuration management -- remote CI evidence for full 500-job reliability proof -- durable artifact storage beyond local filesystem -- production-grade worker process supervision and runbook automation -- explicit egress and secret allowlists per job type -- migration workflow and schema versioning -- dashboards and runbooks for on-call operation -- resource isolation for untrusted or third-party job code +The proof is deterministic, local, synthetic, and zero-model-cost. It does not +establish production throughput, availability, or external adoption. diff --git a/docs/DECISION_LOG.md b/docs/DECISION_LOG.md index 99a456f..e3b41ac 100644 --- a/docs/DECISION_LOG.md +++ b/docs/DECISION_LOG.md @@ -1,7 +1,7 @@ # Decision Log - Agent Runtime Grid Version: 1.0 -Last updated: 2026-06-11 +Last updated: 2026-07-13 Purpose: @@ -26,11 +26,14 @@ This file is not the source of truth. If an entry conflicts with a canonical doc | ID | Date | Status | Decision | Why it matters | Canonical source | Supersedes | |----|------|--------|----------|----------------|------------------|------------| | D-001 | 2026-06-11 | Active | Use Standard mode. | The project is infrastructure with queues, workers, retries, persistence, cost, and evidence needs, but v1 is local-only and not compliance-heavy or privileged. | `docs/ARCHITECTURE.md#solution-shape` | none | -| D-002 | 2026-06-11 | Active | Use T1 Docker Compose runtime. | Local containers are needed for API, worker, Postgres, Redis, Prometheus, and Grafana, while T2/T3 privileges are out of scope. | `docs/ARCHITECTURE.md#runtime-and-isolation-model` | none | +| D-002 | 2026-06-11 | Superseded | Use a six-service T1 Docker Compose runtime. | API/worker containers only imported the package and dashboards had no connected scrape path. | `docs/ARCHITECTURE.md#runtime-components` | none | | D-003 | 2026-06-11 | Active | Keep runtime control plane deterministic. | Scheduling, retries, idempotency, budget, and state transitions must be reproducible and testable. | `docs/ARCHITECTURE.md#deterministic-vs-llm-owned-subproblems` | none | | D-004 | 2026-06-11 | Active | Default benchmark mode uses stub jobs with $0 model cost. | The reliability proof must be runnable by operators without paid LLM credentials. | `docs/COST_BUDGET.md#budget-scope` | none | | D-005 | 2026-06-11 | Active | Capability profiles are OFF for v1. | RAG, Tool-Use, Agentic, Planning, and Compliance profile overhead is not justified for the deterministic control plane; future activation requires ADR and artifacts. | `docs/ARCHITECTURE.md#capability-profiles` | none | | D-006 | 2026-06-11 | Active | Use neutral operator/platform framing. | Documentation must describe the system as an operator-facing runtime, not as an external-evaluation artifact. | `docs/adr/0001-neutral-runtime-framing.md` | none | +| D-007 | 2026-07-13 | Active | Support a CLI/library runtime with Postgres and Redis as the only default Compose services. | It matches the runnable implementation and removes placeholder service claims. | `docs/ARCHITECTURE.md#supported-runtime-path` | D-002 | +| D-008 | 2026-07-13 | Active | Persist rejected finalization attempts separately from duplicate terminal-event defects. | Contention evidence and invariant violations have different meanings and must survive restart. | `docs/ARCHITECTURE.md#finalization-semantics` | none | +| D-009 | 2026-07-13 | Active | Emit portable, tamper-evident Markdown/JSON evidence bundles. | A reviewer can verify run bytes without trusting absolute local paths or prose alone. | `docs/ARCHITECTURE.md#evidence-model` | none | --- diff --git a/docs/EVIDENCE_INDEX.md b/docs/EVIDENCE_INDEX.md index 583e93a..5a5d478 100644 --- a/docs/EVIDENCE_INDEX.md +++ b/docs/EVIDENCE_INDEX.md @@ -1,7 +1,7 @@ # Evidence Index - Agent Runtime Grid Version: 1.0 -Last updated: 2026-06-14 +Last updated: 2026-07-13 Purpose: @@ -29,7 +29,7 @@ Maintain this file for: | T04 job repository | integration tests | `tests/integration/test_job_repository.py` | Postgres job creation, submitted event append, idempotency-key replay and conflict behavior | 2026-06-11 | Yes | | T05 queue adapter | integration tests | `tests/integration/test_queue_adapter.py` | Redis Streams publish, lease, acknowledge, and DLQ behavior | 2026-06-11 | Yes | | T06 worker lifecycle | integration tests | `tests/integration/test_worker_lifecycle.py` | Stub worker success, transient retry/requeue, and policy failure state transitions | 2026-06-11 | Yes | -| T07 idempotent finalization | integration tests | `tests/integration/test_idempotent_finalization.py` | Racing terminal finalization, replayed queue no-op, and duplicate-finalization metric invariant | 2026-06-11 | Yes | +| T07 idempotent finalization | integration tests | `tests/integration/test_idempotent_finalization.py` | Racing terminal finalization, durable rejected-attempt count, replayed queue no-op, and zero duplicate terminal-event invariant | 2026-07-13 | Yes | | T08 artifacts and logs | integration tests | `tests/integration/test_artifacts.py`, `tests/integration/test_logging.py` | JSON artifact write/metadata hashing and sanitized structured job log records | 2026-06-11 | Yes | | T09 timeout and cancellation | integration tests | `tests/integration/test_timeout_cancellation.py` | Timed-out jobs, queued cancellation, and running cancellation terminal paths | 2026-06-11 | Yes | | T10 observability | integration tests | `tests/integration/test_metrics.py`, `tests/integration/test_tracing.py`, `tests/integration/test_observability_safety.py` | Required metric exposure, linked trace spans, and observability secret/payload filtering | 2026-06-11 | Yes | @@ -56,6 +56,8 @@ Maintain this file for: | T31 full-stack live-local proof | integration tests, CLI command, documentation | `tests/integration/test_full_stack_proof.py`, `tests/integration/test_gdev_agent_integration.py`, `src/agent_runtime_grid/cli/proof.py`, `src/agent_runtime_grid/jobs/gdev_agent.py`, `docs/INTEGRATIONS.md` | Optional localhost gdev-agent `/webhook` execution through Grid workers with operator-supplied local config, env-only webhook secret lookup, sanitized artifacts, and no Runtime Grid live model calls | 2026-06-14 | Yes | | T32 operator-run live-local snapshot | local proof, documentation | `docs/evidence/full-stack-live-local.md`, `.gitignore` | Operator-run local gdev-agent demo-mode proof completed 20/20 queued HTTP jobs through Runtime Grid and fixed generated full-stack report ignore rules | 2026-06-15 | Yes | | Bounded routine contract | documentation/test | `docs/AUTONOMOUS_ROUTINE_CONTRACT.md`, `docs/TRIGGER_SECURITY.md`, `docs/evidence/routine-reliability-report.md`, `tests/test_routine_contract_docs.py` | Routine trigger/runtime contract, trigger security controls, and required reliability report metrics for manual/cron/webhook/event bounded routines | 2026-07-06 | Yes | +| T33 machine-readable evidence | implementation/tests | `src/agent_runtime_grid/evidence.py`, `tests/test_evidence_bundle.py` | Versioned JSON run record, portable paths, source/environment/config metadata, atomic writes, SHA-256 manifest, and modified/missing/extra/symlink/path-traversal rejection | 2026-07-13 | Yes | +| T34 database reset safety | implementation/tests | `src/agent_runtime_grid/storage/safety.py`, `tests/test_database_reset_safety.py` | Explicit reset flag and refusal of remote or unrelated database targets | 2026-07-13 | Yes | --- diff --git a/docs/KNOWN_LIMITS.md b/docs/KNOWN_LIMITS.md index f3e4031..d81d2cd 100644 --- a/docs/KNOWN_LIMITS.md +++ b/docs/KNOWN_LIMITS.md @@ -1,6 +1,6 @@ # Known Limits - Agent Runtime Grid -Last updated: 2026-06-14 +Last updated: 2026-07-13 Agent Runtime Grid is a local-first T1 runtime. The project is intentionally scoped to prove queue-backed execution, durable lifecycle state, deterministic worker behavior, artifacts, logs, metrics, cost telemetry, and reliability reports for batches of AI or agent jobs. @@ -38,13 +38,19 @@ v1 targets at-least-once delivery with idempotent finalization. deployment proof, and it expects `gdev-agent` to run in deterministic demo mode for reproducible no-model-cost evidence. - Failure-injection reports are generated from validated scenario evidence; remote chaos testing is not claimed. +- The default Compose file starts Postgres and Redis only. API, worker-service, + Prometheus, and Grafana deployment claims are intentionally absent. +- Reports have portable JSON and checksum manifests, but local files are not + immutable storage or third-party attestations. +- No external user, production workload, throughput SLO, or design-partner + outcome is claimed. ## Local Runtime Boundary The runtime is local-first and T1: - Postgres and Redis run through Docker Compose. -- Workers execute known job runners from the repository. +- Proof commands start bounded in-process workers for known job runners. - Runtime containers do not install packages or mutate toolchains at job execution time. - External LLM and GitHub API calls are disabled unless explicitly configured and approved. - Default stub mode must remain zero model cost. @@ -56,7 +62,7 @@ Before remote or trusted operation, the project would need: - stronger auth and deployment configuration - durable artifact storage beyond local filesystem - remote worker supervision and orchestration -- backpressure and queue lag dashboards +- a connected, tested metrics export and alerting deployment - explicit egress and secret allowlists per job type - production migration workflow - operational runbooks diff --git a/docs/LICENSE_REVIEW.md b/docs/LICENSE_REVIEW.md new file mode 100644 index 0000000..687921d --- /dev/null +++ b/docs/LICENSE_REVIEW.md @@ -0,0 +1,44 @@ +# License review + +Review date: 2026-07-13. Project license decision: Apache-2.0. + +The review inspected declared direct runtime dependencies, installed package +metadata in a clean project environment, and the two default Compose services. +All direct Python dependencies use permissive MIT, Apache-2.0, BSD-2-Clause, or +BSD-3-Clause terms compatible with distribution of this project's original code +under Apache-2.0. + +| Dependency | Declared range | Observed license metadata | +|---|---|---| +| Alembic | `>=1.13,<2.0` | MIT | +| asyncpg | `>=0.29,<1.0` | Apache-2.0 | +| FastAPI | `>=0.115,<1.0` | MIT | +| OpenTelemetry API/SDK | `>=1.28,<2.0` | Apache-2.0 | +| prometheus-client | `>=0.21,<1.0` | Apache-2.0 AND BSD-2-Clause | +| Pydantic / pydantic-settings | `>=2.10,<3.0`, `>=2.6,<3.0` | MIT | +| redis-py | `>=5.2,<6.0` | MIT | +| SQLAlchemy | `>=2.0,<3.0` | MIT | +| Typer | `>=0.13,<1.0` | MIT | +| Uvicorn | `>=0.32,<1.0` | BSD-3-Clause | + +Default services: + +- PostgreSQL `16.14-alpine3.24` uses the PostgreSQL License; its Docker Official + Image packaging is MIT. The Compose/CI image index digest starts `57c72fd2`. +- Redis `7.2.14-alpine3.21` is intentionally pinned to the 7.2 line and index + digest `dfa18828...`. Redis 7.2 + and earlier remain BSD-3-Clause; 7.4 moved to source-available terms, so a + broad `redis:7` tag is not acceptable for this OSS claim. + +Primary references: + +- https://www.postgresql.org/about/licence/ +- https://github.com/docker-library/postgres +- https://redis.io/legal/licenses/ +- https://hub.docker.com/_/redis + +The machine-readable review snapshot is +`docs/license-review/direct-dependencies.json`. Container base packages and +transitive Python dependencies retain their own notices. Dependency updates must +rerun this review; this document is an engineering compatibility record, not +legal advice. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 77255ad..bfb0a53 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -40,11 +40,17 @@ Worker-derived field: - `agent_runtime_grid_queue_wait_p95_seconds` - `agent_runtime_grid_queue_execution_p95_seconds` -Existing counters and histograms still cover retries, timeouts, failures, duplicate finalization, artifacts, estimated cost, and job duration. +Existing counters and histograms cover retries, timeouts, failures, terminal +invariant violations, rejected finalization attempts, artifacts, estimated cost, +and job duration. The authoritative finalization counts in run reports come from +Postgres; the in-process Prometheus objects are not a connected dashboard claim. ## Reports -Smoke and v1 reliability reports include a `queue/backpressure` section. The section is generated from the same `QueueBackpressureSnapshot` when a real queue is available. Configured-only reports fall back to zero queue values and keep the legacy reliability fields for compatibility. +Smoke and v1 reliability reports include a `queue/backpressure` section. Real +proof commands also write versioned JSON plus a SHA-256 manifest. The local +verifier detects modified, missing, or extra sidecars. These files are +tamper-evident, not immutable or independently attested. The report timing fields use these definitions: diff --git a/docs/evidence/RUNTIME_OSS_TRUTH_REPAIR_2026-07-13.md b/docs/evidence/RUNTIME_OSS_TRUTH_REPAIR_2026-07-13.md new file mode 100644 index 0000000..68b0038 --- /dev/null +++ b/docs/evidence/RUNTIME_OSS_TRUTH_REPAIR_2026-07-13.md @@ -0,0 +1,101 @@ +# Runtime Grid OSS and truth-repair verification + +Date: 2026-07-13. Scope: pre-commit verification worktree. + +## Product boundary repaired + +- Default Compose now contains only Postgres and Redis. +- Placeholder API/worker containers and unconnected Prometheus/Grafana services + were removed. +- The API module is documented as experimental library code, not a deployed + service. +- Proof commands require an explicit reset flag and refuse remote or unrelated + database targets. + +## Finalization evidence repaired + +- Rejected competing terminal writes are stored in + `finalization_conflict_attempts`. +- Duplicate terminal events are queried independently and remain an invariant + violation. +- Idempotency submission replays remain a third, separate measure. +- Race, replay, stale recovery, smoke, and reliability tests exercise the + distinction. + +## Open-source review + +- Project license: Apache-2.0. +- Direct Python dependency metadata: MIT, Apache-2.0, BSD-2-Clause, or + BSD-3-Clause; see `docs/LICENSE_REVIEW.md`. +- PostgreSQL image: `16.14-alpine3.24`, index digest + `57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777`. +- Redis image: `7.2.14-alpine3.21`, index digest + `dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8`. +- Redis 7.2 was selected because that line remains BSD-3-Clause; the previous + broad `redis:7` tag resolved to a source-available 7.4 image. + +The old Redis 7.4 RDB was incompatible with Redis 7.2. Before recreating the +test/runtime volume, a mode-600 archive was created and verified: + +```text +redis-data-before-7.2.14.tgz +sha256 da33fb988b42288d269eacce9b759d6546da0b8c6684bb580bacc7c13ee86b13 +contents: dump.rdb +``` + +No Postgres volume was removed. + +## Verification + +Pinned services: + +```text +PostgreSQL 16.14: healthy +Redis 7.2.14: healthy +redis-cli ping: PONG +``` + +Repository checks: + +```text +ruff format --check src tests: pass +ruff check src tests: pass +pytest tests -q: 97 passed, 1 upstream Starlette deprecation warning +git diff --check: pass +``` + +Wheel inspection: + +```text +agent_runtime_grid-0.1.0-py3-none-any.whl +sha256 e2742f2dbef6292227615f06a5c6d9734e12455fc345a9667561bf713974c65b +Apache LICENSE included: yes +console entry point included: yes +storage/migrations/0001_jobs.sql included: yes +``` + +Real local smoke: + +```text +run_id: 3a62f6b7-8da3-4710-bd50-6b2e76b86f13 +jobs: 20 +completed: 20 +artifact completeness: 100% +duplicate terminal events: 0 +estimated model cost: 0 +manifest verification: pass +Markdown sha256: 96031d045d9248efcf4feeb01815245d33ccd01814d8429806d7dbec92dcfbce +JSON sha256: 54d8a40bb8c0e9ce0c984753d12697c3e71349860f422875d6b89adb9ea04ac0 +manifest sha256: 5f8d4f9ffeefbdd001acfa182d2c191142bc00c6f8c44519f761bbaa7654dc58 +absolute-path/secret marker scan: no matches +``` + +The smoke bundle was generated before the atomic commit, so its JSON correctly +marks the source worktree dirty. Release evidence must be regenerated from the +tag and must report `dirty: false`. + +## Claim boundary + +This verifies a local synthetic CLI/library runtime. It does not establish a +hosted service, production throughput, an availability SLO, an external user, +or independent attestation. diff --git a/docs/license-review/direct-dependencies.json b/docs/license-review/direct-dependencies.json new file mode 100644 index 0000000..268f8c7 --- /dev/null +++ b/docs/license-review/direct-dependencies.json @@ -0,0 +1,23 @@ +{ + "review_date": "2026-07-13", + "project_license": "Apache-2.0", + "python_dependencies": { + "alembic": "MIT", + "asyncpg": "Apache-2.0", + "fastapi": "MIT", + "opentelemetry-api": "Apache-2.0", + "opentelemetry-sdk": "Apache-2.0", + "prometheus-client": "Apache-2.0 AND BSD-2-Clause", + "pydantic": "MIT", + "pydantic-settings": "MIT", + "redis": "MIT", + "sqlalchemy": "MIT", + "typer": "MIT", + "uvicorn": "BSD-3-Clause" + }, + "services": { + "postgres:16.14-alpine3.24": "PostgreSQL License; image packaging MIT", + "redis:7.2.14-alpine3.21": "BSD-3-Clause" + }, + "status": "compatible" +} diff --git a/pyproject.toml b/pyproject.toml index 17a4e84..b815203 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,15 @@ version = "0.1.0" description = "Deterministic local runtime grid for agent job execution." readme = "README.md" requires-python = ">=3.12" +license = {file = "LICENSE"} +authors = [{name = "Ashishki"}] +keywords = ["agent-runtime", "reliability", "redis-streams", "postgresql", "evaluation"] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Testing", +] dependencies = [ "alembic>=1.13,<2.0", "asyncpg>=0.29,<1.0", @@ -23,9 +32,19 @@ dependencies = [ "uvicorn[standard]>=0.32,<1.0", ] +[project.urls] +Homepage = "https://github.com/ashishki/Agent-Runtime-Grid" +Issues = "https://github.com/ashishki/Agent-Runtime-Grid/issues" + +[project.scripts] +agent-runtime-grid = "agent_runtime_grid.cli.main:app" + [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +"agent_runtime_grid.storage" = ["migrations/*.sql"] + [tool.pytest.ini_options] addopts = "-ra" pythonpath = ["src"] diff --git a/reports/README.md b/reports/README.md index 68ec6de..a9a92ad 100644 --- a/reports/README.md +++ b/reports/README.md @@ -25,6 +25,13 @@ empty directories. Stable evidence snapshots live under `docs/evidence/`. - `docs/evidence/full-stack-live-local.md` - committed snapshot for the optional live-local proof mode. - `docs/evidence/failure-injection-pack-summary.md` - committed snapshot for the failure-injection report pack. +Real smoke, reliability, and full-stack commands also write a same-stem `.json` +file and `.manifest.json` checksum manifest. Verify a bundle with: + +```bash +agent-runtime-grid verify-evidence --manifest reports/smoke.manifest.json +``` + ## Required Runtime Evidence Reliability reports should include: @@ -35,7 +42,8 @@ Reliability reports should include: - retry count - timeout count - DLQ count -- duplicate-finalization count +- finalization conflict-attempt count +- duplicate terminal-event invariant count - queue lag p95 - execution duration p95 - queue/backpressure section @@ -52,4 +60,6 @@ integrity, and known limits. Smoke and 500-job reliability proof reports are generated from actual runtime state. The committed snapshots are stable summaries with rerun commands; -the generated reports remain the current-run outputs. +the generated reports remain the current-run outputs. Manifests make local +modification detectable; they do not provide immutable storage or independent +attestation. diff --git a/src/agent_runtime_grid/artifacts/store.py b/src/agent_runtime_grid/artifacts/store.py index 05f1df6..5d9165b 100644 --- a/src/agent_runtime_grid/artifacts/store.py +++ b/src/agent_runtime_grid/artifacts/store.py @@ -8,6 +8,7 @@ from typing import Any, Self from agent_runtime_grid.domain.jobs import JobRecord, payload_sha256 +from agent_runtime_grid.evidence import portable_artifact_path @dataclass(frozen=True) @@ -111,6 +112,7 @@ def write_stub_job_artifact( _upsert_eval_result_cross_link( eval_result_path=Path(eval_result_path), artifact_path=artifact_path, + job_id=str(job.id), artifact_payload=artifact_payload or {}, ) @@ -172,6 +174,7 @@ def _upsert_eval_result_cross_link( *, eval_result_path: Path, artifact_path: Path, + job_id: str, artifact_payload: dict[str, Any], ) -> None: eval_result_path.parent.mkdir(parents=True, exist_ok=True) @@ -186,7 +189,7 @@ def _upsert_eval_result_cross_link( { "case_id": artifact_payload.get("case_id"), "quality_status": artifact_payload.get("quality_status"), - "runtime_artifact_path": str(artifact_path), + "runtime_artifact_path": portable_artifact_path(job_id=job_id, path=artifact_path), } ) eval_result_path.write_text(json.dumps(result, sort_keys=True, indent=2), encoding="utf-8") diff --git a/src/agent_runtime_grid/cli/benchmark.py b/src/agent_runtime_grid/cli/benchmark.py index 1be47b6..e4c215b 100644 --- a/src/agent_runtime_grid/cli/benchmark.py +++ b/src/agent_runtime_grid/cli/benchmark.py @@ -18,6 +18,7 @@ ArtifactStore, ) from agent_runtime_grid.domain.jobs import JobSubmission +from agent_runtime_grid.evidence import portable_path, write_evidence_bundle from agent_runtime_grid.jobs.failure_injection import ( FailureMode, FailurePlan, @@ -28,6 +29,7 @@ from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.models import metadata from agent_runtime_grid.storage.repositories import JobRepository +from agent_runtime_grid.storage.safety import UnsafeDatabaseResetError, require_safe_local_reset from agent_runtime_grid.worker.loop import Worker DEFAULT_DATABASE_URL = ( @@ -77,6 +79,7 @@ class ReliabilityReport: queue_lag_seconds: float p95_duration_seconds: float artifact_completeness: float + finalization_conflict_attempt_count: int = 0 failure_classification: dict[str, int] = field(default_factory=dict) estimated_cost_usd: Decimal = Decimal("0") run_id: str | None = None @@ -150,6 +153,7 @@ def render_reliability_report(report: ReliabilityReport) -> str: "## reliability fields", f"- completion rate: {report.completion_rate:.2%}", f"- duplicate-finalization count: {report.duplicate_finalization_count}", + f"- finalization conflict attempts: {report.finalization_conflict_attempt_count}", f"- retry count: {report.retry_count}", f"- timeout count: {report.lifecycle_counts.get('timed_out', 0)}", f"- DLQ count: {report.lifecycle_counts.get('dlq', 0)}", @@ -186,10 +190,13 @@ def render_reliability_report(report: ReliabilityReport) -> str: f"sha256={row.sha256} " f"input_digest={row.input_digest} " f"created_at={row.created_at.isoformat()} " - f"path={row.path}" + f"path=artifact://{row.job_id}/{row.path.name}" ) if row.eval_result_path is not None: - artifact_line = f"{artifact_line} eval_result_path={row.eval_result_path}" + artifact_line = ( + f"{artifact_line} eval_result_path=" + f"{portable_path(row.eval_result_path, namespace='eval-result')}" + ) lines.append(artifact_line) lines.append("") @@ -283,6 +290,7 @@ async def run_reliability_proof( lifecycle_counts=report.lifecycle_counts, completion_rate=report.completion_rate, duplicate_finalization_count=report.duplicate_finalization_count, + finalization_conflict_attempt_count=report.finalization_conflict_attempt_count, retry_count=report.retry_count, queue_lag_seconds=report.queue_lag_seconds, p95_duration_seconds=report.p95_duration_seconds, @@ -312,8 +320,21 @@ async def run_reliability_proof( repeat_idempotency_submissions=repeat_idempotency_submissions, ) - report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(render_reliability_report(report), encoding="utf-8") + write_evidence_bundle( + report_path=report_path, + rendered_report=render_reliability_report(report), + report=report, + command="benchmark v1-proof", + config={ + "jobs": jobs, + "workers": workers, + "failure_rate": failure_rate, + "include_timeouts": include_timeouts, + "repeat_idempotency_submissions": repeat_idempotency_submissions, + "artifact_root": artifact_root, + }, + seed=seed, + ) return ReliabilityProofResult( run_id=run_id, report_path=report_path, @@ -333,7 +354,7 @@ async def run_reliability_proof_from_urls( repeat_idempotency_submissions: bool = True, artifact_root: Path = Path("artifacts"), report_path: Path = Path("reports/v1/reliability_report.md"), - clean_state: bool = True, + clean_state: bool = False, ) -> ReliabilityProofResult: engine = create_async_engine(database_url) session_factory = async_sessionmaker(engine, expire_on_commit=False) @@ -348,6 +369,7 @@ async def run_reliability_proof_from_urls( ) try: if clean_state: + require_safe_local_reset(database_url) lock_connection = await engine.connect() await lock_connection.execute(text("SELECT pg_advisory_lock(7400)")) async with engine.begin() as connection: @@ -479,6 +501,10 @@ def v1_proof_command( artifact_root: Annotated[Path, typer.Option("--artifact-root")] = Path("artifacts"), database_url: Annotated[str, typer.Option("--database-url")] = DEFAULT_DATABASE_URL, redis_url: Annotated[str, typer.Option("--redis-url")] = DEFAULT_REDIS_URL, + reset_local_database: Annotated[ + bool, + typer.Option("--reset-local-database", help="Drop and recreate only the local dev DB."), + ] = False, ) -> None: try: result = asyncio.run( @@ -492,9 +518,15 @@ def v1_proof_command( repeat_idempotency_submissions=repeat_idempotency_submissions, artifact_root=artifact_root, report_path=report_path, + clean_state=reset_local_database, ) ) - except (ArtifactIntegrityError, ReliabilityProofValidationError, ValueError) as exc: + except ( + ArtifactIntegrityError, + ReliabilityProofValidationError, + UnsafeDatabaseResetError, + ValueError, + ) as exc: typer.echo(f"v1 proof failed: {exc}", err=True) raise typer.Exit(code=1) from exc diff --git a/src/agent_runtime_grid/cli/main.py b/src/agent_runtime_grid/cli/main.py index 62b8d95..d3c5edd 100644 --- a/src/agent_runtime_grid/cli/main.py +++ b/src/agent_runtime_grid/cli/main.py @@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from agent_runtime_grid.domain.jobs import JobSubmission +from agent_runtime_grid.evidence import EvidenceVerificationError, verify_evidence_manifest from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.models import job_events_table, jobs_table @@ -203,6 +204,18 @@ def cleanup_command( typer.echo(f"removed: {removed}") +@app.command("verify-evidence") +def verify_evidence_command( + manifest: Annotated[Path, typer.Option("--manifest")], +) -> None: + try: + verify_evidence_manifest(manifest) + except EvidenceVerificationError as exc: + typer.echo(f"evidence verification failed: {exc}", err=True) + raise typer.Exit(code=1) from exc + typer.echo(f"verified: {manifest}") + + from agent_runtime_grid.cli.benchmark import benchmark_app # noqa: E402 from agent_runtime_grid.cli.cost import app as cost_app # noqa: E402 from agent_runtime_grid.cli.failure_reports import app as failure_reports_app # noqa: E402 diff --git a/src/agent_runtime_grid/cli/proof.py b/src/agent_runtime_grid/cli/proof.py index 50f5eff..cdbb0a1 100644 --- a/src/agent_runtime_grid/cli/proof.py +++ b/src/agent_runtime_grid/cli/proof.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import hashlib import json from dataclasses import dataclass, replace from pathlib import Path @@ -16,10 +17,12 @@ from agent_runtime_grid.cli.benchmark import ReliabilityReport, render_reliability_report from agent_runtime_grid.cli.smoke import build_smoke_report from agent_runtime_grid.domain.jobs import JobSubmission +from agent_runtime_grid.evidence import portable_path, write_evidence_bundle from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.models import metadata from agent_runtime_grid.storage.repositories import JobRepository +from agent_runtime_grid.storage.safety import UnsafeDatabaseResetError, require_safe_local_reset from agent_runtime_grid.worker.loop import Worker DEFAULT_DATABASE_URL = ( @@ -158,8 +161,32 @@ async def run_full_stack_proof( inputs=inputs, selected_case_ids=tuple(_case_id(case) for case in cases), ) - report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(render_full_stack_report(result), encoding="utf-8") + write_evidence_bundle( + report_path=report_path, + rendered_report=render_full_stack_report(result), + report=report, + command=inputs.proof_mode, + config={ + "candidate_id": inputs.candidate_id, + "jobs": len(cases), + "workers": workers, + "artifact_root": artifact_root, + "inputs": { + "eval_lab_dataset": { + "path": portable_path(inputs.eval_lab_dataset_path, namespace="eval-dataset"), + "sha256": _file_sha256(inputs.eval_lab_dataset_path), + }, + "eval_lab_report": { + "path": portable_path(inputs.eval_lab_report_path, namespace="eval-report"), + "sha256": _file_sha256(inputs.eval_lab_report_path), + }, + "gdev_artifact": { + "path": portable_path(inputs.gdev_artifact_path, namespace="gdev-artifact"), + "sha256": _file_sha256(inputs.gdev_artifact_path), + }, + }, + }, + ) return result @@ -176,7 +203,7 @@ def validate_cross_project_inputs( raise FullStackProofError(f"Eval Lab dataset not found: {eval_lab_dataset_path}") if not eval_lab_report_path.is_file(): raise FullStackProofError(f"Eval Lab report not found: {eval_lab_report_path}") - if not gdev_artifact_path.exists(): + if not gdev_artifact_path.is_file(): raise FullStackProofError(f"gdev-agent artifact path not found: {gdev_artifact_path}") normalized_candidate_id = candidate_id.strip() if not normalized_candidate_id: @@ -214,22 +241,26 @@ def validate_full_stack_report(*, report: ReliabilityReport, expected_jobs: int) def render_full_stack_report(result: FullStackProofResult) -> str: input_paths = result.inputs + eval_dataset_path = portable_path(input_paths.eval_lab_dataset_path, namespace="eval-dataset") + eval_report_path = portable_path(input_paths.eval_lab_report_path, namespace="eval-report") + gdev_artifact_path = portable_path(input_paths.gdev_artifact_path, namespace="gdev-artifact") + runtime_report_path = portable_path(result.report_path, namespace="runtime-report") lines = [ "# Full Stack Runtime Proof", "", f"grid_run_id: {result.run_id}", f"candidate_id: {input_paths.candidate_id}", f"proof_mode: {input_paths.proof_mode}", - f"eval_lab_dataset_path: {input_paths.eval_lab_dataset_path}", - f"eval_lab_report_path: {input_paths.eval_lab_report_path}", - f"gdev_artifact_path: {input_paths.gdev_artifact_path}", + f"eval_lab_dataset_path: {eval_dataset_path}", + f"eval_lab_report_path: {eval_report_path}", + f"gdev_artifact_path: {gdev_artifact_path}", f"selected_cases: {len(result.selected_case_ids)}", "", "## Cross-Project Links", "", - f"- Eval Lab quality report: `{input_paths.eval_lab_report_path}`", - f"- gdev-agent artifact path: `{input_paths.gdev_artifact_path}`", - f"- Grid runtime report: `{result.report_path}`", + f"- Eval Lab quality report: `{eval_report_path}`", + f"- gdev-agent artifact path: `{gdev_artifact_path}`", + f"- Grid runtime report: `{runtime_report_path}`", f"- Grid run ID: `{result.run_id}`", "", "## Selected Cases", @@ -252,6 +283,10 @@ def render_full_stack_report(result: FullStackProofResult) -> str: return "\n".join(lines) +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + async def run_full_stack_from_urls( *, database_url: str = DEFAULT_DATABASE_URL, @@ -264,7 +299,7 @@ async def run_full_stack_from_urls( artifact_root: Path, report_path: Path, candidate_id: str = DEFAULT_CANDIDATE_ID, - clean_state: bool = True, + clean_state: bool = False, live_local_gdev: LiveLocalGdevConfig | None = None, ) -> FullStackProofResult: engine = create_async_engine(database_url) @@ -280,6 +315,7 @@ async def run_full_stack_from_urls( ) try: if clean_state: + require_safe_local_reset(database_url) lock_connection = await engine.connect() await lock_connection.execute(text("SELECT pg_advisory_lock(7400)")) async with engine.begin() as connection: @@ -321,6 +357,10 @@ def full_stack_command( ), database_url: Annotated[str, typer.Option("--database-url")] = DEFAULT_DATABASE_URL, redis_url: Annotated[str, typer.Option("--redis-url")] = DEFAULT_REDIS_URL, + reset_local_database: Annotated[ + bool, + typer.Option("--reset-local-database", help="Drop and recreate only the local dev DB."), + ] = False, ) -> None: try: result = asyncio.run( @@ -335,9 +375,10 @@ def full_stack_command( candidate_id=candidate_id, artifact_root=artifact_root, report_path=report_path, + clean_state=reset_local_database, ) ) - except FullStackProofError as exc: + except (FullStackProofError, UnsafeDatabaseResetError) as exc: typer.echo(f"full-stack proof failed: {exc}", err=True) raise typer.Exit(code=1) from exc @@ -363,7 +404,7 @@ async def run_full_stack_live_local_from_urls( gdev_tenant_slug: str, gdev_tenant_id: str, gdev_webhook_secret_env: str, - clean_state: bool = True, + clean_state: bool = False, ) -> FullStackProofResult: return await run_full_stack_from_urls( database_url=database_url, @@ -410,6 +451,10 @@ def full_stack_live_local_command( ), database_url: Annotated[str, typer.Option("--database-url")] = DEFAULT_DATABASE_URL, redis_url: Annotated[str, typer.Option("--redis-url")] = DEFAULT_REDIS_URL, + reset_local_database: Annotated[ + bool, + typer.Option("--reset-local-database", help="Drop and recreate only the local dev DB."), + ] = False, ) -> None: try: result = asyncio.run( @@ -428,9 +473,10 @@ def full_stack_live_local_command( gdev_tenant_slug=gdev_tenant_slug, gdev_tenant_id=gdev_tenant_id, gdev_webhook_secret_env=gdev_webhook_secret_env, + clean_state=reset_local_database, ) ) - except FullStackProofError as exc: + except (FullStackProofError, UnsafeDatabaseResetError) as exc: typer.echo(f"full-stack live-local proof failed: {exc}", err=True) raise typer.Exit(code=1) from exc diff --git a/src/agent_runtime_grid/cli/smoke.py b/src/agent_runtime_grid/cli/smoke.py index 798c6c2..23a0909 100644 --- a/src/agent_runtime_grid/cli/smoke.py +++ b/src/agent_runtime_grid/cli/smoke.py @@ -23,12 +23,17 @@ ) from agent_runtime_grid.cli.benchmark import ReliabilityReport, render_reliability_report from agent_runtime_grid.domain.jobs import JobSubmission +from agent_runtime_grid.evidence import write_evidence_bundle from agent_runtime_grid.queue.inspection import inspect_queue_backpressure from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage -from agent_runtime_grid.storage.finalization import duplicate_finalization_metric_value +from agent_runtime_grid.storage.finalization import ( + duplicate_terminal_event_count, + finalization_conflict_attempt_count, +) from agent_runtime_grid.storage.models import job_events_table, jobs_table, metadata from agent_runtime_grid.storage.repositories import JobRepository +from agent_runtime_grid.storage.safety import UnsafeDatabaseResetError, require_safe_local_reset from agent_runtime_grid.worker.loop import Worker TERMINAL_STATUSES = frozenset({"completed", "failed", "timed_out", "cancelled"}) @@ -101,8 +106,19 @@ async def run_smoke( ) validate_smoke_report(report, expected_jobs=jobs) - report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(render_reliability_report(report), encoding="utf-8") + write_evidence_bundle( + report_path=report_path, + rendered_report=render_reliability_report(report), + report=report, + command="smoke", + config={ + "jobs": jobs, + "workers": workers, + "failure_rate": failure_rate, + "mode": mode, + "artifact_root": artifact_root, + }, + ) return SmokeRunResult(run_id=run_id, report_path=report_path, report=report) @@ -148,7 +164,7 @@ async def run_smoke_from_urls( mode: str = "stub", artifact_root: Path = Path("artifacts"), report_path: Path = Path("reports/smoke.md"), - clean_state: bool = True, + clean_state: bool = False, ) -> SmokeRunResult: engine = create_async_engine(database_url) session_factory = async_sessionmaker(engine, expire_on_commit=False) @@ -163,6 +179,7 @@ async def run_smoke_from_urls( ) try: if clean_state: + require_safe_local_reset(database_url) lock_connection = await engine.connect() await lock_connection.execute(text("SELECT pg_advisory_lock(7400)")) async with engine.begin() as connection: @@ -234,6 +251,8 @@ async def build_smoke_report( .order_by(job_events_table.c.id.asc()) ) event_rows = event_result.mappings().all() + duplicate_finalizations = await duplicate_terminal_event_count(session, run_id=run_id) + conflict_attempts = await finalization_conflict_attempt_count(session, run_id=run_id) events_by_job: dict[UUID, list[dict[str, object]]] = defaultdict(list) for row in event_rows: @@ -315,7 +334,8 @@ async def build_smoke_report( submitted_jobs=submitted_jobs, lifecycle_counts=lifecycle_counts, completion_rate=completion_rate, - duplicate_finalization_count=duplicate_finalization_metric_value(), + duplicate_finalization_count=duplicate_finalizations, + finalization_conflict_attempt_count=conflict_attempts, retry_count=retry_count, queue_lag_seconds=( backpressure.p95_queue_wait_seconds if backpressure else _p95(queue_waits) @@ -327,7 +347,7 @@ async def build_smoke_report( failure_classification=dict(failure_classification), estimated_cost_usd=Decimal("0"), run_id=str(run_id), - source=f"runtime_state:{artifact_root}", + source=f"runtime_state:{artifact_root.name}", backpressure=backpressure, artifact_integrity=ArtifactIntegritySummary( checked_count=len(artifact_rows), @@ -377,6 +397,10 @@ def smoke_command( artifact_root: Annotated[Path, typer.Option("--artifact-root")] = Path("artifacts"), database_url: Annotated[str, typer.Option("--database-url")] = DEFAULT_DATABASE_URL, redis_url: Annotated[str, typer.Option("--redis-url")] = DEFAULT_REDIS_URL, + reset_local_database: Annotated[ + bool, + typer.Option("--reset-local-database", help="Drop and recreate only the local dev DB."), + ] = False, ) -> None: try: result = asyncio.run( @@ -389,9 +413,15 @@ def smoke_command( mode=mode, artifact_root=artifact_root, report_path=report_path, + clean_state=reset_local_database, ) ) - except (ArtifactIntegrityError, SmokeValidationError, ValueError) as exc: + except ( + ArtifactIntegrityError, + SmokeValidationError, + UnsafeDatabaseResetError, + ValueError, + ) as exc: typer.echo(f"smoke failed: {exc}", err=True) raise typer.Exit(code=1) from exc diff --git a/src/agent_runtime_grid/evidence.py b/src/agent_runtime_grid/evidence.py new file mode 100644 index 0000000..754eba2 --- /dev/null +++ b/src/agent_runtime_grid/evidence.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import hashlib +import json +import os +import platform +import subprocess +import sys +import tempfile +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any + +RUN_EVIDENCE_SCHEMA = "agent-runtime-grid.run-evidence.v1" +MANIFEST_SCHEMA = "agent-runtime-grid.evidence-manifest.v1" + + +class EvidenceVerificationError(RuntimeError): + pass + + +@dataclass(frozen=True) +class EvidenceBundle: + report_path: Path + data_path: Path + manifest_path: Path + + +def portable_path(value: str | Path, *, namespace: str = "artifact") -> str: + path = Path(value) + if not path.is_absolute(): + return path.as_posix() + try: + return path.resolve().relative_to(Path.cwd().resolve()).as_posix() + except ValueError: + return f"{namespace}://{path.name}" + + +def portable_artifact_path(*, job_id: str, path: str | Path) -> str: + return f"artifact://{job_id}/{Path(path).name}" + + +def write_evidence_bundle( + *, + report_path: Path, + rendered_report: str, + report: Any, + command: str, + config: dict[str, Any], + seed: int | None = None, +) -> EvidenceBundle: + if report_path.suffix != ".md": + raise ValueError("evidence report path must end in .md") + + report_path.parent.mkdir(parents=True, exist_ok=True) + data_path = report_path.with_suffix(".json") + manifest_path = report_path.with_name(f"{report_path.stem}.manifest.json") + payload = { + "schema_version": RUN_EVIDENCE_SCHEMA, + "generated_at": datetime.now(UTC).isoformat(), + "command": command, + "config": _json_safe(config), + "seed": seed, + "source_revision": _source_revision(), + "environment": { + "python": platform.python_version(), + "implementation": platform.python_implementation(), + "platform": platform.system(), + "machine": platform.machine(), + }, + "report": _report_payload(report), + } + + _atomic_write(report_path, rendered_report.encode("utf-8")) + _atomic_write( + data_path, + (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8"), + ) + manifest = { + "schema_version": MANIFEST_SCHEMA, + "files": [ + {"path": report_path.name, "sha256": _sha256(report_path)}, + {"path": data_path.name, "sha256": _sha256(data_path)}, + ], + } + _atomic_write( + manifest_path, + (json.dumps(manifest, sort_keys=True, indent=2) + "\n").encode("utf-8"), + ) + verify_evidence_manifest(manifest_path) + return EvidenceBundle( + report_path=report_path, + data_path=data_path, + manifest_path=manifest_path, + ) + + +def verify_evidence_manifest(manifest_path: Path) -> None: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise EvidenceVerificationError(f"cannot read manifest: {manifest_path}") from exc + if manifest.get("schema_version") != MANIFEST_SCHEMA: + raise EvidenceVerificationError("unsupported evidence manifest schema") + + stem_suffix = ".manifest.json" + if not manifest_path.name.endswith(stem_suffix): + raise EvidenceVerificationError("manifest name must end in .manifest.json") + stem = manifest_path.name[: -len(stem_suffix)] + expected_names = {f"{stem}.md", f"{stem}.json"} + entries = manifest.get("files") + if not isinstance(entries, list) or len(entries) != len(expected_names): + raise EvidenceVerificationError("manifest must contain the report and JSON data") + + declared_names: set[str] = set() + for entry in entries: + if not isinstance(entry, dict): + raise EvidenceVerificationError("invalid manifest file entry") + name = entry.get("path") + expected_sha256 = entry.get("sha256") + if not isinstance(name, str) or Path(name).name != name: + raise EvidenceVerificationError("manifest paths must be sibling file names") + if name in declared_names: + raise EvidenceVerificationError(f"duplicate manifest path: {name}") + declared_names.add(name) + if not isinstance(expected_sha256, str) or len(expected_sha256) != 64: + raise EvidenceVerificationError(f"invalid sha256 for {name}") + candidate = manifest_path.parent / name + if candidate.is_symlink(): + raise EvidenceVerificationError(f"evidence file must not be a symlink: {name}") + if not candidate.is_file(): + raise EvidenceVerificationError(f"missing evidence file: {name}") + if _sha256(candidate) != expected_sha256: + raise EvidenceVerificationError(f"sha256 mismatch: {name}") + + if declared_names != expected_names: + raise EvidenceVerificationError("manifest file set does not match its report stem") + allowed_sidecars = expected_names | {manifest_path.name} + actual_sidecars = { + candidate.name + for candidate in manifest_path.parent.iterdir() + if candidate.is_file() and candidate.name.startswith(f"{stem}.") + } + unexpected = actual_sidecars - allowed_sidecars + if unexpected: + raise EvidenceVerificationError( + f"unexpected evidence sidecar(s): {', '.join(sorted(unexpected))}" + ) + + +def _report_payload(report: Any) -> dict[str, Any]: + artifact_integrity = getattr(report, "artifact_integrity", None) + artifact_rows: list[dict[str, Any]] = [] + if artifact_integrity is not None: + for row in artifact_integrity.rows: + artifact_rows.append( + { + "path": portable_artifact_path(job_id=row.job_id, path=row.path), + "size_bytes": row.size_bytes, + "sha256": row.sha256, + "job_id": row.job_id, + "run_id": row.run_id, + "attempt_number": row.attempt_number, + "input_digest": row.input_digest, + "created_at": row.created_at.isoformat(), + "eval_result_path": ( + portable_path(row.eval_result_path, namespace="eval-result") + if row.eval_result_path is not None + else None + ), + } + ) + backpressure = getattr(report, "backpressure", None) + return { + "title": report.title, + "run_id": report.run_id, + "source": report.source, + "submitted_jobs": report.submitted_jobs, + "lifecycle_counts": report.lifecycle_counts, + "completion_rate": report.completion_rate, + "duplicate_terminal_event_count": report.duplicate_finalization_count, + "finalization_conflict_attempt_count": report.finalization_conflict_attempt_count, + "retry_count": report.retry_count, + "queue_lag_seconds": report.queue_lag_seconds, + "p95_duration_seconds": report.p95_duration_seconds, + "artifact_completeness": report.artifact_completeness, + "failure_classification": report.failure_classification, + "estimated_cost_usd": str(report.estimated_cost_usd), + "idempotency_replay_count": report.idempotency_replay_count, + "injected_failure_count": report.injected_failure_count, + "backpressure": asdict(backpressure) if backpressure is not None else None, + "artifact_integrity": { + "checked_count": artifact_integrity.checked_count if artifact_integrity else 0, + "valid_count": artifact_integrity.valid_count if artifact_integrity else 0, + "artifacts": artifact_rows, + }, + } + + +def _source_revision() -> dict[str, Any]: + commit = _git_output("rev-parse", "HEAD") + status = _git_output("status", "--porcelain=v1", "--untracked-files=no") + return { + "commit": commit or "unavailable", + "dirty": bool(status) if status is not None else None, + } + + +def _git_output(*arguments: str) -> str | None: + result = subprocess.run( + ["git", *arguments], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def _json_safe(value: Any) -> Any: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Path): + return portable_path(value) + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return value + + +def _atomic_write(path: Path, content: bytes) -> None: + with tempfile.NamedTemporaryFile( + dir=path.parent, prefix=f".{path.name}.", delete=False + ) as file: + temporary_path = Path(file.name) + try: + file.write(content) + file.flush() + os.fsync(file.fileno()) + except BaseException: + temporary_path.unlink(missing_ok=True) + raise + os.replace(temporary_path, path) + if sys.platform != "win32": + directory_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/src/agent_runtime_grid/observability/metrics.py b/src/agent_runtime_grid/observability/metrics.py index 19465cf..00e8353 100644 --- a/src/agent_runtime_grid/observability/metrics.py +++ b/src/agent_runtime_grid/observability/metrics.py @@ -90,7 +90,12 @@ def __init__(self) -> None: ) self.duplicate_finalization_total = Counter( "agent_runtime_grid_duplicate_finalization_total", - "Actual duplicate terminal finalization defects.", + "Actual duplicate terminal-event invariant violations.", + registry=self.registry, + ) + self.finalization_conflict_attempt_total = Counter( + "agent_runtime_grid_finalization_conflict_attempt_total", + "Terminal finalization attempts rejected by the database guard.", registry=self.registry, ) self.artifact_total = Counter( @@ -140,6 +145,7 @@ def required_metric_names() -> set[str]: "agent_runtime_grid_failure_total", "agent_runtime_grid_dlq_total", "agent_runtime_grid_duplicate_finalization_total", + "agent_runtime_grid_finalization_conflict_attempt_total", "agent_runtime_grid_artifact_total", "agent_runtime_grid_estimated_cost_usd", } diff --git a/src/agent_runtime_grid/storage/finalization.py b/src/agent_runtime_grid/storage/finalization.py index cf24b69..15d0ed9 100644 --- a/src/agent_runtime_grid/storage/finalization.py +++ b/src/agent_runtime_grid/storage/finalization.py @@ -3,24 +3,25 @@ from dataclasses import dataclass from typing import Any -from sqlalchemy import insert, update +from sqlalchemy import func, insert, select, update from sqlalchemy.dialects.postgresql import insert as postgres_insert from sqlalchemy.ext.asyncio import AsyncSession from agent_runtime_grid.domain.jobs import JobRecord from agent_runtime_grid.storage.models import ( + finalization_conflict_attempts_table, job_events_table, job_finalizations_table, jobs_table, ) TERMINAL_STATUSES = frozenset({"completed", "failed", "timed_out", "cancelled"}) -_duplicate_finalization_metric = 0 @dataclass(frozen=True) class FinalizationResult: finalized: bool + conflict_recorded: bool = False async def finalize_job( @@ -47,7 +48,16 @@ async def finalize_job( .returning(job_finalizations_table.c.job_id) ) if inserted_finalization.scalar_one_or_none() is None: - return FinalizationResult(finalized=False) + await session.execute( + insert(finalization_conflict_attempts_table).values( + job_id=job.id, + run_id=job.run_id, + attempted_status=status, + attempted_event_type=event_type, + trace_id=job.trace_id, + ) + ) + return FinalizationResult(finalized=False, conflict_recorded=True) await session.execute(update(jobs_table).where(jobs_table.c.id == job.id).values(status=status)) await session.execute( @@ -62,10 +72,28 @@ async def finalize_job( return FinalizationResult(finalized=True) -def duplicate_finalization_metric_value() -> int: - return _duplicate_finalization_metric +async def finalization_conflict_attempt_count( + session: AsyncSession, + *, + run_id: Any | None = None, +) -> int: + statement = select(func.count()).select_from(finalization_conflict_attempts_table) + if run_id is not None: + statement = statement.where(finalization_conflict_attempts_table.c.run_id == run_id) + return int(await session.scalar(statement) or 0) -def reset_duplicate_finalization_metric_for_tests() -> None: - global _duplicate_finalization_metric - _duplicate_finalization_metric = 0 +async def duplicate_terminal_event_count( + session: AsyncSession, + *, + run_id: Any | None = None, +) -> int: + statement = select(job_events_table.c.job_id).where( + job_events_table.c.event_type.in_(TERMINAL_STATUSES) + ) + if run_id is not None: + statement = statement.where(job_events_table.c.run_id == run_id) + duplicate_jobs = ( + statement.group_by(job_events_table.c.job_id).having(func.count() > 1).subquery() + ) + return int(await session.scalar(select(func.count()).select_from(duplicate_jobs)) or 0) diff --git a/src/agent_runtime_grid/storage/migrations/0001_jobs.sql b/src/agent_runtime_grid/storage/migrations/0001_jobs.sql index 26159a7..61d3602 100644 --- a/src/agent_runtime_grid/storage/migrations/0001_jobs.sql +++ b/src/agent_runtime_grid/storage/migrations/0001_jobs.sql @@ -35,6 +35,16 @@ CREATE TABLE job_finalizations ( created_at timestamptz NOT NULL DEFAULT now() ); +CREATE TABLE finalization_conflict_attempts ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + run_id uuid NOT NULL, + attempted_status varchar(32) NOT NULL, + attempted_event_type varchar(64) NOT NULL, + trace_id varchar(128) NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + CREATE TABLE artifacts ( id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, diff --git a/src/agent_runtime_grid/storage/models.py b/src/agent_runtime_grid/storage/models.py index e13baa6..152be6f 100644 --- a/src/agent_runtime_grid/storage/models.py +++ b/src/agent_runtime_grid/storage/models.py @@ -67,6 +67,18 @@ Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), ) +finalization_conflict_attempts_table = Table( + "finalization_conflict_attempts", + metadata, + Column("id", BigInteger, primary_key=True, autoincrement=True), + Column("job_id", UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="CASCADE"), nullable=False), + Column("run_id", UUID(as_uuid=True), nullable=False), + Column("attempted_status", String(32), nullable=False), + Column("attempted_event_type", String(64), nullable=False), + Column("trace_id", String(128), nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), +) + artifacts_table = Table( "artifacts", metadata, diff --git a/src/agent_runtime_grid/storage/safety.py b/src/agent_runtime_grid/storage/safety.py new file mode 100644 index 0000000..e887e43 --- /dev/null +++ b/src/agent_runtime_grid/storage/safety.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from sqlalchemy.engine import make_url + + +class UnsafeDatabaseResetError(RuntimeError): + pass + + +_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) +_LOCAL_DATABASES = frozenset({"agent_runtime_grid", "agent_runtime_grid_test"}) + + +def require_safe_local_reset(database_url: str) -> None: + url = make_url(database_url) + host = (url.host or "").lower() + database = (url.database or "").lower() + if host not in _LOCAL_HOSTS or database not in _LOCAL_DATABASES: + raise UnsafeDatabaseResetError( + "database reset is restricted to the local agent_runtime_grid project/test database" + ) diff --git a/tests/integration/test_eval_lab_integration.py b/tests/integration/test_eval_lab_integration.py index 5cb9d64..1beb2e1 100644 --- a/tests/integration/test_eval_lab_integration.py +++ b/tests/integration/test_eval_lab_integration.py @@ -183,7 +183,7 @@ async def test_eval_lab_case_runs_and_writes_cross_linked_artifact( assert artifact["runtime_attempts"] == 1 assert artifact["attempt_count"] == 1 assert artifact["latency_ms"] >= 0 - assert eval_result["runtime_artifact_path"] == str(artifact_path) + assert eval_result["runtime_artifact_path"] == f"artifact://{job.id}/{artifact_path.name}" @pytest.mark.asyncio @@ -218,9 +218,9 @@ async def test_runtime_and_eval_reports_cross_link_without_fixed_checkout( eval_result_path = tmp_path / "eval-results" / "candidate-local" / "case-1.json" eval_result = json.loads(eval_result_path.read_text(encoding="utf-8")) - assert f"path={artifact_path}" in rendered_report + assert f"path=artifact://{job.id}/{artifact_path.name}" in rendered_report assert "eval_result_path=eval-results/candidate-local/case-1.json" in rendered_report - assert eval_result["runtime_artifact_path"] == str(artifact_path) + assert eval_result["runtime_artifact_path"] == f"artifact://{job.id}/{artifact_path.name}" assert "/home/ashishki/Documents/dev/ai-stack/projects/Eval-Ground-Truth-Lab" not in ( rendered_report + json.dumps(eval_result, sort_keys=True) ) diff --git a/tests/integration/test_full_stack_proof.py b/tests/integration/test_full_stack_proof.py index 5e30ed1..39f2e3d 100644 --- a/tests/integration/test_full_stack_proof.py +++ b/tests/integration/test_full_stack_proof.py @@ -220,8 +220,8 @@ async def test_full_stack_report_cross_links_quality_and_runtime_evidence( assert "# Full Stack Runtime Proof" in rendered_report assert f"grid_run_id: {result.run_id}" in rendered_report - assert f"Eval Lab quality report: `{eval_report_path}`" in rendered_report - assert f"gdev-agent artifact path: `{gdev_artifact_path}`" in rendered_report + assert "Eval Lab quality report: `eval-report://baseline_report.md`" in rendered_report + assert "gdev-agent artifact path: `gdev-artifact://last_run.json`" in rendered_report assert "## Runtime Reliability" in rendered_report assert "## artifact integrity" in rendered_report assert "## queue/backpressure" in rendered_report @@ -229,6 +229,7 @@ async def test_full_stack_report_cross_links_quality_and_runtime_evidence( assert "gdev-billing-refund-001" in rendered_report assert "secret-value" not in rendered_report assert "api_token" not in rendered_report + assert str(tmp_path) not in rendered_report @pytest.mark.asyncio diff --git a/tests/integration/test_gdev_agent_integration.py b/tests/integration/test_gdev_agent_integration.py index 9537bed..f2e2aa2 100644 --- a/tests/integration/test_gdev_agent_integration.py +++ b/tests/integration/test_gdev_agent_integration.py @@ -278,10 +278,10 @@ async def test_gdev_runtime_and_eval_outputs_cross_link( eval_result = json.loads(eval_result_path.read_text(encoding="utf-8")) assert "gdev-case-002" in artifact_path.read_text(encoding="utf-8") - assert f"path={artifact_path}" in rendered_report + assert f"path=artifact://{job.id}/{artifact_path.name}" in rendered_report assert "eval_result_path=eval-results/gdev-agent-local/gdev-case-002.json" in rendered_report assert eval_result["case_id"] == "gdev-case-002" - assert eval_result["runtime_artifact_path"] == str(artifact_path) + assert eval_result["runtime_artifact_path"] == f"artifact://{job.id}/{artifact_path.name}" @pytest.mark.asyncio diff --git a/tests/integration/test_idempotent_finalization.py b/tests/integration/test_idempotent_finalization.py index 54f382c..ad8b09c 100644 --- a/tests/integration/test_idempotent_finalization.py +++ b/tests/integration/test_idempotent_finalization.py @@ -13,9 +13,9 @@ from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.finalization import ( - duplicate_finalization_metric_value, + duplicate_terminal_event_count, + finalization_conflict_attempt_count, finalize_job, - reset_duplicate_finalization_metric_for_tests, ) from agent_runtime_grid.storage.models import ( job_events_table, @@ -124,6 +124,16 @@ async def _finalization_count(session_factory: async_sessionmaker[AsyncSession]) return count +async def _finalization_guard_counts( + session_factory: async_sessionmaker[AsyncSession], +) -> tuple[int, int]: + async with session_factory() as session: + return ( + await finalization_conflict_attempt_count(session), + await duplicate_terminal_event_count(session), + ) + + async def _finalize_once( session_factory: async_sessionmaker[AsyncSession], job: JobRecord, @@ -151,8 +161,10 @@ async def test_racing_workers_produce_one_terminal_event( ) assert [result.finalized for result in results].count(True) == 1 + assert [result.conflict_recorded for result in results].count(True) == 1 assert await _terminal_event_count(session_factory) == 1 assert await _finalization_count(session_factory) == 1 + assert await _finalization_guard_counts(session_factory) == (1, 0) @pytest.mark.asyncio @@ -180,11 +192,10 @@ async def test_replayed_message_after_finalization_is_noop( @pytest.mark.asyncio -async def test_duplicate_finalization_metric_stays_zero( +async def test_replayed_worker_message_does_not_reach_finalization_guard( session_factory: async_sessionmaker[AsyncSession], queue: RedisStreamsQueue, ) -> None: - reset_duplicate_finalization_metric_for_tests() job = await _create_job(session_factory) message = QueueJobMessage( job_id=str(job.id), @@ -201,4 +212,4 @@ async def test_duplicate_finalization_metric_stays_zero( assert await worker.process_one() is True assert await _terminal_event_count(session_factory) == 1 - assert duplicate_finalization_metric_value() == 0 + assert await _finalization_guard_counts(session_factory) == (0, 0) diff --git a/tests/integration/test_metrics.py b/tests/integration/test_metrics.py index 3ae2683..d860762 100644 --- a/tests/integration/test_metrics.py +++ b/tests/integration/test_metrics.py @@ -11,6 +11,7 @@ def test_required_runtime_metrics_exposed() -> None: metrics.timeout_total.inc() metrics.failure_total.inc() metrics.dlq_total.inc() + metrics.finalization_conflict_attempt_total.inc() metrics.artifact_total.inc() metrics.estimated_cost_usd.inc(0) diff --git a/tests/integration/test_stale_lease_recovery.py b/tests/integration/test_stale_lease_recovery.py index 020d992..7749867 100644 --- a/tests/integration/test_stale_lease_recovery.py +++ b/tests/integration/test_stale_lease_recovery.py @@ -12,8 +12,8 @@ from agent_runtime_grid.queue.redis_streams import RedisStreamsQueue from agent_runtime_grid.queue.types import QueueJobMessage from agent_runtime_grid.storage.finalization import ( - duplicate_finalization_metric_value, - reset_duplicate_finalization_metric_for_tests, + duplicate_terminal_event_count, + finalization_conflict_attempt_count, ) from agent_runtime_grid.storage.models import ( job_events_table, @@ -158,6 +158,16 @@ async def _finalization_count(session_factory: async_sessionmaker[AsyncSession]) return count +async def _finalization_guard_counts( + session_factory: async_sessionmaker[AsyncSession], +) -> tuple[int, int]: + async with session_factory() as session: + return ( + await finalization_conflict_attempt_count(session), + await duplicate_terminal_event_count(session), + ) + + @pytest.mark.asyncio async def test_stale_lease_is_detected( session_factory: async_sessionmaker[AsyncSession], @@ -181,7 +191,6 @@ async def test_stale_job_requeues_and_completes_once( session_factory: async_sessionmaker[AsyncSession], queue: RedisStreamsQueue, ) -> None: - reset_duplicate_finalization_metric_for_tests() await _lease_and_mark_running(session_factory, queue, max_retries=1) result = await recover_stale_leases( @@ -215,7 +224,7 @@ async def test_stale_job_requeues_and_completes_once( assert events[-1][1]["attempt_number"] == 2 assert await _job_status(session_factory) == "completed" assert await _finalization_count(session_factory) == 1 - assert duplicate_finalization_metric_value() == 0 + assert await _finalization_guard_counts(session_factory) == (0, 0) @pytest.mark.asyncio @@ -224,7 +233,6 @@ async def test_exhausted_stale_recovery_routes_to_dlq_without_duplicate_finaliza session_factory: async_sessionmaker[AsyncSession], queue: RedisStreamsQueue, ) -> None: - reset_duplicate_finalization_metric_for_tests() await _lease_and_mark_running(session_factory, queue, max_retries=0) result = await recover_stale_leases( @@ -246,4 +254,4 @@ async def test_exhausted_stale_recovery_routes_to_dlq_without_duplicate_finaliza assert events[-1][1]["retryable"] is True assert await _job_status(session_factory) == "failed" assert await _finalization_count(session_factory) == 1 - assert duplicate_finalization_metric_value() == 0 + assert await _finalization_guard_counts(session_factory) == (0, 0) diff --git a/tests/test_ci_contract.py b/tests/test_ci_contract.py index 85ad568..bda49fb 100644 --- a/tests/test_ci_contract.py +++ b/tests/test_ci_contract.py @@ -28,9 +28,9 @@ def test_ci_has_required_gates() -> None: job = _test_job() steps = job["steps"] - assert any(step.get("uses") == "actions/checkout@v4" for step in steps) + assert any(step.get("uses") == "actions/checkout@v5" for step in steps) - setup_python = next(step for step in steps if step.get("uses") == "actions/setup-python@v5") + setup_python = next(step for step in steps if step.get("uses") == "actions/setup-python@v6") assert setup_python["with"]["python-version"] == "3.12" install = _step_named(steps, "Install dependencies") @@ -52,7 +52,10 @@ def test_ci_service_env_matches_runtime_contract() -> None: postgres = job["services"]["postgres"] redis = job["services"]["redis"] - assert postgres["image"].startswith("postgres:16") + assert postgres["image"] == ( + "postgres:16.14-alpine3.24@sha256:" + "57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" + ) assert "pg_isready" in postgres["options"] assert postgres["ports"] == ["5432:5432"] assert postgres["env"] == { @@ -61,7 +64,10 @@ def test_ci_service_env_matches_runtime_contract() -> None: "POSTGRES_DB": "agent_runtime_grid_test", } - assert redis["image"].startswith("redis:7") + assert redis["image"] == ( + "redis:7.2.14-alpine3.21@sha256:" + "dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8" + ) assert "redis-cli ping" in redis["options"] assert redis["ports"] == ["6379:6379"] diff --git a/tests/test_compose_contract.py b/tests/test_compose_contract.py index daaea31..3e73e53 100644 --- a/tests/test_compose_contract.py +++ b/tests/test_compose_contract.py @@ -14,14 +14,17 @@ def _load_compose() -> dict[str, Any]: def test_required_services_declared() -> None: compose = _load_compose() - assert set(compose["services"]) >= { - "api", - "worker", - "postgres", - "redis", - "prometheus", - "grafana", - } + assert set(compose["services"]) == {"postgres", "redis"} + assert compose["services"]["postgres"]["image"] == ( + "postgres:16.14-alpine3.24@sha256:" + "57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" + ) + assert compose["services"]["redis"]["image"] == ( + "redis:7.2.14-alpine3.21@sha256:" + "dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8" + ) + assert compose["services"]["postgres"]["ports"] == ["127.0.0.1:5432:5432"] + assert compose["services"]["redis"]["ports"] == ["127.0.0.1:6379:6379"] def test_compose_does_not_commit_real_secret_markers() -> None: diff --git a/tests/test_database_reset_safety.py b/tests/test_database_reset_safety.py new file mode 100644 index 0000000..ff02cfd --- /dev/null +++ b/tests/test_database_reset_safety.py @@ -0,0 +1,28 @@ +import pytest + +from agent_runtime_grid.storage.safety import UnsafeDatabaseResetError, require_safe_local_reset + + +@pytest.mark.parametrize( + "database_url", + [ + "postgresql+asyncpg://user:password@localhost:5432/agent_runtime_grid", + "postgresql+asyncpg://user:password@127.0.0.1:5432/agent_runtime_grid_test", + "postgresql+asyncpg://user:password@postgres:5432/agent_runtime_grid", + ], +) +def test_local_project_databases_are_eligible_for_explicit_reset(database_url: str) -> None: + require_safe_local_reset(database_url) + + +@pytest.mark.parametrize( + "database_url", + [ + "postgresql+asyncpg://user:password@db.example.com:5432/agent_runtime_grid", + "postgresql+asyncpg://user:password@localhost:5432/postgres", + "postgresql+asyncpg://user:password@localhost:5432/customer_data", + ], +) +def test_remote_or_unrelated_databases_are_never_reset(database_url: str) -> None: + with pytest.raises(UnsafeDatabaseResetError, match="restricted"): + require_safe_local_reset(database_url) diff --git a/tests/test_evidence_bundle.py b/tests/test_evidence_bundle.py new file mode 100644 index 0000000..b72a3b7 --- /dev/null +++ b/tests/test_evidence_bundle.py @@ -0,0 +1,93 @@ +import json +from decimal import Decimal +from pathlib import Path + +import pytest + +from agent_runtime_grid.cli.benchmark import ReliabilityReport +from agent_runtime_grid.evidence import ( + EvidenceVerificationError, + portable_path, + verify_evidence_manifest, + write_evidence_bundle, +) + + +def _report() -> ReliabilityReport: + return ReliabilityReport( + submitted_jobs=1, + lifecycle_counts={"completed": 1}, + completion_rate=1, + duplicate_finalization_count=0, + finalization_conflict_attempt_count=1, + retry_count=0, + queue_lag_seconds=0, + p95_duration_seconds=0, + artifact_completeness=1, + estimated_cost_usd=Decimal("0"), + run_id="run-1", + ) + + +def test_evidence_bundle_is_machine_readable_and_verifiable(tmp_path: Path) -> None: + bundle = write_evidence_bundle( + report_path=tmp_path / "runtime.md", + rendered_report="# Runtime\n", + report=_report(), + command="smoke", + config={"jobs": 1, "artifact_root": Path("artifacts")}, + seed=42, + ) + + verify_evidence_manifest(bundle.manifest_path) + payload = json.loads(bundle.data_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == "agent-runtime-grid.run-evidence.v1" + assert payload["report"]["finalization_conflict_attempt_count"] == 1 + assert payload["seed"] == 42 + assert not Path(payload["config"]["artifact_root"]).is_absolute() + + +@pytest.mark.parametrize("target", ["report", "data"]) +def test_verifier_rejects_modified_evidence(tmp_path: Path, target: str) -> None: + bundle = write_evidence_bundle( + report_path=tmp_path / "runtime.md", + rendered_report="# Runtime\n", + report=_report(), + command="smoke", + config={}, + ) + path = bundle.report_path if target == "report" else bundle.data_path + path.write_text("tampered", encoding="utf-8") + + with pytest.raises(EvidenceVerificationError, match="sha256 mismatch"): + verify_evidence_manifest(bundle.manifest_path) + + +def test_verifier_rejects_missing_and_extra_sidecars(tmp_path: Path) -> None: + bundle = write_evidence_bundle( + report_path=tmp_path / "runtime.md", + rendered_report="# Runtime\n", + report=_report(), + command="smoke", + config={}, + ) + bundle.data_path.unlink() + with pytest.raises(EvidenceVerificationError, match="missing evidence"): + verify_evidence_manifest(bundle.manifest_path) + + bundle = write_evidence_bundle( + report_path=tmp_path / "second.md", + rendered_report="# Runtime\n", + report=_report(), + command="smoke", + config={}, + ) + (tmp_path / "second.unexpected").write_text("extra", encoding="utf-8") + with pytest.raises(EvidenceVerificationError, match="unexpected evidence"): + verify_evidence_manifest(bundle.manifest_path) + + +def test_portable_path_redacts_absolute_paths_outside_checkout(tmp_path: Path) -> None: + assert portable_path(tmp_path / "private" / "report.json", namespace="input") == ( + "input://report.json" + )