From a3bd4b2c8f4c80745ab74998d719e23372c1932d Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 08:23:30 +0200 Subject: [PATCH 01/14] docs: specify privacy-preserving Commons sprint --- ...eserving-model-specific-shared-evidence.md | 204 ++++++++++++++++++ ...g-model-specific-shared-evidence-design.md | 101 +++++++++ 2 files changed, 305 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-20-privacy-preserving-model-specific-shared-evidence.md create mode 100644 docs/superpowers/specs/2026-08-20-privacy-preserving-model-specific-shared-evidence-design.md diff --git a/docs/superpowers/plans/2026-08-20-privacy-preserving-model-specific-shared-evidence.md b/docs/superpowers/plans/2026-08-20-privacy-preserving-model-specific-shared-evidence.md new file mode 100644 index 0000000..b481d2c --- /dev/null +++ b/docs/superpowers/plans/2026-08-20-privacy-preserving-model-specific-shared-evidence.md @@ -0,0 +1,204 @@ +# Privacy-Preserving Model-Specific Shared Evidence Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the optional, privacy-safe, model-specific Commons learning loop across Ingress, +Commons, and MARGINAL while preserving local-only Shadow Mode and local enforcement authority. + +**Architecture:** A closed-schema Worker validates and serializes aggregate GitHub updates; a +public Commons repository deterministically compiles aggregate knowledge; the zero-dependency +MARGINAL client finalizes local evidence, compiles bounded atoms, retries a durable outbox, and +loads verified model-specific priors. + +**Tech Stack:** Python 3.10+ stdlib and pytest/Ruff/mypy; TypeScript, Vitest, Wrangler 4, Cloudflare +Durable Objects; GitHub Contents API; JSON Schema 2020-12. + +**Spec:** `docs/superpowers/specs/2026-08-20-privacy-preserving-model-specific-shared-evidence-design.md` + +## Global Constraints + +- Default is `local_only`; existing installations are never opted into network behavior. +- Unknown fields and arbitrary strings are rejected recursively at the Commons boundary. +- No raw context, local pseudonym, persistent client ID, contributor identity, or exact timestamp. +- Commons never contributes to local promotion, trust, coverage, or enforcement counters. +- Network and shared-state failures fail open and cannot block Codex. +- MARGINAL core keeps zero mandatory runtime dependencies. +- Historical benchmark artifacts and claims remain unchanged. +- No production deployment claim without verified credentials and endpoint probe. + +--- + +### Task 1: Freeze the cross-repository contract + +**Files:** +- Create in all repositories: `schemas/commons-evidence-envelope-v1.json` +- Create in Commons and MARGINAL: `schemas/commons-pack-v1.json` +- Create in Commons and MARGINAL: `models/canonical-model-registry-v1.json` +- Test: schema/privacy tests in each repository + +**Interfaces:** +- Produces: exact `schema_version="1.0"`, registry-issued `model_namespace`, and `atoms` with + closed aggregate dimensions; retry identity is a separate base64url `Idempotency-Key` header. +- Consumes: reviewed aggregate-export action/reason/outcome enums. + +- [ ] Write schema tests that reject direct/nested unknown fields, canaries, URLs, paths, hashes, + arbitrary model strings, invalid enums, invalid retry headers, oversized counts, and empty atoms. +- [ ] Run those tests and record RED because the schemas/registry do not exist. +- [ ] Add recursively closed schemas and exact registry entries documented by official sources. +- [ ] Mirror schema bytes where repositories consume the same contract and test equality. +- [ ] Run focused tests and commit the contract separately in each repository. + +### Task 2: Implement Marginal-Ingress + +**Files:** +- Create: `src/index.ts`, `src/schema.ts`, `src/github-sink.ts`, `src/coordinator.ts` +- Create: `wrangler.jsonc`, `package.json`, `package-lock.json`, `tsconfig.json` +- Create: `test/*.test.ts`, `.github/workflows/ci.yml` +- Create: `README.md`, `LICENSE`, `SECURITY.md`, `CONTRIBUTING.md`, `docs/*.md` + +**Interfaces:** +- Consumes: `CommonsEvidenceEnvelopeV1` from Task 1. +- Produces: `{accepted:true, duplicate:boolean}` ACK without evidence echo; GitHub aggregate update. + +- [ ] Write Vitest RED cases for health, content type, malformed/oversized JSON, recursive unknown + fields, unsafe models/free text, no echo/logging, sink failure, and duplicate retry. +- [ ] Implement a dependency-minimal strict parser whose output type contains only allowed fields. +- [ ] Implement `GET /healthz` and `POST /v1/evidence`; never inspect `request.cf`, headers beyond + Content-Type/length, or request-specific logging. +- [ ] Implement one Durable Object coordinator with strong serialized state. Store only SHA-256 + idempotency digests with expiry and a pending target/blob descriptor; reconcile uncertain writes. +- [ ] Implement GitHub Contents reads/writes using blob SHA, bounded 409 retry, fixed repository, + fixed committer, no contributor data, and no envelope persistence. +- [ ] Set `observability.enabled=false` and `observability.logs.invocation_logs=false`; add a static + test that fails on any production logging enablement or request-specific console call. +- [ ] Add Apache-2.0 docs, threat/privacy model, synthetic request/rejection, lockfile, and least- + privilege service credential instructions. +- [ ] Run `npm ci`, format, lint, typecheck, Vitest, and `wrangler deploy --dry-run`; commit. + +### Task 3: Implement Marginal-Commons + +**Files:** +- Create: `models/registry-v1.json`, `models//aggregates.json` +- Create: `validation/schema.py`, `validation/lifecycle.py` +- Create: `tooling/build_pack.py`, `tooling/validate_commons.py` +- Create: `dist/commons-pack-v1.json`, `tests/*`, `.github/workflows/ci.yml` +- Create: `README.md`, `LICENSE`, `SECURITY.md`, `CONTRIBUTING.md`, `docs/*.md`, `pyproject.toml` + +**Interfaces:** +- Consumes: aggregate files written by Ingress. +- Produces: canonical `dist/commons-pack-v1.json` with compatibility, revision, source commit, + models, and digest. + +- [ ] Write RED tests for schema closure, registry isolation, first registered namespace, + aggregate invariants, lifecycle gates, poisoning volume, deterministic bytes, and digest checks. +- [ ] Implement strict stdlib validators and canonical JSON serialization. +- [ ] Implement `candidate → supported → validated → promoted` advancement only from checked-in + validation artifacts; counts never advance status. +- [ ] Implement deterministic pack compilation and rebuild/diff validation. +- [ ] Add docs stating observations are not users and every Commons state is only a prior. +- [ ] Run format, Ruff, mypy, pytest, validation, and deterministic rebuild; commit. + +### Task 4: Add MARGINAL Commons configuration, identity, and compiler + +**Files:** +- Create: `src/marginal/commons/{__init__,config,identity,evidence}.py` +- Create/mirror: `src/marginal/schemas/commons-*.json`, root `schemas/commons-*.json` +- Modify: `src/marginal/integrations/codex/installer.py`, `src/marginal/cli.py` +- Test: `tests/commons/test_config.py`, `test_identity.py`, `test_evidence.py` + +**Interfaces:** +- Produces: `CommonsMode`, `CommonsConfig`, `CanonicalModelIdentity | None`, immutable + `CommonsEvidenceAtom` and `compile_verified_evidence(...)`. +- Consumes: only exact registry model strings and verified local evidence records. + +- [ ] Write RED tests for Local Only default, explicit persistent choice, owner-only atomic config, + exact public registry match, unknown/private/fine-tune rejection, version isolation, conflicting + model attribution, and canary-free serialized atoms. +- [ ] Implement configuration in the existing owner-only `user-config.json` contract without + changing existing Autopilot consent. +- [ ] Implement exact, case-sensitive, registry-backed model resolution; never normalize arbitrary + model strings. +- [ ] Extend safe local action evidence with bounded `action_kind`, applied recommendation, outcome, + and resolved safe model namespace; keep all local hashes out of compiler output. +- [ ] Implement closed compiler construction from typed fields, not arbitrary caller mappings. +- [ ] Run focused tests, schema mirror tests, Ruff, and mypy; commit. + +### Task 5: Add cache, outbox, and bounded client + +**Files:** +- Create: `src/marginal/commons/{cache,outbox,client,sync}.py` +- Test: `tests/commons/test_cache.py`, `test_outbox.py`, `test_client.py`, `test_sync.py` + +**Interfaces:** +- Produces: `CommonsCache.refresh/load_prior`, `CommonsOutbox.enqueue/ack/quarantine`, + `CommonsClient.download/submit`, and fail-open lifecycle results. +- Consumes: Task 1 pack/envelope schemas and Task 4 config/atoms. + +- [ ] Write RED tests for malformed/incompatible/digest-invalid packs, previous-cache fallback, + symlink/path rejection, 0600 atomic queue files, restart retry, 2xx ACK deletion, 4xx quarantine, + 5xx/timeout retention, and zero calls for local/read-only/no-evidence modes. +- [ ] Implement no-follow owner-only safe file operations following GovernanceLedger conventions. +- [ ] Implement a stdlib HTTP client with fixed endpoint paths, no tracking parameters, bounded + connect/read timeout, bounded response size, and no request-body logging. +- [ ] Implement exact outbox ACK/quarantine/retry state transitions. +- [ ] Run focused tests, Ruff, and mypy; commit. + +### Task 6: Integrate SessionStart/SessionEnd without authority escalation + +**Files:** +- Modify: `src/marginal/integrations/codex/service.py`, `events.py`, `evidence.py`, `runtime.py` +- Modify: `src/marginal/diagnostics.py`, plugin control surface as required +- Test: `tests/integrations/codex/test_service.py`, `tests/commons/test_enforcement.py`, + `tests/test_diagnostics.py` + +**Interfaces:** +- SessionStart: retry valid outbox, refresh/cache pack, load same-model prior, continue locally. +- SessionEnd: close runtime, append authoritative end, atomically finalize memory, compile, enqueue, + bounded sync, shutdown; all shared failures return success/fail-open. + +- [ ] Write RED lifecycle tests for finalization in all modes, offline queue/retry, ambiguous model + no-upload, same-model prior visibility, model isolation, and network failure fail-open. +- [ ] Implement idempotent finalization checkpoint bound to verified local ledger root/record count. +- [ ] Add Commons orchestration after local finalization and outside promotion/Autopilot inputs. +- [ ] Prove candidate/supported/validated/promoted packs independently leave enforcement disabled + and local evidence overrides conflicting Commons priors. +- [ ] Extend privacy inspection with mode, endpoint, safe namespace, queue count, last sync, cache + revision, and schema version; never expose a remote identity. +- [ ] Run lifecycle, diagnostics, authority, promotion, and privacy suites; commit. + +### Task 7: Documentation, plugin runtime, and local dogfood installation + +**Files:** +- Modify: `README.md`, `PRIVACY.md`, `CHANGELOG.md`, `docs/operations/privacy.md`, + `docs/product/architecture.md`, `docs/integrations/codex.md`, + `docs/getting-started/quickstart.md`, plugin skill/manifest docs as needed +- Regenerate: `plugins/marginal/runtime/marginal_runtime.pyz`, `provenance.json` + +- [ ] Document Local Only default, Read-Only downloads, Contributor closed-schema upload, Cloudflare + infrastructure limitation, no persistent identity, and no enforcement authority. +- [ ] Remove only statements made inaccurate by optional Contributor mode; make no compliance, + anonymity, performance, or deployment claims. +- [ ] Rebuild runtime and verify provenance determinism. +- [ ] Reinstall/update the local plugin through the real Codex flow, preserve evidence, and verify + status/doctor/privacy plus effective Shadow Mode. +- [ ] Run documentation/publication/plugin tests and commit. + +### Task 8: End-to-end security verification and publication + +**Files:** +- Create synthetic E2E fixtures/tests in the appropriate repositories. +- Update this plan's SDD ledger and final reports. + +- [ ] Run synthetic SessionStart → SessionEnd → outbox → local Ingress → Commons aggregate → pack + → fresh SessionStart and assert same-model prior visibility. +- [ ] Assert another model sees nothing, no canary reaches envelope/Ingress/Commons, and Commons + never enables enforcement. +- [ ] Perform hostile review of all fifteen mandate questions and fix violations. +- [ ] Run all MARGINAL contributor gates, Ingress gates/dry-run, Commons gates/build, plugin + provenance, privacy/model/retry/poisoning tests, and E2E from clean trees. +- [ ] Create/publicize absent GitHub repositories, push focused commits, open non-draft PRs where + appropriate, wait for green CI, and merge only green changes already authorized by the mandate. +- [ ] Attempt production Worker deployment only with verified Wrangler auth and a dedicated + least-privilege Commons service credential; otherwise report that exact external blocker. diff --git a/docs/superpowers/specs/2026-08-20-privacy-preserving-model-specific-shared-evidence-design.md b/docs/superpowers/specs/2026-08-20-privacy-preserving-model-specific-shared-evidence-design.md new file mode 100644 index 0000000..c0b3b95 --- /dev/null +++ b/docs/superpowers/specs/2026-08-20-privacy-preserving-model-specific-shared-evidence-design.md @@ -0,0 +1,101 @@ +# Privacy-Preserving Model-Specific Shared Evidence Design + +Status: approved by the attached 2026-08-20 implementation mandate. + +## Goal + +Add an optional MARGINAL Commons learning loop without turning shared evidence into telemetry, +identity, or enforcement authority. The default remains Local Only and Shadow Mode. + +## Repositories + +- `SignalLayerLabs/Marginal-Ingress`: a minimal Cloudflare Worker privacy boundary. +- `SignalLayerLabs/Marginal-Commons`: public schemas, aggregates, validation, and deterministic pack. +- `SignalLayerLabs/Marginal`: local compiler, cache, outbox, lifecycle integration, diagnostics, and + configuration. + +## Closed wire contract + +The contribution envelope contains only: + +```text +schema_version = 1.0 +model_namespace = exact value from canonical-model-registry-v1 +atoms[] = closed aggregate dimensions and bounded observation counts +``` + +A one-time random base64url retry token is carried only in the `Idempotency-Key` HTTP header. It is +hashed immediately and never enters an envelope, response, GitHub commit, or Commons pack. + +Each atom contains the reviewed `AGGREGATE_EXPORT` dimensions with closed enums: +`record_type`, `action_kind`, `cost_bucket`, `gain_bucket`, `recommendation`, +`applied_decision`, the existing aggregate-export reason class, `outcome_class`, and `count`. There are no timestamps, +identifiers, hashes, metadata objects, free-text values, repository information, or extension +fields. Unknown fields are rejected recursively. + +## Canonical model identity + +Codex supplies an untrusted `model` string and no provider or deployment provenance. It is safe +only when an exact string appears in MARGINAL's reviewed, versioned public-model registry and the +adapter supplies the provider (`openai`). No case folding, prefix matching, alias expansion, or +user-defined entry is allowed. Unknown, custom, private, fine-tuned, conflicting, or ambiguous +model identities remain local and produce no contribution. + +The initial registry includes only exact public identifiers verified in official OpenAI +documentation on 2026-08-20. Registry changes require code review and tests. + +## Ingress + +`POST /v1/evidence` enforces content type, byte limit, strict schema, and the model registry before +any sink call. `GET /healthz` returns static health metadata and no request-derived data. Responses +never echo evidence. Production Wrangler configuration sets `observability.enabled=false` and +disables invocation logs explicitly. Source contains no request-specific logging. + +A single Durable Object serializes GitHub aggregate updates and stores only short-lived digests of +one-time idempotency keys plus a pending write descriptor. The pending descriptor contains the +target aggregate path and expected Git blob digest, not raw envelopes. After an uncertain retry, +the coordinator reconciles the expected blob against GitHub before applying another increment. +GitHub writes use the current blob SHA and bounded 409 retry. The service credential is a +least-privilege secret with Commons Contents write access; contributor credentials are never used. + +## Commons + +Commons persists aggregate knowledge, never envelopes. New observations create or update +`candidate` aggregates. Counts cannot advance lifecycle. `supported`, `validated`, and `promoted` +require checked-in validation artifacts and deterministic validation rules. Every lifecycle state +remains a prior and cannot grant local authority. + +The deterministic JSON pack sorts all namespaces and aggregates, contains schema compatibility, +Commons revision, source commit, and a SHA-256 digest over the canonical payload excluding the +digest field. Consumers reject malformed/incompatible packs and retain the previous valid cache. + +## MARGINAL client + +Persistent user configuration has three explicit modes: + +- `local_only`: zero Commons network calls; default for new and existing installations. +- `read_only`: bounded pack download only. +- `contributor`: bounded download plus automatic safe contribution. + +At `SessionStart`, enabled modes retry a valid outbox and refresh the pack cache. At `SessionEnd`, +all modes atomically finalize local memory. Contributor mode then compiles only verified, +model-attributable local records into closed atoms, writes an owner-only durable outbox envelope, +and attempts bounded synchronization. Empty or unsafe compilation performs no request. + +Network, schema, filesystem, DNS, TLS, GitHub, and Cloudflare failures never block Codex and never +change enforcement state. Malformed queued files are quarantined. ACK removes the exact queued +file; retryable failures retain it. + +## Authority boundary + +Commons priors are loaded through a separate read-only path and are not passed into coverage, +trust, promotion, Autopilot counters, Decision Ledger hashes, or enforcement eligibility. Local +observations override Commons priors. Candidate through promoted Commons evidence independently +has zero enforcement authority. + +## Acceptance + +Synthetic tests must prove SessionEnd → outbox → Ingress → Commons → next SessionStart, model +isolation, no privacy canary in serialized or persisted data, retry idempotency, offline recovery, +and zero enforcement effect. Production deployment is reported only if Wrangler authentication and +the dedicated GitHub service credential are both verified. From bd6972ef496c0495f887b94f08ce7e0f59af3b53 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 08:32:40 +0200 Subject: [PATCH 02/14] Freeze Commons evidence contract v1 --- models/canonical-model-registry-v1.json | 8 + schemas/commons-evidence-envelope-v1.json | 100 +++++++++++++ schemas/commons-pack-v1.json | 140 ++++++++++++++++++ .../schemas/commons-evidence-envelope-v1.json | 100 +++++++++++++ src/marginal/schemas/commons-pack-v1.json | 140 ++++++++++++++++++ tests/test_commons_contract.py | 140 ++++++++++++++++++ tests/test_packaged_schemas_v2.py | 2 + 7 files changed, 630 insertions(+) create mode 100644 models/canonical-model-registry-v1.json create mode 100644 schemas/commons-evidence-envelope-v1.json create mode 100644 schemas/commons-pack-v1.json create mode 100644 src/marginal/schemas/commons-evidence-envelope-v1.json create mode 100644 src/marginal/schemas/commons-pack-v1.json create mode 100644 tests/test_commons_contract.py diff --git a/models/canonical-model-registry-v1.json b/models/canonical-model-registry-v1.json new file mode 100644 index 0000000..6f2c530 --- /dev/null +++ b/models/canonical-model-registry-v1.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "models": { + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna" + } +} diff --git a/schemas/commons-evidence-envelope-v1.json b/schemas/commons-evidence-envelope-v1.json new file mode 100644 index 0000000..5349789 --- /dev/null +++ b/schemas/commons-evidence-envelope-v1.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SignalLayerLabs/Marginal-Commons/schemas/commons-evidence-envelope-v1.json", + "title": "MARGINAL Commons Evidence Envelope v1", + "type": "object", + "required": ["schema_version", "model_namespace", "atoms"], + "properties": { + "schema_version": {"const": "1.0"}, + "model_namespace": { + "enum": [ + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna" + ] + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/atom"} + } + }, + "additionalProperties": false, + "unevaluatedProperties": false, + "$defs": { + "atom": { + "type": "object", + "required": [ + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size" + ], + "properties": { + "record_type": {"enum": ["decision", "outcome"]}, + "action_kind": { + "enum": [ + "command", + "file_read", + "file_write", + "generation", + "llm", + "model_call", + "reasoning", + "research", + "review", + "search", + "subagent", + "test", + "tool", + "verification", + "other" + ] + }, + "cost_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "gain_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "recommendation": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "applied_decision": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "reason_code": { + "enum": [ + "APPROVED", + "BUDGET_REJECTED", + "DENY", + "DUPLICATE_ACTION", + "DUPLICATE_PENDING", + "EXPECTED_GAIN_REJECTED", + "FUNDED", + "MARGINAL_ROI_REJECTED", + "OTHER", + "PARENT_BUDGET_REJECTED", + "RECOMMEND_OVERRIDE", + "SHADOW_OVERRIDE", + "TARGET_REACHED", + "UNSPECIFIED", + "not_applicable" + ] + }, + "outcome_class": { + "enum": [ + "verified_success", + "verified_failure", + "positive_reward", + "non_positive_reward", + "unknown", + "not_applicable" + ] + }, + "count": {"type": "integer", "minimum": 1, "maximum": 1000}, + "minimum_group_size": {"type": "integer", "minimum": 1, "maximum": 1000} + }, + "additionalProperties": false, + "unevaluatedProperties": false + } + } +} diff --git a/schemas/commons-pack-v1.json b/schemas/commons-pack-v1.json new file mode 100644 index 0000000..3a5e0ef --- /dev/null +++ b/schemas/commons-pack-v1.json @@ -0,0 +1,140 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SignalLayerLabs/Marginal-Commons/schemas/commons-pack-v1.json", + "title": "MARGINAL Commons Pack v1", + "type": "object", + "required": [ + "schema_version", + "source_commit", + "commons_revision", + "compatibility", + "models", + "integrity" + ], + "properties": { + "schema_version": {"const": "1.0"}, + "source_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "commons_revision": {"type": "integer", "minimum": 1}, + "compatibility": { + "type": "object", + "required": ["evidence_envelope_schema_version"], + "properties": {"evidence_envelope_schema_version": {"const": "1.0"}}, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "models": { + "type": "object", + "required": [ + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna" + ], + "properties": { + "openai/gpt-5.6-sol": {"$ref": "#/$defs/model"}, + "openai/gpt-5.6-terra": {"$ref": "#/$defs/model"}, + "openai/gpt-5.6-luna": {"$ref": "#/$defs/model"} + }, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "integrity": { + "type": "object", + "required": ["sha256"], + "properties": {"sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}}, + "additionalProperties": false, + "unevaluatedProperties": false + } + }, + "additionalProperties": false, + "unevaluatedProperties": false, + "$defs": { + "model": { + "type": "object", + "required": ["aggregates"], + "properties": { + "aggregates": { + "type": "array", + "items": {"$ref": "#/$defs/aggregate"} + } + }, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "aggregate": { + "type": "object", + "required": [ + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size", + "lifecycle" + ], + "properties": { + "record_type": {"enum": ["decision", "outcome"]}, + "action_kind": { + "enum": [ + "command", + "file_read", + "file_write", + "generation", + "llm", + "model_call", + "reasoning", + "research", + "review", + "search", + "subagent", + "test", + "tool", + "verification", + "other" + ] + }, + "cost_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "gain_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "recommendation": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "applied_decision": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "reason_code": { + "enum": [ + "APPROVED", + "BUDGET_REJECTED", + "DENY", + "DUPLICATE_ACTION", + "DUPLICATE_PENDING", + "EXPECTED_GAIN_REJECTED", + "FUNDED", + "MARGINAL_ROI_REJECTED", + "OTHER", + "PARENT_BUDGET_REJECTED", + "RECOMMEND_OVERRIDE", + "SHADOW_OVERRIDE", + "TARGET_REACHED", + "UNSPECIFIED", + "not_applicable" + ] + }, + "outcome_class": { + "enum": [ + "verified_success", + "verified_failure", + "positive_reward", + "non_positive_reward", + "unknown", + "not_applicable" + ] + }, + "count": {"type": "integer", "minimum": 1, "maximum": 1000}, + "minimum_group_size": {"type": "integer", "minimum": 1, "maximum": 1000}, + "lifecycle": {"enum": ["candidate", "supported", "validated", "promoted"]} + }, + "additionalProperties": false, + "unevaluatedProperties": false + } + } +} diff --git a/src/marginal/schemas/commons-evidence-envelope-v1.json b/src/marginal/schemas/commons-evidence-envelope-v1.json new file mode 100644 index 0000000..5349789 --- /dev/null +++ b/src/marginal/schemas/commons-evidence-envelope-v1.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SignalLayerLabs/Marginal-Commons/schemas/commons-evidence-envelope-v1.json", + "title": "MARGINAL Commons Evidence Envelope v1", + "type": "object", + "required": ["schema_version", "model_namespace", "atoms"], + "properties": { + "schema_version": {"const": "1.0"}, + "model_namespace": { + "enum": [ + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna" + ] + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/atom"} + } + }, + "additionalProperties": false, + "unevaluatedProperties": false, + "$defs": { + "atom": { + "type": "object", + "required": [ + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size" + ], + "properties": { + "record_type": {"enum": ["decision", "outcome"]}, + "action_kind": { + "enum": [ + "command", + "file_read", + "file_write", + "generation", + "llm", + "model_call", + "reasoning", + "research", + "review", + "search", + "subagent", + "test", + "tool", + "verification", + "other" + ] + }, + "cost_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "gain_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "recommendation": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "applied_decision": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "reason_code": { + "enum": [ + "APPROVED", + "BUDGET_REJECTED", + "DENY", + "DUPLICATE_ACTION", + "DUPLICATE_PENDING", + "EXPECTED_GAIN_REJECTED", + "FUNDED", + "MARGINAL_ROI_REJECTED", + "OTHER", + "PARENT_BUDGET_REJECTED", + "RECOMMEND_OVERRIDE", + "SHADOW_OVERRIDE", + "TARGET_REACHED", + "UNSPECIFIED", + "not_applicable" + ] + }, + "outcome_class": { + "enum": [ + "verified_success", + "verified_failure", + "positive_reward", + "non_positive_reward", + "unknown", + "not_applicable" + ] + }, + "count": {"type": "integer", "minimum": 1, "maximum": 1000}, + "minimum_group_size": {"type": "integer", "minimum": 1, "maximum": 1000} + }, + "additionalProperties": false, + "unevaluatedProperties": false + } + } +} diff --git a/src/marginal/schemas/commons-pack-v1.json b/src/marginal/schemas/commons-pack-v1.json new file mode 100644 index 0000000..3a5e0ef --- /dev/null +++ b/src/marginal/schemas/commons-pack-v1.json @@ -0,0 +1,140 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SignalLayerLabs/Marginal-Commons/schemas/commons-pack-v1.json", + "title": "MARGINAL Commons Pack v1", + "type": "object", + "required": [ + "schema_version", + "source_commit", + "commons_revision", + "compatibility", + "models", + "integrity" + ], + "properties": { + "schema_version": {"const": "1.0"}, + "source_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "commons_revision": {"type": "integer", "minimum": 1}, + "compatibility": { + "type": "object", + "required": ["evidence_envelope_schema_version"], + "properties": {"evidence_envelope_schema_version": {"const": "1.0"}}, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "models": { + "type": "object", + "required": [ + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna" + ], + "properties": { + "openai/gpt-5.6-sol": {"$ref": "#/$defs/model"}, + "openai/gpt-5.6-terra": {"$ref": "#/$defs/model"}, + "openai/gpt-5.6-luna": {"$ref": "#/$defs/model"} + }, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "integrity": { + "type": "object", + "required": ["sha256"], + "properties": {"sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}}, + "additionalProperties": false, + "unevaluatedProperties": false + } + }, + "additionalProperties": false, + "unevaluatedProperties": false, + "$defs": { + "model": { + "type": "object", + "required": ["aggregates"], + "properties": { + "aggregates": { + "type": "array", + "items": {"$ref": "#/$defs/aggregate"} + } + }, + "additionalProperties": false, + "unevaluatedProperties": false + }, + "aggregate": { + "type": "object", + "required": [ + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size", + "lifecycle" + ], + "properties": { + "record_type": {"enum": ["decision", "outcome"]}, + "action_kind": { + "enum": [ + "command", + "file_read", + "file_write", + "generation", + "llm", + "model_call", + "reasoning", + "research", + "review", + "search", + "subagent", + "test", + "tool", + "verification", + "other" + ] + }, + "cost_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "gain_bucket": {"enum": ["low", "medium", "high", "unknown"]}, + "recommendation": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "applied_decision": {"enum": ["allow", "deny", "unknown", "not_applicable"]}, + "reason_code": { + "enum": [ + "APPROVED", + "BUDGET_REJECTED", + "DENY", + "DUPLICATE_ACTION", + "DUPLICATE_PENDING", + "EXPECTED_GAIN_REJECTED", + "FUNDED", + "MARGINAL_ROI_REJECTED", + "OTHER", + "PARENT_BUDGET_REJECTED", + "RECOMMEND_OVERRIDE", + "SHADOW_OVERRIDE", + "TARGET_REACHED", + "UNSPECIFIED", + "not_applicable" + ] + }, + "outcome_class": { + "enum": [ + "verified_success", + "verified_failure", + "positive_reward", + "non_positive_reward", + "unknown", + "not_applicable" + ] + }, + "count": {"type": "integer", "minimum": 1, "maximum": 1000}, + "minimum_group_size": {"type": "integer", "minimum": 1, "maximum": 1000}, + "lifecycle": {"enum": ["candidate", "supported", "validated", "promoted"]} + }, + "additionalProperties": false, + "unevaluatedProperties": false + } + } +} diff --git a/tests/test_commons_contract.py b/tests/test_commons_contract.py new file mode 100644 index 0000000..72df35e --- /dev/null +++ b/tests/test_commons_contract.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import copy +import json +import re +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + + +ROOT = Path(__file__).resolve().parents[1] +NAMESPACES = ( + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna", +) +IDEMPOTENCY_KEY_PATTERN = re.compile(r"^[A-Za-z0-9_-]{32,64}$") + + +def _schema(name: str) -> dict[str, object]: + return json.loads((ROOT / "schemas" / name).read_text(encoding="utf-8")) + + +def _valid_atom() -> dict[str, object]: + return { + "record_type": "decision", + "action_kind": "tool", + "cost_bucket": "low", + "gain_bucket": "medium", + "recommendation": "allow", + "applied_decision": "allow", + "reason_code": "APPROVED", + "outcome_class": "not_applicable", + "count": 1, + "minimum_group_size": 1, + } + + +def _valid_envelope() -> dict[str, object]: + return { + "schema_version": "1.0", + "model_namespace": NAMESPACES[0], + "atoms": [_valid_atom()], + } + + +def _assert_invalid(schema_name: str, payload: object) -> None: + with pytest.raises(Exception): + Draft202012Validator(_schema(schema_name)).validate(payload) + + +def test_envelope_accepts_only_closed_aggregate_atoms() -> None: + Draft202012Validator(_schema("commons-evidence-envelope-v1.json")).validate(_valid_envelope()) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda value: value.update({"idempotency_key": "a" * 32}), + lambda value: value.update({"privacy_canary": "customer-acme"}), + lambda value: value.update({"url": "https://example.invalid/private"}), + lambda value: value.update({"path": "/private/customer/acme"}), + lambda value: value.update({"sha256": "a" * 64}), + lambda value: value["atoms"][0].update({"metadata": {"canary": "customer-acme"}}), + lambda value: value["atoms"][0].update({"url": "https://example.invalid/private"}), + lambda value: value["atoms"][0].update({"path": "/private/customer/acme"}), + lambda value: value["atoms"][0].update({"sha256": "a" * 64}), + lambda value: value.update({"model_namespace": "openai/gpt-5.6-sol-custom"}), + lambda value: value.update({"model_namespace": "https://example.invalid/model"}), + lambda value: value.update({"atoms": []}), + lambda value: value["atoms"][0].update({"action_kind": "customer-acme"}), + lambda value: value["atoms"][0].update({"reason_code": "https://example.invalid/reason"}), + lambda value: value["atoms"][0].update({"count": 1001}), + lambda value: value["atoms"][0].update({"minimum_group_size": 1001}), + ], +) +def test_envelope_rejects_unsafe_or_out_of_contract_values(mutate: object) -> None: + payload = _valid_envelope() + mutate(payload) # type: ignore[operator] + _assert_invalid("commons-evidence-envelope-v1.json", payload) + + +@pytest.mark.parametrize("key", ["a" * 31, "a" * 65, "a" * 32 + "+", "a" * 31 + "="]) +def test_idempotency_key_header_is_base64url_and_bounded(key: str) -> None: + assert IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None + + +def test_idempotency_key_is_not_an_envelope_property() -> None: + _assert_invalid( + "commons-evidence-envelope-v1.json", + {**_valid_envelope(), "Idempotency-Key": "a" * 32}, + ) + + +def test_marginal_root_and_packaged_contract_mirrors_are_byte_identical() -> None: + for name in ( + "commons-evidence-envelope-v1.json", + "commons-pack-v1.json", + ): + assert (ROOT / "schemas" / name).read_bytes() == ( + ROOT / "src" / "marginal" / "schemas" / name + ).read_bytes() + + +def test_registry_contains_only_the_reviewed_exact_model_mapping() -> None: + registry = json.loads((ROOT / "models" / "canonical-model-registry-v1.json").read_text()) + assert registry == { + "schema_version": "1.0", + "models": { + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna", + }, + } + + +def test_pack_rejects_noncanonical_or_open_content() -> None: + pack = { + "schema_version": "1.0", + "source_commit": "a" * 40, + "commons_revision": 1, + "compatibility": {"evidence_envelope_schema_version": "1.0"}, + "models": { + namespace: {"aggregates": [{**_valid_atom(), "lifecycle": "candidate"}]} + for namespace in NAMESPACES + }, + "integrity": {"sha256": "b" * 64}, + } + Draft202012Validator(_schema("commons-pack-v1.json")).validate(pack) + for mutation in ( + lambda value: value.update({"metadata": "customer-acme"}), + lambda value: value.update({"source_commit": "A" * 40}), + lambda value: value["compatibility"].update({"url": "https://example.invalid"}), + lambda value: value["models"].update({"custom/model": {"aggregates": []}}), + lambda value: value["integrity"].update({"sha256": "b" * 63}), + ): + candidate = copy.deepcopy(pack) + mutation(candidate) + _assert_invalid("commons-pack-v1.json", candidate) diff --git a/tests/test_packaged_schemas_v2.py b/tests/test_packaged_schemas_v2.py index b218e87..db47563 100644 --- a/tests/test_packaged_schemas_v2.py +++ b/tests/test_packaged_schemas_v2.py @@ -11,6 +11,8 @@ "agent-capabilities-v1.json", "agent-decision-v1.json", "agent-event-v1.json", + "commons-evidence-envelope-v1.json", + "commons-pack-v1.json", "decision-ledger-v2.json", "decision-receipt-v1.json", "governance-ledger-v3.json", From 70782ad988e68d9534b3bdeb7c3d7aec7459f363 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 08:41:31 +0200 Subject: [PATCH 03/14] Harden Commons contract conformance --- schemas/commons-contract-v1.manifest.json | 8 +++ schemas/commons-evidence-envelope-v1.json | 1 + schemas/commons-evidence-envelope-v1.sha256 | 1 + schemas/commons-pack-v1.json | 1 + .../schemas/commons-evidence-envelope-v1.json | 1 + src/marginal/schemas/commons-pack-v1.json | 1 + tests/test_commons_contract.py | 54 +++++++++++++++---- 7 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 schemas/commons-contract-v1.manifest.json create mode 100644 schemas/commons-evidence-envelope-v1.sha256 diff --git a/schemas/commons-contract-v1.manifest.json b/schemas/commons-contract-v1.manifest.json new file mode 100644 index 0000000..9da445e --- /dev/null +++ b/schemas/commons-contract-v1.manifest.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "sha256": { + "canonical-model-registry-v1.json": "142a8bc5645141f9c467279d6935663b5b11af03092ce4bd493edd42f3278ea6", + "commons-evidence-envelope-v1.json": "7a7601748e94107e5a2ccbf54a376a1d074de48d2f7ddd85dfd2b6b335091277", + "commons-pack-v1.json": "2db170ba822cf2dd2052eddd4e3ac84b5a998922201267f0e9fca9bcb06d8305" + } +} diff --git a/schemas/commons-evidence-envelope-v1.json b/schemas/commons-evidence-envelope-v1.json index 5349789..5711b66 100644 --- a/schemas/commons-evidence-envelope-v1.json +++ b/schemas/commons-evidence-envelope-v1.json @@ -54,6 +54,7 @@ "test", "tool", "verification", + "unknown", "other" ] }, diff --git a/schemas/commons-evidence-envelope-v1.sha256 b/schemas/commons-evidence-envelope-v1.sha256 new file mode 100644 index 0000000..cfc8703 --- /dev/null +++ b/schemas/commons-evidence-envelope-v1.sha256 @@ -0,0 +1 @@ +7a7601748e94107e5a2ccbf54a376a1d074de48d2f7ddd85dfd2b6b335091277 diff --git a/schemas/commons-pack-v1.json b/schemas/commons-pack-v1.json index 3a5e0ef..49a7394 100644 --- a/schemas/commons-pack-v1.json +++ b/schemas/commons-pack-v1.json @@ -93,6 +93,7 @@ "test", "tool", "verification", + "unknown", "other" ] }, diff --git a/src/marginal/schemas/commons-evidence-envelope-v1.json b/src/marginal/schemas/commons-evidence-envelope-v1.json index 5349789..5711b66 100644 --- a/src/marginal/schemas/commons-evidence-envelope-v1.json +++ b/src/marginal/schemas/commons-evidence-envelope-v1.json @@ -54,6 +54,7 @@ "test", "tool", "verification", + "unknown", "other" ] }, diff --git a/src/marginal/schemas/commons-pack-v1.json b/src/marginal/schemas/commons-pack-v1.json index 3a5e0ef..49a7394 100644 --- a/src/marginal/schemas/commons-pack-v1.json +++ b/src/marginal/schemas/commons-pack-v1.json @@ -93,6 +93,7 @@ "test", "tool", "verification", + "unknown", "other" ] }, diff --git a/tests/test_commons_contract.py b/tests/test_commons_contract.py index 72df35e..c609f5a 100644 --- a/tests/test_commons_contract.py +++ b/tests/test_commons_contract.py @@ -1,12 +1,15 @@ from __future__ import annotations import copy +import hashlib import json import re from pathlib import Path import pytest -from jsonschema import Draft202012Validator +from jsonschema import Draft202012Validator, ValidationError + +from marginal.privacy import aggregate_ledger_records ROOT = Path(__file__).resolve().parents[1] @@ -45,13 +48,29 @@ def _valid_envelope() -> dict[str, object]: } -def _assert_invalid(schema_name: str, payload: object) -> None: - with pytest.raises(Exception): - Draft202012Validator(_schema(schema_name)).validate(payload) +def _validator(schema_name: str) -> Draft202012Validator: + return Draft202012Validator(_schema(schema_name)) + + +def _assert_invalid(validator: Draft202012Validator, payload: object) -> None: + with pytest.raises(ValidationError): + validator.validate(payload) def test_envelope_accepts_only_closed_aggregate_atoms() -> None: - Draft202012Validator(_schema("commons-evidence-envelope-v1.json")).validate(_valid_envelope()) + _validator("commons-evidence-envelope-v1.json").validate(_valid_envelope()) + + +def test_envelope_accepts_unknown_action_kind_from_real_aggregate_outcome() -> None: + rows = aggregate_ledger_records( + [{"event": "outcome", "outcome": {"resolved": True, "reward": 1.0}}], + minimum_group_size=1, + ) + atom = {key: value for key, value in rows[0].items() if key not in {"schema_version", "privacy_profile"}} + assert atom["action_kind"] == "unknown" + _validator("commons-evidence-envelope-v1.json").validate( + {"schema_version": "1.0", "model_namespace": NAMESPACES[0], "atoms": [atom]} + ) @pytest.mark.parametrize( @@ -78,7 +97,7 @@ def test_envelope_accepts_only_closed_aggregate_atoms() -> None: def test_envelope_rejects_unsafe_or_out_of_contract_values(mutate: object) -> None: payload = _valid_envelope() mutate(payload) # type: ignore[operator] - _assert_invalid("commons-evidence-envelope-v1.json", payload) + _assert_invalid(_validator("commons-evidence-envelope-v1.json"), payload) @pytest.mark.parametrize("key", ["a" * 31, "a" * 65, "a" * 32 + "+", "a" * 31 + "="]) @@ -86,9 +105,14 @@ def test_idempotency_key_header_is_base64url_and_bounded(key: str) -> None: assert IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None +@pytest.mark.parametrize("key", ["a" * 32, "_" * 64]) +def test_idempotency_key_header_accepts_base64url_boundary_lengths(key: str) -> None: + assert IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is not None + + def test_idempotency_key_is_not_an_envelope_property() -> None: _assert_invalid( - "commons-evidence-envelope-v1.json", + _validator("commons-evidence-envelope-v1.json"), {**_valid_envelope(), "Idempotency-Key": "a" * 32}, ) @@ -115,6 +139,17 @@ def test_registry_contains_only_the_reviewed_exact_model_mapping() -> None: } +def test_contract_digest_fixtures_detect_schema_and_registry_drift() -> None: + envelope_digest = (ROOT / "schemas" / "commons-evidence-envelope-v1.sha256").read_text().strip() + assert hashlib.sha256( + (ROOT / "schemas" / "commons-evidence-envelope-v1.json").read_bytes() + ).hexdigest() == envelope_digest + manifest = json.loads((ROOT / "schemas" / "commons-contract-v1.manifest.json").read_text()) + for name, expected in manifest["sha256"].items(): + base = ROOT / ("models" if name.startswith("canonical-model") else "schemas") + assert hashlib.sha256((base / name).read_bytes()).hexdigest() == expected + + def test_pack_rejects_noncanonical_or_open_content() -> None: pack = { "schema_version": "1.0", @@ -127,7 +162,8 @@ def test_pack_rejects_noncanonical_or_open_content() -> None: }, "integrity": {"sha256": "b" * 64}, } - Draft202012Validator(_schema("commons-pack-v1.json")).validate(pack) + validator = _validator("commons-pack-v1.json") + validator.validate(pack) for mutation in ( lambda value: value.update({"metadata": "customer-acme"}), lambda value: value.update({"source_commit": "A" * 40}), @@ -137,4 +173,4 @@ def test_pack_rejects_noncanonical_or_open_content() -> None: ): candidate = copy.deepcopy(pack) mutation(candidate) - _assert_invalid("commons-pack-v1.json", candidate) + _assert_invalid(validator, candidate) From 6d371d224ef0aa01b7f7a4c379bd6ba89d676f84 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 11:38:31 +0200 Subject: [PATCH 04/14] feat: add privacy-safe Commons compiler core --- pyproject.toml | 2 +- src/marginal/cli.py | 6 + src/marginal/commons/__init__.py | 21 ++ .../commons/canonical-model-registry-v1.json | 8 + src/marginal/commons/config.py | 175 ++++++++++++ src/marginal/commons/evidence.py | 265 ++++++++++++++++++ src/marginal/commons/identity.py | 93 ++++++ src/marginal/integrations/codex/evidence.py | 46 +++ src/marginal/integrations/codex/installer.py | 51 ++-- tests/commons/test_config.py | 119 ++++++++ tests/commons/test_evidence.py | 206 ++++++++++++++ tests/commons/test_identity.py | 74 +++++ tests/integrations/codex/test_installer.py | 23 ++ tests/test_cli.py | 30 ++ tests/test_commons_contract.py | 16 +- 15 files changed, 1102 insertions(+), 33 deletions(-) create mode 100644 src/marginal/commons/__init__.py create mode 100644 src/marginal/commons/canonical-model-registry-v1.json create mode 100644 src/marginal/commons/config.py create mode 100644 src/marginal/commons/evidence.py create mode 100644 src/marginal/commons/identity.py create mode 100644 tests/commons/test_config.py create mode 100644 tests/commons/test_evidence.py create mode 100644 tests/commons/test_identity.py diff --git a/pyproject.toml b/pyproject.toml index 17e9257..80813c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ include-package-data = true where = ["src"] [tool.setuptools.package-data] -marginal = ["py.typed", "schemas/*.json"] +marginal = ["py.typed", "commons/*.json", "schemas/*.json"] [tool.pytest.ini_options] addopts = "-ra" diff --git a/src/marginal/cli.py b/src/marginal/cli.py index 3802051..62ea691 100644 --- a/src/marginal/cli.py +++ b/src/marginal/cli.py @@ -159,6 +159,11 @@ def _build_parser() -> argparse.ArgumentParser: install_parser.add_argument("--ref", default="main") install_parser.add_argument("--data-dir", type=Path) install_parser.add_argument("--autopilot-consent", action="store_true") + install_parser.add_argument( + "--commons-mode", + choices=["local_only", "read_only", "contributor"], + help="persist one explicit Commons network posture (default: local_only)", + ) install_parser.add_argument("--json", action="store_true", dest="as_json") uninstall_parser = subparsers.add_parser("uninstall", help="remove a native integration") @@ -208,6 +213,7 @@ def main(argv: Sequence[str] | None = None) -> int: ref=args.ref, data_dir=args.data_dir, autopilot_consent=args.autopilot_consent, + commons_mode=args.commons_mode, ) payload = result.to_dict() if args.as_json: diff --git a/src/marginal/commons/__init__.py b/src/marginal/commons/__init__.py new file mode 100644 index 0000000..918cd3a --- /dev/null +++ b/src/marginal/commons/__init__.py @@ -0,0 +1,21 @@ +"""Privacy-preserving, model-specific MARGINAL Commons primitives.""" + +from .config import CommonsConfig, CommonsMode, configure_commons_mode, load_commons_config +from .evidence import CommonsEvidenceAtom, compile_verified_evidence +from .identity import ( + CanonicalModelIdentity, + resolve_canonical_model, + resolve_model_attribution, +) + +__all__ = [ + "CanonicalModelIdentity", + "CommonsConfig", + "CommonsEvidenceAtom", + "CommonsMode", + "compile_verified_evidence", + "configure_commons_mode", + "load_commons_config", + "resolve_canonical_model", + "resolve_model_attribution", +] diff --git a/src/marginal/commons/canonical-model-registry-v1.json b/src/marginal/commons/canonical-model-registry-v1.json new file mode 100644 index 0000000..6f2c530 --- /dev/null +++ b/src/marginal/commons/canonical-model-registry-v1.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "models": { + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna" + } +} diff --git a/src/marginal/commons/config.py b/src/marginal/commons/config.py new file mode 100644 index 0000000..a9dd6d4 --- /dev/null +++ b/src/marginal/commons/config.py @@ -0,0 +1,175 @@ +"""Owner-controlled configuration for the optional MARGINAL Commons client.""" + +from __future__ import annotations + +import errno +import json +import os +import stat +import tempfile +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any + +_CONFIG_NAME = "user-config.json" +_MAX_CONFIG_BYTES = 65_536 + + +class CommonsMode(str, Enum): + """Explicit network posture for MARGINAL Commons.""" + + LOCAL_ONLY = "local_only" + READ_ONLY = "read_only" + CONTRIBUTOR = "contributor" + + @classmethod + def parse(cls, value: CommonsMode | str) -> CommonsMode: + if isinstance(value, cls): + return value + if not isinstance(value, str): + raise TypeError("Commons mode must be a string or CommonsMode") + try: + return cls(value) + except ValueError as exc: + raise ValueError(f"unknown Commons mode: {value}") from exc + + +@dataclass(frozen=True, slots=True) +class CommonsConfig: + """Validated local Commons configuration.""" + + mode: CommonsMode = CommonsMode.LOCAL_ONLY + + def __post_init__(self) -> None: + object.__setattr__(self, "mode", CommonsMode.parse(self.mode)) + + +def _absolute_path(path: str | Path) -> Path: + supplied = Path(path) + return Path(os.path.abspath(os.fspath(supplied))) + + +def _reject_symlink_components(path: Path) -> None: + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + try: + metadata = current.lstat() + except FileNotFoundError: + continue + if stat.S_ISLNK(metadata.st_mode): + raise ValueError("user configuration path must not contain a symbolic link") + + +def _config_path(data_dir: str | Path) -> Path: + root = _absolute_path(data_dir) + _reject_symlink_components(root) + return root / _CONFIG_NAME + + +def _read_user_config(data_dir: str | Path) -> dict[str, Any] | None: + path = _config_path(data_dir) + if path.is_symlink(): + raise ValueError("user configuration path must not be a symbolic link") + flags = os.O_RDONLY + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except FileNotFoundError: + return None + except OSError as exc: + if exc.errno == errno.ELOOP: + raise ValueError("user configuration path must not be a symbolic link") from exc + raise + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError("user configuration must be a regular file") + if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & 0o077: + raise ValueError("user configuration must have owner-only permissions") + raw = os.read(descriptor, _MAX_CONFIG_BYTES + 1) + finally: + os.close(descriptor) + if len(raw) > _MAX_CONFIG_BYTES: + raise ValueError("user configuration is too large") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("user configuration must be valid JSON") from exc + if ( + not isinstance(payload, dict) + or isinstance(payload.get("schema_version"), bool) + or payload.get("schema_version") != 1 + ): + raise ValueError("user configuration must be a schema version 1 object") + return payload + + +def _write_user_config(data_dir: str | Path, payload: dict[str, Any]) -> None: + path = _config_path(data_dir) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + _reject_symlink_components(path.parent) + if os.name == "posix": + path.parent.chmod(0o700) + encoded = ( + json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + ).encode("utf-8") + descriptor, temporary_name = tempfile.mkstemp( + prefix=".user-config-", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + os.write(descriptor, encoded) + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + if path.is_symlink(): + raise ValueError("user configuration path must not be a symbolic link") + os.replace(temporary, path) + if os.name == "posix": + path.chmod(0o600) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, Any]: + """Atomically merge reviewed choices into the one user configuration file.""" + + payload = _read_user_config(data_dir) + if payload is None: + payload = {"schema_version": 1} + payload.update(changes) + _write_user_config(data_dir, payload) + return payload + + +def load_commons_config(data_dir: str | Path) -> CommonsConfig: + """Load the explicit Commons choice, defaulting absence to Local Only.""" + + payload = _read_user_config(data_dir) + if payload is None or "commons_mode" not in payload: + return CommonsConfig() + try: + return CommonsConfig(mode=CommonsMode.parse(payload["commons_mode"])) + except (TypeError, ValueError): + return CommonsConfig() + + +def configure_commons_mode(data_dir: str | Path, *, mode: CommonsMode | str) -> CommonsConfig: + """Persist one explicit user-level Commons mode without changing Autopilot consent.""" + + selected = CommonsMode.parse(mode) + _update_user_config(data_dir, commons_mode=selected.value) + return CommonsConfig(mode=selected) diff --git a/src/marginal/commons/evidence.py b/src/marginal/commons/evidence.py new file mode 100644 index 0000000..ff14c82 --- /dev/null +++ b/src/marginal/commons/evidence.py @@ -0,0 +1,265 @@ +"""Closed, typed compilation of verified local evidence into Commons atoms.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from enum import Enum +from typing import TypeVar + +from marginal.integrations.codex.evidence import EvidenceStore + +from .identity import CanonicalModelIdentity, identity_is_canonical + + +class RecordType(str, Enum): + DECISION = "decision" + OUTCOME = "outcome" + + +class ActionKind(str, Enum): + COMMAND = "command" + FILE_READ = "file_read" + FILE_WRITE = "file_write" + GENERATION = "generation" + LLM = "llm" + MODEL_CALL = "model_call" + REASONING = "reasoning" + RESEARCH = "research" + REVIEW = "review" + SEARCH = "search" + SUBAGENT = "subagent" + TEST = "test" + TOOL = "tool" + VERIFICATION = "verification" + UNKNOWN = "unknown" + OTHER = "other" + + +class ValueBucket(str, Enum): + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + UNKNOWN = "unknown" + + +class DecisionClass(str, Enum): + ALLOW = "allow" + DENY = "deny" + UNKNOWN = "unknown" + NOT_APPLICABLE = "not_applicable" + + +class AggregateReasonCode(str, Enum): + APPROVED = "APPROVED" + BUDGET_REJECTED = "BUDGET_REJECTED" + DENY = "DENY" + DUPLICATE_ACTION = "DUPLICATE_ACTION" + DUPLICATE_PENDING = "DUPLICATE_PENDING" + EXPECTED_GAIN_REJECTED = "EXPECTED_GAIN_REJECTED" + FUNDED = "FUNDED" + MARGINAL_ROI_REJECTED = "MARGINAL_ROI_REJECTED" + OTHER = "OTHER" + PARENT_BUDGET_REJECTED = "PARENT_BUDGET_REJECTED" + RECOMMEND_OVERRIDE = "RECOMMEND_OVERRIDE" + SHADOW_OVERRIDE = "SHADOW_OVERRIDE" + TARGET_REACHED = "TARGET_REACHED" + UNSPECIFIED = "UNSPECIFIED" + NOT_APPLICABLE = "not_applicable" + + +class OutcomeClass(str, Enum): + VERIFIED_SUCCESS = "verified_success" + VERIFIED_FAILURE = "verified_failure" + POSITIVE_REWARD = "positive_reward" + NON_POSITIVE_REWARD = "non_positive_reward" + UNKNOWN = "unknown" + NOT_APPLICABLE = "not_applicable" + + +@dataclass(frozen=True, slots=True) +class CommonsEvidenceAtom: + """One immutable atom containing only frozen aggregate dimensions and bounded counts.""" + + record_type: RecordType + action_kind: ActionKind + cost_bucket: ValueBucket + gain_bucket: ValueBucket + recommendation: DecisionClass + applied_decision: DecisionClass + reason_code: AggregateReasonCode + outcome_class: OutcomeClass + count: int + minimum_group_size: int + + def __post_init__(self) -> None: + if not isinstance(self.record_type, RecordType): + raise TypeError("record_type must be a typed Commons enum") + if not isinstance(self.action_kind, ActionKind): + raise TypeError("action_kind must be a typed Commons enum") + if not isinstance(self.cost_bucket, ValueBucket): + raise TypeError("cost_bucket must be a typed Commons enum") + if not isinstance(self.gain_bucket, ValueBucket): + raise TypeError("gain_bucket must be a typed Commons enum") + if not isinstance(self.recommendation, DecisionClass): + raise TypeError("recommendation must be a typed Commons enum") + if not isinstance(self.applied_decision, DecisionClass): + raise TypeError("applied_decision must be a typed Commons enum") + if not isinstance(self.reason_code, AggregateReasonCode): + raise TypeError("reason_code must be a typed Commons enum") + if not isinstance(self.outcome_class, OutcomeClass): + raise TypeError("outcome_class must be a typed Commons enum") + for name, integer_value in ( + ("count", self.count), + ("minimum_group_size", self.minimum_group_size), + ): + if isinstance(integer_value, bool) or not isinstance(integer_value, int): + raise TypeError(f"{name} must be an integer") + if not 1 <= integer_value <= 1_000: + raise ValueError(f"{name} must be between 1 and 1000") + + def to_dict(self) -> dict[str, str | int]: + return { + "record_type": self.record_type.value, + "action_kind": self.action_kind.value, + "cost_bucket": self.cost_bucket.value, + "gain_bucket": self.gain_bucket.value, + "recommendation": self.recommendation.value, + "applied_decision": self.applied_decision.value, + "reason_code": self.reason_code.value, + "outcome_class": self.outcome_class.value, + "count": self.count, + "minimum_group_size": self.minimum_group_size, + } + + +_EnumT = TypeVar("_EnumT", bound=Enum) +_DimensionKey = tuple[ + RecordType, + ActionKind, + ValueBucket, + ValueBucket, + DecisionClass, + DecisionClass, + AggregateReasonCode, + OutcomeClass, +] + + +def _exact_enum(enum_type: type[_EnumT], value: object) -> _EnumT | None: + if not isinstance(value, str): + return None + try: + return enum_type(value) + except ValueError: + return None + + +def _decision_key(record: dict[str, object]) -> _DimensionKey | None: + action_kind = _exact_enum(ActionKind, record.get("action_kind")) + cost_bucket = _exact_enum(ValueBucket, record.get("cost_bucket")) + gain_bucket = _exact_enum(ValueBucket, record.get("gain_bucket")) + recommendation = _exact_enum(DecisionClass, record.get("recommendation")) + applied = _exact_enum(DecisionClass, record.get("applied_decision")) + if not isinstance(action_kind, ActionKind): + return None + if not isinstance(cost_bucket, ValueBucket) or not isinstance(gain_bucket, ValueBucket): + return None + if not isinstance(recommendation, DecisionClass) or not isinstance(applied, DecisionClass): + return None + reason = _exact_enum(AggregateReasonCode, record.get("reason_code")) + if reason is None: + reason = AggregateReasonCode.OTHER + return ( + RecordType.DECISION, + action_kind, + cost_bucket, + gain_bucket, + recommendation, + applied, + reason, + OutcomeClass.NOT_APPLICABLE, + ) + + +def _outcome_key(record: dict[str, object]) -> _DimensionKey | None: + outcomes = { + "success": OutcomeClass.VERIFIED_SUCCESS, + "failure": OutcomeClass.VERIFIED_FAILURE, + "unknown": OutcomeClass.UNKNOWN, + } + value = record.get("outcome") + if not isinstance(value, str) or value not in outcomes: + return None + return ( + RecordType.OUTCOME, + ActionKind.UNKNOWN, + ValueBucket.UNKNOWN, + ValueBucket.UNKNOWN, + DecisionClass.NOT_APPLICABLE, + DecisionClass.NOT_APPLICABLE, + AggregateReasonCode.NOT_APPLICABLE, + outcomes[value], + ) + + +def compile_verified_evidence( + evidence_store: EvidenceStore, + *, + model_identity: CanonicalModelIdentity | None, + minimum_group_size: int = 5, +) -> tuple[CommonsEvidenceAtom, ...]: + """Compile a verified local chain; arbitrary caller rows are never accepted.""" + + if isinstance(minimum_group_size, bool) or not isinstance(minimum_group_size, int): + raise TypeError("minimum_group_size must be an integer") + if not 1 <= minimum_group_size <= 1_000: + raise ValueError("minimum_group_size must be between 1 and 1000") + if not isinstance(evidence_store, EvidenceStore): + raise TypeError("evidence_store must be an EvidenceStore") + if model_identity is None or not identity_is_canonical(model_identity): + return () + try: + records, verification = evidence_store.verified_records() + except (OSError, ValueError): + return () + if not verification.valid: + return () + attributed_namespaces = { + namespace + for record in records + if isinstance((namespace := record.get("model_namespace")), str) + } + if attributed_namespaces != {model_identity.namespace}: + return () + + counter: Counter[_DimensionKey] = Counter() + for record in records: + if record.get("model_namespace") != model_identity.namespace: + continue + event = record.get("event") + key = _decision_key(record) if event == "decision" else None + if event == "outcome": + key = _outcome_key(record) + if key is not None: + counter[key] += 1 + + atoms: list[CommonsEvidenceAtom] = [] + for key, count in sorted(counter.items(), key=lambda item: tuple(v.value for v in item[0])): + if count < minimum_group_size: + continue + atoms.append( + CommonsEvidenceAtom( + record_type=key[0], + action_kind=key[1], + cost_bucket=key[2], + gain_bucket=key[3], + recommendation=key[4], + applied_decision=key[5], + reason_code=key[6], + outcome_class=key[7], + count=min(count, 1_000), + minimum_group_size=minimum_group_size, + ) + ) + return tuple(atoms) diff --git a/src/marginal/commons/identity.py b/src/marginal/commons/identity.py new file mode 100644 index 0000000..30f9cb4 --- /dev/null +++ b/src/marginal/commons/identity.py @@ -0,0 +1,93 @@ +"""Exact public-model identity resolution for MARGINAL Commons.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from dataclasses import dataclass +from importlib.resources import files +from typing import Any + +_REVIEWED_MODELS = { + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna", +} + + +@dataclass(frozen=True, slots=True) +class CanonicalModelIdentity: + """One registry-issued public model identity.""" + + provider: str + model: str + namespace: str + registry_version: str + + +def _registry() -> tuple[str, dict[str, str]]: + resource = files("marginal.commons").joinpath("canonical-model-registry-v1.json") + payload: Any = json.loads(resource.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or payload.get("schema_version") != "1.0": + raise RuntimeError("canonical model registry is invalid") + models = payload.get("models") + if not isinstance(models, dict) or not all( + isinstance(model, str) and isinstance(namespace, str) for model, namespace in models.items() + ): + raise RuntimeError("canonical model registry is invalid") + parsed = dict(models) + if parsed != _REVIEWED_MODELS: + raise RuntimeError("canonical model registry contains an unreviewed model") + return "1.0", parsed + + +def resolve_canonical_model(*, provider: str, model: str) -> CanonicalModelIdentity | None: + """Resolve only an exact, case-sensitive reviewed registry entry.""" + + if not isinstance(provider, str) or not isinstance(model, str) or provider != "openai": + return None + version, models = _registry() + namespace = models.get(model) + if namespace is None: + return None + return CanonicalModelIdentity(provider, model, namespace, version) + + +def resolve_model_attribution( + observations: Iterable[tuple[str, str]], +) -> CanonicalModelIdentity | None: + """Resolve a batch only when every observation has one identical safe identity.""" + + if isinstance(observations, (str, bytes)): + return None + resolved: set[CanonicalModelIdentity] = set() + seen = False + try: + for provider, model in observations: + seen = True + identity = resolve_canonical_model(provider=provider, model=model) + if identity is None: + return None + resolved.add(identity) + except (TypeError, ValueError): + return None + if not seen or len(resolved) != 1: + return None + return next(iter(resolved)) + + +def is_canonical_namespace(namespace: object) -> bool: + """Return whether a value is one exact registry-issued namespace.""" + + if not isinstance(namespace, str): + return False + _, models = _registry() + return namespace in models.values() + + +def identity_is_canonical(identity: object) -> bool: + """Reject forged value objects that were not derived from the exact registry mapping.""" + + if not isinstance(identity, CanonicalModelIdentity): + return False + return resolve_canonical_model(provider=identity.provider, model=identity.model) == identity diff --git a/src/marginal/integrations/codex/evidence.py b/src/marginal/integrations/codex/evidence.py index 6f3400c..7a1ae11 100644 --- a/src/marginal/integrations/codex/evidence.py +++ b/src/marginal/integrations/codex/evidence.py @@ -32,6 +32,12 @@ "integration_failure", "pending", "timestamp", + "model_namespace", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", } _FORBIDDEN_FIELDS = { "auth", @@ -45,6 +51,45 @@ "transcript", } +_COMMONS_FIELD_VALUES = { + "action_kind": { + "command", + "file_read", + "file_write", + "generation", + "llm", + "model_call", + "reasoning", + "research", + "review", + "search", + "subagent", + "test", + "tool", + "verification", + "unknown", + "other", + }, + "cost_bucket": {"low", "medium", "high", "unknown"}, + "gain_bucket": {"low", "medium", "high", "unknown"}, + "recommendation": {"allow", "deny", "unknown", "not_applicable"}, + "applied_decision": {"allow", "deny", "unknown", "not_applicable"}, + "outcome": {"success", "failure", "unknown"}, +} + + +def _validate_commons_fields(record: Mapping[str, Any]) -> None: + for field, allowed in _COMMONS_FIELD_VALUES.items(): + if field in record: + value = record[field] + if not isinstance(value, str) or value not in allowed: + raise ValueError(f"invalid Commons evidence {field}") + if "model_namespace" in record: + from marginal.commons.identity import is_canonical_namespace + + if not is_canonical_namespace(record["model_namespace"]): + raise ValueError("invalid Commons evidence model_namespace") + def _canonical_bytes(payload: Mapping[str, Any]) -> bytes: try: @@ -87,6 +132,7 @@ def append(self, record: Mapping[str, Any]) -> None: unsupported = fields - _ALLOWED_EVIDENCE_FIELDS if unsupported: raise ValueError(f"unsupported evidence field: {sorted(unsupported)[0]}") + _validate_commons_fields(record) serialized = _canonical_bytes(record) if len(serialized) > self.max_record_bytes: raise ValueError("evidence record is too large") diff --git a/src/marginal/integrations/codex/installer.py b/src/marginal/integrations/codex/installer.py index b11b411..7a68b1e 100644 --- a/src/marginal/integrations/codex/installer.py +++ b/src/marginal/integrations/codex/installer.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os import re import shutil @@ -11,6 +10,8 @@ from pathlib import Path from typing import Protocol +from marginal.commons.config import CommonsMode, _update_user_config, configure_commons_mode + @dataclass(frozen=True, slots=True) class CommandResult: @@ -83,6 +84,7 @@ class CodexInstallation: error_code: str = "" message: str = "" autopilot_consent: bool = False + commons_mode: str = CommonsMode.LOCAL_ONLY.value def to_dict(self) -> dict[str, object]: return { @@ -92,6 +94,7 @@ def to_dict(self) -> dict[str, object]: "error_code": self.error_code, "message": self.message, "autopilot_consent": self.autopilot_consent, + "commons_mode": self.commons_mode, } @@ -133,9 +136,13 @@ def install( ref: str = "main", data_dir: str | Path | None = None, autopilot_consent: bool = False, + commons_mode: CommonsMode | str | None = None, ) -> CodexInstallation: if not isinstance(autopilot_consent, bool): raise TypeError("autopilot_consent must be a bool") + selected_commons_mode = ( + CommonsMode.LOCAL_ONLY if commons_mode is None else CommonsMode.parse(commons_mode) + ) selected = runner or SubprocessRunner() report = inspect_codex(runner=selected) if report.capability_level != "tool_enforcement": @@ -181,11 +188,22 @@ def install( message="installed in Shadow Mode; Autopilot consent was not persisted", ) configure_autopilot_consent(data_dir, granted=True) + if commons_mode is not None: + if data_dir is None: + return CodexInstallation( + True, + True, + error_code="COMMONS_MODE_DATA_DIR_REQUIRED", + message="installed in Shadow Mode; Commons choice was not persisted", + commons_mode=CommonsMode.LOCAL_ONLY.value, + ) + configure_commons_mode(data_dir, mode=selected_commons_mode) return CodexInstallation( True, True, message="installed in Shadow Mode", autopilot_consent=autopilot_consent, + commons_mode=selected_commons_mode.value, ) @@ -202,10 +220,6 @@ def uninstall(*, runner: CommandRunner | None = None) -> CodexInstallation: return CodexInstallation(False, True, message="plugin removed; local evidence preserved") -def _user_config_path(data_dir: str | Path) -> Path: - return Path(data_dir).resolve() / "user-config.json" - - def configure_autopilot_consent(data_dir: str | Path, *, granted: bool) -> None: """Persist an explicit user-level Autopilot choice outside every repository. @@ -215,33 +229,16 @@ def configure_autopilot_consent(data_dir: str | Path, *, granted: bool) -> None: if not isinstance(granted, bool): raise TypeError("granted must be a bool") - path = _user_config_path(data_dir) - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - path.parent.chmod(0o700) - temporary = path.with_suffix(".tmp") - descriptor = os.open(temporary, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) - try: - os.write( - descriptor, - ( - json.dumps({"schema_version": 1, "autopilot_consent": granted}, sort_keys=True) - + "\n" - ).encode(), - ) - os.fsync(descriptor) - finally: - os.close(descriptor) - os.replace(temporary, path) - if os.name == "posix": - path.chmod(0o600) + _update_user_config(data_dir, autopilot_consent=granted) def autopilot_consent_configured(data_dir: str | Path) -> bool: """Return only a valid, explicit consent bit from the user-owned config.""" try: - value = json.loads(_user_config_path(data_dir).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): + from marginal.commons.config import _read_user_config + + value = _read_user_config(data_dir) + except (OSError, ValueError): return False return bool(isinstance(value, dict) and value.get("autopilot_consent") is True) diff --git a/tests/commons/test_config.py b/tests/commons/test_config.py new file mode 100644 index 0000000..ef4697b --- /dev/null +++ b/tests/commons/test_config.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import json +import os +import stat +from pathlib import Path + +import pytest + +from marginal.commons.config import ( + CommonsMode, + configure_commons_mode, + load_commons_config, +) +from marginal.integrations.codex.installer import ( + autopilot_consent_configured, + configure_autopilot_consent, +) + + +def test_missing_config_defaults_to_local_only_without_creating_a_file(tmp_path: Path) -> None: + assert load_commons_config(tmp_path).mode is CommonsMode.LOCAL_ONLY + assert not (tmp_path / "user-config.json").exists() + + +@pytest.mark.parametrize("mode", list(CommonsMode)) +def test_explicit_mode_choice_persists_in_owner_only_user_config( + tmp_path: Path, mode: CommonsMode +) -> None: + configured = configure_commons_mode(tmp_path, mode=mode) + + path = tmp_path / "user-config.json" + assert configured.mode is mode + assert load_commons_config(tmp_path).mode is mode + assert json.loads(path.read_text(encoding="utf-8")) == { + "commons_mode": mode.value, + "schema_version": 1, + } + if os.name == "posix": + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(tmp_path.stat().st_mode) == 0o700 + + +def test_commons_and_autopilot_updates_preserve_each_others_explicit_choice(tmp_path: Path) -> None: + configure_autopilot_consent(tmp_path, granted=True) + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + assert autopilot_consent_configured(tmp_path) is True + + configure_autopilot_consent(tmp_path, granted=False) + payload = json.loads((tmp_path / "user-config.json").read_text(encoding="utf-8")) + assert payload == { + "autopilot_consent": False, + "commons_mode": "contributor", + "schema_version": 1, + } + assert load_commons_config(tmp_path).mode is CommonsMode.CONTRIBUTOR + + +def test_config_rejects_symlink_targets_and_unsafe_existing_permissions(tmp_path: Path) -> None: + target = tmp_path / "outside.json" + target.write_text('{"schema_version":1,"commons_mode":"read_only"}\n', encoding="utf-8") + target.chmod(0o600) + link_root = tmp_path / "linked" + link_root.mkdir() + try: + (link_root / "user-config.json").symlink_to(target) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available") + + with pytest.raises(ValueError, match="symbolic link"): + load_commons_config(link_root) + with pytest.raises(ValueError, match="symbolic link"): + configure_commons_mode(link_root, mode=CommonsMode.CONTRIBUTOR) + assert "contributor" not in target.read_text(encoding="utf-8") + + unsafe = tmp_path / "unsafe" + unsafe.mkdir() + path = unsafe / "user-config.json" + path.write_text('{"schema_version":1,"commons_mode":"read_only"}\n', encoding="utf-8") + if os.name == "posix": + path.chmod(0o644) + with pytest.raises(ValueError, match="owner-only"): + load_commons_config(unsafe) + + +def test_failed_atomic_replace_preserves_existing_choices( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + configure_autopilot_consent(tmp_path, granted=True) + path = tmp_path / "user-config.json" + before = path.read_bytes() + + def fail_replace(source: Path, destination: Path) -> None: + del source, destination + raise OSError("synthetic replace failure") + + monkeypatch.setattr(os, "replace", fail_replace) + with pytest.raises(OSError, match="synthetic"): + configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) + + assert path.read_bytes() == before + assert not list(tmp_path.glob(".user-config-*.tmp")) + + +def test_config_rejects_boolean_schema_versions_instead_of_treating_them_as_one( + tmp_path: Path, +) -> None: + path = tmp_path / "user-config.json" + path.write_text('{"commons_mode":"contributor","schema_version":true}\n', encoding="utf-8") + path.chmod(0o600) + + with pytest.raises(ValueError, match="schema version 1"): + load_commons_config(tmp_path) + + +@pytest.mark.parametrize("value", ["LOCAL_ONLY", " contributor", "read-only", "custom"]) +def test_mode_parser_does_not_normalize_or_create_namespaces(value: str) -> None: + with pytest.raises(ValueError, match="Commons mode"): + CommonsMode.parse(value) diff --git a/tests/commons/test_evidence.py b/tests/commons/test_evidence.py new file mode 100644 index 0000000..6ce2d22 --- /dev/null +++ b/tests/commons/test_evidence.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest + +from marginal.commons.evidence import CommonsEvidenceAtom, compile_verified_evidence +from marginal.commons.identity import resolve_canonical_model +from marginal.governance_ledger import GovernanceLedger +from marginal.integrations.codex.evidence import EvidenceStore + +CANARY = "privacy-canary-customer-acme-repository-secret" + + +def _identity(model: str = "gpt-5.6-sol"): + identity = resolve_canonical_model(provider="openai", model=model) + assert identity is not None + return identity + + +def _decision(namespace: str) -> dict[str, object]: + return { + "schema_version": 1, + "event": "decision", + "session_hash": CANARY, + "action_hash": f"{CANARY}-action", + "semantic_key": f"{CANARY}-semantic", + "state_hash": f"{CANARY}-state", + "evidence_hash": f"{CANARY}-evidence", + "model_namespace": namespace, + "action_kind": "tool", + "cost_bucket": "low", + "gain_bucket": "medium", + "recommendation": "allow", + "applied_decision": "allow", + "reason_code": "APPROVED", + "latency_ms": 1.0, + "covered": True, + "coverable": True, + "recommended_stop": False, + "reviewed": False, + "false_stop": False, + } + + +def _outcome(namespace: str) -> dict[str, object]: + return { + "schema_version": 1, + "event": "outcome", + "session_hash": CANARY, + "action_hash": f"{CANARY}-action", + "semantic_key": f"{CANARY}-semantic", + "state_hash": f"{CANARY}-state", + "evidence_hash": f"{CANARY}-evidence", + "model_namespace": namespace, + "outcome": "success", + "pending": False, + } + + +def test_compiler_reads_real_verified_records_and_emits_only_closed_atoms(tmp_path: Path) -> None: + identity = _identity() + store = EvidenceStore(tmp_path) + store.append(_decision(identity.namespace)) + store.append(_outcome(identity.namespace)) + + atoms = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) + + assert [atom.to_dict() for atom in atoms] == [ + { + "record_type": "decision", + "action_kind": "tool", + "cost_bucket": "low", + "gain_bucket": "medium", + "recommendation": "allow", + "applied_decision": "allow", + "reason_code": "APPROVED", + "outcome_class": "not_applicable", + "count": 1, + "minimum_group_size": 1, + }, + { + "record_type": "outcome", + "action_kind": "unknown", + "cost_bucket": "unknown", + "gain_bucket": "unknown", + "recommendation": "not_applicable", + "applied_decision": "not_applicable", + "reason_code": "not_applicable", + "outcome_class": "verified_success", + "count": 1, + "minimum_group_size": 1, + }, + ] + serialized = json.dumps([atom.to_dict() for atom in atoms], sort_keys=True) + assert CANARY not in serialized + for forbidden in ( + "hash", + "prompt", + "command", + "path", + "filename", + "repository", + "identity", + "url", + "secret", + "timestamp", + "pseudonym", + "metadata", + ): + assert forbidden not in serialized.casefold() + + +def test_atoms_are_immutable_and_counts_are_bounded(tmp_path: Path) -> None: + identity = _identity() + store = EvidenceStore(tmp_path) + for index in range(1_001): + record = _decision(identity.namespace) + record["action_hash"] = f"action-{index}" + store.append(record) + + atoms = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) + + assert len(atoms) == 1 + assert atoms[0].count == 1_000 + with pytest.raises(dataclasses.FrozenInstanceError): + atoms[0].count = 2 # type: ignore[misc] + + +def test_compiler_rejects_arbitrary_caller_mappings_and_unverified_chains(tmp_path: Path) -> None: + identity = _identity() + with pytest.raises(TypeError, match="EvidenceStore"): + compile_verified_evidence([_decision(identity.namespace)], model_identity=identity) # type: ignore[arg-type] + + store = EvidenceStore(tmp_path) + store.append(_decision(identity.namespace)) + ledger = store.governance_ledger_path + tampered = ledger.read_text(encoding="utf-8").replace("APPROVED", "DENIED") + ledger.write_text(tampered, encoding="utf-8") + assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) == () + + +def test_compiler_fails_closed_on_a_verified_non_record_payload(tmp_path: Path) -> None: + identity = _identity() + store = EvidenceStore(tmp_path) + GovernanceLedger(store.governance_ledger_path).append( + {"event": "codex_evidence", "evidence": [CANARY, {"nested": CANARY}]} + ) + + assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) == () + + +def test_conflicting_or_wrong_model_attribution_compiles_nothing(tmp_path: Path) -> None: + sol = _identity("gpt-5.6-sol") + terra = _identity("gpt-5.6-terra") + store = EvidenceStore(tmp_path) + store.append(_decision(sol.namespace)) + store.append(_decision(terra.namespace)) + + assert compile_verified_evidence(store, model_identity=sol, minimum_group_size=1) == () + assert compile_verified_evidence(store, model_identity=None, minimum_group_size=1) == () + + isolated = EvidenceStore(tmp_path / "isolated") + isolated.append(_decision(sol.namespace)) + assert compile_verified_evidence(isolated, model_identity=terra, minimum_group_size=1) == () + + +def test_small_groups_are_suppressed_and_boundaries_are_validated(tmp_path: Path) -> None: + identity = _identity() + store = EvidenceStore(tmp_path) + store.append(_decision(identity.namespace)) + + assert compile_verified_evidence(store, model_identity=identity) == () + with pytest.raises(TypeError, match="minimum_group_size"): + compile_verified_evidence(store, model_identity=identity, minimum_group_size=True) + with pytest.raises(ValueError, match="between 1 and 1000"): + compile_verified_evidence(store, model_identity=identity, minimum_group_size=1_001) + + +def test_local_evidence_rejects_nested_or_unbounded_commons_values(tmp_path: Path) -> None: + identity = _identity() + store = EvidenceStore(tmp_path) + with pytest.raises(ValueError, match="outcome"): + store.append({**_outcome(identity.namespace), "outcome": {"canary": CANARY}}) + with pytest.raises(ValueError, match="action_kind"): + store.append({**_decision(identity.namespace), "action_kind": CANARY}) + with pytest.raises(ValueError, match="model_namespace"): + store.append({**_decision(identity.namespace), "model_namespace": "private/custom-model"}) + + +def test_atom_constructor_accepts_typed_fields_not_arbitrary_nested_values() -> None: + with pytest.raises(TypeError, match="record_type"): + CommonsEvidenceAtom( # type: ignore[arg-type] + record_type={"canary": CANARY}, + action_kind="tool", + cost_bucket="low", + gain_bucket="medium", + recommendation="allow", + applied_decision="allow", + reason_code="APPROVED", + outcome_class="not_applicable", + count=1, + minimum_group_size=1, + ) diff --git a/tests/commons/test_identity.py b/tests/commons/test_identity.py new file mode 100644 index 0000000..b86e24c --- /dev/null +++ b/tests/commons/test_identity.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from marginal.commons.identity import ( + CanonicalModelIdentity, + resolve_canonical_model, + resolve_model_attribution, +) + +REVIEWED = { + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna", +} + + +@pytest.mark.parametrize(("model", "namespace"), REVIEWED.items()) +def test_exact_public_registry_match_resolves_an_immutable_identity( + model: str, namespace: str +) -> None: + identity = resolve_canonical_model(provider="openai", model=model) + + assert identity == CanonicalModelIdentity( + provider="openai", model=model, namespace=namespace, registry_version="1.0" + ) + with pytest.raises(dataclasses.FrozenInstanceError): + identity.model = "gpt-5.6-sol" # type: ignore[misc] + + +@pytest.mark.parametrize( + ("provider", "model"), + [ + ("OpenAI", "gpt-5.6-sol"), + ("openai", "GPT-5.6-SOL"), + ("openai", " gpt-5.6-sol"), + ("openai", "gpt-5.6-sol "), + ("openai", "gpt-5.6"), + ("openai", "gpt-5.6-sol-latest"), + ("openai", "gpt-5.6-sol-2026-08-21"), + ("openai", "ft:gpt-5.6-sol:private"), + ("openai", "private/gpt-5.6-sol"), + ("custom", "gpt-5.6-sol"), + ], +) +def test_unknown_private_alias_and_version_drift_remain_unresolved( + provider: str, model: str +) -> None: + assert resolve_canonical_model(provider=provider, model=model) is None + + +def test_attribution_requires_one_unambiguous_exact_model() -> None: + expected = resolve_canonical_model(provider="openai", model="gpt-5.6-sol") + + assert resolve_model_attribution([("openai", "gpt-5.6-sol")]) == expected + assert ( + resolve_model_attribution([("openai", "gpt-5.6-sol"), ("openai", "gpt-5.6-sol")]) + == expected + ) + assert ( + resolve_model_attribution([("openai", "gpt-5.6-sol"), ("openai", "gpt-5.6-terra")]) is None + ) + assert resolve_model_attribution([("openai", "gpt-5.6-sol"), ("custom", "private")]) is None + assert resolve_model_attribution([]) is None + + +def test_packaged_registry_is_byte_identical_to_reviewed_root_registry() -> None: + root = Path(__file__).resolve().parents[2] + assert (root / "models" / "canonical-model-registry-v1.json").read_bytes() == ( + root / "src" / "marginal" / "commons" / "canonical-model-registry-v1.json" + ).read_bytes() diff --git a/tests/integrations/codex/test_installer.py b/tests/integrations/codex/test_installer.py index e814eed..5c0e0c9 100644 --- a/tests/integrations/codex/test_installer.py +++ b/tests/integrations/codex/test_installer.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field +from marginal.commons.config import CommonsMode, load_commons_config from marginal.integrations.codex.installer import ( CommandResult, autopilot_consent_configured, @@ -101,3 +102,25 @@ def test_install_can_persist_explicit_user_autopilot_consent(tmp_path) -> None: assert result.installed is True assert result.autopilot_consent is True assert autopilot_consent_configured(tmp_path) is True + + +def test_install_persists_explicit_commons_choice_without_altering_autopilot(tmp_path) -> None: + result = install( + runner=RecordingRunner(), + data_dir=tmp_path, + autopilot_consent=True, + commons_mode=CommonsMode.READ_ONLY, + ) + + assert result.installed is True + assert result.commons_mode == "read_only" + assert load_commons_config(tmp_path).mode is CommonsMode.READ_ONLY + assert autopilot_consent_configured(tmp_path) is True + + +def test_install_defaults_to_local_only_without_persisting_or_enabling_network(tmp_path) -> None: + result = install(runner=RecordingRunner(), data_dir=tmp_path) + + assert result.commons_mode == "local_only" + assert load_commons_config(tmp_path).mode is CommonsMode.LOCAL_ONLY + assert not (tmp_path / "user-config.json").exists() diff --git a/tests/test_cli.py b/tests/test_cli.py index 8e31dbb..c41f7cc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ import json from marginal.cli import main +from marginal.integrations.codex.installer import CodexInstallation def write_trace(path) -> None: @@ -71,3 +72,32 @@ def test_codex_status_dispatches_without_importing_at_cli_module_load(tmp_path, assert exit_code == 0 assert json.loads(capsys.readouterr().out)["mode"] == "shadow" + + +def test_install_cli_persists_only_an_explicit_closed_commons_choice( + tmp_path, capsys, monkeypatch +) -> None: + captured = {} + + def fake_install(**kwargs): + captured.update(kwargs) + return CodexInstallation(True, True, commons_mode="contributor") + + monkeypatch.setattr("marginal.integrations.codex.installer.install", fake_install) + + assert ( + main( + [ + "install", + "codex", + "--data-dir", + str(tmp_path), + "--commons-mode", + "contributor", + "--json", + ] + ) + == 0 + ) + assert captured["commons_mode"] == "contributor" + assert json.loads(capsys.readouterr().out)["commons_mode"] == "contributor" diff --git a/tests/test_commons_contract.py b/tests/test_commons_contract.py index c609f5a..30dff7f 100644 --- a/tests/test_commons_contract.py +++ b/tests/test_commons_contract.py @@ -11,7 +11,6 @@ from marginal.privacy import aggregate_ledger_records - ROOT = Path(__file__).resolve().parents[1] NAMESPACES = ( "openai/gpt-5.6-sol", @@ -66,7 +65,11 @@ def test_envelope_accepts_unknown_action_kind_from_real_aggregate_outcome() -> N [{"event": "outcome", "outcome": {"resolved": True, "reward": 1.0}}], minimum_group_size=1, ) - atom = {key: value for key, value in rows[0].items() if key not in {"schema_version", "privacy_profile"}} + atom = { + key: value + for key, value in rows[0].items() + if key not in {"schema_version", "privacy_profile"} + } assert atom["action_kind"] == "unknown" _validator("commons-evidence-envelope-v1.json").validate( {"schema_version": "1.0", "model_namespace": NAMESPACES[0], "atoms": [atom]} @@ -141,9 +144,12 @@ def test_registry_contains_only_the_reviewed_exact_model_mapping() -> None: def test_contract_digest_fixtures_detect_schema_and_registry_drift() -> None: envelope_digest = (ROOT / "schemas" / "commons-evidence-envelope-v1.sha256").read_text().strip() - assert hashlib.sha256( - (ROOT / "schemas" / "commons-evidence-envelope-v1.json").read_bytes() - ).hexdigest() == envelope_digest + assert ( + hashlib.sha256( + (ROOT / "schemas" / "commons-evidence-envelope-v1.json").read_bytes() + ).hexdigest() + == envelope_digest + ) manifest = json.loads((ROOT / "schemas" / "commons-contract-v1.manifest.json").read_text()) for name, expected in manifest["sha256"].items(): base = ROOT / ("models" if name.startswith("canonical-model") else "schemas") From 211312de87bab8669301c397c7aee101bb9a1b6f Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 11:56:19 +0200 Subject: [PATCH 05/14] fix: harden Commons config persistence --- MANIFEST.in | 1 + .../commons-contract-v1.manifest.json | 0 src/marginal/commons/config.py | 146 +++++++++++------- src/marginal/governance_ledger.py | 5 +- src/marginal/integrations/codex/installer.py | 20 ++- tests/commons/test_config.py | 95 +++++++++++- tests/integrations/codex/test_installer.py | 12 +- tests/test_cli.py | 23 ++- tests/test_commons_contract.py | 3 +- 9 files changed, 243 insertions(+), 62 deletions(-) rename {schemas => contracts}/commons-contract-v1.manifest.json (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 8c4083a..c0853d9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,7 @@ recursive-include .github *.yml *.md include ROADMAP.md recursive-include schemas *.json +recursive-include contracts *.json recursive-include demos *.md *.json *.html *.svg *.jsonl recursive-include assets *.png diff --git a/schemas/commons-contract-v1.manifest.json b/contracts/commons-contract-v1.manifest.json similarity index 100% rename from schemas/commons-contract-v1.manifest.json rename to contracts/commons-contract-v1.manifest.json diff --git a/src/marginal/commons/config.py b/src/marginal/commons/config.py index a9dd6d4..acef003 100644 --- a/src/marginal/commons/config.py +++ b/src/marginal/commons/config.py @@ -5,13 +5,16 @@ import errno import json import os +import secrets import stat -import tempfile +from contextlib import suppress from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Any +from marginal.governance_ledger import _open_parent_directory, _write_all + _CONFIG_NAME = "user-config.json" _MAX_CONFIG_BYTES = 65_536 @@ -47,38 +50,47 @@ def __post_init__(self) -> None: def _absolute_path(path: str | Path) -> Path: supplied = Path(path) - return Path(os.path.abspath(os.fspath(supplied))) - - -def _reject_symlink_components(path: Path) -> None: - current = Path(path.anchor) - for part in path.parts[1:]: - current /= part - try: - metadata = current.lstat() - except FileNotFoundError: - continue - if stat.S_ISLNK(metadata.st_mode): - raise ValueError("user configuration path must not contain a symbolic link") + if ".." in supplied.parts: + raise ValueError("user configuration path must not contain traversal components") + return supplied if supplied.is_absolute() else Path.cwd() / supplied def _config_path(data_dir: str | Path) -> Path: root = _absolute_path(data_dir) - _reject_symlink_components(root) return root / _CONFIG_NAME -def _read_user_config(data_dir: str | Path) -> dict[str, Any] | None: - path = _config_path(data_dir) - if path.is_symlink(): - raise ValueError("user configuration path must not be a symbolic link") - flags = os.O_RDONLY +def _open_config_directory(data_dir: str | Path, *, create: bool) -> int: + try: + return _open_parent_directory(_config_path(data_dir), create_parents=create) + except ValueError as exc: + if "symbolic" in str(exc): + raise ValueError("user configuration path must not contain a symbolic link") from exc + raise + except OSError as exc: + if exc.errno in {errno.ELOOP, errno.ENOTDIR}: + raise ValueError("user configuration path must not contain a symbolic link") from exc + raise + + +def _read_bounded(descriptor: int) -> bytes: + chunks: list[bytes] = [] + remaining = _MAX_CONFIG_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, min(64 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def _read_user_config_at(directory_descriptor: int) -> dict[str, Any] | None: + flags = os.O_RDONLY | os.O_NOFOLLOW if hasattr(os, "O_BINARY"): flags |= os.O_BINARY - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW try: - descriptor = os.open(path, flags) + descriptor = os.open(_CONFIG_NAME, flags, dir_fd=directory_descriptor) except FileNotFoundError: return None except OSError as exc: @@ -91,7 +103,7 @@ def _read_user_config(data_dir: str | Path) -> dict[str, Any] | None: raise ValueError("user configuration must be a regular file") if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & 0o077: raise ValueError("user configuration must have owner-only permissions") - raw = os.read(descriptor, _MAX_CONFIG_BYTES + 1) + raw = _read_bounded(descriptor) finally: os.close(descriptor) if len(raw) > _MAX_CONFIG_BYTES: @@ -109,50 +121,80 @@ def _read_user_config(data_dir: str | Path) -> dict[str, Any] | None: return payload -def _write_user_config(data_dir: str | Path, payload: dict[str, Any]) -> None: - path = _config_path(data_dir) - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - _reject_symlink_components(path.parent) - if os.name == "posix": - path.parent.chmod(0o700) +def _read_user_config(data_dir: str | Path) -> dict[str, Any] | None: + try: + directory_descriptor = _open_config_directory(data_dir, create=False) + except FileNotFoundError: + return None + try: + return _read_user_config_at(directory_descriptor) + finally: + os.close(directory_descriptor) + + +def _temporary_name() -> str: + return f".user-config-{secrets.token_hex(12)}.tmp" + + +def _open_temporary(directory_descriptor: int) -> tuple[int, str]: + for _ in range(16): + name = _temporary_name() + try: + descriptor = os.open( + name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_descriptor, + ) + except FileExistsError: + continue + return descriptor, name + raise FileExistsError("unable to allocate a private user configuration temporary file") + + +def _write_user_config_at(directory_descriptor: int, payload: dict[str, Any]) -> None: encoded = ( json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" ).encode("utf-8") - descriptor, temporary_name = tempfile.mkstemp( - prefix=".user-config-", suffix=".tmp", dir=path.parent - ) - temporary = Path(temporary_name) + descriptor, temporary_name = _open_temporary(directory_descriptor) + renamed = False try: os.fchmod(descriptor, 0o600) - os.write(descriptor, encoded) + _write_all(descriptor, encoded) os.fsync(descriptor) os.close(descriptor) descriptor = -1 - if path.is_symlink(): - raise ValueError("user configuration path must not be a symbolic link") - os.replace(temporary, path) - if os.name == "posix": - path.chmod(0o600) - directory = os.open(path.parent, os.O_RDONLY) - try: - os.fsync(directory) - finally: - os.close(directory) + os.replace( + temporary_name, + _CONFIG_NAME, + src_dir_fd=directory_descriptor, + dst_dir_fd=directory_descriptor, + ) + renamed = True + os.fsync(directory_descriptor) finally: if descriptor >= 0: os.close(descriptor) - temporary.unlink(missing_ok=True) + if not renamed: + with suppress(FileNotFoundError): + os.unlink(temporary_name, dir_fd=directory_descriptor) def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, Any]: """Atomically merge reviewed choices into the one user configuration file.""" - payload = _read_user_config(data_dir) - if payload is None: - payload = {"schema_version": 1} - payload.update(changes) - _write_user_config(data_dir, payload) - return payload + directory_descriptor = _open_config_directory(data_dir, create=True) + try: + if os.name == "posix": + os.fchmod(directory_descriptor, 0o700) + payload = _read_user_config_at(directory_descriptor) + if payload is None: + payload = {"schema_version": 1} + payload.update(changes) + _write_user_config_at(directory_descriptor, payload) + return payload + finally: + os.close(directory_descriptor) def load_commons_config(data_dir: str | Path) -> CommonsConfig: diff --git a/src/marginal/governance_ledger.py b/src/marginal/governance_ledger.py index be70566..aeb73f0 100644 --- a/src/marginal/governance_ledger.py +++ b/src/marginal/governance_ledger.py @@ -492,7 +492,10 @@ def _read_descriptor(descriptor: int) -> bytes: def _write_all(descriptor: int, data: bytes) -> None: offset = 0 while offset < len(data): - offset += os.write(descriptor, data[offset:]) + written = os.write(descriptor, data[offset:]) + if written <= 0: + raise OSError("write made no progress") + offset += written def _write_owner_only_at(directory_descriptor: int, name: str, data: bytes) -> None: diff --git a/src/marginal/integrations/codex/installer.py b/src/marginal/integrations/codex/installer.py index 7a68b1e..af4286f 100644 --- a/src/marginal/integrations/codex/installer.py +++ b/src/marginal/integrations/codex/installer.py @@ -10,7 +10,12 @@ from pathlib import Path from typing import Protocol -from marginal.commons.config import CommonsMode, _update_user_config, configure_commons_mode +from marginal.commons.config import ( + CommonsMode, + _update_user_config, + configure_commons_mode, + load_commons_config, +) @dataclass(frozen=True, slots=True) @@ -140,9 +145,12 @@ def install( ) -> CodexInstallation: if not isinstance(autopilot_consent, bool): raise TypeError("autopilot_consent must be a bool") - selected_commons_mode = ( - CommonsMode.LOCAL_ONLY if commons_mode is None else CommonsMode.parse(commons_mode) - ) + if commons_mode is None: + selected_commons_mode = ( + CommonsMode.LOCAL_ONLY if data_dir is None else load_commons_config(data_dir).mode + ) + else: + selected_commons_mode = CommonsMode.parse(commons_mode) selected = runner or SubprocessRunner() report = inspect_codex(runner=selected) if report.capability_level != "tool_enforcement": @@ -151,6 +159,7 @@ def install( False, error_code="CODEX_CAPABILITIES_UNAVAILABLE", message=", ".join(report.blocking_reasons), + commons_mode=selected_commons_mode.value, ) marketplace = selected.run( [ @@ -170,6 +179,7 @@ def install( False, error_code="MARKETPLACE_ADD_FAILED", message=marketplace.stderr.strip(), + commons_mode=selected_commons_mode.value, ) plugin = selected.run(["codex", "plugin", "add", "marginal@marginal", "--json"]) if plugin.returncode != 0 and "already" not in plugin.stderr.casefold(): @@ -178,6 +188,7 @@ def install( False, error_code="PLUGIN_ADD_FAILED", message=plugin.stderr.strip(), + commons_mode=selected_commons_mode.value, ) if autopilot_consent: if data_dir is None: @@ -186,6 +197,7 @@ def install( True, error_code="AUTOPILOT_CONSENT_DATA_DIR_REQUIRED", message="installed in Shadow Mode; Autopilot consent was not persisted", + commons_mode=selected_commons_mode.value, ) configure_autopilot_consent(data_dir, granted=True) if commons_mode is not None: diff --git a/tests/commons/test_config.py b/tests/commons/test_config.py index ef4697b..5bf32b4 100644 --- a/tests/commons/test_config.py +++ b/tests/commons/test_config.py @@ -7,6 +7,7 @@ import pytest +import marginal.commons.config as commons_config from marginal.commons.config import ( CommonsMode, configure_commons_mode, @@ -90,8 +91,8 @@ def test_failed_atomic_replace_preserves_existing_choices( path = tmp_path / "user-config.json" before = path.read_bytes() - def fail_replace(source: Path, destination: Path) -> None: - del source, destination + def fail_replace(source: object, destination: object, **directory_descriptors: object) -> None: + del source, destination, directory_descriptors raise OSError("synthetic replace failure") monkeypatch.setattr(os, "replace", fail_replace) @@ -102,6 +103,96 @@ def fail_replace(source: Path, destination: Path) -> None: assert not list(tmp_path.glob(".user-config-*.tmp")) +def test_atomic_update_holds_the_open_parent_across_a_path_swap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_root = tmp_path / "data" + configure_commons_mode(data_root, mode=CommonsMode.CONTRIBUTOR) + displaced = tmp_path / "displaced-data" + outside = tmp_path / "outside" + outside.mkdir() + outside_config = outside / "user-config.json" + outside_config.write_text( + '{"commons_mode":"local_only","schema_version":1}\n', encoding="utf-8" + ) + outside_config.chmod(0o600) + original_open = os.open + swapped = False + + def swapping_open( + name: object, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if dir_fd is None: + descriptor = original_open(name, flags, mode) + else: + descriptor = original_open(name, flags, mode, dir_fd=dir_fd) + if not swapped and name == data_root.name and dir_fd is not None and flags & os.O_DIRECTORY: + data_root.rename(displaced) + data_root.symlink_to(outside, target_is_directory=True) + swapped = True + return descriptor + + monkeypatch.setattr(commons_config.os, "open", swapping_open) + + configure_commons_mode(data_root, mode=CommonsMode.READ_ONLY) + + assert swapped is True + assert ( + json.loads((displaced / "user-config.json").read_text(encoding="utf-8"))["commons_mode"] + == "read_only" + ) + assert json.loads(outside_config.read_text(encoding="utf-8"))["commons_mode"] == "local_only" + + +def test_atomic_update_retries_short_writes_until_the_config_is_complete( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + original_write = os.write + writes = 0 + + def short_write(descriptor: int, data: bytes) -> int: + nonlocal writes + writes += 1 + return original_write(descriptor, data[:3]) + + monkeypatch.setattr(commons_config.os, "write", short_write) + + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + + assert writes > 1 + assert load_commons_config(tmp_path).mode is CommonsMode.CONTRIBUTOR + + +def test_atomic_update_preserves_the_old_file_when_a_partial_write_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + path = tmp_path / "user-config.json" + before = path.read_bytes() + original_write = os.write + writes = 0 + + def interrupted_write(descriptor: int, data: bytes) -> int: + nonlocal writes + writes += 1 + if writes == 1: + return original_write(descriptor, data[:4]) + raise OSError("synthetic interrupted write") + + monkeypatch.setattr(commons_config.os, "write", interrupted_write) + + with pytest.raises(OSError, match="interrupted write"): + configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) + + assert path.read_bytes() == before + assert not list(tmp_path.glob(".user-config-*.tmp")) + + def test_config_rejects_boolean_schema_versions_instead_of_treating_them_as_one( tmp_path: Path, ) -> None: diff --git a/tests/integrations/codex/test_installer.py b/tests/integrations/codex/test_installer.py index 5c0e0c9..f9f9728 100644 --- a/tests/integrations/codex/test_installer.py +++ b/tests/integrations/codex/test_installer.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from marginal.commons.config import CommonsMode, load_commons_config +from marginal.commons.config import CommonsMode, configure_commons_mode, load_commons_config from marginal.integrations.codex.installer import ( CommandResult, autopilot_consent_configured, @@ -124,3 +124,13 @@ def test_install_defaults_to_local_only_without_persisting_or_enabling_network(t assert result.commons_mode == "local_only" assert load_commons_config(tmp_path).mode is CommonsMode.LOCAL_ONLY assert not (tmp_path / "user-config.json").exists() + + +def test_reinstall_without_a_mode_reports_and_preserves_persisted_contributor(tmp_path) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + before = (tmp_path / "user-config.json").read_bytes() + + result = install(runner=RecordingRunner(), data_dir=tmp_path) + + assert result.commons_mode == "contributor" + assert (tmp_path / "user-config.json").read_bytes() == before diff --git a/tests/test_cli.py b/tests/test_cli.py index c41f7cc..38f7aab 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,7 +3,8 @@ import json from marginal.cli import main -from marginal.integrations.codex.installer import CodexInstallation +from marginal.commons.config import CommonsMode, configure_commons_mode +from marginal.integrations.codex.installer import CodexInstallation, CommandResult def write_trace(path) -> None: @@ -101,3 +102,23 @@ def fake_install(**kwargs): ) assert captured["commons_mode"] == "contributor" assert json.loads(capsys.readouterr().out)["commons_mode"] == "contributor" + + +def test_reinstall_cli_json_reports_the_effective_persisted_contributor_mode( + tmp_path, capsys, monkeypatch +) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + + class AvailableCodex: + def run(self, args): + if args == ["codex", "--version"]: + return CommandResult(0, "codex-cli 0.147.0\n", "") + if args == ["codex", "features", "list"]: + return CommandResult(0, "hooks stable true\nplugins stable true\n", "") + return CommandResult(0, "{}", "") + + monkeypatch.setattr("marginal.integrations.codex.installer.SubprocessRunner", AvailableCodex) + + assert main(["install", "codex", "--data-dir", str(tmp_path), "--json"]) == 0 + + assert json.loads(capsys.readouterr().out)["commons_mode"] == "contributor" diff --git a/tests/test_commons_contract.py b/tests/test_commons_contract.py index 30dff7f..ec369aa 100644 --- a/tests/test_commons_contract.py +++ b/tests/test_commons_contract.py @@ -150,7 +150,8 @@ def test_contract_digest_fixtures_detect_schema_and_registry_drift() -> None: ).hexdigest() == envelope_digest ) - manifest = json.loads((ROOT / "schemas" / "commons-contract-v1.manifest.json").read_text()) + assert not (ROOT / "schemas" / "commons-contract-v1.manifest.json").exists() + manifest = json.loads((ROOT / "contracts" / "commons-contract-v1.manifest.json").read_text()) for name, expected in manifest["sha256"].items(): base = ROOT / ("models" if name.startswith("canonical-model") else "schemas") assert hashlib.sha256((base / name).read_bytes()).hexdigest() == expected From 0e03d3b5dcd6c5c083da826401226f3f0ccab9a3 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 12:11:00 +0200 Subject: [PATCH 06/14] fix: serialize shared user configuration --- src/marginal/commons/config.py | 78 ++++++++++++++++++---- tests/commons/test_config.py | 116 ++++++++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 14 deletions(-) diff --git a/src/marginal/commons/config.py b/src/marginal/commons/config.py index acef003..8dd3ae4 100644 --- a/src/marginal/commons/config.py +++ b/src/marginal/commons/config.py @@ -13,9 +13,15 @@ from pathlib import Path from typing import Any -from marginal.governance_ledger import _open_parent_directory, _write_all +from marginal.governance_ledger import ( + _lock_exclusive, + _open_parent_directory, + _unlock, + _write_all, +) _CONFIG_NAME = "user-config.json" +_LOCK_NAME = ".user-config.lock" _MAX_CONFIG_BYTES = 65_536 @@ -152,7 +158,50 @@ def _open_temporary(directory_descriptor: int) -> tuple[int, str]: raise FileExistsError("unable to allocate a private user configuration temporary file") +def _require_config_write_safety() -> None: + if os.rename not in os.supports_dir_fd or os.unlink not in os.supports_dir_fd: + raise OSError("descriptor-relative replace and cleanup operations are unavailable") + + +def _open_config_lock(directory_descriptor: int) -> int: + try: + descriptor = os.open( + _LOCK_NAME, + os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_descriptor, + ) + except OSError as exc: + if exc.errno == errno.ELOOP: + raise ValueError("user configuration lock must not be a symbolic link") from exc + raise + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError("user configuration lock must be a regular file") + if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & 0o077: + raise ValueError("user configuration lock must have owner-only permissions") + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def _rename_config_at(directory_descriptor: int, temporary_name: str) -> None: + os.rename( + temporary_name, + _CONFIG_NAME, + src_dir_fd=directory_descriptor, + dst_dir_fd=directory_descriptor, + ) + + +def _unlink_config_at(directory_descriptor: int, temporary_name: str) -> None: + os.unlink(temporary_name, dir_fd=directory_descriptor) + + def _write_user_config_at(directory_descriptor: int, payload: dict[str, Any]) -> None: + _require_config_write_safety() encoded = ( json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" ).encode("utf-8") @@ -164,29 +213,32 @@ def _write_user_config_at(directory_descriptor: int, payload: dict[str, Any]) -> os.fsync(descriptor) os.close(descriptor) descriptor = -1 - os.replace( - temporary_name, - _CONFIG_NAME, - src_dir_fd=directory_descriptor, - dst_dir_fd=directory_descriptor, - ) + _rename_config_at(directory_descriptor, temporary_name) renamed = True os.fsync(directory_descriptor) - finally: + except BaseException: if descriptor >= 0: - os.close(descriptor) + with suppress(OSError): + os.close(descriptor) if not renamed: - with suppress(FileNotFoundError): - os.unlink(temporary_name, dir_fd=directory_descriptor) + with suppress(OSError): + _unlink_config_at(directory_descriptor, temporary_name) + raise def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, Any]: """Atomically merge reviewed choices into the one user configuration file.""" + _require_config_write_safety() directory_descriptor = _open_config_directory(data_dir, create=True) + lock_descriptor = -1 + locked = False try: if os.name == "posix": os.fchmod(directory_descriptor, 0o700) + lock_descriptor = _open_config_lock(directory_descriptor) + _lock_exclusive(lock_descriptor) + locked = True payload = _read_user_config_at(directory_descriptor) if payload is None: payload = {"schema_version": 1} @@ -194,6 +246,10 @@ def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, An _write_user_config_at(directory_descriptor, payload) return payload finally: + if locked: + _unlock(lock_descriptor) + if lock_descriptor >= 0: + os.close(lock_descriptor) os.close(directory_descriptor) diff --git a/tests/commons/test_config.py b/tests/commons/test_config.py index 5bf32b4..caf4565 100644 --- a/tests/commons/test_config.py +++ b/tests/commons/test_config.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import multiprocessing import os import stat from pathlib import Path @@ -19,6 +20,33 @@ ) +def _paused_commons_update( + data_root: str, + replace_reached: multiprocessing.synchronize.Event, + allow_replace: multiprocessing.synchronize.Event, +) -> None: + original_rename = commons_config._rename_config_at + + def paused_rename(directory_descriptor: int, temporary_name: str) -> None: + replace_reached.set() + if not allow_replace.wait(timeout=10): + raise TimeoutError("test did not release Commons replace") + original_rename(directory_descriptor, temporary_name) + + commons_config._rename_config_at = paused_rename + configure_commons_mode(data_root, mode=CommonsMode.CONTRIBUTOR) + + +def _revoke_autopilot( + data_root: str, + revoke_started: multiprocessing.synchronize.Event, + revoke_completed: multiprocessing.synchronize.Event, +) -> None: + revoke_started.set() + configure_autopilot_consent(data_root, granted=False) + revoke_completed.set() + + def test_missing_config_defaults_to_local_only_without_creating_a_file(tmp_path: Path) -> None: assert load_commons_config(tmp_path).mode is CommonsMode.LOCAL_ONLY assert not (tmp_path / "user-config.json").exists() @@ -91,11 +119,11 @@ def test_failed_atomic_replace_preserves_existing_choices( path = tmp_path / "user-config.json" before = path.read_bytes() - def fail_replace(source: object, destination: object, **directory_descriptors: object) -> None: - del source, destination, directory_descriptors + def fail_replace(directory_descriptor: int, temporary_name: str) -> None: + del directory_descriptor, temporary_name raise OSError("synthetic replace failure") - monkeypatch.setattr(os, "replace", fail_replace) + monkeypatch.setattr(commons_config, "_rename_config_at", fail_replace) with pytest.raises(OSError, match="synthetic"): configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) @@ -193,6 +221,88 @@ def interrupted_write(descriptor: int, data: bytes) -> int: assert not list(tmp_path.glob(".user-config-*.tmp")) +@pytest.mark.skipif(os.name == "nt", reason="POSIX advisory locks are required") +def test_commons_update_and_autopilot_revoke_are_one_serialized_transaction( + tmp_path: Path, +) -> None: + try: + context = multiprocessing.get_context("fork") + except ValueError: + pytest.skip("fork multiprocessing context is unavailable") + configure_autopilot_consent(tmp_path, granted=True) + replace_reached = context.Event() + allow_replace = context.Event() + revoke_started = context.Event() + revoke_completed = context.Event() + commons_process = context.Process( + target=_paused_commons_update, + args=(str(tmp_path), replace_reached, allow_replace), + ) + revoke_process = context.Process( + target=_revoke_autopilot, + args=(str(tmp_path), revoke_started, revoke_completed), + ) + + commons_process.start() + assert replace_reached.wait(timeout=10) + revoke_process.start() + assert revoke_started.wait(timeout=10) + assert not revoke_completed.wait(timeout=0.25) + allow_replace.set() + commons_process.join(timeout=10) + revoke_process.join(timeout=10) + + assert commons_process.exitcode == 0 + assert revoke_process.exitcode == 0 + payload = json.loads((tmp_path / "user-config.json").read_text(encoding="utf-8")) + assert payload["commons_mode"] == "contributor" + assert payload["autopilot_consent"] is False + if os.name == "posix": + assert stat.S_IMODE((tmp_path / ".user-config.lock").stat().st_mode) == 0o600 + + +def test_missing_descriptor_cleanup_capability_fails_before_temp_creation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + path = tmp_path / "user-config.json" + before = path.read_bytes() + partial_support = set(os.supports_dir_fd) + partial_support.discard(os.rename) + monkeypatch.setattr(commons_config.os, "supports_dir_fd", partial_support) + + with pytest.raises(OSError, match="descriptor-relative replace and cleanup"): + configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) + + assert path.read_bytes() == before + assert not list(tmp_path.glob(".user-config-*.tmp")) + + +def test_cleanup_failure_does_not_replace_the_original_write_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + original_write = os.write + writes = 0 + + def interrupted_write(descriptor: int, data: bytes) -> int: + nonlocal writes + writes += 1 + if writes == 1: + return original_write(descriptor, data[:4]) + raise OSError("original write failure") + + def failed_cleanup(directory_descriptor: int, temporary_name: str) -> None: + del directory_descriptor, temporary_name + raise OSError("secondary cleanup failure") + + monkeypatch.setattr(commons_config.os, "write", interrupted_write) + monkeypatch.setattr(commons_config, "_unlink_config_at", failed_cleanup) + + with pytest.raises(OSError, match="original write failure"): + configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) + + def test_config_rejects_boolean_schema_versions_instead_of_treating_them_as_one( tmp_path: Path, ) -> None: From 4c24a73674a5608e030335d18daf2fa0656e4467 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 12:18:29 +0200 Subject: [PATCH 07/14] fix: preserve config transaction failures --- src/marginal/commons/config.py | 27 ++++++- tests/commons/test_config.py | 129 +++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/marginal/commons/config.py b/src/marginal/commons/config.py index 8dd3ae4..544a9f1 100644 --- a/src/marginal/commons/config.py +++ b/src/marginal/commons/config.py @@ -183,7 +183,8 @@ def _open_config_lock(directory_descriptor: int) -> int: raise ValueError("user configuration lock must have owner-only permissions") return descriptor except BaseException: - os.close(descriptor) + with suppress(BaseException): + os.close(descriptor) raise @@ -233,6 +234,7 @@ def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, An directory_descriptor = _open_config_directory(data_dir, create=True) lock_descriptor = -1 locked = False + primary_error: BaseException | None = None try: if os.name == "posix": os.fchmod(directory_descriptor, 0o700) @@ -245,12 +247,29 @@ def _update_user_config(data_dir: str | Path, **changes: object) -> dict[str, An payload.update(changes) _write_user_config_at(directory_descriptor, payload) return payload + except BaseException as exc: + primary_error = exc + raise finally: + cleanup_error: BaseException | None = None if locked: - _unlock(lock_descriptor) + try: + _unlock(lock_descriptor) + except BaseException as exc: + cleanup_error = exc if lock_descriptor >= 0: - os.close(lock_descriptor) - os.close(directory_descriptor) + try: + os.close(lock_descriptor) + except BaseException as exc: + if cleanup_error is None: + cleanup_error = exc + try: + os.close(directory_descriptor) + except BaseException as exc: + if cleanup_error is None: + cleanup_error = exc + if primary_error is None and cleanup_error is not None: + raise cleanup_error def load_commons_config(data_dir: str | Path) -> CommonsConfig: diff --git a/tests/commons/test_config.py b/tests/commons/test_config.py index caf4565..7188ca8 100644 --- a/tests/commons/test_config.py +++ b/tests/commons/test_config.py @@ -303,6 +303,135 @@ def failed_cleanup(directory_descriptor: int, temporary_name: str) -> None: configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) +def _open_descriptor_count() -> int: + for descriptor_directory in ("/proc/self/fd", "/dev/fd"): + if os.path.isdir(descriptor_directory): + return len(os.listdir(descriptor_directory)) + pytest.skip("open descriptor enumeration is unavailable") + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX advisory locks are required") +def test_write_failure_survives_unlock_failure_and_all_descriptors_are_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + baseline_descriptors = _open_descriptor_count() + opened: dict[str, int] = {} + unlock_attempts: list[int] = [] + close_attempts: list[int] = [] + original_open_directory = commons_config._open_config_directory + original_open_lock = commons_config._open_config_lock + original_close = os.close + + def tracked_open_directory(data_dir: str | Path, *, create: bool) -> int: + descriptor = original_open_directory(data_dir, create=create) + opened["directory"] = descriptor + return descriptor + + def tracked_open_lock(directory_descriptor: int) -> int: + descriptor = original_open_lock(directory_descriptor) + opened["lock"] = descriptor + return descriptor + + def fail_write(descriptor: int, data: bytes) -> int: + del descriptor, data + raise OSError("primary write failure") + + def fail_unlock(descriptor: int) -> None: + unlock_attempts.append(descriptor) + raise OSError("secondary unlock failure") + + def tracked_close(descriptor: int) -> None: + close_attempts.append(descriptor) + original_close(descriptor) + + monkeypatch.setattr(commons_config, "_open_config_directory", tracked_open_directory) + monkeypatch.setattr(commons_config, "_open_config_lock", tracked_open_lock) + monkeypatch.setattr(commons_config.os, "write", fail_write) + monkeypatch.setattr(commons_config, "_unlock", fail_unlock) + monkeypatch.setattr(commons_config.os, "close", tracked_close) + + with pytest.raises(OSError, match="primary write failure"): + configure_commons_mode(tmp_path, mode=CommonsMode.READ_ONLY) + + assert unlock_attempts == [opened["lock"]] + assert opened["lock"] in close_attempts + assert opened["directory"] in close_attempts + assert _open_descriptor_count() == baseline_descriptors + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX advisory locks are required") +def test_unlock_failure_after_success_closes_all_descriptors_then_surfaces( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + baseline_descriptors = _open_descriptor_count() + opened: dict[str, int] = {} + close_attempts: list[int] = [] + original_open_directory = commons_config._open_config_directory + original_open_lock = commons_config._open_config_lock + original_close = os.close + + def tracked_open_directory(data_dir: str | Path, *, create: bool) -> int: + descriptor = original_open_directory(data_dir, create=create) + opened["directory"] = descriptor + return descriptor + + def tracked_open_lock(directory_descriptor: int) -> int: + descriptor = original_open_lock(directory_descriptor) + opened["lock"] = descriptor + return descriptor + + def fail_unlock(descriptor: int) -> None: + assert descriptor == opened["lock"] + raise OSError("unlock failure after successful write") + + def tracked_close(descriptor: int) -> None: + close_attempts.append(descriptor) + original_close(descriptor) + + monkeypatch.setattr(commons_config, "_open_config_directory", tracked_open_directory) + monkeypatch.setattr(commons_config, "_open_config_lock", tracked_open_lock) + monkeypatch.setattr(commons_config, "_unlock", fail_unlock) + monkeypatch.setattr(commons_config.os, "close", tracked_close) + + with pytest.raises(OSError, match="unlock failure after successful write"): + configure_commons_mode(tmp_path, mode=CommonsMode.CONTRIBUTOR) + + assert opened["lock"] in close_attempts + assert opened["directory"] in close_attempts + assert _open_descriptor_count() == baseline_descriptors + assert load_commons_config(tmp_path).mode is CommonsMode.CONTRIBUTOR + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX descriptor cleanup is required") +def test_open_lock_preserves_validation_failure_when_close_also_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + directory_descriptor = os.open(tmp_path, os.O_RDONLY | os.O_DIRECTORY) + baseline_descriptors = _open_descriptor_count() + close_attempts: list[int] = [] + original_close = os.close + + def fail_metadata(descriptor: int) -> os.stat_result: + del descriptor + raise OSError("primary lock validation failure") + + def close_then_fail(descriptor: int) -> None: + close_attempts.append(descriptor) + original_close(descriptor) + raise OSError("secondary lock close failure") + + monkeypatch.setattr(commons_config.os, "fstat", fail_metadata) + monkeypatch.setattr(commons_config.os, "close", close_then_fail) + try: + with pytest.raises(OSError, match="primary lock validation failure"): + commons_config._open_config_lock(directory_descriptor) + assert len(close_attempts) == 1 + assert _open_descriptor_count() == baseline_descriptors + finally: + original_close(directory_descriptor) + + def test_config_rejects_boolean_schema_versions_instead_of_treating_them_as_one( tmp_path: Path, ) -> None: From c4c7ade977434b8f1f8083b5a883e181504d1d51 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 12:41:27 +0200 Subject: [PATCH 08/14] feat: add bounded Commons synchronization client --- src/marginal/commons/__init__.py | 13 ++ src/marginal/commons/_storage.py | 277 +++++++++++++++++++++++++++++ src/marginal/commons/cache.py | 259 +++++++++++++++++++++++++++ src/marginal/commons/client.py | 203 ++++++++++++++++++++++ src/marginal/commons/outbox.py | 288 +++++++++++++++++++++++++++++++ src/marginal/commons/sync.py | 165 ++++++++++++++++++ tests/commons/test_cache.py | 173 +++++++++++++++++++ tests/commons/test_client.py | 205 ++++++++++++++++++++++ tests/commons/test_outbox.py | 224 ++++++++++++++++++++++++ tests/commons/test_sync.py | 228 ++++++++++++++++++++++++ 10 files changed, 2035 insertions(+) create mode 100644 src/marginal/commons/_storage.py create mode 100644 src/marginal/commons/cache.py create mode 100644 src/marginal/commons/client.py create mode 100644 src/marginal/commons/outbox.py create mode 100644 src/marginal/commons/sync.py create mode 100644 tests/commons/test_cache.py create mode 100644 tests/commons/test_client.py create mode 100644 tests/commons/test_outbox.py create mode 100644 tests/commons/test_sync.py diff --git a/src/marginal/commons/__init__.py b/src/marginal/commons/__init__.py index 918cd3a..ab716e7 100644 --- a/src/marginal/commons/__init__.py +++ b/src/marginal/commons/__init__.py @@ -1,5 +1,7 @@ """Privacy-preserving, model-specific MARGINAL Commons primitives.""" +from .cache import CommonsCache, CommonsLifecycle, CommonsPrior +from .client import CommonsAck, CommonsClient from .config import CommonsConfig, CommonsMode, configure_commons_mode, load_commons_config from .evidence import CommonsEvidenceAtom, compile_verified_evidence from .identity import ( @@ -7,15 +9,26 @@ resolve_canonical_model, resolve_model_attribution, ) +from .outbox import CommonsOutbox, OutboxEntry +from .sync import CommonsSyncResult, synchronize_commons __all__ = [ "CanonicalModelIdentity", + "CommonsAck", + "CommonsCache", + "CommonsClient", "CommonsConfig", "CommonsEvidenceAtom", + "CommonsLifecycle", "CommonsMode", + "CommonsOutbox", + "CommonsPrior", + "CommonsSyncResult", + "OutboxEntry", "compile_verified_evidence", "configure_commons_mode", "load_commons_config", "resolve_canonical_model", "resolve_model_attribution", + "synchronize_commons", ] diff --git a/src/marginal/commons/_storage.py b/src/marginal/commons/_storage.py new file mode 100644 index 0000000..d8c722b --- /dev/null +++ b/src/marginal/commons/_storage.py @@ -0,0 +1,277 @@ +"""Descriptor-relative owner-only storage shared by Commons modules.""" + +from __future__ import annotations + +import errno +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager, suppress +from pathlib import Path +from secrets import token_hex as _token_hex + +from marginal.governance_ledger import ( + _lock_exclusive, + _open_parent_directory, + _unlock, + _write_all, +) + +_OWNER_ONLY_MASK = 0o077 + + +def _validate_directory(descriptor: int) -> None: + metadata = os.fstat(descriptor) + if not stat.S_ISDIR(metadata.st_mode): + raise ValueError("Commons storage path must be a directory") + if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & _OWNER_ONLY_MASK: + raise PermissionError("Commons storage directory must have owner-only permissions") + + +def _open_lock(directory_descriptor: int, name: str) -> int: + try: + descriptor = os.open( + name, + os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_descriptor, + ) + except OSError as exc: + if exc.errno == errno.ELOOP: + raise ValueError("Commons storage lock must not be a symbolic link") from exc + raise + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError("Commons storage lock must be a regular file") + if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & _OWNER_ONLY_MASK: + raise PermissionError("Commons storage lock must have owner-only permissions") + return descriptor + except BaseException: + with suppress(BaseException): + os.close(descriptor) + raise + + +@contextmanager +def locked_directory(path: Path, *, create: bool, lock_name: str) -> Iterator[int]: + """Hold one nofollow directory descriptor and an interprocess lock.""" + + directory_descriptor = -1 + for _ in range(16): + try: + directory_descriptor = _open_parent_directory(path / ".anchor", create_parents=create) + except FileExistsError: + continue + break + if directory_descriptor < 0: + raise FileExistsError("unable to open concurrently created Commons storage") + lock_descriptor = -1 + locked = False + primary_error: BaseException | None = None + try: + _validate_directory(directory_descriptor) + lock_descriptor = _open_lock(directory_descriptor, lock_name) + _lock_exclusive(lock_descriptor) + locked = True + yield directory_descriptor + except BaseException as exc: + primary_error = exc + raise + finally: + cleanup_error: BaseException | None = None + if locked: + try: + _unlock(lock_descriptor) + except BaseException as exc: + cleanup_error = exc + if lock_descriptor >= 0: + try: + os.close(lock_descriptor) + except BaseException as exc: + if cleanup_error is None: + cleanup_error = exc + try: + os.close(directory_descriptor) + except BaseException as exc: + if cleanup_error is None: + cleanup_error = exc + if primary_error is None and cleanup_error is not None: + raise cleanup_error + + +def _validate_regular_owner_only(descriptor: int, *, label: str) -> os.stat_result: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} must be a regular file") + if os.name == "posix" and stat.S_IMODE(metadata.st_mode) & _OWNER_ONLY_MASK: + raise PermissionError(f"{label} must have owner-only permissions") + return metadata + + +def read_bounded_at( + directory_descriptor: int, + name: str, + *, + maximum_bytes: int, + label: str, +) -> tuple[bytes, os.stat_result]: + """Read an owner-only regular leaf without following it or exceeding a bound.""" + + try: + descriptor = os.open( + name, + os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0), + dir_fd=directory_descriptor, + ) + except OSError as exc: + if exc.errno == errno.ELOOP: + raise ValueError(f"{label} must not be a symbolic link") from exc + raise + try: + metadata = _validate_regular_owner_only(descriptor, label=label) + chunks: list[bytes] = [] + remaining = maximum_bytes + 1 + while remaining > 0: + chunk = os.read(descriptor, min(64 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + raw = b"".join(chunks) + if len(raw) > maximum_bytes: + raise ValueError(f"{label} is too large") + return raw, metadata + finally: + os.close(descriptor) + + +def validate_leaf_for_replace(directory_descriptor: int, name: str, *, label: str) -> None: + """Reject an existing unsafe target before an atomic replacement.""" + + try: + descriptor = os.open( + name, + os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0), + dir_fd=directory_descriptor, + ) + except FileNotFoundError: + return + except OSError as exc: + if exc.errno == errno.ELOOP: + raise ValueError(f"{label} must not be a symbolic link") from exc + raise + try: + _validate_regular_owner_only(descriptor, label=label) + finally: + os.close(descriptor) + + +def atomic_replace_at( + directory_descriptor: int, + name: str, + data: bytes, + *, + temporary_prefix: str, + label: str, +) -> None: + """Write completely, fsync, and descriptor-relatively replace one safe leaf.""" + + if os.rename not in os.supports_dir_fd or os.unlink not in os.supports_dir_fd: + raise OSError("descriptor-relative Commons storage operations are unavailable") + validate_leaf_for_replace(directory_descriptor, name, label=label) + descriptor = -1 + temporary_name = "" + for _ in range(16): + temporary_name = f"{temporary_prefix}{_token_hex(12)}.tmp" + try: + descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_descriptor, + ) + except FileExistsError: + continue + break + if descriptor < 0: + raise FileExistsError("unable to allocate a private Commons temporary file") + renamed = False + try: + os.fchmod(descriptor, 0o600) + _write_all(descriptor, data) + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + os.rename( + temporary_name, + name, + src_dir_fd=directory_descriptor, + dst_dir_fd=directory_descriptor, + ) + renamed = True + os.fsync(directory_descriptor) + except BaseException: + if descriptor >= 0: + with suppress(OSError): + os.close(descriptor) + if not renamed: + with suppress(OSError): + os.unlink(temporary_name, dir_fd=directory_descriptor) + raise + + +def atomic_create_at( + directory_descriptor: int, + name: str, + data: bytes, + *, + temporary_prefix: str, + label: str, +) -> os.stat_result: + """Publish a complete private file atomically without overwriting a collision.""" + + if os.link not in os.supports_dir_fd or os.unlink not in os.supports_dir_fd: + raise OSError("descriptor-relative Commons create operations are unavailable") + descriptor = -1 + temporary_name = "" + for _ in range(16): + temporary_name = f"{temporary_prefix}{_token_hex(12)}.tmp" + try: + descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_descriptor, + ) + except FileExistsError: + continue + break + if descriptor < 0: + raise FileExistsError("unable to allocate a private Commons temporary file") + published = False + try: + os.fchmod(descriptor, 0o600) + _write_all(descriptor, data) + os.fsync(descriptor) + metadata = _validate_regular_owner_only(descriptor, label=label) + os.close(descriptor) + descriptor = -1 + os.link( + temporary_name, + name, + src_dir_fd=directory_descriptor, + dst_dir_fd=directory_descriptor, + follow_symlinks=False, + ) + published = True + os.unlink(temporary_name, dir_fd=directory_descriptor) + os.fsync(directory_descriptor) + return metadata + finally: + if descriptor >= 0: + with suppress(OSError): + os.close(descriptor) + if not published: + with suppress(OSError): + os.unlink(temporary_name, dir_fd=directory_descriptor) diff --git a/src/marginal/commons/cache.py b/src/marginal/commons/cache.py new file mode 100644 index 0000000..356a359 --- /dev/null +++ b/src/marginal/commons/cache.py @@ -0,0 +1,259 @@ +"""Verified, model-isolated local cache for deterministic Commons packs.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any + +from ._storage import atomic_replace_at, locked_directory, read_bounded_at +from .evidence import ( + ActionKind, + AggregateReasonCode, + DecisionClass, + OutcomeClass, + RecordType, + ValueBucket, +) +from .identity import is_canonical_namespace + +_PACK_NAME = "commons-pack-v1.json" +_MODEL_NAMESPACES = { + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna", +} +_MAX_PACK_BYTES = 2 * 1024 * 1024 + + +class CommonsLifecycle(str, Enum): + """Non-authoritative lifecycle label carried by a Commons prior.""" + + CANDIDATE = "candidate" + SUPPORTED = "supported" + VALIDATED = "validated" + PROMOTED = "promoted" + + +@dataclass(frozen=True, slots=True) +class CommonsPrior: + """One closed aggregate prior for exactly one canonical model namespace.""" + + model_namespace: str + record_type: RecordType + action_kind: ActionKind + cost_bucket: ValueBucket + gain_bucket: ValueBucket + recommendation: DecisionClass + applied_decision: DecisionClass + reason_code: AggregateReasonCode + outcome_class: OutcomeClass + count: int + minimum_group_size: int + lifecycle: CommonsLifecycle + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("Commons pack contains a duplicate field") + result[key] = value + return result + + +def _exact_keys(payload: object, expected: set[str]) -> bool: + return isinstance(payload, dict) and set(payload) == expected + + +def _positive_bounded_integer(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, int) and 1 <= value <= 1_000 + + +def _parse_aggregate(namespace: str, raw: object) -> CommonsPrior: + expected = { + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size", + "lifecycle", + } + if not _exact_keys(raw, expected): + raise ValueError("Commons aggregate has an invalid shape") + assert isinstance(raw, dict) + if not _positive_bounded_integer(raw["count"]) or not _positive_bounded_integer( + raw["minimum_group_size"] + ): + raise ValueError("Commons aggregate count is invalid") + try: + return CommonsPrior( + model_namespace=namespace, + record_type=RecordType(raw["record_type"]), + action_kind=ActionKind(raw["action_kind"]), + cost_bucket=ValueBucket(raw["cost_bucket"]), + gain_bucket=ValueBucket(raw["gain_bucket"]), + recommendation=DecisionClass(raw["recommendation"]), + applied_decision=DecisionClass(raw["applied_decision"]), + reason_code=AggregateReasonCode(raw["reason_code"]), + outcome_class=OutcomeClass(raw["outcome_class"]), + count=raw["count"], + minimum_group_size=raw["minimum_group_size"], + lifecycle=CommonsLifecycle(raw["lifecycle"]), + ) + except (TypeError, ValueError) as exc: + raise ValueError("Commons aggregate contains an invalid value") from exc + + +def _parse_pack(raw: bytes, *, expected_source_commit: str) -> dict[str, Any]: + try: + payload: Any = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Commons pack is not valid JSON") from exc + expected = { + "schema_version", + "source_commit", + "commons_revision", + "compatibility", + "models", + "integrity", + } + if not _exact_keys(payload, expected): + raise ValueError("Commons pack has an invalid shape") + assert isinstance(payload, dict) + revision = payload["commons_revision"] + if ( + payload["schema_version"] != "1.0" + or payload["source_commit"] != expected_source_commit + or isinstance(revision, bool) + or not isinstance(revision, int) + or revision < 1 + or payload["compatibility"] != {"evidence_envelope_schema_version": "1.0"} + ): + raise ValueError("Commons pack is incompatible") + models = payload["models"] + if not isinstance(models, dict) or set(models) != _MODEL_NAMESPACES: + raise ValueError("Commons pack model registry is invalid") + for namespace, model in models.items(): + if not _exact_keys(model, {"aggregates"}): + raise ValueError("Commons pack model has an invalid shape") + assert isinstance(model, dict) + aggregates = model["aggregates"] + if not isinstance(aggregates, list) or len(aggregates) > 10_000: + raise ValueError("Commons pack model aggregates are invalid") + for aggregate in aggregates: + _parse_aggregate(namespace, aggregate) + integrity = payload["integrity"] + if not _exact_keys(integrity, {"sha256"}): + raise ValueError("Commons pack integrity is invalid") + assert isinstance(integrity, dict) + digest = integrity["sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise ValueError("Commons pack integrity is invalid") + canonical_payload = {key: value for key, value in payload.items() if key != "integrity"} + canonical = json.dumps( + canonical_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + if not hashlib.sha256(canonical).hexdigest() == digest: + raise ValueError("Commons pack integrity mismatch") + return payload + + +class CommonsCache: + """Persist and load only verified priors for one exact canonical model.""" + + def __init__( + self, + data_dir: str | Path, + *, + model_namespace: str, + expected_source_commit: str, + max_pack_bytes: int = _MAX_PACK_BYTES, + ) -> None: + if not is_canonical_namespace(model_namespace): + raise ValueError("Commons cache requires a canonical model namespace") + if ( + not isinstance(expected_source_commit, str) + or len(expected_source_commit) != 40 + or any(character not in "0123456789abcdef" for character in expected_source_commit) + ): + raise ValueError("Commons cache requires an exact source commit") + if ( + isinstance(max_pack_bytes, bool) + or not isinstance(max_pack_bytes, int) + or max_pack_bytes < 1 + ): + raise ValueError("Commons cache byte limit must be positive") + root = Path(data_dir) + if ".." in root.parts: + raise ValueError("Commons cache path must not contain traversal") + self.model_namespace = model_namespace + self.expected_source_commit = expected_source_commit + self.max_pack_bytes = max_pack_bytes + self.path = ( + (root if root.is_absolute() else Path.cwd() / root) / "commons" / "cache" / _PACK_NAME + ) + + def refresh(self, raw: bytes) -> bool: + """Atomically replace the cache only when every frozen-pack check succeeds.""" + + if not isinstance(raw, bytes) or len(raw) > self.max_pack_bytes: + return False + try: + _parse_pack(raw, expected_source_commit=self.expected_source_commit) + with locked_directory( + self.path.parent, create=True, lock_name=".cache.lock" + ) as directory: + atomic_replace_at( + directory, + _PACK_NAME, + raw, + temporary_prefix=".commons-pack-", + label="Commons cache", + ) + except (OSError, ValueError): + return False + return True + + def _load_pack(self) -> dict[str, Any] | None: + try: + with locked_directory( + self.path.parent, create=False, lock_name=".cache.lock" + ) as directory: + raw, _ = read_bounded_at( + directory, + _PACK_NAME, + maximum_bytes=self.max_pack_bytes, + label="Commons cache", + ) + return _parse_pack(raw, expected_source_commit=self.expected_source_commit) + except (FileNotFoundError, OSError, ValueError): + return None + + def load_prior(self) -> tuple[CommonsPrior, ...]: + """Return only this cache instance's exact-model priors, or nothing on failure.""" + + payload = self._load_pack() + if payload is None: + return () + model = payload["models"][self.model_namespace] + return tuple(_parse_aggregate(self.model_namespace, raw) for raw in model["aggregates"]) + + @property + def revision(self) -> int | None: + """Return the verified cached revision without granting it any authority.""" + + payload = self._load_pack() + return payload["commons_revision"] if payload is not None else None diff --git a/src/marginal/commons/client.py b/src/marginal/commons/client.py new file mode 100644 index 0000000..8e956ac --- /dev/null +++ b/src/marginal/commons/client.py @@ -0,0 +1,203 @@ +"""Bounded zero-dependency HTTP transport for MARGINAL Commons.""" + +from __future__ import annotations + +import http.client +import json +from dataclasses import dataclass +from typing import Protocol +from urllib.parse import SplitResult, urlsplit + +from .outbox import _TOKEN_PATTERN, OutboxEntry, _validate_envelope + +_PACK_PATH = "/dist/commons-pack-v1.json" +_EVIDENCE_PATH = "/v1/evidence" +_DEFAULT_MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +_MAX_REQUEST_BYTES = 512 * 1024 + + +class CommonsTransportError(Exception): + """A redacted DNS, TLS, socket, or timeout failure.""" + + +class CommonsProtocolError(Exception): + """A bounded response failed the closed Commons response contract.""" + + +class CommonsHTTPError(Exception): + """A non-success HTTP status, without response content or endpoint details.""" + + def __init__(self, *, status: int) -> None: + self.status = status + category = "client" if 400 <= status < 500 else "server" + super().__init__(f"Commons request failed ({category} response)") + + +@dataclass(frozen=True, slots=True) +class CommonsAck: + """The only accepted evidence response shape.""" + + accepted: bool + duplicate: bool + + +class CommonsClientProtocol(Protocol): + """Narrow transport boundary consumed by fail-open orchestration.""" + + def download(self) -> bytes: ... + + def submit(self, entry: OutboxEntry) -> CommonsAck: ... + + +@dataclass(frozen=True, slots=True) +class _Origin: + scheme: str + host: str + port: int | None + + +def _parse_origin(value: str) -> _Origin: + if not isinstance(value, str): + raise ValueError("Commons origin must be an absolute HTTP origin") + parsed: SplitResult = urlsplit(value) + try: + port = parsed.port + except ValueError as exc: + raise ValueError("Commons origin is invalid") from exc + if ( + parsed.scheme not in {"http", "https"} + or parsed.hostname is None + or parsed.username is not None + or parsed.password is not None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + ): + raise ValueError("Commons origin must not contain credentials, paths, or query parameters") + if parsed.scheme == "http" and parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise ValueError("Commons origin must use TLS unless it is loopback") + return _Origin(parsed.scheme, parsed.hostname, port) + + +def _positive_timeout(value: float, *, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= 30: + raise ValueError(f"Commons {label} timeout must be between 0 and 30 seconds") + return float(value) + + +class CommonsClient: + """Issue only fixed-path GET and POST requests with strict resource limits.""" + + def __init__( + self, + *, + pack_origin: str, + ingress_origin: str, + connect_timeout: float = 2.0, + read_timeout: float = 3.0, + max_response_bytes: int = _DEFAULT_MAX_RESPONSE_BYTES, + ) -> None: + self._pack_origin = _parse_origin(pack_origin) + self._ingress_origin = _parse_origin(ingress_origin) + self._connect_timeout = _positive_timeout(connect_timeout, label="connect") + self._read_timeout = _positive_timeout(read_timeout, label="read") + if ( + isinstance(max_response_bytes, bool) + or not isinstance(max_response_bytes, int) + or not 1 <= max_response_bytes <= _DEFAULT_MAX_RESPONSE_BYTES + ): + raise ValueError("Commons response byte limit is invalid") + self._max_response_bytes = max_response_bytes + + def _connection(self, origin: _Origin) -> http.client.HTTPConnection: + connection_type: type[http.client.HTTPConnection] + connection_type = ( + http.client.HTTPSConnection if origin.scheme == "https" else http.client.HTTPConnection + ) + return connection_type(origin.host, port=origin.port, timeout=self._connect_timeout) + + def _request( + self, + origin: _Origin, + *, + method: str, + path: str, + body: bytes | None = None, + headers: dict[str, str] | None = None, + ) -> tuple[int, bytes]: + connection = self._connection(origin) + try: + connection.request(method, path, body=body, headers=headers or {}) + if connection.sock is not None: + connection.sock.settimeout(self._read_timeout) + response = connection.getresponse() + declared = response.getheader("Content-Length") + if declared is not None: + try: + declared_size = int(declared) + except ValueError: + raise CommonsProtocolError("invalid Commons response") from None + if declared_size < 0: + raise CommonsProtocolError("invalid Commons response") + raw = response.read(self._max_response_bytes + 1) + if len(raw) > self._max_response_bytes: + raise CommonsProtocolError("invalid Commons response") + return response.status, raw + except CommonsProtocolError: + raise + except (TimeoutError, OSError, http.client.HTTPException): + raise CommonsTransportError("Commons transport failed") from None + finally: + connection.close() + + def download(self) -> bytes: + """Download the pack from its fixed public path.""" + + status, raw = self._request( + self._pack_origin, + method="GET", + path=_PACK_PATH, + headers={"Accept": "application/json"}, + ) + if status != 200: + raise CommonsHTTPError(status=status) + return raw + + def submit(self, entry: OutboxEntry) -> CommonsAck: + """Submit one validated envelope with retry identity only in its header.""" + + if not isinstance(entry, OutboxEntry): + raise TypeError("Commons submission requires an outbox entry") + if _TOKEN_PATTERN.fullmatch(entry.retry_token) is None: + raise ValueError("Commons retry token is invalid") + envelope = _validate_envelope(entry.envelope) + body = ( + json.dumps(envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + ).encode("utf-8") + if len(body) > _MAX_REQUEST_BYTES: + raise ValueError("Commons evidence envelope is too large") + status, raw = self._request( + self._ingress_origin, + method="POST", + path=_EVIDENCE_PATH, + body=body, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "Idempotency-Key": entry.retry_token, + }, + ) + if not 200 <= status < 300: + raise CommonsHTTPError(status=status) + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise CommonsProtocolError("invalid Commons response") from None + if ( + not isinstance(payload, dict) + or set(payload) != {"accepted", "duplicate"} + or payload.get("accepted") is not True + or type(payload.get("duplicate")) is not bool + ): + raise CommonsProtocolError("invalid Commons response") + return CommonsAck(accepted=True, duplicate=payload["duplicate"]) diff --git a/src/marginal/commons/outbox.py b/src/marginal/commons/outbox.py new file mode 100644 index 0000000..94bc9f1 --- /dev/null +++ b/src/marginal/commons/outbox.py @@ -0,0 +1,288 @@ +"""Durable owner-only queue for closed Commons evidence envelopes.""" + +from __future__ import annotations + +import json +import os +import re +import secrets +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ._storage import atomic_create_at, locked_directory, read_bounded_at +from .evidence import ( + ActionKind, + AggregateReasonCode, + CommonsEvidenceAtom, + DecisionClass, + OutcomeClass, + RecordType, + ValueBucket, +) +from .identity import is_canonical_namespace + +_MAX_QUEUE_BYTES = 512 * 1024 +_MAX_ATOMS = 1_000 +_TOKEN_PATTERN = re.compile(r"^[A-Za-z0-9_-]{43}$") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("queued Commons record contains a duplicate field") + result[key] = value + return result + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + """One validated queued envelope plus local-only retry metadata.""" + + name: str + retry_token: str + envelope: dict[str, object] + device: int + inode: int + + def body(self) -> bytes: + """Serialize only the closed wire envelope, excluding local metadata.""" + + return ( + json.dumps(self.envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + "\n" + ).encode("utf-8") + + +@dataclass(frozen=True, slots=True) +class OutboxScan: + """Bounded valid entries plus the count of malformed leaves quarantined.""" + + entries: tuple[OutboxEntry, ...] + quarantined: int = 0 + + +def _atom_from_mapping(raw: object) -> CommonsEvidenceAtom: + expected = { + "record_type", + "action_kind", + "cost_bucket", + "gain_bucket", + "recommendation", + "applied_decision", + "reason_code", + "outcome_class", + "count", + "minimum_group_size", + } + if not isinstance(raw, dict) or set(raw) != expected: + raise ValueError("queued Commons atom has an invalid shape") + try: + return CommonsEvidenceAtom( + record_type=RecordType(raw["record_type"]), + action_kind=ActionKind(raw["action_kind"]), + cost_bucket=ValueBucket(raw["cost_bucket"]), + gain_bucket=ValueBucket(raw["gain_bucket"]), + recommendation=DecisionClass(raw["recommendation"]), + applied_decision=DecisionClass(raw["applied_decision"]), + reason_code=AggregateReasonCode(raw["reason_code"]), + outcome_class=OutcomeClass(raw["outcome_class"]), + count=raw["count"], + minimum_group_size=raw["minimum_group_size"], + ) + except (TypeError, ValueError) as exc: + raise ValueError("queued Commons atom has an invalid value") from exc + + +def _validate_envelope(raw: object) -> dict[str, object]: + if not isinstance(raw, dict) or set(raw) != {"schema_version", "model_namespace", "atoms"}: + raise ValueError("queued Commons envelope has an invalid shape") + namespace = raw["model_namespace"] + atoms = raw["atoms"] + if raw["schema_version"] != "1.0" or not is_canonical_namespace(namespace): + raise ValueError("queued Commons envelope is incompatible") + if not isinstance(atoms, list) or not 1 <= len(atoms) <= _MAX_ATOMS: + raise ValueError("queued Commons envelope must contain bounded evidence") + parsed = [_atom_from_mapping(atom).to_dict() for atom in atoms] + return {"schema_version": "1.0", "model_namespace": namespace, "atoms": parsed} + + +def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> OutboxEntry: + try: + payload: Any = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("queued Commons record is malformed") from exc + if not isinstance(payload, dict) or set(payload) != {"retry_token", "envelope"}: + raise ValueError("queued Commons record has an invalid shape") + retry_token = payload["retry_token"] + if not isinstance(retry_token, str) or _TOKEN_PATTERN.fullmatch(retry_token) is None: + raise ValueError("queued Commons retry token is invalid") + envelope = _validate_envelope(payload["envelope"]) + return OutboxEntry(name, retry_token, envelope, device, inode) + + +class CommonsOutbox: + """Queue, recover, acknowledge, and quarantine exact evidence files.""" + + def __init__(self, data_dir: str | Path) -> None: + root = Path(data_dir) + if ".." in root.parts: + raise ValueError("Commons outbox path must not contain traversal") + absolute = root if root.is_absolute() else Path.cwd() / root + base = absolute / "commons" / "outbox" + self.queue_path = base / "queue" + self.quarantine_path = base / "quarantine" + + def enqueue( + self, + *, + model_namespace: str, + atoms: tuple[CommonsEvidenceAtom, ...], + ) -> OutboxEntry | None: + """Atomically queue nonempty typed evidence with one random retry token.""" + + if not is_canonical_namespace(model_namespace) or not atoms: + return None + if len(atoms) > _MAX_ATOMS or not all( + isinstance(atom, CommonsEvidenceAtom) for atom in atoms + ): + return None + envelope: dict[str, object] = { + "schema_version": "1.0", + "model_namespace": model_namespace, + "atoms": [atom.to_dict() for atom in atoms], + } + retry_token = secrets.token_urlsafe(32) + record = {"envelope": envelope, "retry_token": retry_token} + encoded = ( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + ).encode("utf-8") + if len(encoded) > _MAX_QUEUE_BYTES: + return None + with locked_directory(self.queue_path, create=True, lock_name=".outbox.lock") as directory: + for _ in range(16): + name = f"queue-{secrets.token_hex(16)}.json" + try: + metadata = atomic_create_at( + directory, + name, + encoded, + temporary_prefix=".outbox-", + label="Commons outbox entry", + ) + except FileExistsError: + continue + return OutboxEntry( + name=name, + retry_token=retry_token, + envelope=envelope, + device=metadata.st_dev, + inode=metadata.st_ino, + ) + raise FileExistsError("unable to allocate a unique Commons outbox entry") + + def _quarantine_name(self, queue_descriptor: int, name: str) -> bool: + with locked_directory( + self.quarantine_path, create=True, lock_name=".quarantine.lock" + ) as quarantine_descriptor: + for _ in range(16): + target = f"quarantine-{secrets.token_hex(16)}.json" + try: + os.stat(target, dir_fd=quarantine_descriptor, follow_symlinks=False) + except FileNotFoundError: + pass + else: + continue + try: + os.rename( + name, + target, + src_dir_fd=queue_descriptor, + dst_dir_fd=quarantine_descriptor, + ) + except FileNotFoundError: + return False + os.fsync(queue_descriptor) + os.fsync(quarantine_descriptor) + return True + raise FileExistsError("unable to allocate a Commons quarantine entry") + + def pending(self, *, limit: int = 8) -> OutboxScan: + """Return bounded valid work and quarantine malformed leaves during the scan.""" + + if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1 or limit > 1_000: + raise ValueError("Commons outbox scan limit must be between 1 and 1000") + try: + context = locked_directory(self.queue_path, create=False, lock_name=".outbox.lock") + with context as directory: + entries: list[OutboxEntry] = [] + quarantined = 0 + names = sorted( + name + for name in os.listdir(directory) + if isinstance(name, str) and not name.startswith(".") + ) + for name in names[: limit * 4 + 16]: + try: + raw, metadata = read_bounded_at( + directory, + name, + maximum_bytes=_MAX_QUEUE_BYTES, + label="Commons outbox entry", + ) + entry = _parse_queue_record( + raw, name=name, device=metadata.st_dev, inode=metadata.st_ino + ) + except (OSError, ValueError): + quarantined += int(self._quarantine_name(directory, name)) + continue + if len(entries) < limit: + entries.append(entry) + return OutboxScan(tuple(entries), quarantined) + except FileNotFoundError: + return OutboxScan(()) + + def _matches(self, directory: int, entry: OutboxEntry) -> bool: + try: + _, metadata = read_bounded_at( + directory, + entry.name, + maximum_bytes=_MAX_QUEUE_BYTES, + label="Commons outbox entry", + ) + except (FileNotFoundError, OSError, ValueError): + return False + return (metadata.st_dev, metadata.st_ino) == (entry.device, entry.inode) + + def ack(self, entry: OutboxEntry) -> bool: + """Delete only the exact inode whose valid envelope received an ACK.""" + + if not isinstance(entry, OutboxEntry): + return False + try: + with locked_directory( + self.queue_path, create=False, lock_name=".outbox.lock" + ) as directory: + if not self._matches(directory, entry): + return False + os.unlink(entry.name, dir_fd=directory) + os.fsync(directory) + return True + except FileNotFoundError: + return False + + def quarantine(self, entry: OutboxEntry) -> bool: + """Move only the exact inode to the owner-only quarantine directory.""" + + if not isinstance(entry, OutboxEntry): + return False + try: + with locked_directory( + self.queue_path, create=False, lock_name=".outbox.lock" + ) as directory: + if not self._matches(directory, entry): + return False + return self._quarantine_name(directory, entry.name) + except FileNotFoundError: + return False diff --git a/src/marginal/commons/sync.py b/src/marginal/commons/sync.py new file mode 100644 index 0000000..f083fdf --- /dev/null +++ b/src/marginal/commons/sync.py @@ -0,0 +1,165 @@ +"""Bounded fail-open orchestration for optional Commons network modes.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from .cache import CommonsCache +from .client import ( + CommonsClientProtocol, + CommonsHTTPError, + CommonsProtocolError, + CommonsTransportError, +) +from .config import CommonsConfig, CommonsMode +from .evidence import CommonsEvidenceAtom +from .outbox import CommonsOutbox + + +class SyncFailure(str, Enum): + """Closed local diagnostic categories that never contain raw error details.""" + + DOWNLOAD_HTTP = "download_http" + DOWNLOAD_PROTOCOL = "download_protocol" + DOWNLOAD_TRANSPORT = "download_transport" + CACHE_REJECTED = "cache_rejected" + OUTBOX_WRITE = "outbox_write" + OUTBOX_READ = "outbox_read" + SUBMIT_PROTOCOL = "submit_protocol" + SUBMIT_TRANSPORT = "submit_transport" + OUTBOX_TRANSITION = "outbox_transition" + + +@dataclass(frozen=True, slots=True) +class CommonsSyncResult: + """Fail-open bounded outcomes with no endpoint, evidence, or exception text.""" + + network_calls: int = 0 + cache_refreshed: bool = False + submitted: int = 0 + acked: int = 0 + quarantined: int = 0 + retained: int = 0 + failures: tuple[SyncFailure, ...] = () + + +def synchronize_commons( + config: CommonsConfig, + *, + cache: CommonsCache, + outbox: CommonsOutbox, + client: CommonsClientProtocol, + model_namespace: str | None = None, + atoms: tuple[CommonsEvidenceAtom, ...] = (), + max_submissions: int = 8, +) -> CommonsSyncResult: + """Download and optionally contribute without allowing any failure to block local work.""" + + if not isinstance(config, CommonsConfig): + raise TypeError("Commons sync requires a CommonsConfig") + if ( + isinstance(max_submissions, bool) + or not isinstance(max_submissions, int) + or not 1 <= max_submissions <= 100 + ): + raise ValueError("Commons submission bound must be between 1 and 100") + if config.mode is CommonsMode.LOCAL_ONLY: + return CommonsSyncResult() + + network_calls = 1 + cache_refreshed = False + submitted = 0 + acked = 0 + quarantined = 0 + retained = 0 + failures: list[SyncFailure] = [] + try: + pack = client.download() + except CommonsHTTPError: + failures.append(SyncFailure.DOWNLOAD_HTTP) + except CommonsProtocolError: + failures.append(SyncFailure.DOWNLOAD_PROTOCOL) + except (CommonsTransportError, OSError, TimeoutError): + failures.append(SyncFailure.DOWNLOAD_TRANSPORT) + except Exception: + failures.append(SyncFailure.DOWNLOAD_TRANSPORT) + else: + try: + cache_refreshed = cache.refresh(pack) + except Exception: + cache_refreshed = False + if not cache_refreshed: + failures.append(SyncFailure.CACHE_REJECTED) + + if config.mode is CommonsMode.READ_ONLY: + return CommonsSyncResult( + network_calls=network_calls, + cache_refreshed=cache_refreshed, + failures=tuple(failures), + ) + + if model_namespace is not None and atoms: + try: + outbox.enqueue(model_namespace=model_namespace, atoms=atoms) + except Exception: + failures.append(SyncFailure.OUTBOX_WRITE) + + try: + scan = outbox.pending(limit=max_submissions) + except Exception: + failures.append(SyncFailure.OUTBOX_READ) + return CommonsSyncResult( + network_calls=network_calls, + cache_refreshed=cache_refreshed, + failures=tuple(failures), + ) + quarantined += scan.quarantined + + for entry in scan.entries: + network_calls += 1 + submitted += 1 + try: + client.submit(entry) + except CommonsHTTPError as exc: + if 400 <= exc.status < 500: + try: + moved = outbox.quarantine(entry) + except Exception: + moved = False + if moved: + quarantined += 1 + else: + retained += 1 + failures.append(SyncFailure.OUTBOX_TRANSITION) + else: + retained += 1 + except CommonsProtocolError: + retained += 1 + failures.append(SyncFailure.SUBMIT_PROTOCOL) + except (CommonsTransportError, OSError, TimeoutError): + retained += 1 + failures.append(SyncFailure.SUBMIT_TRANSPORT) + except Exception: + retained += 1 + failures.append(SyncFailure.SUBMIT_TRANSPORT) + else: + try: + deleted = outbox.ack(entry) + except Exception: + deleted = False + if deleted: + acked += 1 + else: + retained += 1 + failures.append(SyncFailure.OUTBOX_TRANSITION) + + return CommonsSyncResult( + network_calls=network_calls, + cache_refreshed=cache_refreshed, + submitted=submitted, + acked=acked, + quarantined=quarantined, + retained=retained, + failures=tuple(failures), + ) diff --git a/tests/commons/test_cache.py b/tests/commons/test_cache.py new file mode 100644 index 0000000..96a52ca --- /dev/null +++ b/tests/commons/test_cache.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import hashlib +import json +import os +import stat +from pathlib import Path + +import pytest + +from marginal.commons import _storage as storage_module +from marginal.commons.cache import CommonsCache + +SOURCE_COMMIT = "a" * 40 +MODEL_NAMESPACE = "openai/gpt-5.6-sol" + + +def _aggregate(*, count: int = 7) -> dict[str, object]: + return { + "record_type": "decision", + "action_kind": "test", + "cost_bucket": "low", + "gain_bucket": "high", + "recommendation": "allow", + "applied_decision": "allow", + "reason_code": "APPROVED", + "outcome_class": "not_applicable", + "count": count, + "minimum_group_size": 5, + "lifecycle": "candidate", + } + + +def _pack_bytes( + *, + count: int = 7, + source_commit: str = SOURCE_COMMIT, + revision: int = 1, + compatibility: str = "1.0", + extra: tuple[str, object] | None = None, +) -> bytes: + payload: dict[str, object] = { + "schema_version": "1.0", + "source_commit": source_commit, + "commons_revision": revision, + "compatibility": {"evidence_envelope_schema_version": compatibility}, + "models": { + "openai/gpt-5.6-sol": {"aggregates": [_aggregate(count=count)]}, + "openai/gpt-5.6-terra": {"aggregates": []}, + "openai/gpt-5.6-luna": {"aggregates": []}, + }, + } + if extra is not None: + payload[extra[0]] = extra[1] + canonical = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + payload["integrity"] = {"sha256": hashlib.sha256(canonical).hexdigest()} + return ( + json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + ).encode("utf-8") + + +def _cache(tmp_path: Path) -> CommonsCache: + return CommonsCache( + tmp_path, + model_namespace=MODEL_NAMESPACE, + expected_source_commit=SOURCE_COMMIT, + ) + + +def test_refresh_loads_only_the_selected_model_from_a_canonical_pack(tmp_path: Path) -> None: + cache = _cache(tmp_path) + + assert cache.refresh(_pack_bytes()) is True + + priors = cache.load_prior() + assert len(priors) == 1 + assert priors[0].model_namespace == MODEL_NAMESPACE + assert priors[0].action_kind.value == "test" + assert priors[0].count == 7 + assert priors[0].lifecycle.value == "candidate" + assert cache.revision == 1 + assert stat.S_IMODE(cache.path.stat().st_mode) == 0o600 + + +@pytest.mark.parametrize( + "candidate", + [ + b"not-json", + _pack_bytes(revision=0), + _pack_bytes(compatibility="2.0"), + _pack_bytes(source_commit="b" * 40), + _pack_bytes(extra=("privacy-canary", "customer-acme")), + ], +) +def test_rejected_refresh_preserves_and_uses_the_last_valid_pack( + tmp_path: Path, candidate: bytes +) -> None: + cache = _cache(tmp_path) + assert cache.refresh(_pack_bytes(count=7)) is True + before = cache.path.read_bytes() + + assert cache.refresh(candidate) is False + + assert cache.path.read_bytes() == before + assert [prior.count for prior in cache.load_prior()] == [7] + + +def test_digest_is_over_canonical_payload_and_detects_post_digest_mutation(tmp_path: Path) -> None: + cache = _cache(tmp_path) + parsed = json.loads(_pack_bytes()) + parsed["models"][MODEL_NAMESPACE]["aggregates"][0]["count"] = 999 + attacked = json.dumps(parsed, separators=(",", ":")).encode("utf-8") + + assert cache.refresh(attacked) is False + assert cache.load_prior() == () + + +def test_refresh_rejects_oversized_and_cross_model_or_incomplete_packs(tmp_path: Path) -> None: + cache = _cache(tmp_path) + missing_model = json.loads(_pack_bytes()) + missing_model["models"].pop("openai/gpt-5.6-terra") + without_integrity = {key: value for key, value in missing_model.items() if key != "integrity"} + canonical = json.dumps(without_integrity, sort_keys=True, separators=(",", ":")).encode() + missing_model["integrity"] = {"sha256": hashlib.sha256(canonical).hexdigest()} + + assert cache.refresh(json.dumps(missing_model).encode()) is False + assert cache.refresh(b"{" + b" " * (cache.max_pack_bytes + 1)) is False + + +def test_cache_rejects_symlink_leaf_without_touching_its_target(tmp_path: Path) -> None: + cache = _cache(tmp_path) + cache.path.parent.mkdir(parents=True) + outside = tmp_path / "outside.json" + outside.write_bytes(b"outside") + cache.path.symlink_to(outside) + + assert cache.refresh(_pack_bytes()) is False + assert outside.read_bytes() == b"outside" + + +def test_atomic_cache_write_retries_short_writes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache = _cache(tmp_path) + original_write = os.write + + def short_write(descriptor: int, data: bytes) -> int: + return original_write(descriptor, data[: max(1, len(data) // 3)]) + + monkeypatch.setattr(storage_module.os, "write", short_write) + + assert cache.refresh(_pack_bytes()) is True + assert [prior.count for prior in cache.load_prior()] == [7] + + +def test_partial_cache_write_failure_keeps_previous_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache = _cache(tmp_path) + assert cache.refresh(_pack_bytes(count=7)) is True + before = cache.path.read_bytes() + + def fail_after_partial_write(descriptor: int, data: bytes) -> None: + os.write(descriptor, data[: len(data) // 2]) + raise OSError("synthetic partial write") + + monkeypatch.setattr(storage_module, "_write_all", fail_after_partial_write) + + assert cache.refresh(_pack_bytes(count=8)) is False + assert cache.path.read_bytes() == before + assert [prior.count for prior in cache.load_prior()] == [7] diff --git a/tests/commons/test_client.py b/tests/commons/test_client.py new file mode 100644 index 0000000..ace85ae --- /dev/null +++ b/tests/commons/test_client.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import ClassVar + +import pytest + +from marginal.commons.client import ( + CommonsClient, + CommonsHTTPError, + CommonsProtocolError, + CommonsTransportError, +) +from marginal.commons.evidence import ( + ActionKind, + AggregateReasonCode, + CommonsEvidenceAtom, + DecisionClass, + OutcomeClass, + RecordType, + ValueBucket, +) +from marginal.commons.outbox import CommonsOutbox, OutboxEntry + + +class _Handler(BaseHTTPRequestHandler): + requests: ClassVar[list[dict[str, object]]] = [] + response_status: ClassVar[int] = 200 + response_body: ClassVar[bytes] = b"{}" + response_delay: ClassVar[float] = 0.0 + + def do_GET(self) -> None: + type(self).requests.append( + {"method": "GET", "path": self.path, "headers": dict(self.headers)} + ) + self.send_response(type(self).response_status) + self.send_header("Content-Length", str(len(type(self).response_body))) + self.end_headers() + time.sleep(type(self).response_delay) + try: + self.wfile.write(type(self).response_body) + except BrokenPipeError: + return + + def do_POST(self) -> None: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + type(self).requests.append( + {"method": "POST", "path": self.path, "headers": dict(self.headers), "body": body} + ) + self.send_response(type(self).response_status) + self.send_header("Content-Length", str(len(type(self).response_body))) + self.end_headers() + time.sleep(type(self).response_delay) + try: + self.wfile.write(type(self).response_body) + except BrokenPipeError: + return + + def log_message(self, _format: str, *_args: object) -> None: + return + + +@contextmanager +def _server(*, status: int = 200, body: bytes = b"{}", delay: float = 0.0) -> Iterator[str]: + _Handler.requests = [] + _Handler.response_status = status + _Handler.response_body = body + _Handler.response_delay = delay + server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +def _entry(tmp_path: Path) -> OutboxEntry: + atom = CommonsEvidenceAtom( + record_type=RecordType.DECISION, + action_kind=ActionKind.TEST, + cost_bucket=ValueBucket.LOW, + gain_bucket=ValueBucket.HIGH, + recommendation=DecisionClass.ALLOW, + applied_decision=DecisionClass.ALLOW, + reason_code=AggregateReasonCode.APPROVED, + outcome_class=OutcomeClass.NOT_APPLICABLE, + count=7, + minimum_group_size=5, + ) + entry = CommonsOutbox(tmp_path).enqueue(model_namespace="openai/gpt-5.6-sol", atoms=(atom,)) + assert entry is not None + return entry + + +def test_download_uses_only_the_fixed_pack_path_without_query_or_tracking_headers() -> None: + with _server(body=b"pack") as origin: + client = CommonsClient(pack_origin=origin, ingress_origin=origin) + + assert client.download() == b"pack" + + request = _Handler.requests[0] + assert request["path"] == "/dist/commons-pack-v1.json" + headers = {key.lower(): value for key, value in request["headers"].items()} + assert "cookie" not in headers + assert "referer" not in headers + assert "x-request-id" not in headers + + +def test_submit_sends_only_the_closed_envelope_and_retry_header(tmp_path: Path) -> None: + response = json.dumps({"accepted": True, "duplicate": False}).encode() + with _server(body=response) as origin: + entry = _entry(tmp_path) + ack = CommonsClient(pack_origin=origin, ingress_origin=origin).submit(entry) + + assert ack.accepted is True + assert ack.duplicate is False + request = _Handler.requests[0] + assert request["path"] == "/v1/evidence" + assert json.loads(request["body"]) == entry.envelope + assert request["headers"]["Idempotency-Key"] == entry.retry_token + assert entry.retry_token.encode() not in request["body"] + + +@pytest.mark.parametrize( + "body", + [ + b"not-json", + b'{"accepted":true}', + b'{"accepted":true,"duplicate":false,"extra":true}', + b'{"accepted":1,"duplicate":false}', + ], +) +def test_submit_requires_the_exact_ack_shape(tmp_path: Path, body: bytes) -> None: + with ( + _server(body=body) as origin, + pytest.raises(CommonsProtocolError, match="invalid Commons response"), + ): + CommonsClient(pack_origin=origin, ingress_origin=origin).submit(_entry(tmp_path)) + + +def test_invalid_response_errors_do_not_retain_raw_body_details(tmp_path: Path) -> None: + with ( + _server(body=b"privacy-canary-not-json") as origin, + pytest.raises(CommonsProtocolError) as captured, + ): + CommonsClient(pack_origin=origin, ingress_origin=origin).submit(_entry(tmp_path)) + + assert "privacy-canary" not in str(captured.value) + assert captured.value.__cause__ is None + + +def test_read_timeout_is_separate_and_redacted() -> None: + with ( + _server(body=b"late", delay=0.1) as origin, + pytest.raises(CommonsTransportError, match="Commons transport failed") as captured, + ): + CommonsClient( + pack_origin=origin, + ingress_origin=origin, + connect_timeout=1.0, + read_timeout=0.01, + ).download() + + assert "timed out" not in str(captured.value).lower() + + +def test_client_bounds_response_bytes_and_reports_only_status_category(tmp_path: Path) -> None: + with _server(body=b"x" * 65) as origin: + client = CommonsClient(pack_origin=origin, ingress_origin=origin, max_response_bytes=64) + with pytest.raises(CommonsProtocolError, match="invalid Commons response"): + client.download() + + with ( + _server(status=503, body=b"privacy-canary-secret-error") as origin, + pytest.raises(CommonsHTTPError) as captured, + ): + CommonsClient(pack_origin=origin, ingress_origin=origin).submit(_entry(tmp_path)) + assert captured.value.status == 503 + assert "privacy-canary" not in str(captured.value) + + +@pytest.mark.parametrize( + "origin", + [ + "http://example.test", + "https://example.test/base", + "https://example.test?tracking=yes", + "https://user:secret@example.test", + "ftp://example.test", + ], +) +def test_client_rejects_origins_that_could_change_fixed_request_targets(origin: str) -> None: + with pytest.raises(ValueError, match="origin"): + CommonsClient(pack_origin=origin, ingress_origin="https://example.test") diff --git a/tests/commons/test_outbox.py b/tests/commons/test_outbox.py new file mode 100644 index 0000000..cc10ca7 --- /dev/null +++ b/tests/commons/test_outbox.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import json +import multiprocessing +import os +import re +import stat +import threading +from pathlib import Path + +import pytest + +from marginal.commons.evidence import ( + ActionKind, + AggregateReasonCode, + CommonsEvidenceAtom, + DecisionClass, + OutcomeClass, + RecordType, + ValueBucket, +) +from marginal.commons.outbox import CommonsOutbox + +MODEL_NAMESPACE = "openai/gpt-5.6-sol" + + +def _atom() -> CommonsEvidenceAtom: + return CommonsEvidenceAtom( + record_type=RecordType.DECISION, + action_kind=ActionKind.TEST, + cost_bucket=ValueBucket.LOW, + gain_bucket=ValueBucket.HIGH, + recommendation=DecisionClass.ALLOW, + applied_decision=DecisionClass.ALLOW, + reason_code=AggregateReasonCode.APPROVED, + outcome_class=OutcomeClass.NOT_APPLICABLE, + count=7, + minimum_group_size=5, + ) + + +def _enqueue_child(data_dir: str, results: multiprocessing.Queue[str]) -> None: + entry = CommonsOutbox(data_dir).enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + results.put(entry.name if entry is not None else "") + + +def test_enqueue_is_private_atomic_and_keeps_retry_identity_outside_evidence( + tmp_path: Path, +) -> None: + outbox = CommonsOutbox(tmp_path) + + entry = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + + assert entry is not None + assert re.fullmatch(r"[A-Za-z0-9_-]{43}", entry.retry_token) + assert "retry" not in json.dumps(entry.envelope, sort_keys=True).lower() + raw = json.loads((outbox.queue_path / entry.name).read_text(encoding="utf-8")) + assert raw == {"envelope": entry.envelope, "retry_token": entry.retry_token} + assert stat.S_IMODE((outbox.queue_path / entry.name).stat().st_mode) == 0o600 + assert stat.S_IMODE(outbox.queue_path.stat().st_mode) == 0o700 + + +def test_restart_recovers_the_same_one_time_retry_token_and_ack_deletes_exact_entry( + tmp_path: Path, +) -> None: + first = CommonsOutbox(tmp_path) + queued = first.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + assert queued is not None + + restarted = CommonsOutbox(tmp_path) + pending = restarted.pending(limit=8) + + assert len(pending.entries) == 1 + assert pending.entries[0].retry_token == queued.retry_token + assert restarted.ack(pending.entries[0]) is True + assert restarted.pending(limit=8).entries == () + + +def test_empty_or_unregistered_evidence_is_never_queued(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + + assert outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=()) is None + assert outbox.enqueue(model_namespace="private/customer-model", atoms=(_atom(),)) is None + assert not outbox.queue_path.exists() + + +def test_pending_quarantines_malformed_files_without_following_symlinks(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + assert queued is not None + malformed = outbox.queue_path / "queue-malformed.json" + malformed.write_text('{"retry_token":"privacy-canary"}', encoding="utf-8") + malformed.chmod(0o600) + outside = tmp_path / "outside" + outside.write_text("do-not-read", encoding="utf-8") + symlink = outbox.queue_path / "queue-symlink.json" + symlink.symlink_to(outside) + + scan = outbox.pending(limit=8) + + assert [entry.name for entry in scan.entries] == [queued.name] + assert scan.quarantined == 2 + assert outside.read_text(encoding="utf-8") == "do-not-read" + assert not malformed.exists() + assert not symlink.exists() + assert ( + len([path for path in outbox.quarantine_path.iterdir() if not path.name.startswith(".")]) + == 2 + ) + + +def test_pending_quarantines_duplicate_json_fields_instead_of_accepting_last_value( + tmp_path: Path, +) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + assert queued is not None + path = outbox.queue_path / queued.name + raw = path.read_text(encoding="utf-8") + attacked = raw.replace( + '"schema_version":"1.0"', + '"schema_version":"2.0","schema_version":"1.0"', + ) + path.write_text(attacked, encoding="utf-8") + path.chmod(0o600) + + scan = outbox.pending(limit=8) + + assert scan.entries == () + assert scan.quarantined == 1 + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFO leaves are POSIX-specific") +def test_pending_rejects_a_fifo_leaf_without_blocking_for_a_writer(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + assert queued is not None + fifo = outbox.queue_path / "queue-fifo.json" + os.mkfifo(fifo, mode=0o600) + completed = threading.Event() + + def scan() -> None: + outbox.pending(limit=8) + completed.set() + + worker = threading.Thread(target=scan, daemon=True) + worker.start() + returned_without_writer = completed.wait(timeout=0.2) + if not returned_without_writer: + writer = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK) + os.close(writer) + worker.join(timeout=1) + + assert returned_without_writer is True + + +def test_ack_refuses_a_different_inode_reusing_the_same_name(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + assert queued is not None + path = outbox.queue_path / queued.name + path.unlink() + path.write_text("replacement", encoding="utf-8") + path.chmod(0o600) + + assert outbox.ack(queued) is False + assert path.read_text(encoding="utf-8") == "replacement" + + +def test_enqueue_handles_random_name_collision_without_overwrite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + outbox = CommonsOutbox(tmp_path) + tokens = iter(["same-name", "same-name", "different-name"]) + monkeypatch.setattr("marginal.commons.outbox.secrets.token_hex", lambda _size: next(tokens)) + + first = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + second = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + + assert first is not None and second is not None + assert first.name != second.name + assert len(outbox.pending(limit=8).entries) == 2 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permissions are required") +def test_outbox_rejects_weak_existing_queue_permissions(tmp_path: Path) -> None: + queue = tmp_path / "commons" / "outbox" / "queue" + queue.mkdir(parents=True) + queue.chmod(0o755) + + with pytest.raises(PermissionError, match="owner-only"): + CommonsOutbox(tmp_path).pending(limit=1) + + +def test_outbox_rejects_a_symlinked_queue_directory_without_writing_outside(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + outbox.queue_path.parent.mkdir(parents=True) + outside = tmp_path / "outside-queue" + outside.mkdir() + outbox.queue_path.symlink_to(outside, target_is_directory=True) + + with pytest.raises((OSError, ValueError)): + outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + + assert list(outside.iterdir()) == [] + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX process locks are required") +def test_concurrent_processes_publish_complete_unique_entries(tmp_path: Path) -> None: + context = multiprocessing.get_context("fork") + results: multiprocessing.Queue[str] = context.Queue() + processes = [ + context.Process(target=_enqueue_child, args=(str(tmp_path), results)) for _ in range(6) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=5) + + assert all(process.exitcode == 0 for process in processes) + names = [results.get(timeout=1) for _ in processes] + assert len(set(names)) == 6 + assert len(CommonsOutbox(tmp_path).pending(limit=8).entries) == 6 diff --git a/tests/commons/test_sync.py b/tests/commons/test_sync.py new file mode 100644 index 0000000..81e91a9 --- /dev/null +++ b/tests/commons/test_sync.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import marginal.commons as commons +from marginal.commons.cache import CommonsCache +from marginal.commons.client import CommonsAck, CommonsHTTPError, CommonsProtocolError +from marginal.commons.config import CommonsConfig, CommonsMode +from marginal.commons.evidence import ( + ActionKind, + AggregateReasonCode, + CommonsEvidenceAtom, + DecisionClass, + OutcomeClass, + RecordType, + ValueBucket, +) +from marginal.commons.outbox import CommonsOutbox, OutboxEntry +from marginal.commons.sync import SyncFailure, synchronize_commons + +MODEL_NAMESPACE = "openai/gpt-5.6-sol" +SOURCE_COMMIT = "a" * 40 + + +def _pack_bytes() -> bytes: + payload: dict[str, object] = { + "schema_version": "1.0", + "source_commit": SOURCE_COMMIT, + "commons_revision": 1, + "compatibility": {"evidence_envelope_schema_version": "1.0"}, + "models": { + MODEL_NAMESPACE: {"aggregates": []}, + "openai/gpt-5.6-terra": {"aggregates": []}, + "openai/gpt-5.6-luna": {"aggregates": []}, + }, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + payload["integrity"] = {"sha256": hashlib.sha256(canonical).hexdigest()} + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + + +class _RecordingClient: + def __init__(self, *, pack: bytes | Exception, submit: CommonsAck | Exception) -> None: + self.pack = pack + self.submit_result = submit + self.download_calls = 0 + self.submitted: list[OutboxEntry] = [] + + def download(self) -> bytes: + self.download_calls += 1 + if isinstance(self.pack, Exception): + raise self.pack + return self.pack + + def submit(self, entry: OutboxEntry) -> CommonsAck: + self.submitted.append(entry) + if isinstance(self.submit_result, Exception): + raise self.submit_result + return self.submit_result + + +def _atom() -> CommonsEvidenceAtom: + return CommonsEvidenceAtom( + record_type=RecordType.DECISION, + action_kind=ActionKind.TEST, + cost_bucket=ValueBucket.LOW, + gain_bucket=ValueBucket.HIGH, + recommendation=DecisionClass.ALLOW, + applied_decision=DecisionClass.ALLOW, + reason_code=AggregateReasonCode.APPROVED, + outcome_class=OutcomeClass.NOT_APPLICABLE, + count=7, + minimum_group_size=5, + ) + + +def _components(tmp_path: Path) -> tuple[CommonsCache, CommonsOutbox]: + return ( + CommonsCache( + tmp_path, + model_namespace=MODEL_NAMESPACE, + expected_source_commit=SOURCE_COMMIT, + ), + CommonsOutbox(tmp_path), + ) + + +def test_task_five_interfaces_are_available_from_the_commons_package() -> None: + assert commons.CommonsCache is CommonsCache + assert commons.CommonsOutbox is CommonsOutbox + assert commons.synchronize_commons is synchronize_commons + + +def test_local_only_makes_zero_network_calls_and_does_not_enqueue(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + client = _RecordingClient( + pack=AssertionError("download must not run"), + submit=AssertionError("submit must not run"), + ) + + result = synchronize_commons( + CommonsConfig(CommonsMode.LOCAL_ONLY), + cache=cache, + outbox=outbox, + client=client, + model_namespace=MODEL_NAMESPACE, + atoms=(_atom(),), + ) + + assert result.network_calls == 0 + assert result.failures == () + assert client.download_calls == 0 + assert client.submitted == [] + assert not outbox.queue_path.exists() + + +def test_read_only_downloads_but_never_enqueues_or_submits(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + client = _RecordingClient(pack=_pack_bytes(), submit=AssertionError("submit must not run")) + + result = synchronize_commons( + CommonsConfig(CommonsMode.READ_ONLY), + cache=cache, + outbox=outbox, + client=client, + model_namespace=MODEL_NAMESPACE, + atoms=(_atom(),), + ) + + assert result.network_calls == 1 + assert result.cache_refreshed is True + assert result.submitted == 0 + assert not outbox.queue_path.exists() + + +def test_contributor_without_new_or_queued_evidence_only_downloads(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + client = _RecordingClient(pack=_pack_bytes(), submit=AssertionError("submit must not run")) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + model_namespace=MODEL_NAMESPACE, + atoms=(), + ) + + assert result.network_calls == 1 + assert result.submitted == 0 + assert client.submitted == [] + + +def test_contributor_ack_deletes_queued_evidence(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + client = _RecordingClient(pack=_pack_bytes(), submit=CommonsAck(True, False)) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + model_namespace=MODEL_NAMESPACE, + atoms=(_atom(),), + ) + + assert result.acked == 1 + assert result.submitted == 1 + assert outbox.pending(limit=8).entries == () + + +def test_4xx_quarantines_but_5xx_and_protocol_failures_retain_for_retry(tmp_path: Path) -> None: + for status, expected_quarantined, expected_retained in ((422, 1, 0), (503, 0, 1)): + case = tmp_path / str(status) + cache, outbox = _components(case) + outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + client = _RecordingClient(pack=_pack_bytes(), submit=CommonsHTTPError(status=status)) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + ) + + assert result.quarantined == expected_quarantined + assert result.retained == expected_retained + assert len(outbox.pending(limit=8).entries) == expected_retained + + protocol_case = tmp_path / "protocol" + cache, outbox = _components(protocol_case) + outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=_RecordingClient( + pack=_pack_bytes(), submit=CommonsProtocolError("invalid Commons response") + ), + ) + assert result.retained == 1 + assert SyncFailure.SUBMIT_PROTOCOL in result.failures + + +def test_sync_is_bounded_and_download_failure_does_not_block_outbox_retry(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + for _ in range(3): + outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + client = _RecordingClient( + pack=TimeoutError("privacy-canary-network-detail"), + submit=CommonsAck(True, False), + ) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + max_submissions=2, + ) + + assert result.network_calls == 3 + assert result.acked == 2 + assert len(outbox.pending(limit=8).entries) == 1 + assert result.failures == (SyncFailure.DOWNLOAD_TRANSPORT,) + assert "privacy-canary" not in repr(result) From d45b422e0f3314e8b6b9e67845d909c9a79cfd10 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:10:58 +0200 Subject: [PATCH 09/14] fix: harden Commons synchronization boundaries --- src/marginal/commons/__init__.py | 3 +- src/marginal/commons/_storage.py | 38 ++++++- src/marginal/commons/cache.py | 35 ++++++- src/marginal/commons/client.py | 172 ++++++++++++++++++++++++------- src/marginal/commons/evidence.py | 42 ++++++-- src/marginal/commons/identity.py | 12 +++ src/marginal/commons/outbox.py | 141 +++++++++++++++++++------ src/marginal/commons/sync.py | 32 ++++-- tests/commons/test_cache.py | 46 +++++++++ tests/commons/test_client.py | 87 +++++++++++++--- tests/commons/test_evidence.py | 40 ++++--- tests/commons/test_outbox.py | 104 ++++++++++++++++--- tests/commons/test_sync.py | 123 +++++++++++++++++++--- 13 files changed, 736 insertions(+), 139 deletions(-) diff --git a/src/marginal/commons/__init__.py b/src/marginal/commons/__init__.py index ab716e7..df28e5b 100644 --- a/src/marginal/commons/__init__.py +++ b/src/marginal/commons/__init__.py @@ -3,7 +3,7 @@ from .cache import CommonsCache, CommonsLifecycle, CommonsPrior from .client import CommonsAck, CommonsClient from .config import CommonsConfig, CommonsMode, configure_commons_mode, load_commons_config -from .evidence import CommonsEvidenceAtom, compile_verified_evidence +from .evidence import CommonsEvidenceAtom, CommonsEvidenceBatch, compile_verified_evidence from .identity import ( CanonicalModelIdentity, resolve_canonical_model, @@ -19,6 +19,7 @@ "CommonsClient", "CommonsConfig", "CommonsEvidenceAtom", + "CommonsEvidenceBatch", "CommonsLifecycle", "CommonsMode", "CommonsOutbox", diff --git a/src/marginal/commons/_storage.py b/src/marginal/commons/_storage.py index d8c722b..43b9eba 100644 --- a/src/marginal/commons/_storage.py +++ b/src/marginal/commons/_storage.py @@ -5,19 +5,44 @@ import errno import os import stat +import time from collections.abc import Iterator from contextlib import contextmanager, suppress from pathlib import Path from secrets import token_hex as _token_hex from marginal.governance_ledger import ( - _lock_exclusive, _open_parent_directory, _unlock, _write_all, ) _OWNER_ONLY_MASK = 0o077 +_LOCK_TIMEOUT_SECONDS = 0.2 +_LOCK_POLL_SECONDS = 0.01 + + +class CommonsStorageBusy(OSError): + """A closed bounded result for lock contention.""" + + +def _lock_exclusive_bounded(descriptor: int) -> None: + try: + import fcntl + except ImportError as exc: + raise OSError("OS-level file locking is unavailable") from exc + deadline = time.monotonic() + _LOCK_TIMEOUT_SECONDS + while True: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + return + except OSError as exc: + if exc.errno not in {errno.EACCES, errno.EAGAIN}: + raise + remaining = deadline - time.monotonic() + if remaining <= 0: + raise CommonsStorageBusy("Commons storage is busy") + time.sleep(min(_LOCK_POLL_SECONDS, remaining)) def _validate_directory(descriptor: int) -> None: @@ -71,8 +96,15 @@ def locked_directory(path: Path, *, create: bool, lock_name: str) -> Iterator[in primary_error: BaseException | None = None try: _validate_directory(directory_descriptor) - lock_descriptor = _open_lock(directory_descriptor, lock_name) - _lock_exclusive(lock_descriptor) + for _ in range(16): + try: + lock_descriptor = _open_lock(directory_descriptor, lock_name) + except FileNotFoundError: + continue + break + if lock_descriptor < 0: + raise FileNotFoundError("unable to open concurrently created Commons lock") + _lock_exclusive_bounded(lock_descriptor) locked = True yield directory_descriptor except BaseException as exc: diff --git a/src/marginal/commons/cache.py b/src/marginal/commons/cache.py index 356a359..c56405c 100644 --- a/src/marginal/commons/cache.py +++ b/src/marginal/commons/cache.py @@ -212,10 +212,32 @@ def refresh(self, raw: bytes) -> bool: if not isinstance(raw, bytes) or len(raw) > self.max_pack_bytes: return False try: - _parse_pack(raw, expected_source_commit=self.expected_source_commit) + candidate = _parse_pack(raw, expected_source_commit=self.expected_source_commit) with locked_directory( self.path.parent, create=True, lock_name=".cache.lock" ) as directory: + try: + existing_raw, _ = read_bounded_at( + directory, + _PACK_NAME, + maximum_bytes=self.max_pack_bytes, + label="Commons cache", + ) + except FileNotFoundError: + existing = None + else: + try: + existing = _parse_pack( + existing_raw, + expected_source_commit=self.expected_source_commit, + ) + except (ValueError, RecursionError, MemoryError, OverflowError): + existing = None + if ( + existing is not None + and candidate["commons_revision"] < existing["commons_revision"] + ): + return False atomic_replace_at( directory, _PACK_NAME, @@ -223,7 +245,7 @@ def refresh(self, raw: bytes) -> bool: temporary_prefix=".commons-pack-", label="Commons cache", ) - except (OSError, ValueError): + except (OSError, ValueError, RecursionError, MemoryError, OverflowError): return False return True @@ -239,7 +261,14 @@ def _load_pack(self) -> dict[str, Any] | None: label="Commons cache", ) return _parse_pack(raw, expected_source_commit=self.expected_source_commit) - except (FileNotFoundError, OSError, ValueError): + except ( + FileNotFoundError, + OSError, + ValueError, + RecursionError, + MemoryError, + OverflowError, + ): return None def load_prior(self) -> tuple[CommonsPrior, ...]: diff --git a/src/marginal/commons/client.py b/src/marginal/commons/client.py index 8e956ac..aa3e792 100644 --- a/src/marginal/commons/client.py +++ b/src/marginal/commons/client.py @@ -4,11 +4,15 @@ import http.client import json +import socket +import threading +import time +from contextlib import suppress from dataclasses import dataclass from typing import Protocol from urllib.parse import SplitResult, urlsplit -from .outbox import _TOKEN_PATTERN, OutboxEntry, _validate_envelope +from .outbox import OutboxEntry, _entry_boundary_valid, _reject_duplicate_keys _PACK_PATH = "/dist/commons-pack-v1.json" _EVIDENCE_PATH = "/v1/evidence" @@ -40,6 +44,10 @@ class CommonsAck: accepted: bool duplicate: bool + def __post_init__(self) -> None: + if self.accepted is not True or type(self.duplicate) is not bool: + raise ValueError("invalid Commons ACK") + class CommonsClientProtocol(Protocol): """Narrow transport boundary consumed by fail-open orchestration.""" @@ -86,7 +94,13 @@ def _positive_timeout(value: float, *, label: str) -> float: class CommonsClient: - """Issue only fixed-path GET and POST requests with strict resource limits.""" + """Issue fixed-path bounded requests. + + The monotonic request deadline covers socket connect, request send, headers, and body once + ``getaddrinfo`` returns. Python's synchronous stdlib resolver cannot preempt a blocked DNS + lookup without a helper thread, so DNS delay is checked and failed open immediately after + resolution rather than claimed as a hard cancellable deadline. + """ def __init__( self, @@ -95,12 +109,14 @@ def __init__( ingress_origin: str, connect_timeout: float = 2.0, read_timeout: float = 3.0, + request_timeout: float = 5.0, max_response_bytes: int = _DEFAULT_MAX_RESPONSE_BYTES, ) -> None: self._pack_origin = _parse_origin(pack_origin) self._ingress_origin = _parse_origin(ingress_origin) self._connect_timeout = _positive_timeout(connect_timeout, label="connect") self._read_timeout = _positive_timeout(read_timeout, label="read") + self._request_timeout = _positive_timeout(request_timeout, label="request") if ( isinstance(max_response_bytes, bool) or not isinstance(max_response_bytes, int) @@ -109,12 +125,68 @@ def __init__( raise ValueError("Commons response byte limit is invalid") self._max_response_bytes = max_response_bytes - def _connection(self, origin: _Origin) -> http.client.HTTPConnection: + def _connection(self, origin: _Origin, *, timeout: float) -> http.client.HTTPConnection: connection_type: type[http.client.HTTPConnection] connection_type = ( http.client.HTTPSConnection if origin.scheme == "https" else http.client.HTTPConnection ) - return connection_type(origin.host, port=origin.port, timeout=self._connect_timeout) + return connection_type(origin.host, port=origin.port, timeout=timeout) + + @staticmethod + def _abort_socket(socket_holder: list[socket.socket | None]) -> None: + sock = socket_holder[0] + if sock is None: + return + with suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + with suppress(OSError): + sock.close() + + @staticmethod + def _remaining(deadline: float, *, cap: float | None = None) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise CommonsTransportError("Commons transport failed") + return min(remaining, cap) if cap is not None else remaining + + def _set_read_timeout(self, connection: http.client.HTTPConnection, *, deadline: float) -> None: + if connection.sock is not None: + connection.sock.settimeout(self._remaining(deadline, cap=self._read_timeout)) + + def _read_bounded_response( + self, + response: http.client.HTTPResponse, + connection: http.client.HTTPConnection, + *, + deadline: float, + validate_length: bool, + ) -> bytes: + declared_size: int | None = None + if validate_length: + declared = response.getheader("Content-Length") + if declared is not None: + try: + declared_size = int(declared) + except ValueError: + raise CommonsProtocolError("invalid Commons response") from None + if declared_size < 0 or declared_size > self._max_response_bytes: + raise CommonsProtocolError("invalid Commons response") + chunks: list[bytes] = [] + remaining_bytes = self._max_response_bytes + 1 + while remaining_bytes > 0: + self._set_read_timeout(connection, deadline=deadline) + chunk = response.read1(min(64 * 1024, remaining_bytes)) + if not chunk: + break + chunks.append(chunk) + remaining_bytes -= len(chunk) + raw = b"".join(chunks) + self._remaining(deadline) + if validate_length and len(raw) > self._max_response_bytes: + raise CommonsProtocolError("invalid Commons response") + if validate_length and declared_size is not None and len(raw) != declared_size: + raise CommonsProtocolError("invalid Commons response") + return raw def _request( self, @@ -124,59 +196,77 @@ def _request( path: str, body: bytes | None = None, headers: dict[str, str] | None = None, - ) -> tuple[int, bytes]: - connection = self._connection(origin) + success_status: object, + ) -> bytes: + deadline = time.monotonic() + self._request_timeout + connection = self._connection( + origin, + timeout=self._remaining(deadline, cap=self._connect_timeout), + ) + socket_holder: list[socket.socket | None] = [None] + deadline_guard = threading.Timer( + self._request_timeout, + self._abort_socket, + args=(socket_holder,), + ) + deadline_guard.daemon = True + deadline_guard.start() try: - connection.request(method, path, body=body, headers=headers or {}) + connection.connect() + socket_holder[0] = connection.sock if connection.sock is not None: - connection.sock.settimeout(self._read_timeout) + connection.sock.settimeout(self._remaining(deadline)) + connection.request(method, path, body=body, headers=headers or {}) + self._set_read_timeout(connection, deadline=deadline) response = connection.getresponse() - declared = response.getheader("Content-Length") - if declared is not None: - try: - declared_size = int(declared) - except ValueError: - raise CommonsProtocolError("invalid Commons response") from None - if declared_size < 0: - raise CommonsProtocolError("invalid Commons response") - raw = response.read(self._max_response_bytes + 1) - if len(raw) > self._max_response_bytes: - raise CommonsProtocolError("invalid Commons response") - return response.status, raw - except CommonsProtocolError: + status_is_success = ( + response.status == success_status + if isinstance(success_status, int) + else 200 <= response.status < 300 + ) + if not status_is_success: + with suppress(CommonsTransportError, OSError, http.client.HTTPException): + self._read_bounded_response( + response, + connection, + deadline=deadline, + validate_length=False, + ) + raise CommonsHTTPError(status=response.status) + return self._read_bounded_response( + response, + connection, + deadline=deadline, + validate_length=True, + ) + except (CommonsProtocolError, CommonsHTTPError): raise except (TimeoutError, OSError, http.client.HTTPException): raise CommonsTransportError("Commons transport failed") from None finally: + deadline_guard.cancel() connection.close() def download(self) -> bytes: """Download the pack from its fixed public path.""" - status, raw = self._request( + return self._request( self._pack_origin, method="GET", path=_PACK_PATH, headers={"Accept": "application/json"}, + success_status=200, ) - if status != 200: - raise CommonsHTTPError(status=status) - return raw def submit(self, entry: OutboxEntry) -> CommonsAck: """Submit one validated envelope with retry identity only in its header.""" - if not isinstance(entry, OutboxEntry): - raise TypeError("Commons submission requires an outbox entry") - if _TOKEN_PATTERN.fullmatch(entry.retry_token) is None: - raise ValueError("Commons retry token is invalid") - envelope = _validate_envelope(entry.envelope) - body = ( - json.dumps(envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" - ).encode("utf-8") + if not _entry_boundary_valid(entry): + raise ValueError("Commons submission requires a valid outbox entry") + body = entry.body_bytes if len(body) > _MAX_REQUEST_BYTES: raise ValueError("Commons evidence envelope is too large") - status, raw = self._request( + raw = self._request( self._ingress_origin, method="POST", path=_EVIDENCE_PATH, @@ -186,12 +276,18 @@ def submit(self, entry: OutboxEntry) -> CommonsAck: "Content-Type": "application/json", "Idempotency-Key": entry.retry_token, }, + success_status=range(200, 300), ) - if not 200 <= status < 300: - raise CommonsHTTPError(status=status) try: - payload = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): + payload = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except ( + UnicodeDecodeError, + json.JSONDecodeError, + ValueError, + RecursionError, + MemoryError, + OverflowError, + ): raise CommonsProtocolError("invalid Commons response") from None if ( not isinstance(payload, dict) diff --git a/src/marginal/commons/evidence.py b/src/marginal/commons/evidence.py index ff14c82..4091a5a 100644 --- a/src/marginal/commons/evidence.py +++ b/src/marginal/commons/evidence.py @@ -81,6 +81,7 @@ class OutcomeClass(str, Enum): class CommonsEvidenceAtom: """One immutable atom containing only frozen aggregate dimensions and bounded counts.""" + model_identity: CanonicalModelIdentity record_type: RecordType action_kind: ActionKind cost_bucket: ValueBucket @@ -93,6 +94,8 @@ class CommonsEvidenceAtom: minimum_group_size: int def __post_init__(self) -> None: + if not identity_is_canonical(self.model_identity): + raise ValueError("model_identity must be one exact canonical model") if not isinstance(self.record_type, RecordType): raise TypeError("record_type must be a typed Commons enum") if not isinstance(self.action_kind, ActionKind): @@ -133,6 +136,30 @@ def to_dict(self) -> dict[str, str | int]: } +@dataclass(frozen=True, slots=True) +class CommonsEvidenceBatch: + """Immutable compiled atoms bound to one exact canonical model identity.""" + + identity: CanonicalModelIdentity + atoms: tuple[CommonsEvidenceAtom, ...] + + def __post_init__(self) -> None: + if not identity_is_canonical(self.identity): + raise ValueError("Commons evidence batch requires a canonical model identity") + if not isinstance(self.atoms, tuple) or not all( + isinstance(atom, CommonsEvidenceAtom) for atom in self.atoms + ): + raise TypeError("Commons evidence batch atoms must be an immutable typed tuple") + if any(atom.model_identity != self.identity for atom in self.atoms): + raise ValueError("Commons evidence batch atoms must use the same canonical model") + + @property + def model_namespace(self) -> str: + """Return the namespace inseparably carried by the compiled batch.""" + + return self.identity.namespace + + _EnumT = TypeVar("_EnumT", bound=Enum) _DimensionKey = tuple[ RecordType, @@ -208,7 +235,7 @@ def compile_verified_evidence( *, model_identity: CanonicalModelIdentity | None, minimum_group_size: int = 5, -) -> tuple[CommonsEvidenceAtom, ...]: +) -> CommonsEvidenceBatch | None: """Compile a verified local chain; arbitrary caller rows are never accepted.""" if isinstance(minimum_group_size, bool) or not isinstance(minimum_group_size, int): @@ -218,20 +245,20 @@ def compile_verified_evidence( if not isinstance(evidence_store, EvidenceStore): raise TypeError("evidence_store must be an EvidenceStore") if model_identity is None or not identity_is_canonical(model_identity): - return () + return None try: records, verification = evidence_store.verified_records() except (OSError, ValueError): - return () + return None if not verification.valid: - return () + return None attributed_namespaces = { namespace for record in records if isinstance((namespace := record.get("model_namespace")), str) } if attributed_namespaces != {model_identity.namespace}: - return () + return None counter: Counter[_DimensionKey] = Counter() for record in records: @@ -250,6 +277,7 @@ def compile_verified_evidence( continue atoms.append( CommonsEvidenceAtom( + model_identity=model_identity, record_type=key[0], action_kind=key[1], cost_bucket=key[2], @@ -262,4 +290,6 @@ def compile_verified_evidence( minimum_group_size=minimum_group_size, ) ) - return tuple(atoms) + if not atoms: + return None + return CommonsEvidenceBatch(identity=model_identity, atoms=tuple(atoms)) diff --git a/src/marginal/commons/identity.py b/src/marginal/commons/identity.py index 30f9cb4..dacbbde 100644 --- a/src/marginal/commons/identity.py +++ b/src/marginal/commons/identity.py @@ -85,6 +85,18 @@ def is_canonical_namespace(namespace: object) -> bool: return namespace in models.values() +def resolve_canonical_namespace(namespace: object) -> CanonicalModelIdentity | None: + """Resolve one exact reviewed namespace back to its canonical identity.""" + + if not isinstance(namespace, str): + return None + version, models = _registry() + for model, registered_namespace in models.items(): + if namespace == registered_namespace: + return CanonicalModelIdentity("openai", model, namespace, version) + return None + + def identity_is_canonical(identity: object) -> bool: """Reject forged value objects that were not derived from the exact registry mapping.""" diff --git a/src/marginal/commons/outbox.py b/src/marginal/commons/outbox.py index 94bc9f1..8eeb86f 100644 --- a/src/marginal/commons/outbox.py +++ b/src/marginal/commons/outbox.py @@ -2,29 +2,33 @@ from __future__ import annotations +import hashlib import json import os import re import secrets from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, cast from ._storage import atomic_create_at, locked_directory, read_bounded_at from .evidence import ( ActionKind, AggregateReasonCode, CommonsEvidenceAtom, + CommonsEvidenceBatch, DecisionClass, OutcomeClass, RecordType, ValueBucket, ) -from .identity import is_canonical_namespace +from .identity import is_canonical_namespace, resolve_canonical_namespace _MAX_QUEUE_BYTES = 512 * 1024 _MAX_ATOMS = 1_000 _TOKEN_PATTERN = re.compile(r"^[A-Za-z0-9_-]{43}$") +_ENTRY_NAME_PATTERN = re.compile(r"^queue-[0-9a-f]{32}\.json$") +_DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: @@ -42,17 +46,28 @@ class OutboxEntry: name: str retry_token: str - envelope: dict[str, object] + body_bytes: bytes + canonical_record: bytes + record_sha256: str device: int inode: int def body(self) -> bytes: """Serialize only the closed wire envelope, excluding local metadata.""" - return ( - json.dumps(self.envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - + "\n" - ).encode("utf-8") + return self.body_bytes + + @property + def envelope(self) -> dict[str, object]: + """Return a fresh mapping decoded from immutable canonical body bytes.""" + + return cast(dict[str, object], json.loads(self.body_bytes.decode("utf-8"))) + + @property + def model_namespace(self) -> str: + """Return the exact model namespace bound into canonical body bytes.""" + + return cast(str, self.envelope["model_namespace"]) @dataclass(frozen=True, slots=True) @@ -63,7 +78,7 @@ class OutboxScan: quarantined: int = 0 -def _atom_from_mapping(raw: object) -> CommonsEvidenceAtom: +def _atom_from_mapping(raw: object, *, model_namespace: str) -> CommonsEvidenceAtom: expected = { "record_type", "action_kind", @@ -78,8 +93,12 @@ def _atom_from_mapping(raw: object) -> CommonsEvidenceAtom: } if not isinstance(raw, dict) or set(raw) != expected: raise ValueError("queued Commons atom has an invalid shape") + identity = resolve_canonical_namespace(model_namespace) + if identity is None: + raise ValueError("queued Commons atom has an invalid model") try: return CommonsEvidenceAtom( + model_identity=identity, record_type=RecordType(raw["record_type"]), action_kind=ActionKind(raw["action_kind"]), cost_bucket=ValueBucket(raw["cost_bucket"]), @@ -104,11 +123,14 @@ def _validate_envelope(raw: object) -> dict[str, object]: raise ValueError("queued Commons envelope is incompatible") if not isinstance(atoms, list) or not 1 <= len(atoms) <= _MAX_ATOMS: raise ValueError("queued Commons envelope must contain bounded evidence") - parsed = [_atom_from_mapping(atom).to_dict() for atom in atoms] + assert isinstance(namespace, str) + parsed = [_atom_from_mapping(atom, model_namespace=namespace).to_dict() for atom in atoms] return {"schema_version": "1.0", "model_namespace": namespace, "atoms": parsed} def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> OutboxEntry: + if _ENTRY_NAME_PATTERN.fullmatch(name) is None: + raise ValueError("queued Commons record name is invalid") try: payload: Any = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) except (UnicodeDecodeError, json.JSONDecodeError) as exc: @@ -119,7 +141,52 @@ def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> Ou if not isinstance(retry_token, str) or _TOKEN_PATTERN.fullmatch(retry_token) is None: raise ValueError("queued Commons retry token is invalid") envelope = _validate_envelope(payload["envelope"]) - return OutboxEntry(name, retry_token, envelope, device, inode) + body_bytes = ( + json.dumps(envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + ).encode("utf-8") + canonical_record = ( + json.dumps( + {"envelope": envelope, "retry_token": retry_token}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + + "\n" + ).encode("utf-8") + return OutboxEntry( + name=name, + retry_token=retry_token, + body_bytes=body_bytes, + canonical_record=canonical_record, + record_sha256=hashlib.sha256(canonical_record).hexdigest(), + device=device, + inode=inode, + ) + + +def _entry_boundary_valid(entry: object) -> bool: + if not isinstance(entry, OutboxEntry): + return False + if ( + _ENTRY_NAME_PATTERN.fullmatch(entry.name) is None + or _TOKEN_PATTERN.fullmatch(entry.retry_token) is None + or not isinstance(entry.body_bytes, bytes) + or not isinstance(entry.canonical_record, bytes) + or _DIGEST_PATTERN.fullmatch(entry.record_sha256) is None + or hashlib.sha256(entry.canonical_record).hexdigest() != entry.record_sha256 + or len(entry.canonical_record) > _MAX_QUEUE_BYTES + ): + return False + try: + rebound = _parse_queue_record( + entry.canonical_record, + name=entry.name, + device=entry.device, + inode=entry.inode, + ) + except (UnicodeDecodeError, ValueError, RecursionError, MemoryError, OverflowError): + return False + return rebound == entry class CommonsOutbox: @@ -137,26 +204,28 @@ def __init__(self, data_dir: str | Path) -> None: def enqueue( self, *, - model_namespace: str, - atoms: tuple[CommonsEvidenceAtom, ...], + batch: CommonsEvidenceBatch, ) -> OutboxEntry | None: """Atomically queue nonempty typed evidence with one random retry token.""" - if not is_canonical_namespace(model_namespace) or not atoms: + if not isinstance(batch, CommonsEvidenceBatch) or not batch.atoms: return None - if len(atoms) > _MAX_ATOMS or not all( - isinstance(atom, CommonsEvidenceAtom) for atom in atoms - ): + if len(batch.atoms) > _MAX_ATOMS: return None envelope: dict[str, object] = { "schema_version": "1.0", - "model_namespace": model_namespace, - "atoms": [atom.to_dict() for atom in atoms], + "model_namespace": batch.model_namespace, + "atoms": [atom.to_dict() for atom in batch.atoms], } retry_token = secrets.token_urlsafe(32) - record = {"envelope": envelope, "retry_token": retry_token} encoded = ( - json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + json.dumps( + {"envelope": envelope, "retry_token": retry_token}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + + "\n" ).encode("utf-8") if len(encoded) > _MAX_QUEUE_BYTES: return None @@ -173,10 +242,9 @@ def enqueue( ) except FileExistsError: continue - return OutboxEntry( + return _parse_queue_record( + encoded, name=name, - retry_token=retry_token, - envelope=envelope, device=metadata.st_dev, inode=metadata.st_ino, ) @@ -234,7 +302,7 @@ def pending(self, *, limit: int = 8) -> OutboxScan: entry = _parse_queue_record( raw, name=name, device=metadata.st_dev, inode=metadata.st_ino ) - except (OSError, ValueError): + except (OSError, ValueError, RecursionError, MemoryError, OverflowError): quarantined += int(self._quarantine_name(directory, name)) continue if len(entries) < limit: @@ -244,21 +312,36 @@ def pending(self, *, limit: int = 8) -> OutboxScan: return OutboxScan(()) def _matches(self, directory: int, entry: OutboxEntry) -> bool: + if not _entry_boundary_valid(entry): + return False try: - _, metadata = read_bounded_at( + raw, metadata = read_bounded_at( directory, entry.name, maximum_bytes=_MAX_QUEUE_BYTES, label="Commons outbox entry", ) - except (FileNotFoundError, OSError, ValueError): + rebound = _parse_queue_record( + raw, + name=entry.name, + device=metadata.st_dev, + inode=metadata.st_ino, + ) + except ( + FileNotFoundError, + OSError, + ValueError, + RecursionError, + MemoryError, + OverflowError, + ): return False - return (metadata.st_dev, metadata.st_ino) == (entry.device, entry.inode) + return rebound == entry def ack(self, entry: OutboxEntry) -> bool: """Delete only the exact inode whose valid envelope received an ACK.""" - if not isinstance(entry, OutboxEntry): + if not _entry_boundary_valid(entry): return False try: with locked_directory( @@ -275,7 +358,7 @@ def ack(self, entry: OutboxEntry) -> bool: def quarantine(self, entry: OutboxEntry) -> bool: """Move only the exact inode to the owner-only quarantine directory.""" - if not isinstance(entry, OutboxEntry): + if not _entry_boundary_valid(entry): return False try: with locked_directory( diff --git a/src/marginal/commons/sync.py b/src/marginal/commons/sync.py index f083fdf..b3f4058 100644 --- a/src/marginal/commons/sync.py +++ b/src/marginal/commons/sync.py @@ -7,13 +7,14 @@ from .cache import CommonsCache from .client import ( + CommonsAck, CommonsClientProtocol, CommonsHTTPError, CommonsProtocolError, CommonsTransportError, ) from .config import CommonsConfig, CommonsMode -from .evidence import CommonsEvidenceAtom +from .evidence import CommonsEvidenceBatch from .outbox import CommonsOutbox @@ -29,6 +30,7 @@ class SyncFailure(str, Enum): SUBMIT_PROTOCOL = "submit_protocol" SUBMIT_TRANSPORT = "submit_transport" OUTBOX_TRANSITION = "outbox_transition" + EVIDENCE_MODEL_MISMATCH = "evidence_model_mismatch" @dataclass(frozen=True, slots=True) @@ -50,8 +52,7 @@ def synchronize_commons( cache: CommonsCache, outbox: CommonsOutbox, client: CommonsClientProtocol, - model_namespace: str | None = None, - atoms: tuple[CommonsEvidenceAtom, ...] = (), + evidence: CommonsEvidenceBatch | None = None, max_submissions: int = 8, ) -> CommonsSyncResult: """Download and optionally contribute without allowing any failure to block local work.""" @@ -99,9 +100,16 @@ def synchronize_commons( failures=tuple(failures), ) - if model_namespace is not None and atoms: + if ( + evidence is not None + and evidence.atoms + and evidence.model_namespace != cache.model_namespace + ): + failures.append(SyncFailure.EVIDENCE_MODEL_MISMATCH) + evidence = None + if evidence is not None and evidence.atoms: try: - outbox.enqueue(model_namespace=model_namespace, atoms=atoms) + outbox.enqueue(batch=evidence) except Exception: failures.append(SyncFailure.OUTBOX_WRITE) @@ -117,10 +125,14 @@ def synchronize_commons( quarantined += scan.quarantined for entry in scan.entries: + if entry.model_namespace != cache.model_namespace: + retained += 1 + failures.append(SyncFailure.EVIDENCE_MODEL_MISMATCH) + continue network_calls += 1 submitted += 1 try: - client.submit(entry) + ack = client.submit(entry) except CommonsHTTPError as exc: if 400 <= exc.status < 500: try: @@ -144,6 +156,14 @@ def synchronize_commons( retained += 1 failures.append(SyncFailure.SUBMIT_TRANSPORT) else: + if ( + not isinstance(ack, CommonsAck) + or ack.accepted is not True + or type(ack.duplicate) is not bool + ): + retained += 1 + failures.append(SyncFailure.SUBMIT_PROTOCOL) + continue try: deleted = outbox.ack(entry) except Exception: diff --git a/tests/commons/test_cache.py b/tests/commons/test_cache.py index 96a52ca..1f107ea 100644 --- a/tests/commons/test_cache.py +++ b/tests/commons/test_cache.py @@ -4,6 +4,8 @@ import json import os import stat +import threading +import time from pathlib import Path import pytest @@ -129,6 +131,50 @@ def test_refresh_rejects_oversized_and_cross_model_or_incomplete_packs(tmp_path: assert cache.refresh(b"{" + b" " * (cache.max_pack_bytes + 1)) is False +def test_refresh_rejects_recursive_json_as_a_bounded_parser_failure(tmp_path: Path) -> None: + cache = _cache(tmp_path) + + assert cache.refresh(("[" * 2_000 + "]" * 2_000).encode()) is False + assert cache.load_prior() == () + + +def test_refresh_rejects_revision_rollback_under_the_cache_lock(tmp_path: Path) -> None: + cache = _cache(tmp_path) + assert cache.refresh(_pack_bytes(count=8, revision=2)) is True + before = cache.path.read_bytes() + + assert cache.refresh(_pack_bytes(count=7, revision=1)) is False + + assert cache.path.read_bytes() == before + assert cache.revision == 2 + assert [prior.count for prior in cache.load_prior()] == [8] + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX advisory locks are required") +def test_cache_lock_contention_returns_fail_open_within_a_bound(tmp_path: Path) -> None: + import fcntl + + cache = _cache(tmp_path) + assert cache.refresh(_pack_bytes()) is True + lock = (cache.path.parent / ".cache.lock").open("r+b") + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + + def release_later() -> None: + time.sleep(0.5) + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + release = threading.Thread(target=release_later, daemon=True) + release.start() + started = time.monotonic() + refreshed = cache.refresh(_pack_bytes(count=8, revision=2)) + elapsed = time.monotonic() - started + release.join(timeout=1) + lock.close() + + assert refreshed is False + assert elapsed < 0.4 + + def test_cache_rejects_symlink_leaf_without_touching_its_target(tmp_path: Path) -> None: cache = _cache(tmp_path) cache.path.parent.mkdir(parents=True) diff --git a/tests/commons/test_client.py b/tests/commons/test_client.py index ace85ae..1296a16 100644 --- a/tests/commons/test_client.py +++ b/tests/commons/test_client.py @@ -21,11 +21,13 @@ ActionKind, AggregateReasonCode, CommonsEvidenceAtom, + CommonsEvidenceBatch, DecisionClass, OutcomeClass, RecordType, ValueBucket, ) +from marginal.commons.identity import resolve_canonical_model from marginal.commons.outbox import CommonsOutbox, OutboxEntry @@ -34,6 +36,20 @@ class _Handler(BaseHTTPRequestHandler): response_status: ClassVar[int] = 200 response_body: ClassVar[bytes] = b"{}" response_delay: ClassVar[float] = 0.0 + trickle_delay: ClassVar[float] = 0.0 + + def _write_body(self) -> None: + time.sleep(type(self).response_delay) + try: + if type(self).trickle_delay: + for byte in type(self).response_body: + time.sleep(type(self).trickle_delay) + self.wfile.write(bytes([byte])) + self.wfile.flush() + else: + self.wfile.write(type(self).response_body) + except (BrokenPipeError, ConnectionResetError): + return def do_GET(self) -> None: type(self).requests.append( @@ -42,11 +58,7 @@ def do_GET(self) -> None: self.send_response(type(self).response_status) self.send_header("Content-Length", str(len(type(self).response_body))) self.end_headers() - time.sleep(type(self).response_delay) - try: - self.wfile.write(type(self).response_body) - except BrokenPipeError: - return + self._write_body() def do_POST(self) -> None: length = int(self.headers.get("Content-Length", "0")) @@ -57,22 +69,25 @@ def do_POST(self) -> None: self.send_response(type(self).response_status) self.send_header("Content-Length", str(len(type(self).response_body))) self.end_headers() - time.sleep(type(self).response_delay) - try: - self.wfile.write(type(self).response_body) - except BrokenPipeError: - return + self._write_body() def log_message(self, _format: str, *_args: object) -> None: return @contextmanager -def _server(*, status: int = 200, body: bytes = b"{}", delay: float = 0.0) -> Iterator[str]: +def _server( + *, + status: int = 200, + body: bytes = b"{}", + delay: float = 0.0, + trickle_delay: float = 0.0, +) -> Iterator[str]: _Handler.requests = [] _Handler.response_status = status _Handler.response_body = body _Handler.response_delay = delay + _Handler.trickle_delay = trickle_delay server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -86,7 +101,10 @@ def _server(*, status: int = 200, body: bytes = b"{}", delay: float = 0.0) -> It def _entry(tmp_path: Path) -> OutboxEntry: + identity = resolve_canonical_model(provider="openai", model="gpt-5.6-sol") + assert identity is not None atom = CommonsEvidenceAtom( + model_identity=identity, record_type=RecordType.DECISION, action_kind=ActionKind.TEST, cost_bucket=ValueBucket.LOW, @@ -98,7 +116,9 @@ def _entry(tmp_path: Path) -> OutboxEntry: count=7, minimum_group_size=5, ) - entry = CommonsOutbox(tmp_path).enqueue(model_namespace="openai/gpt-5.6-sol", atoms=(atom,)) + entry = CommonsOutbox(tmp_path).enqueue( + batch=CommonsEvidenceBatch(identity=identity, atoms=(atom,)) + ) assert entry is not None return entry @@ -139,6 +159,7 @@ def test_submit_sends_only_the_closed_envelope_and_retry_header(tmp_path: Path) b'{"accepted":true}', b'{"accepted":true,"duplicate":false,"extra":true}', b'{"accepted":1,"duplicate":false}', + b'{"accepted":true,"duplicate":true,"duplicate":false}', ], ) def test_submit_requires_the_exact_ack_shape(tmp_path: Path, body: bytes) -> None: @@ -160,6 +181,15 @@ def test_invalid_response_errors_do_not_retain_raw_body_details(tmp_path: Path) assert captured.value.__cause__ is None +def test_recursive_ack_is_a_redacted_protocol_failure(tmp_path: Path) -> None: + body = ("[" * 2_000 + "]" * 2_000).encode() + with ( + _server(body=body) as origin, + pytest.raises(CommonsProtocolError, match="invalid Commons response"), + ): + CommonsClient(pack_origin=origin, ingress_origin=origin).submit(_entry(tmp_path)) + + def test_read_timeout_is_separate_and_redacted() -> None: with ( _server(body=b"late", delay=0.1) as origin, @@ -175,6 +205,39 @@ def test_read_timeout_is_separate_and_redacted() -> None: assert "timed out" not in str(captured.value).lower() +def test_slow_trickle_hits_one_monotonic_request_deadline() -> None: + with _server(body=b"slow-body", trickle_delay=0.04) as origin: + started = time.monotonic() + with pytest.raises(CommonsTransportError, match="Commons transport failed"): + CommonsClient( + pack_origin=origin, + ingress_origin=origin, + connect_timeout=1.0, + read_timeout=0.2, + request_timeout=0.12, + ).download() + elapsed = time.monotonic() - started + + assert elapsed < 0.3 + + +@pytest.mark.parametrize("status", [422, 503]) +def test_non_2xx_status_is_classified_before_oversized_body_validation( + tmp_path: Path, status: int +) -> None: + with ( + _server(status=status, body=b"x" * 65) as origin, + pytest.raises(CommonsHTTPError) as captured, + ): + CommonsClient( + pack_origin=origin, + ingress_origin=origin, + max_response_bytes=64, + ).submit(_entry(tmp_path)) + + assert captured.value.status == status + + def test_client_bounds_response_bytes_and_reports_only_status_category(tmp_path: Path) -> None: with _server(body=b"x" * 65) as origin: client = CommonsClient(pack_origin=origin, ingress_origin=origin, max_response_bytes=64) diff --git a/tests/commons/test_evidence.py b/tests/commons/test_evidence.py index 6ce2d22..d899871 100644 --- a/tests/commons/test_evidence.py +++ b/tests/commons/test_evidence.py @@ -6,7 +6,10 @@ import pytest -from marginal.commons.evidence import CommonsEvidenceAtom, compile_verified_evidence +from marginal.commons.evidence import ( + CommonsEvidenceAtom, + compile_verified_evidence, +) from marginal.commons.identity import resolve_canonical_model from marginal.governance_ledger import GovernanceLedger from marginal.integrations.codex.evidence import EvidenceStore @@ -66,9 +69,14 @@ def test_compiler_reads_real_verified_records_and_emits_only_closed_atoms(tmp_pa store.append(_decision(identity.namespace)) store.append(_outcome(identity.namespace)) - atoms = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) + batch = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) - assert [atom.to_dict() for atom in atoms] == [ + assert batch is not None + assert batch.identity == identity + assert batch.model_namespace == identity.namespace + with pytest.raises(ValueError, match="same canonical model"): + dataclasses.replace(batch, identity=_identity("gpt-5.6-terra")) + assert [atom.to_dict() for atom in batch.atoms] == [ { "record_type": "decision", "action_kind": "tool", @@ -94,7 +102,7 @@ def test_compiler_reads_real_verified_records_and_emits_only_closed_atoms(tmp_pa "minimum_group_size": 1, }, ] - serialized = json.dumps([atom.to_dict() for atom in atoms], sort_keys=True) + serialized = json.dumps([atom.to_dict() for atom in batch.atoms], sort_keys=True) assert CANARY not in serialized for forbidden in ( "hash", @@ -121,12 +129,15 @@ def test_atoms_are_immutable_and_counts_are_bounded(tmp_path: Path) -> None: record["action_hash"] = f"action-{index}" store.append(record) - atoms = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) + batch = compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) - assert len(atoms) == 1 - assert atoms[0].count == 1_000 + assert batch is not None + assert len(batch.atoms) == 1 + assert batch.atoms[0].count == 1_000 with pytest.raises(dataclasses.FrozenInstanceError): - atoms[0].count = 2 # type: ignore[misc] + batch.atoms[0].count = 2 # type: ignore[misc] + with pytest.raises(dataclasses.FrozenInstanceError): + batch.identity = _identity("gpt-5.6-terra") # type: ignore[misc] def test_compiler_rejects_arbitrary_caller_mappings_and_unverified_chains(tmp_path: Path) -> None: @@ -139,7 +150,7 @@ def test_compiler_rejects_arbitrary_caller_mappings_and_unverified_chains(tmp_pa ledger = store.governance_ledger_path tampered = ledger.read_text(encoding="utf-8").replace("APPROVED", "DENIED") ledger.write_text(tampered, encoding="utf-8") - assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) == () + assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) is None def test_compiler_fails_closed_on_a_verified_non_record_payload(tmp_path: Path) -> None: @@ -149,7 +160,7 @@ def test_compiler_fails_closed_on_a_verified_non_record_payload(tmp_path: Path) {"event": "codex_evidence", "evidence": [CANARY, {"nested": CANARY}]} ) - assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) == () + assert compile_verified_evidence(store, model_identity=identity, minimum_group_size=1) is None def test_conflicting_or_wrong_model_attribution_compiles_nothing(tmp_path: Path) -> None: @@ -159,12 +170,12 @@ def test_conflicting_or_wrong_model_attribution_compiles_nothing(tmp_path: Path) store.append(_decision(sol.namespace)) store.append(_decision(terra.namespace)) - assert compile_verified_evidence(store, model_identity=sol, minimum_group_size=1) == () - assert compile_verified_evidence(store, model_identity=None, minimum_group_size=1) == () + assert compile_verified_evidence(store, model_identity=sol, minimum_group_size=1) is None + assert compile_verified_evidence(store, model_identity=None, minimum_group_size=1) is None isolated = EvidenceStore(tmp_path / "isolated") isolated.append(_decision(sol.namespace)) - assert compile_verified_evidence(isolated, model_identity=terra, minimum_group_size=1) == () + assert compile_verified_evidence(isolated, model_identity=terra, minimum_group_size=1) is None def test_small_groups_are_suppressed_and_boundaries_are_validated(tmp_path: Path) -> None: @@ -172,7 +183,7 @@ def test_small_groups_are_suppressed_and_boundaries_are_validated(tmp_path: Path store = EvidenceStore(tmp_path) store.append(_decision(identity.namespace)) - assert compile_verified_evidence(store, model_identity=identity) == () + assert compile_verified_evidence(store, model_identity=identity) is None with pytest.raises(TypeError, match="minimum_group_size"): compile_verified_evidence(store, model_identity=identity, minimum_group_size=True) with pytest.raises(ValueError, match="between 1 and 1000"): @@ -193,6 +204,7 @@ def test_local_evidence_rejects_nested_or_unbounded_commons_values(tmp_path: Pat def test_atom_constructor_accepts_typed_fields_not_arbitrary_nested_values() -> None: with pytest.raises(TypeError, match="record_type"): CommonsEvidenceAtom( # type: ignore[arg-type] + model_identity=_identity(), record_type={"canary": CANARY}, action_kind="tool", cost_bucket="low", diff --git a/tests/commons/test_outbox.py b/tests/commons/test_outbox.py index cc10ca7..82ccebc 100644 --- a/tests/commons/test_outbox.py +++ b/tests/commons/test_outbox.py @@ -1,5 +1,7 @@ from __future__ import annotations +import dataclasses +import hashlib import json import multiprocessing import os @@ -14,18 +16,23 @@ ActionKind, AggregateReasonCode, CommonsEvidenceAtom, + CommonsEvidenceBatch, DecisionClass, OutcomeClass, RecordType, ValueBucket, ) +from marginal.commons.identity import resolve_canonical_model from marginal.commons.outbox import CommonsOutbox MODEL_NAMESPACE = "openai/gpt-5.6-sol" -def _atom() -> CommonsEvidenceAtom: +def _atom(model: str = "gpt-5.6-sol") -> CommonsEvidenceAtom: + identity = resolve_canonical_model(provider="openai", model=model) + assert identity is not None return CommonsEvidenceAtom( + model_identity=identity, record_type=RecordType.DECISION, action_kind=ActionKind.TEST, cost_bucket=ValueBucket.LOW, @@ -39,8 +46,14 @@ def _atom() -> CommonsEvidenceAtom: ) +def _batch(model: str = "gpt-5.6-sol") -> CommonsEvidenceBatch: + identity = resolve_canonical_model(provider="openai", model=model) + assert identity is not None + return CommonsEvidenceBatch(identity=identity, atoms=(_atom(model),)) + + def _enqueue_child(data_dir: str, results: multiprocessing.Queue[str]) -> None: - entry = CommonsOutbox(data_dir).enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + entry = CommonsOutbox(data_dir).enqueue(batch=_batch()) results.put(entry.name if entry is not None else "") @@ -49,13 +62,17 @@ def test_enqueue_is_private_atomic_and_keeps_retry_identity_outside_evidence( ) -> None: outbox = CommonsOutbox(tmp_path) - entry = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + entry = outbox.enqueue(batch=_batch()) assert entry is not None assert re.fullmatch(r"[A-Za-z0-9_-]{43}", entry.retry_token) assert "retry" not in json.dumps(entry.envelope, sort_keys=True).lower() - raw = json.loads((outbox.queue_path / entry.name).read_text(encoding="utf-8")) + persisted = (outbox.queue_path / entry.name).read_bytes() + raw = json.loads(persisted) assert raw == {"envelope": entry.envelope, "retry_token": entry.retry_token} + assert entry.canonical_record == persisted + assert entry.record_sha256 == hashlib.sha256(entry.canonical_record).hexdigest() + assert json.loads(entry.body_bytes) == entry.envelope assert stat.S_IMODE((outbox.queue_path / entry.name).stat().st_mode) == 0o600 assert stat.S_IMODE(outbox.queue_path.stat().st_mode) == 0o700 @@ -64,7 +81,7 @@ def test_restart_recovers_the_same_one_time_retry_token_and_ack_deletes_exact_en tmp_path: Path, ) -> None: first = CommonsOutbox(tmp_path) - queued = first.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + queued = first.enqueue(batch=_batch()) assert queued is not None restarted = CommonsOutbox(tmp_path) @@ -79,14 +96,14 @@ def test_restart_recovers_the_same_one_time_retry_token_and_ack_deletes_exact_en def test_empty_or_unregistered_evidence_is_never_queued(tmp_path: Path) -> None: outbox = CommonsOutbox(tmp_path) - assert outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=()) is None - assert outbox.enqueue(model_namespace="private/customer-model", atoms=(_atom(),)) is None + empty = CommonsEvidenceBatch(identity=_batch().identity, atoms=()) + assert outbox.enqueue(batch=empty) is None assert not outbox.queue_path.exists() def test_pending_quarantines_malformed_files_without_following_symlinks(tmp_path: Path) -> None: outbox = CommonsOutbox(tmp_path) - queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + queued = outbox.enqueue(batch=_batch()) assert queued is not None malformed = outbox.queue_path / "queue-malformed.json" malformed.write_text('{"retry_token":"privacy-canary"}', encoding="utf-8") @@ -113,7 +130,7 @@ def test_pending_quarantines_duplicate_json_fields_instead_of_accepting_last_val tmp_path: Path, ) -> None: outbox = CommonsOutbox(tmp_path) - queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + queued = outbox.enqueue(batch=_batch()) assert queued is not None path = outbox.queue_path / queued.name raw = path.read_text(encoding="utf-8") @@ -130,10 +147,24 @@ def test_pending_quarantines_duplicate_json_fields_instead_of_accepting_last_val assert scan.quarantined == 1 +def test_pending_quarantines_recursive_json_without_stopping_other_work(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + valid = outbox.enqueue(batch=_batch()) + assert valid is not None + recursive = outbox.queue_path / f"queue-{'f' * 32}.json" + recursive.write_text("[" * 2_000 + "]" * 2_000, encoding="utf-8") + recursive.chmod(0o600) + + scan = outbox.pending(limit=8) + + assert [entry.name for entry in scan.entries] == [valid.name] + assert scan.quarantined == 1 + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFO leaves are POSIX-specific") def test_pending_rejects_a_fifo_leaf_without_blocking_for_a_writer(tmp_path: Path) -> None: outbox = CommonsOutbox(tmp_path) - queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + queued = outbox.enqueue(batch=_batch()) assert queued is not None fifo = outbox.queue_path / "queue-fifo.json" os.mkfifo(fifo, mode=0o600) @@ -156,7 +187,7 @@ def scan() -> None: def test_ack_refuses_a_different_inode_reusing_the_same_name(tmp_path: Path) -> None: outbox = CommonsOutbox(tmp_path) - queued = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + queued = outbox.enqueue(batch=_batch()) assert queued is not None path = outbox.queue_path / queued.name path.unlink() @@ -167,15 +198,58 @@ def test_ack_refuses_a_different_inode_reusing_the_same_name(tmp_path: Path) -> assert path.read_text(encoding="utf-8") == "replacement" +def test_ack_and_quarantine_reject_forged_traversal_names(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(batch=_batch()) + assert queued is not None + victim = tmp_path / "victim.json" + victim.write_bytes((outbox.queue_path / queued.name).read_bytes()) + victim.chmod(0o600) + metadata = victim.stat() + forged = dataclasses.replace( + queued, + name="../../../victim.json", + device=metadata.st_dev, + inode=metadata.st_ino, + ) + + assert outbox.ack(forged) is False + assert outbox.quarantine(forged) is False + assert victim.exists() + + +def test_transition_revalidates_canonical_content_immediately_before_delete(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(batch=_batch()) + assert queued is not None + path = outbox.queue_path / queued.name + mutated = path.read_bytes().replace(b'"action_kind":"test"', b'"action_kind":"search"') + path.write_bytes(mutated) + path.chmod(0o600) + + assert outbox.ack(queued) is False + assert path.exists() + + +def test_transition_revalidates_bound_retry_token_and_digest(tmp_path: Path) -> None: + outbox = CommonsOutbox(tmp_path) + queued = outbox.enqueue(batch=_batch()) + assert queued is not None + + assert outbox.ack(dataclasses.replace(queued, retry_token="x" * 43)) is False + assert outbox.quarantine(dataclasses.replace(queued, record_sha256="0" * 64)) is False + assert (outbox.queue_path / queued.name).exists() + + def test_enqueue_handles_random_name_collision_without_overwrite( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: outbox = CommonsOutbox(tmp_path) - tokens = iter(["same-name", "same-name", "different-name"]) + tokens = iter(["a" * 32, "a" * 32, "b" * 32]) monkeypatch.setattr("marginal.commons.outbox.secrets.token_hex", lambda _size: next(tokens)) - first = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) - second = outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + first = outbox.enqueue(batch=_batch()) + second = outbox.enqueue(batch=_batch()) assert first is not None and second is not None assert first.name != second.name @@ -200,7 +274,7 @@ def test_outbox_rejects_a_symlinked_queue_directory_without_writing_outside(tmp_ outbox.queue_path.symlink_to(outside, target_is_directory=True) with pytest.raises((OSError, ValueError)): - outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + outbox.enqueue(batch=_batch()) assert list(outside.iterdir()) == [] diff --git a/tests/commons/test_sync.py b/tests/commons/test_sync.py index 81e91a9..e65985f 100644 --- a/tests/commons/test_sync.py +++ b/tests/commons/test_sync.py @@ -2,8 +2,13 @@ import hashlib import json +import os +import threading +import time from pathlib import Path +import pytest + import marginal.commons as commons from marginal.commons.cache import CommonsCache from marginal.commons.client import CommonsAck, CommonsHTTPError, CommonsProtocolError @@ -12,11 +17,13 @@ ActionKind, AggregateReasonCode, CommonsEvidenceAtom, + CommonsEvidenceBatch, DecisionClass, OutcomeClass, RecordType, ValueBucket, ) +from marginal.commons.identity import resolve_canonical_model from marginal.commons.outbox import CommonsOutbox, OutboxEntry from marginal.commons.sync import SyncFailure, synchronize_commons @@ -61,8 +68,11 @@ def submit(self, entry: OutboxEntry) -> CommonsAck: return self.submit_result -def _atom() -> CommonsEvidenceAtom: +def _atom(model: str = "gpt-5.6-sol") -> CommonsEvidenceAtom: + identity = resolve_canonical_model(provider="openai", model=model) + assert identity is not None return CommonsEvidenceAtom( + model_identity=identity, record_type=RecordType.DECISION, action_kind=ActionKind.TEST, cost_bucket=ValueBucket.LOW, @@ -76,6 +86,12 @@ def _atom() -> CommonsEvidenceAtom: ) +def _batch(model: str = "gpt-5.6-sol") -> CommonsEvidenceBatch: + identity = resolve_canonical_model(provider="openai", model=model) + assert identity is not None + return CommonsEvidenceBatch(identity=identity, atoms=(_atom(model),)) + + def _components(tmp_path: Path) -> tuple[CommonsCache, CommonsOutbox]: return ( CommonsCache( @@ -105,8 +121,7 @@ def test_local_only_makes_zero_network_calls_and_does_not_enqueue(tmp_path: Path cache=cache, outbox=outbox, client=client, - model_namespace=MODEL_NAMESPACE, - atoms=(_atom(),), + evidence=_batch(), ) assert result.network_calls == 0 @@ -125,8 +140,7 @@ def test_read_only_downloads_but_never_enqueues_or_submits(tmp_path: Path) -> No cache=cache, outbox=outbox, client=client, - model_namespace=MODEL_NAMESPACE, - atoms=(_atom(),), + evidence=_batch(), ) assert result.network_calls == 1 @@ -144,8 +158,7 @@ def test_contributor_without_new_or_queued_evidence_only_downloads(tmp_path: Pat cache=cache, outbox=outbox, client=client, - model_namespace=MODEL_NAMESPACE, - atoms=(), + evidence=None, ) assert result.network_calls == 1 @@ -153,6 +166,42 @@ def test_contributor_without_new_or_queued_evidence_only_downloads(tmp_path: Pat assert client.submitted == [] +def test_contributor_cannot_relabel_a_model_bound_batch_for_another_cache(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + client = _RecordingClient(pack=_pack_bytes(), submit=AssertionError("submit must not run")) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + evidence=_batch("gpt-5.6-terra"), + ) + + assert result.submitted == 0 + assert result.failures == (SyncFailure.EVIDENCE_MODEL_MISMATCH,) + assert not outbox.queue_path.exists() + + +def test_sync_does_not_submit_a_queued_envelope_for_another_cache_model(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + queued = outbox.enqueue(batch=_batch("gpt-5.6-terra")) + assert queued is not None + client = _RecordingClient(pack=_pack_bytes(), submit=AssertionError("submit must not run")) + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + ) + + assert result.submitted == 0 + assert result.retained == 1 + assert result.failures == (SyncFailure.EVIDENCE_MODEL_MISMATCH,) + assert len(outbox.pending(limit=8).entries) == 1 + + def test_contributor_ack_deletes_queued_evidence(tmp_path: Path) -> None: cache, outbox = _components(tmp_path) client = _RecordingClient(pack=_pack_bytes(), submit=CommonsAck(True, False)) @@ -162,8 +211,7 @@ def test_contributor_ack_deletes_queued_evidence(tmp_path: Path) -> None: cache=cache, outbox=outbox, client=client, - model_namespace=MODEL_NAMESPACE, - atoms=(_atom(),), + evidence=_batch(), ) assert result.acked == 1 @@ -175,7 +223,7 @@ def test_4xx_quarantines_but_5xx_and_protocol_failures_retain_for_retry(tmp_path for status, expected_quarantined, expected_retained in ((422, 1, 0), (503, 0, 1)): case = tmp_path / str(status) cache, outbox = _components(case) - outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + outbox.enqueue(batch=_batch()) client = _RecordingClient(pack=_pack_bytes(), submit=CommonsHTTPError(status=status)) result = synchronize_commons( @@ -191,7 +239,7 @@ def test_4xx_quarantines_but_5xx_and_protocol_failures_retain_for_retry(tmp_path protocol_case = tmp_path / "protocol" cache, outbox = _components(protocol_case) - outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + outbox.enqueue(batch=_batch()) result = synchronize_commons( CommonsConfig(CommonsMode.CONTRIBUTOR), cache=cache, @@ -204,10 +252,29 @@ def test_4xx_quarantines_but_5xx_and_protocol_failures_retain_for_retry(tmp_path assert SyncFailure.SUBMIT_PROTOCOL in result.failures +def test_unvalidated_ack_object_never_deletes_queued_evidence(tmp_path: Path) -> None: + cache, outbox = _components(tmp_path) + queued = outbox.enqueue(batch=_batch()) + assert queued is not None + client = _RecordingClient(pack=_pack_bytes(), submit=object()) # type: ignore[arg-type] + + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=client, + ) + + assert result.acked == 0 + assert result.retained == 1 + assert result.failures == (SyncFailure.SUBMIT_PROTOCOL,) + assert len(outbox.pending(limit=8).entries) == 1 + + def test_sync_is_bounded_and_download_failure_does_not_block_outbox_retry(tmp_path: Path) -> None: cache, outbox = _components(tmp_path) for _ in range(3): - outbox.enqueue(model_namespace=MODEL_NAMESPACE, atoms=(_atom(),)) + outbox.enqueue(batch=_batch()) client = _RecordingClient( pack=TimeoutError("privacy-canary-network-detail"), submit=CommonsAck(True, False), @@ -226,3 +293,35 @@ def test_sync_is_bounded_and_download_failure_does_not_block_outbox_retry(tmp_pa assert len(outbox.pending(limit=8).entries) == 1 assert result.failures == (SyncFailure.DOWNLOAD_TRANSPORT,) assert "privacy-canary" not in repr(result) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX advisory locks are required") +def test_outbox_lock_contention_returns_a_closed_fail_open_sync_result(tmp_path: Path) -> None: + import fcntl + + cache, outbox = _components(tmp_path) + queued = outbox.enqueue(batch=_batch()) + assert queued is not None + lock = (outbox.queue_path / ".outbox.lock").open("r+b") + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + + def release_later() -> None: + time.sleep(0.5) + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + release = threading.Thread(target=release_later, daemon=True) + release.start() + started = time.monotonic() + result = synchronize_commons( + CommonsConfig(CommonsMode.CONTRIBUTOR), + cache=cache, + outbox=outbox, + client=_RecordingClient(pack=_pack_bytes(), submit=CommonsAck(True, False)), + ) + elapsed = time.monotonic() - started + release.join(timeout=1) + lock.close() + + assert elapsed < 0.4 + assert result.network_calls == 1 + assert result.failures == (SyncFailure.OUTBOX_READ,) From db4b15ff23ecea1b525df2281798b7760657d066 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:18:56 +0200 Subject: [PATCH 10/14] fix: bound Commons DNS resolution --- src/marginal/commons/client.py | 100 +++++++++++++++++++++++++++++++-- tests/commons/test_client.py | 40 +++++++++++++ 2 files changed, 134 insertions(+), 6 deletions(-) diff --git a/src/marginal/commons/client.py b/src/marginal/commons/client.py index aa3e792..31711ce 100644 --- a/src/marginal/commons/client.py +++ b/src/marginal/commons/client.py @@ -4,12 +4,13 @@ import http.client import json +import queue import socket import threading import time from contextlib import suppress from dataclasses import dataclass -from typing import Protocol +from typing import Any, Protocol, cast from urllib.parse import SplitResult, urlsplit from .outbox import OutboxEntry, _entry_boundary_valid, _reject_duplicate_keys @@ -37,6 +38,69 @@ def __init__(self, *, status: int) -> None: super().__init__(f"Commons request failed ({category} response)") +_AddressInfo = tuple[Any, Any, int, str, Any] +_ResolveResult = tuple[tuple[_AddressInfo, ...] | None, Exception | None] + + +@dataclass(slots=True) +class _ResolveRequest: + host: str + port: int + result: queue.Queue[_ResolveResult] + canceled: threading.Event + + +class _SharedResolver: + """One daemon resolver so stalled DNS never creates retry-proportional threads.""" + + def __init__(self) -> None: + self._requests: queue.Queue[_ResolveRequest] = queue.Queue(maxsize=1) + self._thread = threading.Thread( + target=self._run, + name="marginal-commons-resolver", + daemon=True, + ) + self._thread.start() + + def _run(self) -> None: + while True: + request = self._requests.get() + if request.canceled.is_set(): + continue + try: + addresses = cast( + list[_AddressInfo], + socket.getaddrinfo(request.host, request.port, type=socket.SOCK_STREAM), + ) + result: _ResolveResult = (tuple(addresses), None) + except Exception as exc: + result = (None, exc) + if not request.canceled.is_set(): + with suppress(queue.Full): + request.result.put_nowait(result) + + def resolve(self, host: str, port: int, *, deadline: float) -> tuple[_AddressInfo, ...]: + request = _ResolveRequest(host, port, queue.Queue(maxsize=1), threading.Event()) + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise queue.Full + self._requests.put(request, timeout=remaining) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise queue.Empty + addresses, error = request.result.get(timeout=remaining) + except (queue.Empty, queue.Full): + request.canceled.set() + raise CommonsTransportError("Commons transport failed") from None + if error is not None or not addresses or time.monotonic() >= deadline: + raise CommonsTransportError("Commons transport failed") from None + return addresses + + +_SHARED_RESOLVER = _SharedResolver() + + @dataclass(frozen=True, slots=True) class CommonsAck: """The only accepted evidence response shape.""" @@ -96,10 +160,9 @@ def _positive_timeout(value: float, *, label: str) -> float: class CommonsClient: """Issue fixed-path bounded requests. - The monotonic request deadline covers socket connect, request send, headers, and body once - ``getaddrinfo`` returns. Python's synchronous stdlib resolver cannot preempt a blocked DNS - lookup without a helper thread, so DNS delay is checked and failed open immediately after - resolution rather than claimed as a hard cancellable deadline. + The monotonic request deadline covers queued resolution, connect, request send, headers, and + body. Resolution runs on one shared daemon worker, so a stalled resolver cannot block callers + or create retry-proportional threads; later calls fail within their remaining budget. """ def __init__( @@ -199,13 +262,38 @@ def _request( success_status: object, ) -> bytes: deadline = time.monotonic() + self._request_timeout + port = origin.port or (443 if origin.scheme == "https" else 80) + addresses = _SHARED_RESOLVER.resolve(origin.host, port, deadline=deadline) connection = self._connection( origin, timeout=self._remaining(deadline, cap=self._connect_timeout), ) + + def connect_resolved(*args: Any, **kwargs: Any) -> socket.socket: + source_address = cast( + tuple[str, int] | None, + args[2] if len(args) > 2 else kwargs.get("source_address"), + ) + last_error: OSError | None = None + for family, socktype, proto, _canonical_name, sockaddr in addresses: + sock = socket.socket(family, socktype, proto) + try: + sock.settimeout(self._remaining(deadline, cap=self._connect_timeout)) + if source_address is not None: + sock.bind(source_address) + sock.connect(sockaddr) + return sock + except OSError as exc: + last_error = exc + sock.close() + if last_error is not None: + raise last_error + raise OSError("Commons connection failed") + + connection._create_connection = connect_resolved # type: ignore[attr-defined] socket_holder: list[socket.socket | None] = [None] deadline_guard = threading.Timer( - self._request_timeout, + self._remaining(deadline), self._abort_socket, args=(socket_holder,), ) diff --git a/tests/commons/test_client.py b/tests/commons/test_client.py index 1296a16..c3d7144 100644 --- a/tests/commons/test_client.py +++ b/tests/commons/test_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import socket import threading import time from collections.abc import Iterator @@ -221,6 +222,45 @@ def test_slow_trickle_hits_one_monotonic_request_deadline() -> None: assert elapsed < 0.3 +def test_stalled_dns_is_bounded_and_retries_do_not_grow_threads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release = threading.Event() + entered = threading.Event() + + def stalled_getaddrinfo(*_args: object, **_kwargs: object) -> object: + entered.set() + release.wait(timeout=2) + raise OSError("synthetic stalled resolver") + + monkeypatch.setattr(socket, "getaddrinfo", stalled_getaddrinfo) + client = CommonsClient( + pack_origin="https://stalled.example", + ingress_origin="https://stalled.example", + connect_timeout=1.0, + read_timeout=1.0, + request_timeout=0.08, + ) + resolver_threads_before = [ + thread for thread in threading.enumerate() if thread.name == "marginal-commons-resolver" + ] + started = time.monotonic() + try: + for _ in range(3): + with pytest.raises(CommonsTransportError, match="Commons transport failed"): + client.download() + elapsed = time.monotonic() - started + assert entered.wait(timeout=0.2) + resolver_threads_after = [ + thread for thread in threading.enumerate() if thread.name == "marginal-commons-resolver" + ] + assert len(resolver_threads_before) == 1 + assert resolver_threads_after == resolver_threads_before + assert elapsed < 0.4 + finally: + release.set() + + @pytest.mark.parametrize("status", [422, 503]) def test_non_2xx_status_is_classified_before_oversized_body_validation( tmp_path: Path, status: int From 14235da66812bc7aa8f0d3cab9fa2c25171f1498 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:33:33 +0200 Subject: [PATCH 11/14] feat: integrate Commons with Codex lifecycle --- src/marginal/cli.py | 3 +- src/marginal/diagnostics.py | 70 ++++++- src/marginal/integrations/codex/runtime.py | 10 + src/marginal/integrations/codex/service.py | 222 ++++++++++++++++++++- tests/commons/test_enforcement.py | 42 ++++ tests/integrations/codex/test_service.py | 119 ++++++++++- tests/test_diagnostics.py | 40 ++++ 7 files changed, 492 insertions(+), 14 deletions(-) create mode 100644 tests/commons/test_enforcement.py diff --git a/src/marginal/cli.py b/src/marginal/cli.py index 62ea691..500c025 100644 --- a/src/marginal/cli.py +++ b/src/marginal/cli.py @@ -197,6 +197,7 @@ def _build_parser() -> argparse.ArgumentParser: privacy = subparsers.add_parser("privacy", help="inspect local persistence categories") privacy_commands = privacy.add_subparsers(dest="privacy_command", required=True) privacy_inspect = privacy_commands.add_parser("inspect", help="show persisted data categories") + privacy_inspect.add_argument("--data-dir", type=Path) privacy_inspect.add_argument("--json", action="store_true", dest="as_json") return parser @@ -274,7 +275,7 @@ def main(argv: Sequence[str] | None = None) -> int: ).to_dict() exit_code = 0 if payload["found"] is True else 1 else: - payload = inspect_privacy().to_dict() + payload = inspect_privacy(data_root=data_dir).to_dict() exit_code = 0 if args.as_json: print(json.dumps(payload, sort_keys=True)) diff --git a/src/marginal/diagnostics.py b/src/marginal/diagnostics.py index a7ba027..67bd2b0 100644 --- a/src/marginal/diagnostics.py +++ b/src/marginal/diagnostics.py @@ -9,6 +9,9 @@ from pathlib import Path from typing import Any +from .commons.config import CommonsMode, load_commons_config +from .commons.identity import is_canonical_namespace +from .commons.sync import SyncFailure from .governance_ledger import GovernanceLedger from .integrations.codex.evidence import ( EvidenceStore, @@ -23,7 +26,7 @@ evaluate_promotion, read_promotion_receipt, ) -from .integrations.codex.service import read_mode +from .integrations.codex.service import _COMMONS_INGRESS_ORIGIN, read_mode _PERSISTED_CATEGORIES = ( "derived_enums", @@ -92,11 +95,24 @@ def to_dict(self) -> dict[str, object]: class PrivacyInspectionReport: """The local persistence contract, without inspecting user content.""" + commons: dict[str, object] | None = None + def to_dict(self) -> dict[str, object]: + commons = self.commons or { + "mode": CommonsMode.LOCAL_ONLY.value, + "endpoint": _COMMONS_INGRESS_ORIGIN, + "model_namespace": None, + "sharing_allowed": False, + "safe_queue_count": 0, + "last_sync_status": "not_attempted", + "cache_revision": None, + "schema_version": "1.0", + } return { "persisted_categories": list(_PERSISTED_CATEGORIES), "never_persisted": list(_NEVER_PERSISTED), - "local_only": True, + "local_only": commons["mode"] == CommonsMode.LOCAL_ONLY.value, + "commons": dict(commons), "dictionary_attack_limit": ( "Derived hashes can reveal low-entropy inputs to a party with local ledger access." ), @@ -291,8 +307,54 @@ def decision_explanation( return DecisionExplanationReport(decision_id, False, "DECISION_NOT_FOUND") -def inspect_privacy() -> PrivacyInspectionReport: - return PrivacyInspectionReport() +def inspect_privacy(*, data_root: str | Path | None = None) -> PrivacyInspectionReport: + if data_root is None: + return PrivacyInspectionReport() + root = Path(data_root).resolve() + try: + configured_mode = load_commons_config(root).mode + except Exception: + configured_mode = CommonsMode.LOCAL_ONLY + try: + raw = json.loads((root / "commons" / "status.json").read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + raw = {} + if not isinstance(raw, dict): + raw = {} + try: + mode = CommonsMode.parse(raw.get("mode", configured_mode.value)) + except (TypeError, ValueError): + mode = configured_mode + namespace = raw.get("model_namespace") + if not is_canonical_namespace(namespace): + namespace = None + queue_count = raw.get("safe_queue_count") + if ( + isinstance(queue_count, bool) + or not isinstance(queue_count, int) + or not 0 <= queue_count <= 100 + ): + queue_count = 0 + revision = raw.get("cache_revision") + if isinstance(revision, bool) or not isinstance(revision, int) or revision < 1: + revision = None + sync_status = raw.get("last_sync_status") + allowed_statuses = {"not_attempted", "ok"} | {failure.value for failure in SyncFailure} + if not isinstance(sync_status, str) or any( + part not in allowed_statuses for part in sync_status.split("+") + ): + sync_status = "not_attempted" + commons = { + "mode": mode.value, + "endpoint": _COMMONS_INGRESS_ORIGIN, + "model_namespace": namespace, + "sharing_allowed": mode is CommonsMode.CONTRIBUTOR and namespace is not None, + "safe_queue_count": queue_count, + "last_sync_status": sync_status, + "cache_revision": revision, + "schema_version": "1.0", + } + return PrivacyInspectionReport(commons) def render_human(payload: dict[str, object]) -> str: diff --git a/src/marginal/integrations/codex/runtime.py b/src/marginal/integrations/codex/runtime.py index 5523270..d772d35 100644 --- a/src/marginal/integrations/codex/runtime.py +++ b/src/marginal/integrations/codex/runtime.py @@ -284,11 +284,21 @@ def _is_autopilot_eligible(self, event: PreToolUseEvent, action: AgentAction) -> @staticmethod def _safe_action_evidence(action: AgentAction) -> dict[str, str]: + action_kinds = { + "edit": "file_write", + "shell": "command", + "verification": "verification", + } + tool_name = str(action.metadata.get("tool_name", "")).casefold() + action_kind = action_kinds.get(action.kind, "tool") + if tool_name in {"read", "read_file"}: + action_kind = "file_read" return { "action_hash": hashlib.sha256(action.action_id.encode("utf-8")).hexdigest(), "semantic_key": str(action.metadata.get("semantic_key", "")), "state_hash": action.state_hash, "evidence_hash": str(action.metadata.get("evidence_hash", "")), + "action_kind": action_kind, } @staticmethod diff --git a/src/marginal/integrations/codex/service.py b/src/marginal/integrations/codex/service.py index daa7514..4d2738a 100644 --- a/src/marginal/integrations/codex/service.py +++ b/src/marginal/integrations/codex/service.py @@ -8,6 +8,7 @@ import secrets import subprocess import sys +import tempfile import threading import time from contextlib import suppress @@ -16,6 +17,13 @@ from typing import Any from marginal import BudgetLimits, Treasury +from marginal.commons.cache import CommonsCache, CommonsPrior +from marginal.commons.client import CommonsClient, CommonsClientProtocol +from marginal.commons.config import CommonsConfig, CommonsMode, load_commons_config +from marginal.commons.evidence import compile_verified_evidence +from marginal.commons.identity import CanonicalModelIdentity, resolve_canonical_model +from marginal.commons.outbox import CommonsOutbox +from marginal.commons.sync import CommonsSyncResult, synchronize_commons from marginal.protocol import AgentCapabilities from marginal.reason_codes import ReasonCode from marginal.runtime import UniversalRuntime @@ -37,6 +45,20 @@ from .transport import ConnectionInfo, SessionServer, connection_filename, request_session _SERVERS: dict[tuple[Path, str], tuple[SessionServer, CodexSessionRuntime]] = {} +_COMMONS_PACK_ORIGIN = "https://marginal-commons.pages.dev" +_COMMONS_INGRESS_ORIGIN = "https://marginal-ingress.signallayerlabs.workers.dev" +_COMMONS_SOURCE_COMMIT = "7347a1b4024329780139d17494430f2ccac94fec" + + +@dataclass(slots=True) +class _CommonsSession: + config: CommonsConfig + identity: CanonicalModelIdentity | None = None + cache: CommonsCache | None = None + outbox: CommonsOutbox | None = None + client: CommonsClientProtocol | None = None + priors: tuple[CommonsPrior, ...] = () + attribution_valid: bool = True @dataclass(frozen=True, slots=True) @@ -46,6 +68,168 @@ class HookResult: warning_code: str = "" +def _commons_client() -> CommonsClientProtocol: + return CommonsClient( + pack_origin=_COMMONS_PACK_ORIGIN, + ingress_origin=_COMMONS_INGRESS_ORIGIN, + ) + + +def _commons_status_path(data_root: Path) -> Path: + return data_root / "commons" / "status.json" + + +def _safe_queue_count(outbox: CommonsOutbox | None) -> int: + if outbox is None: + return 0 + try: + return len(outbox.pending(limit=100).entries) + except Exception: + return 0 + + +def _write_commons_status( + data_root: Path, + commons: _CommonsSession, + result: CommonsSyncResult | None, +) -> None: + path = _commons_status_path(data_root) + payload = { + "schema_version": "1.0", + "mode": commons.config.mode.value, + "endpoint": _COMMONS_INGRESS_ORIGIN, + "model_namespace": commons.identity.namespace if commons.identity is not None else None, + "sharing_allowed": ( + commons.config.mode is CommonsMode.CONTRIBUTOR and commons.identity is not None + ), + "safe_queue_count": _safe_queue_count(commons.outbox), + "last_sync_status": ( + "not_attempted" + if result is None + else "ok" + if not result.failures + else "+".join(failure.value for failure in result.failures) + ), + "cache_revision": commons.cache.revision if commons.cache is not None else None, + } + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if os.name == "posix": + path.parent.chmod(0o700) + encoded = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + descriptor, name = tempfile.mkstemp(prefix=".status-", suffix=".tmp", dir=path.parent) + temporary = Path(name) + try: + os.fchmod(descriptor, 0o600) + os.write(descriptor, encoded) + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + os.replace(temporary, path) + if os.name == "posix": + path.chmod(0o600) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +def _start_commons(data_root: Path, *, model: str) -> _CommonsSession: + try: + config = load_commons_config(data_root) + except Exception: + config = CommonsConfig() + identity = resolve_canonical_model(provider="openai", model=model) + commons = _CommonsSession(config=config, identity=identity) + if config.mode is CommonsMode.LOCAL_ONLY or identity is None: + with suppress(Exception): + _write_commons_status(data_root, commons, None) + return commons + try: + commons.cache = CommonsCache( + data_root, + model_namespace=identity.namespace, + expected_source_commit=_COMMONS_SOURCE_COMMIT, + ) + commons.outbox = CommonsOutbox(data_root) + commons.client = _commons_client() + result = synchronize_commons( + config, + cache=commons.cache, + outbox=commons.outbox, + client=commons.client, + ) + commons.priors = commons.cache.load_prior() + _write_commons_status(data_root, commons, result) + except Exception: + with suppress(Exception): + _write_commons_status(data_root, commons, None) + return commons + + +def _finalize_local_session( + runtime: CodexSessionRuntime, + evidence_store: EvidenceStore, + *, + session_hash: str, +) -> bool: + runtime.close() + report = evidence_store.verified_governance_root() + try: + checkpoint = evidence_store.read_checkpoint() + except (OSError, ValueError, json.JSONDecodeError): + checkpoint = None + if ( + report.valid + and checkpoint is not None + and checkpoint.get("event") == "session_finalized" + and checkpoint.get("session_hash") == session_hash + and checkpoint.get("ledger_root") == report.root_hash + and checkpoint.get("ledger_records") == report.records + ): + return False + evidence_store.append( + {"schema_version": 1, "event": "session_end", "session_hash": session_hash} + ) + finalized = evidence_store.verified_governance_root() + if not finalized.valid: + return False + evidence_store.write_checkpoint( + { + "schema_version": 1, + "event": "session_finalized", + "session_hash": session_hash, + "ledger_root": finalized.root_hash, + "ledger_records": finalized.records, + } + ) + return True + + +def _end_commons( + data_root: Path, + commons: _CommonsSession, + evidence_store: EvidenceStore, +) -> None: + if ( + commons.config.mode is not CommonsMode.CONTRIBUTOR + or commons.identity is None + or not commons.attribution_valid + or commons.cache is None + or commons.outbox is None + or commons.client is None + ): + return + evidence = compile_verified_evidence(evidence_store, model_identity=commons.identity) + result = synchronize_commons( + commons.config, + cache=commons.cache, + outbox=commons.outbox, + client=commons.client, + evidence=evidence, + ) + _write_commons_status(data_root, commons, result) + + def _connection_path(data_root: Path, session_id: str) -> Path: return data_root / "sessions" / connection_filename(session_id) @@ -57,6 +241,7 @@ def _handler( session_hash: str, data_root: Path, identity: PromotionIdentity, + commons: _CommonsSession, shutdown_event: threading.Event | None = None, ) -> Any: def handle(operation: str, payload: dict[str, Any]) -> dict[str, Any] | None: @@ -64,20 +249,21 @@ def handle(operation: str, payload: dict[str, Any]) -> dict[str, Any] | None: return { **runtime.summary(), "repository_hash": identity.repository_hash, + "commons_prior_count": len(commons.priors), } if operation == "close": - runtime.close() - evidence_store.append( - { - "schema_version": 1, - "event": "session_end", - "session_hash": session_hash, - } - ) + finalized = _finalize_local_session(runtime, evidence_store, session_hash=session_hash) + if finalized: + with suppress(Exception): + _end_commons(data_root, commons, evidence_store) if shutdown_event is not None: threading.Timer(0.05, shutdown_event.set).start() return runtime.summary() event = parse_hook_event(payload) + if getattr(event, "model", None) != ( + commons.identity.model if commons.identity is not None else None + ): + commons.attribution_valid = False if operation == "prompt" and isinstance(event, UserPromptSubmitEvent): runtime.user_prompt_submit(event) return None @@ -93,6 +279,17 @@ def handle(operation: str, payload: dict[str, Any]) -> dict[str, Any] | None: "event": "decision", "session_hash": session_hash, **action_evidence, + **( + { + "model_namespace": commons.identity.namespace, + "cost_bucket": "unknown", + "gain_bucket": "unknown", + "recommendation": "allow" if decision.allowed else "deny", + "applied_decision": "allow" if decision.allowed else "deny", + } + if commons.identity is not None and commons.attribution_valid + else {} + ), "reason_code": decision.reason_code, "latency_ms": latency_ms, "covered": decision.reason_code != ReasonCode.CONTROL_PLANE_BYPASS.value, @@ -120,6 +317,11 @@ def handle(operation: str, payload: dict[str, Any]) -> dict[str, Any] | None: "event": "outcome", "session_hash": session_hash, **action_evidence, + **( + {"model_namespace": commons.identity.namespace} + if commons.identity is not None and commons.attribution_valid + else {} + ), "outcome": outcome.value, "pending": False, } @@ -276,6 +478,7 @@ def _serve_bootstrap(path: Path) -> int: evidence_store.append( {"schema_version": 1, "event": "session_start", "session_hash": session_hash} ) + commons = _start_commons(data_root, model=event.model) treasury = Treasury(BudgetLimits(), mode="shadow") universal = UniversalRuntime( treasury, @@ -313,6 +516,7 @@ def _serve_bootstrap(path: Path) -> int: session_hash=session_hash, data_root=data_root, identity=identity, + commons=commons, shutdown_event=shutdown_event, ), ) @@ -345,6 +549,7 @@ def start_session_service( evidence_store.append( {"schema_version": 1, "event": "session_start", "session_hash": session_hash} ) + commons = _start_commons(root, model=event.model) treasury = Treasury(BudgetLimits(), mode="shadow") universal = UniversalRuntime( treasury, @@ -381,6 +586,7 @@ def start_session_service( session_hash=session_hash, data_root=root, identity=identity, + commons=commons, ), ) connection = server.start() diff --git a/tests/commons/test_enforcement.py b/tests/commons/test_enforcement.py new file mode 100644 index 0000000..760e4d4 --- /dev/null +++ b/tests/commons/test_enforcement.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from marginal.diagnostics import status_report + + +@pytest.mark.parametrize("lifecycle", ["candidate", "supported", "validated", "promoted"]) +def test_commons_lifecycle_state_has_no_local_enforcement_authority( + tmp_path: Path, lifecycle: str +) -> None: + workspace = tmp_path / "repository" + workspace.mkdir() + commons = tmp_path / "data" / "commons" + commons.mkdir(parents=True) + (commons / "status.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "mode": "read_only", + "endpoint": "https://marginal-ingress.signallayerlabs.workers.dev", + "model_namespace": "openai/gpt-5.6-sol", + "sharing_allowed": False, + "safe_queue_count": 0, + "last_sync_status": "ok", + "cache_revision": 1, + "lifecycle": lifecycle, + "count": 1000, + } + ), + encoding="utf-8", + ) + + payload = status_report(data_root=tmp_path / "data", workspace=workspace).to_dict() + + assert payload["authority"]["current"] == "L0" + assert payload["authority"]["eligible"] == "L0" + assert payload["trust"]["components"]["covered_actions"] == 0 + assert payload["trust"]["components"]["completed_sessions"] == 0 diff --git a/tests/integrations/codex/test_service.py b/tests/integrations/codex/test_service.py index 9d50412..6427755 100644 --- a/tests/integrations/codex/test_service.py +++ b/tests/integrations/codex/test_service.py @@ -5,7 +5,11 @@ from dataclasses import asdict, replace from pathlib import Path +import pytest + import marginal.integrations.codex.service as service_module +from marginal.commons.config import CommonsMode, configure_commons_mode +from marginal.commons.outbox import CommonsOutbox from marginal.integrations.codex.events import SessionEvent from marginal.integrations.codex.evidence import ( EvidenceStore, @@ -28,7 +32,7 @@ start_session_service, stop_session_service, ) -from marginal.integrations.codex.transport import connection_filename +from marginal.integrations.codex.transport import connection_filename, request_session def _git(repo: Path, *args: str) -> None: @@ -228,6 +232,119 @@ def test_session_start_and_end_are_complete_hook_lifecycle(tmp_path: Path) -> No assert not (data / "sessions" / connection_filename("session-1")).exists() +class _OfflineCommonsClient: + def __init__(self) -> None: + self.downloads = 0 + self.submissions = 0 + + def download(self) -> bytes: + self.downloads += 1 + raise TimeoutError("private network detail") + + def submit(self, _entry) -> object: + self.submissions += 1 + raise TimeoutError("private network detail") + + +@pytest.mark.parametrize( + "mode,expected_downloads", + [ + (CommonsMode.LOCAL_ONLY, 0), + (CommonsMode.READ_ONLY, 1), + (CommonsMode.CONTRIBUTOR, 2), + ], +) +def test_session_lifecycle_finalizes_locally_in_every_commons_mode_and_fails_open( + tmp_path: Path, monkeypatch, mode: CommonsMode, expected_downloads: int +) -> None: + workspace = tmp_path / "repo" + workspace.mkdir() + _repository(workspace) + data = tmp_path / "data" + configure_commons_mode(data, mode=mode) + client = _OfflineCommonsClient() + monkeypatch.setattr(service_module, "_commons_client", lambda: client) + + connection = start_session_service(_start(workspace), data_root=data) + stop_session_service("session-1", data_root=data) + stop_session_service("session-1", data_root=data) + + identity = current_promotion_identity(workspace) + store = EvidenceStore(data / "evidence" / identity.repository_hash) + records = store.read_all() + assert sum(record.get("event") == "session_end" for record in records) == 1 + checkpoint = store.read_checkpoint() + assert checkpoint is not None + ledger = store.verified_governance_root() + assert checkpoint == { + "schema_version": 1, + "event": "session_finalized", + "session_hash": service_module._session_hash("session-1"), + "ledger_root": ledger.root_hash, + "ledger_records": ledger.records, + } + assert client.downloads == expected_downloads + assert connection.connection_file.exists() is False + + +def test_contributor_session_end_queues_model_bound_evidence_for_offline_retry( + tmp_path: Path, monkeypatch +) -> None: + workspace = tmp_path / "repo" + workspace.mkdir() + _repository(workspace) + data = tmp_path / "data" + configure_commons_mode(data, mode=CommonsMode.CONTRIBUTOR) + client = _OfflineCommonsClient() + monkeypatch.setattr(service_module, "_commons_client", lambda: client) + connection = start_session_service(_start(workspace), data_root=data) + common = { + "session_id": "session-1", + "cwd": str(workspace), + "model": "gpt-5.6-sol", + "permission_mode": "default", + "turn_id": "turn-1", + "tool_name": "Read", + "tool_input": {"path": str(workspace / "tracked.txt")}, + } + for index in range(5): + pre = {**common, "hook_event_name": "PreToolUse", "tool_use_id": f"call-{index}"} + post = {**pre, "hook_event_name": "PostToolUse", "tool_response": {"exit_code": 0}} + assert request_session(connection, operation="pre", payload=pre)["ok"] is True + assert request_session(connection, operation="post", payload=post)["ok"] is True + + stop_session_service("session-1", data_root=data) + + queued = CommonsOutbox(data).pending(limit=8).entries + assert len(queued) == 1 + assert queued[0].model_namespace == "openai/gpt-5.6-sol" + assert client.submissions == 1 + identity = current_promotion_identity(workspace) + records = EvidenceStore(data / "evidence" / identity.repository_hash).read_all() + dimensions = [record for record in records if record.get("event") == "decision"] + assert {record.get("model_namespace") for record in dimensions} == {"openai/gpt-5.6-sol"} + assert {record.get("action_kind") for record in dimensions} == {"file_read"} + + +def test_ambiguous_model_session_remains_local_and_never_queues( + tmp_path: Path, monkeypatch +) -> None: + workspace = tmp_path / "repo" + workspace.mkdir() + _repository(workspace) + data = tmp_path / "data" + configure_commons_mode(data, mode=CommonsMode.CONTRIBUTOR) + client = _OfflineCommonsClient() + monkeypatch.setattr(service_module, "_commons_client", lambda: client) + event = replace(_start(workspace), model="private/fine-tune") + + start_session_service(event, data_root=data) + stop_session_service("session-1", data_root=data) + + assert CommonsOutbox(data).pending(limit=8).entries == () + assert client.submissions == 0 + + def test_user_prompt_submit_reaches_only_the_authenticated_session_and_is_not_persisted( tmp_path: Path, ) -> None: diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 5b3008c..f0cedf4 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path from marginal.diagnostics import ( @@ -98,6 +99,45 @@ def test_privacy_inspection_lists_persisted_categories_and_exclusions() -> None: assert "credentials" in payload["never_persisted"] +def test_privacy_inspection_reports_only_safe_commons_operational_state(tmp_path: Path) -> None: + status = tmp_path / "commons" / "status.json" + status.parent.mkdir(parents=True) + status.write_text( + json.dumps( + { + "schema_version": "1.0", + "mode": "contributor", + "endpoint": "https://marginal-ingress.signallayerlabs.workers.dev", + "model_namespace": "openai/gpt-5.6-sol", + "sharing_allowed": True, + "safe_queue_count": 2, + "last_sync_status": "submit_transport", + "cache_revision": 7, + "repository_hash": "must-not-escape", + "remote_identity": "must-not-escape", + } + ), + encoding="utf-8", + ) + + payload = inspect_privacy(data_root=tmp_path).to_dict() + + assert payload["commons"] == { + "mode": "contributor", + "endpoint": "https://marginal-ingress.signallayerlabs.workers.dev", + "model_namespace": "openai/gpt-5.6-sol", + "sharing_allowed": True, + "safe_queue_count": 2, + "last_sync_status": "submit_transport", + "cache_revision": 7, + "schema_version": "1.0", + } + serialized = json.dumps(payload) + assert "must-not-escape" not in serialized + assert "remote_identity" not in serialized + assert "repository_hash" not in serialized + + def test_status_reports_unvalidated_enforcement_as_configured_but_not_effective( tmp_path: Path, ) -> None: From 272fc4d79401de7131ea9eb98b7dad5d344b45f6 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:43:33 +0200 Subject: [PATCH 12/14] fix: export each finalized Commons range once --- src/marginal/commons/evidence.py | 18 +++++ src/marginal/commons/outbox.py | 48 +++++++++++- src/marginal/integrations/codex/service.py | 91 +++++++++++++++++++++- tests/integrations/codex/test_service.py | 43 ++++++++++ 4 files changed, 195 insertions(+), 5 deletions(-) diff --git a/src/marginal/commons/evidence.py b/src/marginal/commons/evidence.py index 4091a5a..4ccfe38 100644 --- a/src/marginal/commons/evidence.py +++ b/src/marginal/commons/evidence.py @@ -235,6 +235,8 @@ def compile_verified_evidence( *, model_identity: CanonicalModelIdentity | None, minimum_group_size: int = 5, + after_records: int = 0, + through_records: int | None = None, ) -> CommonsEvidenceBatch | None: """Compile a verified local chain; arbitrary caller rows are never accepted.""" @@ -244,6 +246,8 @@ def compile_verified_evidence( raise ValueError("minimum_group_size must be between 1 and 1000") if not isinstance(evidence_store, EvidenceStore): raise TypeError("evidence_store must be an EvidenceStore") + if isinstance(after_records, bool) or not isinstance(after_records, int) or after_records < 0: + raise ValueError("after_records must be a non-negative integer") if model_identity is None or not identity_is_canonical(model_identity): return None try: @@ -252,6 +256,20 @@ def compile_verified_evidence( return None if not verification.valid: return None + end = verification.records if through_records is None else through_records + if ( + isinstance(end, bool) + or not isinstance(end, int) + or not after_records <= end <= len(records) + ): + return None + records = records[after_records:end] + if after_records != 0 or through_records is not None: + records = [ + record + for record in records + if record.get("model_namespace") == model_identity.namespace + ] attributed_namespaces = { namespace for record in records diff --git a/src/marginal/commons/outbox.py b/src/marginal/commons/outbox.py index 8eeb86f..88ca7e3 100644 --- a/src/marginal/commons/outbox.py +++ b/src/marginal/commons/outbox.py @@ -51,6 +51,7 @@ class OutboxEntry: record_sha256: str device: int inode: int + export_receipt: str | None = None def body(self) -> bytes: """Serialize only the closed wire envelope, excluding local metadata.""" @@ -135,18 +136,30 @@ def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> Ou payload: Any = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ValueError("queued Commons record is malformed") from exc - if not isinstance(payload, dict) or set(payload) != {"retry_token", "envelope"}: + if not isinstance(payload, dict) or set(payload) not in ( + {"retry_token", "envelope"}, + {"retry_token", "envelope", "export_receipt"}, + ): raise ValueError("queued Commons record has an invalid shape") retry_token = payload["retry_token"] if not isinstance(retry_token, str) or _TOKEN_PATTERN.fullmatch(retry_token) is None: raise ValueError("queued Commons retry token is invalid") + export_receipt = payload.get("export_receipt") + if export_receipt is not None and ( + not isinstance(export_receipt, str) or _DIGEST_PATTERN.fullmatch(export_receipt) is None + ): + raise ValueError("queued Commons export receipt is invalid") envelope = _validate_envelope(payload["envelope"]) body_bytes = ( json.dumps(envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" ).encode("utf-8") canonical_record = ( json.dumps( - {"envelope": envelope, "retry_token": retry_token}, + { + "envelope": envelope, + "retry_token": retry_token, + **({"export_receipt": export_receipt} if export_receipt is not None else {}), + }, sort_keys=True, separators=(",", ":"), ensure_ascii=False, @@ -161,6 +174,7 @@ def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> Ou record_sha256=hashlib.sha256(canonical_record).hexdigest(), device=device, inode=inode, + export_receipt=export_receipt, ) @@ -205,6 +219,7 @@ def enqueue( self, *, batch: CommonsEvidenceBatch, + export_receipt: str | None = None, ) -> OutboxEntry | None: """Atomically queue nonempty typed evidence with one random retry token.""" @@ -212,6 +227,8 @@ def enqueue( return None if len(batch.atoms) > _MAX_ATOMS: return None + if export_receipt is not None and _DIGEST_PATTERN.fullmatch(export_receipt) is None: + raise ValueError("Commons export receipt is invalid") envelope: dict[str, object] = { "schema_version": "1.0", "model_namespace": batch.model_namespace, @@ -220,7 +237,11 @@ def enqueue( retry_token = secrets.token_urlsafe(32) encoded = ( json.dumps( - {"envelope": envelope, "retry_token": retry_token}, + { + "envelope": envelope, + "retry_token": retry_token, + **({"export_receipt": export_receipt} if export_receipt is not None else {}), + }, sort_keys=True, separators=(",", ":"), ensure_ascii=False, @@ -230,6 +251,27 @@ def enqueue( if len(encoded) > _MAX_QUEUE_BYTES: return None with locked_directory(self.queue_path, create=True, lock_name=".outbox.lock") as directory: + if export_receipt is not None: + for existing_name in os.listdir(directory): + if existing_name.startswith("."): + continue + try: + raw, metadata = read_bounded_at( + directory, + existing_name, + maximum_bytes=_MAX_QUEUE_BYTES, + label="Commons outbox entry", + ) + existing = _parse_queue_record( + raw, + name=existing_name, + device=metadata.st_dev, + inode=metadata.st_ino, + ) + except (OSError, ValueError): + continue + if existing.export_receipt == export_receipt: + return existing for _ in range(16): name = f"queue-{secrets.token_hex(16)}.json" try: diff --git a/src/marginal/integrations/codex/service.py b/src/marginal/integrations/codex/service.py index 4d2738a..f75baad 100644 --- a/src/marginal/integrations/codex/service.py +++ b/src/marginal/integrations/codex/service.py @@ -17,6 +17,7 @@ from typing import Any from marginal import BudgetLimits, Treasury +from marginal.commons._storage import atomic_replace_at, locked_directory, read_bounded_at from marginal.commons.cache import CommonsCache, CommonsPrior from marginal.commons.client import CommonsClient, CommonsClientProtocol from marginal.commons.config import CommonsConfig, CommonsMode, load_commons_config @@ -219,17 +220,103 @@ def _end_commons( or commons.client is None ): return - evidence = compile_verified_evidence(evidence_store, model_identity=commons.identity) + cursor = _read_export_cursor(evidence_store, commons.identity) + if cursor is None: + return + report = evidence_store.verified_governance_root() + if not report.valid or report.root_hash is None or report.records <= cursor[0]: + return + evidence = compile_verified_evidence( + evidence_store, + model_identity=commons.identity, + after_records=cursor[0], + through_records=report.records, + ) + receipt = hashlib.sha256( + f"{commons.identity.namespace}:{cursor[0]}:{report.records}:{report.root_hash}".encode() + ).hexdigest() + if evidence is not None: + queued = commons.outbox.enqueue(batch=evidence, export_receipt=receipt) + if queued is None: + return + _write_export_cursor(evidence_store, commons.identity, report.records, report.root_hash) result = synchronize_commons( commons.config, cache=commons.cache, outbox=commons.outbox, client=commons.client, - evidence=evidence, ) _write_commons_status(data_root, commons, result) +def _export_cursor_name(identity: CanonicalModelIdentity) -> str: + return f"{identity.model}.json" + + +def _read_export_cursor( + store: EvidenceStore, identity: CanonicalModelIdentity +) -> tuple[int, str] | None: + directory_path = store.root / "commons-export" + try: + with locked_directory(directory_path, create=False, lock_name=".export.lock") as directory: + raw, _ = read_bounded_at( + directory, + _export_cursor_name(identity), + maximum_bytes=4096, + label="Commons export cursor", + ) + except FileNotFoundError: + return (0, "") + except OSError: + return None + try: + payload = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if ( + not isinstance(payload, dict) + or set(payload) != {"schema_version", "model_namespace", "ledger_records", "ledger_root"} + or payload.get("schema_version") != 1 + or payload.get("model_namespace") != identity.namespace + or isinstance(payload.get("ledger_records"), bool) + or not isinstance(payload.get("ledger_records"), int) + or not isinstance(payload.get("ledger_root"), str) + ): + return None + records = payload["ledger_records"] + root = payload["ledger_root"] + if records < 0 or ( + records and not store.verifies_governance_prefix(root_hash=root, records=records) + ): + return None + return records, root + + +def _write_export_cursor( + store: EvidenceStore, + identity: CanonicalModelIdentity, + records: int, + root: str, +) -> None: + payload = { + "schema_version": 1, + "model_namespace": identity.namespace, + "ledger_records": records, + "ledger_root": root, + } + encoded = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + with locked_directory( + store.root / "commons-export", create=True, lock_name=".export.lock" + ) as directory: + atomic_replace_at( + directory, + _export_cursor_name(identity), + encoded, + temporary_prefix=".export-cursor-", + label="Commons export cursor", + ) + + def _connection_path(data_root: Path, session_id: str) -> Path: return data_root / "sessions" / connection_filename(session_id) diff --git a/tests/integrations/codex/test_service.py b/tests/integrations/codex/test_service.py index 6427755..d925c3e 100644 --- a/tests/integrations/codex/test_service.py +++ b/tests/integrations/codex/test_service.py @@ -326,6 +326,49 @@ def test_contributor_session_end_queues_model_bound_evidence_for_offline_retry( assert {record.get("action_kind") for record in dimensions} == {"file_read"} +def test_contributor_exports_only_each_new_finalized_range(tmp_path: Path, monkeypatch) -> None: + workspace = tmp_path / "repo" + workspace.mkdir() + _repository(workspace) + data = tmp_path / "data" + configure_commons_mode(data, mode=CommonsMode.CONTRIBUTOR) + client = _OfflineCommonsClient() + monkeypatch.setattr(service_module, "_commons_client", lambda: client) + + def run_session(session_id: str, actions: int) -> None: + connection = start_session_service( + replace(_start(workspace), session_id=session_id), data_root=data + ) + for index in range(actions): + pre = { + "session_id": session_id, + "cwd": str(workspace), + "model": "gpt-5.6-sol", + "permission_mode": "default", + "turn_id": "turn-1", + "tool_name": "Read", + "tool_input": {"path": str(workspace / "tracked.txt")}, + "hook_event_name": "PreToolUse", + "tool_use_id": f"{session_id}-call-{index}", + } + post = {**pre, "hook_event_name": "PostToolUse", "tool_response": {"exit_code": 0}} + request_session(connection, operation="pre", payload=pre) + request_session(connection, operation="post", payload=post) + stop_session_service(session_id, data_root=data) + + run_session("session-1", 5) + first = CommonsOutbox(data).pending(limit=8).entries + assert len(first) == 1 + run_session("session-empty", 0) + assert [entry.name for entry in CommonsOutbox(data).pending(limit=8).entries] == [first[0].name] + run_session("session-2", 5) + + queued = CommonsOutbox(data).pending(limit=8).entries + assert len(queued) == 2 + exported_counts = sorted(atom["count"] for entry in queued for atom in entry.envelope["atoms"]) + assert exported_counts == [5, 5] + + def test_ambiguous_model_session_remains_local_and_never_queues( tmp_path: Path, monkeypatch ) -> None: From 657f9040d680d72cbc40cd97639a520a44ffd951 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:45:19 +0200 Subject: [PATCH 13/14] fix: recover Commons cursor from pending receipt --- src/marginal/commons/outbox.py | 6 ++- src/marginal/integrations/codex/service.py | 60 +++++++++++++++++++--- tests/integrations/codex/test_service.py | 48 +++++++++++++++++ 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/src/marginal/commons/outbox.py b/src/marginal/commons/outbox.py index 88ca7e3..2ab53ad 100644 --- a/src/marginal/commons/outbox.py +++ b/src/marginal/commons/outbox.py @@ -29,6 +29,7 @@ _TOKEN_PATTERN = re.compile(r"^[A-Za-z0-9_-]{43}$") _ENTRY_NAME_PATTERN = re.compile(r"^queue-[0-9a-f]{32}\.json$") _DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_EXPORT_RECEIPT_PATTERN = re.compile(r"^v1\.[0-9]+\.[0-9]+\.[0-9a-f]{64}\.[0-9a-f]{64}$") def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: @@ -146,7 +147,8 @@ def _parse_queue_record(raw: bytes, *, name: str, device: int, inode: int) -> Ou raise ValueError("queued Commons retry token is invalid") export_receipt = payload.get("export_receipt") if export_receipt is not None and ( - not isinstance(export_receipt, str) or _DIGEST_PATTERN.fullmatch(export_receipt) is None + not isinstance(export_receipt, str) + or _EXPORT_RECEIPT_PATTERN.fullmatch(export_receipt) is None ): raise ValueError("queued Commons export receipt is invalid") envelope = _validate_envelope(payload["envelope"]) @@ -227,7 +229,7 @@ def enqueue( return None if len(batch.atoms) > _MAX_ATOMS: return None - if export_receipt is not None and _DIGEST_PATTERN.fullmatch(export_receipt) is None: + if export_receipt is not None and _EXPORT_RECEIPT_PATTERN.fullmatch(export_receipt) is None: raise ValueError("Commons export receipt is invalid") envelope: dict[str, object] = { "schema_version": "1.0", diff --git a/src/marginal/integrations/codex/service.py b/src/marginal/integrations/codex/service.py index f75baad..38707b4 100644 --- a/src/marginal/integrations/codex/service.py +++ b/src/marginal/integrations/codex/service.py @@ -134,7 +134,9 @@ def _write_commons_status( temporary.unlink(missing_ok=True) -def _start_commons(data_root: Path, *, model: str) -> _CommonsSession: +def _start_commons( + data_root: Path, *, model: str, evidence_store: EvidenceStore +) -> _CommonsSession: try: config = load_commons_config(data_root) except Exception: @@ -153,6 +155,7 @@ def _start_commons(data_root: Path, *, model: str) -> _CommonsSession: ) commons.outbox = CommonsOutbox(data_root) commons.client = _commons_client() + _recover_export_cursor(evidence_store, identity, commons.outbox) result = synchronize_commons( config, cache=commons.cache, @@ -232,9 +235,13 @@ def _end_commons( after_records=cursor[0], through_records=report.records, ) - receipt = hashlib.sha256( - f"{commons.identity.namespace}:{cursor[0]}:{report.records}:{report.root_hash}".encode() - ).hexdigest() + receipt_payload = ( + f"{commons.identity.namespace}:{cursor[0]}:{report.records}:{report.root_hash}" + ) + receipt = ( + f"v1.{cursor[0]}.{report.records}.{report.root_hash}." + f"{hashlib.sha256(receipt_payload.encode()).hexdigest()}" + ) if evidence is not None: queued = commons.outbox.enqueue(batch=evidence, export_receipt=receipt) if queued is None: @@ -317,6 +324,47 @@ def _write_export_cursor( ) +def _recover_export_cursor( + store: EvidenceStore, + identity: CanonicalModelIdentity, + outbox: CommonsOutbox, +) -> None: + cursor = _read_export_cursor(store, identity) + if cursor is None: + return + entries = sorted( + outbox.pending(limit=100).entries, + key=lambda entry: entry.export_receipt or "", + ) + advanced = True + while advanced: + advanced = False + for entry in entries: + receipt = entry.export_receipt + if receipt is None or entry.model_namespace != identity.namespace: + continue + parts = receipt.split(".") + if len(parts) != 5 or parts[0] != "v1": + continue + try: + after, through = int(parts[1]), int(parts[2]) + except ValueError: + continue + root, digest = parts[3], parts[4] + payload = f"{identity.namespace}:{after}:{through}:{root}" + if ( + after != cursor[0] + or through <= after + or hashlib.sha256(payload.encode()).hexdigest() != digest + or not store.verifies_governance_prefix(root_hash=root, records=through) + ): + continue + _write_export_cursor(store, identity, through, root) + cursor = (through, root) + advanced = True + break + + def _connection_path(data_root: Path, session_id: str) -> Path: return data_root / "sessions" / connection_filename(session_id) @@ -565,7 +613,7 @@ def _serve_bootstrap(path: Path) -> int: evidence_store.append( {"schema_version": 1, "event": "session_start", "session_hash": session_hash} ) - commons = _start_commons(data_root, model=event.model) + commons = _start_commons(data_root, model=event.model, evidence_store=evidence_store) treasury = Treasury(BudgetLimits(), mode="shadow") universal = UniversalRuntime( treasury, @@ -636,7 +684,7 @@ def start_session_service( evidence_store.append( {"schema_version": 1, "event": "session_start", "session_hash": session_hash} ) - commons = _start_commons(root, model=event.model) + commons = _start_commons(root, model=event.model, evidence_store=evidence_store) treasury = Treasury(BudgetLimits(), mode="shadow") universal = UniversalRuntime( treasury, diff --git a/tests/integrations/codex/test_service.py b/tests/integrations/codex/test_service.py index d925c3e..c660b01 100644 --- a/tests/integrations/codex/test_service.py +++ b/tests/integrations/codex/test_service.py @@ -369,6 +369,54 @@ def run_session(session_id: str, actions: int) -> None: assert exported_counts == [5, 5] +def test_pending_export_receipt_recovers_cursor_after_enqueue_crash( + tmp_path: Path, monkeypatch +) -> None: + workspace = tmp_path / "repo" + workspace.mkdir() + _repository(workspace) + data = tmp_path / "data" + configure_commons_mode(data, mode=CommonsMode.CONTRIBUTOR) + monkeypatch.setattr(service_module, "_commons_client", _OfflineCommonsClient) + original_write = service_module._write_export_cursor + failed = False + + def crash_once(*args, **kwargs): + nonlocal failed + if not failed: + failed = True + raise OSError("crash after enqueue") + return original_write(*args, **kwargs) + + monkeypatch.setattr(service_module, "_write_export_cursor", crash_once) + connection = start_session_service(_start(workspace), data_root=data) + for index in range(5): + pre = { + "session_id": "session-1", + "cwd": str(workspace), + "model": "gpt-5.6-sol", + "permission_mode": "default", + "turn_id": "turn", + "tool_name": "Read", + "tool_input": {"path": str(workspace / "tracked.txt")}, + "hook_event_name": "PreToolUse", + "tool_use_id": f"call-{index}", + } + request_session(connection, operation="pre", payload=pre) + request_session( + connection, + operation="post", + payload={**pre, "hook_event_name": "PostToolUse", "tool_response": {"exit_code": 0}}, + ) + stop_session_service("session-1", data_root=data) + assert len(CommonsOutbox(data).pending(limit=8).entries) == 1 + + monkeypatch.setattr(service_module, "_write_export_cursor", original_write) + start_session_service(replace(_start(workspace), session_id="session-2"), data_root=data) + stop_session_service("session-2", data_root=data) + assert len(CommonsOutbox(data).pending(limit=8).entries) == 1 + + def test_ambiguous_model_session_remains_local_and_never_queues( tmp_path: Path, monkeypatch ) -> None: From 1877d7d850170af5014c047a0ec773d140c01b94 Mon Sep 17 00:00:00 2001 From: Renato Date: Thu, 20 Aug 2026 13:53:11 +0200 Subject: [PATCH 14/14] feat: finalize privacy-safe Commons integration --- CHANGELOG.md | 15 ++ PRIVACY.md | 26 ++- README.md | 23 +++ docs/getting-started/quickstart.md | 14 ++ docs/integrations/codex.md | 20 +++ docs/operations/privacy.md | 27 +++ docs/product/architecture.md | 19 ++ plugins/marginal/runtime/marginal_runtime.pyz | Bin 525623 -> 634185 bytes plugins/marginal/runtime/provenance.json | 2 +- tests/commons/test_local_e2e.py | 169 ++++++++++++++++++ 10 files changed, 309 insertions(+), 6 deletions(-) create mode 100644 tests/commons/test_local_e2e.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8db83a8..7e93d03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to MARGINAL are documented here. The project follows Semanti ## [Unreleased] +### Added + +- optional model-specific Commons modes: Local Only by default, bounded Read-Only pack refresh, and + explicit Contributor upload through a recursively closed aggregate schema; +- owner-only durable outbox retry, exact reviewed public-model attribution, verified cache fallback, + and synthetic lifecycle-to-aggregate-to-next-session acceptance coverage. + +### Security + +- Commons priors remain outside all local trust, promotion, Autopilot, and Tool Enforcement inputs; +- shared envelopes exclude prompts, source, commands, outputs, repository data, local hashes, + timestamps, free text, credentials, and persistent contributor identity; +- Commons network and shared-state failures fail open; production contribution remains blocked on + verified Wrangler authentication and a dedicated least-privilege GitHub service credential. + ## [0.3.3] - 2026-08-13 ### Fixed diff --git a/PRIVACY.md b/PRIVACY.md index a6da618..8b62a51 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -2,8 +2,9 @@ **Effective date:** 2026-08-13 -MARGINAL is local-first open-source software. The Codex plugin makes no network request and does -not operate a SignalLayer Labs telemetry service. +MARGINAL is local-first open-source software. Commons is `local_only` by default for new and +existing Codex plugin installations, so the plugin makes no Commons network request unless the user +explicitly selects `read_only` or `contributor`. ## Data processed locally @@ -19,13 +20,28 @@ until the user deletes it or runs an explicit purge. ## Sharing and remote processing -MARGINAL does not transmit plugin evidence to SignalLayer Labs. GitHub, Codex, package registries, -and any model provider remain governed by their own policies. Exporting a ledger or attaching files -to an issue is an explicit user action; inspect exports before sharing them. +`read_only` downloads a bounded, verified aggregate pack. `contributor` also sends a recursively +closed envelope containing an exact reviewed public-model namespace and bounded aggregate counts. +It excludes prompts, source, commands, outputs, paths, repository data, local hashes, timestamps, +free text, credentials, and persistent client or contributor identity. A one-time random retry +token is carried only in the `Idempotency-Key` HTTP header and is not part of the envelope or pack. + +Contributor transport uses Cloudflare infrastructure. Cloudflare's handling of transport metadata, +including source IP addresses, is outside MARGINAL's application-level guarantees; MARGINAL makes +no anonymity claim. The application disables Worker observability and invocation logs and does not +persist request-derived metadata. Production contribution is not active until Wrangler +authentication and a dedicated least-privilege GitHub service credential are both verified. + +Commons aggregates are model-specific priors only. They cannot affect local coverage, trust, +promotion, Autopilot, Decision Ledger identity, or Tool Enforcement. GitHub, Codex, Cloudflare, +package registries, and model providers remain governed by their own policies. Exporting a ledger +or attaching files to an issue is a separate explicit user action; inspect exports before sharing. ## User controls - `$marginal` in Codex uses the bundled native control plane to show status or demote. +- `marginal install codex --commons-mode local_only|read_only|contributor` records an explicit + Commons network posture when the optional Python package is installed. - `codex plugin remove marginal@marginal` removes the plugin and preserves evidence. - the optional Python package command `marginal uninstall codex --purge-data --yes` removes plugin data explicitly. diff --git a/README.md b/README.md index 45e1084..ce11cf9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,27 @@ marginal install codex --autopilot-consent Installation alone never enables enforcement. Earned Enforcement requires verified evidence and explicit promotion. +### Optional Commons modes + +Commons is **Local Only by default** for new and existing installations. An explicit Python +installer choice can enable one of two network postures: + +```bash +marginal install codex --commons-mode read_only +marginal install codex --commons-mode contributor +``` + +Read-Only downloads a bounded, verified model-specific aggregate pack. Contributor also sends only +closed-schema aggregate counts for an exact reviewed public model; it sends no prompt, source, +command, output, repository data, local hash, timestamp, or persistent contributor identity. A +one-time retry token exists only in an HTTP header. Commons data is a prior only and cannot affect +local trust, promotion, Autopilot, or Tool Enforcement. Network failures fail open. + +Contributor transport uses Cloudflare infrastructure, whose processing of network-layer metadata is +outside MARGINAL's application boundary. The production contribution endpoint is not active until +Wrangler authentication and a dedicated least-privilege GitHub service credential are both +verified. See the [privacy model](docs/operations/privacy.md). + ## How Autopilot works 1. **Observe.** Hooks collect derived state, outcome, and coverage signals in Shadow Mode. @@ -127,6 +148,8 @@ that did not run. - Integration errors demote enforcement and allow the requested tool action. - `SAFE_TELEMETRY` exports derived pseudonyms and approved measurements, never raw private payloads. - `AGGREGATE_EXPORT` publishes only grouped statistics that meet the configured minimum group size. +- Optional Commons sharing remains Local Only unless the user explicitly selects Read-Only or + Contributor; shared Commons priors never grant enforcement authority. Read the [privacy model](docs/operations/privacy.md) and [governance evidence standard](docs/evaluation/governance-evidence.md). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d95aa7e..c34c6be 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -6,6 +6,20 @@ python -m pip install -e ".[dev]" ``` +For the native Codex plugin, Commons remains Local Only unless you explicitly choose otherwise: + +```bash +marginal install codex # local_only; no Commons network calls +marginal install codex --commons-mode read_only # verified pack download only +marginal install codex --commons-mode contributor # download plus closed aggregate submission +``` + +Contributor mode sends no prompt, source, command, output, repository data, local hash, timestamp, +free text, or persistent identity. Its Cloudflare transport is not an anonymity boundary, and +Commons priors never affect local Tool Enforcement. Production contribution remains unavailable +until both Wrangler authentication and a dedicated least-privilege GitHub service credential are +verified. + ## Shadow first ```python diff --git a/docs/integrations/codex.md b/docs/integrations/codex.md index 110f98c..2dd6119 100644 --- a/docs/integrations/codex.md +++ b/docs/integrations/codex.md @@ -24,6 +24,26 @@ An installed Python package can perform the same native transaction: marginal install codex ``` +## Commons network posture + +The plugin defaults to `local_only`, including upgrades of existing installations. The optional +Python installer records a different posture only when explicitly requested: + +```bash +marginal install codex --commons-mode read_only +marginal install codex --commons-mode contributor +``` + +Read-Only downloads a bounded verified pack. Contributor also queues and submits recursively closed +aggregate counts for an exact reviewed public model. It sends no prompt, source, command, output, +path, repository data, local hash, timestamp, free text, credential, or persistent identity. The +one-time retry token remains an HTTP header and queued local metadata, never shared evidence. + +Commons priors are diagnostics only and cannot enable Tool Enforcement. All shared failures fail +open. Contributor transport depends on Cloudflare network infrastructure, so MARGINAL does not make +an anonymity claim. Production contribution is blocked until Wrangler authentication and a +dedicated least-privilege GitHub Commons write credential are both verified. + ## Remove ```bash diff --git a/docs/operations/privacy.md b/docs/operations/privacy.md index 53665cf..5eee1f6 100644 --- a/docs/operations/privacy.md +++ b/docs/operations/privacy.md @@ -189,3 +189,30 @@ explicitly allowlists them. Use `load_schema("safe-telemetry-v1.json")` and `load_schema("aggregate-export-v1.json")` to validate shareable outputs from an installed wheel. + +## Optional Commons sharing + +Commons has three persistent modes, independent of Shadow or Enforce Mode: + +- `local_only` is the default for new and existing installations and performs zero Commons network + calls; +- `read_only` downloads only a bounded, digest-verified, model-specific aggregate pack; +- `contributor` also submits verified local aggregate counts through a recursively closed envelope. + +The envelope contains only schema version `1.0`, one exact namespace from the reviewed public-model +registry, and closed aggregate atoms. It contains no prompt, source, command, output, path, +repository data, local pseudonym or hash, timestamp, free text, credential, or persistent identity. +The one-time random retry token is an `Idempotency-Key` header only; it is not written into the +envelope, response, Commons aggregate, or pack. + +Contributor transport crosses Cloudflare infrastructure. Network-layer metadata processing by +Cloudflare, including source IP addresses, is outside MARGINAL's application boundary, so this is +not an anonymity guarantee. Worker observability and invocation logs are disabled at the +application configuration boundary, and request-derived metadata is not persisted by the service. +Production contribution remains inactive until Wrangler authentication and a dedicated +least-privilege GitHub Commons write credential are both verified. + +Every Commons lifecycle state is a prior only. Candidate, supported, validated, and promoted shared +aggregates have zero authority over local coverage, trust, promotion, Autopilot, or Tool +Enforcement. Network, schema, filesystem, DNS, TLS, GitHub, and Cloudflare failures fail open and do +not change local enforcement state. diff --git a/docs/product/architecture.md b/docs/product/architecture.md index 8cce259..84a6761 100644 --- a/docs/product/architecture.md +++ b/docs/product/architecture.md @@ -74,3 +74,22 @@ protection. Keys remain local and are not part of a trace transaction. Losing a key prevents future stable correlation but does not make existing pseudonyms anonymous. + +## Commons boundary + +The optional Commons loop is separate from the authority path: + +```text +verified local finalization → closed aggregate compiler → owner-only outbox + → Ingress-compatible boundary → aggregate-only Commons pack → same-model prior +``` + +`local_only` is the default and performs no Commons network calls. `read_only` downloads a bounded, +digest-verified pack. `contributor` additionally submits only closed atoms for an exact reviewed +public-model namespace. It carries a one-time retry token in an HTTP header and no persistent +client identity. Cloudflare remains an external network processor; the application cannot promise +anonymity at that layer. + +Downloaded priors enter a separate read-only diagnostic path. They are not inputs to coverage, +trust, promotion, Autopilot, Decision Ledger hashes, or enforcement eligibility, and local evidence +takes precedence. Shared failures fail open without changing the local mode. diff --git a/plugins/marginal/runtime/marginal_runtime.pyz b/plugins/marginal/runtime/marginal_runtime.pyz index 750ced97341008355c1dbcf4f0e0facb9273b413..a57544e189ecbacbf8d48bcf4627793c908657c9 100644 GIT binary patch delta 75808 zcmeFa34EN@c`xkuoz0e4S-a$AG#E27(pd5aC?1R~3*ul~vMd{jG2^i`wg-0IQdCu}K z8X1!=-}n1{ANdh9@7d3J&a#T{Ye@2hyv!j_}a(^tBA5G^{`H9?gtTP%bWYS~i$7p_{ zRLCBhD&-5Y15VqcncQSDHkm0Dv&B+0Kaq)M?wrhJN3*49zZ@+(ktvB5m> z!u{nl7d}%vZ|PXNlui}$`BIYq8p{^MBdhHC%I`nk`k5t9{B>*jm+yRSZE(&U`ghr! zIdjI*Pj(`mTdTTSn@VLTvZYiiF*zNJ#fA#mlOj7Rc?8Ld?&Up)Y`%~$X1jKB9qO+z$ zEjm9{I+VZDx9CCs*cmO$KL!B7)2e=PdScX9dk;PhXNmyK&L|ZhEnv29Gh0(rlS-v? zxl}5ejPB?4#HueyL2^v(70#*b`^0DO|Hen?|H$X#`>6MsHh-2PJ_4t#%-7z)m{Y*t z=`?(wMVpiy+7X6VL1upUT){%Ie&CjEQa=_}^Up?|BRjw0}4`bb8ij%}IZXdyd^ zPhEvfE?r6hBSiBjCNhOC!mUwNIa)lLE@Z}{ho&_eAOtg&6BoV{Ss%(23KMy8$9n>G z;sc)xF0ixX;%q*=V$=3grjSP05|9ZkXYMTJvWGM@t1si}3CyPeP@b9u3KMsf?e^8h z%xED~D(b4G{ISeL>S*T9Xu23pnO|b*jdn+f@2oiReXFS~Z((X`0{Hrv`f;L=EoJCJ z&>gHv4c@eCU^q3nYsW3Aoqc<5#2x9*cVBf?O=`zr|Bb1U?K=ks_l~6Y4D=7~+O~&E z6YEsj(BO_8o`UY4nwpx?T)J3P+j6Y9NAB*{sp52eaL+)YkS}!f$~zc~^+iW>`6BIO zerjS2{{>Ls5=Rf`3q)X!5tb_vuE|eGtXu>76HQT3Ds$&(ZmNjmNvX!-^pC9nD&n^V zR$N|n>OAO)quE?0I#QU*^omD+C(tT3Ja1jTB>G>Xhesz$xdbnH*H&@&8oOSce=Rsq z+cA zx22F7PlG&79Kq&|fiUN?ptW7mQg%F(08!7E2(`vLcz#4NF_P0{bF#8~y0NS(xxntR~Z?qhi-rttOG)XSZAy zO;3!W3^!{}YWvQ?Z3CuxqFCa~-XUK8nzdo4zaKN1==c=)m7{6z#RMNSo-GyuDMf66 zT1kEqmxQ22^nKrI5Np0=EfJ4D7+EAH4h9yA>AiM?xc6JuJW=_UwNyO$J!`&r?=zU_ zi{G;v#Vg;juBFp|>x>m~hb=z|t`I*y9qJGt{&{G=*zwa~kF!669}IO=Z+7md!I(Jp zH^KR0a-Ususk^bfa${?aSo>nIc9VZw_?o=#j)?RH>x#tIbTKn<=V)e(9%t>`tqxW)M#aUi&z6Y23x zFV4Bp5$(D*suAD*>_q8+x&3d>=f;Td6pOxNt!QDuNKfEe0ZUF6@?dL=+?VM5j@91U z)x-CLFhh#2ULiekBopu1)S>S2tKYTObwt-jV~I5AKR8{mG2ns$#go9)6QyEO{*;$P zoSh7<65kuP8WtQvtz+DG_ON*6yH;ahbF^FZ{8uYlj!jLZ4*^}2@=<~$!bYQ01?mly zKz27~@V}xA>h|#_j2U8ro39oicMQL;Pv?pm9+z0pbYVIL^pP+0M$G{P>6j{y7V5+w z{h_sJ>C9_MP~9Y+TWZf+;@=i=`nJHT5YIwHzi!nB^d$((0}^Lfh3drMw}TO}{aNcW z@%#hf2J!io_M)Z0KEBCU?)X*nvFm|1cjZgh6QUlViN@d1?ogL5#54;EkIM;PSZdd| z(Vw6QNk-NNH%@0WxiLDXh9&$BC>h|)dsf-CvbEi_muQS~)3%V#7Bk|}gMr3ou$EzemvTz^pEYVW&|JtInaZULDQ2=*W`&Tc znXq0$2c6M10NV7SOwPe5`69?gx|EU(gG4Es4q?hDK1}7*mALt*!G%Pv>Kz0PO0{a< z;n+#;^^^csULn5nHQ@g@f|b|bWO}$1nncG47H+CA5!EwQo59k$;$pz*#zoI}td_D| zJ43I*rjmnOBmXm=zB4;MHJ&;&UCI<)wVka3gGcDMG&Pya?B{Bov^B1sbFlF+I1`+r zVRo+pj#Fn71&AGHGLfGm<}8=XpCEPyngZY*0<6p!{Z30hPPBjAZmL3Eo37Y3cs*Fg z!JEKu9RV)^X)vBILR~SK+BLXqD=}X{+udLQzGq!SwD%U*8F>kAZ|c$jNC8fg_eA0T zPkh5_@0X|HFlL_3j9wX)8&#_Azge!&4)9$gIOd9%G?Qs62ZgNL@|9B zLkA;A$c%%Y{yTxpd;Y?TiC=xz4vT|735I5cpbftc@Q8;iq0VNxTXF$viy%t9{*@)R z84MELRL>nQWk@qb)E|=G;q0A^;be^&*Vwb&H<1v>nwngul*vtZMh_QvLG{tenApLf zQjCR|#MhaQ6!I+aHJ=O}wiK)g_yAEr4q$J!vu*KsOQ8lQ|w>G8mG)+F(^P>H%;`&j759D1YMe`@#lmRHSzvGsk{$iL^1*|4aHW}q zm$Sn_he@TwOzmRo#l^9RiFX7V#oj~qaxru=5ETy(Tfxft<3R!LX?ZgMmjrUvu>+Z6 z$H~AdQ@6!Haojh1@#?p%W#R`*?HY0ZIcq_!y535D_)ogsn4OC*8?-t!btsoD9>odY zJPKVLU3PQUbg~MW0urVTh0a4;M@k+^lSe_Xvm_Y%Z@T9~&vMV--GPJ`#LEPpTwYQH}*PB$$L`(E!th z&```z^pi!3iRw0(*9g`0lMYyZf>i;CE93eyoEgm*#zv+mRil0B+*D@k6vX|KV-+GR zRVgbLHP|)fY|&+0BI;hm@-t9f5A{Jqy{m8M04_ibqp*pTwx;WNPvUmWELM{u9petJ zvB@v2FVWhsB<&3u8 zGQ#7mQ&NHY52QV}ZoH_iI>M?CHI%{Kb)~0DP+esq8Ijo`r!9+8_ytT93NXQt(IM>s z0vQ_}%--L(Yuom1eIs&5Ae6HvS8_so_6`jV4v!3MqunCg0&JR@FDrH5z|hGY$by;OSrv<4KSSpa4N=**qzQBt507iw&V z(mu=l9v`l=B564^r_KyApC5G7Wx!FY1VHMIIzYvR7y@9JItGyKjXL1VWuq`=rw%c^ z^+sL5$Aw3rs-LwGLjuSkyWxPKL5-F@@Gb|Qgbkf8mbwYO(V2B{(*T>W(vD%`RqhgZ z1W}vrO^QLLH|oAdY86uxVHo0}AQ8D|gUbf_tAUwb4IvuJa9^Z)zrcc8Y79yNSahKv zJ(ig+#wXL+0+a~g6iWM9Um%|OMR1ik|54~kA8NCgET>`~f(SY=C|a=|mMu1-_N!KO zpkKKS&<$kM4fe7KST#d|7lr!Hr3aMioBinP0Dux;HwAOoM?qmiu}^Nm2sJ26_i3^F=oG z8*E29XIR|CJ%47c62p&LZQ}6fLQ5)7To);$sDnkSp6F&ZL;9m9)!p4~`b!szkODQV z$GyM`%$h!-XV3Z=wNI{Fqi1N46KyiQI7a7d2nUK0k9;81uv}tQE_VU4LQdkNFNT}M z*Z(n87tvpy|9QC4M&rSipp-Aa7!He9UktY_R2@uBl&I(C`$EgMxOz5Vq+q8vK+)@RilUu0z@JpWY*45i(v-&zh@0OL+MPF+ST`(9lqB|ZAfM4&tRu>AIujtql_OiTlF5W|xo zR9U2c5}-gJFfh}U4tHH0>wxB$@h1iXDo-8F=Z__!!6KCEqXiu`V%g`djpFxy7OE95 z?Xs6%tuHkrc3cp;cJemAWzXO)_mXJ;^sFJY+|eCr6Hnb}hgl4ad6uD23$@U^8|HJI z`A1(FY{=O|*&JMs+y#97itaCr$H718Y7W|3L5RYIw-kBMTgsV( zBwt27U+li>k?<2E7H2II=K#4m*hAF5V7HW`?O?=u65Wyoqm2McGyQls>i|{OJL`o9 zgBH*lyEg$f0vx>JdTyO;mv!{a3TJFdNL;QL>SChF?LG4FlMzUJXcK4r78r<$%r5Q zgWb4M_2&>!8rR^UBDRZb>{sLQa0kTc{gGu?YF-7_7h?ICn36fcX-!szo^G%?;)U8! zd$qlC>PBH=H&oLBBHA)jiI_$6!^KL{oXJLCor8ipo2R-Dlb0QvJMM-C^D$;F8?y$k z%{Nu(0KkN-+qemPPmDm+4SvK45(aSt05dCByE<@od@Oq;187*PZK`h&7)N zwf4x0@acj}g!`7!rpq#h)J+?7VddNnE#l_a>{Sa?F&FTJuX?=+%Mpxlh3#c#Sp3#W61nu8|{WTuY&c>J&I%WIs5;^S`(B*gb_vFh7-_>Kg8N8MC~ze-RS51+SN znmQ72N|Q5=GxII~<_ovd1LS7dC>(dj$@5ffX|bDc^T^2TZ)qgUA?2)0#%3aPkRc&R zYTraDV-{J+B;k?|-!Ec{yFMRIh_U-2E4=CFR>M)gA*{q^xzlBTA;7GTaDA0vdt>VD zGQs9L{vNl@%(oJt(Aj)3)btMl6s&*=8h@&Vv#Im&PHJ%gOXwo z-W6yQ8$Tafd`tCN`(TfjhxBZ})DdpLWq4?%q+i$zKbw3`&y2H)l7wvY-aJ5dtG0^#qVr2{zXDg`Q4C=F3)eolCYYW^#w&-v@tb z2fQ>&cD~ezdxN3IJsx6@TGjX0OiA|bfjjEj^Fc)Khs5{gKMgM07pHmCmFM|Got8cX zZP!#OgDaYW8&Y%#PI`&a6Jz+jmMgI<6`c+F(~(;hA0z$cDBUXUuQoOw23Q`A(|nYi z%QjBYzAjL=9gg&UZ0cp}gfjO=OYo?dK*6{7#8G%wWpIb5i7|d#W}=J9F_;Dr<>J&R zITuu}eQ%?<>sD+2m0)ZE6J$m;BnzL2R?XNFk$(i{tjLf^8?xdI1*4!V| zHJ<%Xa$nVtzzb&VIww7q993#uT=-J3N&NS7!I~}F3oS`5r&8<6XP=D8w*}zW;TUDk zl|UEjh9Wo_h>4}VIC~qkyVpEq)r$L`wdS^K(|U^cf&AHhTHDI22U_PDrEaC?mR6UR zlzw_V1&VT|9=H+KC#|N+`J01vY=BN0u9(hDq<03IMfVZ7P{Io;LDlB!If-Z^=KAv| zA}gE;Q-wxd!FvESQz~Adi)l?XL;6@bcwk=qQqQE)-d0z6WdB?_uX`V`=3b*rhjA8+ z*dCT59eS5mDgC5DM85=st!LG{Z;rH! z=id?tZU$egy90Pv4qXtiRo^T|kD983iw;<``64tf70Fvt$o+Q9|_i5dEPY+j@1 zm>XTkEZV;uXtH@T8(oH4X&xiMlh>N>Sy$!4txdC?G}IDDA~oXYOYG&BU;6He^C8>4 zicf_D3&fU)T{DkJHd9J*;gH=?S<+J5_<9%UI&*>8U-H+yLCqJrF0=0=fU)t?uFFL? zaFMugjU6ffyW$=V-yXyGi(=867>qo`OZPJiOi-aN$}cOB3Yqv19?K`s_54q( zWuX?Z1Kkt*H zubkcaKc2jL&K&x;wd!*D!I-oIZ&fc}-j*rkyT&r)7zhz-I{N03k)db_CsX97ha^hN zH;XmSqeW5sx51wIM@yy2#3*-JIsL)?V?YH{D^f~{iy=Ewq(NCtYu zKvK1`Y_z+vPNvAN(3RX=;6;w=rUCw8b%q~H3L zBy_N@7ktps;@1GZ*z1JOjs7bQE#&3KT0?yHDIUwx3Z+fDLwzG}2DcDfI|ki^e&i|X zm1Ec)*tdP#z^?uQFr=#X@t(C>{RG4Bwt?&W_U;&AIc#`f&(PqmJp-v(OtWQay&WFd zy?0>Gh^e))N1Je=tM$mGBl=_*)x`(&s|xqqCOi;>paX;Lwq1A}G_nK#$HxH^7EY7i z=3wI_MuHknT0Fg@3>EM=2>v@48AKWEajO9+6F4x!a2S)hZ27oKiz%iGJ>)20YR-`3 z=EYW#0PiB>;i7giSB@48qvW(S1vL^;a$1jP6r4V-6+k_ihm1zf449sH;V!$qMm8h^ zrs$cl>cvxafzXirh_jb}80tcX$|GRc2r4A3LmoRE-O$}l);QH_b95tqvp^H0hrgLZ zta9(t*79M)!;&CP`Wm%K@sqmWQ)b>B5a9raePd%-Npbtc;XKBa9@J2?capBXle9f7 zPV>(LHK}3w1{VLJ=WDvYY?Zpw)fdyT1Mq()XU@*3fgAPZK;`{^7hU3X$FuDfn;rMGh)1gyB**^hlt23+_5E z1J>p0Q|D5$|GJ9fkV%So!kg?7|JQ#DZ0jye;l(hKQT-3;J(4l)Vbi5~T0L)Z!|w+A z7jg@JGKlt~=X|Jnqw4{nr#}rD049>#&7>1>U5*Ru=0I~TBsRIT2}sbSVXmAzUdJzI zDv1XgmZT-bKueQkWJ)}GE!)%Gz$EVeW}uEFi0dUTp#G_47JvqoJ?};0C_EI%AxHkA zz3q&Wn?q8y*+VZg_lyh=^z9U{z6TF^SR#8l77iyQjCkP!6N*c|!^{!pAc8>A_w8VV z_~cEId6mv9YWXe8kB)?E6%@xMkb>DHU-o7_pK9CC?0rNgPxAVyT#onXsTG0nO1T&b z#fizOQfeZ9BAqQ^(KK9#c=q9NbD35y)oW2&wtUq)wbGs3vjUqodh&>^YjaY%_4os zZa7o1!ZmU@+8ojM_g3A0HOV4viUt9UE$Ktj4Cd85n9n#|y*D!-mCiPYvYf-f_+)9? za}5lDKsQ3KywJNoc6D-eA1*RCEt$9m=u{V*R;=jb3Vn@r=LV+eUUgLN{$-uEIQS^7 zLFF#qh$)`7LK4iHC4of9WhkR16)K4!YL=?0vHLM@^xC<4U&V#rj5O+Zmv3-3;^2l* zL$?`;2+N)~_YDthOMxR9+_7(97#r+E$5=jGtBxwF-8Xv7sJ}i4V2okNhU?UJzO*^+38Z**?g8LOLtn9H1U4p%_n|4qb=d ze6shCxNng?S07pwj(JQ+Jxr5~b)lJ>xce#?f zh$up9p?{qAhY<^6T=d+i@iN#an0AZS)3ffXL^uA|LqB?ZdtkNm zShL-8#qN>dftOf0dQJ@iA6uQv=O+)r;W!7eZ!95ClNQB4^-T+yTj74ZE~5xJr!#sKJU5!~Lo7b_L;11k1k}%LazWE5 zn9(9Bqlg#*UsE6v#dcG)C64@&hpA8U-D*(gy3-T1b%)4i5FMom9vFv*z^2MK7g)e7 z9C`6ga3cmVf;Y|OU?c?+Wtj~C23au+6_W{5acF7`RKD`+|Gi8cd^iv*lR+jW!OCa) zglqxIQ>Vt-@To|IYqr-8LQs@jFPs}j|L z(wiPXC*fJd0H^LPZGo0mpjTIoc|QIQ-Fs|$yM?}1>5Fg2>vZR-V|rBgz!-BcpLs&F z7<3O+t69ycOYqO0xHB6NBDsjt$1(o4X*{EWhYgC#{u|wIHl;6=`g26Er1~PoF||QS z^t;~W z|4xFbM?i@5q~`5eT4HsYspkTANNPa84(3rH?;LjT48U?gqcb|+CDe--CL^sBdBka) z;4`5Z65?xT0yU%jJApA3RPbn|rcna6*=AmelxIBCqxePy>{1;>^HvN2YDw8ioXtj> zhxMXTtf(^I$!GI@26JnJ(D02#~I7doHrBo)zVB{dh7E>K= zy$F{Ioa@@60r#a@`cSL9de3Pm*DYL+*eN+eE2r2hYPr8C2i8 zIY;&ok34P#9YcS!F63!p03dH1$tZNa2O~`)ZX@#2-OpNWoAk33E&NGUGknl^=DO6( zPf7J+?IV$vUhf!q5KYXa_oMOyN%?TMum&Ry0&1vGWyyxnMnwQFc@Dgb;uDA)F+zJ` zfKCzsLN2T`^bHCpm5T+EP>m5zLY#ihst3{W-6sdS9Q9XHgOS5n(2O{gg!@?OV)k&* zuWGR=jd`9i~7U$}}JFF9O=TFdJNt&k?tDwo!o$xIk#*@)cKU;{OdKK=tP zW#Ez3pas}BcCfSovkdD6E~OS>w8$=qdQ$MBemgR+?E zVkEWL$5$|hSr1wBXr5k0l?o1uBGJe9iR2OW%0MhhgqkBtViDh85?CQtBtkWv%EE(p zq-5ODkC&<=C9&!@$OxJf^>Bs;zLeI%Mds=1#y4zO|N8vJ)!hzs7)=^r-J?lc2UQwE zS3tx%o|(yCRJ}itqMGzrd<_N1MbsZQFs)g0>;(PhpG*db9%f2R1~VmxE2Qp}qmWKS4X$7yLk@v4VKxac2kABk4TPGPlIWsnl=ocMo=$PmmiY#A> z$G;^VT1z^e2Xnj?|a`}rRQYHFO1tZ+9WT(o?o=0_?E zX)py(V4%e*Or%t#4CfQUpcG+|mAZCKgelr4VN+HhnmO4Ye4bRw7d38;l_|=g}xI>8; zUIaiFAnBDlIz{`xh8n<H$`V&3*?%#0fG&{tkE58crro;(w1x(XPT@p)X57OCPS5q@ksN&jb* zT}eek2&<5G{~60%A17ne>@6fk-9Pwv^mimuNWDOr*FQuD;@xR3s#pe>*rgECrCt>A!BS zoIel}Z@&{kQejkPo>|M5j(dpY4Pc-2>^go$q2CZ`Lz1hPCJyxx2Sr}$g^%hZHNmh|ZWq=Vp^kR!D*bCY|xr)k`_cpXu^<0ay ziI89HX;}k{fC-T!4})1gsD$_~<-+Fj zO#`wK^d*@u}STqO>4eqH910L5qZ(EwP zqzz^@)TIFr_4P9PDF2xiU_|<cr-fD<8{(8z9 zw&gx}H@H@MjVA7UcVLnD&402MLbd`ZxOjsrsNS`nO3`#!`vOjGH>px&*8yWY@}#dGhno67J_=)>cEJuCIr z$BdOUPrA(~1S;@34W~YLA(31?XAb>a-D#K;El8)~K~6{jUa3?-aJOr~T;p#}2g1mf zUqBLV$aAD`IOR##;l3S4zb1Vs*@0N4E2u5oE5tu+54CxGgqJ>eKWONaZwa-Fp$`U@ zRF>=uiKjnj&l9m5?Uq&v$i86C45pWyeE?YnuHSxx3F1WNemL{dVsro}8IxZHc}ln| z$hT9B9aT5ly?4{b)W-FjmKcvmb*SQB>q5=t0iEW?q_=@(owD0FPS(y&Ij%IAN8b+4 zlK_J{+B{!57#WIZ*X*z;*Dp$hHVf-6#!p6N0C3_?Ndu(Gh>L!cPNiIQ2e?TfQPrhqs-pK zXKoiWfHQnx#Me)U5)^8;b0ztzWGMGZVqnMM;1IhI(vMw(BghkRirvKCP`_)$7cN+l z1#XdtvV1g%Q{Obert(`I@S#{NaXZ}KWiaQWah~C2(KhQNPM0`OMUbsJuRe$eFF!kEFBE_D zVz7=d<)GHQ#s`%BjHu8T|* zjz&BfJrtt|EqEt?b&I`fi61;g&4=t(@xCT|kpxQ)K!mg#%dhZbsJO;+8U-c!@uu(r zVQx1V!#~J}7A%^137W)DpSD&~`hsPPz0rE(;{LtihQ7lwH!IL}QsqZVm{dqksAdl~ zHJ$NQL+m8-GI5;A@rxgJsJQsV5%|)Lm;+yvA%xEe<1ceK}e2JUn5r4V-8@Ce?_0##D^%-ZtT$mXNb-&HfvAA9xvCuH(zz|AJu)#t6K zj>hM_7gepjhMc=pTO{nnp z7DFawAYk8Yw=zWFm4u0(f05Z__FAsxj)gg2mR=^t|b}M4i65!5^q$935Xtys;@dYr;;%Mj` ztWiZbkNP4(H;Lvmhz@~Z$SuAo?#o;uoiXhyM=owSXfF}Z9kgp!n%-x`Vg8Q|%3e#U z=@1lJ@23JrXJ+D$FIWq&nVJMi$)sE-WM~XSD7YgM2H;z@X7ng2a*$*PVB=Xuc&%?z%JW3Id7^!YRs+Igz z-7f3;%xOqyR7;^^hDQ##-QW}ZUAdb!m}z^|GD;0%7&q5xB87(R`M^GVe#G_mzOnN1 z<2BYv@xpw&Zp$fyL9{kV!UY&>K68gXO3iws=aCUhM#I8G_7;Dlf&V{=6gVDfQ~rNd z5~(YyC6VHw1f$XtqYwx)saiGP?^Sc`pon3l7DHqoZ=Pd`1;|T;vl~M<9);izM9#V2 zc5p=YL8x;a3~&)xj*qSNi1}cmbZzy{h4h&wOe9^$SNrWY@w0c?^J}0=l(UYT;Ir2_ z-$+HGgOcvV-|~&V)^R#98K@owO}*m@dsSuUxf*fT4af!d%De1f-04gcSb4jl|H2ls zQH!%Ml9mBJqUgx0>Y1UQ@@=vX(MEXG>@(f;t{h$t=d|*NBUzX?<9vT<*ZYv98mWh& zJHf1qnev2sqfQhPt{NQn8Q?ingG3XR*FSX;S^;^)wM23fB+hu=q^NBQwQ)u&Z~OUGzI@P2id)LHEY-Go_;0GT05T20GUtIG z6DYNAjt5jE9uiXKu0PM*1l%GGbWoF57WR1<)eaOorz#BCE;1bU}q`5LezB zSSH4s;f?w1i`F7xHP}s?`?b%_POQ>N?rbb%WaKt}IYy-lI|DtxI``+HhWc#-{oD6! zAKZnbLH-|^0zka|%b~Wiif7Pg5(vcE|7c)hYFs`0>mS^?6G7{#DIE?30?8mFzZ7^g5j@~lRuQC|;KFNBBBytPQm;S51?B$7!45a${8J3db^rldq zRH|$Ri-KO3q(cq2aW8msylwC%S6MXX z`y_iF=^K`t)z|-KIX&+eS!?gEJwv$f*KZ%#CO$Pk)X;P3$2KT)8t~aKm4J(<(0le$ z`s6*B^7%gwH9Oz0@7um(?=UT8!RJEFNm!$|k8IyZNa3cwVMYt)hujgQVxxfgPSR@*QeJd;3yFfVV`bn0M4|=#t%{)eXbX&m|6_7sl*tiqW`+zJ z(uoFNEx(07*n71)5|x7Gm^}wZCk(%rkOV^piQpU<^p^dqc7Bmtn)Z+_$HbINWyKR) zpgWI#)vA?|qf^EgL@RG4_d;Px4dG*sB^MLtJ{4Xjo(hCQjjp}t=z<`JpthptT~>RC ztIT=8%@%1VV{isU(A8fPh68MH&Jx6xF%(n0P#0b;et*K6-%PK-U#AmST7XVhauad- zi{bVTSK%ek%9SKwW{0lAOYKnn>uaHT;=W)ww5ZdQ8>MpYPWXf!wdQwvA?#8+Uf~D3 zS-Pk6o1T0HmpXu0yD$`PVUWf7XbLo}Qy}rdi{HK8Zmc8EgIUkH)dSSp^yun7BI{T0IZ^9@G+OCeA%)w~G&a zAhc*P&%(JP;_i3gbAX(1~rc7WM%b5>O@oxF?lE2wBHwM2=N%=?7QI9`09J?Yxc41rD@Gpiid58&2~d* zq__a!B8pRlQ3&a{IX^jzTU8L2sC_b6ccVsd7o{cKC89>bM<%mhi5hx7MEhiL9mcV3 zv*TD7b~Sk;zyecLyG-;DEz9jD=R>bb<`8ECp;bYek0}3Pq^pPiGJXT@DN%Gp#F$4h zJrIzmh98iP;LmgBcB@va0vtg~PJx;fotp*kKeNU1^71aYT=pQvBQ|23+Nn5=dQU0A z0FP`wxaL&jObVAfwnc60MWzE;nUxepXy0d29}4!kOfm?h(B+urOxxreky1ficaz<` z-jrcmiWxWnfJ;FT6(@lhsX+`Z@s=1z(Xw7g!t?@ZAw?n^2_s6NjK|~`_;WB2CRwCI z)IH7PzzYwxB``6oGR&+BHJ6TT4uzAywIoso*WvwQK4{^AaHUEwSLM@8)T~ByVOq34 zpzVS(NdQq}xOYyCZ*V`amWw>lDPvXWhfuuPYDZ)!H*XM^`J(f2c)pQ=vrcCGrjEH^ zj>PipXYDm)h!2TR9k(M3w8%^;l|nU%!`avKv<$-HngPdzaxoRnfEB zZrmkBc(oKo!+jI*HuEgRPxQslyKMqNxtxs?(Z@rv%JPx<&923}S)Wqn zT&cd=QiX*eiR_*Dp~zh`8>!0Ajd zM$em|0j=KRO^{KQEb3}Pnhp5j`PJSYl9xr4Gu9XOWKj6_@<*cynhT_g8J3(Oanm!tnh9DX(! zkI=B`!zJ-Ae{HucMTEcn)RCjmEfUUy)i5hw-Wq7qY=h=Cm>pBrHN%yGmP2(*BAK3_ z)Y`(R2sOZ)kQC=o7VkqSdKri6L1Kjyko@6jDuZZhe_==R$8D7foW$h2LY2!1LS(QL zKf~^D-UytuOl){}Xj$E%OzA`hLDAeCcx@U;?&-JM;Ra3#OA$vEE4HVEH+bUiFWM_s zN#0CWaI-}2s^XO|+l^st8(4pld&O#BE&o>g#xF`aBZKF2bh9|!9bU1}^HDtaZF}{Z zcZY&~tZLQVh7JNo#u2_6Ras2$Q8Th+G8>iW2RhB5BGsS33L0Qi;+9Dy4`02_YOCxx z2Z4qRBIF9!f<33<-0jF7kR9U-IgF4_=*Cm3)=Z9~KJYPX;D=NMu&lZy0MH9n+_+h+UyllgJvRluQB0Vr<@Aq`ji`MepBqro3 zA#RrAD)?qk&qf>Jb;_M#NA`1yL-E3D``RNueiShq@bu1mQZ8PS?jQXl$ER1I*+WQ0 zGRCeW%+Pw#x#2>^uT}$tMqdhs#VeQF^D5t8x2Wvo+Kl$OBpA{ zrsso8E8R~vi;q2vmzoFr?ciKs9^P6;Hf4|Tv8XqgiBH_OHToyQ%#kRYlzY#AgGKfR zq(Aec*3<2a=FFjgs~RX(XIrPGg>v9d2pH-Sp^V6HY=MAz6UMIEa|wP?A|8GVM4R1z zZZDg=9R!0kIi zr201*3h?r83hEfnB|j?fF{@nWBBBA2+JboaYP+RukQnqpr2=0m-G$DKC`*h_rp$nV zR|h80F$44whxSJC~5)zT#Yh8G7V6xM76;-231P^57&7fK318cOj`xlr3$ z#!;%ooDebPvMZg`k&g@JYMyfQY*Rk-w|4zi6f6^Sq$%hGl-wF9m7ltdaXLyL@T;E7 zv-PYkOPoh}YQi|L@hsI@Db&H6OD;|{5vAbZD|;O79s1PS!6O$(04@!r40}id!9DU1 zhP!^dOZJexfxzlj3UTEU|kqj!46<_PK zmqzrpdAi1Kk=A&1YG8!)bZHaS4Hmn12ik_fpOIZnEr6f|9KF6ld;rezDr#F~2;~J! z0K*ZPN+P{B+)=K+{J5Tl8cjN7GvmVSCrv~PFna5G7&y5}UI!JV>uerQMup?7*nsGv zaiz?^zQ0(eNG*KJJ8OQ8<47xQE88MnS=L;Ga&B(2bx8sMF(pc_tXd1({A6%}IP$L0 zq83*h1U8k90vwl2tb9DQ>}s`4ianGm?XSLV6uyF9f6C1+lBA+sS%-et3O4vQl_ldt zl~25<`4(B#>z7L0m!^Q|S8uZ)!gZ%f{f9a-$zF)=#deF>{Xn=?Cf+(f6sfC|!AcSc z$6_e1&y9+thBg8g&~Ba^08?_|vaNXXX}gJJi}~WUr|revdUMR%Hn%&C!}JZZ%OAa5ozc|od!Xe5dgr5Z2-qydYW&}mdUGFTnQ|dt0h%y`SlS1 zaVvZcNx?~|Aw$8soUvwLeTUC?V+P`|XSTOS#jC#^Xna$3(2pxz1zowCff?4p7~*5c z?WPqPsJ+H({Z%~oj{q%zMre@ob`$wJ0(v9CM4mwrGHJRLJ%QCRgn|RfQuG~;5Fwo( ziMi8Ygdn<;lkiv&1U>C~9N1c_V*U9SWeWR zd_3#-8a{gDuaa}-(7(R-(7V`DwiXf$Vk=0Fb=wqknNcX@;AzMh55W#7{xeRUV{Ew0 zvDz4=LM4a`--)ajPi+odF4mUq`QZ$r8I_9S?A*YxchAZcGGsAXcCCIqO`!d`Kfxm6PHNe(cnq#RTXuaET|$VRV`roK$dtEbn?@nF)y#y zHj`nN8XZf_KCd`gNyn<@e;=e%tmND9h8#QPrSmRpw@!HmCt_ z9vU1bFS7oD?L)ISJl=C_f`&YBrT4$OFZosfqNx~epRRMtga=3+YsgF&;n$ljfG^-2 zGA!STXMPb}CC+~|v`ReGW-nPz#mwC>JvdIZ-fK#&_NM5QSc<~(jw##h5$~bQo6BQGe-R8_>xhz)bim3< zQF@S)Ha%994pYu_{NYbJ#HE>{H_B|!jPD)(S~)IY=kpF>uQtimQq(0^550~1m5;8$ zJMeU8cRdM`d$4E&kz2_>LQ=>qg#iNpji9UyiP8cNRb3Qr*W>9NCf)K;8_9<0KyWK&WB4qzH7UJiavYv-s$X;U@9* ze+<<{RNyD^{LjOUHX0AE#GCt9;T-kfyU16dW^n)V8$ss=mLXf6YF(nZ8&Lsuca1=)E|s3khHmdlJy zwO)RAWZCUW4;$gk*zx+DF%xA2c2uWTy71Sb@lFqfTCb?W0qH@>V9g$PRQYrFgxdO( z4ntEc%4&JHm*|YCNyRL0JO3FwAkIA%Y9x(pkiLrOeW7~mfVkxyp`lx1mjDn<33a+D zpmcq?HR(g-Fp2PKcKULQb1ST6mBx{})>*H|NpZF_QrGIk01z5tbT^7ImFLAbLp|8V z?v6o~ba@E={9|Za3^_%qDsoI&00D@bkveblcv@>LV+U9f)060qX?0t5w$X@+;_Od? zE3TMr1+4QU_755n(ixLhkA1(Ob08+jDD~JV%8_YlV>&qV1F@dWTrzgqG}{>L;j28g zB^3hF2VcPcX)$qEIowQQ<>G^U(Ui^(<}N62{HUA0D8a@-*h#b39f(B$kpPN7ua_Ec z?137wWGb+ttn_s=4y4zimFE0=m0ItV1~{HwidU+bAPI5Ip)fU}$uJ3_w{3>`A#1JV ze3B~ofs2$$qJk5z!e%E9BZb>I`S9_HipdW}E?@22X^Gl+`fepI@MoR4u4G3Lasd&J z0c9>;1Hk3yFQ7|4#7r?W!ASLxKbFKI{d_Xm-K_VHE}48GQlvB zE2UuWkY?x!SGcy5K-`8wrHiRAU(6NRaP~^n;7%8dS<=n;DU|q)FImldtIz#tnsvX` z=%Q;z*i|0M3==wL$3P#+qa%V|tXXKWezy??45jyb+mpxM(zV_=Vex;-7~D^XBRf z#3x>{JH!uevFbKnJW>FYeyyNUHy=IAa04Ix%iu!s@##PvTnr=`f{T*+Q&^n{3}pZT z1)AxqzhS#i#pY#JMntP!36nQ>)KM@d+K9sgsTAfdxQoITWToix*Cw5BQcHRy@ zc+d`Ryl5N*S{(i=c9KQ zlir`Qno$yv^ud|5YbVA`Ds2^zl=OmNE|%7DC4N(&H;#37ZDkyp19=xm?c4`DY z9Sq$&kIu$@-DrY{7GqAu)jUj%qfzp4QFb5apx{9~jTeqa@j#X6?GrkiaXP-LJRh$* zr^Q61t2l3Y=oI~u9GRza$4>W5cqAYoRP z`yf|R-Kcoz=D>o>hcly7Y=BdNm3JbCbYWWk7=&&f=>SiN^dH-C7%$b5)c%2&@#F}V zSKbKlyGe^HU6RoaoE=8V>k`?mkKijkqI0Bepd`|4YhI2m;gp7Yg_!>#*m>IS$9v~j z+UH7gdcG zNR`ashhQE>2f9IYtqBPbqrc_Q^2H?`#TY*^U^2|H1ksdxKvn+e)KnF<{$gfrU83x+ z6m>c?!JSZYk;~Q{;hHJrNM)&WIZfORYI^BT91=)RTupQlHhuJqg@znJ-nJKi2?6w$jQ^C$ zZc}M5^&OI1Cn;U!!mVvG>-`U(4lHVtyMU>A>ExNq!fllKAOHbV_lhulp?Ov2)q;5` zf~c2@ZGuAUxNg}X$)URxU#L{h{6Pk3-9`)Yc> zP=>TdnZ|)~z=-2F2WlG#80e@yVF=}k-5Tt{k`GG)P&EYNrjAo8j52jY|F{z1tUMQ( zTZaw^SGet&be_Jm^n@;?9S!kq5w(*M2jxm^e#H7*Ez{1xqoLT zVb^;9v(^If`~%?z@yRQI5NRJ4%{)~1@HkJguqU*XvtBgLdW4NlRgVwa?T7D4^iVMc5)8oEbclv-CXLW#h|bxZAKRBiqk-G4b%vHcU) zWtA+1_VN}ouAHQ4{ z8Sb?u>T34UkxUv{%ET)lx9V?TmfV=hC<(Qnw>FwHZ6F()Dv%wMtY}5_!$z{og>_9K z9!}I=9d3>DpU%?|{)x2TJtT96QE}P3k7m+DGX^b;>!dZJZf-N4oEf8uUQ3)1CWwV#tJV27Hc;U84 zo%o+4;mAU5SHy(7@v83WKz#s8cQN=?;JnFgfuMm|N)#qq+QJJOs!;W5nT1SDd>av> zTULbTFC%G)bTCy2JQAiKiL4Y2ZQ&(NR9xIoGXz*f*F-l^j>=63B<;E8Ay}CI?R4LJ z<^H3{{=x>@S#0A>`ast=38F83Gg&9EHQ-4%{RXvA%5cwTPzuq$GSK4C^~zVi)OH0& z`M3l@_x@pGxQ+w!?4R4~+|{w@l`=0XqxKo(!f;P=(#XNZQ(#(1Ohls;U=)bK=pm1g z4~}4_UX@UM3*Z2y8G0dxFye%yRy&tIE*}f%o`^f^#W`}j^rpGC8yf6Nw&7v@>@0lIfkbTsX_IKSzYp{SpWFrG1S-Iex=My14< z33^Q-nl<0+25>2Q3Q8U5fieO<52m4^t1>BgG!iX)s$LH_|6Tdg_2l1BB?-*no+=;N zLR{s$PkZAT$I;|0b0rm}iE6bDcPLZFoDU%~hI59W3I`$^XX<;XF@X|PISPuZLrp*Y zF18^;U&-ZGBUL=SN6{0=AFji!YgM-tu*u&M#G|kO#@Ef{R0ZrBtBZKz^kTd5#ws%2 z>kt#qcDf}#arU+VM7eS#A`ZT0g__|K0>c`vmVshx_QGHX!+!+Dy02KB;uSj(xss<&N9Ev$i=G%U0Xy@@XE`q&VAr^4icfvl zYWV-a2^m3#)8ehZ7~*#hknu1dgZQIJV9~K>e{;*-Uuv8)hyHDzGiS~?Sn2FUI=6OI z`UR~mPEU+VXV|UE>4_e-cTqTL@LL2}c#F(If{D>LnSnIdrE}pxzeEnPofKFyRQbW) z7{!vTE!VLBF!b^0Nc+L6rv3Cymm{zeJgTzD#Z}OGu2eN-NRrr2H{+^9l=pQspYs%% z9E0ZTZR7B|(RE!YO!YMao(fqA3o5Z+O^2Qi2Y}Vn(F5~GqeI*YOE|mdZdc_Vg{SIB z@uX^9s-mQFMuEKt2XH;d5fD~=sIe?P0NLRYgei@rr%=_5!rcu4l#pc@PG;<>E#HD` zj|32pXr@rW35~%qGMjUSg+mCxT{{MmWr(&1)G($~1*VRcN|RFmYw8RQmj_1%`(?05 zQxWdiyxYFYBg1`&=?4#2Pp<$Mxv8>tKLrK!1X5&RfO?8-@);F;5Ah1N4&IzXY|gR_ z&M9|J#$%QpsVX|Or>a1-U6vyp=FaVKvK@97AoBr2wcwtR3G7j)K8=hCx6f z@1d5nF!X31Y`ByyCi38$#wL*=hBzptZs)gjGG82~hB)_kxNr*t{MgoP+=i6L5{y!G$H$25?7CBYG>%zWn|4vwTEecNNru*WuJ4BqPh^ zxtWqdw$aEUnbph`^;bbLdVftNbKUVodGu)ImF4wiu#3nntFt<)4NgX{*3Yoyq^q2? zNz6&Im`1S_cnQ*@eC2LJ9>XA#KS2^SqDpagWo#-vbvcyFFT|r4i8e?Q=a(4`a>ESL z;te}O;Tx7+uJ$lWn+^WQWT15K?GbU;??7+-{)Zy-SNK-v%Pd32*MS*T0AY^)AIOQg zc^R3)yHy+z4u1Uvh{9^IS03c^b?3;Gob#+Az2gufm(1_w$-O`=htRbv!&t9){#}T2 z1nS@%2Z0|=z;jCgmNIm!?|$IB6I)-R58M-cbzKqj%i)8Z8gxy%=$@Eb&J|(LeVUl0 z3mLi``wyIXcc31zHSxEE*M!E;>TR67^?JdE7&_WIJ%E<}1a#8mFAo=; z(POY&fBdG%YP_}uN&)#&h!V%)00M!J5?X!LoDHKo7h9Z{U@DkIrwXoDkR{I#*hCaL zi(pl_W~ioN=^qbtCq2I;GL+S35CnL%x&p`B@9NuC4TfSyTOMp&7l8TU{$UTRy4dD!$Moes%6(ViP+il zNvajHcybkzn@I0!mO=+gHJE((VGNmCk7oQOPFtFXWPU1-i2C&x+u2!|Me)qAwQ`l` zPLl3V5+Lf{5E}7lDY9}Su#!U!WSLaoxu9024pyj^kh|FZjbJNKT zBYH+B!U9D;owRfOPHsC>?KD$9tEiKbWc$*?s`5{8)D(W zod1=VupG@hh^HS2Ur|Xs(9jC(5AB`{C@~48w(k3O5EkH@B5h$=i9wMPJE}I`+e(W; zTb|C~7b1m#@rP&bA1)q%s6Gbb#_>9@Yx8Mvg9+t?}oP2 zyb?{^`N6l$ne)N7JXY6aHCg3RDFET6E=h4d^$IcQWLm@fXH^gXrFupu!y-Y#d5|TQ zMb%%9_!<>S##yS=FfO)NJaak{6*qp}u6=Bz+4_XljKp(0cMk5_lR|EWVI&tw4GwR= zVf(JmKWeq!6^^KhiTACrZfa9MfEw8@qkrOJ;6Y9>+|bl7mrDzCNggu1^JVaQlA=3Y zn`RkQ)IJrgU+eMZBF4ix2?F03oKMN3nqt8aa{K88cGKQb-~aSG<0$6eKtL$KU zE|z=)c^|3nZKzv)ntXjd%oZu#JnvDsN54m=++OKU(*b8zB+^0Tqpfpg>y!NUF-Imx ze*5SkC|y2tTw-&5(_}Yadd1@evBNtPd4ZrI7-Q9w%2#CKl5mDI**IWjll{OT9cJmJc?7;)A4M(*7((=K>*59)_yM-oaJN zt%h(-*Ih#k_!M=7v);P4$9F@-|N63BKUY^|*M_qG<#;#fFXZ6hI((;lvp=rO>fWCj z7g-W^g%8y2hRzVLd}CXEHyr-L$6}`tnCYZKI?D8E{sXw+EjK;#d#G(an22m z13<#1fWTrRI%6uth;s$v!yOcqC?df^DIU900ha3sT(z)}vJX@7s0v@uI?XKjeH;)9 z-59wv2dP~G26k1Z9t`N0M=9j5xgxG=K5VRp_KWMtH9j)Dee2$l!C`ioF+FLP%Zuk@ z{wr_*{knkCHU11>z$-pygIyJUuu3m_BtXLkUHgI1<<=?j$QEmrCV7v2bBncp$Glb3 zC;r#jeRJlVeQe1+))Q}@N7=JV9Fghyy|Bao;+u$4OjL>f0cO_4o0FB@(~WOZ*5!DN zlWxJA6KD)dY^djtK+A$qbLQOghY#aL9kKDMi^qQXP3xaZ3%_@K{jdJ<$~klB-z&t0 z2wqD^UyICtY|UHj^R{?uZX~qCb5J-ZQTwxSo%qcc!~4W9ci4}tHN+N)h<0pC?d#jI zcVLeJnwb09bT1%D{8l5p{QkHRi6-WzLO&5-{#kgTSn&>3uIGL}t?MDhq^c)Qe-m^5!i(YXS`^0k{!lM?Y&z}k4WIZzQ^JW+3YQoY z&;2~SEcr%S=ou;7LabN(^I*M^QX+SEyr<`4Rgg9W_Va^47Q9b=jOb zcU>m#`-v43UwtjIK>YUG?J042*?xDq`m(*;xy0ydUORyVCA?S3mM9AhFJZWNT7oR| z@n8|+?qM4tGuUI_D%7zWk5gYey~4(WUuF zW_f@p-P}2Ud#v$&wg|7J6Y~z`8U$}pUN~aaiy!?d(!6~7e*2yD5YAnjFg0Om2Z%5Z z8#Fx&$J8B~NF%z&a4m$8C9$#OHmZA;?;-V`T}GB{B@vk%#*%q8SW}Qfd(&+5aE~n6 zMfZ9hTYK8xZ6S4*cT-IAO3$h!DShnt2kn1;gH4V+VDEi{O+Njgz1gZfe`|1sd%iO# zCf@#zNHf3G5U-uFuU_OmFp)luC?(H7XuorT7V*eXD@teVH=B1`GM}sX%QJR64YNpW^lpmmw~Ky($^FW{ViThsSo`v?=gQ(PG;RD3R5&$97*8_+qf; zvlB~>?0VarIlJB_u>P3XvN`aL2xN0=FYfSEUUHX)t4YK$EF#v%@mP&%>s66eZ4Sj0arTK~_6U5=a_Q+z0Y0+D z-3jtgK*(q=L81B zUXxOu7$>p0FOVy<;tXm3$cs*f{#LzuN0&R|RZz1^hSr$ zrK6f1=A$7UuSAgwHf1U91_CY`P;u_A3YFbivx(o`IE*I6VaMtFCctmCB~@jeW{r4e zz1=MSOLt@`t7gku4>Bqzp)~NU`uulPi?CKi520@txfQp{gmc8@`isSd#eqiU#z{cs z*Jp!)cVLq$=nmw)<|VwIdPO&pQxebrEYc+2x6fLT_k$7!OZmScEdw8+dV%Y5Xoyu& zws`0If7-k9=&FwM{@$JU?tKDnM*D`O2Ld6qAhrNT5F`j*!sgT1Ar6ZW0#qZA5s+d9 z3)M+Yk{*|s;4h8S#5AOC>$-;2IWkUL;*+M08wafTn8wNRk^oK|bk2o$=gvOg_suuod`n17MNt^@Slc%4+b{CdWT+Pwzws$I_cjFz=5{c^ z;Haz*mm|X|1t9#oz zW%#+ZvkR}_OPUzU*{cFDQL$CVOcZr8a!IMO`w8j~dwa&F(L1ZHD%yR<%A>*uV2yP1 z@n{hn%f;xf)3Gwz@i8mIs%xao99&lvpT$kZKirQyi>_ONCoN~KQS`0VAVjyUu`1~3 z15kKBH~_0VT4RlwA-Q4}qr%0fJ8!8Gz5+&b%*$_7p0lX6S&Yt89@sEW~Zwh&oRu0Y8M7m=`Eur34 zw?Xj(M|Jmn1JSjPwxWNi3MBM!#*|3u9humSPnXn(UGEqztth&;bpcY~r*DB&U!ciH zY`D)!NqY_1Ub`zjSQEKco4p2tO~6mL8fD?x0MEXoV^4829>@5fx-N@s077R_{lnthW~U4wc+!M zxhaMPF!Tdh-v*V8EeQ4~J&3WxmCakKHNOkECTfgkAcljkRBep@>Elfbjig{FebXbm zr6fACkM2K`jsS8L4V7;m2f=w%bB!j8dJpDN<1|_1>)u0$^V)R1ilagwb%Jb2dZdN+ zH@GbqRk*=x2O~*LB?9K8fJ$mSV&|;B_S$$8SF;)dNY|P3IyTE-jc2LDyQ*6;u`~q% z>u6207!$3b-eyrWQgQ^$1wy5wZoVi)JaK6?0mnG5Dw;e5(#1HW7@r!OQ426`yF;pp zV}b&C+ClpHzTIS~mRFN4UII#Rza=;UzrFQVjq)bcvDN?^tG&5pY4OLM!W(@FxPNmU zxpBZL_^{xRQ|=%kQC4GjRrz_Df<J`8}~wq!*b% zgr`|9MINS)rNbtdoIBb-%_9O-|Hu8wz_Z zj4ge#Syc6gxYhvV7!;M8irFi8Zdw$X;A3>X?~cqYGd2!gI=q4S^K9xxT{`2SBa}a< z&Lqi&-iuOYP*iES94;-EVqr%+3taMQEdQg571rbPwpK~FIUH*t zt$r0j^@c%vYTS>d{>>urF$-<0WF+XKEPeZ=6}0r#SYEFNICKhAz|kmE#L6`DpmR&Q z;?qA-d!g>BAZ55NJo%PBz zh{+kc7IGH{+QR;M^gwXzY};JDwjE~Ee*b_wRTTyd+_jM6?B((fn5RIGT7l^h`o-)= zUh)8L%a~-@164P75_qbrWKUF;)7-u7g8?ihzaCARdAt_ZL3ds;xgJbNq%=?^-@K1% zq~>dgYv^@NB+Pe(oqjmF_KjPesf++4FddQ;G!cJrl2?+XtdJ~__I%9Fn*kUwU@37& zJJ~Ve9IzP&s#l;*jm^>AMvw0{&4gM}XANavg>8^pM~R$XBj=Su0>DZ)M{up@9l@um zRHvF%z!u<+Q(H9$YX#W?BeAtzxm*c2$mn&w7rnpdDrgzi!duY?G1vFNfuWcGg42i9 zu{sgNa-Aq-UtaT|0q^r4HFeU=>4R$s+3>4*ap`9g2KBjg?+ zV5fL$Dt(Tt{C*OYSR^JvOA*fd`=W4Y8`yDK8tYWAnkOs(r0D`7Xl}G*t3S;k|i4JfWtIt(|Mu zwrY1VEU4cR_KpWq#@b6f!Kz2CYd^ldbyJ(P7ciy_pV|PPx1x2e`r=8SbPBo}wCW4k z4!Ji3Ft$7H!d%B)Pp2Cg}G`n~GY=qSxxcAdvbsb;aI8PZmt_^F3ymV7O(Q_}^vznI{Kj*_ouq z#Uj}&ze?LaeRc`VD31{Xa?7&jt}Ku$Pl)l>`=UbP5Wme0%JGzWE_b;p{zG$mPoPC{ ztE`;6n`%37J4KsciC4-+)3VDT?GGW4nSHfucx*YhZ7{LgtvppGAM_eX=fV{UJZr$B z-O>NGd17j`&}F#?T}$;p(=4V(X-}z$O)z>?Qh)}LfU+ZCpwb+>61!1fH*_u1=ZmV@ zMz8>2FhZT6K&yHS>gnjjQ8#p~q$BG^@!a630mPa@Xt^}Iq5@-jmY3Q%9CXn zF@`y0hr2)}3&hjQ;c1X%1`v?~J;7f|^aSh0NGlP^qh8z*Jr|nnN`c14p{}}c_mCaW z%F%^xH>zUfid-!yPk6x$o& zzo};@wH_0-)bzY?Xk~dc(f|An;&RNLwT7M)oKCn|kK@@XU8eo6ySB6`ng&liR;8Mq zxpUCTw8i4x-05yl7)gTnN;*dfUHJ=k4V`RoMiqvU2Z$~0vgxTP^t%sUu}i75&PmS<=A$<=obqMt z3#qlEVSU%ux@q;(>u|;94^Mw-YT7fw-hb)K`Who(Nf(ELbvGmm!?fgu|_^T*mI1K!LL%`9&i}r*@ zf0L~8-o9yDC;E3gRNmW^y%`n^%pxGTJ-Grp>)fjX>N*K+@a%FYla72lTF`$u-iAp}!!`dUeS}bE%nD_sqZi@@F`hMw9=A(O zCCCQqwMvF5Q(E}CS`LI)Br})j7E@&I|M_SKr!$P{%x-ukefHP3L+=(i#dQ9cb|!5) z3F*EZ;ulUqLpu8;#0t;9?EXlb=mItO+FOK%pjT%bAs zRW5z-!RWD)>_~*4Dx_##vO(C)nnu(ooKKexiSRND<9NEp@uge{Xgz`cbppme`O801mISTaw3z>Upk|Luv_&ktNQ$&{IC zE8}PCR_U#iLqRfTsv==lT~osJG}cB!XcvV3<27*HSF`dZ!%RMqqeCL2KNp*qr+?!x-|BShVjmahK8Gj zZke7s0}`XV^atJ?O#eWhH+DeLb4MF=pBdi%v}63S>=-!74c<@~4*AbpHHnvITWg16v-%Z1bKWWa(9G6@X_vgE=mIk#Op zgcAHT21@>Ueae!Xl-hZnB&u~F6o_X5-^F)E6NS^B-oNO3M`lDK{1gW(Ye-d@)E!nM zSao=yzG++}!cVYh>xNWC`68coopW;QOJARH;&M$S!cTDGj^8)9B$dD8lum!(<>D_^ zW`c9T6D*VQy#eA{UjpLp1!``&gwEF#A&8hG2OE;EL=FupAd6l(sN zMfvBQ8^q|zf?Cfz9~IwFKMLqzj%Cx2&ZC3lr^|0{Qu2Z`UDURDN!QUg&tl8jdjTUm z_E|3-r7t)|WdW<(K;!%V)}I-sxYNYI_x(vf>#|1Du6HnO&lWKf0cM!L1AX^B=Pv;p z#p&YDF8Grw-gTyk*Rx_gyaIab30$#ZlCx>|e_PoZx}pPp|5WDYP~(0Vow2L^Ny&>& zX_;qL6o~-7&EJ8(Q!D+M-*UtV`tnuu+_S?8m-wGA`Afh)X9OL;h}B=76<2+gJ*l|l zOcrfL{-jHLz^C7R30a$`$LT*Fg?W-3(Q}uu0^8>yUiK5Q96FA9qi>&$`3HGp528K2 z=QN4e9<-_JJ!iJ~-q&q<>^*0$_}o9E9Dn-CfK82;ojKyC^Bvl98K1fqJM=jJc>HS) zHRK~!ZRQm;{uk#Qy8a64co;wKy5h8m`!6{3-7CoWtEvQ9SMlSfi3ys26}7IOm!QvF zMXhDa67+pe`NpjY8hz9*qo162viD6+6pO0c67{@j=sN`fGOhj;e&}+ z+d$tiwaGIGDwr%9?oZI6-`XX#`!_h49gAX&&X;9UjT+Hw@7dW|UbTz&qgvW?8Hbg- zEo9FAW`Z8E5;u#-A5PFpILcO~rV7kl{E-AL6Ny`d_)&r$t#+#DhayoEf8!WeQCsVb zqly@+xaa>8v?!LCEq?rRg8n*|m@D>AO{1fn@=Q+}WyBMunM)!|BE_IAMGvH*FaHln CqR7ku delta 2010 zcmaJ>dr(x@9lrOTbMM`=JY0E;S$5eG1i=*yk_?f6Ws(qVM%yMvN8`F9>!aeXl{HQ^ z4PlIi)-gfW_#yIGVR;nPAw$x`w3w21G>OwFs3~n6i_vP*QBg{)A*Pd_Wogj<(K~bR zobUJhzTe~A-#H(Dmdbx+?Y^_$=>B4?WaueL8hWbdoBb;iduKPY|&g@y# z6OggVB^!HYzV~kf)_6&bZ@4efx8t1MSN7gQU-uam170$_XXO1QiF}_~i77C?RDW#X z^Fm4bypVUTQsQNtt0U=r*J|as%uCiNyEO2&jhTYeqgyT_g?* zW|3*UYZm$1YwUREE8q51N!p%@56slY&mJNd6{IzgZvIX)$L=Rl_ z5H0hp%VeL1JJ(4SUKk{HUi&36%lyVa$f(StZj!|^_uV4%W%Tc%MzGuDaXflMq)Zqg zk)co4D0zYNQIe~uj&0?xt%&r<+3=jC+T5j1?}oHer`PEy-MBT~zSULkDSO$KmTsMA zwFRkj*LvLRHm4~GLJVHwA|(P+VZUHw>sTk zSLrWMQLV(``OTVzrQ2QZwVqN}8rHbvSlmw`1(^3NU5StGl13CU5KCu&DlB1?dOkD`NqOnY(j9G%81 zPSf)W2I3h-_ilyprnB@rdT^~u;g8Z)_quW8#G1bFW#ZI zNO0XLWAUpb3Z8Ll!QGqmY7iFlrhD{;jMF1D;*EF5sd!cKYW&@Cz4w(j@@ot{zlMFK zK5(Y-L@mp~%EN4ia(xJOX2pVzT4qA)VV0=u{0#G~iVX)23d@+Vkh(Dmrg~OFe*c|> zm+G0_Jaw8SDPEd-0+sjUWOT1m63}1I=8);jW!$P~g@jkg$ZB9|=I@WIII`v`p%kbb z-z&p$QAxrAzewEDAQHQ;$oSbk5|5zQ^Vo7G(HY4oUG(pj1@(MgM)hlVMX#)D+rLZl;xOgU7#n~3NkR->exZI@0 z;xSP-(D{(=i)0l`1A_i_v5M*ddy+ifr(!U`3dot0DxzD(A}UV9V56RZ&8?!0KX1}d z*DB(wWg7n4%8JMxw+2g_AU@ro!5Lh1G;3&S6H(vq)o`^3BO^ruGeL(q@=e8d5Q@gUwOr{ACiy!P~(eBhOtE#158+!@m}?zFQh1 zJK0R!uQ5a-w@Wk^TWff5Qx1$xFrFtDztynmh#>>jouY#VNymv!wwNp;ItJT?x%Q9{ zKi$O=)f!C~CGaj$!rm^ytxnTn{lt)hO9H7oGJ@Ww0pXpH{NSjl!}O>Q$5F8%jwf{7 p9yY|{W%1MT$Dq$%t>bdgw_-?#=3.10","schema_version":1,"sha256":"57b0f86a9c49daff2c0213d5606d2c369c4e39580b52cd76cfd20fdb2de33167","source_hash":"fccff1138073eb96d48fbb60470bb7f4d6d27770da7a2f61bc490b83ee3a912a"} +{"builder":"scripts/build_codex_plugin.py","python_requires":">=3.10","schema_version":1,"sha256":"1aea1f9a6c3f2077b8f7877972aab9a16868678a755288a8c3fbaaf1217c1d06","source_hash":"3582f402ae25c9054e828e6554358ccfcbbfb85a89cde7aed8cd2f6345ad72aa"} diff --git a/tests/commons/test_local_e2e.py b/tests/commons/test_local_e2e.py new file mode 100644 index 0000000..5be984d --- /dev/null +++ b/tests/commons/test_local_e2e.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from dataclasses import replace +from pathlib import Path + +import marginal.integrations.codex.service as service_module +from marginal.commons.client import CommonsAck +from marginal.commons.config import CommonsMode, configure_commons_mode +from marginal.commons.outbox import CommonsOutbox, OutboxEntry +from marginal.integrations.codex.events import SessionEvent +from marginal.integrations.codex.identity import current_promotion_identity +from marginal.integrations.codex.service import ( + read_mode, + start_session_service, + stop_session_service, +) +from marginal.integrations.codex.transport import request_session + +_SOURCE_COMMIT = "7347a1b4024329780139d17494430f2ccac94fec" +_MODEL_NAMESPACES = ( + "openai/gpt-5.6-luna", + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", +) + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + + +def _pack(models: dict[str, list[dict[str, object]]], *, revision: int) -> bytes: + payload: dict[str, object] = { + "schema_version": "1.0", + "source_commit": _SOURCE_COMMIT, + "commons_revision": revision, + "compatibility": {"evidence_envelope_schema_version": "1.0"}, + "models": { + namespace: {"aggregates": models.get(namespace, [])} for namespace in _MODEL_NAMESPACES + }, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + payload["integrity"] = {"sha256": hashlib.sha256(canonical).hexdigest()} + return (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +class _LocalIngressCommonsAdapter: + """Synthetic closed Ingress boundary backed by aggregate-only local Commons files.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.models: dict[str, list[dict[str, object]]] = {} + self.revision = 1 + self.submitted_bodies: list[bytes] = [] + self.retry_headers: list[str] = [] + self._write_pack() + + def _write_pack(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + (self.root / "commons-pack-v1.json").write_bytes(_pack(self.models, revision=self.revision)) + + def download(self) -> bytes: + return (self.root / "commons-pack-v1.json").read_bytes() + + def submit(self, entry: OutboxEntry) -> CommonsAck: + envelope = json.loads(entry.body_bytes) + assert set(envelope) == {"schema_version", "model_namespace", "atoms"} + assert entry.retry_token.encode() not in entry.body_bytes + namespace = envelope["model_namespace"] + aggregates = self.models.setdefault(namespace, []) + for atom in envelope["atoms"]: + aggregates.append({**atom, "lifecycle": "candidate"}) + self.submitted_bodies.append(entry.body_bytes) + self.retry_headers.append(entry.retry_token) + self.revision += 1 + self._write_pack() + aggregate_path = self.root / namespace / "aggregates.json" + aggregate_path.parent.mkdir(parents=True, exist_ok=True) + aggregate_path.write_text( + json.dumps({"aggregates": aggregates}, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + return CommonsAck(accepted=True, duplicate=False) + + +def test_local_lifecycle_round_trip_is_model_isolated_private_and_non_authoritative( + tmp_path: Path, monkeypatch +) -> None: + """Catches uploads of raw context, cross-model priors, or Commons authority escalation.""" + + canary = "MARGINAL-PRIVACY-CANARY-7f93" + workspace = tmp_path / canary / "repository" + workspace.mkdir(parents=True) + _git(workspace, "init", "-q") + _git(workspace, "config", "user.email", "test@example.com") + _git(workspace, "config", "user.name", "Test User") + tracked = workspace / f"{canary}.txt" + tracked.write_text("private source\n", encoding="utf-8") + _git(workspace, "add", tracked.name) + _git(workspace, "commit", "-qm", "initial") + + data = tmp_path / "plugin-data" + boundary = _LocalIngressCommonsAdapter(tmp_path / "local-boundary") + configure_commons_mode(data, mode=CommonsMode.CONTRIBUTOR) + monkeypatch.setattr(service_module, "_commons_client", lambda: boundary) + start = SessionEvent( + session_id="contributor-session", + cwd=str(workspace), + hook_event_name="SessionStart", + model="gpt-5.6-sol", + permission_mode="default", + source="startup", + ) + connection = start_session_service(start, data_root=data) + for index in range(5): + pre = { + "session_id": start.session_id, + "cwd": str(workspace), + "model": start.model, + "permission_mode": "default", + "turn_id": canary, + "tool_name": "Read", + "tool_input": {"path": str(tracked)}, + "hook_event_name": "PreToolUse", + "tool_use_id": f"{canary}-{index}", + } + assert request_session(connection, operation="pre", payload=pre)["ok"] is True + assert ( + request_session( + connection, + operation="post", + payload={ + **pre, + "hook_event_name": "PostToolUse", + "tool_response": {"exit_code": 0}, + }, + )["ok"] + is True + ) + stop_session_service(start.session_id, data_root=data) + + assert len(boundary.submitted_bodies) == 1 + assert len(boundary.retry_headers) == 1 + assert CommonsOutbox(data).pending(limit=8).entries == () + assert canary.encode() not in boundary.submitted_bodies[0] + persisted_boundary = b"".join( + path.read_bytes() for path in boundary.root.rglob("*") if path.is_file() + ) + assert canary.encode() not in persisted_boundary + + same_model = start_session_service( + replace(start, session_id="same-model-session"), data_root=data + ) + same_status = request_session(same_model, operation="status", payload={})["result"] + stop_session_service("same-model-session", data_root=data) + assert same_status["commons_prior_count"] == 1 + + other_model = start_session_service( + replace(start, session_id="other-model-session", model="gpt-5.6-terra"), + data_root=data, + ) + other_status = request_session(other_model, operation="status", payload={})["result"] + stop_session_service("other-model-session", data_root=data) + assert other_status["commons_prior_count"] == 0 + + repository_hash = current_promotion_identity(workspace).repository_hash + assert read_mode(data, repository_hash=repository_hash)["mode"] == "shadow"