diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8766d990..82a161c6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,11 +14,8 @@ # (a Virtual-Scientists fork) and ari-skill-paper-re/vendor/paperbench # (openai/preparedness) are pinned external forks; auto-bumping their # submodule SHAs is undesirable. -# * pip "/ari-skill-orchestrator" is NOT listed. That one skill ships no -# pyproject.toml (13 of 14 skills have one); pointing Dependabot at a -# missing manifest would error. Only first-party directories are listed, -# so the vendored pyproject.toml files under the two vendor/ submodule -# trees above are also excluded. +# * Only first-party directories are listed, so vendored pyproject.toml files +# under the two vendor/submodule trees above are excluded. # * docker ecosystem is NOT added. There are no in-tree Dockerfiles outside # vendor/ (containers/ holds only a README). # @@ -45,10 +42,9 @@ updates: - "github-actions" # --- Python (pip) --------------------------------------------------------- - # One block covering the 15 verified first-party manifest directories: - # the root requirements.txt, ari-core, and the 13 skills that ship a - # pyproject.toml. ari-skill-orchestrator is intentionally absent (no - # manifest). Minor/patch bumps are grouped per directory to cap PR volume + # One block covering the 17 verified first-party manifest directories: + # the root requirements.txt, ari-core, and all 15 skills. Minor/patch bumps + # are grouped per directory to cap PR volume # across the tree; a low open-PR limit adds a second guardrail. - package-ecosystem: "pip" directories: @@ -60,10 +56,12 @@ updates: - "/ari-skill-hpc" - "/ari-skill-idea" - "/ari-skill-memory" + - "/ari-skill-orchestrator" - "/ari-skill-paper" - "/ari-skill-paper-re" - "/ari-skill-plot" - "/ari-skill-replicate" + - "/ari-skill-tool-registry" - "/ari-skill-transform" - "/ari-skill-vlm" - "/ari-skill-web" diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 8f36aef4..9cd653ca 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -106,6 +106,17 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.13" + - name: Install manifest validation dependencies + run: python -m pip install pydantic pyyaml + - name: Canonical Skill manifest conformance + # Hard gate: an invalid/unversioned manifest, runtime tool drift, + # workflow drift, version drift, generated mcp.json drift, or a + # default-enabled collision is an admission failure. + run: python scripts/check_skill_manifests.py + - name: Generated Skill and execution schema drift + run: python scripts/sync_skill_metadata.py + - name: Generated HPC contract schema drift + run: python ari-skill-hpc/scripts/sync_contracts.py - name: MCP tool-schema snapshot verify (advisory at Stage 1) continue-on-error: true run: python scripts/snapshot_contracts.py --surface mcp --check diff --git a/.gitignore b/.gitignore index ec74236f..f42aa209 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ slurm-*.out # Checkpoint directories (generated at runtime) *_ckpt_*/ checkpoints/ +# Versioned migration fixtures exercise legacy checkpoint readers in CI. +!/ari-core/tests/fixtures/checkpoints/ +!/ari-core/tests/fixtures/checkpoints/** output*/ results*/ @@ -92,6 +95,8 @@ ari-core/checkpoints/ # ── Runtime / temp files (generated at runtime, never track) ──── experiment.md ari-core/experiment.md +# The migration contract intentionally contains historical experiment.md. +!/ari-core/tests/fixtures/checkpoints/v0_7_golden/experiment.md tmp*.md GUI_TASK.md *_TASK.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 246bdbfd..c57babcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to ARI are documented here. Versions follow `MAJOR.MINOR.PATCH`. +## Unreleased — Skill-platform P6 removal + +- Removed the deprecated web provider aliases, mutable backend selector, + iterative LLM collector, and `papers`/`results` retrieval projections. Use + `search_papers(provider=...)`, canonical `records`, and explicit broker/workflow + composition. +- Removed coding-skill's flat measurement writer and permissive JSON coercion. + `emit_results` now writes only the typed `measurement_set`; the old reader is + read-only migration support. +- Removed replicate-skill's low-coverage single-call generator, prompt, API/env + switches, GUI controls, and report field. Rubric generation is always + `hierarchical-v2`/`calibrated`, with model-call budgets and repair provenance. +- Removed the five container-specific HPC public aliases and their compilers. + Container work uses `container_submit`. The core-agent `slurm_submit` bridge + remains narrowly supported until that agent emits `JobRequestV1` directly. +- Added the permanent + [compatibility support policy](docs/reference/compatibility_support.md) with + owners and objective re-evaluation gates for retained readers, deployment + paths, PaperBench adaptations, and registry repair. +- Externalized the remaining paper claim-declaration guidance without changing + the bytes sent to the model; prompt and composition digests are regression + tested. +- Removed the completed temporary skill-platform plan set after its + architecture, contracts, migration rules, and retained-support decisions were + transferred to permanent documentation. + +Pre-removal rollback boundary: `c487ea9` (`feat(orchestrator): add durable +authenticated run control`). + ## v0.9.1 — Contract-preserving refactoring program (73 subtasks) + DONE-verification audit (2026-07-05) - **73-subtask refactoring program, every contract preserved.** ari-core, the 14 diff --git a/README.ja.md b/README.ja.md index a9ca8c37..77547ca6 100644 --- a/README.ja.md +++ b/README.ja.md @@ -95,12 +95,12 @@ MCP ツール・描画出力のいずれも変更なし。 でドッグフードに使用可能。 - **`container_image` のエンドツーエンド配線** — ウィザード → API ワーカー → MCP ツール → サンドボックスランナーまで同じ 1 フィールドが流れる。 - `pb-env` / `pb-reproducer` の短縮エイリアスは - `scripts/build_pb_images.sh` でビルドされる `image:latest` タグに解決。 + v1.0 は immutable なローカル SIF、完全な Docker + `sha256:`、または digest 固定 URI のみを許可し、mutable な + `pb-env` / `pb-reproducer` エイリアスは削除した。 - **fail-loud な事前条件チェック** — サンドボックス / GPU の不整合は - 既定で `RuntimeError` を送出(従来サイレントに CPU 実行へ降格していた - 4 箇所を修正)。互換挙動は `ARI_PHASE1_ALLOW_FALLBACK=1` と - `ARI_SLURM_ALLOW_NO_GRES=1` でオプトイン可能。 + エラーで停止する。旧 host-local 再現 fallback は v1.0 で削除され、GPU + 要求も黙って降格できない。 - **PaperBench env-truth ガードレール** — Stage 1 のプロンプトに 「scaffold 前にホストを probe する」「言語選択を Python 偏重から 打ち消す」「ホスト実機を反映した `ADDITIONAL NOTES`(バイナリ / GPU / diff --git a/README.md b/README.md index 18c9355a..e0656d86 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,12 @@ See [CHANGELOG.md](CHANGELOG.md#v081--structural-refactor-frontend-decomposition surface. Drives the dogfood CLI via `scripts/sc_paper_dogfood.py --with-rollout / --with-reproduction`. - **`container_image` end-to-end** — one field flows wizard → API worker → - MCP tool → sandbox runner, with `pb-env` / `pb-reproducer` aliases - resolved by `scripts/build_pb_images.sh`. -- **Fail-loud preconditions** — sandbox / GPU mismatches now raise - actionable `RuntimeError`s by default (four silent-downgrade sites - fixed); legacy fallbacks behind `ARI_PHASE1_ALLOW_FALLBACK=1` and - `ARI_SLURM_ALLOW_NO_GRES=1`. + MCP tool → sandbox runner. v1.0 admits only a local immutable SIF, a full + Docker `sha256:`, or a registry URI pinned by digest; mutable + `pb-env` / `pb-reproducer` aliases were removed. +- **Fail-loud preconditions** — sandbox / GPU mismatches raise actionable + errors. The former host-local reproduction fallback was removed in v1.0; + GPU requests likewise cannot be silently downgraded. - **PaperBench env-truth** — Stage 1 prompts now probe-before-scaffold, counter-prime the language choice, and inject a host-truthful `ADDITIONAL NOTES` block (binaries / GPU / network / Phase-2 isolation). @@ -352,7 +352,9 @@ After a run completes, outputs are saved in `./checkpoints//`: ### Skills (MCP plugin servers) -13 skills total. 12 are registered by default in `workflow.yaml`; 1 additional skill (orchestrator) can be enabled by adding it to the config. +15 skills total. 13 are registered by default in `workflow.yaml`; 2 additional +skills (orchestrator and the federated tool registry) are default-off and can be +enabled explicitly. In v0.6.0 two skills were retired: `ari-skill-figure-router` was folded into `ari-skill-plot` (a single skill now owns both matplotlib plots and SVG architecture diagrams, both feeding the same VLM review loop), and `ari-skill-review` (rebuttal generation) was deleted — the rubric-driven review score is the final quality signal. @@ -370,7 +372,9 @@ In v0.6.0 two skills were retired: `ari-skill-figure-router` was folded into `ar | `ari-skill-benchmark` | CSV/JSON analysis, plotting, statistical tests | ✗ | ✓ | | `ari-skill-vlm` | Vision-Language model figure/table review | ✓ | ✓ | | `ari-skill-coding` | Code generation + execution + file read + bash | ✗ | ✓ | +| `ari-skill-replicate` | PaperBench-compatible rubric generation and audit | ✓ | ✓ | | `ari-skill-orchestrator` | Expose ARI as MCP server, recursive sub-experiments, dual stdio+HTTP transport | ✗ | — | +| `ari-skill-tool-registry` | Immutable federation, scientific admission, and replay for large MCP collections | ✗ | — | ✗ = no LLM, △ = LLM used in some tools only, ✓ = primary tools use LLM. @@ -425,4 +429,3 @@ What makes this paper different is not a headline number but a property: **every ## License MIT. See [LICENSE](LICENSE). - diff --git a/README.zh.md b/README.zh.md index bbfddd03..deef685c 100644 --- a/README.zh.md +++ b/README.zh.md @@ -89,12 +89,11 @@ ARI 围绕一个原则设计:**用 Markdown 描述目标 — 其余的交给 A `scripts/sc_paper_dogfood.py --with-rollout / --with-reproduction` 在自测中驱动。 - **`container_image` 端到端贯通** — 同一个字段从向导 → API worker → - MCP 工具 → 沙盒运行器一路传递;`pb-env` / `pb-reproducer` 短别名 - 通过 `scripts/build_pb_images.sh` 解析为对应的 `image:latest` 标签。 -- **失败时立即报错的前置检查** — 沙盒 / GPU 不匹配的四处原本静默 - 降级到主机 CPU 的位置,现在默认抛出可操作的 `RuntimeError`; - 兼容回退路径通过 `ARI_PHASE1_ALLOW_FALLBACK=1` 与 - `ARI_SLURM_ALLOW_NO_GRES=1` 显式启用。 + MCP 工具 → 沙盒运行器一路传递。v1.0 仅接受不可变的本地 SIF、完整 + Docker `sha256:` 或按摘要固定的 registry URI;可变的 + `pb-env` / `pb-reproducer` 别名已删除。 +- **失败时立即报错的前置检查** — 沙盒 / GPU 不匹配会直接报错。旧的 + host-local 再现回退已在 v1.0 删除,GPU 请求也不能静默降级。 - **PaperBench env-truth 护栏** — Stage 1 提示加入 「先 probe 再 scaffold」「对抗语言选择的 Python 偏置」 「按主机实测注入 `ADDITIONAL NOTES`(二进制 / GPU / 网络 / 第二阶段隔离)」 diff --git a/ari-core/ari/README.md b/ari-core/ari/README.md index 553e7a13..d457ee23 100644 --- a/ari-core/ari/README.md +++ b/ari-core/ari/README.md @@ -9,18 +9,38 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `__init__.py` — package marker. - `_deprecation.py` — `DeprecationWarning` helpers for v0.5→v1.0 legacy paths/aliases. - `_factory.py` — TODO +- `analysis.py` — TODO - `artifact_store.py` — TODO +- `async_tools.py` — portable asynchronous handle, timeout budget, and lifecycle contracts. +- `call_context.py` — explicit run/node lineage models and tool-bound signed MCP capabilities. - `checkpoint.py` — checkpoint JSON I/O (`tree.json` / `nodes_tree.json` / `results.json`). +- `claim_gate_contract.py` — TODO - `cli_ear.py` — `ari ear …` curation / publish / promote / status CLI surface. - `container.py` — unified container runtime abstraction (Docker / Singularity / bare subprocess). - `core.py` — generic ARI runtime builder (zero domain-specific code; `cli` calls into it). - `cost_tracker.py` — per-call logs + per-experiment cost summaries. +- `credential_scope.py` — value-free credential-scope contract and environment-name classification. - `env_detect.py` — detect schedulers, container runtimes, and HPC resources. +- `execution.py` — TODO +- `figure_contract.py` — TODO +- `latex_claims.py` — TODO - `lineage.py` — recursion lineage helpers; walk `parent_run_id` chains for ancestor artifacts. - `memory_cli.py` — `ari memory` subcommand (migrate / backup / …). +- `memory_contract.py` — TODO +- `paper_contract.py` — TODO - `paths.py` — centralised `PathManager` for directory layout/resolution. - `pidfile.py` — `.ari_pid` write/read/cleanup for run-liveness detection. +- `research_contract.py` — TODO +- `result.py` — typed MCP result envelope, artifact, error, context, and provenance contract. +- `science_data_base.py` — TODO +- `science_data_contract.py` — TODO +- `science_data_derived.py` — TODO +- `science_data_migration.py` — TODO +- `science_data_validation.py` — TODO +- `skill_lock.py` — deterministic run-level MCP provider/schema/phase snapshot. +- `skill_manifest.py` — canonical versioned Skill package and tool-policy contract. - `trace_store.py` — TODO +- `visual_review_contract.py` — TODO - `agent/` — ReAct loop, environment capture, per-stage workflow guidance. - `README.md` — agent index. - `__init__.py` — package module-map docstring. @@ -35,6 +55,7 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `shims/` — executable `PATH` shims for the reproducibility sandbox. - `README.md` — shims index. - `git.sh` — intercepts only `git clone` of the paper's ref; other git passes through. +- `calibration/` — `evaluator_v1.json` is the permanent, versioned calibration input for ARI's - `cli/` — CLI entry point (thin Typer wrapper; delegates to `ari.core`). - `README.md` — cli index. - `__init__.py` — Typer app entry point. @@ -60,6 +81,7 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `README.md` — config index. - `__init__.py` — Pydantic config models + env-var overrides. - `finder.py` — workflow / profile YAML discovery. + - `skill_runtime.py` — canonical manifest to runtime registration metadata adapter. - `configs/` — external config tables (Phase PC). - `README.md` — configs index. - `__init__.py` — config-table exports + loader plumbing. @@ -80,11 +102,18 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `mcp/` — MCP client talking to `ari-skill-*` subprocesses. - `README.md` — mcp index. - `__init__.py` — public `MCPClient` + contract. - - `client.py` — stdio connection pooling, retry, per-thread asyncio loop. + - `child_environment.py` — exact child allowlist, credential scopes, and redaction. + - `claude_bridge.py` — value-free Claude CLI MCP config and allowed-tool rendering. + - `client.py` — registry, typed dispatch, retry, and connection pooling. + - `connection.py` — one provider's stdio lifecycle and event-loop thread. + - `dispatch_support.py` — pure identity, phase, timeout, and tracing policy. + - `invoke_runtime.py` — retries, cancellation, and typed transport normalization. + - `lock_runtime.py` — per-client exact/subset `SKILLS.lock` reconciliation state. + - `registry_runtime.py` — live discovery, enrichment, and collision admission. + - `secure_stdio_proxy.py` — exact-env/redacting boundary for direct MCP clients. - `memory/` — backend abstraction for ancestor-scoped node memory. - `README.md` — memory index. - `__init__.py` — `MemoryClient` protocol, backends, migration map. - - `auto_migrate.py` — v0.5.x → v0.6.0 auto-migration on first launch. - `backend.py` — sanctioned core→skill funnel: lazy forwards (`get_backend` / `clear_backend_cache` / `build_verified_context`) to the rich `MemoryBackend`. - `client.py` — abstract `MemoryClient` ABC. - `file_client.py` — `FileMemoryClient` (legacy JSONL). @@ -93,6 +122,8 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `migrations/` — migration shims for older checkpoint formats. - `README.md` — migrations index. - `__init__.py` — rationale + layout. + - `checkpoint.py` — digest-bound, read-only view of legacy paper/replay inputs. + - `skill_manifest.py` — in-memory conversion of unversioned Skill metadata; - `v05_to_v07/` — v0.5 → v0.7 migration helpers. - `README.md` — v05_to_v07 index. - `__init__.py` — subsystem map + deprecation plan. @@ -175,14 +206,32 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `public/` — public API surface for ARI skills (import-only contract). - `README.md` — public index. - `__init__.py` — exported sub-modules + rationale. - - `claim_gate.py` — re-exports five symbols from `ari.pipeline.claim_gate`: `run_hard_gate` (→ ari-skill-evaluator), `check_emission` (→ ari-skill-coding), `scan_science_data` (→ ari-skill-transform), plus `classify_concept` / `CONCEPT_INVARIANTS` (shared concept→invariant registry). + - `analysis.py` — versioned deterministic analysis requests and result contracts. + - `call_context.py` — explicit run/node/lineage models plus signed transport capability helpers. + - `claim_gate.py` — canonical deterministic gate plus versioned metric + - `clone.py` — digest-verified EAR bundle retrieval and safe extraction. - `config_schema.py` — re-export of `ari.config` models. - `container.py` — re-export of `ari.container`. - `cost_tracker.py` — re-export of `ari.cost_tracker`. + - `evaluation.py` — stable evaluator-contract surface shared by idea, + - `execution.py` — versioned workspace, bounded execution, complete-log + - `figures.py` — declarative `FigureSpecV1`, digest-bound render/batch + - `latex_claims.py` — canonical lexical LaTeX claim/number/citation/figure parser. + - `lineage.py` — TODO - `llm.py` — re-export of `ari.llm.client.LLMClient`. + - `memory.py` — content-addressed memory records, retrievals, events, and backups. + - `node_selection.py` — deterministic downstream node/source selection. + - `paper.py` — immutable paper build, revision, model-call, compile, review, and - `paths.py` — re-export of `ari.paths.PathManager`. + - `publish.py` — staged EAR publication and promotion. + - `research_contract.py` — TODO + - `result.py` — versioned `ResultEnvelopeV1`, artifact references, typed errors, - `run_env.py` — re-export of `ari.agent.run_env` capture helpers. + - `science_data.py` — canonical `ScienceDataV1` raw/derived/interpretation + - `skill_lock.py` — immutable run-level provider/schema/phase snapshot contract + - `skill_manifest.py` — canonical Skill package, entrypoint, and tool-policy - `verified_context.py` — re-export of `ari.pipeline.verified_context` (`render_grounded_block` / `write_verified_context`; used by ari-skill-paper). + - `visual_review.py` — criteria profiles, artifact-bound review findings, - `publish/` — `ari ear publish`: package + ship a curated EAR. - `README.md` — publish index. - `__init__.py` — publish flow + artifacts. @@ -203,13 +252,45 @@ Core engine package for ARI. Each sub-package carries its own `README.md` - `schemas/` — JSON Schemas shipped with ari-core. - `README.md` — schemas index. - `__init__.py` — `load(name)` loader. + - `analysis_request_v1.schema.json` — TODO + - `analysis_result_v1.schema.json` — TODO + - `async_tool_handle_v1.schema.json` — immutable submit/status/result/cancel handle contract. + - `call_context_v1.schema.json` — explicit run, node, ordered-lineage, and call provenance context. + - `execution_request_v1.schema.json` — exact command, workspace, input digest, environment, resource, network, and container request. + - `execution_result_v1.schema.json` — attempt identity, enforcement report, bounded previews, and complete-log artifacts. + - `figure_batch_v1.schema.json` — declarative specs, render environment, + - `gate_report_v1.schema.json` — deterministic policy/evidence/formula-bound hard-gate report. + - `idea_candidate_v1.schema.json` — admitted falsifiable hypothesis candidate. + - `idea_set_v1.schema.json` — generation lock, admitted candidates, and explicit rejections. + - `measurement_set_v1.schema.json` — typed parameter/measurement/unit/execution/artifact separation. + - `memory_backup_v1.schema.json` — TODO + - `memory_record_v1.schema.json` — TODO + - `memory_retrieval_v1.schema.json` — TODO + - `metric_admission_decision_v1.schema.json` — explicit human admission/rejection record. + - `metric_contract_proposal_v1.schema.json` — provenance-bound, untrusted LLM metric proposal. + - `metric_contract_v1.schema.json` — immutable metric, unit, direction, comparison, and evidence vocabulary. + - `metric_gate_contract_v1.schema.json` — evaluator projection of one admitted metric contract. - `node_report.schema.json` — per-node report schema. + - `paper_build_v1.schema.json` — TODO + - `paper_model_call_batch_v1.schema.json` — TODO - `publish.schema.json` — publish record / manifest schema. + - `research_contract_v1.schema.json` — selected mint-once scientific hand-off consumed by evaluators. + - `result_envelope_v1.schema.json` — typed MCP result plus value-free credential-scope provenance. + - `retrieval_record_v1.schema.json` — provider-neutral literature/web record identity and payload digest. + - `run_comparison_request_v1.schema.json` — TODO + - `science_data_v1.schema.json` — separately digest-bound raw measurement, + - `semantic_review_v1.schema.json` — independent provenance-bound semantic advisory. + - `skill_manifest_v1.schema.json` — canonical Skill package, environment, and credential-scope contract. + - `skills_lock_v1.schema.json` — immutable provider/schema/phase/credential-authority snapshot. + - `statistical_test_request_v1.schema.json` — TODO + - `survey_snapshot_v1.schema.json` — digest-bound record/replay retrieval input and citation graph. + - `visual_review_batch_v1.schema.json` — criteria profiles, artifact identity, - `viz_checkpoint.schema.json` — TODO - `viz_checkpoint_summary.schema.json` — TODO - `viz_settings.schema.json` — TODO - `viz_state.schema.json` — TODO - `viz_tree_node.schema.json` — TODO + - `workspace_ref_v1.schema.json` — canonical closed workspace root. - `viz/` — HTTP + WebSocket dashboard server + React frontend. - `README.md` — viz index. - `__init__.py` — package docstring + module map / public symbols. diff --git a/ari-core/ari/agent/loop.py b/ari-core/ari/agent/loop.py index aa37206f..71800211 100644 --- a/ari-core/ari/agent/loop.py +++ b/ari-core/ari/agent/loop.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any from ari.agent.workflow import WorkflowHints +from ari.call_context import ToolCallContextV1 from ari.llm.client import LLMClient, LLMMessage from ari.mcp.client import MCPClient from ari.memory.client import MemoryClient @@ -28,11 +29,6 @@ MAX_REACT_STEPS = 80 # default; overridden per-instance via AgentLoop(max_react_steps=...) MIN_TOOL_CALLS = 2 -# MCP tools that the parent (ari-core) drives itself and must never be -# exposed to the LLM — otherwise the model could set an arbitrary node -# id and bypass the memory skill's CoW check. -_INTERNAL_MCP_TOOLS = frozenset({"_set_current_node"}) - # Clearly placeholder strings (used to detect LLM-fabricated values) _FAKE_PATTERNS = [ "found n papers", "[title1]", "[title2]", @@ -397,15 +393,54 @@ def __init__( # Tool filtering (Phase 3D — bodies in ari.agent.tool_manager) # ------------------------------------------------------------------ - def _available_tools_openai(self, suppress: set | None = None, phase: str | None = None) -> list[dict]: + def _available_tools_openai( + self, + suppress: set | None = None, + phase: str | None = None, + context: ToolCallContextV1 | None = None, + ) -> list[dict]: from ari.agent.tool_manager import available_tools_openai as _at - return _at(self.mcp, suppress=suppress, phase=phase) + return _at(self.mcp, suppress=suppress, phase=phase, context=context) def _execute_tool_calls( - self, tool_calls: list[dict], node_id: str | None = None, + self, + tool_calls: list[dict], + context: ToolCallContextV1 | None = None, ) -> list[dict]: from ari.agent.tool_manager import execute_tool_calls as _et - return _et(self.mcp, tool_calls, node_id=node_id) + return _et(self.mcp, tool_calls, context=context) + + def _node_tool_context( + self, + node: Node, + *, + phase: str, + run_id: str | None = None, + ) -> ToolCallContextV1: + """Build one immutable context shared by this node's tool calls.""" + + import os + + explicit_run_id = str( + run_id or getattr(self, "run_id", "") or "" + ).strip() + checkpoint = str( + getattr(self, "checkpoint_dir", "") + or os.environ.get("ARI_CHECKPOINT_DIR", "") + ).strip() + run_id = explicit_run_id or ( + Path(checkpoint.rstrip(os.sep)).name if checkpoint else "" + ) + if not run_id: + root_id = (node.ancestor_ids or [node.id])[0] + run_id = f"node-lineage:{root_id}" + return ToolCallContextV1.for_node( + run_id=run_id, + node_id=node.id, + parent_node_id=node.parent_id, + ancestor_node_ids=node.ancestor_ids or [], + phase=phase, + ) def _active_tools( self, @@ -469,11 +504,6 @@ def run(self, node: Node, experiment: dict) -> Node: # Notify the orchestrator so tree.json picks up the RUNNING state # immediately (before the first LLM round-trip, which can take >30 s). self._notify_progress(force=True) - # NB: ARI_CURRENT_NODE_ID synchronization is now done per-call via - # MCPClient.call_tool(..., cow_node_id=node.id), which locks the - # (_set_current_node, write) pair so concurrent BFTS nodes don't - # race on the shared memory-skill env var. The previous once-per- - # run _set_current_node was unsafe at max_parallel_nodes > 1. # Inject work_dir BEFORE forking MCP servers (env snapshot taken at fork time). # Directory creation is handled by PathManager in cli.py; this only sets the env var. _work_dir_early = experiment.get("work_dir", "") if isinstance(experiment, dict) else "" @@ -489,7 +519,16 @@ def run(self, node: Node, experiment: dict) -> Node: if _ckpt_early: import os as _os_ckpt _os_ckpt.environ["ARI_CHECKPOINT_DIR"] = str(_ckpt_early) - tools = self._available_tools_openai(suppress=getattr(self, "_suppress_tools", set()), phase="bfts") + tool_context = self._node_tool_context( + node, + phase="bfts", + run_id=str(experiment.get("run_id") or ""), + ) + tools = self._available_tools_openai( + suppress=getattr(self, "_suppress_tools", set()), + phase="bfts", + context=tool_context, + ) tool_names = [t["function"]["name"] for t in tools] if tools else [] tool_desc = ", ".join(tool_names) if tool_names else "none" has_exec = any(n in ("run_bash", "run_code") for n in tool_names) @@ -654,7 +693,11 @@ def run(self, node: Node, experiment: dict) -> Node: # the legacy `self.experiment_goal` attribute is never assigned in this class # (the old call sites only survived via short-circuit eval + try/except). messages.extend(build_working_context_messages( - self.mcp.call_tool, + lambda name, args: self.mcp.call_tool( + name, + args, + context=tool_context, + ), depth=node.depth, ancestor_ids=node.ancestor_ids or [], eval_summary=node.eval_summary, @@ -853,6 +896,7 @@ def _validate_pairs(lst): response = self.llm.complete( llm_msgs, tools=effective_tools, require_tool=(active is not None), node_id=node.id, phase="react", skill="agent_loop", + call_context=tool_context, ) if response.tool_calls: @@ -884,7 +928,10 @@ def _validate_pairs(lst): ], }) - results = self._execute_tool_calls(response.tool_calls, node_id=node.id) + results = self._execute_tool_calls( + response.tool_calls, + context=tool_context, + ) # Build args lookup by tool name for trace logging _tc_args_by_name = { tc.get("function", {}).get("name", ""): tc.get("function", {}).get("arguments", "") @@ -938,7 +985,7 @@ def _validate_pairs(lst): "node_id": node.id, "text": f"Tool {r['name']}: {rc[:1000]}", "metadata": {"step": step, "tool": r["name"]}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: self.memory.add( f"Tool {r['name']}: {rc[:1000]}", @@ -977,7 +1024,7 @@ def _validate_pairs(lst): "node_id": node.id, "text": summary, "metadata": {"type": "survey_papers"}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: self.memory.add( summary, @@ -1011,8 +1058,6 @@ def _validate_pairs(lst): # generate_ideas call: capture primary_metric and higher_is_better # Track that generate_ideas was called to prevent repeated calls if r["name"] == "generate_ideas": - self._ideas_generated = True - self._suppress_tools = {"generate_ideas"} try: idea_raw = r["result"] if isinstance(idea_raw, str): @@ -1024,18 +1069,61 @@ def _validate_pairs(lst): if isinstance(idea_data, dict) and "result" in idea_data: _inner = idea_data["result"] idea_data = json.loads(_inner) if isinstance(_inner, str) else _inner + _typed_idea = ( + idea_data.get("typed_schema_version") + == "ari.research-contract/v1" + ) + _idea_admitted = ( + not _typed_idea + or idea_data.get("contract_status") == "admitted" + ) + self._ideas_generated = _idea_admitted + self._suppress_tools = ( + {"generate_ideas"} if _idea_admitted else set() + ) # Persist full idea data to checkpoint for Idea tab try: _ckpt = getattr(self, "checkpoint_dir", None) if _ckpt: + from ari.public.execution import WorkspaceRefV1 + + _idea_workspace = WorkspaceRefV1( + root=str(Path(_ckpt).expanduser().resolve()) + ) + _idea_workspace.atomic_write_bytes( + "idea.json", + ( + json.dumps( + idea_data, + ensure_ascii=False, + sort_keys=True, + indent=2, + ) + + "\n" + ).encode("utf-8"), + ) _idea_path = Path(_ckpt) / "idea.json" - _idea_path.write_text(json.dumps(idea_data, ensure_ascii=False, indent=2)) logger.info("Saved idea.json to %s", _idea_path) except Exception as _se: logger.warning("Failed to save idea.json: %s", _se) - pm = idea_data.get("primary_metric", "") - hib = idea_data.get("higher_is_better", True) - mr = idea_data.get("metric_rationale", "") + _research_contract = None + if _typed_idea: + from ari.public.research_contract import ( + parse_research_contract_document, + ) + + _research_contract = parse_research_contract_document( + idea_data + ) + if _research_contract is not None: + _metric = _research_contract.metric_contract + pm = _metric.name + hib = _metric.direction != "lower" + mr = _metric.rationale + else: + pm = idea_data.get("primary_metric", "") + hib = idea_data.get("higher_is_better", True) + mr = idea_data.get("metric_rationale", "") if pm: # Persist to memory so pipeline.py can read it try: @@ -1071,7 +1159,14 @@ def _validate_pairs(lst): # including descendants that never re-run generate_ideas — # inherits the design intent (planned mechanism, target # workloads), not just the metric. Run-level invariant. - _best_idea = (idea_data.get("ideas") or [{}])[0] if isinstance(idea_data, dict) else {} + if _research_contract is not None: + _best_idea = { + "title": _research_contract.title, + "description": _research_contract.hypothesis, + "experiment_plan": _research_contract.experiment_plan, + } + else: + _best_idea = (idea_data.get("ideas") or [{}])[0] if isinstance(idea_data, dict) else {} _idea_summary = f"{_best_idea.get('title','')}: {(_best_idea.get('description','') or '')[:400]}" try: from ari.pipeline import _extract_plan_sections as _eps_seed @@ -1371,7 +1466,10 @@ def _dyn_extractor(text: str, p=_art_pat) -> dict: "arguments": json.dumps({"job_id": job_ids[-1]}), }, }] - poll_results = self._execute_tool_calls(poll_tc, node_id=node.id) + poll_results = self._execute_tool_calls( + poll_tc, + context=tool_context, + ) rc2 = json.dumps(poll_results[0]["result"], ensure_ascii=False) logger.info("Auto-poll job %s: %s", job_ids[-1], rc2[:100]) # OpenAI requires tool message to follow assistant message with tool_calls @@ -1497,7 +1595,7 @@ def _dyn_extractor(text: str, p=_art_pat) -> dict: "node_id": node.id, "text": f"RESULT SUMMARY node={node.id} label={node.label}: metrics=[{metrics_str}] summary={summary[:300]}", "metadata": {"type": "result_summary", "metrics": node.metrics}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: pass node.mark_success(artifacts=artifacts, eval_summary=summary) @@ -1566,7 +1664,7 @@ def _dyn_extractor(text: str, p=_art_pat) -> dict: "node_id": node.id, "text": f"RESULT SUMMARY node={node.id} label={node.label}: metrics=[{_ms}] stdout={self._slurm_real_stdout[:300]}", "metadata": {"type": "result_summary", "metrics": node.metrics}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: pass node.mark_success( @@ -1584,7 +1682,7 @@ def _dyn_extractor(text: str, p=_art_pat) -> dict: "node_id": node.id, "text": f"RESULT SUMMARY node={node.id} label={node.label}: metrics=[{_ms}] summary={summary[:300]}", "metadata": {"type": "result_summary", "metrics": node.metrics}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: pass node.mark_success( @@ -1632,7 +1730,7 @@ def _dyn_extractor(text: str, p=_art_pat) -> dict: "node_id": node.id, "text": f"RESULT SUMMARY node={node.id} label={node.label}: metrics=[{_ms}] summary={summary[:300]}", "metadata": {"type": "result_summary", "metrics": node.metrics}, - }, cow_node_id=node.id) + }, context=tool_context) except Exception: pass node.mark_success( diff --git a/ari-core/ari/agent/react_driver.py b/ari-core/ari/agent/react_driver.py index 2b18684b..0fb9fbda 100644 --- a/ari-core/ari/agent/react_driver.py +++ b/ari-core/ari/agent/react_driver.py @@ -23,6 +23,7 @@ from pathlib import Path from typing import Any +from ari.call_context import ToolCallContextV1 from ari.llm.client import LLMClient from ari.mcp.client import MCPClient @@ -249,7 +250,30 @@ def run_react( messages: list[dict] — full conversation log tool_calls_count: int """ + checkpoint = os.environ.get("ARI_CHECKPOINT_DIR", "").strip() + run_id = Path(checkpoint.rstrip(os.sep)).name if checkpoint else "" + call_context = ( + ToolCallContextV1.for_run(run_id, phase=agent_phase) + if run_id + else None + ) raw_tools = mcp.list_tools(phase=agent_phase) + + def _context_admits(tool: dict) -> bool: + requirement = str( + (tool.get("policy") or {}).get("context_requirement") or "none" + ) + return requirement == "none" if call_context is None else call_context.satisfies( + requirement + ) + + raw_tools = [tool for tool in raw_tools if _context_admits(tool)] + context_requirements = { + str(tool.get("name") or ""): str( + (tool.get("policy") or {}).get("context_requirement") or "none" + ) + for tool in raw_tools + } tool_defs: list[dict] = [ { "type": "function", @@ -286,10 +310,18 @@ def run_react( for step in range(1, max_steps + 1): try: + completion_kwargs = { + "phase": agent_phase, + "skill": "react_driver", + "work_dir": _work_dir, + } + if call_context is not None: + completion_kwargs["call_context"] = call_context resp = llm.complete( - _build_window(messages), tools=tool_defs, require_tool=False, - phase=agent_phase, skill="react_driver", - work_dir=_work_dir, + _build_window(messages), + tools=tool_defs, + require_tool=False, + **completion_kwargs, ) except Exception as e: log.error("react_driver step %d LLM error: %s", step, e) @@ -372,7 +404,12 @@ def run_react( # Regular MCP dispatch. try: - result = mcp.call_tool(tool_name, args) + requirement = context_requirements.get(tool_name, "none") + result = ( + mcp.call_tool(tool_name, args, context=call_context) + if requirement != "none" and call_context is not None + else mcp.call_tool(tool_name, args) + ) except Exception as e: result = {"error": f"{tool_name} failed: {type(e).__name__}: {e}"} text = json.dumps(result, ensure_ascii=False, default=str) diff --git a/ari-core/ari/agent/tool_manager.py b/ari-core/ari/agent/tool_manager.py index 4cb659f6..43eccca4 100644 --- a/ari-core/ari/agent/tool_manager.py +++ b/ari-core/ari/agent/tool_manager.py @@ -5,10 +5,9 @@ don't need to change. - :func:`available_tools_openai` — convert MCP tool list to the OpenAI - function-calling shape, filtering ``_set_current_node`` and any - user-supplied suppress set. + function-calling shape, filtering any user-supplied suppress set. - :func:`execute_tool_calls` — dispatch a batch of tool calls, - routing CoW-guarded memory tools through ``cow_node_id``. + attaching the explicit run/node context to every dispatch. - :func:`active_tools` — phase-aware filter over the available tool list (post-survey vs post-job-submit vs final output, etc.). @@ -19,27 +18,23 @@ import json as _json from typing import Any +from ari.call_context import ToolCallContextV1 from ari.agent.message_utils import _tool_was_called from ari.agent.workflow import WorkflowHints -# MCP tools that the parent (ari-core) drives itself and must never be -# exposed to the LLM — otherwise the model could set an arbitrary node -# id and bypass the memory skill's CoW check. -_INTERNAL_MCP_TOOLS = frozenset({"_set_current_node"}) - - def available_tools_openai( mcp: Any, suppress: set | None = None, phase: str | None = None, + context: ToolCallContextV1 | None = None, ) -> list[dict]: """Return the MCP tool list in OpenAI function-calling format. ``suppress`` excludes tools by name (e.g. already-called once-only tools); ``phase`` filters to tools whose declared phase matches. """ - suppress = (suppress or set()) | _INTERNAL_MCP_TOOLS + suppress = suppress or set() return [ { "type": "function", @@ -49,7 +44,7 @@ def available_tools_openai( "parameters": t.get("inputSchema") or t.get("parameters") or {"type": "object", "properties": {}}, }, } - for t in mcp.list_tools(phase=phase) + for t in mcp.list_tools(phase=phase, context=context) if t.get("name", "") not in suppress ] @@ -57,14 +52,12 @@ def available_tools_openai( def execute_tool_calls( mcp: Any, tool_calls: list[dict], - node_id: str | None = None, + context: ToolCallContextV1 | None = None, ) -> list[dict]: """Execute a batch of tool calls and return results. - When *node_id* is provided and the call targets a CoW-guarded - memory tool, ``cow_node_id`` is forwarded to ``mcp.call_tool`` so - the ``(_set_current_node, write)`` pair is locked atomically — - prevents the env-var race when ``max_parallel_nodes > 1``. + ``context`` is forwarded unchanged. The MCP control plane uses manifest + policy to require and sign it only for tools that need run/node authority. """ results = [] for tc in tool_calls: @@ -74,10 +67,7 @@ def execute_tool_calls( args = _json.loads(func.get("arguments", "{}")) except _json.JSONDecodeError: args = {} - if node_id and name in mcp._COW_TOOLS: - result = mcp.call_tool(name, args, cow_node_id=node_id) - else: - result = mcp.call_tool(name, args) + result = mcp.call_tool(name, args, context=context) results.append({"tool_call_id": tc.get("id", ""), "name": name, "result": result}) return results diff --git a/ari-core/ari/analysis.py b/ari-core/ari/analysis.py new file mode 100644 index 00000000..c29c0b84 --- /dev/null +++ b/ari-core/ari/analysis.py @@ -0,0 +1,509 @@ +"""Versioned deterministic statistical-analysis contracts. + +The models in this module describe scientific inputs and outputs independently +of a particular statistics MCP implementation. File-backed samples are always +bound to a closed workspace and an expected digest; callers therefore cannot +silently analyse a different file on replay. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import PurePosixPath +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ari.execution import WorkspaceRefV1 + + +ANALYSIS_REQUEST_V1 = "ari.analysis-request/v1" +STATISTICAL_TEST_REQUEST_V1 = "ari.statistical-test-request/v1" +RUN_COMPARISON_REQUEST_V1 = "ari.run-comparison-request/v1" +ANALYSIS_RESULT_V1 = "ari.analysis-result/v1" + +_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$") + + +def canonical_analysis_digest(value: Any) -> str: + """Return the canonical SHA-256 identity of a JSON-compatible value.""" + + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _validate_digest(value: str | None, field: str) -> str | None: + if value is not None and not _DIGEST_RE.fullmatch(value): + raise ValueError(f"{field} must use sha256:<64 lowercase hex> format") + return value + + +def _validate_id(value: str | None, field: str) -> str | None: + if value is not None and not _ID_RE.fullmatch(value): + raise ValueError(f"{field} contains unsupported characters") + return value + + +class AnalysisDataSourceV1(BaseModel): + """Digest-bound numeric column in a closed workspace.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + workspace: WorkspaceRefV1 + relative_path: str + format: Literal["auto", "csv", "json", "npy"] = "auto" + value_column: str | None = None + replicate_id_column: str | None = None + pair_id_column: str | None = None + backend_id_column: str | None = None + environment_digest_column: str | None = None + expected_digest: str + max_bytes: int = Field(default=64 * 1024 * 1024, ge=1, le=256 * 1024 * 1024) + + @field_validator("relative_path") + @classmethod + def _safe_path(cls, value: str) -> str: + pure = PurePosixPath(value) + if ( + not value + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + ): + raise ValueError("analysis source path must be safe and relative") + return value + + @field_validator("expected_digest") + @classmethod + def _digest(cls, value: str) -> str: + assert _validate_digest(value, "expected_digest") is not None + return value + + @field_validator( + "value_column", + "replicate_id_column", + "pair_id_column", + "backend_id_column", + "environment_digest_column", + ) + @classmethod + def _column(cls, value: str | None) -> str | None: + if value is not None and (not value.strip() or len(value) > 256): + raise ValueError("analysis source column name is invalid") + return value + + +class AnalysisObservationV1(BaseModel): + """One observed value and the identities needed to assess independence.""" + + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=True) + + value: float | None + replicate_id: str | None = None + pair_id: str | None = None + backend_id: str | None = None + environment_digest: str | None = None + + @field_validator("replicate_id", "pair_id", "backend_id") + @classmethod + def _identity(cls, value: str | None, info: Any) -> str | None: + return _validate_id(value, info.field_name) + + @field_validator("environment_digest") + @classmethod + def _environment_digest(cls, value: str | None) -> str | None: + return _validate_digest(value, "environment_digest") + + +class MetricSampleSetV1(BaseModel): + """A unit-bearing metric sample supplied inline or by immutable source.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + metric_id: str + unit: str + observations: list[AnalysisObservationV1] | None = Field( + default=None, max_length=1_000_000 + ) + source: AnalysisDataSourceV1 | None = None + + @field_validator("metric_id") + @classmethod + def _metric_id(cls, value: str) -> str: + checked = _validate_id(value, "metric_id") + assert checked is not None + return checked + + @field_validator("unit") + @classmethod + def _unit(cls, value: str) -> str: + value = value.strip() + if not value or len(value) > 128: + raise ValueError("measurement unit must be explicit and non-empty") + return value + + @model_validator(mode="after") + def _one_input(self) -> "MetricSampleSetV1": + if (self.observations is None) == (self.source is None): + raise ValueError("sample set requires exactly one of observations or source") + return self + + +class AnalysisArtifactTargetV1(BaseModel): + """Optional closed-workspace destination for deterministic result artifacts.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + workspace: WorkspaceRefV1 + relative_directory: str = "analysis" + + @field_validator("relative_directory") + @classmethod + def _directory(cls, value: str) -> str: + pure = PurePosixPath(value) + if ( + not value + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + ): + raise ValueError("analysis artifact directory must be safe and relative") + return value.rstrip("/") + + +class AnalysisRequestV1(BaseModel): + """Deterministic summary request for one or more metrics.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.analysis-request/v1"] = ANALYSIS_REQUEST_V1 + datasets: list[MetricSampleSetV1] = Field(min_length=1, max_length=1_024) + missing_policy: Literal["error", "drop"] = "error" + confidence_level: float = Field(default=0.95, gt=0.0, lt=1.0) + analysis_plan_digest: str | None = None + artifact_target: AnalysisArtifactTargetV1 | None = None + + @field_validator("analysis_plan_digest") + @classmethod + def _plan_digest(cls, value: str | None) -> str | None: + return _validate_digest(value, "analysis_plan_digest") + + @model_validator(mode="after") + def _unique_metrics(self) -> "AnalysisRequestV1": + metric_ids = [dataset.metric_id for dataset in self.datasets] + if len(metric_ids) != len(set(metric_ids)): + raise ValueError("analysis request metric_id values must be unique") + return self + + +class StatisticalComparisonV1(BaseModel): + """One pre-declared two-sample or paired comparison.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + comparison_id: str + group_a: MetricSampleSetV1 + group_b: MetricSampleSetV1 + test_family: Literal[ + "auto", + "welch_t", + "student_t", + "paired_t", + "mann_whitney", + "wilcoxon", + ] = "auto" + pairing: Literal["unpaired", "ordered", "pair_id"] = "unpaired" + alternative: Literal["two-sided", "less", "greater"] = "two-sided" + alpha: float = Field(default=0.05, gt=0.0, lt=1.0) + confidence_level: float = Field(default=0.95, gt=0.0, lt=1.0) + + @field_validator("comparison_id") + @classmethod + def _comparison_id(cls, value: str) -> str: + checked = _validate_id(value, "comparison_id") + assert checked is not None + return checked + + @model_validator(mode="after") + def _compatible_samples(self) -> "StatisticalComparisonV1": + if self.group_a.metric_id != self.group_b.metric_id: + raise ValueError("comparison metric identities do not match") + if self.group_a.unit != self.group_b.unit: + raise ValueError("comparison units do not match") + paired_family = self.test_family in {"paired_t", "wilcoxon"} + if paired_family and self.pairing == "unpaired": + raise ValueError("paired test family requires ordered or pair_id pairing") + if not paired_family and self.pairing != "unpaired" and self.test_family != "auto": + raise ValueError("paired input requires paired_t, wilcoxon, or auto") + if ( + self.pairing == "ordered" + and self.group_a.observations is not None + and self.group_b.observations is not None + and len(self.group_a.observations) != len(self.group_b.observations) + ): + raise ValueError("ordered paired samples must have equal lengths") + return self + + +class StatisticalTestRequestV1(BaseModel): + """A family of comparisons with an explicit multiplicity policy.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.statistical-test-request/v1"] = ( + STATISTICAL_TEST_REQUEST_V1 + ) + comparisons: list[StatisticalComparisonV1] = Field(min_length=1, max_length=10_000) + correction: Literal["none", "bonferroni", "holm", "benjamini_hochberg"] = "none" + missing_policy: Literal["error", "drop"] = "error" + analysis_plan_digest: str | None = None + artifact_target: AnalysisArtifactTargetV1 | None = None + + @field_validator("analysis_plan_digest") + @classmethod + def _plan_digest(cls, value: str | None) -> str | None: + return _validate_digest(value, "analysis_plan_digest") + + @model_validator(mode="after") + def _family_policy(self) -> "StatisticalTestRequestV1": + ids = [item.comparison_id for item in self.comparisons] + if len(ids) != len(set(ids)): + raise ValueError("comparison_id values must be unique") + if len(ids) > 1 and self.correction == "none": + raise ValueError("multiple comparisons require an explicit correction") + return self + + +class RunRecordV1(BaseModel): + """One scalar run outcome with environment and provenance identity.""" + + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + run_id: str + metric_id: str + unit: str + value: float + backend_id: str + environment_digest: str + replicate_id: str | None = None + execution_attempt_id: str | None = None + input_digest: str | None = None + provenance_digest: str | None = None + library_versions: dict[str, str] = Field(default_factory=dict, max_length=256) + + @field_validator( + "run_id", "metric_id", "backend_id", "replicate_id", "execution_attempt_id" + ) + @classmethod + def _ids(cls, value: str | None, info: Any) -> str | None: + checked = _validate_id(value, info.field_name) + if info.field_name in {"run_id", "metric_id", "backend_id"}: + assert checked is not None + return checked + + @field_validator("unit") + @classmethod + def _unit(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("run comparison unit must be explicit") + return value + + @field_validator("environment_digest", "input_digest", "provenance_digest") + @classmethod + def _digests(cls, value: str | None, info: Any) -> str | None: + checked = _validate_digest(value, info.field_name) + if info.field_name == "environment_digest": + assert checked is not None + return checked + + @field_validator("library_versions") + @classmethod + def _versions(cls, value: dict[str, str]) -> dict[str, str]: + if any(not key or not item for key, item in value.items()): + raise ValueError("library version names and values must be non-empty") + return dict(sorted(value.items())) + + +class RunComparisonRequestV1(BaseModel): + """Rank runs while retaining compatibility and independence caveats.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.run-comparison-request/v1"] = ( + RUN_COMPARISON_REQUEST_V1 + ) + runs: list[RunRecordV1] = Field(min_length=2, max_length=100_000) + direction: Literal["higher", "lower"] + baseline_run_id: str | None = None + require_compatible_environment: bool = True + analysis_plan_digest: str | None = None + artifact_target: AnalysisArtifactTargetV1 | None = None + + @field_validator("baseline_run_id") + @classmethod + def _baseline(cls, value: str | None) -> str | None: + return _validate_id(value, "baseline_run_id") + + @field_validator("analysis_plan_digest") + @classmethod + def _plan_digest(cls, value: str | None) -> str | None: + return _validate_digest(value, "analysis_plan_digest") + + @model_validator(mode="after") + def _compatible_runs(self) -> "RunComparisonRequestV1": + run_ids = [run.run_id for run in self.runs] + if len(run_ids) != len(set(run_ids)): + raise ValueError("run_id values must be unique") + if len({run.metric_id for run in self.runs}) != 1: + raise ValueError("run comparison metric identities do not match") + if len({run.unit for run in self.runs}) != 1: + raise ValueError("run comparison units do not match") + if self.baseline_run_id is not None and self.baseline_run_id not in run_ids: + raise ValueError("baseline_run_id does not identify a supplied run") + return self + + +class AnalysisArtifactV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + relative_path: str + digest: str + size_bytes: int = Field(ge=0) + media_type: str + logical_role: Literal["analysis-json", "analysis-table"] + + @field_validator("digest") + @classmethod + def _digest(cls, value: str) -> str: + assert _validate_digest(value, "artifact digest") is not None + return value + + +class AnalysisSummaryV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + metric_id: str + unit: str + count: int = Field(ge=1) + missing_count: int = Field(ge=0) + mean: float + std: float | None + variance: float | None + minimum: float + q25: float + median: float + q75: float + maximum: float + mean_confidence_interval: tuple[float, float] | None + constant_data: bool + independence_status: Literal["verified", "declared", "not-established"] + source_digest: str + + +class StatisticalComparisonResultV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + comparison_id: str + metric_id: str + unit: str + test_family: str + alternative: str + pairing: str + sample_count_a: int = Field(ge=1) + sample_count_b: int = Field(ge=1) + missing_count_a: int = Field(ge=0) + missing_count_b: int = Field(ge=0) + statistic: float + p_value: float + adjusted_p_value: float + alpha: float + significant: bool + effect_size_name: str + effect_size: float + confidence_interval_name: str + confidence_interval: tuple[float, float] + assumptions: dict[str, Any] + input_digest: str + + +class RunComparisonResultV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + metric_id: str + unit: str + direction: Literal["higher", "lower"] + ranking: list[dict[str, Any]] + baseline_run_id: str + environment_compatible: bool + environment_groups: list[dict[str, Any]] + independence_status: Literal["declared", "not-established"] + independent_replicate_count: int | None + provenance_differences: list[dict[str, Any]] + + +class AnalysisResultV1(BaseModel): + """Machine-readable deterministic output shared by analysis providers.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.analysis-result/v1"] = ANALYSIS_RESULT_V1 + kind: Literal["summary", "statistical-test", "run-comparison"] + input_digest: str + analysis_plan_digest: str | None = None + library_versions: dict[str, str] + summaries: list[AnalysisSummaryV1] = Field(default_factory=list) + comparisons: list[StatisticalComparisonResultV1] = Field(default_factory=list) + run_comparison: RunComparisonResultV1 | None = None + artifacts: list[AnalysisArtifactV1] = Field(default_factory=list) + + @field_validator("input_digest", "analysis_plan_digest") + @classmethod + def _digests(cls, value: str | None, info: Any) -> str | None: + checked = _validate_digest(value, info.field_name) + if info.field_name == "input_digest": + assert checked is not None + return checked + + @model_validator(mode="after") + def _kind_content(self) -> "AnalysisResultV1": + populated = { + "summary": bool(self.summaries), + "statistical-test": bool(self.comparisons), + "run-comparison": self.run_comparison is not None, + } + if not populated[self.kind] or sum(populated.values()) != 1: + raise ValueError("analysis result content does not match its kind") + return self + + +__all__ = [ + "ANALYSIS_REQUEST_V1", + "ANALYSIS_RESULT_V1", + "RUN_COMPARISON_REQUEST_V1", + "STATISTICAL_TEST_REQUEST_V1", + "AnalysisArtifactTargetV1", + "AnalysisArtifactV1", + "AnalysisDataSourceV1", + "AnalysisObservationV1", + "AnalysisRequestV1", + "AnalysisResultV1", + "AnalysisSummaryV1", + "MetricSampleSetV1", + "RunComparisonRequestV1", + "RunComparisonResultV1", + "RunRecordV1", + "StatisticalComparisonResultV1", + "StatisticalComparisonV1", + "StatisticalTestRequestV1", + "canonical_analysis_digest", +] diff --git a/ari-core/ari/async_tools.py b/ari-core/ari/async_tools.py new file mode 100644 index 00000000..925170ad --- /dev/null +++ b/ari-core/ari/async_tools.py @@ -0,0 +1,233 @@ +"""Provider-neutral contracts for asynchronous Skill execution. + +The manifest-side lifecycle names semantic capabilities. At admission time the +MCP control plane resolves those capabilities to immutable tool references and +places them in :class:`AsyncToolHandleV1`. A serialized handle therefore never +depends on a provider's bare tool names or on a mutable catalog lookup. +""" + +from __future__ import annotations + +import re +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +ASYNC_TOOL_HANDLE_V1 = "ari.async-tool-handle/v1" + +_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_CAPABILITY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") + +AsyncStateV1 = Literal[ + "submitted", + "running", + "succeeded", + "failed", + "cancelled", + "unknown", +] + + +class TimeoutBudgetV1(BaseModel): + """A caller-controlled timeout argument explicitly admitted by a manifest.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + argument: str + unit: Literal["seconds", "minutes", "hours"] = "seconds" + overhead_seconds: int = Field(default=0, ge=0, le=2_678_400) + maximum_seconds: int = Field(gt=0, le=2_678_400) + + @field_validator("argument") + @classmethod + def _valid_argument(cls, value: str) -> str: + if not _FIELD_RE.fullmatch(value): + raise ValueError("timeout budget argument must be an identifier") + return value + + @model_validator(mode="after") + def _valid_limit(self) -> "TimeoutBudgetV1": + if self.maximum_seconds <= self.overhead_seconds: + raise ValueError("maximum_seconds must exceed overhead_seconds") + return self + + def requested_seconds(self, arguments: dict[str, Any]) -> int | None: + """Return the bounded outer timeout requested by one call, if present.""" + + raw = arguments.get(self.argument) + if isinstance(raw, bool) or not isinstance(raw, (int, float)) or raw <= 0: + return None + multiplier = {"seconds": 1, "minutes": 60, "hours": 3_600}[self.unit] + requested = int(raw * multiplier) + self.overhead_seconds + return min(requested, self.maximum_seconds) + + +class AsyncOperationV1(BaseModel): + """Manifest reference to one operation in an async lifecycle.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + capability_ref: str + handle_argument: str = "handle_id" + + @field_validator("capability_ref") + @classmethod + def _valid_capability_ref(cls, value: str) -> str: + value = value.strip() + if not _CAPABILITY_RE.fullmatch(value): + raise ValueError("capability_ref must be a lowercase dotted identifier") + return value + + @field_validator("handle_argument") + @classmethod + def _valid_handle_argument(cls, value: str) -> str: + if not _FIELD_RE.fullmatch(value): + raise ValueError("handle_argument must be an identifier") + return value + + +class AsyncStateMapV1(BaseModel): + """Case-insensitive provider state mapping used while polling a handle.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + submitted_states: list[str] = Field( + default_factory=lambda: ["submitted", "queued", "pending", "started"] + ) + running_states: list[str] = Field(default_factory=lambda: ["running", "active"]) + succeeded_states: list[str] = Field( + default_factory=lambda: ["succeeded", "success", "completed", "complete", "done", "ok"] + ) + failed_states: list[str] = Field( + default_factory=lambda: ["failed", "error", "timed_out", "timeout"] + ) + cancelled_states: list[str] = Field( + default_factory=lambda: ["cancelled", "canceled"] + ) + + @field_validator( + "submitted_states", + "running_states", + "succeeded_states", + "failed_states", + "cancelled_states", + ) + @classmethod + def _valid_states(cls, values: list[str]) -> list[str]: + normalized = [str(value).strip() for value in values] + if any(not value for value in normalized): + raise ValueError("async state names cannot be empty") + folded = [value.casefold() for value in normalized] + if len(folded) != len(set(folded)): + raise ValueError("async state names must be unique") + return normalized + + @model_validator(mode="after") + def _disjoint_states(self) -> "AsyncStateMapV1": + owners: dict[str, str] = {} + for field_name in ( + "submitted_states", + "running_states", + "succeeded_states", + "failed_states", + "cancelled_states", + ): + for state in getattr(self, field_name): + key = state.casefold() + if key in owners: + raise ValueError( + f"async state {state!r} occurs in {owners[key]} and {field_name}" + ) + owners[key] = field_name + return self + + def classify(self, value: Any) -> AsyncStateV1: + """Map one provider value to ARI's state machine without guessing.""" + + key = str(value).strip().casefold() + for result, field_name in ( + ("submitted", "submitted_states"), + ("running", "running_states"), + ("succeeded", "succeeded_states"), + ("failed", "failed_states"), + ("cancelled", "cancelled_states"), + ): + if key in {state.casefold() for state in getattr(self, field_name)}: + return result # type: ignore[return-value] + return "unknown" + + +class AsyncLifecycleV1(BaseModel): + """Manifest-declared submit/status/result/cancel protocol.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + handle_field: str = "handle_id" + state_field: str = "status" + status: AsyncOperationV1 + result: AsyncOperationV1 | None = None + cancel: AsyncOperationV1 | None = None + states: AsyncStateMapV1 = Field(default_factory=AsyncStateMapV1) + poll_interval_seconds: float = Field(default=5.0, ge=0.01, le=3_600) + max_wait_seconds: int = Field(default=86_400, ge=1, le=2_678_400) + + @field_validator("handle_field", "state_field") + @classmethod + def _valid_field(cls, value: str) -> str: + if not _FIELD_RE.fullmatch(value): + raise ValueError("async handle/state fields must be identifiers") + return value + + +class AsyncToolEndpointV1(BaseModel): + """One immutable endpoint embedded in a runtime async handle.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + tool_ref: str = Field(min_length=1) + handle_argument: str + + @field_validator("handle_argument") + @classmethod + def _valid_handle_argument(cls, value: str) -> str: + if not _FIELD_RE.fullmatch(value): + raise ValueError("handle_argument must be an identifier") + return value + + +class AsyncToolHandleV1(BaseModel): + """Portable handle bound to immutable lifecycle tool references.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.async-tool-handle/v1"] = ASYNC_TOOL_HANDLE_V1 + handle_id: str = Field(min_length=1, max_length=1_024) + submission_tool_ref: str = Field(min_length=1) + status: AsyncToolEndpointV1 + result: AsyncToolEndpointV1 | None = None + cancel: AsyncToolEndpointV1 | None = None + state_field: str + states: AsyncStateMapV1 + poll_interval_seconds: float = Field(ge=0.01, le=3_600) + max_wait_seconds: int = Field(ge=1, le=2_678_400) + submitted_at: str = Field(min_length=1) + + @field_validator("state_field") + @classmethod + def _valid_state_field(cls, value: str) -> str: + if not _FIELD_RE.fullmatch(value): + raise ValueError("state_field must be an identifier") + return value + + +__all__ = [ + "ASYNC_TOOL_HANDLE_V1", + "AsyncLifecycleV1", + "AsyncOperationV1", + "AsyncStateMapV1", + "AsyncStateV1", + "AsyncToolEndpointV1", + "AsyncToolHandleV1", + "TimeoutBudgetV1", +] diff --git a/ari-core/ari/calibration/README.md b/ari-core/ari/calibration/README.md new file mode 100644 index 00000000..0d09f865 --- /dev/null +++ b/ari-core/ari/calibration/README.md @@ -0,0 +1,12 @@ +# Evaluator calibration corpus + +`evaluator_v1.json` is the permanent, versioned calibration input for ARI's +scientific evaluator boundary. The hard-gate cases execute in CI and cover +numeric agreement, formula resolution, unit conversion, evidence identity, +artifact integrity, and policy behavior. The semantic cases are human labels +for advisory reviewer calibration; they are not converted into deterministic +scientific truth and are not used to change a hard-gate result. + +Changes require a schema-version decision, a human review of every expected +label, and the evaluator calibration test. Removing a case requires recording +which replacement preserves its positive or negative boundary. diff --git a/ari-core/ari/calibration/__init__.py b/ari-core/ari/calibration/__init__.py new file mode 100644 index 00000000..fa231112 --- /dev/null +++ b/ari-core/ari/calibration/__init__.py @@ -0,0 +1,17 @@ +"""Versioned, human-curated evaluator calibration assets.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + + +def load_evaluator_calibration_v1() -> dict[str, Any]: + """Load the bundled v1 corpus without consulting network or model state.""" + + resource = files(__package__).joinpath("evaluator_v1.json") + return json.loads(resource.read_text(encoding="utf-8")) + + +__all__ = ["load_evaluator_calibration_v1"] diff --git a/ari-core/ari/calibration/evaluator_v1.json b/ari-core/ari/calibration/evaluator_v1.json new file mode 100644 index 00000000..fec91e21 --- /dev/null +++ b/ari-core/ari/calibration/evaluator_v1.json @@ -0,0 +1,168 @@ +{ + "schema_version": "ari.evaluator-calibration/v1", + "provenance": { + "curation": "human-authored boundary cases derived from the evaluator contract", + "last_reviewed": "2026-08-02", + "semantic_labels_are_advisory": true + }, + "hard_gate_cases": [ + { + "id": "numeric-exact-negative-control", + "category": "numeric", + "mutation": "none", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "passed", + "expected_should_block": false, + "expected_finding_types": [], + "negative_control": true + }, + { + "id": "allowed-unit-conversion-negative-control", + "category": "unit", + "mutation": "none", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "passed", + "expected_should_block": false, + "expected_finding_types": [], + "negative_control": true + }, + { + "id": "numeric-mismatch", + "category": "numeric", + "mutation": "reported-value-mismatch", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["numeric_mismatch"], + "negative_control": false + }, + { + "id": "unknown-formula", + "category": "formula", + "mutation": "unknown-formula", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["operand_unresolved"], + "negative_control": false + }, + { + "id": "unknown-unit", + "category": "unit", + "mutation": "unknown-unit", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["unit_mismatch"], + "negative_control": false + }, + { + "id": "cross-run-operand", + "category": "evidence", + "mutation": "cross-run-operand", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["cross_run_evidence"], + "negative_control": false + }, + { + "id": "unknown-node", + "category": "evidence", + "mutation": "unknown-node", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["cross_run_or_unknown_node"], + "negative_control": false + }, + { + "id": "missing-evidence", + "category": "evidence", + "mutation": "missing-evidence", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["missing_evidence"], + "negative_control": false + }, + { + "id": "untyped-artifact-reference", + "category": "evidence", + "mutation": "untyped-artifact", + "artifact_fixture": false, + "policy_mode": "strict", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["artifact_reference_untyped"], + "negative_control": false + }, + { + "id": "tampered-artifact", + "category": "evidence", + "mutation": "tamper-artifact", + "artifact_fixture": true, + "policy_mode": "warn", + "expected_status": "failed", + "expected_should_block": true, + "expected_finding_types": ["artifact_digest_mismatch"], + "negative_control": false + }, + { + "id": "off-policy-does-not-block", + "category": "policy", + "mutation": "tamper-artifact", + "artifact_fixture": true, + "policy_mode": "off", + "expected_status": "failed", + "expected_should_block": false, + "expected_finding_types": ["artifact_digest_mismatch"], + "negative_control": true + } + ], + "semantic_cases": [ + { + "id": "bounded-observation-negative-control", + "evidence_summary": "On the recorded fixture, latency was 100 ms.", + "paper_text": "On the recorded fixture, latency was 100 ms.", + "expected_finding_types": [], + "negative_control": true + }, + { + "id": "explicit-limitation-negative-control", + "evidence_summary": "Only one simulator configuration was measured.", + "paper_text": "The result is limited to the measured simulator configuration.", + "expected_finding_types": [], + "negative_control": true + }, + { + "id": "universal-generalization", + "evidence_summary": "One simulator configuration improved in one run.", + "paper_text": "The method improves every simulator and physical device.", + "expected_finding_types": ["overgeneralization"], + "negative_control": false + }, + { + "id": "unsupported-causal-claim", + "evidence_summary": "The experiment recorded an association only.", + "paper_text": "The intervention proves that the mechanism causes the effect.", + "expected_finding_types": ["unsupported_claim"], + "negative_control": false + }, + { + "id": "magnitude-overclaim", + "evidence_summary": "The measured improvement was 2 percent.", + "paper_text": "The method delivers a dramatic order-of-magnitude improvement.", + "expected_finding_types": ["overclaim"], + "negative_control": false + } + ] +} diff --git a/ari-core/ari/call_context.py b/ari-core/ari/call_context.py new file mode 100644 index 00000000..a0f4b460 --- /dev/null +++ b/ari-core/ari/call_context.py @@ -0,0 +1,357 @@ +"""Explicit, verifiable run and node context for Skill calls. + +The context models are provider-neutral public data. A per-connection HMAC +turns that data into a narrowly-scoped capability at the MCP transport +boundary; the authority key never appears in a tool argument, result, lock, or +Claude shim configuration. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import secrets +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +RUN_CONTEXT_V1 = "ari.run-context/v1" +NODE_CONTEXT_V1 = "ari.node-context/v1" +AUTHORIZED_TOOL_CONTEXT_V1 = "ari.authorized-tool-context/v1" +CALL_CONTEXT_ARGUMENT = "ari_context" +CONTEXT_AUTHORITY_ENV = "ARI_CONTEXT_AUTHORITY_KEY" +SHA256_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" + + +class CallContextAuthorizationError(ValueError): + """Raised when a call context is missing, malformed, or not authorized.""" + + +def _canonical_digest(payload: dict[str, Any]) -> str: + encoded = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def run_scope_digest(run_id: str) -> str: + """Return the canonical digest for a logical run identity.""" + + return _canonical_digest({"run_id": run_id}) + + +def lineage_digest( + *, + run_id: str, + node_id: str, + parent_node_id: str | None, + ancestor_node_ids: list[str], +) -> str: + """Bind an ordered root-to-parent lineage to one run and node.""" + + return _canonical_digest( + { + "run_id": run_id, + "node_id": node_id, + "parent_node_id": parent_node_id, + "ancestor_node_ids": ancestor_node_ids, + } + ) + + +class RunContextV1(BaseModel): + """Immutable identity for one ARI run.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.run-context/v1"] = RUN_CONTEXT_V1 + run_id: str = Field(min_length=1) + run_scope_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + + @classmethod + def create(cls, run_id: str) -> "RunContextV1": + normalized = str(run_id).strip() + if not normalized: + raise ValueError("run_id cannot be empty") + return cls(run_id=normalized, run_scope_digest=run_scope_digest(normalized)) + + @model_validator(mode="after") + def _digest_matches(self) -> "RunContextV1": + if self.run_scope_digest != run_scope_digest(self.run_id): + raise ValueError("run_scope_digest does not match run_id") + return self + + +class NodeContextV1(BaseModel): + """Immutable node identity and its ordered, run-scoped ancestry.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.node-context/v1"] = NODE_CONTEXT_V1 + run_id: str = Field(min_length=1) + node_id: str = Field(min_length=1) + parent_node_id: str | None = None + ancestor_node_ids: list[str] = Field(default_factory=list) + lineage_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + + @classmethod + def create( + cls, + *, + run_id: str, + node_id: str, + parent_node_id: str | None = None, + ancestor_node_ids: list[str] | tuple[str, ...] = (), + ) -> "NodeContextV1": + ancestors = [str(item) for item in ancestor_node_ids] + return cls( + run_id=str(run_id), + node_id=str(node_id), + parent_node_id=str(parent_node_id) if parent_node_id is not None else None, + ancestor_node_ids=ancestors, + lineage_digest=lineage_digest( + run_id=str(run_id), + node_id=str(node_id), + parent_node_id=( + str(parent_node_id) if parent_node_id is not None else None + ), + ancestor_node_ids=ancestors, + ), + ) + + @field_validator("ancestor_node_ids") + @classmethod + def _valid_ancestors(cls, values: list[str]) -> list[str]: + if any(not value for value in values): + raise ValueError("ancestor node IDs cannot be empty") + if len(values) != len(set(values)): + raise ValueError("ancestor node IDs must be unique") + return values + + @model_validator(mode="after") + def _consistent_lineage(self) -> "NodeContextV1": + if self.node_id in self.ancestor_node_ids: + raise ValueError("node cannot be its own ancestor") + expected_parent = self.ancestor_node_ids[-1] if self.ancestor_node_ids else None + if self.parent_node_id != expected_parent: + raise ValueError( + "parent_node_id must equal the final ancestor, or be null at root" + ) + expected = lineage_digest( + run_id=self.run_id, + node_id=self.node_id, + parent_node_id=self.parent_node_id, + ancestor_node_ids=self.ancestor_node_ids, + ) + if self.lineage_digest != expected: + raise ValueError("lineage_digest does not match the ordered lineage") + return self + + @property + def readable_node_ids(self) -> frozenset[str]: + """Nodes this context may read: its ancestors and itself.""" + + return frozenset([*self.ancestor_node_ids, self.node_id]) + + +class ToolCallContextV1(BaseModel): + """Explicit context supplied at a tool-call boundary. + + ``run_id`` and ``node_id`` remain as flattened provenance fields for the + v1 result envelope. Authorization requires the corresponding structured + context, whose digests prevent accidental lineage corruption. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + run_id: str = "" + node_id: str | None = None + phase: str | None = None + selection_reason: str = "" + credential_scope_ids: list[str] = Field(default_factory=list) + run_context: RunContextV1 | None = None + node_context: NodeContextV1 | None = None + + @classmethod + def for_run( + cls, + run_id: str, + *, + phase: str | None = None, + ) -> "ToolCallContextV1": + run = RunContextV1.create(run_id) + return cls(run_id=run.run_id, phase=phase, run_context=run) + + @classmethod + def for_node( + cls, + *, + run_id: str, + node_id: str, + parent_node_id: str | None = None, + ancestor_node_ids: list[str] | tuple[str, ...] = (), + phase: str | None = None, + ) -> "ToolCallContextV1": + run = RunContextV1.create(run_id) + node = NodeContextV1.create( + run_id=run.run_id, + node_id=node_id, + parent_node_id=parent_node_id, + ancestor_node_ids=ancestor_node_ids, + ) + return cls( + run_id=run.run_id, + node_id=node.node_id, + phase=phase, + run_context=run, + node_context=node, + ) + + @model_validator(mode="after") + def _structured_context_matches_flattened(self) -> "ToolCallContextV1": + if self.run_context is not None and self.run_id != self.run_context.run_id: + raise ValueError("run_id does not match run_context") + if self.node_context is not None: + if self.node_id != self.node_context.node_id: + raise ValueError("node_id does not match node_context") + if self.run_id != self.node_context.run_id: + raise ValueError("run_id does not match node_context") + if self.run_context is None: + raise ValueError("node_context requires run_context") + return self + + def satisfies(self, requirement: str) -> bool: + if requirement == "none": + return True + if requirement == "run": + return self.run_context is not None + if requirement == "node": + return self.run_context is not None and self.node_context is not None + return False + + +class AuthorizedToolContextV1(BaseModel): + """Signed context capability injected by an ARI-controlled transport.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.authorized-tool-context/v1"] = ( + AUTHORIZED_TOOL_CONTEXT_V1 + ) + tool_name: str = Field(min_length=1) + authority_id: str = Field(pattern=SHA256_DIGEST_PATTERN) + nonce: str = Field(min_length=16) + context: ToolCallContextV1 + signature: str = Field(pattern=r"^hmac-sha256:[0-9a-f]{64}$") + + +def new_context_authority_key() -> str: + """Create a 256-bit per-connection authority key.""" + + return secrets.token_hex(32) + + +def _key_bytes(key: str) -> bytes: + try: + payload = bytes.fromhex(key) + except ValueError as exc: + raise CallContextAuthorizationError("context authority key is malformed") from exc + if len(payload) < 32: + raise CallContextAuthorizationError("context authority key is too short") + return payload + + +def _authority_id(key: str) -> str: + return f"sha256:{hashlib.sha256(_key_bytes(key)).hexdigest()}" + + +def _signature_payload(document: dict[str, Any]) -> bytes: + unsigned = {key: value for key, value in document.items() if key != "signature"} + return json.dumps( + unsigned, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def authorize_tool_context( + context: ToolCallContextV1, + *, + tool_name: str, + authority_key: str, +) -> dict[str, Any]: + """Sign one context for exactly one MCP tool name.""" + + document: dict[str, Any] = { + "schema_version": AUTHORIZED_TOOL_CONTEXT_V1, + "tool_name": tool_name, + "authority_id": _authority_id(authority_key), + "nonce": secrets.token_hex(16), + "context": context.model_dump(mode="json"), + } + signature = hmac.new( + _key_bytes(authority_key), + _signature_payload(document), + hashlib.sha256, + ).hexdigest() + document["signature"] = f"hmac-sha256:{signature}" + AuthorizedToolContextV1.model_validate(document) + return document + + +def verify_tool_context( + document: Any, + *, + tool_name: str, + authority_key: str, + requirement: Literal["run", "node"] = "node", +) -> ToolCallContextV1: + """Verify a transport capability and return its structured context.""" + + try: + authorized = AuthorizedToolContextV1.model_validate(document) + except Exception as exc: + raise CallContextAuthorizationError("authorized call context is malformed") from exc + if authorized.tool_name != tool_name: + raise CallContextAuthorizationError("call context is bound to another tool") + if authorized.authority_id != _authority_id(authority_key): + raise CallContextAuthorizationError("call context authority does not match") + supplied = authorized.model_dump(mode="json") + expected = hmac.new( + _key_bytes(authority_key), + _signature_payload(supplied), + hashlib.sha256, + ).hexdigest() + actual = authorized.signature.removeprefix("hmac-sha256:") + if not hmac.compare_digest(actual, expected): + raise CallContextAuthorizationError("call context signature is invalid") + if not authorized.context.satisfies(requirement): + raise CallContextAuthorizationError( + f"tool requires explicit {requirement} context" + ) + return authorized.context + + +__all__ = [ + "AUTHORIZED_TOOL_CONTEXT_V1", + "CALL_CONTEXT_ARGUMENT", + "CONTEXT_AUTHORITY_ENV", + "NODE_CONTEXT_V1", + "RUN_CONTEXT_V1", + "AuthorizedToolContextV1", + "CallContextAuthorizationError", + "NodeContextV1", + "RunContextV1", + "ToolCallContextV1", + "authorize_tool_context", + "lineage_digest", + "new_context_authority_key", + "run_scope_digest", + "verify_tool_context", +] diff --git a/ari-core/ari/claim_gate_contract.py b/ari-core/ari/claim_gate_contract.py new file mode 100644 index 00000000..45a58076 --- /dev/null +++ b/ari-core/ari/claim_gate_contract.py @@ -0,0 +1,626 @@ +"""Canonical contracts for metric admission and claim/evidence evaluation. + +The evaluator Skill is a transport boundary. Scientific identity, report +shape, and digest verification live here so idea, transform, evaluator, paper, +and offline readers all consume the same models. +""" + +from __future__ import annotations + +import json +import math +import re +from typing import Any, ClassVar, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator + +from ari.research_contract import ( + SHA256_DIGEST_PATTERN, + MetricContractV1, + MetricCorrectnessV1, + MetricFormulaProvenanceV1, + MetricToleranceV1, + canonical_digest, +) + + +METRIC_GATE_CONTRACT_V1 = "ari.metric-gate-contract/v1" +METRIC_CONTRACT_PROPOSAL_V1 = "ari.metric-contract-proposal/v1" +METRIC_ADMISSION_DECISION_V1 = "ari.metric-admission-decision/v1" +GATE_FINDING_V1 = "ari.gate-finding/v1" +GATE_REPORT_V1 = "ari.gate-report/v1" +SEMANTIC_REVIEW_V1 = "ari.semantic-review/v1" + +_ZERO_DIGEST = "sha256:" + ("0" * 64) +_SAFE_CODE = re.compile(r"^[a-z][a-z0-9_.-]{0,127}$") + + +class ClaimGateContractError(ValueError): + """A gate, proposal, or review document is malformed or was changed.""" + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class _DigestBoundModel(_StrictModel): + _digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values[cls._digest_field] = _ZERO_DIGEST + return cls.model_validate(values, context={"bind_claim_gate_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self._digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_digest(self.digest_payload()) + if info.context and info.context.get("bind_claim_gate_digest"): + object.__setattr__(self, self._digest_field, expected) + elif getattr(self, self._digest_field) != expected: + raise ValueError( + f"{self._digest_field} does not match the canonical payload" + ) + return self + + +class MetricClaimV1(_StrictModel): + claim: str = Field(min_length=1, max_length=10_000) + required_evidence: tuple[str, ...] = Field(min_length=1, max_length=128) + + @field_validator("required_evidence") + @classmethod + def _unique_evidence(cls, value: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in value if item.strip()) + if not normalized or len(normalized) != len(set(normalized)): + raise ValueError("claim evidence must be non-empty and unique") + return normalized + + +class MetricGateContractV1(_DigestBoundModel): + """Evaluator projection of one immutable idea-owned metric contract.""" + + _digest_field = "projection_digest" + + schema_version: Literal["ari.metric-gate-contract/v1"] = ( + METRIC_GATE_CONTRACT_V1 + ) + projection_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + source: Literal["research-contract", "human-admitted", "legacy-migrated"] + source_idea_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + research_contract_digest: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + metric_contract: MetricContractV1 + claims: tuple[MetricClaimV1, ...] = Field(default_factory=tuple, max_length=128) + + @model_validator(mode="after") + def _source_binding(self): + if self.source == "research-contract" and self.research_contract_digest is None: + raise ValueError("research-contract projection requires its digest") + if self.source != "research-contract" and self.research_contract_digest is not None: + raise ValueError("only research-contract projections carry its digest") + return self + + def gate_projection(self) -> dict[str, Any]: + """Return the single compatibility view consumed by gate mathematics.""" + + metric = self.metric_contract + return { + "schema_version": self.schema_version, + "projection_digest": self.projection_digest, + "research_contract_digest": self.research_contract_digest, + "metric_contract_digest": metric.contract_digest, + "key": metric.name, + "unit": metric.unit, + "direction": metric.direction, + "comparison_scope": metric.comparison_scope, + "formula": metric.formula, + "formula_operands": dict(metric.operands), + "formula_provenance": metric.formula_provenance.model_dump(mode="json"), + "tolerance": metric.tolerance.model_dump(mode="json"), + "claims": [item.model_dump(mode="json") for item in self.claims], + "correctness_required": metric.correctness_required, + "ceiling_must_be_measured": metric.normalization_ceiling == "measured", + "required_measured": list(metric.required_measured), + "invariants": list(metric.invariants), + "correctness": ( + metric.correctness.model_dump(mode="json") + if metric.correctness is not None + else {} + ), + } + + +class MetricContractProposalV1(_DigestBoundModel): + """Untrusted LLM suggestion; never an admitted scientific contract.""" + + _digest_field = "proposal_digest" + + schema_version: Literal["ari.metric-contract-proposal/v1"] = ( + METRIC_CONTRACT_PROPOSAL_V1 + ) + proposal_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + source_idea_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + evidence_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + model: str = Field(min_length=1, max_length=512) + model_revision: str | None = Field(default=None, max_length=512) + prompt_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + proposed_contract: dict[str, Any] + confidence: float = Field(ge=0, le=1) + requires_human_review: Literal[True] = True + + @field_validator("proposed_contract") + @classmethod + def _finite_proposal(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, sort_keys=True, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("proposal must be finite JSON") from exc + return value + + +class MetricAdmissionDecisionV1(_DigestBoundModel): + _digest_field = "decision_digest" + + schema_version: Literal["ari.metric-admission-decision/v1"] = ( + METRIC_ADMISSION_DECISION_V1 + ) + decision_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + proposal_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + decision: Literal["admitted", "review-required", "rejected"] + reviewer: str | None = Field(default=None, max_length=512) + reasons: tuple[str, ...] = Field(min_length=1, max_length=64) + admitted_contract_digest: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + + @model_validator(mode="after") + def _admission_is_explicit(self): + if self.decision == "admitted": + if not self.reviewer or not self.admitted_contract_digest: + raise ValueError("admission requires reviewer and contract digest") + elif self.admitted_contract_digest is not None: + raise ValueError("non-admission cannot carry an admitted contract") + return self + + +class GateFindingV1(_StrictModel): + schema_version: Literal["ari.gate-finding/v1"] = GATE_FINDING_V1 + severity: Literal["blocking", "advisory"] + type: str = Field(min_length=1, max_length=128) + message: str = Field(min_length=1, max_length=20_000) + claim_id: str | None = Field(default=None, max_length=512) + numeric_id: str | None = Field(default=None, max_length=512) + node_id: str | None = Field(default=None, max_length=512) + artifact_path: str | None = Field(default=None, max_length=4096) + details: dict[str, Any] = Field(default_factory=dict) + + @field_validator("type") + @classmethod + def _finding_code(cls, value: str) -> str: + if not _SAFE_CODE.fullmatch(value): + raise ValueError("finding type must be a stable lowercase code") + return value + + @field_validator("details") + @classmethod + def _finite_details(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, sort_keys=True, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("finding details must be finite JSON") from exc + return value + + +class GateFormulaProvenanceV1(_StrictModel): + registry_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + formulas_used: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + metric_contract_digest: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + unit_conversions: tuple[str, ...] = Field(default_factory=tuple, max_length=128) + + +class GateReportV1(_DigestBoundModel): + """Deterministic hard-gate result with typed finding separation.""" + + _digest_field = "report_digest" + + schema_version: Literal["ari.gate-report/v1"] = GATE_REPORT_V1 + report_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + gate: Literal["claim_evidence_hard_gate"] = "claim_evidence_hard_gate" + source_run_id: str = Field(min_length=1, max_length=512) + phase: Literal["draft", "final"] + policy_mode: Literal["off", "warn", "strict"] + comparison_scope: Literal["any", "same_environment"] + status: Literal["passed", "warn", "failed"] + should_block: bool + policy_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + evidence_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + formula_provenance: GateFormulaProvenanceV1 + blocking_findings: tuple[GateFindingV1, ...] = Field(default_factory=tuple) + advisory_findings: tuple[GateFindingV1, ...] = Field(default_factory=tuple) + metrics: dict[str, int | float] + + @field_validator("metrics") + @classmethod + def _finite_metrics(cls, value: dict[str, int | float]): + if any( + isinstance(item, bool) or not math.isfinite(float(item)) + for item in value.values() + ): + raise ValueError("gate metrics must be finite numbers") + return value + + @model_validator(mode="after") + def _outcome_consistency(self): + if any(item.severity != "blocking" for item in self.blocking_findings): + raise ValueError("blocking_findings contains a non-blocking finding") + if any(item.severity != "advisory" for item in self.advisory_findings): + raise ValueError("advisory_findings contains a non-advisory finding") + if self.should_block and ( + self.phase != "final" + or self.policy_mode == "off" + or not self.blocking_findings + ): + raise ValueError("gate cannot block in this phase/policy/outcome") + if self.status == "passed" and ( + self.blocking_findings or self.advisory_findings + ): + raise ValueError("passed gate cannot contain findings") + if self.status == "failed" and not self.blocking_findings: + raise ValueError("failed gate requires a blocking finding") + return self + + +class SemanticFindingV1(_StrictModel): + type: Literal[ + "overclaim", + "overgeneralization", + "unsupported_claim", + "interpretation", + "visual_semantics", + ] + section: str = Field(min_length=1, max_length=256) + message: str = Field(min_length=1, max_length=20_000) + + +class SemanticRevisionV1(_StrictModel): + section: str = Field(min_length=1, max_length=256) + instruction: str = Field(min_length=1, max_length=20_000) + + +class SemanticReviewV1(_DigestBoundModel): + """Advisory review whose provenance cannot mutate the hard-gate result.""" + + _digest_field = "review_digest" + + schema_version: Literal["ari.semantic-review/v1"] = SEMANTIC_REVIEW_V1 + review_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + stage: Literal["evidence_grounded_semantic_review"] = ( + "evidence_grounded_semantic_review" + ) + phase: str = Field(min_length=1, max_length=128) + status: Literal["ok", "revise", "unavailable"] + model: str = Field(min_length=1, max_length=512) + model_revision: str | None = Field(default=None, max_length=512) + prompt_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + evidence_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + hard_gate_report_digest: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + scores: dict[str, float] = Field(default_factory=dict) + findings: tuple[SemanticFindingV1, ...] = Field(default_factory=tuple) + suggested_revisions: tuple[SemanticRevisionV1, ...] = Field(default_factory=tuple) + detected_overclaim_count: int = Field(ge=0) + previous_review_digest: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + score_delta: float | None = None + resolved_overclaim_count: int = 0 + human_verified_overclaim_precision: float | None = Field( + default=None, ge=0, le=1 + ) + note: str | None = Field(default=None, max_length=20_000) + + @field_validator("scores") + @classmethod + def _bounded_scores(cls, value: dict[str, float]) -> dict[str, float]: + if any(not math.isfinite(float(item)) or not 0 <= float(item) <= 1 for item in value.values()): + raise ValueError("semantic-review scores must be finite values in [0, 1]") + return value + + +def parse_metric_gate_contract(document: dict[str, Any]) -> MetricGateContractV1: + try: + return MetricGateContractV1.model_validate(document) + except Exception as exc: + raise ClaimGateContractError(f"invalid metric gate contract: {exc}") from exc + + +def parse_gate_report(document: dict[str, Any]) -> GateReportV1: + try: + return GateReportV1.model_validate(document) + except Exception as exc: + raise ClaimGateContractError(f"invalid gate report: {exc}") from exc + + +_LEGACY_FINDING_FIELDS = { + "type", + "message", + "claim_id", + "numeric_id", + "node_id", + "artifact_path", +} + + +def _legacy_gate_findings( + document: dict[str, Any], key: str, severity: str +) -> tuple[GateFindingV1, ...]: + raw_items = document.get(key) or () + if not isinstance(raw_items, (list, tuple)): + raise ClaimGateContractError(f"legacy {key} must be a list") + migrated: list[GateFindingV1] = [] + for raw in raw_items: + if not isinstance(raw, dict): + raise ClaimGateContractError(f"legacy {key} contains a non-object") + finding_type = str(raw.get("type") or "legacy_untyped_finding") + optional = { + name: (str(raw[name]) if raw.get(name) is not None else None) + for name in ("claim_id", "numeric_id", "node_id", "artifact_path") + } + try: + migrated.append( + GateFindingV1( + severity=severity, + type=finding_type, + message=str(raw.get("message") or finding_type), + **optional, + details={ + name: value + for name, value in raw.items() + if name not in _LEGACY_FINDING_FIELDS + }, + ) + ) + except Exception as exc: + raise ClaimGateContractError( + f"invalid legacy {key} finding: {exc}" + ) from exc + return tuple(migrated) + + +def migrate_legacy_gate_report( + document: dict[str, Any], *, source_run_id: str +) -> GateReportV1: + """Read a pre-v1 gate report without pretending it was fully replayable. + + Old reports did not record policy, evidence, or formula implementation + digests. The migration therefore binds each unknown provenance field to a + digest of an explicit legacy marker plus the exact old document. It never + rewrites the published report on disk and never upgrades its findings. + """ + + if not isinstance(document, dict) or not document: + raise ClaimGateContractError("legacy gate report must be an object") + if document.get("schema_version") == GATE_REPORT_V1: + return parse_gate_report(document) + if document.get("gate") != "claim_evidence_hard_gate": + raise ClaimGateContractError("unrecognized legacy gate report") + source_run_id = source_run_id.strip() + if not source_run_id: + raise ClaimGateContractError("legacy gate report needs its source run id") + + phase = document.get("phase") + policy_mode = document.get("policy_mode", document.get("policy")) + if phase not in {"draft", "final"}: + raise ClaimGateContractError("legacy gate report has an invalid phase") + if policy_mode not in {"off", "warn", "strict"}: + raise ClaimGateContractError("legacy gate report has an invalid policy mode") + + blocking = _legacy_gate_findings(document, "errors", "blocking") + advisory = _legacy_gate_findings(document, "warnings", "advisory") + status = "failed" if blocking else ("warn" if advisory else "passed") + should_block = bool(document.get("should_block")) + if should_block and (phase != "final" or policy_mode == "off" or not blocking): + raise ClaimGateContractError("legacy gate report has an impossible block state") + comparison_scope = document.get("comparison_scope", "any") + if comparison_scope not in {"any", "same_environment"}: + raise ClaimGateContractError("legacy gate report has an invalid comparison scope") + raw_metrics = document.get("metrics") or {} + if not isinstance(raw_metrics, dict): + raise ClaimGateContractError("legacy gate report metrics must be an object") + + legacy_digest = canonical_digest(document) + marker = {"format": "pre-ari.gate-report/v1", "document": legacy_digest} + try: + return GateReportV1.create( + source_run_id=source_run_id, + phase=phase, + policy_mode=policy_mode, + comparison_scope=comparison_scope, + status=status, + should_block=should_block, + policy_digest=canonical_digest({**marker, "unknown": "policy"}), + evidence_digest=canonical_digest({**marker, "unknown": "evidence"}), + formula_provenance=GateFormulaProvenanceV1( + registry_digest=canonical_digest( + {**marker, "unknown": "formula-registry"} + ), + formulas_used=("legacy-unrecorded",), + ), + blocking_findings=blocking, + advisory_findings=advisory, + metrics=raw_metrics, + ) + except Exception as exc: + raise ClaimGateContractError(f"invalid legacy gate report: {exc}") from exc + + +def parse_semantic_review(document: dict[str, Any]) -> SemanticReviewV1: + try: + return SemanticReviewV1.model_validate(document) + except Exception as exc: + raise ClaimGateContractError(f"invalid semantic review: {exc}") from exc + + +def migrate_legacy_metric_gate_contract( + document: dict[str, Any], *, source_idea_digest: str | None = None +) -> MetricGateContractV1: + """Conservatively read one pre-v1 metric-gate document. + + This is a read/migration boundary, never an admission shortcut. A missing + metric name or unit is rejected instead of guessed, and the resulting + contract remains ``human-review-required`` with legacy provenance. + """ + + if not isinstance(document, dict) or not document: + raise ClaimGateContractError("legacy metric contract must be an object") + if document.get("schema_version") == METRIC_GATE_CONTRACT_V1: + return parse_metric_gate_contract(document) + key = str(document.get("key") or document.get("name") or "").strip() + unit = str(document.get("unit") or "").strip() + if not key or not unit: + raise ClaimGateContractError( + "legacy metric contract needs an explicit metric name and unit" + ) + raw_claims = document.get("claims") or [] + claims: list[MetricClaimV1] = [] + evidence: list[str] = [key] + for raw in raw_claims: + if not isinstance(raw, dict): + continue + required = tuple( + str(item).strip() + for item in raw.get("required_evidence") or () + if str(item).strip() + ) + claim = str(raw.get("claim") or "").strip() + if claim and required: + claims.append(MetricClaimV1(claim=claim, required_evidence=required)) + evidence.extend(required) + evidence = list(dict.fromkeys(evidence)) + formula = str(document.get("formula") or "value").strip() + operands = document.get("formula_operands") or document.get("operands") + if not isinstance(operands, dict): + operands = {"value": key} + tolerance_raw = document.get("tolerance") or { + "absolute": 0.0, + "relative": 0.02, + } + if not isinstance(tolerance_raw, dict): + raise ClaimGateContractError("legacy tolerance must be an object") + correctness_raw = document.get("correctness") or None + correctness = None + if isinstance(correctness_raw, dict) and correctness_raw: + correctness = MetricCorrectnessV1( + expr=str(correctness_raw.get("expr") or "").strip(), + requires=tuple(correctness_raw.get("requires") or ()), + ) + source_digest = canonical_digest(document) + metric = MetricContractV1.create( + name=key, + unit=unit, + direction=document.get("direction") or "none", + comparison_scope=document.get("comparison_scope") or "not-applicable", + rationale=str(document.get("rationale") or "Legacy contract; human review required."), + required_evidence=tuple(evidence), + correctness_required=bool(document.get("correctness_required")), + normalization_ceiling=( + "measured" if document.get("ceiling_must_be_measured") else "not-applicable" + ), + target_value=document.get("target_value"), + formula=formula, + operands={str(role): str(name) for role, name in operands.items()}, + tolerance=MetricToleranceV1( + absolute=tolerance_raw.get("absolute", 0.0), + relative=tolerance_raw.get("relative", 0.02), + ), + formula_provenance=MetricFormulaProvenanceV1( + source="legacy-migration", + source_digest=source_digest, + ), + required_measured=tuple(document.get("required_measured") or ()), + invariants=tuple(document.get("invariants") or ()), + correctness=correctness, + confidence=0.0, + admission_status="human-review-required", + ) + return MetricGateContractV1.create( + source="legacy-migrated", + source_idea_digest=source_idea_digest or source_digest, + metric_contract=metric, + claims=tuple(claims), + ) + + +def admit_metric_contract_proposal( + proposal: MetricContractProposalV1, *, reviewer: str +) -> tuple[MetricGateContractV1, MetricAdmissionDecisionV1]: + """Human-admit an exact proposal; all scientific fields remain unchanged.""" + + reviewer = reviewer.strip() + if not reviewer: + raise ClaimGateContractError("metric admission requires a reviewer identity") + raw = dict(proposal.proposed_contract) + raw.pop("contract_digest", None) + raw.pop("schema_version", None) + raw_claims = raw.pop("claims", ()) + raw["formula_provenance"] = MetricFormulaProvenanceV1( + source="human-admission", + source_digest=proposal.proposal_digest, + model=proposal.model, + prompt_digests=(proposal.prompt_digest,), + ) + raw["confidence"] = proposal.confidence + raw["admission_status"] = "admitted" + try: + metric = MetricContractV1.create(**raw) + except Exception as exc: + raise ClaimGateContractError(f"proposal cannot be admitted: {exc}") from exc + claims = tuple(MetricClaimV1.model_validate(item) for item in raw_claims) + projection = MetricGateContractV1.create( + source="human-admitted", + source_idea_digest=proposal.source_idea_digest, + metric_contract=metric, + claims=claims, + ) + decision = MetricAdmissionDecisionV1.create( + proposal_digest=proposal.proposal_digest, + decision="admitted", + reviewer=reviewer, + reasons=("explicit-human-confirmation",), + admitted_contract_digest=metric.contract_digest, + ) + return projection, decision + + +__all__ = [ + "GATE_FINDING_V1", + "GATE_REPORT_V1", + "METRIC_ADMISSION_DECISION_V1", + "METRIC_CONTRACT_PROPOSAL_V1", + "METRIC_GATE_CONTRACT_V1", + "SEMANTIC_REVIEW_V1", + "ClaimGateContractError", + "GateFindingV1", + "GateFormulaProvenanceV1", + "GateReportV1", + "MetricAdmissionDecisionV1", + "MetricClaimV1", + "MetricContractProposalV1", + "MetricGateContractV1", + "SemanticFindingV1", + "SemanticReviewV1", + "SemanticRevisionV1", + "parse_gate_report", + "parse_metric_gate_contract", + "parse_semantic_review", + "admit_metric_contract_proposal", + "migrate_legacy_gate_report", + "migrate_legacy_metric_gate_contract", +] diff --git a/ari-core/ari/cli/bfts_loop.py b/ari-core/ari/cli/bfts_loop.py index 44b660c4..6f2e0bd3 100644 --- a/ari-core/ari/cli/bfts_loop.py +++ b/ari-core/ari/cli/bfts_loop.py @@ -28,6 +28,7 @@ from rich.console import Console +from ari.call_context import ToolCallContextV1 from ari.cli.lineage import ( _LINEAGE_LOG, _build_idea_ctx_for_expand, @@ -266,8 +267,14 @@ def _flush_tree_progress(force: bool = False) -> None: isinstance(_idea_data_pre, dict) and "_root_choice" in _idea_data_pre ) + _contract_already_minted = ( + isinstance(_idea_data_pre, dict) + and _idea_data_pre.get("research_contract") + is not None + ) if (not _already_inherited and not _already_chosen + and not _contract_already_minted and len(_idea_data_pre.get("ideas") or []) > 1): import asyncio as _asyncio_root from ari.orchestrator.root_idea_selector import ( @@ -527,6 +534,9 @@ def _is_output_artifact(rel_path: str, name: str) -> bool: def _node_exp(n): d = dict(experiment_data) d["work_dir"] = n.work_dir + # Custom checkpoint templates need not end in run_id. Carry + # the scheduler's canonical identity into each node context. + d["run_id"] = str(run_id) # Inject HPC settings so the agent knows without reading the .md again if _partition: d["slurm_partition"] = _partition @@ -685,7 +695,8 @@ def _node_exp(n): # typed store feeds the verifiable / paper-context layer # (search_research_memory, get_verified_context), NOT Phase 0 # working-context injection, which keeps using result_summary. - # Best-effort: never breaks the loop. CoW via cow_node_id=result.id. + # Best-effort: never breaks the loop. The signed NodeContext + # authorizes this completed node and its ordered lineage. from ari.config import consolidation_enabled as _cons_on if _cons_on(): try: @@ -696,6 +707,15 @@ def _node_exp(n): _nr_path = _cwd / "node_report.json" _nr = json.loads(_nr_path.read_text()) if _nr_path.exists() else None if _nr and getattr(agent, "mcp", None) is not None: + _memory_context = ToolCallContextV1.for_node( + run_id=run_id, + node_id=result.id, + parent_node_id=getattr(result, "parent_id", None), + ancestor_node_ids=list( + getattr(result, "ancestor_ids", []) or [] + ), + phase="bfts", + ) agent.mcp.call_tool( "consolidate_node_memory", { @@ -704,7 +724,7 @@ def _node_exp(n): "work_dir": str(_cwd), "run_id": run_id, }, - cow_node_id=result.id, + context=_memory_context, ) except Exception as _ce: logging.getLogger(__name__).warning( @@ -922,4 +942,3 @@ def _save_checkpoint(checkpoint_dir, run_id, experiment_file, nodes): _save_pv(checkpoint_dir, _build_pv(checkpoint_dir)) except Exception: log.debug("prompt_versions rollup write failed", exc_info=True) - diff --git a/ari-core/ari/cli/run.py b/ari-core/ari/cli/run.py index 438b3d7b..e50dc18f 100644 --- a/ari-core/ari/cli/run.py +++ b/ari-core/ari/cli/run.py @@ -362,22 +362,9 @@ def run( "trajectory is NOT guaranteed reproducible " "(recorded in bfts_web_provenance.json).[/yellow]" ) - # auto-migrate v0.5.x sources on first launch. - try: - from ari.memory.auto_migrate import maybe_auto_migrate - _am = maybe_auto_migrate(checkpoint_dir) - if _am.get("ran") and _am.get("imported"): - logging.getLogger(__name__).info( - "v0.5.x auto-migration: %s", _am["imported"] - ) - except Exception as _amerr: - logging.getLogger(__name__).warning( - "auto-migrate skipped: %s", _amerr - ) # — on-exit backup. try: import atexit as _atexit_bk - from ari.memory_cli import _do_backup as _do_bk _atexit_bk.register(lambda _p=checkpoint_dir: _safe_backup(_p)) except Exception: pass @@ -526,13 +513,7 @@ def resume( console.print("[yellow]No pending nodes.[/yellow]") raise typer.Exit(0) - #+ — auto-migrate v0.5.x sources - # and auto-restore from memory_backup.jsonl.gz when Letta is empty. - try: - from ari.memory.auto_migrate import maybe_auto_migrate - maybe_auto_migrate(checkpoint_dir) - except Exception as _amerr: - logging.getLogger(__name__).warning("auto-migrate skipped: %s", _amerr) + # Restore only the verified v1 portable backup when Letta is empty. if os.environ.get("ARI_MEMORY_AUTO_RESTORE", "true").lower() != "false": try: from ari.memory_cli import _do_restore, _backup_path @@ -576,4 +557,3 @@ def resume( console.print(f"[bold red]Paper pipeline failed:[/bold red] {_paper_err}") import traceback traceback.print_exc() - diff --git a/ari-core/ari/clone/__init__.py b/ari-core/ari/clone/__init__.py index 797fb326..10536059 100644 --- a/ari-core/ari/clone/__init__.py +++ b/ari-core/ari/clone/__init__.py @@ -53,10 +53,8 @@ def _sha256_file(p: Path) -> str: def _recompute_manifest_digest(extracted_dir: Path, manifest_path: Path) -> str: """Re-derive the bundle digest from the extracted tree. - Mirrors the curator's logic in ari-skill-transform/src/curate.py: - canonical JSON of {"version":1,"files":[{"path","sha256","size"}, ...]} - sorted by path, hashed with sha256. This is what we compare against - `manifest.lock`'s `bundle_sha256` field after extraction. + Mirrors both supported curator formats. v2 also binds each file's logical + role; v1 remains a read-only published-bundle compatibility format. """ manifest = json.loads(manifest_path.read_text(encoding="utf-8")) files = manifest.get("files") or [] @@ -71,8 +69,20 @@ def _recompute_manifest_digest(extracted_dir: Path, manifest_path: Path) -> str: raise CloneError( f"sha256 mismatch for {rel}: expected {entry.get('sha256')[:16]}…, got {h[:16]}…" ) - rebuilt.append({"path": rel, "sha256": h, "size": p.stat().st_size}) - canonical_payload = {"version": 1, "files": sorted(rebuilt, key=lambda r: r["path"])} + record = {"path": rel, "sha256": h, "size": p.stat().st_size} + if int(manifest.get("version") or 1) == 2: + role = entry.get("role") + if not isinstance(role, str) or not role: + raise CloneError(f"v2 manifest lacks role for {rel}") + record["role"] = role + rebuilt.append(record) + version = int(manifest.get("version") or 1) + if version not in {1, 2}: + raise CloneError(f"unsupported manifest version: {version}") + canonical_payload = { + "version": version, + "files": sorted(rebuilt, key=lambda r: r["path"]), + } canonical = json.dumps(canonical_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/ari-core/ari/config/README.md b/ari-core/ari/config/README.md index a07ab2a4..88a568b4 100644 --- a/ari-core/ari/config/README.md +++ b/ari-core/ari/config/README.md @@ -8,6 +8,7 @@ their env-var overrides (`ARI_BACKEND`, `ARI_MODEL`, …). - `README.md` — this file. - `__init__.py` — Pydantic config models + env-var overrides. - `finder.py` — workflow / profile YAML discovery. +- `skill_runtime.py` — canonical manifest to runtime registration metadata adapter. ## See also diff --git a/ari-core/ari/config/__init__.py b/ari-core/ari/config/__init__.py index 2fb9322b..75018338 100644 --- a/ari-core/ari/config/__init__.py +++ b/ari-core/ari/config/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import os import re from pathlib import Path @@ -10,6 +11,17 @@ import yaml from pydantic import BaseModel, Field +from ari.config.skill_runtime import manifest_runtime_metadata +from ari.skill_manifest import ( + MANIFEST_FILENAME, + SkillManifestV1, + load_skill_manifest, + resolve_skill_entrypoint, +) + + +logger = logging.getLogger(__name__) + class LLMConfig(BaseModel): backend: str = Field( @@ -43,7 +55,7 @@ class LLMConfig(BaseModel): class SkillConfig(BaseModel): name: str = Field( ..., - description="Skill package directory name (e.g. `ari-skill-coding`).", + description="Runtime Skill alias (e.g. `coding-skill`).", ) path: str = Field( ..., @@ -61,6 +73,66 @@ class SkillConfig(BaseModel): "/ `reproduce` / `all` / `none`) or a list. `all` " "matches any phase; `none` disables the skill.", ) + package: str = Field( + "", + description="Canonical package identity from skill.yaml.", + ) + version: str = Field( + "", + description="Canonical package version from skill.yaml.", + ) + manifest_path: str | None = Field( + None, + description="Resolved path to the canonical skill.yaml, when present.", + ) + entrypoint: str = Field( + "src/server.py", + description="Package-relative Python MCP server entrypoint.", + ) + enabled_by_default: bool = Field( + True, + description="Whether auto-discovery admits this Skill by default.", + ) + environment_policy: Literal["audit-pending", "complete"] = Field( + "audit-pending", + description="Whether manifest environment declarations are exhaustive.", + ) + required_env: list[str] = Field( + default_factory=list, + description="Environment names required by the Skill contract.", + ) + optional_env: list[str] = Field( + default_factory=list, + description="Optional environment names declared by the Skill contract.", + ) + credential_scopes: dict[str, dict[str, list[str]]] = Field( + default_factory=dict, + description="Named credential scopes and their required/optional env names.", + ) + credential_scope_identities: list[dict] = Field( + default_factory=list, + description="Runtime-only value-free credential authority identities.", + exclude=True, + ) + tool_timeout_classes: dict[str, str] = Field( + default_factory=dict, + description="Resolved manifest timeout class keyed by runtime tool name.", + ) + manifest_digest: str = Field( + "", description="SHA-256 digest of the normalized canonical manifest." + ) + tool_refs: dict[str, str] = Field( + default_factory=dict, + description="Declared immutable tool references keyed by runtime name.", + ) + tool_capabilities: dict[str, str] = Field( + default_factory=dict, + description="Semantic capability references keyed by runtime tool name.", + ) + tool_policies: dict[str, dict] = Field( + default_factory=dict, + description="Resolved side-effect, determinism, phase, and permission policy.", + ) class BFTSConfig(BaseModel): @@ -146,7 +218,7 @@ class BFTSConfig(BaseModel): allow_web: bool = Field( False, description="Opt-in: expose web-skill (web_search / fetch_url / " - "search_arxiv / search_semantic_scholar) to the BFTS node " + "search_papers / walk_citations) to the BFTS node " "agent during exploration. Default False keeps the search " "loop reproducible (P5) — live web results are " "time-varying. When True, ARI records a " @@ -316,7 +388,14 @@ def _apply_memory_section(raw: dict) -> None: ) return backend = (mem.get("backend") or "letta").strip().lower() - os.environ.setdefault("ARI_MEMORY_BACKEND", backend) + # ``local`` and ``cloud`` select a Letta deployment topology; neither is + # an alternate storage implementation. Only the test-only in-memory + # implementation is forbidden in workflow configuration. + if backend not in {"letta", "local", "cloud"}: + raise ValueError( + "workflow memory.backend must select Letta (letta/local/cloud); " + "the in-memory backend is available only inside the test namespace" + ) letta = mem.get("letta") or {} if letta.get("base_url"): os.environ.setdefault("LETTA_BASE_URL", str(letta["base_url"])) @@ -359,6 +438,7 @@ def _resolve_ari_root(data): _apply_web_phase_for_bfts(cfg) return cfg cfg = ARIConfig(**{k: v for k, v in raw.items() if k in ARIConfig.model_fields}) + _hydrate_skill_manifests(cfg.skills) _merge_bfts_disabled_tools(cfg, raw) _apply_llm_env_overrides(cfg) _apply_checkpoint_env_overrides(cfg) @@ -438,24 +518,34 @@ def apply_bfts_env_overrides(cfg: "ARIConfig") -> None: """ _n = os.environ.get("ARI_MAX_NODES") if _n: - try: cfg.bfts.max_total_nodes = int(_n) - except ValueError: pass + try: + cfg.bfts.max_total_nodes = int(_n) + except ValueError: + pass _d = os.environ.get("ARI_MAX_DEPTH") if _d: - try: cfg.bfts.max_depth = int(_d) - except ValueError: pass + try: + cfg.bfts.max_depth = int(_d) + except ValueError: + pass _r = os.environ.get("ARI_MAX_REACT") if _r: - try: cfg.bfts.max_react_steps = int(_r) - except ValueError: pass + try: + cfg.bfts.max_react_steps = int(_r) + except ValueError: + pass _p = os.environ.get("ARI_PARALLEL") if _p: - try: cfg.bfts.max_parallel_nodes = int(_p) - except ValueError: pass + try: + cfg.bfts.max_parallel_nodes = int(_p) + except ValueError: + pass _t = os.environ.get("ARI_TIMEOUT_NODE") if _t: - try: cfg.bfts.timeout_per_node = int(_t) - except ValueError: pass + try: + cfg.bfts.timeout_per_node = int(_t) + except ValueError: + pass # GUI wizard's frontier-selection strategy choice. Pydantic does not # validate on assignment, so guard against unknown values from env. _fs = os.environ.get("ARI_FRONTIER_SCORE") @@ -562,19 +652,90 @@ def _merge_bfts_disabled_tools(cfg: "ARIConfig", raw: dict) -> None: def _discover_skills(base_dir: Path | None = None) -> list[SkillConfig]: - """Auto-detect ari-skill-* directories and return a list of SkillConfig.""" + """Auto-detect canonical, default-enabled ``ari-skill-*`` packages. + + Directory presence is not registration. Only a canonical, default-enabled + manifest enters production discovery; an unmanifested local Skill must be + named explicitly in development configuration. + """ if base_dir is None: # Phase 2 — file moved into a package; ``parents[3]`` reaches # the repo root (alongside the ``ari-skill-*`` directories). base_dir = Path(__file__).resolve().parents[3] skills = [] - for skill_dir in sorted(base_dir.glob("ari-skill-*")): - server = skill_dir / "src" / "server.py" - if server.exists(): - skills.append(SkillConfig(name=skill_dir.name, path=str(skill_dir))) + for manifest_path in sorted(base_dir.glob(f"ari-skill-*/{MANIFEST_FILENAME}")): + skill_dir = manifest_path.parent + manifest = load_skill_manifest(manifest_path) + resolve_skill_entrypoint(skill_dir, manifest) + if not manifest.enabled_by_default: + logger.info( + "Skipping default-off Skill '%s' during auto-discovery", + manifest.name, + ) + continue + skills.append(_skill_config_from_manifest(skill_dir, manifest_path, manifest)) return skills +def _skill_config_from_manifest( + skill_dir: Path, + manifest_path: Path, + manifest: SkillManifestV1, + *, + phase: str | list[str] = "all", +) -> SkillConfig: + return SkillConfig( + name=manifest.name, + path=str(skill_dir), + description=manifest.description, + phase=phase, + package=manifest.package, + version=manifest.version, + manifest_path=str(manifest_path), + entrypoint=manifest.entrypoint.module, + enabled_by_default=manifest.enabled_by_default, + environment_policy=manifest.environment_policy, + required_env=list(manifest.required_env), + optional_env=list(manifest.optional_env), + **manifest_runtime_metadata(manifest), + ) + + +def _hydrate_skill_manifests(skills: list[SkillConfig]) -> None: + """Attach canonical metadata to explicitly configured Skills in place. + + ``workflow.yaml`` remains the authority for aliases and phase exposure during + the compatibility window. Package identity, entrypoint, version, environment + declarations, and execution class come from the manifest. + """ + + for skill in skills: + skill_dir = Path(skill.path) + manifest_path = skill_dir / MANIFEST_FILENAME + if not manifest_path.is_file(): + continue + manifest = load_skill_manifest(manifest_path) + resolve_skill_entrypoint(skill_dir, manifest) + if skill.name != manifest.name: + logger.warning( + "Configured Skill alias '%s' differs from canonical name '%s'", + skill.name, + manifest.name, + ) + skill.package = manifest.package + skill.version = manifest.version + skill.manifest_path = str(manifest_path) + skill.entrypoint = manifest.entrypoint.module + skill.enabled_by_default = manifest.enabled_by_default + skill.environment_policy = manifest.environment_policy + skill.required_env = list(manifest.required_env) + skill.optional_env = list(manifest.optional_env) + for field_name, value in manifest_runtime_metadata(manifest).items(): + setattr(skill, field_name, value) + if not skill.description: + skill.description = manifest.description + + def auto_config() -> ARIConfig: """Default configuration when config.yaml is omitted. Can be overridden by environment variables.""" # Determine backend from model name diff --git a/ari-core/ari/config/skill_runtime.py b/ari-core/ari/config/skill_runtime.py new file mode 100644 index 00000000..ee84bf23 --- /dev/null +++ b/ari-core/ari/config/skill_runtime.py @@ -0,0 +1,33 @@ +"""Translate canonical Skill manifests into runtime registration metadata.""" + +from ari.skill_manifest import SkillManifestV1, manifest_digest, manifest_tool_ref + + +def manifest_runtime_metadata(manifest: SkillManifestV1) -> dict[str, object]: + """Resolve the immutable identity and per-tool policy consumed by MCPClient.""" + + tools = manifest.resolved_tools() + return { + "tool_timeout_classes": {tool.name: tool.timeout_class for tool in tools}, + "manifest_digest": manifest_digest(manifest), + "tool_refs": { + tool.name: manifest_tool_ref(manifest, tool.name) for tool in tools + }, + "tool_capabilities": {tool.name: tool.capability_ref for tool in tools}, + "tool_policies": { + tool.name: tool.model_dump( + mode="json", exclude={"name", "capability_ref", "description"} + ) + for tool in tools + }, + "credential_scopes": { + scope.id: { + "required_env": list(scope.required_env), + "optional_env": list(scope.optional_env), + } + for scope in manifest.credential_scopes + }, + } + + +__all__ = ["manifest_runtime_metadata"] diff --git a/ari-core/ari/container.py b/ari-core/ari/container.py index 9bf00ad8..a3134871 100644 --- a/ari-core/ari/container.py +++ b/ari-core/ari/container.py @@ -1,19 +1,28 @@ -from __future__ import annotations """Unified container runtime abstraction. Detects Docker (local) or Singularity/Apptainer (HPC), provides image pull -and command execution helpers. Falls back to bare subprocess when no -container runtime is selected. +and command execution helpers, and refuses an implicit host fallback when a +configured runtime cannot be honored. """ +from __future__ import annotations + import glob as _glob import os -import signal import shutil import subprocess from dataclasses import dataclass, field +from pathlib import Path as _Path +from typing import Literal as _Literal from typing import Optional +from ari.execution import ( + ExecutionLimitsV1 as _ExecutionLimitsV1, + ExecutionRequestV1 as _ExecutionRequestV1, + WorkspaceRefV1 as _WorkspaceRefV1, + build_minimal_environment as _build_minimal_environment, + execute_local as _execute_local, +) # ── Fail-safe: process sandbox ────────────────────────── # RLIMIT_NPROC is per real-uid, not per process: capping the child also counts @@ -22,21 +31,9 @@ # …). Only honor the cap when the operator explicitly opts in via # ARI_MAX_CHILD_PROCS. _MAX_CHILD_PROCS_ENV = os.environ.get("ARI_MAX_CHILD_PROCS", "").strip() -_MAX_CHILD_PROCS: int | None = int(_MAX_CHILD_PROCS_ENV) if _MAX_CHILD_PROCS_ENV else None - - -def _sandbox_preexec() -> None: - """Pre-exec hook: new process group (and optional RLIMIT_NPROC cap).""" - os.setsid() - if _MAX_CHILD_PROCS is None: - return - try: - import resource - _soft, hard = resource.getrlimit(resource.RLIMIT_NPROC) - cap = min(hard, _MAX_CHILD_PROCS) - resource.setrlimit(resource.RLIMIT_NPROC, (cap, hard)) - except Exception: - pass +_MAX_CHILD_PROCS: int | None = ( + int(_MAX_CHILD_PROCS_ENV) if _MAX_CHILD_PROCS_ENV else None +) # ── Runtime detection ──────────────────────────────── @@ -70,18 +67,37 @@ def _detect_container_shell(mode: str, image_ref: str) -> str: if mode == "docker": # `docker exec` targets a running container, not an image — use `run --rm`. probe_cmd = [ - "docker", "run", "--rm", image_ref, - "sh", "-c", "command -v bash >/dev/null 2>&1 && echo bash || echo sh", + "docker", + "run", + "--rm", + "--network", + "none", + image_ref, + "sh", + "-c", + "command -v bash >/dev/null 2>&1 && echo bash || echo sh", ] elif mode in ("singularity", "apptainer"): probe_cmd = [ - mode, "exec", image_ref, - "sh", "-c", "command -v bash >/dev/null 2>&1 && echo bash || echo sh", + mode, + "exec", + "--cleanenv", + "--containall", + "--net", + "--network", + "none", + image_ref, + "sh", + "-c", + "command -v bash >/dev/null 2>&1 && echo bash || echo sh", ] if probe_cmd is not None: try: probe = subprocess.run( - probe_cmd, capture_output=True, text=True, timeout=30, + probe_cmd, + capture_output=True, + text=True, + timeout=30, ) if probe.returncode == 0 and probe.stdout.strip() == "sh": shell = "sh" @@ -100,9 +116,17 @@ def detect_runtime() -> str: """ on_hpc = bool(os.environ.get("SLURM_JOB_ID")) - have_docker = shutil.which("docker") is not None and _cmd_ok(["docker", "info"]) is not None - have_singularity = shutil.which("singularity") is not None and _cmd_ok(["singularity", "--version"]) is not None - have_apptainer = shutil.which("apptainer") is not None and _cmd_ok(["apptainer", "--version"]) is not None + have_docker = ( + shutil.which("docker") is not None and _cmd_ok(["docker", "info"]) is not None + ) + have_singularity = ( + shutil.which("singularity") is not None + and _cmd_ok(["singularity", "--version"]) is not None + ) + have_apptainer = ( + shutil.which("apptainer") is not None + and _cmd_ok(["apptainer", "--version"]) is not None + ) if on_hpc: if have_apptainer: @@ -160,11 +184,13 @@ def _resolve_singularity_ref(image: str) -> str: env_dir = os.environ.get("ARI_CONTAINERS_DIR", "") if env_dir: search_dirs.append(env_dir) - search_dirs.extend([ - os.path.join(os.getcwd(), "containers"), - os.getcwd(), - os.path.expanduser("~/containers"), - ]) + search_dirs.extend( + [ + os.path.join(os.getcwd(), "containers"), + os.getcwd(), + os.path.expanduser("~/containers"), + ] + ) # Walk up from cwd looking for a ``containers/`` sibling — covers the # case where MCP skills are invoked from an experiment workdir nested # several levels below the ARI project root. @@ -177,7 +203,9 @@ def _resolve_singularity_ref(image: str) -> str: _cur = _parent # Finally, the ARI package itself lives at /ari-core/ari/, so # three levels up is the canonical ``/containers``. - _pkg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + _pkg_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) search_dirs.append(os.path.join(_pkg_root, "containers")) seen: set[str] = set() @@ -210,7 +238,9 @@ def pull_image(config: ContainerConfig) -> bool: if mode == "docker": r = subprocess.run( ["docker", "pull", config.image], - capture_output=True, text=True, timeout=600, + capture_output=True, + text=True, + timeout=600, ) return r.returncode == 0 @@ -229,7 +259,9 @@ def pull_image(config: ContainerConfig) -> bool: out_path = os.path.join("containers", _name) r = subprocess.run( [exe, "pull", "--force", out_path, f"docker://{config.image}"], - capture_output=True, text=True, timeout=600, + capture_output=True, + text=True, + timeout=600, ) return r.returncode == 0 @@ -252,25 +284,38 @@ def run_in_container( mode = config.mode if mode == "auto": mode = detect_runtime() if config.image else "none" - - run_env = dict(os.environ) - if env: - run_env.update(env) + run_env = _build_minimal_environment(env) if mode == "none" or not config.image: - return subprocess.Popen(cmd, env=run_env, cwd=workdir, preexec_fn=_sandbox_preexec) + return subprocess.Popen( + cmd, + env=run_env, + cwd=workdir, + start_new_session=True, + close_fds=True, + ) workdir = os.path.abspath(workdir) if mode == "docker": - docker_cmd = ["docker", "run", "--rm", "-v", f"{workdir}:{workdir}", "-w", workdir] + docker_cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{workdir}:{workdir}", + "-w", + workdir, + ] if env: for k, v in env.items(): docker_cmd.extend(["-e", f"{k}={v}"]) docker_cmd.extend(config.extra_args) docker_cmd.append(config.image) docker_cmd.extend(cmd) - return subprocess.Popen(docker_cmd, env=run_env, preexec_fn=_sandbox_preexec) + return subprocess.Popen( + docker_cmd, env=run_env, start_new_session=True, close_fds=True + ) if mode in ("singularity", "apptainer"): exe = mode @@ -283,10 +328,11 @@ def run_in_container( sif_cmd.extend(config.extra_args) sif_cmd.append(_resolve_singularity_ref(config.image)) sif_cmd.extend(cmd) - return subprocess.Popen(sif_cmd, env=run_env, preexec_fn=_sandbox_preexec) + return subprocess.Popen( + sif_cmd, env=run_env, start_new_session=True, close_fds=True + ) - # Fallback — direct execution - return subprocess.Popen(cmd, env=run_env, cwd=workdir, preexec_fn=_sandbox_preexec) + raise ValueError(f"unsupported container mode {mode!r}; refusing host fallback") def _run_shell_sandboxed( @@ -296,33 +342,41 @@ def _run_shell_sandboxed( timeout: int = 120, cwd: str | None = None, ) -> subprocess.CompletedProcess: - """Run with process-group isolation. Kill entire tree on timeout.""" - proc = subprocess.Popen( + """Compatibility adapter over the canonical execution contract.""" + + workdir = _Path(cwd or ".").resolve(strict=True) + if not workdir.is_dir(): + raise NotADirectoryError(workdir) + common = { + "workspace": _WorkspaceRefV1(root=str(workdir)), + "timeout_seconds": timeout, + "limits": _ExecutionLimitsV1(max_processes=_MAX_CHILD_PROCS), + } + if isinstance(cmd, str): + if not shell: + raise ValueError("string commands require explicit shell=True") + request = _ExecutionRequestV1(shell_command=cmd, **common) + else: + request = _ExecutionRequestV1(argv=cmd, **common) + result = _execute_local(request) + logs: dict[str, str] = {} + for artifact in result.artifacts: + logs[artifact.logical_role] = ( + _Path(request.workspace.root) / artifact.relative_path + ).read_text(encoding="utf-8", errors="replace") + if result.status == "timed_out": + raise subprocess.TimeoutExpired( + cmd, + timeout, + output=logs.get("stdout", ""), + stderr=logs.get("stderr", ""), + ) + return subprocess.CompletedProcess( cmd, - shell=shell, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - cwd=cwd, - preexec_fn=_sandbox_preexec, + result.exit_code if result.exit_code is not None else -1, + logs.get("stdout", ""), + logs.get("stderr", ""), ) - try: - stdout, stderr = proc.communicate(timeout=timeout) - return subprocess.CompletedProcess(cmd, proc.returncode, stdout or "", stderr or "") - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGTERM) - except OSError: - pass - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGKILL) - except OSError: - proc.kill() - proc.wait() - raise def run_shell_in_container( @@ -331,6 +385,7 @@ def run_shell_in_container( *, cwd: str | None = None, timeout: int = 120, + network: _Literal["inherit", "deny"] = "inherit", ) -> subprocess.CompletedProcess: """Run a shell command string inside the container (blocking). @@ -345,47 +400,99 @@ def run_shell_in_container( When ``config.image`` is empty or mode resolves to ``"none"``, the command runs directly on the host. """ + full_cmd = container_shell_argv( + config, + shell_cmd, + cwd=cwd, + network=network, + ) + if full_cmd is None: + if network == "deny": + raise ValueError( + "network denial requires a container; refusing host fallback" + ) + return _run_shell_sandboxed( + shell_cmd, + shell=True, + timeout=timeout, + cwd=cwd, + ) + return _run_shell_sandboxed( + full_cmd, + timeout=timeout, + cwd=cwd, + ) + + +def container_shell_argv( + config: ContainerConfig, + shell_cmd: str, + *, + cwd: str | None = None, + network: _Literal["inherit", "deny"] = "inherit", +) -> list[str] | None: + """Build the exact container argv for a shell command. + + ``None`` means the configuration selects host execution. Callers that + require a container must treat that value as an error rather than silently + weakening isolation. + """ + workdir = os.path.abspath(cwd) if cwd else os.path.abspath(".") mode = config.mode if mode == "auto": mode = detect_runtime() if config.image else "none" + if network == "deny" and any( + argument == "--net" + or argument.startswith("--network") + or argument.startswith("--netns") + for argument in config.extra_args + ): + raise ValueError( + "container extra_args cannot override an explicit network denial" + ) if mode == "none" or not config.image: - return _run_shell_sandboxed( - shell_cmd, shell=True, timeout=timeout, cwd=cwd, - ) + return None - # Build the container command that wraps the shell command if mode == "docker": shell = _detect_container_shell("docker", config.image) full_cmd = [ - "docker", "run", "--rm", - "-v", f"{workdir}:{workdir}", "-w", workdir, + "docker", + "run", + "--rm", + "-v", + f"{workdir}:{workdir}", + "-w", + workdir, + *(["--network", "none"] if network == "deny" else []), *config.extra_args, config.image, - shell, "-c", shell_cmd, + shell, + "-c", + shell_cmd, ] elif mode in ("singularity", "apptainer"): image_ref = _resolve_singularity_ref(config.image) shell = _detect_container_shell(mode, image_ref) - # --writable-tmpfs: see comment in run_in_container. Needed so the - # agent can install missing tools (git, build-essentials, …) - # without rebuilding the SIF. full_cmd = [ - mode, "exec", "--writable-tmpfs", "--bind", workdir, + mode, + "exec", + "--cleanenv", + "--containall", + "--writable-tmpfs", + *(["--net", "--network", "none"] if network == "deny" else []), + "--bind", + workdir, *config.extra_args, image_ref, - shell, "-c", shell_cmd, + shell, + "-c", + shell_cmd, ] else: - # Unknown mode — fall back to sandboxed direct execution - return _run_shell_sandboxed( - shell_cmd, shell=True, timeout=timeout, cwd=cwd, - ) - - return _run_shell_sandboxed( - full_cmd, timeout=timeout, cwd=cwd, - ) + raise ValueError(f"unsupported container mode {mode!r}; refusing host fallback") + return full_cmd def config_from_env() -> ContainerConfig | None: @@ -424,7 +531,9 @@ def list_images(mode: str = "auto") -> list[dict]: name = parts[0] if name == ":": continue - images.append({"name": name, "size": parts[1] if len(parts) > 1 else ""}) + images.append( + {"name": name, "size": parts[1] if len(parts) > 1 else ""} + ) elif mode in ("singularity", "apptainer"): # Scan common SIF cache locations diff --git a/ari-core/ari/core.py b/ari-core/ari/core.py index 3807195b..4b50f1ef 100644 --- a/ari-core/ari/core.py +++ b/ari-core/ari/core.py @@ -6,10 +6,8 @@ from __future__ import annotations -import importlib.util import logging import os -import re from pathlib import Path from typing import TYPE_CHECKING @@ -99,13 +97,13 @@ def build_runtime(cfg, experiment_text: str = "", checkpoint_dir: "str | Path | from ari.agent.workflow import from_experiment_text, enrich_hints_from_mcp from ari.evaluator import LLMEvaluator from ari.llm.client import LLMClient - from ari.mcp.client import MCPClient + from ari.mcp.client import MCPClient, ToolNameCollisionError # ReAct trace now lives in Letta. The # v0.5.x FileMemoryClient is kept only as a v0.5.x → v0.6.0 migration # source (`ari memory migrate --react`). from ari.memory.letta_client import LettaMemoryClient from ari.orchestrator.bfts import BFTS - from ari.paths import PathManager + from ari.skill_lock import SKILLS_LOCK_FILENAME, SkillLockError if checkpoint_dir is None: raise ValueError( @@ -137,12 +135,16 @@ def _phase_llm(phase: str) -> LLMClient: _skills = list(cfg.skills) if not cfg.resources.get("hpc_enabled", True): # Laptop profile: drop the hpc-skill entirely. Its SLURM/Singularity - # tools (slurm_submit, singularity_build/pull/run/run_gpu) would + # scheduler tools (job_submit, container_submit, slurm_submit) would # otherwise submit sbatch jobs from inside the skill even when the # agent is not supposed to use HPC at all. run_bash lives in # coding-skill, so removing hpc-skill does not remove shell access. _skills = [s for s in _skills if getattr(s, "name", "") != "hpc-skill"] - mcp = MCPClient(_skills, disabled_tools=_disabled) + mcp = MCPClient( + _skills, + disabled_tools=_disabled, + skill_lock_path=Path(checkpoint_dir) / SKILLS_LOCK_FILENAME, + ) # Wire the MCPClient into both LLMClients so cli-shim-targeted calls can # forward (--mcp-config + --allowedTools mcp__*) to the Claude # subprocess. With this, the text-catalog tool protocol is bypassed and @@ -214,7 +216,9 @@ def _phase_llm(phase: str) -> LLMClient: try: bfts_tools = mcp.list_tools(phase="bfts") enrich_hints_from_mcp(wf_hints, bfts_tools, hpc_enabled=hpc_enabled) - except Exception: + except Exception as exc: + if isinstance(exc, (SkillLockError, ToolNameCollisionError)): + raise pass # Graceful fallback — static hints still work # metric_extractor: generated by workflow.py from metric_keyword; falls back to MetricSpec extractor @@ -247,7 +251,7 @@ def generate_paper_section( log.info("Starting paper pipeline (config_path=%s, checkpoint=%s)", config_path, checkpoint_dir) print(f"\n{'='*60}") - print(f" Paper Pipeline Starting") + print(" Paper Pipeline Starting") print(f" Checkpoint: {checkpoint_dir}") print(f"{'='*60}", flush=True) diff --git a/ari-core/ari/credential_scope.py b/ari-core/ari/credential_scope.py new file mode 100644 index 00000000..01ee29c9 --- /dev/null +++ b/ari-core/ari/credential_scope.py @@ -0,0 +1,70 @@ +"""Credential-name classification and value-free manifest scope contract.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +ENVIRONMENT_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") +_CREDENTIAL_ENV_RE = re.compile( + r"(?:^|_)(?:API_?KEY|APIKEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?|" + r"PRIVATE_KEY|SSH_KEY)(?:$|_)" +) + + +def looks_like_credential_environment_name(name: str) -> bool: + """Return whether an environment name requires credential classification.""" + + return bool(_CREDENTIAL_ENV_RE.search(name)) + + +class CredentialScopeV1(BaseModel): + """Named credential authority whose values are never serialized to locks.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + id: str + required_env: list[str] = Field(default_factory=list) + optional_env: list[str] = Field(default_factory=list) + + @field_validator("id") + @classmethod + def _valid_id(cls, value: str) -> str: + value = value.strip() + if not re.fullmatch(r"^[a-z0-9][a-z0-9._-]*$", value): + raise ValueError("credential scope id must be a lowercase dotted identifier") + return value + + @field_validator("required_env", "optional_env") + @classmethod + def _valid_env_names(cls, values: list[str]) -> list[str]: + if len(values) != len(set(values)): + raise ValueError("credential environment variable names must be unique") + invalid = [value for value in values if not ENVIRONMENT_NAME_RE.fullmatch(value)] + if invalid: + raise ValueError(f"invalid credential environment names: {invalid}") + return values + + @model_validator(mode="after") + def _disjoint_names(self) -> "CredentialScopeV1": + overlap = sorted(set(self.required_env) & set(self.optional_env)) + if overlap: + raise ValueError( + "credential environment variables cannot be required and optional: " + f"{overlap}" + ) + return self + + def environment_names(self) -> tuple[str, ...]: + """Return required then optional credential variable names.""" + + return tuple(self.required_env + self.optional_env) + + +__all__ = [ + "CredentialScopeV1", + "ENVIRONMENT_NAME_RE", + "looks_like_credential_environment_name", +] diff --git a/ari-core/ari/evaluator/dynamic_axes.py b/ari-core/ari/evaluator/dynamic_axes.py index a8b25612..114dec33 100644 --- a/ari-core/ari/evaluator/dynamic_axes.py +++ b/ari-core/ari/evaluator/dynamic_axes.py @@ -471,12 +471,22 @@ def build_axes_for_run( plan_text = "" if idea_data and isinstance(idea_data, dict): - ideas = idea_data.get("ideas") or [] - if ideas and isinstance(ideas[0], dict): + if idea_data.get("research_contract") is not None: + from ari.research_contract import parse_research_contract_document + + contract = parse_research_contract_document(idea_data) + _plan = contract.experiment_plan if contract is not None else "" + plan_text = _plan + else: + ideas = idea_data.get("ideas") or [] + _plan = ( + ideas[0].get("experiment_plan") + if ideas and isinstance(ideas[0], dict) + else "" + ) or "" # Newer generate_ideas variants emit a structured plan # ({"Design Steps": [...], "Ideal Outcomes": ...}); flatten to # text so the regex-based plan_to_axes still finds keywords. - _plan = ideas[0].get("experiment_plan") or "" plan_text = _plan if isinstance(_plan, str) else json.dumps( _plan, ensure_ascii=False, default=str ) diff --git a/ari-core/ari/evaluator/llm_evaluator.py b/ari-core/ari/evaluator/llm_evaluator.py index bc1a74c5..10f13f48 100644 --- a/ari-core/ari/evaluator/llm_evaluator.py +++ b/ari-core/ari/evaluator/llm_evaluator.py @@ -14,6 +14,7 @@ penalizes any single weak axis and naturally spreads scores away from the centre — directly countering the single-scalar collapse problem. """ + from __future__ import annotations import json @@ -24,6 +25,11 @@ import litellm from ari._factory import BaseRegistry +from ari.execution import ( + MeasurementDocumentError, + measurement_document_format, + parse_measurement_document, +) logger = logging.getLogger(__name__) @@ -219,7 +225,9 @@ class MetricSpec: expected_metrics: list[str] = field(default_factory=list) expected_params: list[str] = field(default_factory=list) scoring_guide: str = "" - artifact_extractor: object = field(default=None) # callable(artifacts_text: str) -> dict + artifact_extractor: object = field( + default=None + ) # callable(artifacts_text: str) -> dict # Optional per-axis weights for the harmonic-mean composite. When None, # the evaluator falls back to constructor-supplied weights and then to # the hardcoded equal-weight default. Keys must be a subset of AXIS_NAMES. @@ -240,9 +248,13 @@ def extract_from_artifacts(self, artifacts_text: str) -> dict: def to_prompt_section(self) -> str: lines = [f"Experiment type: {self.name}"] if self.expected_params: - lines.append(f"Expected params (inputs, NOT measurements): {', '.join(self.expected_params)}") + lines.append( + f"Expected params (inputs, NOT measurements): {', '.join(self.expected_params)}" + ) if self.expected_metrics: - lines.append(f"Expected metrics (measurements): {', '.join(self.expected_metrics)}") + lines.append( + f"Expected metrics (measurements): {', '.join(self.expected_metrics)}" + ) if self.scoring_guide: lines.append(f"Domain-specific scoring guide:\n{self.scoring_guide}") return "\n".join(lines) @@ -263,6 +275,7 @@ class LLMEvaluator: @staticmethod def _load_base_system() -> str: from ari.prompts import FilesystemPromptLoader + text = FilesystemPromptLoader().load("evaluator/extract_metrics") # The Python constant did not have a trailing newline; the file # storage layer may add one — strip a single trailing ``\n`` so @@ -278,6 +291,7 @@ def _load_base_system_hash() -> str: # ``load_versioned`` / the snapshot test compute. Never renders or # calls an LLM. from ari.prompts import FilesystemPromptLoader + return FilesystemPromptLoader().load_versioned("evaluator/extract_metrics")[1] BASE_SYSTEM = _load_base_system.__func__() # type: ignore[func-returns-value] @@ -331,10 +345,9 @@ def __init__( # ``_refresh_axes_if_needed`` hook in evaluate() picks up plan # axes once the root node has produced idea.json. from ari.evaluator.dynamic_axes import build_axes_for_run + self._dynamic_axes = list( - build_axes_for_run( - rubric=rubric, idea_data=self._read_idea_data() - ) + build_axes_for_run(rubric=rubric, idea_data=self._read_idea_data()) ) self._axis_names = tuple(a.name for a in self._dynamic_axes) self._axes_idea_mtime = self._idea_json_signature() @@ -348,6 +361,7 @@ def __init__( def _idea_json_path(self): from pathlib import Path as _Path + if not self._checkpoint_dir: return None return _Path(self._checkpoint_dir) / "idea.json" @@ -366,6 +380,7 @@ def _idea_json_signature(self) -> str | None: return None try: import hashlib + data = p.read_bytes() mt = p.stat().st_mtime h = hashlib.md5(data).hexdigest()[:16] @@ -399,8 +414,11 @@ def _refresh_axes_if_needed(self) -> None: return # cached (signature unchanged) try: from ari.evaluator.dynamic_axes import build_axes_for_run + self._dynamic_axes = list( - build_axes_for_run(rubric=self._rubric, idea_data=self._read_idea_data()) + build_axes_for_run( + rubric=self._rubric, idea_data=self._read_idea_data() + ) ) self._axis_names = tuple(a.name for a in self._dynamic_axes) self._axes_idea_mtime = cur_sig @@ -436,6 +454,7 @@ def _build_system_prompt(self) -> str: # is no longer duplicated between code and the prompt file. from ari.evaluator.dynamic_axes import axes_to_prompt_section from ari.prompts import FilesystemPromptLoader as _PL_pr + _pr_text, _pr_hash = _PL_pr().load_versioned("evaluator/peer_review") base = _pr_text.format( axes_block=axes_to_prompt_section(self._dynamic_axes), @@ -458,8 +477,11 @@ def _build_system_prompt(self) -> str: else: system = head + f"\n\nDomain context:\n{spec_section}" _record_prompt_use( - "evaluator/peer_review", _pr_hash, rendered_text=system, - model=self.model, phase="evaluation", + "evaluator/peer_review", + _pr_hash, + rendered_text=system, + model=self.model, + phase="evaluation", ) return system @@ -477,8 +499,11 @@ def _build_system_prompt(self) -> str: + f"\n\nDomain context:\n{spec_section}" ) _record_prompt_use( - "evaluator/extract_metrics", self.BASE_SYSTEM_HASH, rendered_text=system, - model=self.model, phase="evaluation", + "evaluator/extract_metrics", + self.BASE_SYSTEM_HASH, + rendered_text=system, + model=self.model, + phase="evaluation", ) return system @@ -516,13 +541,17 @@ def _build_score_context(self) -> str: ) return "\n".join(lines) + "\n\n" - def _record_score(self, node_id: str | None, score: float, label: str | None) -> None: + def _record_score( + self, node_id: str | None, score: float, label: str | None + ) -> None: """Record a freshly assigned score so future evaluations can calibrate.""" if not node_id or score is None: return try: entry = { - "node_id": (str(node_id)[-8:] if len(str(node_id)) > 8 else str(node_id)), + "node_id": ( + str(node_id)[-8:] if len(str(node_id)) > 8 else str(node_id) + ), "score": float(score), "label": str(label or ""), } @@ -546,6 +575,7 @@ def evaluate_sync( import asyncio import concurrent.futures import logging + _log = logging.getLogger(__name__) def _run_in_thread(): @@ -554,8 +584,9 @@ def _run_in_thread(): asyncio.set_event_loop(loop) try: return loop.run_until_complete( - self.evaluate(goal, artifacts, summary, - node_id=node_id, node_label=node_label) + self.evaluate( + goal, artifacts, summary, node_id=node_id, node_label=node_label + ) ) finally: loop.close() @@ -572,17 +603,25 @@ def _run_in_thread(): with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: future = pool.submit(_run_in_thread) result = future.result(timeout=120) - _log.info("evaluate_sync (thread): metrics=%s", result.get("metrics", {})) + _log.info( + "evaluate_sync (thread): metrics=%s", result.get("metrics", {}) + ) return result else: return asyncio.run( - self.evaluate(goal, artifacts, summary, - node_id=node_id, node_label=node_label) + self.evaluate( + goal, artifacts, summary, node_id=node_id, node_label=node_label + ) ) except Exception as e: _log.warning("evaluate_sync failed: %s", e) - return {"score": None, "reason": f"sync error: {e}", - "has_real_data": False, "has_paper_section": False, "metrics": {}} + return { + "score": None, + "reason": f"sync error: {e}", + "has_real_data": False, + "has_paper_section": False, + "metrics": {}, + } async def evaluate( self, @@ -651,7 +690,9 @@ async def evaluate( # Supplement with raw artifact text via MetricSpec artifact_extractor # (domain-specific fallback when LLM misses some metrics) artifacts_text = " ".join( - (a.get("stdout", "") or a.get("content", "") or str(a)) if isinstance(a, dict) else str(a) + (a.get("stdout", "") or a.get("content", "") or str(a)) + if isinstance(a, dict) + else str(a) for a in (artifacts if isinstance(artifacts, list) else []) ) extra_metrics = self.metric_spec.extract_from_artifacts(artifacts_text) @@ -689,12 +730,11 @@ async def evaluate( from ari.migrations.v05_to_v07.legacy_axes import ( legacy_uniform_axis_scores, ) + axis_scores = legacy_uniform_axis_scores(data, iter_names) weights = self._resolve_axis_weights() - composite = self._compose_fn( - axis_scores, weights, axis_names=iter_names - ) + composite = self._compose_fn(axis_scores, weights, axis_names=iter_names) comparison_found = bool(data.get("comparison_found", False)) if composite > 0: @@ -714,19 +754,34 @@ async def evaluate( try: import os as _os_rj from pathlib import Path as _Path_rj + _wd = _os_rj.environ.get("ARI_WORK_DIR", "") if _wd: _rj_path = _Path_rj(_wd) / "results.json" if _rj_path.is_file(): - _rj_meas = (json.loads(_rj_path.read_text()) or {}).get("measurements") - if isinstance(_rj_meas, dict): - for _k, _v in _rj_meas.items(): - if isinstance(_k, str) and isinstance(_v, (int, float)) and not isinstance(_v, bool): - extracted_metrics[_k] = float(_v) - measurements_dict[_k] = float(_v) - _rj_has_real = True - except Exception: - pass + _rj_document = json.loads(_rj_path.read_text()) or {} + _rj_set = parse_measurement_document(_rj_document) + for _record in _rj_set.measurements: + extracted_metrics[_record.metric_id] = float(_record.value) + measurements_dict[_record.metric_id] = float(_record.value) + _rj_has_real = True + extracted_metrics["_measurement_schema"] = ( + _rj_set.schema_version + ) + extracted_metrics["_measurement_compatibility"] = ( + measurement_document_format(_rj_document) + ) + extracted_metrics["_measurement_admissible"] = bool( + _rj_set.measurements + ) and all( + record.unit_status == "declared" + and record.execution_status == "completed" + and record.exit_code == 0 + and bool(record.artifact_digests) + for record in _rj_set.measurements + ) + except (MeasurementDocumentError, OSError, ValueError) as exc: + logger.warning("Ignoring invalid results.json: %s", exc) # Typed views — present iff the LLM honoured the new contract. # Stored under reserved underscore keys so they don't collide diff --git a/ari-core/ari/execution.py b/ari-core/ari/execution.py new file mode 100644 index 00000000..0b3acf31 --- /dev/null +++ b/ari-core/ari/execution.py @@ -0,0 +1,1238 @@ +"""Closed workspace and local-process execution contracts. + +This module owns the low-level process-group, environment, resource, log, and +workspace primitives shared by Skills. It intentionally does not own scheduler +or container lifecycle; those adapters bind their own substrate identity to the +same request/result records. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import secrets +import signal +import stat +import subprocess +import tempfile +import threading +import time +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +EXECUTION_REQUEST_V1 = "ari.execution-request/v1" +EXECUTION_RESULT_V1 = "ari.execution-result/v1" +MEASUREMENT_SET_V1 = "ari.measurement-set/v1" +WORKSPACE_REF_V1 = "ari.workspace-ref/v1" +_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") +_SECRET_NAME_RE = re.compile( + r"(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|PRIVATE_?KEY|CREDENTIAL)", + re.IGNORECASE, +) +_SAFE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$") +_DEFAULT_PATH = "/usr/local/bin:/usr/bin:/bin" +_PLATFORM_ENV = ("LANG", "LC_ALL", "SSL_CERT_DIR", "SSL_CERT_FILE") +_MAX_INPUT_BYTES = 256 * 1024 * 1024 + + +class ExecutionPolicyError(RuntimeError): + """The request cannot be executed under its declared security policy.""" + + +class MeasurementDocumentError(ValueError): + """A typed or supported legacy measurement document is inconsistent.""" + + +MeasurementDocumentFormat = Literal[ + "canonical", "legacy-v1", "legacy-unversioned" +] + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _sha256_bytes(value: bytes) -> str: + return "sha256:" + hashlib.sha256(value).hexdigest() + + +def _canonical_digest(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return _sha256_bytes(payload) + + +class WorkspaceRefV1(BaseModel): + """One canonical writable root with no caller-controlled escape path.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.workspace-ref/v1"] = WORKSPACE_REF_V1 + root: str + + @field_validator("root") + @classmethod + def _canonical_root(cls, value: str) -> str: + path = Path(value) + if not path.is_absolute(): + raise ValueError("workspace root must be absolute") + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.is_symlink() or not path.is_dir(): + raise ValueError("workspace root must be a real directory") + return str(path.resolve(strict=True)) + + def _parts(self, value: str, *, allow_absolute: bool) -> tuple[str, ...]: + path = Path(value) + root = Path(self.root) + if path.is_absolute(): + if not allow_absolute: + raise ExecutionPolicyError("absolute workspace paths are not allowed") + try: + path = path.relative_to(root) + except ValueError as exc: + raise ExecutionPolicyError("path escapes the workspace root") from exc + pure = PurePosixPath(path.as_posix()) + if not pure.parts or pure == PurePosixPath("."): + raise ExecutionPolicyError("workspace path must name a file") + if any(part in {"", ".", ".."} for part in pure.parts): + raise ExecutionPolicyError("workspace path traversal is not allowed") + return tuple(pure.parts) + + def resolve( + self, + value: str, + *, + allow_absolute: bool = True, + must_exist: bool = True, + require_file: bool = False, + ) -> Path: + parts = self._parts(value, allow_absolute=allow_absolute) + current = Path(self.root) + for index, part in enumerate(parts): + current = current / part + if current.is_symlink(): + raise ExecutionPolicyError("workspace symlinks are not allowed") + if current.exists() and index < len(parts) - 1 and not current.is_dir(): + raise ExecutionPolicyError("workspace parent is not a directory") + if must_exist and not current.exists(): + raise FileNotFoundError(current) + if require_file and (not current.is_file() or current.is_symlink()): + raise ExecutionPolicyError("workspace path is not a regular file") + try: + current.resolve(strict=must_exist).relative_to(Path(self.root)) + except ValueError as exc: + raise ExecutionPolicyError("resolved path escapes the workspace") from exc + return current + + def read_bytes(self, value: str, *, max_bytes: int) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be positive") + fd = self.open_read_fd(value) + try: + size = os.fstat(fd).st_size + if size > max_bytes: + raise ExecutionPolicyError("workspace file exceeds the read limit") + chunks: list[bytes] = [] + remaining = max_bytes + 1 + while remaining: + chunk = os.read(fd, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + finally: + os.close(fd) + if len(payload) > max_bytes: + raise ExecutionPolicyError("workspace file grew beyond the read limit") + return payload + + def open_read_fd(self, value: str) -> int: + """Open a regular file through dirfds so path swaps cannot escape root.""" + + parts = self._parts(value, allow_absolute=True) + directory_fd = os.open( + self.root, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + for part in parts[:-1]: + child_fd = os.open( + part, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + os.close(directory_fd) + directory_fd = child_fd + fd = os.open( + parts[-1], + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + status = os.fstat(fd) + if not stat.S_ISREG(status.st_mode): + os.close(fd) + raise ExecutionPolicyError("workspace path is not a regular file") + return fd + except OSError as exc: + raise ExecutionPolicyError( + "workspace path changed or contains a symlink" + ) from exc + finally: + os.close(directory_fd) + + def file_digest(self, value: str) -> str: + fd = self.open_read_fd(value) + digest = hashlib.sha256() + try: + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + finally: + os.close(fd) + return "sha256:" + digest.hexdigest() + + def ensure_directory(self, value: str) -> Path: + """Create a caller-selected subdirectory without following symlinks.""" + + parts = self._parts(value, allow_absolute=True) + directory_fd = os.open( + self.root, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + for part in parts: + try: + os.mkdir(part, mode=0o700, dir_fd=directory_fd) + except FileExistsError: + pass + child_fd = os.open( + part, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + os.close(directory_fd) + directory_fd = child_fd + except OSError as exc: + raise ExecutionPolicyError( + "workspace directory changed or contains a symlink" + ) from exc + finally: + os.close(directory_fd) + return Path(self.root).joinpath(*parts) + + def atomic_write_bytes(self, value: str, payload: bytes) -> Path: + parts = self._parts(value, allow_absolute=False) + directory_fd = os.open( + self.root, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + for part in parts[:-1]: + try: + os.mkdir(part, mode=0o700, dir_fd=directory_fd) + except FileExistsError: + pass + child_fd = os.open( + part, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + os.close(directory_fd) + directory_fd = child_fd + final_name = parts[-1] + try: + status = os.stat(final_name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + status = None + if status is not None and not stat.S_ISREG(status.st_mode): + raise ExecutionPolicyError("write target is not a regular file") + temporary = f".ari-write-{secrets.token_hex(12)}.tmp" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(temporary, flags, 0o600, dir_fd=directory_fd) + try: + try: + view = memoryview(payload) + while view: + written = os.write(fd, view) + view = view[written:] + os.fsync(fd) + finally: + os.close(fd) + os.replace( + temporary, + final_name, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + ) + os.fsync(directory_fd) + except Exception: + try: + os.unlink(temporary, dir_fd=directory_fd) + except OSError: + pass + raise + finally: + os.close(directory_fd) + return Path(self.root).joinpath(*parts) + + def atomic_write_text(self, value: str, text: str) -> Path: + return self.atomic_write_bytes(value, text.encode("utf-8")) + + +class ExecutionLimitsV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + cpu_seconds: int | None = Field(default=None, ge=1, le=86_400) + memory_bytes: int | None = Field(default=None, ge=16 * 1024 * 1024) + max_processes: int | None = Field(default=None, ge=1, le=65_536) + max_output_bytes: int = Field(default=64 * 1024 * 1024, ge=1024, le=1024**3) + + +class ResourceLimitReportV1(BaseModel): + """What the launcher actually enforced, distinct from requested limits.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + substrate: Literal["posix-kernel", "external-unverified"] + wall_time: Literal["executor", "external-unverified"] + process_group: Literal["executor", "external-unverified"] + enforced: list[Literal["cpu", "memory", "processes", "output"]] = Field( + default_factory=list + ) + + @field_validator("enforced") + @classmethod + def _unique_enforced(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise ValueError("enforced resource limits must be unique") + return value + + @model_validator(mode="after") + def _report_is_coherent(self) -> "ResourceLimitReportV1": + if self.substrate == "external-unverified": + if ( + self.wall_time != "external-unverified" + or self.process_group != "external-unverified" + or self.enforced + ): + raise ValueError("external resource report cannot claim enforcement") + elif ( + self.wall_time != "executor" + or self.process_group != "executor" + or "output" not in self.enforced + ): + raise ValueError("POSIX resource report is incomplete") + return self + + +class ContainerIdentityV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + runtime: Literal["docker", "singularity", "apptainer", "unknown"] + reference: str + digest: str | None = None + resolution_status: Literal["resolved", "unresolved"] + + @field_validator("digest") + @classmethod + def _digest(cls, value: str | None) -> str | None: + if value is not None and not _DIGEST_RE.fullmatch(value): + raise ValueError("container digest must be SHA-256") + return value + + @model_validator(mode="after") + def _status_matches_digest(self) -> "ContainerIdentityV1": + if (self.digest is not None) != (self.resolution_status == "resolved"): + raise ValueError("container resolution status and digest differ") + return self + + +class ExecutionRequestV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.execution-request/v1"] = EXECUTION_REQUEST_V1 + workspace: WorkspaceRefV1 + argv: list[str] | None = Field(default=None, min_length=1, max_length=256) + shell_command: str | None = Field(default=None, max_length=100_000) + timeout_seconds: float = Field(default=60, gt=0, le=86_400) + environment: dict[str, str] = Field(default_factory=dict, max_length=128) + limits: ExecutionLimitsV1 = Field(default_factory=ExecutionLimitsV1) + network: Literal["inherit", "deny"] = "inherit" + request_id: str | None = None + input_digests: dict[str, str] = Field(default_factory=dict, max_length=1_024) + container: ContainerIdentityV1 | None = None + + @field_validator("argv") + @classmethod + def _argv(cls, value: list[str] | None) -> list[str] | None: + if value is not None and any( + not isinstance(item, str) or "\x00" in item or len(item) > 100_000 + for item in value + ): + raise ValueError("execution argv contains an invalid item") + return value + + @field_validator("shell_command") + @classmethod + def _shell(cls, value: str | None) -> str | None: + if value is not None and (not value.strip() or "\x00" in value): + raise ValueError("shell command is empty or invalid") + return value + + @field_validator("environment") + @classmethod + def _environment(cls, value: dict[str, str]) -> dict[str, str]: + for name, item in value.items(): + if ( + not _ENV_NAME_RE.fullmatch(name) + or _SECRET_NAME_RE.search(name) + or not isinstance(item, str) + or "\x00" in item + ): + raise ValueError("execution environment contains an unsafe entry") + return dict(sorted(value.items())) + + @field_validator("request_id") + @classmethod + def _request_id(cls, value: str | None) -> str | None: + if value is not None and not _SAFE_ID_RE.fullmatch(value): + raise ValueError("execution request_id is invalid") + return value + + @field_validator("input_digests") + @classmethod + def _input_digests(cls, value: dict[str, str]) -> dict[str, str]: + for path, digest in value.items(): + pure = PurePosixPath(path) + if ( + not path + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + ): + raise ValueError("execution input digest path is unsafe") + if not _DIGEST_RE.fullmatch(digest): + raise ValueError("execution input digests must be SHA-256") + return dict(sorted(value.items())) + + @model_validator(mode="after") + def _one_command(self) -> "ExecutionRequestV1": + if (self.argv is None) == (self.shell_command is None): + raise ValueError("execution requires exactly one of argv or shell_command") + return self + + @property + def execution_identity(self) -> str: + payload = self.model_dump(mode="json") + payload.pop("request_id", None) + return _canonical_digest(payload) + + +class ExecutionArtifactV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + logical_role: Literal["stdout", "stderr"] + relative_path: str + digest: str + size_bytes: int = Field(ge=0) + media_type: Literal["text/plain; charset=utf-8"] = "text/plain; charset=utf-8" + + @field_validator("digest") + @classmethod + def _digest(cls, value: str) -> str: + if not _DIGEST_RE.fullmatch(value): + raise ValueError("execution artifact digest must be SHA-256") + return value + + @field_validator("relative_path") + @classmethod + def _relative_path(cls, value: str) -> str: + pure = PurePosixPath(value) + if ( + not value + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + ): + raise ValueError("execution artifact path must be safe and relative") + return value + + +class ExecutionResultV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.execution-result/v1"] = EXECUTION_RESULT_V1 + status: Literal["completed", "failed", "timed_out", "cancelled"] + execution_identity: str + request_id: str | None = None + attempt_id: str + exit_code: int | None + started_at: str + completed_at: str + stdout_preview: str + stderr_preview: str + stdout_truncated: bool + stderr_truncated: bool + artifacts: list[ExecutionArtifactV1] + environment_names: list[str] + network: Literal["inherit", "deny"] + network_report: Literal["inherited", "isolated", "external-unverified"] + limits: ExecutionLimitsV1 + limit_report: ResourceLimitReportV1 + input_digests: dict[str, str] = Field(default_factory=dict) + input_bindings: dict[ + str, + Literal["immutable-snapshot", "verified-at-launch", "external-unverified"], + ] = Field(default_factory=dict) + container: ContainerIdentityV1 | None = None + + @field_validator("execution_identity") + @classmethod + def _identity(cls, value: str) -> str: + if not _DIGEST_RE.fullmatch(value): + raise ValueError("execution identity must be SHA-256") + return value + + @field_validator("attempt_id") + @classmethod + def _attempt_id(cls, value: str) -> str: + if not _SAFE_ID_RE.fullmatch(value): + raise ValueError("execution attempt_id is invalid") + return value + + @field_validator("started_at", "completed_at") + @classmethod + def _timestamp(cls, value: str) -> str: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("execution timestamp must be ISO-8601") from exc + if parsed.utcoffset() is None: + raise ValueError("execution timestamp must include a timezone") + return value + + @field_validator("environment_names") + @classmethod + def _environment_names(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)) or any( + not _ENV_NAME_RE.fullmatch(name) or _SECRET_NAME_RE.search(name) + for name in value + ): + raise ValueError("execution environment names are unsafe or duplicated") + return value + + @field_validator("input_digests") + @classmethod + def _result_input_digests(cls, value: dict[str, str]) -> dict[str, str]: + for path, digest in value.items(): + pure = PurePosixPath(path) + if ( + not path + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + or not _DIGEST_RE.fullmatch(digest) + ): + raise ValueError("execution result input digest is invalid") + return value + + @model_validator(mode="after") + def _input_reports_match(self) -> "ExecutionResultV1": + if set(self.input_bindings) != set(self.input_digests): + raise ValueError("execution input binding report is incomplete") + roles = [artifact.logical_role for artifact in self.artifacts] + if sorted(roles) != ["stderr", "stdout"]: + raise ValueError("execution result requires one stdout and one stderr artifact") + paths = [artifact.relative_path for artifact in self.artifacts] + if len(paths) != len(set(paths)): + raise ValueError("execution artifact paths must be unique") + started = datetime.fromisoformat(self.started_at.replace("Z", "+00:00")) + completed = datetime.fromisoformat(self.completed_at.replace("Z", "+00:00")) + if completed < started: + raise ValueError("execution completion predates its start") + return self + + +def build_minimal_environment( + explicit: dict[str, str] | None = None, +) -> dict[str, str]: + """Build the non-secret environment inherited by reviewed launchers.""" + + explicit = explicit or {} + for name, item in explicit.items(): + if ( + not _ENV_NAME_RE.fullmatch(name) + or _SECRET_NAME_RE.search(name) + or not isinstance(item, str) + or "\x00" in item + ): + raise ExecutionPolicyError("execution environment contains an unsafe entry") + environment = { + "HOME": "/nonexistent", + "LOGNAME": "ari-executor", + "USER": "ari-executor", + "SHELL": "", + "TERM": "dumb", + "PATH": _DEFAULT_PATH, + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONUNBUFFERED": "1", + } + for name in _PLATFORM_ENV: + value = os.environ.get(name) + if value: + environment[name] = value + environment.update(explicit) + return environment + + +def _resource_limit_report(limits: ExecutionLimitsV1) -> ResourceLimitReportV1: + if os.name != "posix": + raise ExecutionPolicyError("local execution requires POSIX resource controls") + try: + import resource + except ImportError as exc: # pragma: no cover - POSIX Python always provides it + raise ExecutionPolicyError("POSIX resource controls are unavailable") from exc + + requested = ["output"] + required = {"output": "RLIMIT_FSIZE"} + if limits.cpu_seconds is not None: + requested.append("cpu") + required["cpu"] = "RLIMIT_CPU" + if limits.memory_bytes is not None: + requested.append("memory") + required["memory"] = "RLIMIT_AS" + if limits.max_processes is not None: + requested.append("processes") + required["processes"] = "RLIMIT_NPROC" + missing = [name for name, attr in required.items() if not hasattr(resource, attr)] + if missing: + raise ExecutionPolicyError( + "kernel cannot enforce requested limits: " + ", ".join(sorted(missing)) + ) + return ResourceLimitReportV1( + substrate="posix-kernel", + wall_time="executor", + process_group="executor", + enforced=requested, + ) + + +def _preexec(limits: ExecutionLimitsV1) -> None: + os.setsid() + import resource + + if limits.cpu_seconds is not None: + resource.setrlimit( + resource.RLIMIT_CPU, (limits.cpu_seconds, limits.cpu_seconds) + ) + if limits.memory_bytes is not None: + resource.setrlimit( + resource.RLIMIT_AS, (limits.memory_bytes, limits.memory_bytes) + ) + if limits.max_processes is not None: + _soft, hard = resource.getrlimit(resource.RLIMIT_NPROC) + cap = ( + limits.max_processes + if hard == resource.RLIM_INFINITY + else min(hard, limits.max_processes) + ) + resource.setrlimit(resource.RLIMIT_NPROC, (cap, cap)) + resource.setrlimit( + resource.RLIMIT_FSIZE, + (limits.max_output_bytes, limits.max_output_bytes), + ) + + +def _terminate_group(process: subprocess.Popen[Any]) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except OSError: + pass + try: + process.wait(timeout=2) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + process.kill() + process.wait() + + +def _preview(value: bytes, limit: int) -> tuple[str, bool]: + text = value.decode("utf-8", errors="replace") + if len(text) <= limit: + return text, False + marker_template = "\n... [{} chars truncated; full log is an artifact] ...\n" + marker = marker_template.format(len(text)) + if len(marker) >= limit: + return marker[:limit], True + content_budget = limit - len(marker) + omitted = len(text) - content_budget + marker = marker_template.format(omitted) + # The omitted digit count can make the final marker a few characters wider. + content_budget = max(0, limit - len(marker)) + head = (content_budget + 1) // 2 + tail = content_budget - head + return text[:head] + marker + (text[-tail:] if tail else ""), True + + +def _store_log( + workspace: WorkspaceRefV1, role: Literal["stdout", "stderr"], value: bytes +) -> ExecutionArtifactV1: + digest = _sha256_bytes(value) + name = f".ari-execution/{digest.removeprefix('sha256:')}.{role}.log" + path = workspace.atomic_write_bytes(name, value) + return ExecutionArtifactV1( + logical_role=role, + relative_path=path.relative_to(workspace.root).as_posix(), + digest=digest, + size_bytes=len(value), + ) + + +def _snapshot_inputs( + request: ExecutionRequestV1, + temp: Path, +) -> tuple[list[str] | None, dict[str, str]]: + """Verify declared inputs and bind argv file operands to private snapshots.""" + + argv = list(request.argv) if request.argv is not None else None + bindings: dict[str, str] = {} + for relative_path, expected_digest in request.input_digests.items(): + payload = request.workspace.read_bytes( + relative_path, max_bytes=_MAX_INPUT_BYTES + ) + actual_digest = _sha256_bytes(payload) + if actual_digest != expected_digest: + raise ExecutionPolicyError( + f"input digest changed before launch: {relative_path}" + ) + snapshot_dir = temp / "inputs" / actual_digest.removeprefix("sha256:") + snapshot_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + snapshot = snapshot_dir / Path(relative_path).name + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + try: + fd = os.open(snapshot, flags, 0o400) + except FileExistsError: + if snapshot.read_bytes() != payload: + raise ExecutionPolicyError("content-addressed input snapshot collision") + else: + try: + view = memoryview(payload) + while view: + written = os.write(fd, view) + view = view[written:] + os.fsync(fd) + finally: + os.close(fd) + binding = "verified-at-launch" + if argv is not None: + absolute = str(Path(request.workspace.root) / relative_path) + replacements = {relative_path, absolute} + replaced = False + for index, item in enumerate(argv): + if item in replacements: + argv[index] = str(snapshot) + replaced = True + if replaced: + binding = "immutable-snapshot" + bindings[relative_path] = binding + return argv, bindings + + +def execute_local( + request: ExecutionRequestV1, + *, + cancel_event: threading.Event | None = None, + network_isolation_verified: bool = False, +) -> ExecutionResultV1: + """Execute one exact request without a parent environment copy. + + A request for network denial is rejected unless a reviewed caller has + already wrapped the command in an isolation boundary (for example Docker's + ``--network none``). Merely asking for denial never changes provenance. + """ + + if request.network == "deny" and not network_isolation_verified: + raise ExecutionPolicyError( + "host execution cannot prove network denial; use a reviewed isolated substrate" + ) + limit_report = _resource_limit_report(request.limits) + started_at = _now() + attempt_id = secrets.token_hex(16) + with tempfile.TemporaryDirectory(prefix="ari-execution-log-") as temp_text: + temp = Path(temp_text) + argv, input_bindings = _snapshot_inputs(request, temp) + command = ( + argv + if argv is not None + else [ + "/bin/bash", + "--noprofile", + "--norc", + "-c", + request.shell_command or "", + ] + ) + stdout_path = temp / "stdout" + stderr_path = temp / "stderr" + terminal_reason: Literal["exited", "timed_out", "cancelled"] = "exited" + with ( + stdout_path.open("wb") as stdout_stream, + stderr_path.open("wb") as stderr_stream, + ): + process = subprocess.Popen( + command, + cwd=request.workspace.root, + env=build_minimal_environment(request.environment), + stdin=subprocess.DEVNULL, + stdout=stdout_stream, + stderr=stderr_stream, + shell=False, + close_fds=True, + preexec_fn=lambda: _preexec(request.limits), + ) + deadline = time.monotonic() + request.timeout_seconds + while process.poll() is None: + if cancel_event is not None and cancel_event.is_set(): + terminal_reason = "cancelled" + _terminate_group(process) + break + remaining = deadline - time.monotonic() + if remaining <= 0: + terminal_reason = "timed_out" + _terminate_group(process) + break + time.sleep(min(0.05, remaining)) + stdout = stdout_path.read_bytes() + stderr = stderr_path.read_bytes() + stdout_preview, stdout_truncated = _preview(stdout, 4_000) + stderr_preview, stderr_truncated = _preview(stderr, 2_000) + artifacts = [ + _store_log(request.workspace, "stdout", stdout), + _store_log(request.workspace, "stderr", stderr), + ] + if terminal_reason == "timed_out": + status: Literal["completed", "failed", "timed_out", "cancelled"] = "timed_out" + elif terminal_reason == "cancelled": + status = "cancelled" + else: + status = "completed" if process.returncode == 0 else "failed" + return ExecutionResultV1( + status=status, + execution_identity=request.execution_identity, + request_id=request.request_id, + attempt_id=attempt_id, + exit_code=None if terminal_reason != "exited" else process.returncode, + started_at=started_at, + completed_at=_now(), + stdout_preview=stdout_preview, + stderr_preview=stderr_preview, + stdout_truncated=stdout_truncated, + stderr_truncated=stderr_truncated, + artifacts=artifacts, + environment_names=sorted(build_minimal_environment(request.environment)), + network=request.network, + network_report="isolated" if request.network == "deny" else "inherited", + limits=request.limits, + limit_report=limit_report, + input_digests=request.input_digests, + input_bindings=input_bindings, + container=request.container, + ) + + +def record_completed_execution( + request: ExecutionRequestV1, + *, + stdout: str | bytes, + stderr: str | bytes, + returncode: int, + started_at: str | None = None, + inputs_verified: bool = False, + network_verified: bool = False, +) -> ExecutionResultV1: + """Normalize an already-executed reviewed substrate into the same record.""" + + stdout_bytes = stdout.encode("utf-8") if isinstance(stdout, str) else stdout + stderr_bytes = stderr.encode("utf-8") if isinstance(stderr, str) else stderr + if ( + len(stdout_bytes) > request.limits.max_output_bytes + or len(stderr_bytes) > request.limits.max_output_bytes + ): + raise ExecutionPolicyError("external execution log exceeds the declared limit") + stdout_preview, stdout_truncated = _preview(stdout_bytes, 4_000) + stderr_preview, stderr_truncated = _preview(stderr_bytes, 2_000) + return ExecutionResultV1( + status="completed" if returncode == 0 else "failed", + execution_identity=request.execution_identity, + request_id=request.request_id, + attempt_id=secrets.token_hex(16), + exit_code=returncode, + started_at=started_at or _now(), + completed_at=_now(), + stdout_preview=stdout_preview, + stderr_preview=stderr_preview, + stdout_truncated=stdout_truncated, + stderr_truncated=stderr_truncated, + artifacts=[ + _store_log(request.workspace, "stdout", stdout_bytes), + _store_log(request.workspace, "stderr", stderr_bytes), + ], + environment_names=sorted(build_minimal_environment(request.environment)), + network=request.network, + network_report=( + "isolated" + if request.network == "deny" and network_verified + else "external-unverified" + ), + limits=request.limits, + limit_report=ResourceLimitReportV1( + substrate="external-unverified", + wall_time="external-unverified", + process_group="external-unverified", + ), + input_digests=request.input_digests, + input_bindings={ + path: "verified-at-launch" if inputs_verified else "external-unverified" + for path in request.input_digests + }, + container=request.container, + ) + + +class MeasurementRecordV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + metric_id: str + value: int | float + unit: str | None = None + unit_status: Literal["declared", "missing"] + provenance: str | None = None + parameters: dict[str, Any] = Field(default_factory=dict) + artifact_digests: list[str] = Field(default_factory=list) + execution_identity: str | None = None + execution_attempt_id: str | None = None + execution_status: Literal[ + "completed", "failed", "timed_out", "cancelled", "unreported" + ] = "unreported" + exit_code: int | None = None + + @field_validator("metric_id") + @classmethod + def _metric_id(cls, value: str) -> str: + if not value.strip() or len(value) > 256 or value.startswith("_"): + raise ValueError("measurement metric_id is invalid") + return value + + @field_validator("value", mode="before") + @classmethod + def _value(cls, value: Any) -> int | float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError("measurement value must be a finite number") + return value + + @field_validator("unit") + @classmethod + def _unit(cls, value: str | None) -> str | None: + if value is not None and (not value.strip() or len(value) > 100): + raise ValueError("measurement unit is invalid") + return value + + @field_validator("provenance") + @classmethod + def _provenance(cls, value: str | None) -> str | None: + if value is not None and (not value.strip() or len(value) > 1_000): + raise ValueError("measurement provenance is invalid") + return value + + @field_validator("parameters") + @classmethod + def _parameters(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("measurement parameters must be finite JSON") from exc + return value + + @field_validator("artifact_digests") + @classmethod + def _artifact_digests(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)) or any( + not _DIGEST_RE.fullmatch(item) for item in value + ): + raise ValueError("measurement artifact digests must be unique SHA-256") + return sorted(value) + + @field_validator("execution_identity") + @classmethod + def _execution_identity(cls, value: str | None) -> str | None: + if value is not None and not _DIGEST_RE.fullmatch(value): + raise ValueError("measurement execution identity must be SHA-256") + return value + + @field_validator("execution_attempt_id") + @classmethod + def _execution_attempt_id(cls, value: str | None) -> str | None: + if value is not None and not _SAFE_ID_RE.fullmatch(value): + raise ValueError("measurement execution attempt ID is invalid") + return value + + @model_validator(mode="after") + def _unit_status(self) -> "MeasurementRecordV1": + if (self.unit is not None) != (self.unit_status == "declared"): + raise ValueError("measurement unit status differs from unit") + if self.execution_status == "unreported": + if ( + self.execution_identity is not None + or self.execution_attempt_id is not None + or self.exit_code is not None + ): + raise ValueError( + "unreported execution cannot carry identity, attempt, or exit code" + ) + elif self.execution_identity is None or self.execution_attempt_id is None: + raise ValueError( + "reported execution requires an execution identity and attempt ID" + ) + if self.execution_status in {"completed", "failed"}: + if self.exit_code is None: + raise ValueError("completed or failed execution requires an exit code") + elif self.exit_code is not None: + raise ValueError("non-exited execution cannot carry an exit code") + return self + + +class MeasurementSetV1(BaseModel): + """Canonical typed scientific-measurement document.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.measurement-set/v1"] = MEASUREMENT_SET_V1 + parameters: dict[str, Any] = Field(default_factory=dict) + measurements: list[MeasurementRecordV1] = Field(default_factory=list) + predictions: dict[str, Any] = Field(default_factory=dict) + scores: dict[str, Any] = Field(default_factory=dict) + artifact_digests: list[str] = Field(default_factory=list) + + @field_validator("parameters", "predictions", "scores") + @classmethod + def _json_groups(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("measurement groups must be finite JSON") from exc + return value + + @field_validator("artifact_digests") + @classmethod + def _set_artifacts(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)) or any( + not _DIGEST_RE.fullmatch(item) for item in value + ): + raise ValueError("measurement-set artifact digests must be unique SHA-256") + return sorted(value) + + @model_validator(mode="after") + def _disjoint_names(self) -> "MeasurementSetV1": + measurement_names = [item.metric_id for item in self.measurements] + if len(measurement_names) != len(set(measurement_names)): + raise ValueError("measurement IDs must be unique") + groups = [ + set(self.parameters), + set(measurement_names), + set(self.predictions), + set(self.scores), + ] + if any( + groups[left] & groups[right] + for left in range(4) + for right in range(left + 1, 4) + ): + raise ValueError( + "parameter, measurement, prediction, and score names overlap" + ) + record_artifacts = { + digest for record in self.measurements for digest in record.artifact_digests + } + if not record_artifacts.issubset(set(self.artifact_digests)): + raise ValueError("measurement artifact is absent from measurement set") + if any(record.parameters != self.parameters for record in self.measurements): + raise ValueError("measurement record parameters differ from measurement set") + return self + + +def measurement_document_format(document: dict[str, Any]) -> MeasurementDocumentFormat: + """Classify compatibility usage without parsing values a second time.""" + + if document.get("typed_schema_version") is not None: + return "canonical" + if document.get("schema_version") is None: + return "legacy-unversioned" + return "legacy-v1" + + +def _parse_canonical_measurement_document( + document: dict[str, Any], typed_version: Any +) -> MeasurementSetV1: + if typed_version != MEASUREMENT_SET_V1: + raise MeasurementDocumentError( + f"unsupported typed measurement schema: {typed_version!r}" + ) + projection_version = document.get("schema_version") + if projection_version is not None and ( + not isinstance(projection_version, str) + or projection_version.split(".", 1)[0] != "1" + ): + raise MeasurementDocumentError( + "typed measurement document projection is not v1" + ) + canonical = document.get("measurement_set") + if not isinstance(canonical, dict): + raise MeasurementDocumentError("typed measurement_set is missing") + try: + value = MeasurementSetV1.model_validate(canonical) + except Exception as exc: + raise MeasurementDocumentError("typed measurement_set is invalid") from exc + projections: tuple[tuple[str, Any], ...] = ( + ("params", value.parameters), + ( + "measurements", + {record.metric_id: record.value for record in value.measurements}, + ), + ("predictions", value.predictions), + ("scores", value.scores), + ( + "measurement_records", + [record.model_dump(mode="json") for record in value.measurements], + ), + ) + for key, expected in projections: + if key in document and document[key] != expected: + raise MeasurementDocumentError( + f"typed measurement projection differs at {key}" + ) + projected_provenance = { + record.metric_id: record.provenance + for record in value.measurements + if record.provenance is not None + } + if ( + "_provenance" in document + and document["_provenance"] != projected_provenance + ): + raise MeasurementDocumentError( + "typed measurement provenance projection differs" + ) + return value + + +def _parse_legacy_measurement_document(document: dict[str, Any]) -> MeasurementSetV1: + if "measurement_set" in document or "measurement_records" in document: + raise MeasurementDocumentError( + "canonical measurement fields require typed_schema_version" + ) + legacy_version = document.get("schema_version") + if legacy_version is not None and ( + not isinstance(legacy_version, str) + or legacy_version.split(".", 1)[0] != "1" + ): + raise MeasurementDocumentError( + f"unsupported legacy measurement schema: {legacy_version!r}" + ) + parameters = document.get("params", {}) + measurements = document.get("measurements", {}) + predictions = document.get("predictions", {}) + scores = document.get("scores", {}) + provenance = document.get("_provenance", {}) + if not all( + isinstance(group, dict) + for group in (parameters, measurements, predictions, scores, provenance) + ): + raise MeasurementDocumentError("legacy measurement groups must be objects") + if set(provenance) - set(measurements): + raise MeasurementDocumentError("legacy provenance names an unknown measurement") + try: + records = [ + MeasurementRecordV1( + metric_id=str(metric_id), + value=value, + unit=None, + unit_status="missing", + provenance=provenance.get(metric_id), + parameters=parameters, + ) + for metric_id, value in sorted(measurements.items()) + ] + return MeasurementSetV1( + parameters=parameters, + measurements=records, + predictions=predictions, + scores=scores, + ) + except Exception as exc: + raise MeasurementDocumentError( + "legacy measurement document is invalid" + ) from exc + + +def parse_measurement_document( + document: dict[str, Any], *, allow_legacy: bool = True +) -> MeasurementSetV1: + """Validate a canonical measurement set or migrate one supported v1 file. + + Historical typed documents are cross-checked against any flat fields they + contain so old and new readers cannot observe different values. Unversioned + and ``schema_version: 1.x`` files are read-only migration inputs; their + absent units and execution provenance stay explicitly absent. + """ + + if not isinstance(document, dict): + raise MeasurementDocumentError("measurement document must be an object") + typed_version = document.get("typed_schema_version") + if typed_version is not None: + return _parse_canonical_measurement_document(document, typed_version) + + if not allow_legacy: + raise MeasurementDocumentError("legacy measurement document is not admitted") + return _parse_legacy_measurement_document(document) + + +__all__ = [ + "ContainerIdentityV1", + "EXECUTION_REQUEST_V1", + "EXECUTION_RESULT_V1", + "ExecutionArtifactV1", + "ExecutionLimitsV1", + "ExecutionPolicyError", + "ExecutionRequestV1", + "ExecutionResultV1", + "MEASUREMENT_SET_V1", + "MeasurementRecordV1", + "MeasurementDocumentError", + "MeasurementDocumentFormat", + "MeasurementSetV1", + "ResourceLimitReportV1", + "WORKSPACE_REF_V1", + "WorkspaceRefV1", + "build_minimal_environment", + "execute_local", + "measurement_document_format", + "parse_measurement_document", + "record_completed_execution", +] diff --git a/ari-core/ari/figure_contract.py b/ari-core/ari/figure_contract.py new file mode 100644 index 00000000..1f80e5dc --- /dev/null +++ b/ari-core/ari/figure_contract.py @@ -0,0 +1,437 @@ +"""Digest-bound contracts for scientific figure specifications and artifacts. + +The contract intentionally accepts declarative plotting specifications only. +Model-generated Python, SVG, shell commands, and embedded image payloads are +not part of the public surface: a stochastic planner may select admitted fields, +but the fixed renderer remains the sole owner of numeric values and bytes. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import PurePosixPath +from typing import Any, ClassVar, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + + +FIGURE_SPEC_V1 = "ari.figure-spec/v1" +FIGURE_MANIFEST_V1 = "ari.figure-manifest/v1" +FIGURE_BATCH_V1 = "ari.figure-batch/v1" +FIGURE_FEEDBACK_V1 = "ari.figure-feedback/v1" +FIGURE_ARTIFACT_V1 = "ari.figure-artifact/v1" +FIGURE_ENVIRONMENT_V1 = "ari.figure-environment/v1" +LEGACY_FIGURE_BATCH_V0 = "ari.figure-batch/legacy-v0" +SHA256_PATTERN = r"^sha256:[0-9a-f]{64}$" +ZERO_DIGEST = "sha256:" + "0" * 64 +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + + +class FigureContractError(ValueError): + """A figure document is malformed, unbound, or scientifically unsafe.""" + + +def canonical_figure_digest(value: Any) -> str: + """Return the canonical SHA-256 identity of a JSON-compatible value.""" + + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _finite_json(value: Any, field: str) -> Any: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be finite JSON") from exc + return value + + +def _safe_relative(value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError("figure artifact path must be safe and relative") + return value + + +class _StrictFigureModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class _DigestBoundFigureModel(_StrictFigureModel): + digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values = dict(values) + values[cls.digest_field] = ZERO_DIGEST + return cls.model_validate(values, context={"bind_figure_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self.digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_figure_digest(self.digest_payload()) + if info.context and info.context.get("bind_figure_digest"): + object.__setattr__(self, self.digest_field, expected) + elif getattr(self, self.digest_field) != expected: + raise ValueError( + f"{self.digest_field} does not match the canonical payload" + ) + return self + + +class FigureArtifactV1(_StrictFigureModel): + schema_version: Literal["ari.figure-artifact/v1"] = FIGURE_ARTIFACT_V1 + role: Literal[ + "source-data", + "spec", + "png", + "pdf", + "raw-planner-response", + "prompt", + ] + relative_path: str + digest: str = Field(pattern=SHA256_PATTERN) + media_type: str = Field(min_length=1, max_length=128) + size_bytes: int = Field(ge=0) + + @field_validator("relative_path") + @classmethod + def _path(cls, value: str) -> str: + return _safe_relative(value) + + +class FigureSourceV1(_StrictFigureModel): + artifact_digest: str = Field(pattern=SHA256_PATTERN) + data_digest: str = Field(pattern=SHA256_PATTERN) + record_ids: tuple[str, ...] = Field(default_factory=tuple, max_length=100_000) + node_ids: tuple[str, ...] = Field(default_factory=tuple, max_length=100_000) + + @field_validator("record_ids", "node_ids") + @classmethod + def _identities(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if any(not item or len(item) > 256 for item in value): + raise ValueError("figure source identities must be non-empty and bounded") + if len(value) != len(set(value)): + raise ValueError("figure source identities must be unique") + return value + + @model_validator(mode="after") + def _has_pointer(self) -> "FigureSourceV1": + if not self.record_ids and not self.node_ids: + raise ValueError("figure source requires a record or node pointer") + return self + + +class FigureAxisV1(_StrictFigureModel): + label: str = Field(min_length=1, max_length=256) + unit: str = Field(min_length=1, max_length=128) + scale: Literal["linear", "log", "symlog"] = "linear" + + @field_validator("label", "unit") + @classmethod + def _trimmed(cls, value: str) -> str: + if value != value.strip(): + raise ValueError("figure axis fields must not have surrounding whitespace") + return value + + +class FigureUncertaintyV1(_StrictFigureModel): + kind: Literal["none", "standard-deviation", "standard-error", "confidence-interval"] + field: str | None = Field(default=None, max_length=256) + confidence: float | None = Field(default=None, gt=0, lt=1) + + @model_validator(mode="after") + def _coherent(self) -> "FigureUncertaintyV1": + if self.kind == "none" and (self.field is not None or self.confidence is not None): + raise ValueError("uncertainty=none cannot name an uncertainty field") + if self.kind != "none" and self.field is None: + raise ValueError("declared uncertainty requires a source field") + if self.kind == "confidence-interval" and self.confidence is None: + raise ValueError("confidence intervals require a confidence level") + if self.kind != "confidence-interval" and self.confidence is not None: + raise ValueError("confidence is valid only for confidence intervals") + return self + + +class FigureSpecV1(_DigestBoundFigureModel): + """One declarative plot over an immutable, content-addressed data slice.""" + + digest_field = "spec_digest" + schema_version: Literal["ari.figure-spec/v1"] = FIGURE_SPEC_V1 + figure_id: str + revision: int = Field(default=0, ge=0, le=100) + chart_type: Literal["bar", "line", "scatter", "hist", "errorbar", "heatmap"] + data: dict[str, Any] = Field(max_length=10_000) + source: FigureSourceV1 + x_field: str | None = Field(default=None, max_length=256) + y_field: str = Field(min_length=1, max_length=256) + x_axis: FigureAxisV1 + y_axis: FigureAxisV1 + value_unit: str = Field(min_length=1, max_length=128) + aggregation: Literal["none", "mean", "median", "sum"] = "none" + uncertainty: FigureUncertaintyV1 = Field( + default_factory=lambda: FigureUncertaintyV1(kind="none") + ) + title: str = Field(default="", max_length=512) + caption: str = Field(default="", max_length=4_096) + style_profile: Literal["ari-publication-v1"] = "ari-publication-v1" + spec_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("figure_id") + @classmethod + def _figure_id(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("figure_id is invalid") + return value + + @field_validator("data") + @classmethod + def _data_json(cls, value: dict[str, Any]) -> dict[str, Any]: + if not value: + raise ValueError("figure data must not be empty") + return _finite_json(value, "figure data") + + @model_validator(mode="after") + def _data_and_fields_match(self) -> "FigureSpecV1": + if canonical_figure_digest(self.data) != self.source.data_digest: + raise ValueError("figure source data digest does not match the data slice") + required = {self.y_field} + if self.x_field is not None: + required.add(self.x_field) + if self.uncertainty.field is not None: + required.add(self.uncertainty.field) + missing = sorted(required - set(self.data)) + if missing: + raise ValueError(f"figure fields are absent from the data slice: {missing}") + if self.chart_type in {"bar", "line", "scatter", "errorbar"} and self.x_field is None: + raise ValueError("the selected chart type requires an x field") + if self.chart_type == "errorbar" and self.uncertainty.kind == "none": + raise ValueError("errorbar charts require declared uncertainty") + if self.chart_type != "errorbar" and self.uncertainty.kind != "none": + raise ValueError("uncertainty is rendered only by errorbar charts") + if self.chart_type == "heatmap" and self.y_field != "values": + raise ValueError("heatmap data must use the canonical values field") + return self + + +class FigureEnvironmentV1(_DigestBoundFigureModel): + digest_field = "environment_digest" + schema_version: Literal["ari.figure-environment/v1"] = FIGURE_ENVIRONMENT_V1 + renderer_version: str = Field(min_length=1, max_length=128) + python_version: str = Field(min_length=1, max_length=128) + matplotlib_version: str = Field(min_length=1, max_length=128) + backend: Literal["agg"] = "agg" + font_family: str = Field(min_length=1, max_length=256) + font_digest: str = Field(pattern=SHA256_PATTERN) + platform: str = Field(min_length=1, max_length=512) + container_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + environment_digest: str = Field(pattern=SHA256_PATTERN) + + +class FigureFeedbackV1(_DigestBoundFigureModel): + digest_field = "feedback_digest" + schema_version: Literal["ari.figure-feedback/v1"] = FIGURE_FEEDBACK_V1 + figure_id: str + source_manifest_digest: str = Field(pattern=SHA256_PATTERN) + review_digest: str = Field(pattern=SHA256_PATTERN) + iteration: int = Field(ge=1, le=2) + issues: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + suggestions: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + feedback_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("figure_id") + @classmethod + def _id(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("feedback figure_id is invalid") + return value + + +class FigureManifestV1(_DigestBoundFigureModel): + """Reproducible identity of one rendered figure revision.""" + + digest_field = "manifest_digest" + schema_version: Literal["ari.figure-manifest/v1"] = FIGURE_MANIFEST_V1 + spec: FigureSpecV1 + execution_mode: Literal["declarative-fixed-renderer"] = "declarative-fixed-renderer" + planner: Literal["none", "llm-spec-only"] = "none" + planner_model: str | None = Field(default=None, max_length=512) + planner_model_revision: str | None = Field(default=None, max_length=512) + prompt_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + sampling: dict[str, Any] = Field(default_factory=dict, max_length=128) + feedback: FigureFeedbackV1 | None = None + parent_manifest_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + environment: FigureEnvironmentV1 + artifacts: tuple[FigureArtifactV1, ...] = Field(min_length=4, max_length=16) + limitations: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + manifest_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("sampling") + @classmethod + def _sampling_json(cls, value: dict[str, Any]) -> dict[str, Any]: + return _finite_json(value, "figure planner sampling") + + @model_validator(mode="after") + def _manifest_coherent(self) -> "FigureManifestV1": + roles = [artifact.role for artifact in self.artifacts] + required = {"source-data", "spec", "png", "pdf"} + if not required.issubset(roles): + raise ValueError("figure manifest lacks a required artifact role") + if len(roles) != len(set(roles)): + raise ValueError("figure artifact roles must be unique") + if self.planner == "none": + if any( + item is not None + for item in (self.planner_model, self.prompt_digest, self.feedback) + ) or self.sampling: + raise ValueError("deterministic figures cannot claim planner provenance") + else: + if not self.planner_model or not self.prompt_digest: + raise ValueError("LLM-planned figures require model and prompt identity") + if "raw-planner-response" not in roles or "prompt" not in roles: + raise ValueError("LLM-planned figures require raw response and prompt artifacts") + if self.feedback is None: + if self.spec.revision != 0 or self.parent_manifest_digest is not None: + raise ValueError("the first figure revision cannot claim a parent") + else: + if self.feedback.figure_id != self.spec.figure_id: + raise ValueError("feedback belongs to a different figure") + if self.feedback.iteration != self.spec.revision: + raise ValueError("feedback iteration and figure revision differ") + if self.parent_manifest_digest != self.feedback.source_manifest_digest: + raise ValueError("feedback does not bind the parent manifest") + return self + + +class FigureBatchV1(_DigestBoundFigureModel): + digest_field = "batch_digest" + schema_version: Literal["ari.figure-batch/v1"] = FIGURE_BATCH_V1 + revision: int = Field(ge=0, le=2) + manifests: tuple[FigureManifestV1, ...] = Field(min_length=1, max_length=100) + figures: dict[str, str] = Field(min_length=1, max_length=100) + latex_snippets: dict[str, str] = Field(min_length=1, max_length=100) + figure_kinds: dict[str, str] = Field(min_length=1, max_length=100) + batch_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("figures") + @classmethod + def _figure_paths(cls, value: dict[str, str]) -> dict[str, str]: + return {key: _safe_relative(path) for key, path in value.items()} + + @model_validator(mode="after") + def _maps_match_manifests(self) -> "FigureBatchV1": + ids = [item.spec.figure_id for item in self.manifests] + if len(ids) != len(set(ids)): + raise ValueError("figure batch IDs must be unique") + expected = set(ids) + if any(set(mapping) != expected for mapping in ( + self.figures, + self.latex_snippets, + self.figure_kinds, + )): + raise ValueError("figure batch maps must exactly match manifest IDs") + for manifest in self.manifests: + if manifest.spec.revision != self.revision: + raise ValueError("figure batch mixes revisions") + pdf = next(item for item in manifest.artifacts if item.role == "pdf") + if self.figures[manifest.spec.figure_id] != pdf.relative_path: + raise ValueError("figure path does not match its PDF artifact") + if self.figure_kinds[manifest.spec.figure_id] != manifest.spec.chart_type: + raise ValueError("figure kind does not match its specification") + return self + + +class LegacyFigureBatchV0(_StrictFigureModel): + """Read-only index for pre-v1 batches; never accepted as native evidence.""" + + schema_version: Literal["ari.figure-batch/legacy-v0"] = LEGACY_FIGURE_BATCH_V0 + figures: dict[str, str] + latex_snippets: dict[str, str] = Field(default_factory=dict) + limitations: tuple[str, ...] = ( + "Legacy figures lack source/spec/environment/artifact digest binding.", + ) + + +def parse_figure_manifest(value: Any) -> FigureManifestV1: + try: + if isinstance(value, str): + value = json.loads(value) + return FigureManifestV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise FigureContractError(f"invalid FigureManifestV1: {exc}") from exc + + +def parse_figure_batch(value: Any) -> FigureBatchV1: + try: + if isinstance(value, str): + value = json.loads(value) + return FigureBatchV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise FigureContractError(f"invalid FigureBatchV1: {exc}") from exc + + +def read_legacy_figure_batch(value: Any) -> LegacyFigureBatchV0: + """Explicit offline reader for the old schema-less figures mapping.""" + + try: + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, dict) or value.get("schema_version"): + raise ValueError("legacy figure input must be an unversioned object") + return LegacyFigureBatchV0.model_validate( + { + "figures": value.get("figures") or {}, + "latex_snippets": value.get("latex_snippets") or {}, + } + ) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise FigureContractError(f"invalid legacy figure batch: {exc}") from exc + + +__all__ = [ + "FIGURE_ARTIFACT_V1", + "FIGURE_BATCH_V1", + "FIGURE_ENVIRONMENT_V1", + "FIGURE_FEEDBACK_V1", + "FIGURE_MANIFEST_V1", + "FIGURE_SPEC_V1", + "LEGACY_FIGURE_BATCH_V0", + "FigureArtifactV1", + "FigureAxisV1", + "FigureBatchV1", + "FigureContractError", + "FigureEnvironmentV1", + "FigureFeedbackV1", + "FigureManifestV1", + "FigureSourceV1", + "FigureSpecV1", + "FigureUncertaintyV1", + "LegacyFigureBatchV0", + "canonical_figure_digest", + "parse_figure_batch", + "parse_figure_manifest", + "read_legacy_figure_batch", +] diff --git a/ari-core/ari/latex_claims.py b/ari-core/ari/latex_claims.py new file mode 100644 index 00000000..dfa1b7dd --- /dev/null +++ b/ari-core/ari/latex_claims.py @@ -0,0 +1,264 @@ +"""Canonical deterministic parsing of scientific claims in LaTeX. + +This module is the single implementation shared by paper authoring and the +independent hard gate. It deliberately performs lexical classification only; +whether a numeric assertion is scientifically valid remains the gate's job. +""" + +from __future__ import annotations + +import hashlib +import math +import re + + +ANCHOR_RE = re.compile(r"%\s*CLAIM:(C\w+):(NC\w+)") + +_NUMBER_RE = re.compile( + r"(? str: + """Normalize prose for stable claim-span hashing.""" + + value = ANCHOR_RE.sub("", text) + value = re.sub(r"%.*", "", value) + for region in _STRIP_REGIONS: + value = region.sub(" ", value) + value = re.sub(r"\\[a-zA-Z]+\*?(?:\[[^\]]*\])?", " ", value) + value = re.sub(r"[{}$~\\]", " ", value) + return re.sub(r"\s+", " ", value).strip().lower() + + +def claim_span_hash(text: str) -> str: + payload = normalize_claim_sentence(text).encode("utf-8") + return "sha256-" + hashlib.sha256(payload).hexdigest() + + +def _canonical_section(title: str) -> str: + normalized = title.strip().lower() + aliases = ( + ("introduction", "introduction"), + ("related", "related_work"), + ("background", "related_work"), + ("prior work", "related_work"), + ("method", "methodology"), + ("approach", "methodology"), + ("design", "methodology"), + ("implementation", "methodology"), + ("experiment", "experiments"), + ("evaluation", "experiments"), + ("setup", "experiments"), + ("result", "results"), + ("discussion", "discussion"), + ("limitation", "limitations"), + ("conclusion", "conclusion"), + ("summary", "conclusion"), + ("future work", "conclusion"), + ("reference", "references"), + ("bibliograph", "references"), + ("acknowled", "acknowledgements"), + ("abstract", "abstract"), + ("appendix", "appendix"), + ) + for needle, canonical in aliases: + if needle in normalized: + return canonical + return re.sub(r"[^a-z0-9]+", "_", normalized).strip("_") or "body" + + +def build_section_map(tex: str) -> list[str]: + """Return one canonical section name per source line.""" + + result: list[str] = [] + current = "preamble" + appendix = False + section_pattern = re.compile(r"\\(?:sub)*section\*?\s*\{([^}]*)\}") + for line in tex.split("\n"): + stripped = line.strip() + if "\\begin{abstract}" in stripped: + current = "abstract" + result.append(current) + continue + if "\\end{abstract}" in stripped: + result.append(current) + current = "body" + continue + if re.search(r"\\appendix\b", stripped): + appendix = True + match = section_pattern.search(stripped) + if match: + current = "appendix" if appendix else _canonical_section(match.group(1)) + elif re.search(r"\\bibliography\b|\\begin\{thebibliography\}", stripped): + current = "references" + result.append(current) + return result + + +def section_at(section_map: list[str], line_number: int) -> str: + index = line_number - 1 + return section_map[index] if 0 <= index < len(section_map) else "body" + + +def find_claim_anchors(tex: str) -> list[dict]: + result: list[dict] = [] + for line_number, line in enumerate(tex.split("\n"), start=1): + for match in ANCHOR_RE.finditer(line): + result.append( + { + "anchor": f"CLAIM:{match.group(1)}:{match.group(2)}", + "claim_id": match.group(1), + "numeric_id": match.group(2), + "line": line_number, + } + ) + return result + + +def sentence_for_anchor(lines: list[str], line_number: int) -> tuple[str, list[int]]: + index = line_number - 1 + if 0 <= index < len(lines): + raw = lines[index] + if not raw.lstrip().startswith("%"): + own = re.sub(r"^%+", "", ANCHOR_RE.sub("", raw)).strip() + if own: + return raw, [line_number, line_number] + for next_index in range(index + 1, min(index + 4, len(lines))): + if lines[next_index].strip() and not lines[next_index].strip().startswith("%"): + return lines[next_index], [next_index + 1, next_index + 1] + fallback = lines[index] if 0 <= index < len(lines) else "" + return fallback, [line_number, line_number] + + +def _strip_for_scan(line: str) -> str: + value = re.sub(r"(? ]", " ", value) + value = value.replace("\\(", " ").replace("\\)", " ") + return value.replace("~", " ").replace("$", " ") + + +def _classify( + numeric_text: str, + has_percent: bool, + before: str, + after: str, +) -> tuple[str, bool]: + try: + value = float(numeric_text.replace(",", "")) + except ValueError: + return "ambiguous", False + is_integer = "." not in numeric_text + if ( + is_integer + and 1900 <= value <= 2099 + and not has_percent + and not _PERF_UNIT_RE.match(after) + and not _SETTING_UNIT_RE.match(after) + ): + return "citation_year", False + if _REF_WORD_RE.search(before): + return "figure_table_ref", False + if _SETTING_UNIT_RE.match(after): + return "experimental_setting", False + if has_percent or _PERF_UNIT_RE.match(after): + return "result_claim", True + return "ambiguous", False + + +def extract_numeric_mentions( + tex: str, + section_map: list[str] | None = None, +) -> list[dict]: + """Extract finite numeric tokens and their lexical evidence class.""" + + sections = section_map if section_map is not None else build_section_map(tex) + mentions: list[dict] = [] + for line_number, raw in enumerate(tex.split("\n"), start=1): + line = _strip_for_scan(raw) + for match in _NUMBER_RE.finditer(line): + integer, fraction = match.group(1), match.group(2) or "" + exponent = match.group(3) or match.group(4) or "" + numeric_text = integer + fraction + has_percent = match.group(5) == "%" + before = line[max(0, match.start() - 24) : match.start()] + after = line[match.end() : match.end() + 24] + mention_type, requires_assertion = _classify( + numeric_text, + has_percent, + before, + after, + ) + try: + value = float(numeric_text.replace(",", "")) + if exponent: + value *= 10.0 ** int(exponent) + except (ValueError, OverflowError): + continue + if not math.isfinite(value): + continue + if match.group(4): + mention_type, requires_assertion = "result_claim", True + mentions.append( + { + "value": value, + "unit": "%" if has_percent else "", + "type": mention_type, + "requires_assertion": requires_assertion, + "section": section_at(sections, line_number), + "line": line_number, + } + ) + return mentions + + +def figure_references(tex: str) -> list[str]: + result: list[str] = [] + for match in re.finditer(r"\\(?:ref|autoref|cref|Cref)\{([^}]*)\}", tex): + label = match.group(1) + if label not in result: + result.append(label) + return result + + +__all__ = [ + "ANCHOR_RE", + "build_section_map", + "claim_span_hash", + "extract_numeric_mentions", + "figure_references", + "find_claim_anchors", + "normalize_claim_sentence", + "section_at", + "sentence_for_anchor", +] diff --git a/ari-core/ari/lineage.py b/ari-core/ari/lineage.py index e8d7ffd9..d2a73cff 100644 --- a/ari-core/ari/lineage.py +++ b/ari-core/ari/lineage.py @@ -24,6 +24,7 @@ from __future__ import annotations import json +import hashlib import logging import os from pathlib import Path @@ -204,12 +205,31 @@ def _try_load(d: Path, depth: int) -> None: pass else: run_id = d.name + payload_digest = "sha256:" + hashlib.sha256(ip.read_bytes()).hexdigest() + contract_digest = None + contract_verified = False + if isinstance(data, dict) and data.get("research_contract") is not None: + try: + from ari.research_contract import parse_research_contract_document + + contract = parse_research_contract_document(data) + if contract is not None: + contract_digest = contract.contract_digest + contract_verified = True + except Exception as exc: + log.warning("lineage: invalid research contract at %s: %s", ip, exc) pool.append( { "run_id": run_id, "depth": depth, "ckpt_dir": str(d), "ideas": ideas, + "artifact_ref": { + "logical_name": "idea.json", + "digest": payload_digest, + "research_contract_digest": contract_digest, + "contract_verified": contract_verified, + }, } ) @@ -237,11 +257,22 @@ def format_ancestor_pool_for_virsci(pool: list[dict], *, max_per_run: int = 3) - for entry in ancestors: rid = str(entry.get("run_id", ""))[-12:] depth = entry.get("depth", "?") + artifact = entry.get("artifact_ref") or {} + if artifact.get("contract_verified"): + evidence_label = ( + "verified artifact " + + str(artifact.get("research_contract_digest") or "")[:20] + ) + else: + evidence_label = ( + "legacy artifact (contract unverified) " + + str(artifact.get("digest") or "")[:20] + ) for idea in (entry.get("ideas") or [])[:max_per_run]: title = (idea.get("title") or "").strip().replace("\n", " ")[:140] score = idea.get("overall_score", "") lines.append( - f"- run {rid} (depth {depth}, score {score}): {title}" + f"- run {rid} (depth {depth}, {evidence_label}, score {score}): {title}" ) lines.append( "Treat these as context — refine, extend, or explicitly pivot from " diff --git a/ari-core/ari/llm/cli_server.py b/ari-core/ari/llm/cli_server.py index cc5f5678..1b8869b2 100644 --- a/ari-core/ari/llm/cli_server.py +++ b/ari-core/ari/llm/cli_server.py @@ -1,4 +1,3 @@ -from __future__ import annotations """OpenAI-compatible HTTP shim that serves agentic CLIs (`claude -p`, `codex exec`) as chat-completion backends. @@ -52,6 +51,8 @@ whatever auth those CLIs use (subscription login *or* API key). """ +from __future__ import annotations + import argparse import json import logging @@ -355,6 +356,133 @@ def _run(cmd: list[str], stdin_text: str, cwd: str) -> subprocess.CompletedProce ) +def _materialize_mcp_credential_env( + mcp_config: dict, + source_env: dict[str, str] | None = None, +) -> dict: + """Resolve value-free credential references in a local MCP config copy.""" + + source = source_env if source_env is not None else os.environ + materialized = json.loads(json.dumps(mcp_config)) + servers = materialized.get("mcpServers") + if not isinstance(servers, dict): + raise ValueError("mcp_config.mcpServers must be an object") + for name, server in servers.items(): + if not isinstance(server, dict): + raise ValueError(f"MCP server {name!r} must be an object") + refs = server.pop("_ariCredentialEnv", []) + if not isinstance(refs, list) or any( + not isinstance(ref, str) or not re.fullmatch(r"[A-Z_][A-Z0-9_]*", ref) + for ref in refs + ): + raise ValueError(f"MCP server {name!r} has invalid credential env refs") + environment = server.setdefault("env", {}) + if not isinstance(environment, dict): + raise ValueError(f"MCP server {name!r} env must be an object") + for ref in refs: + value = source.get(ref) + if not value: + raise ValueError( + f"MCP server {name!r} credential env ref {ref!r} is unavailable" + ) + environment[ref] = value + return materialized + + +def _mcp_credential_values( + mcp_config: dict, + source_env: dict[str, str] | None = None, +) -> tuple[str, ...]: + """Return present local values referenced by an already validated config.""" + + source = source_env if source_env is not None else os.environ + values: set[str] = set() + for server in (mcp_config.get("mcpServers") or {}).values(): + if not isinstance(server, dict): + continue + for name in server.get("_ariCredentialEnv") or []: + value = source.get(name) + if value: + values.add(value) + return tuple(sorted(values, key=len, reverse=True)) + + +def _redact_mcp_credential_values(text: str | None, values: tuple[str, ...]) -> str: + rendered = text or "" + for value in values: + rendered = rendered.replace(value, "") + escaped = json.dumps(value, ensure_ascii=False)[1:-1] + rendered = rendered.replace(escaped, "") + return rendered + + +def _write_claude_mcp_config(mcp_config: dict, cwd: str) -> str: + """Write a mode-0600 local config and remove partial files on failure.""" + + materialized = _materialize_mcp_credential_env(mcp_config) + path: str | None = None + try: + with tempfile.NamedTemporaryFile( + "w", suffix=".mcp.json", dir=cwd, delete=False, encoding="utf-8" + ) as fh: + path = fh.name + json.dump(materialized, fh) + os.chmod(path, 0o600) + return path + except BaseException: + if path: + try: + os.unlink(path) + except OSError: + pass + raise + + +def _build_claude_command( + *, + system: str, + agent: bool, + real_model: str | None, + use_mcp: bool, + mcp_json_file: str | None, + allowed_mcp_tools: list[str] | None, + debug_log: str | None, +) -> list[str]: + """Build the Claude CLI argv after any secret-bearing file is materialized.""" + + output_format = "stream-json" if use_mcp else "json" + cmd = [CLAUDE_BIN, "-p", "--output-format", output_format] + if use_mcp: + cmd.append("--verbose") + if CLAUDE_BARE: + cmd.append("--bare") + if real_model: + cmd += ["--model", real_model] + if system: + cmd += ["--system-prompt", system] + if use_mcp: + if not mcp_json_file or not debug_log: + raise ValueError("MCP Claude invocation requires config and debug paths") + cmd += [ + "--mcp-config", + mcp_json_file, + "--strict-mcp-config", + "--allowedTools", + " ".join(allowed_mcp_tools or []), + "--permission-mode", + CLAUDE_AGENT_PERMISSION, + "--debug-file", + debug_log, + ] + elif agent: + cmd += ["--permission-mode", CLAUDE_AGENT_PERMISSION] + else: + cmd += ["--allowedTools", ""] + if MAX_BUDGET_USD: + cmd += ["--max-budget-usd", MAX_BUDGET_USD] + return cmd + + def run_claude( system: str, prompt: str, @@ -386,44 +514,43 @@ def run_claude( """ mcp_json_file: str | None = None use_mcp = bool(mcp_config and allowed_mcp_tools) - debug_log = os.path.join(cwd, "claude_debug.log") if use_mcp else None - + credential_values: tuple[str, ...] = () if use_mcp: - cmd = [CLAUDE_BIN, "-p", "--output-format", "stream-json", "--verbose"] - else: - cmd = [CLAUDE_BIN, "-p", "--output-format", "json"] - if CLAUDE_BARE: - cmd.append("--bare") - if real_model: - cmd += ["--model", real_model] - if system: - cmd += ["--system-prompt", system] + credential_values = _mcp_credential_values(mcp_config) + debug_log = ( + os.devnull + if credential_values + else os.path.join(cwd, "claude_debug.log") if use_mcp else None + ) if use_mcp: - # Materialise the MCP server config as a tmp JSON file in cwd so it - # survives for post-mortem inspection alongside tool_calls.jsonl. - fh = tempfile.NamedTemporaryFile( - "w", suffix=".mcp.json", dir=cwd, delete=False, encoding="utf-8", - ) + # Credential values are materialized only inside the local shim and the + # temporary file exists only while Claude is running. + assert mcp_config is not None + mcp_json_file = _write_claude_mcp_config(mcp_config, cwd) + cmd = _build_claude_command( + system=system, + agent=agent, + real_model=real_model, + use_mcp=use_mcp, + mcp_json_file=mcp_json_file, + allowed_mcp_tools=allowed_mcp_tools, + debug_log=debug_log, + ) + try: try: - json.dump(mcp_config, fh) - finally: - fh.close() - mcp_json_file = fh.name - cmd += [ - "--mcp-config", mcp_json_file, - "--strict-mcp-config", - "--allowedTools", " ".join(allowed_mcp_tools or []), - "--permission-mode", CLAUDE_AGENT_PERMISSION, - "--debug-file", debug_log, - ] - elif agent: - cmd += ["--permission-mode", CLAUDE_AGENT_PERMISSION] - else: - # No tools => pure text/JSON generation. - cmd += ["--allowedTools", ""] - if MAX_BUDGET_USD: - cmd += ["--max-budget-usd", MAX_BUDGET_USD] - proc = _run(cmd, prompt, cwd) + proc = _run(cmd, prompt, cwd) + except subprocess.TimeoutExpired as exc: + exc.stdout = _redact_mcp_credential_values(exc.stdout, credential_values) + exc.stderr = _redact_mcp_credential_values(exc.stderr, credential_values) + raise + finally: + if mcp_json_file: + try: + os.unlink(mcp_json_file) + except OSError: + pass + proc.stdout = _redact_mcp_credential_values(proc.stdout, credential_values) + proc.stderr = _redact_mcp_credential_values(proc.stderr, credential_values) if proc.returncode != 0: raise RuntimeError( f"claude exited {proc.returncode}: {(proc.stderr or proc.stdout)[:500]}" diff --git a/ari-core/ari/llm/client.py b/ari-core/ari/llm/client.py index 93619a16..e9b387ea 100644 --- a/ari-core/ari/llm/client.py +++ b/ari-core/ari/llm/client.py @@ -7,6 +7,7 @@ import litellm +from ari.call_context import ToolCallContextV1 from ari.config import LLMConfig @@ -31,6 +32,7 @@ def __init__(self, config: LLMConfig) -> None: self._phase: str = "" self._skill: str = "" self._work_dir: str = "" + self._call_context: ToolCallContextV1 | None = None # Optional MCPClient injected post-construction (see core.py). When # set AND the backend is the cli-shim, complete() forwards a # --mcp-config payload to the shim so Claude can call the same @@ -46,6 +48,7 @@ def set_context( phase: str | None = None, skill: str | None = None, work_dir: str | None = None, + call_context: ToolCallContextV1 | None = None, ) -> None: """Attach context that will be sent as litellm metadata on every subsequent ``complete()`` call. Pass ``None`` to leave a field @@ -63,6 +66,8 @@ def set_context( self._skill = str(skill) if work_dir is not None: self._work_dir = str(work_dir) + if call_context is not None: + self._call_context = call_context def _model_name(self) -> str: from ari.llm.routing import resolve_litellm_model @@ -95,6 +100,7 @@ def complete( phase: str | None = None, skill: str | None = None, work_dir: str | None = None, + call_context: ToolCallContextV1 | None = None, ) -> LLMResponse: """Send messages to the LLM and return a response. @@ -116,6 +122,11 @@ def complete( _phase = phase if phase is not None else getattr(self, "_phase", "") _skill = skill if skill is not None else getattr(self, "_skill", "") _work_dir = work_dir if work_dir is not None else getattr(self, "_work_dir", "") + _call_context = ( + call_context + if call_context is not None + else getattr(self, "_call_context", None) + ) kwargs: dict = { "model": _model, "messages": msgs, @@ -157,7 +168,12 @@ def complete( ): try: mcp_cfg, allowed = self.mcp_client.to_claude_mcp_config( - phase=(_phase or None), + phase=( + _call_context.phase + if _call_context is not None and _call_context.phase + else (_phase or None) + ), + context=_call_context, ) except Exception as _e: # noqa: BLE001 — never block the LLM call import logging as _l diff --git a/ari-core/ari/mcp/README.md b/ari-core/ari/mcp/README.md index 0eae8fc6..9fe55d6f 100644 --- a/ari-core/ari/mcp/README.md +++ b/ari-core/ari/mcp/README.md @@ -7,7 +7,15 @@ lifecycle of each `ari-skill-*` subprocess and routes tool calls + results. - `README.md` — this file. - `__init__.py` — public `MCPClient` + contract. -- `client.py` — stdio connection pooling, retry, per-thread asyncio loop. +- `child_environment.py` — exact child allowlist, credential scopes, and redaction. +- `claude_bridge.py` — value-free Claude CLI MCP config and allowed-tool rendering. +- `client.py` — registry, typed dispatch, retry, and connection pooling. +- `connection.py` — one provider's stdio lifecycle and event-loop thread. +- `dispatch_support.py` — pure identity, phase, timeout, and tracing policy. +- `invoke_runtime.py` — retries, cancellation, and typed transport normalization. +- `lock_runtime.py` — per-client exact/subset `SKILLS.lock` reconciliation state. +- `registry_runtime.py` — live discovery, enrichment, and collision admission. +- `secure_stdio_proxy.py` — exact-env/redacting boundary for direct MCP clients. ## See also diff --git a/ari-core/ari/mcp/child_environment.py b/ari-core/ari/mcp/child_environment.py new file mode 100644 index 00000000..cd72233b --- /dev/null +++ b/ari-core/ari/mcp/child_environment.py @@ -0,0 +1,415 @@ +"""Minimal child-process environment and value-free credential authority. + +The MCP SDK adds a small parent-environment baseline even when callers provide +``StdioServerParameters.env``. This module therefore supplies explicit safe +overrides for that baseline as well as the manifest allowlist. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, TextIO + +from ari.config import SkillConfig +from ari.call_context import CONTEXT_AUTHORITY_ENV +from ari.skill_manifest import looks_like_credential_environment_name + + +SAFE_INHERITED_ENV_NAMES = ( + "PATH", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "TMPDIR", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", +) +MANAGED_CHILD_ENV_NAMES = frozenset( + { + "HOME", + "BASH_ENV", + "CDPATH", + "ENV", + "LD_PRELOAD", + "LOGNAME", + "USER", + "SHELL", + "TERM", + "PYTHONPATH", + "PYTHONIOENCODING", + "PYTHONBREAKPOINT", + "PYTHONHOME", + "PYTHONINSPECT", + "PYTHONNOUSERSITE", + "PYTHONSTARTUP", + "PYTHONUNBUFFERED", + "PYTHONUTF8", + "PYTHONWARNINGS", + "VIRTUAL_ENV", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + CONTEXT_AUTHORITY_ENV, + } +) + + +class ChildEnvironmentError(RuntimeError): + """Base class for manifest/environment admission failures.""" + + +class MissingRequiredEnvironmentError(ChildEnvironmentError): + """Raised when a required ordinary or credential variable is unavailable.""" + + +class UnclassifiedCredentialError(ChildEnvironmentError): + """Raised when a complete policy lists a probable secret as ordinary env.""" + + +class ManagedEnvironmentOverrideError(ChildEnvironmentError): + """Raised when a manifest attempts to override core-owned isolation names.""" + + +class CredentialScopeDriftError(ChildEnvironmentError): + """Raised when reconnect would change the run's credential authority.""" + + +class SecretRedactor: + """Redact known credential values from text and structured MCP responses.""" + + def __init__(self, replacements: Mapping[str, str] | None = None) -> None: + self._replacements = { + str(value): str(marker) + for value, marker in (replacements or {}).items() + if value + } + pairs: list[tuple[str, str]] = [] + for value, marker in self._replacements.items(): + pairs.append((value, marker)) + escaped = json.dumps(value, ensure_ascii=False)[1:-1] + if escaped != value: + pairs.append((escaped, marker)) + self._pairs = tuple(sorted(pairs, key=lambda item: len(item[0]), reverse=True)) + + def merged(self, replacements: Mapping[str, str]) -> "SecretRedactor": + """Return a redactor extended with core-owned ephemeral secrets.""" + + return SecretRedactor({**self._replacements, **dict(replacements)}) + + def text(self, value: str) -> str: + rendered = value + for secret, marker in self._pairs: + rendered = rendered.replace(secret, marker) + return rendered + + def value(self, value: Any) -> Any: + if isinstance(value, str): + return self.text(value) + if isinstance(value, dict): + return {key: self.value(item) for key, item in value.items()} + if isinstance(value, list): + return [self.value(item) for item in value] + if isinstance(value, tuple): + return tuple(self.value(item) for item in value) + return value + + +class SecretRedactingPipe: + """Pipe child stderr through value redaction before it reaches a log sink. + + Subprocess launchers consume a stream's file descriptor directly, so a + ``TextIOBase.write`` wrapper alone cannot intercept child output. This + class gives the child a private pipe descriptor and drains the other end on + a thread, redacting complete lines before forwarding them. + """ + + def __init__(self, target: TextIO, redactor: SecretRedactor) -> None: + read_fd, write_fd = os.pipe() + self._reader = os.fdopen( + read_fd, + "r", + encoding="utf-8", + errors="replace", + ) + self.child_writer = os.fdopen( + write_fd, + "w", + encoding="utf-8", + errors="replace", + ) + self._target = target + self._redactor = redactor + self._thread = threading.Thread( + target=self._drain, + name="ari-mcp-redacted-stderr", + daemon=True, + ) + self._thread.start() + + def _drain(self) -> None: + try: + with self._reader: + for line in self._reader: + try: + self._target.write(self._redactor.text(line)) + self._target.flush() + except (OSError, ValueError): + return + except (OSError, ValueError): + return + + def close_parent_writer(self) -> None: + """Close the launcher's copy after the subprocess inherits the fd.""" + + if not self.child_writer.closed: + self.child_writer.close() + + def close(self) -> None: + self.close_parent_writer() + self._thread.join(timeout=5) + + +@dataclass(frozen=True) +class ChildEnvironment: + """Resolved environment plus non-secret scope identities and redaction.""" + + values: dict[str, str] + inherited_names: tuple[str, ...] + credential_env_names: tuple[str, ...] + credential_scope_identities: tuple[dict[str, Any], ...] + redactor: SecretRedactor + core_secret_env_names: tuple[str, ...] = () + + @property + def active_credential_scope_ids(self) -> tuple[str, ...]: + return tuple( + identity["scope_id"] + for identity in self.credential_scope_identities + if identity["present_env"] + ) + + def transport_values(self) -> dict[str, str]: + """Return non-credential values safe to serialize to a local shim.""" + + credential_names = set(self.credential_env_names) + credential_names.update(self.core_secret_env_names) + return { + name: value + for name, value in self.values.items() + if name not in credential_names + } + + def with_core_secret( + self, + name: str, + value: str, + *, + marker: str, + ) -> "ChildEnvironment": + """Add a core-owned ephemeral secret without changing manifest scopes.""" + + if name not in MANAGED_CHILD_ENV_NAMES: + raise ManagedEnvironmentOverrideError( + f"core secret name is not managed by ARI: {name}" + ) + values = dict(self.values) + values[name] = value + names = tuple(sorted({*self.core_secret_env_names, name})) + return ChildEnvironment( + values=values, + inherited_names=self.inherited_names, + credential_env_names=self.credential_env_names, + credential_scope_identities=self.credential_scope_identities, + redactor=self.redactor.merged({value: marker}), + core_secret_env_names=names, + ) + + +def _identity_digest(scope_id: str, present_env: list[str]) -> str: + payload = json.dumps( + {"scope_id": scope_id, "present_env": sorted(present_env)}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def _runtime_root(skill: SkillConfig, parent: Mapping[str, str]) -> Path: + safe_name = re.sub(r"[^a-zA-Z0-9_.-]+", "_", skill.name).strip("._-") + if not safe_name: + safe_name = hashlib.sha256(skill.name.encode("utf-8")).hexdigest()[:16] + checkpoint = str(parent.get("ARI_CHECKPOINT_DIR") or "").strip() + if checkpoint: + return Path(checkpoint) / ".ari" / "skill-runtime" / safe_name + uid = str(os.getuid()) if hasattr(os, "getuid") else "user" + return Path(tempfile.gettempdir()) / "ari-skill-runtime" / uid / safe_name + + +def _prepare_runtime_dirs(skill: SkillConfig, parent: Mapping[str, str]) -> dict[str, str]: + root = _runtime_root(skill, parent) + directories = { + "HOME": root / "home", + "XDG_CACHE_HOME": root / "cache", + "XDG_CONFIG_HOME": root / "config", + "XDG_DATA_HOME": root / "data", + "XDG_STATE_HOME": root / "state", + } + for path in directories.values(): + path.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + path.chmod(0o700) + except OSError: + pass + return {name: str(path) for name, path in directories.items()} + + +def _scope_entries(skill: SkillConfig) -> list[tuple[str, list[str], list[str]]]: + entries: list[tuple[str, list[str], list[str]]] = [] + for scope_id, declaration in sorted(skill.credential_scopes.items()): + entries.append( + ( + scope_id, + list(declaration.get("required_env") or []), + list(declaration.get("optional_env") or []), + ) + ) + return entries + + +def build_child_environment( + skill: SkillConfig, + *, + skill_path: Path, + ari_core_root: Path, + parent_env: Mapping[str, str] | None = None, +) -> ChildEnvironment: + """Resolve one fail-closed child environment from manifest declarations.""" + + parent = parent_env if parent_env is not None else os.environ + declared_ordinary = set(skill.required_env) | set(skill.optional_env) + managed = sorted(declared_ordinary & MANAGED_CHILD_ENV_NAMES) + if managed: + raise ManagedEnvironmentOverrideError( + f"Skill '{skill.name}' cannot override managed environment names: {managed}" + ) + if skill.environment_policy == "complete": + unclassified = sorted( + name + for name in declared_ordinary + if looks_like_credential_environment_name(name) + ) + if unclassified: + raise UnclassifiedCredentialError( + f"Skill '{skill.name}' must classify credential-like env names: " + f"{unclassified}" + ) + + missing = sorted( + name for name in skill.required_env if not str(parent.get(name) or "") + ) + scope_entries = _scope_entries(skill) + missing.extend( + name + for _scope_id, required, _optional in scope_entries + for name in required + if not str(parent.get(name) or "") + ) + if missing: + raise MissingRequiredEnvironmentError( + f"Skill '{skill.name}' is missing required environment names: " + f"{sorted(set(missing))}" + ) + + inherited = { + name + for name in SAFE_INHERITED_ENV_NAMES + if parent.get(name) is not None + } + values: dict[str, str] = { + name: str(parent[name]) + for name in inherited + } + values.setdefault("PATH", os.defpath) + values.setdefault("LANG", "C.UTF-8") + values.update(_prepare_runtime_dirs(skill, parent)) + values.update( + { + "LOGNAME": "ari-skill", + "USER": "ari-skill", + "SHELL": "/bin/sh", + "TERM": "dumb", + "BASH_ENV": "", + "CDPATH": "", + "ENV": "", + "LD_PRELOAD": "", + "PYTHONPATH": os.pathsep.join([str(skill_path), str(ari_core_root)]), + "PYTHONBREAKPOINT": "0", + "PYTHONHOME": "", + "PYTHONINSPECT": "0", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONSTARTUP": "", + "PYTHONUNBUFFERED": "1", + "PYTHONUTF8": "1", + "PYTHONWARNINGS": "default", + "VIRTUAL_ENV": "", + } + ) + for name in sorted(declared_ordinary): + if parent.get(name) is not None: + values[name] = str(parent[name]) + + credential_names: set[str] = set() + identities: list[dict[str, Any]] = [] + replacements: dict[str, str] = {} + for scope_id, required, optional in scope_entries: + declared = sorted(set(required) | set(optional)) + present = [name for name in declared if str(parent.get(name) or "")] + credential_names.update(present) + for name in present: + value = str(parent[name]) + values[name] = value + replacements[value] = f"" + identities.append( + { + "scope_id": scope_id, + "declared_env": declared, + "present_env": present, + "identity_digest": _identity_digest(scope_id, present), + } + ) + + return ChildEnvironment( + values=values, + inherited_names=tuple(sorted(inherited)), + credential_env_names=tuple(sorted(credential_names)), + credential_scope_identities=tuple(identities), + redactor=SecretRedactor(replacements), + ) + + +__all__ = [ + "SAFE_INHERITED_ENV_NAMES", + "MANAGED_CHILD_ENV_NAMES", + "ChildEnvironment", + "ChildEnvironmentError", + "CredentialScopeDriftError", + "ManagedEnvironmentOverrideError", + "MissingRequiredEnvironmentError", + "SecretRedactingPipe", + "SecretRedactor", + "UnclassifiedCredentialError", + "build_child_environment", +] diff --git a/ari-core/ari/mcp/claude_bridge.py b/ari-core/ari/mcp/claude_bridge.py new file mode 100644 index 00000000..81f23944 --- /dev/null +++ b/ari-core/ari/mcp/claude_bridge.py @@ -0,0 +1,102 @@ +"""Render a discovered ARI registry for Claude CLI's native MCP interface.""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from ari.call_context import ToolCallContextV1 +from ari.config import SkillConfig +from ari.mcp.dispatch_support import phase_is_disabled, phase_matches + + +def build_claude_mcp_config( + *, + skills: list[SkillConfig], + connections: dict[str, Any], + visible_tools: list[dict], + phase: str | None, + context: ToolCallContextV1 | None = None, +) -> tuple[dict, list[str]]: + """Return Claude's ``mcpServers`` document and fully-qualified allowlist.""" + + servers: dict[str, dict] = {} + allowed: list[str] = [] + for skill in skills: + if phase_is_disabled(skill.phase): + continue + if phase is not None and not phase_matches(skill.phase, phase): + continue + connection = connections.get(skill.name) + if connection is None: + continue + params = connection._server_params() + child_environment = connection.child_environment + skill_tools = [] + for tool in visible_tools: + if tool.get("skill_name") != skill.name: + continue + requirement = str( + (tool.get("policy") or {}).get("context_requirement") or "none" + ) + if requirement != "none" and ( + context is None or not context.satisfies(requirement) + ): + continue + skill_tools.append(tool) + if not skill_tools: + continue + context_requirements = { + str(tool["name"]): str( + (tool.get("policy") or {}).get("context_requirement") or "none" + ) + for tool in skill_tools + if str((tool.get("policy") or {}).get("context_requirement") or "none") + != "none" + } + markers = { + env_name: str(identity["scope_id"]) + for identity in child_environment.credential_scope_identities + for env_name in identity["present_env"] + } + proxy_spec = { + "command": params.command, + "args": list(params.args), + "env_names": sorted( + set(child_environment.transport_values()) + | set(child_environment.credential_env_names) + ), + "credential_markers": markers, + "context_requirements": context_requirements, + } + if context_requirements: + assert context is not None + proxy_spec["call_context"] = context.model_dump(mode="json") + servers[skill.name] = { + "command": sys.executable, + "args": [ + "-m", + "ari.mcp.secure_stdio_proxy", + "--spec", + json.dumps( + proxy_spec, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ), + ], + "env": child_environment.transport_values(), + } + if child_environment.credential_env_names: + servers[skill.name]["_ariCredentialEnv"] = list( + child_environment.credential_env_names + ) + allowed.extend( + f"mcp__{skill.name}__{tool['name']}" + for tool in skill_tools + ) + return {"mcpServers": servers}, allowed + + +__all__ = ["build_claude_mcp_config"] diff --git a/ari-core/ari/mcp/client.py b/ari-core/ari/mcp/client.py index 59adc476..7f2dfbe0 100644 --- a/ari-core/ari/mcp/client.py +++ b/ari-core/ari/mcp/client.py @@ -1,287 +1,93 @@ -"""MCP client for calling Skills (MCP Servers) via stdio protocol. - -Features: -- Connection pooling: MCP server processes stay alive across calls -- Retry logic: up to MAX_RETRIES attempts per tool call -- Thread-safe: uses asyncio loop per thread -""" +"""Thread-safe pooled MCP stdio client with bounded retries and typed results.""" from __future__ import annotations -import asyncio import atexit import logging -import sys -import threading +import os import time -from contextlib import asynccontextmanager from pathlib import Path -from typing import Any - -from mcp import ClientSession, StdioServerParameters -from mcp.client.stdio import stdio_client +from ari.async_tools import ( + AsyncLifecycleV1, + AsyncToolEndpointV1, + AsyncToolHandleV1, +) +from ari.call_context import ToolCallContextV1, new_context_authority_key from ari.config import SkillConfig +from ari.mcp.connection import SkillConnection +from ari.mcp.dispatch_support import ( + DEFAULT_TOOL_TIMEOUT as DEFAULT_TOOL_TIMEOUT, + SLOW_TOOL_TIMEOUT as SLOW_TOOL_TIMEOUT, + VERY_SLOW_TOOL_TIMEOUT as VERY_SLOW_TOOL_TIMEOUT, + ToolNameCollisionError, + default_call_context, + enrich_call_context, + log_tool_call, + phase_matches as _phase_matches, + resolve_registration as _resolve_registration, + resolve_tool_timeout as _resolve_tool_timeout, + runtime_tool_ref, + unresolved_tool_ref as _unresolved_tool_ref, +) +from ari.mcp.lock_runtime import SkillLockController +from ari.mcp.invoke_runtime import invoke_with_retries +from ari.mcp.registry_runtime import discover_registry +from ari.protocols.stores import ArtifactStore +from ari.result import ( + DEFAULT_INLINE_RESULT_LIMIT, + ResultEnvelopeNormalizer, + ResultEnvelopeV1, + ResultErrorV1, + utc_now_iso, +) +from ari.skill_lock import SkillLockError, SkillsLockV1 logger = logging.getLogger(__name__) - -def _normalize_phases(phase: str | list[str] | None) -> list[str]: - """Coerce SkillConfig.phase into a flat list of phase strings.""" - if phase is None: - return ["all"] - if isinstance(phase, str): - return [phase] - return [str(p) for p in phase] - - -def _phase_matches(skill_phase: str | list[str], want: str) -> bool: - """True iff a skill declared `skill_phase` should be exposed for `want`.""" - phases = _normalize_phases(skill_phase) - return want in phases or "all" in phases - - -def _phase_is_disabled(skill_phase: str | list[str]) -> bool: - """True iff the skill is fully disabled (phase == 'none' or ['none']).""" - phases = [p for p in _normalize_phases(skill_phase) if p] - return phases == ["none"] - -MAX_RETRIES = 3 -RETRY_DELAY = 0.5 -DEFAULT_TOOL_TIMEOUT = 300 # seconds -# Tools that perform internal LLM calls or heavy processing need longer timeouts -SLOW_TOOL_TIMEOUT = 3600 # seconds — multi-agent ideation / iterative LLM tools -# Agent/sandbox tools whose internal budgets are measured in hours. Keep the -# MCP-level timeout above the tool's own ceiling — otherwise MCP times out -# mid-rollout, retries, hits any idempotent-skip path inside the tool (e.g. -# build_reproduce_sh skipping when the partial first attempt already wrote -# reproduce.sh), and surfaces a misleading "skipped" result. -VERY_SLOW_TOOL_TIMEOUT = 13 * 3600 # 13 h ≥ build_reproduce_sh's 12 h default -_VERY_SLOW_TOOLS = frozenset({ - "build_reproduce_sh", # PaperBench BasicAgent rollout, default 12 h - "run_reproduce", # Phase 1 sbatch, default up to 4 h - "grade_with_simplejudge", # Phase 2 judge, n_runs × minutes -}) -_SLOW_TOOLS = frozenset({"generate_ideas", "write_paper_iterative", "review_compiled_paper", - "collect_references_iterative", "reproduce_from_paper", - # paper_refine does an internal LLM call (S2P refiner); without - # this it inherited the 300s default and timed out under CLI-shim - # congestion while write_paper (already slow-tiered) did not. - "paper_refine", - # compile_paper (render_paper / A_rend) runs pdflatex×3 + bibtex - # (each up to 120s) — the 4-pass sequence can exceed 300s. - "compile_paper"}) - - -def _resolve_tool_timeout(tool_name: str, args: dict) -> int: - """Resolve MCP-level timeout for a tool call. - - Priority: explicit per-call budget in args > _VERY_SLOW_TOOLS tier > - _SLOW_TOOLS tier > DEFAULT_TOOL_TIMEOUT. - """ - for k in ("time_limit_sec", "timeout_global_sec", "wall_time_sec"): - v = args.get(k) - if isinstance(v, (int, float)) and v > 0: - return int(v) + 600 # +10 min buffer for setup / teardown - if tool_name in _VERY_SLOW_TOOLS: - return VERY_SLOW_TOOL_TIMEOUT - if tool_name in _SLOW_TOOLS: - return SLOW_TOOL_TIMEOUT - return DEFAULT_TOOL_TIMEOUT - - -class _SkillConnection: - """Persistent connection to a single MCP Skill server.""" - - def __init__(self, skill: SkillConfig) -> None: - self.skill = skill - self._session: ClientSession | None = None - self._loop: asyncio.AbstractEventLoop | None = None - self._loop_thread: threading.Thread | None = None - self._context_stack: Any = None - - def _skill_path(self) -> Path: - import os as _os - path = self.skill.path - # Resolve {{ari_root}} template in skill path - ari_root = _os.environ.get("ARI_ROOT", str(Path(__file__).parents[3])) - path = path.replace("{{ari_root}}", ari_root) - return Path(path) - - @staticmethod - def _resolve_python(skill_path: Path) -> str: - """Return the best Python interpreter for a skill. - - Priority: - 1. Skill-local venv (/.venv/bin/python) - 2. Python recorded by setup.sh ($ARI_ROOT/.ari_python) - 3. sys.executable (fallback) - """ - # 1. Skill-local venv - skill_python = skill_path / ".venv" / "bin" / "python" - if skill_python.is_file(): - return str(skill_python) - - # 2. Recorded by setup.sh - import os as _os - ari_root = _os.environ.get("ARI_ROOT", str(Path(__file__).parents[3])) - marker = Path(ari_root) / ".ari_python" - if marker.is_file(): - recorded = marker.read_text().strip() - if recorded and Path(recorded).is_file(): - return recorded - - # 3. Fallback - return sys.executable - - def _server_params(self) -> StdioServerParameters: - import os - skill_path = self._skill_path() - python = self._resolve_python(skill_path) - # Expose ari-core on the skill subprocess's PYTHONPATH so the skill - # can `from ari import cost_tracker` and wire itself into the shared - # cost_trace.jsonl. ari-core is kept last so the skill's own src/ - # layout wins on name collisions. - ari_core_root = str(Path(__file__).parents[2]) - pythonpath = os.pathsep.join([str(skill_path), ari_core_root]) - return StdioServerParameters( - command=python, - args=[str(skill_path / "src" / "server.py")], - env={**os.environ, "PYTHONPATH": pythonpath}, - ) - - async def _start(self) -> None: - """Start the MCP server process and establish session.""" - import contextlib - stack = contextlib.AsyncExitStack() - read, write = await stack.enter_async_context(stdio_client(self._server_params())) - session = await stack.enter_async_context(ClientSession(read, write)) - await session.initialize() - self._session = session - self._context_stack = stack - - async def _stop(self) -> None: - if self._context_stack is not None: - try: - await self._context_stack.aclose() - except Exception: - pass - self._context_stack = None - self._session = None - - def _ensure_loop(self) -> None: - """Ensure the dedicated event loop thread is running. - - A single daemon thread runs ``loop.run_forever()`` for the - lifetime of this connection. All coroutines are submitted via - ``asyncio.run_coroutine_threadsafe`` and therefore serialised on - the loop — no concurrent ``run_until_complete`` conflicts. - """ - if ( - self._loop is not None - and not self._loop.is_closed() - and self._loop_thread is not None - and self._loop_thread.is_alive() - ): - return - self._loop = asyncio.new_event_loop() - self._loop_thread = threading.Thread( - target=self._loop.run_forever, daemon=True, - ) - self._loop_thread.start() - - def _run(self, coro: Any, timeout: int = DEFAULT_TOOL_TIMEOUT) -> Any: - """Run a coroutine on the connection's dedicated event loop thread. - - Thread-safe: concurrent callers are queued on the single loop via - ``asyncio.run_coroutine_threadsafe``, so there is no risk of - "This event loop is already running". - """ - self._ensure_loop() - assert self._loop is not None - future = asyncio.run_coroutine_threadsafe(coro, self._loop) - return future.result(timeout=timeout) - - def ensure_connected(self) -> None: - if self._session is None: - self._run(self._start()) - - def list_tools(self) -> list[dict]: - self.ensure_connected() - - async def _list() -> list[dict]: - assert self._session is not None - result = await self._session.list_tools() - return [ - { - "name": t.name, - "description": t.description or "", - "inputSchema": t.inputSchema if t.inputSchema else {}, - "skill_name": self.skill.name, - } - for t in result.tools - ] - - return self._run(_list()) - - def call_tool(self, tool_name: str, args: dict, timeout: int = DEFAULT_TOOL_TIMEOUT) -> dict: - self.ensure_connected() - - async def _call() -> dict: - assert self._session is not None - result = await self._session.call_tool(tool_name, args) - parts = [p.text for p in result.content if hasattr(p, "text")] - text = "\n".join(parts) if parts else "" - if not text: - return {"error": f"Tool '{tool_name}' returned empty response — the tool may have crashed or timed out."} - return {"result": text} - - return self._run(_call(), timeout=timeout) - - def close(self) -> None: - if self._loop and not self._loop.is_closed(): - # Submit _stop() to the loop thread (same path as _run) - future = asyncio.run_coroutine_threadsafe(self._stop(), self._loop) - try: - future.result(timeout=30) - except Exception: - pass - self._loop.call_soon_threadsafe(self._loop.stop) - if self._loop_thread is not None: - self._loop_thread.join(timeout=5) - self._loop.close() - self._loop_thread = None +# Private compatibility alias. Connection ownership moved to connection.py; +# callers outside ari.mcp should use MCPClient rather than this implementation. +_SkillConnection = SkillConnection +_runtime_tool_ref = runtime_tool_ref class MCPClient: """MCP client with connection pooling and retry logic.""" - # Tools whose CoW guard reads ARI_CURRENT_NODE_ID inside the - # pooled memory-skill MCP server. The (set_current_node, write) - # pair must be atomic across all parallel nodes that share this - # MCPClient — see ``call_tool(cow_node_id=...)`` below. - _COW_TOOLS: frozenset = frozenset({ - "add_memory", "clear_node_memory", - # Typed write tools (Phase 1) — all delegate to backend.add_memory, - # which enforces node_id == $ARI_CURRENT_NODE_ID. Keep in sync with - # ari-skill-memory/src/server.py. - "add_experiment_result", "add_failure_case", "add_procedure_memory", - "add_reflection", "add_reproducibility_event", - "consolidate_node_memory", - }) - - def __init__(self, skills: list[SkillConfig], disabled_tools: list[str] | None = None) -> None: + def __init__( + self, + skills: list[SkillConfig], + disabled_tools: list[str] | None = None, + *, + artifact_store: ArtifactStore | None = None, + result_inline_limit: int = DEFAULT_INLINE_RESULT_LIMIT, + skill_lock_path: str | Path | None = None, + skill_lock_scope: str = "exact", + strict_provider_loading: bool | None = None, + ) -> None: import threading as _t + self.skills = skills self.disabled_tools: set[str] = set(disabled_tools or []) self._connections: dict[str, _SkillConnection] = {} self._conn_lock = _t.Lock() - # Serialises (_set_current_node, memory write) pairs across - # parallel BFTS nodes. RLock so the same thread can re-enter - # if a future caller wraps higher-level helpers. - self._cow_lock = _t.RLock() + self._context_authority_keys: dict[str, str] = { + skill.name: new_context_authority_key() for skill in skills + } self._tool_registry: dict[str, str] = {} # tool_name -> skill.name + self._tool_ref_registry: dict[str, str] = {} # tool_ref -> skill.name + self._tool_name_by_ref: dict[str, str] = {} + self._tool_ref_by_name: dict[str, str] = {} + self._tool_metadata_by_ref: dict[str, dict] = {} self._tools_cache: list[dict] | None = None + self._artifact_store = artifact_store + self._derived_artifact_store: tuple[str, ArtifactStore] | None = None + self._result_inline_limit = result_inline_limit + self._skill_lock = SkillLockController( + skill_lock_path, + scope=skill_lock_scope, + strict_provider_loading=strict_provider_loading, + ) atexit.register(self.close_all) def _get_conn(self, skill_name: str) -> _SkillConnection | None: @@ -290,12 +96,22 @@ def _get_conn(self, skill_name: str) -> _SkillConnection | None: def _init_connection(self, skill: SkillConfig) -> _SkillConnection: with self._conn_lock: if skill.name not in self._connections: - conn = _SkillConnection(skill) + conn = _SkillConnection( + skill, + context_authority_key=self._context_authority_keys.setdefault( + skill.name, new_context_authority_key() + ), + ) self._connections[skill.name] = conn return self._connections[skill.name] - def list_tools(self, phase: str | None = None) -> list[dict]: - """Return skill tools, optionally filtered by phase and disabled_tools.""" + def list_tools( + self, + phase: str | None = None, + *, + context: ToolCallContextV1 | None = None, + ) -> list[dict]: + """Return tools admitted by phase and, when supplied, call context.""" if self._tools_cache is None: self._build_tools_cache() @@ -304,126 +120,604 @@ def list_tools(self, phase: str | None = None) -> list[dict]: if self.disabled_tools: tools = [t for t in tools if t["name"] not in self.disabled_tools] # Filter by phase. Skill `phase` may be a string or a list; matching is - # any-of with "all" as wildcard. + # any-of with "all" as wildcard. Canonical per-tool policy is an + # additional constraint rather than a replacement for Skill exposure. if phase is not None: - _pm = self._phase_map - tools = [t for t in tools if _phase_matches(_pm.get(t["name"], "all"), phase)] + tools = [t for t in tools if self._tool_admits_phase(t["tool_ref"], phase)] + if context is not None: + tools = [ + tool + for tool in tools + if context.satisfies(self._tool_context_requirement(tool["tool_ref"])) + ] return tools def _build_tools_cache(self) -> None: - """Discover tools from all enabled skills (called once, lazily).""" - tools: list[dict] = [] - for skill in self.skills: - # Skip disabled skills (phase: none / [none]) — don't start MCP server - if _phase_is_disabled(getattr(skill, "phase", "all")): - logger.info("Skipping disabled skill '%s' (phase=none)", skill.name) - continue - try: - conn = self._init_connection(skill) - skill_tools = conn.list_tools() - for t in skill_tools: - self._tool_registry[t["name"]] = skill.name - tools.extend(skill_tools) - logger.info("Loaded %d tools from skill '%s'", len(skill_tools), skill.name) - except Exception as e: - logger.warning("Failed to load skill '%s': %s", skill.name, e) - - self._tools_cache = tools - self._phase_map = {t["name"]: getattr( - next((s for s in self.skills if s.name == self._tool_registry.get(t["name"],"")), None), - "phase", "all") for t in tools} + """Discover tools from all enabled skills exactly once.""" + + discovered = discover_registry( + self.skills, + init_connection=self._init_connection, + close_all=self.close_all, + strict_provider_loading=self._skill_lock.strict_provider_loading, + ) + self._tool_registry = discovered.owner_by_name + self._tool_ref_registry = discovered.owner_by_ref + self._tool_name_by_ref = discovered.name_by_ref + self._tool_ref_by_name = discovered.ref_by_name + self._tool_metadata_by_ref = discovered.metadata_by_ref + self._tools_cache = discovered.tools + self._reconcile_skills_lock() + + def _reconcile_skills_lock(self) -> None: + """Create or verify the run's immutable live-provider snapshot.""" + + try: + self._skill_lock.reconcile( + skills=self.skills, + tools=self._tools_cache or [], + disabled_tools=self.disabled_tools, + ) + except SkillLockError: + self.close_all() + self._tool_registry = {} + self._tool_ref_registry = {} + self._tool_name_by_ref = {} + self._tool_ref_by_name = {} + self._tool_metadata_by_ref = {} + self._tools_cache = None + self._skill_lock.clear() + raise + + @property + def skills_lock(self) -> SkillsLockV1 | None: + """Return the reconciled snapshot after discovery, if locking is enabled.""" + + return self._skill_lock.snapshot def call_tool( self, tool_name: str, args: dict, *, - cow_node_id: str | None = None, + context: ToolCallContextV1 | None = None, ) -> dict: - """Call a tool. Reuses connection pool and retries on failure. - - ``cow_node_id`` (optional): when set and ``tool_name`` is a - CoW-guarded memory tool (``add_memory`` / ``clear_node_memory``), - ``_set_current_node({node_id: cow_node_id})`` is invoked under a - process-wide lock immediately before the actual call so the two - operations are atomic across parallel BFTS nodes that share this - MCPClient. Without this, the memory skill's ``ARI_CURRENT_NODE_ID`` - env var (set by ``_set_current_node``) is racy and one node's - write can be rejected by another node's set. + """Call a tool with explicit run/node context when policy requires it.""" + + envelope = self.call_tool_envelope(tool_name, args, context=context) + return envelope.to_legacy(self._artifact_store_for_call()) + + def call_tool_envelope( + self, + tool_name_or_ref: str, + args: dict, + *, + context: ToolCallContextV1 | None = None, + ) -> ResultEnvelopeV1: + """Call a tool and return the canonical typed result envelope. + + ``tool_name_or_ref`` accepts an immutable ``tool_ref`` or a unique bare + alias during migration. New federation callers should always pass the + immutable reference returned by :meth:`list_tools`. """ - if cow_node_id and tool_name in self._COW_TOOLS: - with self._cow_lock: - self._call_tool_unlocked( - "_set_current_node", {"node_id": cow_node_id}, + + if self._tools_cache is None: + started_at = utc_now_iso() + try: + self._build_tools_cache() + except (ToolNameCollisionError, SkillLockError) as exc: + return self._result_normalizer().error( + tool_ref=_unresolved_tool_ref(tool_name_or_ref), + kind="admission", + message=str(exc), + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), ) - return self._call_tool_unlocked(tool_name, args) - return self._call_tool_unlocked(tool_name, args) - def _call_tool_unlocked(self, tool_name: str, args: dict) -> dict: - """Internal: same as call_tool but without the CoW gate. + return self._call_tool_envelope_unlocked( + tool_name_or_ref, + args, + context=context, + ) - Holds no locks; safe to call from inside ``_cow_lock`` for the - atomic (set + write) sequence. - """ - # ── Trace: log tool call args for propagation debugging ──── - _TRACE_TOOLS = {"make_metric_spec", "generate_ideas", "survey"} - if tool_name in _TRACE_TOOLS: - import json as _json_trace - _args_str = _json_trace.dumps(args, ensure_ascii=False) - logger.info( - "[mcp] call_tool %s: args_len=%d args=%s", - tool_name, len(_args_str), _args_str[:500], + def _call_tool_envelope_unlocked( + self, + tool_name_or_ref: str, + args: dict, + *, + context: ToolCallContextV1 | None = None, + ) -> ResultEnvelopeV1: + """Typed dispatch implementation.""" + + started_at = utc_now_iso() + normalizer = self._result_normalizer() + if self._tools_cache is None: + try: + self._build_tools_cache() + except (ToolNameCollisionError, SkillLockError) as exc: + return normalizer.error( + tool_ref=_unresolved_tool_ref(tool_name_or_ref), + kind="admission", + message=str(exc), + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + + tool_name, tool_ref, skill_name, selection_reason = _resolve_registration( + tool_name_or_ref, + tool_ref_registry=self._tool_ref_registry, + tool_name_by_ref=self._tool_name_by_ref, + tool_registry=self._tool_registry, + tool_ref_by_name=self._tool_ref_by_name, + ) + effective_context = context or self._default_call_context() + effective_context = enrich_call_context( + effective_context, + selection_reason=selection_reason, + skill=next( + (skill for skill in self.skills if skill.name == skill_name), + None, + ), + ) + + admission_error = self._registration_admission_error( + requested=tool_name_or_ref, + tool_name=tool_name, + tool_ref=tool_ref, + skill_name=skill_name, + context=effective_context, + normalizer=normalizer, + started_at=started_at, + ) + if admission_error is not None: + return admission_error + assert skill_name is not None + + return self._invoke_registered_tool( + tool_name=tool_name, + tool_ref=tool_ref, + skill_name=skill_name, + args=args, + context=effective_context, + normalizer=normalizer, + started_at=started_at, + ) + + def get_async_status( + self, + handle: AsyncToolHandleV1 | dict, + *, + context: ToolCallContextV1 | None = None, + ) -> ResultEnvelopeV1: + """Poll an async handle through its immutable status endpoint.""" + + parsed = self._parse_async_handle(handle, context=context) + if isinstance(parsed, ResultEnvelopeV1): + return parsed + envelope = self.call_tool_envelope( + parsed.status.tool_ref, + {parsed.status.handle_argument: parsed.handle_id}, + context=context, + ) + return self._normalize_async_status(envelope, parsed) + + def get_async_result( + self, + handle: AsyncToolHandleV1 | dict, + *, + context: ToolCallContextV1 | None = None, + ) -> ResultEnvelopeV1: + """Fetch an async result, or use the terminal status payload as its result.""" + + parsed = self._parse_async_handle(handle, context=context) + if isinstance(parsed, ResultEnvelopeV1): + return parsed + if parsed.result is None or parsed.result == parsed.status: + return self.get_async_status(parsed, context=context) + envelope = self.call_tool_envelope( + parsed.result.tool_ref, + {parsed.result.handle_argument: parsed.handle_id}, + context=context, + ) + return envelope.model_copy(update={"async_handle": parsed}) + + def cancel_async( + self, + handle: AsyncToolHandleV1 | dict, + *, + context: ToolCallContextV1 | None = None, + ) -> ResultEnvelopeV1: + """Cancel an async operation through its declared immutable endpoint.""" + + parsed = self._parse_async_handle(handle, context=context) + if isinstance(parsed, ResultEnvelopeV1): + return parsed + if parsed.cancel is None: + return self._result_normalizer().error( + tool_ref=parsed.submission_tool_ref, + kind="admission", + message="Async operation does not declare a cancel capability", + retryable=False, + context=context, + details={"handle_id": parsed.handle_id}, ) - else: - logger.debug("[mcp] call_tool %s: args_keys=%s", tool_name, list(args.keys())) - skill_name = self._tool_registry.get(tool_name) + envelope = self.call_tool_envelope( + parsed.cancel.tool_ref, + {parsed.cancel.handle_argument: parsed.handle_id}, + context=context, + ) + if envelope.status == "error": + return envelope.model_copy(update={"async_handle": parsed}) + return envelope.model_copy( + update={"status": "cancelled", "async_handle": parsed} + ) + + def wait_for_async( + self, + handle: AsyncToolHandleV1 | dict, + *, + context: ToolCallContextV1 | None = None, + timeout_seconds: float | None = None, + cancel_on_timeout: bool = False, + ) -> ResultEnvelopeV1: + """Poll until terminal state, then retrieve the declared result.""" + + parsed = self._parse_async_handle(handle, context=context) + if isinstance(parsed, ResultEnvelopeV1): + return parsed + wait_budget = ( + float(parsed.max_wait_seconds) + if timeout_seconds is None + else max(0.0, float(timeout_seconds)) + ) + started = time.monotonic() + while True: + status = self.get_async_status(parsed, context=context) + if status.status not in {"submitted", "running"}: + if status.status == "ok" and parsed.result not in { + None, + parsed.status, + }: + return self.get_async_result(parsed, context=context) + return status + elapsed = time.monotonic() - started + if elapsed >= wait_budget: + if cancel_on_timeout and parsed.cancel is not None: + self.cancel_async(parsed, context=context) + return self._result_normalizer().error( + tool_ref=parsed.status.tool_ref, + kind="timeout", + message=( + f"Async operation {parsed.handle_id!r} did not reach a " + f"terminal state within {wait_budget:g} seconds" + ), + retryable=True, + context=context, + details={"handle_id": parsed.handle_id}, + ) + time.sleep(min(parsed.poll_interval_seconds, wait_budget - elapsed)) + + def _registration_admission_error( + self, + *, + requested: str, + tool_name: str, + tool_ref: str, + skill_name: str | None, + context: ToolCallContextV1, + normalizer: ResultEnvelopeNormalizer, + started_at: str, + ) -> ResultEnvelopeV1 | None: + """Return a typed policy rejection, or ``None`` when dispatch is admitted.""" + + message = "" if not skill_name: - registered = list(self._tool_registry.keys()) - return { - "error": ( - f"Tool '{tool_name}' not found. " - f"Available: {registered}" + message = f"Tool '{requested}' not found. Available: {sorted(self._tool_registry)}" + elif tool_name in self.disabled_tools or tool_ref in self.disabled_tools: + message = f"Tool '{tool_name}' is disabled by run configuration" + elif context.phase and not self._tool_admits_phase(tool_ref, context.phase): + message = f"Tool '{tool_name}' is not admitted in phase '{context.phase}'" + else: + requirement = self._tool_context_requirement(tool_ref) + if not context.satisfies(requirement): + message = ( + f"Tool '{tool_name}' requires explicit {requirement} context" ) - } + if not message: + return None + return normalizer.error( + tool_ref=tool_ref, + kind="admission", + message=message, + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + def _invoke_registered_tool( + self, + *, + tool_name: str, + tool_ref: str, + skill_name: str, + args: dict, + context: ToolCallContextV1, + normalizer: ResultEnvelopeNormalizer, + started_at: str, + ) -> ResultEnvelopeV1: + """Invoke an admitted registration and normalize transport outcomes.""" + + log_tool_call(logger, tool_name, args) conn = self._connections.get(skill_name) if conn is None: skill = next((s for s in self.skills if s.name == skill_name), None) if skill is None: - return {"error": f"Skill '{skill_name}' not found"} + return normalizer.error( + tool_ref=tool_ref, + kind="admission", + message=f"Skill '{skill_name}' not found", + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) conn = self._init_connection(skill) - timeout = _resolve_tool_timeout(tool_name, args) + skill = next((s for s in self.skills if s.name == skill_name), None) + timeout_class = None + timeout_budget = None + if skill is not None: + timeout_class = skill.tool_timeout_classes.get(tool_name) + policy = skill.tool_policies.get(tool_name, {}) + if isinstance(policy, dict): + timeout_budget = policy.get("timeout_budget") + timeout = _resolve_tool_timeout( + args, + timeout_class=timeout_class, + timeout_budget=timeout_budget, + ) + requirement = self._tool_context_requirement(tool_ref) + call_args = ( + conn.authorize_args(tool_name, args, context) + if requirement != "none" + else dict(args) + ) - last_error = "" - for attempt in range(1, MAX_RETRIES + 1): - try: - return conn.call_tool(tool_name, args, timeout=timeout) - except Exception as e: - last_error = f"{type(e).__name__}: {e}" - logger.warning( - "Tool '%s' attempt %d/%d failed: %s", - tool_name, attempt, MAX_RETRIES, last_error, - ) - # Reconnect in case the connection was dropped - try: - conn.close() - self._connections.pop(skill_name, None) # invalidate before re-init - conn = self._init_connection( - next(s for s in self.skills if s.name == skill_name) - ) - self._connections[skill_name] = conn - except Exception: - pass - if attempt < MAX_RETRIES: - time.sleep(RETRY_DELAY * attempt) - - return { - "error": ( - f"Tool '{tool_name}' failed after {MAX_RETRIES} attempts. " - f"Last: {last_error}" + def _reconnect(failed_connection): + failed_connection.close() + self._connections.pop(skill_name, None) + return self._init_connection( + next(item for item in self.skills if item.name == skill_name) ) - } + + envelope = invoke_with_retries( + connection=conn, + reconnect=_reconnect, + tool_name=tool_name, + tool_ref=tool_ref, + args=call_args, + timeout=timeout, + context=context, + normalizer=normalizer, + started_at=started_at, + logger=logger, + ) + policy = self._tool_policy(tool_ref) + lifecycle_raw = policy.get("async_lifecycle") + if policy.get("timeout_class") != "async" or lifecycle_raw is None: + return envelope + return self._attach_async_handle( + envelope, + lifecycle=AsyncLifecycleV1.model_validate(lifecycle_raw), + skill_name=skill_name, + context=context, + ) + + def _attach_async_handle( + self, + envelope: ResultEnvelopeV1, + *, + lifecycle: AsyncLifecycleV1, + skill_name: str, + context: ToolCallContextV1, + ) -> ResultEnvelopeV1: + """Bind a successful provider submission to immutable lifecycle refs.""" + + if envelope.status == "error": + return envelope + raw_handle = envelope.structured_content.get(lifecycle.handle_field) + if raw_handle is None or not str(raw_handle).strip(): + return self._result_normalizer().error( + tool_ref=envelope.provenance.tool_ref, + kind="protocol", + message=( + "Async submission omitted declared handle field " + f"{lifecycle.handle_field!r}" + ), + retryable=False, + context=context, + details={"handle_field": lifecycle.handle_field}, + ) + try: + status = self._resolve_async_endpoint(skill_name, lifecycle.status) + result = ( + self._resolve_async_endpoint(skill_name, lifecycle.result) + if lifecycle.result is not None + else None + ) + cancel = ( + self._resolve_async_endpoint(skill_name, lifecycle.cancel) + if lifecycle.cancel is not None + else None + ) + except ValueError as exc: + return self._result_normalizer().error( + tool_ref=envelope.provenance.tool_ref, + kind="protocol", + message=str(exc), + retryable=False, + context=context, + ) + handle = AsyncToolHandleV1( + handle_id=str(raw_handle), + submission_tool_ref=envelope.provenance.tool_ref, + status=status, + result=result, + cancel=cancel, + state_field=lifecycle.state_field, + states=lifecycle.states, + poll_interval_seconds=lifecycle.poll_interval_seconds, + max_wait_seconds=lifecycle.max_wait_seconds, + submitted_at=envelope.provenance.completed_at or utc_now_iso(), + ) + return envelope.model_copy( + update={"status": "submitted", "async_handle": handle} + ) + + def _resolve_async_endpoint( + self, + skill_name: str, + operation, + ) -> AsyncToolEndpointV1: + matches = [ + tool_ref + for tool_ref, metadata in self._tool_metadata_by_ref.items() + if self._tool_ref_registry.get(tool_ref) == skill_name + and metadata.get("capability_ref") == operation.capability_ref + ] + if len(matches) != 1: + raise ValueError( + f"Async capability {operation.capability_ref!r} resolved to " + f"{len(matches)} runtime tools for Skill {skill_name!r}" + ) + return AsyncToolEndpointV1( + tool_ref=matches[0], + handle_argument=operation.handle_argument, + ) + + def _parse_async_handle( + self, + handle: AsyncToolHandleV1 | dict, + *, + context: ToolCallContextV1 | None, + ) -> AsyncToolHandleV1 | ResultEnvelopeV1: + try: + return AsyncToolHandleV1.model_validate(handle) + except (TypeError, ValueError) as exc: + return self._result_normalizer().error( + tool_ref=_unresolved_tool_ref("async-handle"), + kind="protocol", + message=f"Invalid async handle: {exc}", + retryable=False, + context=context, + ) + + def _normalize_async_status( + self, + envelope: ResultEnvelopeV1, + handle: AsyncToolHandleV1, + ) -> ResultEnvelopeV1: + if envelope.status == "error": + return envelope.model_copy(update={"async_handle": handle}) + raw_state = envelope.structured_content.get(handle.state_field) + state = handle.states.classify(raw_state) + if state == "unknown": + return envelope.model_copy( + update={ + "status": "error", + "async_handle": handle, + "error": ResultErrorV1( + kind="protocol", + message=( + "Async status response contains an undeclared state " + f"{raw_state!r} in field {handle.state_field!r}" + ), + retryable=True, + details={"provider_state": raw_state}, + ), + } + ) + if state == "failed": + structured = envelope.structured_content + message = next( + ( + str(structured[key]) + for key in ("message", "error", "stderr") + if structured.get(key) + ), + f"Async operation {handle.handle_id!r} failed", + ) + return envelope.model_copy( + update={ + "status": "error", + "async_handle": handle, + "error": ResultErrorV1( + kind="tool", message=message, retryable=False + ), + } + ) + status = { + "submitted": "submitted", + "running": "running", + "succeeded": "ok", + "cancelled": "cancelled", + }[state] + return envelope.model_copy(update={"status": status, "async_handle": handle}) + + def _tool_policy(self, tool_ref: str) -> dict: + metadata = self._tool_metadata_by_ref.get(tool_ref, {}) + policy = metadata.get("policy") + return policy if isinstance(policy, dict) else {} + + def _tool_admits_phase(self, tool_ref: str, phase: str) -> bool: + skill_name = self._tool_ref_registry.get(tool_ref) + skill = next((item for item in self.skills if item.name == skill_name), None) + if skill is None or not _phase_matches(skill.phase, phase): + return False + metadata = self._tool_metadata_by_ref.get(tool_ref, {}) + policy = metadata.get("policy") + tool_phases = ( + policy.get("phases", ["all"]) if isinstance(policy, dict) else ["all"] + ) + return _phase_matches(tool_phases, phase) + + def _tool_context_requirement(self, tool_ref: str) -> str: + policy = self._tool_policy(tool_ref) + requirement = str(policy.get("context_requirement") or "none") + return requirement if requirement in {"none", "run", "node"} else "none" + + def _artifact_store_for_call(self) -> ArtifactStore | None: + if self._artifact_store is not None: + return self._artifact_store + checkpoint_dir = os.environ.get("ARI_CHECKPOINT_DIR", "").strip() + if not checkpoint_dir: + return None + if ( + self._derived_artifact_store is None + or self._derived_artifact_store[0] != checkpoint_dir + ): + from ari.artifact_store import CheckpointArtifactStore + + self._derived_artifact_store = ( + checkpoint_dir, + CheckpointArtifactStore(checkpoint_dir), + ) + return self._derived_artifact_store[1] + + def _result_normalizer(self) -> ResultEnvelopeNormalizer: + return ResultEnvelopeNormalizer( + self._artifact_store_for_call(), + inline_limit=self._result_inline_limit, + ) + + @staticmethod + def _default_call_context(node_id: str | None = None) -> ToolCallContextV1: + return default_call_context(node_id) def close_all(self) -> None: """Close all connections.""" @@ -435,49 +729,21 @@ def close_all(self) -> None: self._connections.clear() def to_claude_mcp_config( - self, phase: str | None = None, + self, + phase: str | None = None, + *, + context: ToolCallContextV1 | None = None, ) -> tuple[dict, list[str]]: - """Build the ``--mcp-config`` payload + ``--allowedTools`` list for - spawning a Claude CLI subprocess against the same ari-skill servers - this client manages. - - Returns ``(mcp_config, allowed_tools)``: - - ``mcp_config``: ``{"mcpServers": {: {command, args, env}}}`` - — claude reads this via ``--mcp-config ``. - - ``allowed_tools``: list of fully-qualified MCP tool names - (``mcp____``) to pass to ``--allowedTools`` so claude - can ONLY call ari skills (no native Bash/Write/Edit). - - ``phase`` filters skills exactly as ``list_tools(phase=...)`` does. - Skills are spawned with the same python interpreter + PYTHONPATH the - in-process MCPClient uses, so they see ari-core (for cost_tracker). - """ - # Ensure connections + registry are populated (lazy). + """Render this registry for Claude CLI's native MCP interface.""" + if self._tools_cache is None: self._build_tools_cache() - servers: dict[str, dict] = {} - allowed: list[str] = [] - for skill in self.skills: - if _phase_is_disabled(getattr(skill, "phase", "all")): - continue - if phase is not None and not _phase_matches( - getattr(skill, "phase", "all"), phase, - ): - continue - conn = self._connections.get(skill.name) - if conn is None: - # _build_tools_cache may have skipped a failing skill; skip too. - continue - params = conn._server_params() - servers[skill.name] = { - "command": params.command, - "args": list(params.args), - "env": dict(params.env or {}), - } - for tool in self._tools_cache or []: - if self._tool_registry.get(tool["name"]) != skill.name: - continue - if tool["name"] in self.disabled_tools: - continue - allowed.append(f"mcp__{skill.name}__{tool['name']}") - return {"mcpServers": servers}, allowed + from ari.mcp.claude_bridge import build_claude_mcp_config + + return build_claude_mcp_config( + skills=self.skills, + connections=self._connections, + visible_tools=self.list_tools(phase=phase, context=context), + phase=phase, + context=context, + ) diff --git a/ari-core/ari/mcp/connection.py b/ari-core/ari/mcp/connection.py new file mode 100644 index 00000000..ee3823d6 --- /dev/null +++ b/ari-core/ari/mcp/connection.py @@ -0,0 +1,316 @@ +"""Lifecycle and transport for one isolated MCP Skill subprocess.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import threading +from pathlib import Path +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +from ari.call_context import ( + CALL_CONTEXT_ARGUMENT, + CONTEXT_AUTHORITY_ENV, + ToolCallContextV1, + authorize_tool_context, + new_context_authority_key, +) +from ari.config import SkillConfig +from ari.mcp.child_environment import ( + ChildEnvironment, + CredentialScopeDriftError, + SecretRedactingPipe, + build_child_environment, +) +from ari.mcp.dispatch_support import DEFAULT_TOOL_TIMEOUT + + +class SkillConnection: + """Persistent, thread-safe connection to one MCP Skill server.""" + + def __init__( + self, + skill: SkillConfig, + *, + context_authority_key: str | None = None, + ) -> None: + self.skill = skill + self._context_authority_key = ( + context_authority_key or new_context_authority_key() + ) + self._session: ClientSession | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._loop_thread: threading.Thread | None = None + self._context_stack: Any = None + self._child_environment: ChildEnvironment | None = None + self._stderr_pipe: SecretRedactingPipe | None = None + self._server_parameters: StdioServerParameters | None = None + self._expected_credential_scopes = tuple( + dict(identity) for identity in skill.credential_scope_identities + ) + + def _skill_path(self) -> Path: + import os + + path = self.skill.path + ari_root = os.environ.get("ARI_ROOT", str(Path(__file__).parents[3])) + path = path.replace("{{ari_root}}", ari_root) + return Path(path) + + @staticmethod + def _resolve_python(skill_path: Path) -> str: + """Return the skill-local, setup-recorded, or current interpreter.""" + + skill_python = skill_path / ".venv" / "bin" / "python" + if skill_python.is_file(): + return str(skill_python) + + import os + + ari_root = os.environ.get("ARI_ROOT", str(Path(__file__).parents[3])) + marker = Path(ari_root) / ".ari_python" + if marker.is_file(): + recorded = marker.read_text(encoding="utf-8").strip() + if recorded and Path(recorded).is_file(): + return recorded + return sys.executable + + def _server_params(self) -> StdioServerParameters: + if self._server_parameters is not None: + return self._server_parameters + skill_path = self._skill_path() + child_environment = build_child_environment( + self.skill, + skill_path=skill_path, + ari_core_root=Path(__file__).parents[2], + ) + child_environment = child_environment.with_core_secret( + CONTEXT_AUTHORITY_ENV, + self._context_authority_key, + marker="", + ) + resolved_scopes = tuple( + dict(identity) + for identity in child_environment.credential_scope_identities + ) + if ( + self._expected_credential_scopes + and resolved_scopes != self._expected_credential_scopes + ): + raise CredentialScopeDriftError( + f"Skill '{self.skill.name}' credential authority changed during run" + ) + self._child_environment = child_environment + self.skill.credential_scope_identities = [ + dict(identity) for identity in resolved_scopes + ] + self._server_parameters = StdioServerParameters( + command=self._resolve_python(skill_path), + args=[str(skill_path / self.skill.entrypoint)], + env=dict(child_environment.values), + ) + return self._server_parameters + + @property + def child_environment(self) -> ChildEnvironment: + """Return the environment resolved by the latest parameter build.""" + + if self._child_environment is None: + self._server_params() + assert self._child_environment is not None + return self._child_environment + + def redact_text(self, value: str) -> str: + if self._child_environment is None: + return value + return self._child_environment.redactor.text(value) + + async def _start(self) -> None: + import contextlib + + stack = contextlib.AsyncExitStack() + params = self._server_params() + assert self._child_environment is not None + stderr_pipe = SecretRedactingPipe( + sys.stderr, + self._child_environment.redactor, + ) + try: + read, write = await stack.enter_async_context( + stdio_client(params, errlog=stderr_pipe.child_writer) + ) + except BaseException: + stderr_pipe.close() + raise + stderr_pipe.close_parent_writer() + try: + session = await stack.enter_async_context(ClientSession(read, write)) + await session.initialize() + except BaseException: + await stack.aclose() + stderr_pipe.close() + raise + self._session = session + self._context_stack = stack + self._stderr_pipe = stderr_pipe + + async def _stop(self) -> None: + if self._context_stack is not None: + try: + await self._context_stack.aclose() + except Exception: + pass + self._context_stack = None + self._session = None + if self._stderr_pipe is not None: + self._stderr_pipe.close() + self._stderr_pipe = None + + def _ensure_loop(self) -> None: + if ( + self._loop is not None + and not self._loop.is_closed() + and self._loop_thread is not None + and self._loop_thread.is_alive() + ): + return + self._loop = asyncio.new_event_loop() + self._loop_thread = threading.Thread( + target=self._loop.run_forever, + daemon=True, + ) + self._loop_thread.start() + + def _run(self, coro: Any, timeout: int = DEFAULT_TOOL_TIMEOUT) -> Any: + """Submit a coroutine to the connection's dedicated event loop.""" + + self._ensure_loop() + assert self._loop is not None + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result(timeout=timeout) + + def ensure_connected(self) -> None: + if self._session is None: + self._run(self._start()) + + def list_tools(self) -> list[dict]: + self.ensure_connected() + + async def _list() -> list[dict]: + assert self._session is not None + assert self._child_environment is not None + result = await self._session.list_tools() + return [ + { + "name": tool.name, + "description": self._child_environment.redactor.text( + tool.description or "" + ), + "inputSchema": _public_input_schema( + self._child_environment.redactor.value( + tool.inputSchema if tool.inputSchema else {} + ) + ), + "outputSchema": self._child_environment.redactor.value( + tool.outputSchema if tool.outputSchema else {} + ), + "skill_name": self.skill.name, + } + for tool in result.tools + ] + + return self._run(_list()) + + def authorize_args( + self, + tool_name: str, + args: dict, + context: ToolCallContextV1, + ) -> dict: + """Inject a signed, connection-scoped context capability.""" + + authorized = dict(args) + authorized[CALL_CONTEXT_ARGUMENT] = authorize_tool_context( + context, + tool_name=tool_name, + authority_key=self._context_authority_key, + ) + return authorized + + def call_tool( + self, tool_name: str, args: dict, timeout: int = DEFAULT_TOOL_TIMEOUT + ) -> dict: + self.ensure_connected() + + async def _call() -> dict: + assert self._session is not None + result = await self._session.call_tool(tool_name, args) + parts = [part.text for part in result.content if hasattr(part, "text")] + rendered = "\n".join(parts) if parts else "" + structured = getattr(result, "structuredContent", None) + if not isinstance(structured, dict): + structured = None + assert self._child_environment is not None + rendered = self._child_environment.redactor.text(rendered) + structured = self._child_environment.redactor.value(structured) + if not rendered and structured: + rendered = json.dumps(structured, ensure_ascii=False) + if not rendered: + return { + "error": ( + f"Tool '{tool_name}' returned empty response — the tool " + "may have crashed or timed out." + ), + "_error_kind": "protocol", + "_retryable": True, + } + return { + "result": rendered, + "_structured_content": structured, + "_mcp_is_error": bool(getattr(result, "isError", False)), + } + + return self._run(_call(), timeout=timeout) + + def close(self) -> None: + if self._loop and not self._loop.is_closed(): + future = asyncio.run_coroutine_threadsafe(self._stop(), self._loop) + try: + future.result(timeout=30) + except Exception: + pass + self._loop.call_soon_threadsafe(self._loop.stop) + if self._loop_thread is not None: + self._loop_thread.join(timeout=5) + self._loop.close() + self._loop_thread = None + + +__all__ = ["SkillConnection"] + + +def _public_input_schema(schema: Any) -> Any: + """Hide the transport-managed context argument from model-facing schemas.""" + + if not isinstance(schema, dict): + return schema + rendered = dict(schema) + properties = rendered.get("properties") + if isinstance(properties, dict) and CALL_CONTEXT_ARGUMENT in properties: + rendered["properties"] = { + name: value + for name, value in properties.items() + if name != CALL_CONTEXT_ARGUMENT + } + required = rendered.get("required") + if isinstance(required, list): + kept = [name for name in required if name != CALL_CONTEXT_ARGUMENT] + if kept: + rendered["required"] = kept + else: + rendered.pop("required", None) + return rendered diff --git a/ari-core/ari/mcp/dispatch_support.py b/ari-core/ari/mcp/dispatch_support.py new file mode 100644 index 00000000..9ae99be8 --- /dev/null +++ b/ari-core/ari/mcp/dispatch_support.py @@ -0,0 +1,202 @@ +"""Pure identity, phase, timeout, and tracing policy for MCP dispatch.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +from pathlib import Path + +from ari.async_tools import TimeoutBudgetV1 +from ari.config import SkillConfig +from ari.result import ToolCallContextV1 + + +class ToolNameCollisionError(RuntimeError): + """Raised when more than one admitted Skill owns the same bare tool name.""" + + +def runtime_tool_ref(skill: SkillConfig, tool: dict) -> str: + """Bind declared identity to the schemas returned by ``tools/list``.""" + + name = str(tool.get("name") or "unknown") + provider = skill.package or skill.name + declared = skill.tool_refs.get(name) + if not declared: + declared = { + "package": provider, + "version": skill.version, + "entrypoint": skill.entrypoint, + "tool": name, + } + payload = json.dumps( + { + "declared": declared, + "input_schema": tool.get("inputSchema") or {}, + "output_schema": tool.get("outputSchema") or {}, + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + digest = hashlib.sha256(payload).hexdigest() + return f"{provider}/{name}@sha256:{digest}" + + +def unresolved_tool_ref(tool_name: str) -> str: + """Return a deterministic identity for a rejected unresolved request.""" + + digest = hashlib.sha256(tool_name.encode("utf-8")).hexdigest() + return f"unresolved/{tool_name}@sha256:{digest}" + + +def resolve_registration( + requested: str, + *, + tool_ref_registry: dict[str, str], + tool_name_by_ref: dict[str, str], + tool_registry: dict[str, str], + tool_ref_by_name: dict[str, str], +) -> tuple[str, str, str | None, str]: + """Resolve an immutable reference or the migration-only unique bare alias.""" + + if requested in tool_ref_registry: + return ( + tool_name_by_ref[requested], + requested, + tool_ref_registry[requested], + "immutable-tool-ref", + ) + skill_name = tool_registry.get(requested) + return ( + requested, + tool_ref_by_name.get(requested, unresolved_tool_ref(requested)), + skill_name, + "unique-bare-alias" if skill_name else "unresolved", + ) + + +def normalize_phases(phase: str | list[str] | None) -> list[str]: + """Coerce SkillConfig.phase into a flat list of phase strings.""" + + if phase is None: + return ["all"] + if isinstance(phase, str): + return [phase] + return [str(item) for item in phase] + + +def phase_matches(skill_phase: str | list[str], want: str) -> bool: + """Return whether a declared phase admits ``want``.""" + + phases = normalize_phases(skill_phase) + return want in phases or "all" in phases + + +def phase_is_disabled(skill_phase: str | list[str]) -> bool: + """Return whether a Skill is fully disabled by its phase declaration.""" + + phases = [phase for phase in normalize_phases(skill_phase) if phase] + return phases == ["none"] + + +MAX_RETRIES = 3 +RETRY_DELAY = 0.5 +DEFAULT_TOOL_TIMEOUT = 300 +SLOW_TOOL_TIMEOUT = 3_600 +VERY_SLOW_TOOL_TIMEOUT = 13 * 3_600 + +_TIMEOUT_CLASS_SECONDS = { + "default": DEFAULT_TOOL_TIMEOUT, + "bounded": DEFAULT_TOOL_TIMEOUT, + "slow": SLOW_TOOL_TIMEOUT, + "very-slow": VERY_SLOW_TOOL_TIMEOUT, + "async": DEFAULT_TOOL_TIMEOUT, +} + + +def resolve_tool_timeout( + args: dict, + *, + timeout_class: str | None = None, + timeout_budget: TimeoutBudgetV1 | dict | None = None, +) -> int: + """Resolve a manifest class and an explicitly declared per-call budget.""" + + if timeout_budget is not None: + budget = ( + timeout_budget + if isinstance(timeout_budget, TimeoutBudgetV1) + else TimeoutBudgetV1.model_validate(timeout_budget) + ) + requested = budget.requested_seconds(args) + if requested is not None: + return requested + return _TIMEOUT_CLASS_SECONDS.get(timeout_class or "default", DEFAULT_TOOL_TIMEOUT) + + +def log_tool_call(log: logging.Logger, tool_name: str, args: dict) -> None: + """Emit bounded argument diagnostics for selected propagation-sensitive tools.""" + + if tool_name not in {"make_metric_spec", "generate_ideas", "survey"}: + log.debug("[mcp] call_tool %s: args_keys=%s", tool_name, list(args)) + return + rendered = json.dumps(args, ensure_ascii=False) + log.info( + "[mcp] call_tool %s: args_len=%d args=%s", + tool_name, + len(rendered), + rendered[:500], + ) + + +def default_call_context(node_id: str | None = None) -> ToolCallContextV1: + """Derive compatibility context from the scoped checkpoint environment.""" + + checkpoint_dir = os.environ.get("ARI_CHECKPOINT_DIR", "").strip() + run_id = Path(checkpoint_dir.rstrip(os.sep)).name if checkpoint_dir else "" + return ToolCallContextV1( + run_id=run_id, + node_id=node_id, + ) + + +def enrich_call_context( + context: ToolCallContextV1, + *, + selection_reason: str, + skill: SkillConfig | None, +) -> ToolCallContextV1: + """Attach core-owned selection and value-free credential authority metadata.""" + + updates: dict[str, object] = {} + if not context.selection_reason: + updates["selection_reason"] = selection_reason + if not context.credential_scope_ids and skill is not None: + updates["credential_scope_ids"] = sorted( + str(identity.get("scope_id")) + for identity in skill.credential_scope_identities + if identity.get("scope_id") and identity.get("present_env") + ) + return context.model_copy(update=updates) if updates else context + + +__all__ = [ + "DEFAULT_TOOL_TIMEOUT", + "MAX_RETRIES", + "RETRY_DELAY", + "SLOW_TOOL_TIMEOUT", + "ToolNameCollisionError", + "VERY_SLOW_TOOL_TIMEOUT", + "default_call_context", + "enrich_call_context", + "log_tool_call", + "normalize_phases", + "phase_is_disabled", + "phase_matches", + "resolve_registration", + "resolve_tool_timeout", + "runtime_tool_ref", + "unresolved_tool_ref", +] diff --git a/ari-core/ari/mcp/invoke_runtime.py b/ari-core/ari/mcp/invoke_runtime.py new file mode 100644 index 00000000..ec149aea --- /dev/null +++ b/ari-core/ari/mcp/invoke_runtime.py @@ -0,0 +1,134 @@ +"""Retry, cancellation, and typed normalization for one admitted tool call.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import logging +import time +from typing import Callable, Protocol + +from ari.mcp.child_environment import ChildEnvironmentError +from ari.mcp.dispatch_support import MAX_RETRIES, RETRY_DELAY +from ari.result import ( + ResultArtifactIntegrityError, + ResultEnvelopeNormalizer, + ResultEnvelopeV1, + ResultErrorKind, + ToolCallContextV1, + utc_now_iso, +) + + +class ToolCallConnection(Protocol): + def call_tool(self, tool_name: str, args: dict, timeout: int) -> dict: ... + + def redact_text(self, value: str) -> str: ... + + +def _redact(connection: ToolCallConnection, message: str) -> str: + redact = getattr(connection, "redact_text", None) + return redact(message) if callable(redact) else message + + +def invoke_with_retries( + *, + connection: ToolCallConnection, + reconnect: Callable[[ToolCallConnection], ToolCallConnection], + tool_name: str, + tool_ref: str, + args: dict, + timeout: int, + context: ToolCallContextV1, + normalizer: ResultEnvelopeNormalizer, + started_at: str, + logger: logging.Logger, +) -> ResultEnvelopeV1: + """Invoke a provider and convert every transport outcome to one envelope.""" + + last_error = "" + last_kind: ResultErrorKind = "transport" + for attempt in range(1, MAX_RETRIES + 1): + try: + response = connection.call_tool(tool_name, args, timeout=timeout) + except ChildEnvironmentError as exc: + detail = _redact(connection, f"{type(exc).__name__}: {exc}".rstrip()) + return normalizer.error( + tool_ref=tool_ref, + kind="admission", + message=f"Tool '{tool_name}' provider environment was refused. {detail}", + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + except (asyncio.CancelledError, concurrent.futures.CancelledError) as exc: + detail = _redact(connection, f"{type(exc).__name__}: {exc}".rstrip()) + return normalizer.error( + tool_ref=tool_ref, + kind="cancelled", + message=f"Tool '{tool_name}' was cancelled. {detail}", + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + except TimeoutError as exc: + last_error = _redact( + connection, f"{type(exc).__name__}: {exc}".rstrip() + ) + last_kind = "timeout" + except Exception as exc: + last_error = _redact( + connection, f"{type(exc).__name__}: {exc}".rstrip() + ) + last_kind = "transport" + else: + try: + return normalizer.normalize_legacy( + response, + tool_ref=tool_ref, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + except (ResultArtifactIntegrityError, OSError) as exc: + return normalizer.error( + tool_ref=tool_ref, + kind="artifact-integrity", + message=str(exc), + retryable=False, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + + logger.warning( + "Tool '%s' attempt %d/%d failed: %s", + tool_name, + attempt, + MAX_RETRIES, + last_error, + ) + try: + connection = reconnect(connection) + except Exception: + pass + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY * attempt) + + return normalizer.error( + tool_ref=tool_ref, + kind=last_kind, + message=( + f"Tool '{tool_name}' failed after {MAX_RETRIES} attempts. " + f"Last: {last_error}" + ), + retryable=True, + context=context, + started_at=started_at, + completed_at=utc_now_iso(), + ) + + +__all__ = ["ToolCallConnection", "invoke_with_retries"] diff --git a/ari-core/ari/mcp/lock_runtime.py b/ari-core/ari/mcp/lock_runtime.py new file mode 100644 index 00000000..cd9afe17 --- /dev/null +++ b/ari-core/ari/mcp/lock_runtime.py @@ -0,0 +1,66 @@ +"""Small stateful bridge between MCP discovery and the run lock contract.""" + +from __future__ import annotations + +from pathlib import Path + +from ari.config import SkillConfig +from ari.skill_lock import ( + SkillsLockV1, + build_skills_lock, + verify_skills_lock_subset, + write_or_verify_skills_lock, +) + + +class SkillLockController: + """Own lock configuration and the snapshot reconciled for one MCP client.""" + + def __init__( + self, + path: str | Path | None, + *, + scope: str, + strict_provider_loading: bool | None, + ) -> None: + if scope not in {"exact", "subset"}: + raise ValueError("skill_lock_scope must be 'exact' or 'subset'") + self.path = Path(path) if path else None + self.scope = scope + self.strict_provider_loading = ( + self.path is not None + if strict_provider_loading is None + else strict_provider_loading + ) + self.snapshot: SkillsLockV1 | None = None + + def reconcile( + self, + *, + skills: list[SkillConfig], + tools: list[dict], + disabled_tools: set[str], + ) -> SkillsLockV1 | None: + """Create/verify an exact snapshot or verify a stage-worker subset.""" + + if self.path is None: + return None + current = build_skills_lock( + run_id=self.path.parent.name, + skills=skills, + tools=tools, + disabled_tools=disabled_tools, + ) + if self.scope == "subset": + self.snapshot = verify_skills_lock_subset(self.path, current) + else: + self.snapshot = write_or_verify_skills_lock(self.path, current) + return self.snapshot + + def clear(self) -> None: + """Forget a failed reconciliation without altering the on-disk lock.""" + + self.snapshot = None + + +__all__ = ["SkillLockController"] diff --git a/ari-core/ari/mcp/registry_runtime.py b/ari-core/ari/mcp/registry_runtime.py new file mode 100644 index 00000000..b3902508 --- /dev/null +++ b/ari-core/ari/mcp/registry_runtime.py @@ -0,0 +1,115 @@ +"""Live MCP discovery and immutable runtime registry assembly.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Callable, Protocol + +from ari.config import SkillConfig +from ari.mcp.dispatch_support import ( + ToolNameCollisionError, + phase_is_disabled, + runtime_tool_ref, +) +from ari.skill_lock import SkillProviderAdmissionError + + +logger = logging.getLogger(__name__) + + +class ToolDiscoveryConnection(Protocol): + def list_tools(self) -> list[dict]: ... + + +@dataclass(frozen=True) +class DiscoveredRegistry: + tools: list[dict] + owner_by_name: dict[str, str] + owner_by_ref: dict[str, str] + name_by_ref: dict[str, str] + ref_by_name: dict[str, str] + metadata_by_ref: dict[str, dict] + + +def discover_registry( + skills: list[SkillConfig], + *, + init_connection: Callable[[SkillConfig], ToolDiscoveryConnection], + close_all: Callable[[], None], + strict_provider_loading: bool, +) -> DiscoveredRegistry: + """Discover enabled providers and reject ambiguous runtime identities.""" + + tools: list[dict] = [] + owner_by_name: dict[str, str] = {} + owner_by_ref: dict[str, str] = {} + name_by_ref: dict[str, str] = {} + ref_by_name: dict[str, str] = {} + collisions: dict[str, set[str]] = {} + + for skill in skills: + if phase_is_disabled(getattr(skill, "phase", "all")): + logger.info("Skipping disabled skill '%s' (phase=none)", skill.name) + continue + try: + skill_tools = init_connection(skill).list_tools() + for raw_tool in skill_tools: + tool = dict(raw_tool) + tool_ref = runtime_tool_ref(skill, tool) + tool["tool_ref"] = tool_ref + capability_ref = skill.tool_capabilities.get(tool["name"]) + if capability_ref: + tool["capability_ref"] = capability_ref + policy = skill.tool_policies.get(tool["name"]) + if policy: + tool["policy"] = policy + + previous = owner_by_name.get(tool["name"]) + if previous is not None and previous != skill.name: + collisions.setdefault(tool["name"], {previous}).add(skill.name) + else: + owner_by_name[tool["name"]] = skill.name + ref_by_name[tool["name"]] = tool_ref + previous_ref = owner_by_ref.get(tool_ref) + if previous_ref is not None and previous_ref != skill.name: + raise ToolNameCollisionError( + f"immutable tool_ref collision: {tool_ref}" + ) + owner_by_ref[tool_ref] = skill.name + name_by_ref[tool_ref] = tool["name"] + tools.append(tool) + logger.info("Loaded %d tools from skill '%s'", len(skill_tools), skill.name) + except ToolNameCollisionError: + raise + except Exception as exc: + if strict_provider_loading: + close_all() + raise SkillProviderAdmissionError( + f"required MCP Skill '{skill.name}' failed live discovery: " + f"{type(exc).__name__}: {exc}" + ) from exc + logger.warning("Failed to load skill '%s': %s", skill.name, exc) + + if collisions: + rendered = "; ".join( + f"{name}: {', '.join(sorted(owners))}" + for name, owners in sorted(collisions.items()) + ) + close_all() + raise ToolNameCollisionError( + "Ambiguous MCP tool names are not admitted; configure one owner " + f"or use a namespaced registry: {rendered}" + ) + + return DiscoveredRegistry( + tools=tools, + owner_by_name=owner_by_name, + owner_by_ref=owner_by_ref, + name_by_ref=name_by_ref, + ref_by_name=ref_by_name, + metadata_by_ref={tool["tool_ref"]: tool for tool in tools}, + ) + + +__all__ = ["DiscoveredRegistry", "ToolDiscoveryConnection", "discover_registry"] diff --git a/ari-core/ari/mcp/secure_stdio_proxy.py b/ari-core/ari/mcp/secure_stdio_proxy.py new file mode 100644 index 00000000..d74bc92a --- /dev/null +++ b/ari-core/ari/mcp/secure_stdio_proxy.py @@ -0,0 +1,364 @@ +"""Value-redacting, exact-environment stdio proxy for direct MCP clients. + +Claude CLI and similar clients may merge their own parent environment into an +MCP server declaration. This proxy is the trust boundary: it launches the real +provider with exactly the environment names admitted by ARI, and removes known +credential values from provider stdout/stderr before forwarding either stream. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import signal +import subprocess +import sys +import threading +from typing import BinaryIO + +from ari.call_context import ( + CALL_CONTEXT_ARGUMENT, + CONTEXT_AUTHORITY_ENV, + ToolCallContextV1, + authorize_tool_context, + new_context_authority_key, +) + + +_ENV_NAME_RE = re.compile(r"[A-Z_][A-Z0-9_]*") + + +class _ByteRedactor: + def __init__(self, markers: dict[str, str], environment: dict[str, str]) -> None: + replacements: dict[bytes, bytes] = {} + for name, marker in markers.items(): + secret = environment.get(name, "") + if not secret: + continue + rendered = f"".encode("utf-8") + replacements[secret.encode("utf-8")] = rendered + escaped = json.dumps(secret, ensure_ascii=False)[1:-1].encode("utf-8") + replacements[escaped] = rendered + self._pairs = tuple( + sorted(replacements.items(), key=lambda item: len(item[0]), reverse=True) + ) + + def apply(self, payload: bytes) -> bytes: + for secret, marker in self._pairs: + payload = payload.replace(secret, marker) + return payload + + +def _load_spec( + raw: str, +) -> tuple[ + str, + list[str], + list[str], + dict[str, str], + dict[str, str], + ToolCallContextV1 | None, +]: + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid proxy spec JSON: {exc}") from exc + if not isinstance(document, dict): + raise ValueError("proxy spec must be an object") + command = document.get("command") + args = document.get("args", []) + env_names = document.get("env_names", []) + markers = document.get("credential_markers", {}) + context_requirements = document.get("context_requirements", {}) + raw_context = document.get("call_context") + if not isinstance(command, str) or not command: + raise ValueError("proxy command must be a non-empty string") + if not isinstance(args, list) or any(not isinstance(item, str) for item in args): + raise ValueError("proxy args must be strings") + if not isinstance(env_names, list) or any( + not isinstance(name, str) or not _ENV_NAME_RE.fullmatch(name) + for name in env_names + ): + raise ValueError("proxy env_names must contain canonical names") + if len(env_names) != len(set(env_names)): + raise ValueError("proxy env_names must be unique") + if not isinstance(markers, dict) or any( + name not in env_names + or not isinstance(marker, str) + or not marker + for name, marker in markers.items() + ): + raise ValueError("proxy credential_markers must reference admitted env names") + if not isinstance(context_requirements, dict) or any( + not isinstance(name, str) + or not name + or requirement not in {"run", "node"} + for name, requirement in context_requirements.items() + ): + raise ValueError("proxy context_requirements must map tools to run/node") + context = None + if context_requirements: + if raw_context is None: + raise ValueError("proxy context-requiring tools need call_context") + try: + context = ToolCallContextV1.model_validate(raw_context) + except Exception as exc: + raise ValueError("proxy call_context is malformed") from exc + unsatisfied = sorted( + name + for name, requirement in context_requirements.items() + if not context.satisfies(requirement) + ) + if unsatisfied: + raise ValueError( + f"proxy call_context does not authorize tools: {unsatisfied}" + ) + elif raw_context is not None: + raise ValueError("proxy call_context requires context_requirements") + return command, args, env_names, markers, context_requirements, context + + +def _copy_input( + source: BinaryIO, + target: BinaryIO, + *, + context_requirements: dict[str, str] | None = None, + call_context: ToolCallContextV1 | None = None, + authority_key: str | None = None, +) -> None: + try: + while chunk := source.readline(): + target.write( + _inject_call_context( + chunk, + context_requirements=context_requirements or {}, + call_context=call_context, + authority_key=authority_key, + ) + ) + target.flush() + except (BrokenPipeError, OSError, ValueError): + pass + finally: + try: + target.close() + except (OSError, ValueError): + pass + + +def _copy_redacted( + source: BinaryIO, + target: BinaryIO, + redactor: _ByteRedactor, + *, + sanitize_tool_schemas: bool = False, +) -> None: + try: + while chunk := source.readline(): + if sanitize_tool_schemas: + chunk = _strip_context_from_tool_schemas(chunk) + target.write(redactor.apply(chunk)) + target.flush() + except (BrokenPipeError, OSError, ValueError): + pass + + +def run_proxy(raw_spec: str) -> int: + ( + command, + args, + env_names, + markers, + context_requirements, + call_context, + ) = _load_spec(raw_spec) + environment = { + name: os.environ[name] + for name in env_names + if name in os.environ + } + environment.setdefault("PATH", os.defpath) + authority_key = None + if context_requirements: + authority_key = new_context_authority_key() + environment[CONTEXT_AUTHORITY_ENV] = authority_key + markers = { + **markers, + CONTEXT_AUTHORITY_ENV: "core.call-context-authority", + } + redactor = _ByteRedactor(markers, environment) + + popen_kwargs: dict[str, object] = { + "stdin": subprocess.PIPE, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "env": environment, + "bufsize": 0, + } + if os.name == "nt": # pragma: no cover - exercised in Windows CI + popen_kwargs["creationflags"] = getattr( + subprocess, "CREATE_NEW_PROCESS_GROUP", 0 + ) + else: + popen_kwargs["start_new_session"] = True + process = subprocess.Popen([command, *args], **popen_kwargs) + assert process.stdin is not None + assert process.stdout is not None + assert process.stderr is not None + + def _terminate(_signum, _frame) -> None: + try: + if os.name != "nt": + os.killpg(process.pid, signal.SIGTERM) + else: # pragma: no cover - exercised in Windows CI + process.terminate() + except (OSError, ProcessLookupError): + pass + + for signal_name in ("SIGTERM", "SIGINT"): + if hasattr(signal, signal_name): + signal.signal(getattr(signal, signal_name), _terminate) + + input_thread = threading.Thread( + target=_copy_input, + args=(sys.stdin.buffer, process.stdin), + kwargs={ + "context_requirements": context_requirements, + "call_context": call_context, + "authority_key": authority_key, + }, + daemon=True, + ) + stderr_thread = threading.Thread( + target=_copy_redacted, + args=(process.stderr, sys.stderr.buffer, redactor), + daemon=True, + ) + input_thread.start() + stderr_thread.start() + _copy_redacted( + process.stdout, + sys.stdout.buffer, + redactor, + sanitize_tool_schemas=True, + ) + return_code = process.wait() + stderr_thread.join(timeout=5) + return return_code + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", required=True, help="value-free JSON launch spec") + args = parser.parse_args(argv) + try: + return run_proxy(args.spec) + except (OSError, ValueError) as exc: + print(f"secure stdio proxy refused launch: {exc}", file=sys.stderr) + return 2 + + +def _inject_call_context( + payload: bytes, + *, + context_requirements: dict[str, str], + call_context: ToolCallContextV1 | None, + authority_key: str | None, +) -> bytes: + """Override any caller context with a proxy-issued capability.""" + + try: + document = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError): + return payload + + def inject(message): + if not isinstance(message, dict) or message.get("method") != "tools/call": + return message + params = message.get("params") + if not isinstance(params, dict): + return message + tool_name = params.get("name") + arguments = params.get("arguments") + if not isinstance(arguments, dict): + arguments = {} + else: + arguments = dict(arguments) + arguments.pop(CALL_CONTEXT_ARGUMENT, None) + requirement = context_requirements.get(str(tool_name)) + if requirement: + if call_context is None or authority_key is None: + return message + arguments[CALL_CONTEXT_ARGUMENT] = authorize_tool_context( + call_context, + tool_name=str(tool_name), + authority_key=authority_key, + ) + params = dict(params) + params["arguments"] = arguments + message = dict(message) + message["params"] = params + return message + + if isinstance(document, list): + document = [inject(message) for message in document] + else: + document = inject(document) + suffix = b"\n" if payload.endswith(b"\n") else b"" + return json.dumps( + document, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + suffix + + +def _strip_context_from_tool_schemas(payload: bytes) -> bytes: + """Remove the transport-only argument from tools/list responses.""" + + try: + document = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError): + return payload + if not isinstance(document, dict): + return payload + result = document.get("result") + tools = result.get("tools") if isinstance(result, dict) else None + if not isinstance(tools, list): + return payload + changed = False + for tool in tools: + schema = tool.get("inputSchema") if isinstance(tool, dict) else None + if not isinstance(schema, dict): + continue + properties = schema.get("properties") + if isinstance(properties, dict) and CALL_CONTEXT_ARGUMENT in properties: + schema["properties"] = { + name: value + for name, value in properties.items() + if name != CALL_CONTEXT_ARGUMENT + } + changed = True + required = schema.get("required") + if isinstance(required, list) and CALL_CONTEXT_ARGUMENT in required: + kept = [name for name in required if name != CALL_CONTEXT_ARGUMENT] + if kept: + schema["required"] = kept + else: + schema.pop("required", None) + changed = True + if not changed: + return payload + suffix = b"\n" if payload.endswith(b"\n") else b"" + return json.dumps( + document, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + suffix + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ari-core/ari/memory/README.md b/ari-core/ari/memory/README.md index e00214be..6b7c0329 100644 --- a/ari-core/ari/memory/README.md +++ b/ari-core/ari/memory/README.md @@ -2,13 +2,12 @@ Backend abstraction for ancestor-scoped node memory: `LettaMemoryClient` (default), `FileMemoryClient` (legacy JSONL), `LocalMemoryClient` (tests), -plus v0.5→v0.6 auto-migration. +plus explicit offline v0.5→v1 migration. ## Contents - `README.md` — this file. - `__init__.py` — `MemoryClient` protocol, backends, migration map. -- `auto_migrate.py` — v0.5.x → v0.6.0 auto-migration on first launch. - `backend.py` — sanctioned core→skill funnel: lazy forwards (`get_backend` / `clear_backend_cache` / `build_verified_context`) to the rich `MemoryBackend`. - `client.py` — abstract `MemoryClient` ABC. - `file_client.py` — `FileMemoryClient` (legacy JSONL). diff --git a/ari-core/ari/memory/__init__.py b/ari-core/ari/memory/__init__.py index 5d034d4f..cf9d95fa 100644 --- a/ari-core/ari/memory/__init__.py +++ b/ari-core/ari/memory/__init__.py @@ -22,7 +22,8 @@ **Funnel policy (the one allowed core→skill edge).** ``ari_skill_memory`` may be imported **only** from within ``ari/memory/**`` — specifically ``backend.py`` (the sanctioned forwards ``get_backend`` / ``clear_backend_cache`` / -``build_verified_context``), ``letta_client.py``, and ``auto_migrate.py``. +``build_verified_context``) and ``letta_client.py``. Legacy conversion is an +explicit offline command in ``ari.memory_cli``. Every other ``ari-core`` module reaches the skill backend through the ``ari.memory`` re-exports below, so the sanctioned edge (introduced v0.6.0) is confined to one directory and a future import-boundary checker (subtask 026) can @@ -38,7 +39,6 @@ """ from __future__ import annotations -from ari.memory.auto_migrate import maybe_auto_migrate from ari.memory.backend import ( build_verified_context, clear_backend_cache, @@ -54,7 +54,6 @@ "LettaMemoryClient", "FileMemoryClient", "LocalMemoryClient", - "maybe_auto_migrate", "get_backend", "clear_backend_cache", "build_verified_context", diff --git a/ari-core/ari/memory/auto_migrate.py b/ari-core/ari/memory/auto_migrate.py deleted file mode 100644 index 709909fd..00000000 --- a/ari-core/ari/memory/auto_migrate.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Auto-migrate v0.5.x checkpoints on first v0.6.0 launch. - -LETTA_BACKEND_SPEC.md §16.2. Callers (cli `run` / `resume` / `viz`) invoke -``maybe_auto_migrate(checkpoint_dir)`` once at startup. -""" -from __future__ import annotations - -import json -import logging -import os -import time -from pathlib import Path - -log = logging.getLogger(__name__) - - -def _has_source(ckpt: Path) -> dict[str, Path]: - sources: dict[str, Path] = {} - node = ckpt / "memory_store.jsonl" - if node.exists(): - sources["node_scope"] = node - react = ckpt / "memory.json" - if react.exists(): - sources["react_step"] = react - return sources - - -def maybe_auto_migrate(checkpoint_dir: str | Path) -> dict: - """Run a one-shot import if v0.5.x files exist and Letta is empty. - - Returns a status dict; never raises — the caller keeps going either - way (failed migrations surface in the dashboard banner per §16.2). - """ - ckpt = Path(checkpoint_dir).expanduser().resolve() - if not ckpt.is_dir(): - return {"ran": False, "reason": "checkpoint_dir not a directory"} - - sources = _has_source(ckpt) - if not sources: - return {"ran": False, "reason": "no v0.5.x source files"} - - # Surface legacy global_memory.jsonl (removed in v0.6.0 per §3). - global_path = Path.home() / ".ari" / "global_memory.jsonl" - if global_path.exists(): - log.warning( - "WARNING: %s found — global memory is removed in v0.6.0. " - "See LETTA_BACKEND_SPEC.md §3. File left untouched.", - global_path, - ) - - from ari.paths import PathManager - PathManager.set_checkpoint_dir_env(ckpt) - try: - from ari_skill_memory.backends import get_backend - backend = get_backend(checkpoint_dir=ckpt) - except Exception as e: - return {"ran": False, "reason": f"backend unavailable: {e}"} - - # If the checkpoint's Letta collections already have content, skip. - try: - already_node = sum( - len(v) for v in backend.list_all_nodes().get("by_node", {}).values() - ) - except Exception: - already_node = 0 - try: - already_react = len(backend.list_react_entries()) - except Exception: - already_react = 0 - - imported: dict[str, int] = {} - ts = int(time.time()) - - if "node_scope" in sources and already_node == 0: - src = sources["node_scope"] - entries = _load_jsonl(src) - if entries: - res = backend.bulk_import(entries, kind="node_scope") - imported["node_scope"] = int(res.get("imported", 0)) - src.rename(ckpt / f"memory_store.jsonl.migrated-{ts}") - - if "react_step" in sources and already_react == 0: - src = sources["react_step"] - entries = _load_json_list(src) - if entries: - res = backend.bulk_import(entries, kind="react_step") - imported["react_step"] = int(res.get("imported", 0)) - src.rename(ckpt / f"memory.json.migrated-{ts}") - - if not imported: - return {"ran": True, "imported": {}} - - log.info("auto-migrate imported %s", imported) - return {"ran": True, "imported": imported} - - -def _load_jsonl(path: Path) -> list[dict]: - out: list[dict] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - try: - out.append(json.loads(line)) - except json.JSONDecodeError: - continue - return out - - -def _load_json_list(path: Path) -> list[dict]: - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return [] - return data if isinstance(data, list) else [] - - -__all__ = ["maybe_auto_migrate"] diff --git a/ari-core/ari/memory/backend.py b/ari-core/ari/memory/backend.py index 0c140a74..c86e988c 100644 --- a/ari-core/ari/memory/backend.py +++ b/ari-core/ari/memory/backend.py @@ -1,6 +1,6 @@ """Sanctioned funnel for the core→skill memory edge. -Together with the sibling ``letta_client.py`` and ``auto_migrate.py``, this +Together with the sibling ``letta_client.py``, this module is the **sole** place in ``ari-core`` that reaches into the ``ari-skill-memory`` package. Every other ``ari-core`` module must obtain a ``MemoryBackend`` (or build verified context) through ``ari.memory`` — which diff --git a/ari-core/ari/memory_cli.py b/ari-core/ari/memory_cli.py index 633bd5b9..3fa7dc5f 100644 --- a/ari-core/ari/memory_cli.py +++ b/ari-core/ari/memory_cli.py @@ -14,12 +14,13 @@ from __future__ import annotations import gzip +import hashlib import json import logging import os import shutil import subprocess -import sys +import tempfile import time from pathlib import Path @@ -29,6 +30,7 @@ log = logging.getLogger(__name__) memory_app = typer.Typer(name="memory", help="Letta memory admin commands") console = Console() +_MAX_BACKUP_BYTES = 512 * 1024 * 1024 def _resolve_ckpt(path: "str | Path | None", scan: bool = False) -> Path: @@ -52,62 +54,255 @@ def _get_backend(checkpoint_dir: Path): # ─ migrate ──────────────────────────────────────────────────────────── +def _load_legacy_jsonl(path: Path) -> list[dict]: + entries: list[dict] = [] + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"{path.name}:{line_number} is not valid JSON" + ) from exc + if not isinstance(value, dict): + raise ValueError(f"{path.name}:{line_number} must contain an object") + entries.append(value) + return entries + + +def _legacy_record(entry: dict, *, ordinal: int) -> dict: + """Convert one v0.5 node entry into a conservative v1 index record.""" + + from ari.public.memory import build_memory_record + + metadata = dict(entry.get("metadata") or {}) + node_id = str(entry.get("node_id") or metadata.get("node_id") or "") + text = str(entry.get("text") or entry.get("content") or "") + if not node_id or not text: + raise ValueError(f"legacy memory entry {ordinal} lacks node_id or text") + legacy_kind = str(metadata.get("mem_kind") or metadata.get("type") or "") + supported = { + "observation", + "experiment_result", + "failure_case", + "procedure", + "reflection", + "artifact_summary", + "paper_claim", + } + kind = legacy_kind if legacy_kind in supported else "observation" + ancestors = entry.get("ancestor_ids") or metadata.get("ancestor_ids") or [] + if not isinstance(ancestors, list): + raise ValueError(f"legacy memory entry {ordinal} has invalid ancestor_ids") + run_id = str(entry.get("run_id") or metadata.get("run_id") or "legacy-v0.5") + + artifact_refs: list[dict] = [] + for raw_ref in metadata.get("artifact_refs") or []: + if not isinstance(raw_ref, dict): + continue + digest = str(raw_ref.get("digest") or raw_ref.get("sha256") or "") + if digest and not digest.startswith("sha256:"): + digest = "sha256:" + digest + path = str(raw_ref.get("relative_path") or raw_ref.get("path") or "") + if len(digest) != 71 or not path: + continue + artifact_refs.append( + { + "relative_path": path, + "digest": digest, + "size_bytes": int(raw_ref.get("size_bytes") or 0), + "role": str(raw_ref.get("role") or "legacy-unknown"), + # A legacy hash has not been re-read from the migrated host. + "integrity_status": "unverified", + } + ) + + metric_ptr = metadata.get("metric_ptr") + if not ( + isinstance(metric_ptr, dict) + and metric_ptr.get("name") + and metric_ptr.get("unit") + and isinstance(metric_ptr.get("value"), (int, float)) + and not isinstance(metric_ptr.get("value"), bool) + ): + metric_ptr = None + attributes = { + "legacy_schema": "ari.memory-store/v0.5", + "legacy_entry_digest": _canonical_digest(entry), + "legacy_kind": legacy_kind or None, + "legacy_metadata": metadata, + } + return build_memory_record( + kind=kind, + text=text, + source_run_id=run_id, + source_node_id=node_id, + ancestor_node_ids=[str(value) for value in ancestors], + artifact_refs=artifact_refs, + metric_ptr=metric_ptr, + confidence=( + metadata.get("confidence") + if isinstance(metadata.get("confidence"), (int, float)) + and not isinstance(metadata.get("confidence"), bool) + else None + ), + created_by_tool_ref="ari-memory-migrate:v05", + attributes=attributes, + ).model_dump(mode="json") + + +def _archive_legacy_source(path: Path) -> Path: + stamp = time.time_ns() + target = path.with_name(f"{path.name}.migrated-{stamp}") + path.rename(target) + return target + + +def _do_migrate( + ckpt: Path, + *, + include_react: bool = False, + dry_run: bool = False, +) -> dict: + """Offline, idempotent v0.5 JSONL -> MemoryRecordV1 migration.""" + + src_node = ckpt / "memory_store.jsonl" + src_react = ckpt / "memory.json" + legacy_nodes = _load_legacy_jsonl(src_node) if src_node.exists() else [] + records = [ + _legacy_record(entry, ordinal=index) + for index, entry in enumerate(legacy_nodes, start=1) + ] + react_entries: list[dict] = [] + if include_react and src_react.exists(): + try: + raw_react = json.loads(src_react.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError("memory.json is not valid JSON") from exc + if not isinstance(raw_react, list) or not all( + isinstance(item, dict) for item in raw_react + ): + raise ValueError("memory.json must contain a list of objects") + react_entries = raw_react + if dry_run: + return { + "records": len(records), + "react_entries": len(react_entries), + "imported": 0, + "archived_sources": [], + "dry_run": True, + } + + backend = _get_backend(ckpt) + existing_entries = [ + entry + for entries in backend.list_all_nodes().get("by_node", {}).values() + for entry in entries + ] + unversioned = [ + entry + for entry in existing_entries + if not (entry.get("metadata") or {}).get("record_digest") + ] + if unversioned: + raise MemoryBackupIntegrityError( + "target backend already contains unversioned memory; migrate into " + "an empty backend or restore a canonical backup" + ) + existing_digests = _validated_backend_record_digests(backend) + node_entries = [ + { + "node_id": record["source_node_id"], + "text": record["text"], + "metadata": _record_metadata(record), + "ts": float(index), + } + for index, record in enumerate(records) + if record["record_digest"] not in existing_digests + ] + imported = 0 + if node_entries: + imported += backend.bulk_import(node_entries, kind="node_scope")["imported"] + existing_react_digests = { + _canonical_digest( + { + "content": str(entry.get("content") or ""), + "metadata": dict(entry.get("metadata") or {}), + "ts": float(entry.get("ts") or 0.0), + } + ) + for entry in backend.list_react_entries() + } + normalized_react = [] + for entry in react_entries: + payload = { + "content": str(entry.get("content") or entry.get("text") or ""), + "metadata": dict(entry.get("metadata") or {}), + "ts": float(entry.get("ts") or 0.0), + } + if _canonical_digest(payload) not in existing_react_digests: + normalized_react.append( + { + "text": payload["content"], + "metadata": payload["metadata"], + "ts": payload["ts"], + } + ) + if normalized_react: + imported += backend.bulk_import( + normalized_react, kind="react_step" + )["imported"] + + backup = _do_backup(ckpt) + archived: list[str] = [] + if src_node.exists(): + archived.append(str(_archive_legacy_source(src_node))) + if include_react and src_react.exists(): + archived.append(str(_archive_legacy_source(src_react))) + return { + "records": len(records), + "react_entries": len(react_entries), + "imported": imported, + "skipped": ( + len(records) - len(node_entries) + + len(react_entries) - len(normalized_react) + ), + "archived_sources": archived, + "backup": backup, + "dry_run": False, + } + + @memory_app.command("migrate") def migrate_cmd( checkpoint: "Path | None" = typer.Option(None, help="Checkpoint directory"), react: bool = typer.Option(False, "--react", help="Also migrate memory.json"), dry_run: bool = typer.Option(False, "--dry-run"), ) -> None: - """Import v0.5.x JSONL data into the checkpoint's Letta collections.""" + """Offline import of v0.5 JSONL as content-addressed v1 records.""" ckpt = _resolve_ckpt(checkpoint) if not ckpt.is_dir(): console.print(f"[red]Not a directory: {ckpt}[/red]") raise typer.Exit(1) - src_node = ckpt / "memory_store.jsonl" - src_react = ckpt / "memory.json" - - node_entries: list[dict] = [] - react_entries: list[dict] = [] - - if src_node.exists(): - for line in src_node.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - try: - node_entries.append(json.loads(line)) - except json.JSONDecodeError: - continue - - if react and src_react.exists(): - try: - data = json.loads(src_react.read_text(encoding="utf-8")) - if isinstance(data, list): - react_entries = data - except json.JSONDecodeError as e: - console.print(f"[red]memory.json parse failed: {e}[/red]") - raise typer.Exit(2) - - console.print(f"[cyan]node entries: {len(node_entries)}[/cyan]") + try: + result = _do_migrate(ckpt, include_react=react, dry_run=dry_run) + except (ValueError, OSError) as exc: + console.print(f"[red]migration failed: {exc}[/red]") + raise typer.Exit(2) from exc + console.print(f"[cyan]node records: {result['records']}[/cyan]") if react: - console.print(f"[cyan]react entries: {len(react_entries)}[/cyan]") + console.print(f"[cyan]react entries: {result['react_entries']}[/cyan]") if dry_run: - console.print("[yellow]--dry-run: no writes.[/yellow]") - return - - backend = _get_backend(ckpt) - ts = int(time.time()) - - if node_entries: - backend.bulk_import(node_entries, kind="node_scope") - src_node.rename(ckpt / f"memory_store.jsonl.migrated-{ts}") - console.print(f"[green]✓ imported {len(node_entries)} node entries[/green]") - - if react and react_entries: - backend.bulk_import(react_entries, kind="react_step") - src_react.rename(ckpt / f"memory.json.migrated-{ts}") - console.print(f"[green]✓ imported {len(react_entries)} react entries[/green]") + console.print("[yellow]--dry-run: validated without writes.[/yellow]") + else: + console.print( + f"[green]✓ imported {result['imported']} entries and wrote " + f"{result['backup']['backup_digest']}[/green]" + ) # Global memory: detect but do not migrate. # Phase 5 (REFACTORING.md §8) parks the legacy path in @@ -118,96 +313,259 @@ def migrate_cmd( if global_path.exists(): console.print( f"[yellow]WARNING: {global_path} found — global memory is removed in " - "v0.6.0.[/yellow]" + "v0.6.0 and is intentionally not imported.[/yellow]" ) # ─ backup / restore ─────────────────────────────────────────────────── +class MemoryBackupIntegrityError(ValueError): + """A portable memory backup is corrupt or contains unsupported records.""" + + +def _canonical_digest(value: object) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _record_metadata(record: dict) -> dict: + return { + "memory_record": record, + "record_digest": record["record_digest"], + "type": record["kind"], + "mem_kind": record["kind"], + "metric_ptr": record.get("metric_ptr"), + "artifact_refs": record.get("artifact_refs") or [], + "node_report_ref": record.get("node_report_ref"), + "repro_target_id": record.get("repro_target_id"), + "repro_status": record.get("repro_status"), + "confidence": record.get("confidence"), + } + + +def _validated_backend_record_digests(backend: object) -> set[str]: + from ari.public.memory import MemoryRecordV1 + + digests: set[str] = set() + by_node = backend.list_all_nodes().get("by_node", {}) + for node_id, entries in by_node.items(): + for entry in entries: + metadata = entry.get("metadata") or {} + raw = metadata.get("memory_record") + if not isinstance(raw, dict): + raise MemoryBackupIntegrityError( + "target backend contains an unversioned memory record" + ) + try: + record = MemoryRecordV1.model_validate(raw) + except ValueError as exc: + raise MemoryBackupIntegrityError( + "target backend contains an invalid canonical memory record" + ) from exc + if ( + record.source_node_id != node_id + or record.text != entry.get("text", "") + or metadata.get("record_digest") != record.record_digest + ): + raise MemoryBackupIntegrityError( + "target backend projection disagrees with MemoryRecordV1" + ) + if record.record_digest in digests: + raise MemoryBackupIntegrityError( + "target backend contains duplicate canonical memory records" + ) + digests.add(record.record_digest) + return digests + + def _backup_path(ckpt: Path) -> Path: - return ckpt / "memory_backup.jsonl.gz" + return ckpt / "memory_backup.v1.json.gz" -def _do_backup(ckpt: Path) -> int: +def _do_backup(ckpt: Path) -> dict: + from ari.public.memory import ( + MemoryRecordV1, + build_memory_backup, + build_memory_react_entry, + ) + backend = _get_backend(ckpt) path = _backup_path(ckpt) - n = 0 - with gzip.open(path, "wt", encoding="utf-8") as f: - # node_scope - for nid, entries in backend.list_all_nodes().get("by_node", {}).items(): - for e in entries: - f.write(json.dumps({ - "kind": "node_scope", - "node_id": nid, - "text": e["text"], - "metadata": e["metadata"], - "ts": e["ts"], - }, ensure_ascii=False) + "\n") - n += 1 - # react_step - for e in backend.list_react_entries(): - f.write(json.dumps({ - "kind": "react_step", - "text": e["content"], - "metadata": e["metadata"], - "ts": e["ts"], - }, ensure_ascii=False) + "\n") - n += 1 - # core_seed - ctx = backend.get_experiment_context() - if ctx: - f.write(json.dumps({ - "kind": "core_seed", - "persona": "", - "human": "", - "context": ctx, - "seeded_at": ctx.get("seeded_at", 0.0), - }, ensure_ascii=False) + "\n") - n += 1 - return n + records: list[dict] = [] + record_order: list[str] = [] + by_node = backend.list_all_nodes().get("by_node", {}) + for node_id in sorted(by_node): + entries = by_node[node_id] + for entry in entries: + raw = (entry.get("metadata") or {}).get("memory_record") + if not isinstance(raw, dict): + raise MemoryBackupIntegrityError( + "runtime backup refuses an unversioned memory record; " + "run the offline `ari memory migrate` command first" + ) + record = MemoryRecordV1.model_validate(raw) + if record.source_node_id != node_id or record.text != entry.get("text", ""): + raise MemoryBackupIntegrityError( + "memory backend projection disagrees with MemoryRecordV1" + ) + normalized = record.model_dump(mode="json") + records.append(normalized) + record_order.append(normalized["record_digest"]) + records.sort(key=lambda item: item["record_digest"]) + + react: list[dict] = [] + for entry in backend.list_react_entries(): + item = build_memory_react_entry( + content=str(entry.get("content") or ""), + metadata=dict(entry.get("metadata") or {}), + ts=float(entry.get("ts") or 0.0), + ).model_dump(mode="json") + react.append(item) + react.sort(key=lambda item: item["entry_digest"]) + context = dict(backend.get_experiment_context() or {}) + # Empty backends expose ``seeded_at=0`` as a convenience projection; it is + # not a real core-memory record and must not create a phantom backup entry. + if set(context) <= {"seeded_at"} and not context.get("seeded_at"): + context = {} + document = build_memory_backup( + records=records, + react_entries=react, + core_context=context, + record_digests=[item["record_digest"] for item in records], + record_order=record_order, + ).model_dump(mode="json") + raw_payload = ( + json.dumps( + document, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + path.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: + temp_path = Path(temporary.name) + with gzip.GzipFile( + fileobj=temporary, mode="wb", filename="", mtime=0 + ) as stream: + stream.write(raw_payload) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temp_path, path) + temp_path = None + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + compressed = path.read_bytes() + return { + "entries": len(records) + len(react) + int(bool(context)), + "records": len(records), + "react_entries": len(react), + "backup_digest": document["backup_digest"], + "artifact_digest": "sha256:" + hashlib.sha256(compressed).hexdigest(), + "size_bytes": len(compressed), + "path": str(path), + } + + +def _load_backup(path: Path) -> dict: + from ari.public.memory import MemoryBackupV1 + + try: + if path.stat().st_size > _MAX_BACKUP_BYTES: + raise MemoryBackupIntegrityError("compressed memory backup exceeds size limit") + with gzip.open(path, "rb") as stream: + payload = stream.read(_MAX_BACKUP_BYTES + 1) + if len(payload) > _MAX_BACKUP_BYTES: + raise MemoryBackupIntegrityError("expanded memory backup exceeds size limit") + raw_document = json.loads(payload.decode("utf-8")) + except MemoryBackupIntegrityError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise MemoryBackupIntegrityError("memory backup is not valid gzip JSON") from exc + try: + return MemoryBackupV1.model_validate(raw_document).model_dump(mode="json") + except ValueError as exc: + raise MemoryBackupIntegrityError(f"memory backup validation failed: {exc}") from exc def _do_restore(ckpt: Path, on_conflict: str = "skip") -> dict: path = _backup_path(ckpt) if not path.exists(): return {"restored": 0, "reason": "no backup"} + if on_conflict not in {"skip", "overwrite", "merge"}: + raise ValueError("on_conflict must be skip, overwrite, or merge") + document = _load_backup(path) backend = _get_backend(ckpt) if on_conflict == "overwrite": backend.purge_checkpoint() - node_entries: list[dict] = [] - react_entries: list[dict] = [] - core_entries: list[dict] = [] - with gzip.open(path, "rt", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - e = json.loads(line) - except json.JSONDecodeError: - continue - kind = e.get("kind") - if kind == "node_scope": - node_entries.append(e) - elif kind == "react_step": - react_entries.append(e) - elif kind == "core_seed": - core_entries.append(e) - - if on_conflict == "skip": - # naive dedupe by (ts, text) presence - existing_node = { - (e.get("ts"), e.get("text")) - for entries in backend.list_all_nodes().get("by_node", {}).values() - for e in entries - } - node_entries = [e for e in node_entries - if (e.get("ts"), e.get("text")) not in existing_node] - existing_react = { - (e.get("ts"), e.get("content")) for e in backend.list_react_entries() - } - react_entries = [e for e in react_entries - if (e.get("ts"), e.get("text")) not in existing_react] + existing_records = _validated_backend_record_digests(backend) + node_entries = [] + skipped = 0 + records_by_digest = { + record["record_digest"]: record for record in document["records"] + } + for order_index, record_digest in enumerate(document["record_order"]): + record = records_by_digest[record_digest] + if record["record_digest"] in existing_records: + skipped += 1 + continue + node_entries.append( + { + "node_id": record["source_node_id"], + "text": record["text"], + "metadata": _record_metadata(record), + # A portable logical clock preserves append order, including + # the latest reproducibility event, without wall-clock noise. + "ts": float(order_index), + } + ) + existing_react = { + _canonical_digest( + { + "content": str(entry.get("content") or ""), + "metadata": dict(entry.get("metadata") or {}), + "ts": float(entry.get("ts") or 0.0), + } + ) + for entry in backend.list_react_entries() + } + react_entries = [] + for item in document.get("react_entries") or []: + if item["entry_digest"] in existing_react: + skipped += 1 + continue + react_entries.append( + { + "text": item["content"], + "metadata": item["metadata"], + "ts": item["ts"], + } + ) + target_context = document.get("core_context") or {} + current_context = dict(backend.get_experiment_context() or {}) + if set(current_context) <= {"seeded_at"} and not current_context.get("seeded_at"): + current_context = {} + core_entries = [] + if target_context and target_context != current_context: + core_entries = [{"persona": "", "human": "", "context": target_context}] + elif target_context: + skipped += 1 total = 0 if node_entries: @@ -216,21 +574,37 @@ def _do_restore(ckpt: Path, on_conflict: str = "skip") -> dict: total += backend.bulk_import(react_entries, kind="react_step")["imported"] if core_entries: total += backend.bulk_import(core_entries, kind="core_seed")["imported"] - return {"restored": total} + restored_digests = _validated_backend_record_digests(backend) + missing = sorted(set(document["record_digests"]) - restored_digests) + if missing: + raise MemoryBackupIntegrityError( + f"restored backend is missing {len(missing)} memory record digests" + ) + return { + "restored": total, + "skipped": skipped, + "backup_digest": document["backup_digest"], + "record_digests": document["record_digests"], + "record_order": document["record_order"], + "conflict_policy": on_conflict, + } @memory_app.command("backup") def backup_cmd( checkpoint: "Path | None" = typer.Option(None, help="Checkpoint directory"), ) -> None: - """Snapshot Letta-stored memory to ``{ckpt}/memory_backup.jsonl.gz``.""" + """Snapshot canonical memory to ``{ckpt}/memory_backup.v1.json.gz``.""" ckpt = _resolve_ckpt(checkpoint) try: - n = _do_backup(ckpt) + result = _do_backup(ckpt) except Exception as e: console.print(f"[red]backup failed: {e}[/red]") raise typer.Exit(1) - console.print(f"[green]✓ wrote {_backup_path(ckpt)} ({n} entries)[/green]") + console.print( + f"[green]✓ wrote {_backup_path(ckpt)} " + f"({result['entries']} entries, {result['backup_digest']})[/green]" + ) @memory_app.command("restore") @@ -238,7 +612,7 @@ def restore_cmd( checkpoint: "Path | None" = typer.Option(None, help="Checkpoint directory"), on_conflict: str = typer.Option("skip", help="skip|overwrite|merge"), ) -> None: - """Import ``memory_backup.jsonl.gz`` into Letta.""" + """Validate and import ``memory_backup.v1.json.gz`` into Letta.""" ckpt = _resolve_ckpt(checkpoint) res = _do_restore(ckpt, on_conflict=on_conflict) console.print(f"[green]✓ restored {res['restored']} entries[/green]") @@ -402,6 +776,8 @@ def _detect_deployment() -> str: __all__ = [ "memory_app", "_do_backup", + "_do_migrate", "_do_restore", "_detect_deployment", + "MemoryBackupIntegrityError", ] diff --git a/ari-core/ari/memory_contract.py b/ari-core/ari/memory_contract.py new file mode 100644 index 00000000..bc9deb3f --- /dev/null +++ b/ari-core/ari/memory_contract.py @@ -0,0 +1,424 @@ +"""Versioned, provider-neutral research-memory contracts.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import PurePosixPath +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +MEMORY_RECORD_V1 = "ari.memory-record/v1" +MEMORY_RETRIEVAL_V1 = "ari.memory-retrieval/v1" +MEMORY_BACKUP_V1 = "ari.memory-backup/v1" +_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$") + +MemoryKindV1 = Literal[ + "observation", + "experiment_result", + "failure_case", + "procedure", + "reflection", + "artifact_summary", + "paper_claim", + "reproducibility_event", +] +ReproStatusV1 = Literal[ + "unverified", + "rerun_passed", + "rerun_failed", + "paper_only_reproduced", +] + + +def canonical_memory_digest(value: Any) -> str: + value = _json_value(value) + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _json_value(value: Any) -> Any: + if isinstance(value, BaseModel): + return _json_value(value.model_dump(mode="json")) + if isinstance(value, dict): + return {key: _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + +def _digest(value: str, field: str) -> str: + if not _DIGEST_RE.fullmatch(value): + raise ValueError(f"{field} must use sha256:<64 lowercase hex> format") + return value + + +def _identifier(value: str, field: str) -> str: + if not _ID_RE.fullmatch(value): + raise ValueError(f"{field} contains unsupported characters") + return value + + +class MemoryArtifactRefV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + relative_path: str + digest: str + size_bytes: int = Field(ge=0) + role: str = Field(min_length=1, max_length=128) + integrity_status: Literal["verified", "unverified"] + + @field_validator("relative_path") + @classmethod + def _path(cls, value: str) -> str: + pure = PurePosixPath(value) + if ( + not value + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + ): + raise ValueError("memory artifact path must be safe and relative") + return value + + @field_validator("digest") + @classmethod + def _sha256(cls, value: str) -> str: + return _digest(value, "memory artifact digest") + + +class MemoryNodeReportRefV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + run_id: str + node_id: str + digest: str + + @field_validator("run_id", "node_id") + @classmethod + def _ids(cls, value: str, info: Any) -> str: + return _identifier(value, info.field_name) + + @field_validator("digest") + @classmethod + def _sha256(cls, value: str) -> str: + return _digest(value, "node report digest") + + +class MemoryMetricPointerV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + name: str = Field(min_length=1, max_length=256) + value: float + unit: str = Field(min_length=1, max_length=128) + + @field_validator("value") + @classmethod + def _finite(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("memory metric value must be finite") + return value + + @field_validator("unit") + @classmethod + def _unit(cls, value: str) -> str: + if not value.strip(): + raise ValueError("memory metric unit must be explicit") + return value.strip() + + +class _MemoryRecordPayloadV1(BaseModel): + """Validated canonical payload before its content address is assigned.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.memory-record/v1"] = MEMORY_RECORD_V1 + kind: MemoryKindV1 + text: str = Field(min_length=1, max_length=100_000) + source_run_id: str + source_node_id: str + ancestor_node_ids: list[str] = Field(default_factory=list, max_length=10_000) + artifact_refs: list[MemoryArtifactRefV1] = Field(default_factory=list, max_length=1_024) + node_report_ref: MemoryNodeReportRefV1 | None = None + metric_ptr: MemoryMetricPointerV1 | None = None + confidence: float | None = Field(default=None, ge=0.0, le=1.0) + repro_target_id: str | None = None + repro_status: ReproStatusV1 | None = None + created_by_tool_ref: str + attributes: dict[str, Any] = Field(default_factory=dict, max_length=256) + + @field_validator("source_run_id", "source_node_id", "created_by_tool_ref") + @classmethod + def _ids(cls, value: str, info: Any) -> str: + return _identifier(value, info.field_name) + + @field_validator("ancestor_node_ids") + @classmethod + def _ancestors(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise ValueError("memory ancestor lineage contains duplicates") + for node_id in value: + _identifier(node_id, "ancestor_node_ids") + return value + + @field_validator("repro_target_id") + @classmethod + def _target(cls, value: str | None) -> str | None: + if value is not None: + _digest(value, "repro_target_id") + return value + + @field_validator("attributes") + @classmethod + def _json_attributes(cls, value: dict[str, Any]) -> dict[str, Any]: + # Round-trip validation also rejects NaN and non-JSON runtime objects. + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + return value + + @model_validator(mode="after") + def _consistent(self) -> "_MemoryRecordPayloadV1": + if self.source_node_id in self.ancestor_node_ids: + raise ValueError("memory source node cannot appear in its ancestor lineage") + if self.node_report_ref is not None and ( + self.node_report_ref.run_id != self.source_run_id + or self.node_report_ref.node_id != self.source_node_id + ): + raise ValueError( + "memory node-report reference must match its source run and node" + ) + if self.kind == "reproducibility_event": + if self.repro_target_id is None or self.repro_status is None: + raise ValueError( + "reproducibility_event requires target and status" + ) + elif self.repro_target_id is not None or self.repro_status is not None: + raise ValueError( + "reproducibility status is only valid on reproducibility events" + ) + return self + + +class MemoryRecordV1(_MemoryRecordPayloadV1): + """Immutable index record; artifacts, not text, remain evidence.""" + + record_id: str + record_digest: str + + @field_validator("record_id", "record_digest") + @classmethod + def _digests(cls, value: str, info: Any) -> str: + return _digest(value, info.field_name) + + @model_validator(mode="after") + def _content_addressed(self) -> "MemoryRecordV1": + payload = self.model_dump(mode="json", exclude={"record_id", "record_digest"}) + expected = canonical_memory_digest(payload) + if self.record_id != expected or self.record_digest != expected: + raise ValueError(f"memory record digest mismatch: expected {expected}") + return self + + @property + def artifact_grounded(self) -> bool: + return bool(self.artifact_refs) and all( + ref.integrity_status == "verified" for ref in self.artifact_refs + ) + + +def build_memory_record(**values: Any) -> MemoryRecordV1: + values = dict(values) + values.setdefault("schema_version", MEMORY_RECORD_V1) + unsigned = { + key: value + for key, value in values.items() + if key not in {"record_id", "record_digest"} + } + payload = _MemoryRecordPayloadV1.model_validate(unsigned).model_dump(mode="json") + digest = canonical_memory_digest(payload) + return MemoryRecordV1.model_validate( + {**payload, "record_id": digest, "record_digest": digest} + ) + + +class MemoryRetrievalProvenanceV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + backend: str = Field(min_length=1, max_length=256) + backend_version: str = Field(min_length=1, max_length=256) + server_version: str = Field(min_length=1, max_length=256) + model: str = Field(min_length=1, max_length=512) + model_version: str = Field(min_length=1, max_length=256) + ranking: str = Field(min_length=1, max_length=1_024) + deterministic: bool + query_digest: str + candidate_count: int = Field(ge=0) + returned_count: int = Field(ge=0) + limit: int = Field(ge=1, le=1_000) + filter_evidence: dict[str, Any] + + @field_validator("query_digest") + @classmethod + def _query_digest(cls, value: str) -> str: + return _digest(value, "memory query digest") + + @field_validator("filter_evidence") + @classmethod + def _filter_json(cls, value: dict[str, Any]) -> dict[str, Any]: + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + return value + + @model_validator(mode="after") + def _candidate_bound(self) -> "MemoryRetrievalProvenanceV1": + if self.returned_count > self.candidate_count: + raise ValueError("memory retrieval returned more rows than candidates") + return self + + +class MemoryRetrievalV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.memory-retrieval/v1"] = MEMORY_RETRIEVAL_V1 + results: list[dict[str, Any]] = Field(max_length=1_000) + provenance: MemoryRetrievalProvenanceV1 + + @model_validator(mode="after") + def _count(self) -> "MemoryRetrievalV1": + if len(self.results) != self.provenance.returned_count: + raise ValueError("memory retrieval returned_count is inconsistent") + return self + + +class _MemoryReactEntryPayloadV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + content: str = Field(max_length=1_000_000) + metadata: dict[str, Any] = Field(default_factory=dict, max_length=256) + ts: float + + @field_validator("metadata") + @classmethod + def _metadata_json(cls, value: dict[str, Any]) -> dict[str, Any]: + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + return value + + +class MemoryReactEntryV1(_MemoryReactEntryPayloadV1): + entry_digest: str + + @field_validator("entry_digest") + @classmethod + def _entry_digest_format(cls, value: str) -> str: + return _digest(value, "memory react entry digest") + + @model_validator(mode="after") + def _content_addressed(self) -> "MemoryReactEntryV1": + payload = self.model_dump(mode="json", exclude={"entry_digest"}) + expected = canonical_memory_digest(payload) + if self.entry_digest != expected: + raise ValueError( + f"memory react entry digest mismatch: expected {expected}" + ) + return self + + +def build_memory_react_entry(**values: Any) -> MemoryReactEntryV1: + payload = _MemoryReactEntryPayloadV1.model_validate(values).model_dump(mode="json") + return MemoryReactEntryV1.model_validate( + {**payload, "entry_digest": canonical_memory_digest(payload)} + ) + + +class _MemoryBackupPayloadV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.memory-backup/v1"] = MEMORY_BACKUP_V1 + records: list[MemoryRecordV1] = Field(max_length=1_000_000) + react_entries: list[MemoryReactEntryV1] = Field(max_length=1_000_000) + core_context: dict[str, Any] = Field(default_factory=dict, max_length=256) + record_digests: list[str] = Field(max_length=1_000_000) + record_order: list[str] = Field(max_length=1_000_000) + + @field_validator("record_digests", "record_order") + @classmethod + def _record_digest_formats(cls, value: list[str], info: Any) -> list[str]: + for digest in value: + _digest(digest, info.field_name) + return value + + @field_validator("core_context") + @classmethod + def _context_json(cls, value: dict[str, Any]) -> dict[str, Any]: + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + return value + + @model_validator(mode="after") + def _indexes(self) -> "_MemoryBackupPayloadV1": + record_digests = [record.record_digest for record in self.records] + if len(record_digests) != len(set(record_digests)): + raise ValueError("memory backup contains duplicate records") + if self.record_digests != sorted(record_digests): + raise ValueError("memory backup record digest index is inconsistent") + if ( + len(self.record_order) != len(set(self.record_order)) + or sorted(self.record_order) != self.record_digests + ): + raise ValueError("memory backup record order is inconsistent") + react_digests = [entry.entry_digest for entry in self.react_entries] + if react_digests != sorted(react_digests): + raise ValueError("memory backup react entry order is not canonical") + return self + + +class MemoryBackupV1(_MemoryBackupPayloadV1): + backup_digest: str + + @field_validator("backup_digest") + @classmethod + def _backup_digest_format(cls, value: str) -> str: + return _digest(value, "memory backup digest") + + @model_validator(mode="after") + def _content_addressed(self) -> "MemoryBackupV1": + payload = self.model_dump(mode="json", exclude={"backup_digest"}) + expected = canonical_memory_digest(payload) + if self.backup_digest != expected: + raise ValueError(f"memory backup digest mismatch: expected {expected}") + return self + + +def build_memory_backup(**values: Any) -> MemoryBackupV1: + values = dict(values) + values.setdefault("schema_version", MEMORY_BACKUP_V1) + unsigned = {key: value for key, value in values.items() if key != "backup_digest"} + payload = _MemoryBackupPayloadV1.model_validate(unsigned).model_dump(mode="json") + return MemoryBackupV1.model_validate( + {**payload, "backup_digest": canonical_memory_digest(payload)} + ) + + +__all__ = [ + "MEMORY_RECORD_V1", + "MEMORY_RETRIEVAL_V1", + "MEMORY_BACKUP_V1", + "MemoryBackupV1", + "MemoryArtifactRefV1", + "MemoryMetricPointerV1", + "MemoryNodeReportRefV1", + "MemoryRecordV1", + "MemoryRetrievalProvenanceV1", + "MemoryRetrievalV1", + "MemoryReactEntryV1", + "build_memory_backup", + "build_memory_react_entry", + "build_memory_record", + "canonical_memory_digest", +] diff --git a/ari-core/ari/migrations/README.md b/ari-core/ari/migrations/README.md index 35b078cd..1e6f40a1 100644 --- a/ari-core/ari/migrations/README.md +++ b/ari-core/ari/migrations/README.md @@ -1,13 +1,15 @@ # ari.migrations Migration shims that keep checkpoints from older releases readable, so new -feature code stays free of v0.5/v0.6 branching. Sub-packages host the -branching; canonical modules ship thin re-export shims. +feature code stays free of compatibility branching. Runtime admission never +imports these readers; they only produce read-only migration views. ## Contents - `README.md` — this file. - `__init__.py` — rationale + layout. +- `checkpoint.py` — digest-bound, read-only view of legacy paper/replay inputs. +- `skill_manifest.py` — in-memory conversion of unversioned Skill metadata; - `v05_to_v07/` — v0.5 → v0.7 migration helpers. - `README.md` — v05_to_v07 index. - `__init__.py` — subsystem map + deprecation plan. diff --git a/ari-core/ari/migrations/__init__.py b/ari-core/ari/migrations/__init__.py index fd2dbe07..cbc7eb7a 100644 --- a/ari-core/ari/migrations/__init__.py +++ b/ari-core/ari/migrations/__init__.py @@ -1,8 +1,10 @@ -"""Migration shims for older ARI checkpoint formats (Phase 5). +"""Read-only migration shims for older ARI formats. -Sub-packages here host code that ARI keeps around purely so existing -checkpoints created on older releases stay readable. The plan -(REFACTORING.md §8) is to keep new feature code free of v0.5 / v0.6 -branching by funnelling that branching through this package and -shipping thin re-exports at the historical import paths. +``checkpoint`` normalizes historical checkpoint paper and replay inputs while +digesting every consumed file. ``skill_manifest`` converts unversioned package +metadata in memory. Neither module is a runtime registration path; production +discovery accepts canonical manifests only. + +The ``v05_to_v07`` package contains the older memory and node-report migration +helpers retained for supported checkpoints. """ diff --git a/ari-core/ari/migrations/checkpoint.py b/ari-core/ari/migrations/checkpoint.py new file mode 100644 index 00000000..43e95f68 --- /dev/null +++ b/ari-core/ari/migrations/checkpoint.py @@ -0,0 +1,181 @@ +"""Read-only compatibility view for checkpoints created before Skill locks. + +Legacy layouts remain replayable data, but they are not runtime registration +inputs. This reader resolves the historical tree/paper locations into one +typed snapshot, records the digest of every consumed source, and never mutates +the checkpoint. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field + + +LEGACY_CHECKPOINT_VIEW_V1 = "ari.legacy-checkpoint-view/v1" +_MAX_MIGRATION_FILE_BYTES = 10_000_000 + + +class LegacyCheckpointError(ValueError): + """Raised when a legacy checkpoint cannot be read safely and completely.""" + + +class LegacyCheckpointViewV1(BaseModel): + """Normalized, immutable view used by migration and replay tooling.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.legacy-checkpoint-view/v1"] = ( + LEGACY_CHECKPOINT_VIEW_V1 + ) + checkpoint_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + tree_layout: Literal["tree.json", "nodes_tree.json", "node_*/tree.json"] + tree: dict[str, Any] + results: dict[str, Any] = Field(default_factory=dict) + paper_source: str = "" + paper_relative_path: str | None = None + replay_inputs: dict[str, Any] = Field(default_factory=dict) + source_digests: dict[str, str] + + +def _read_bytes(root: Path, path: Path) -> bytes: + try: + resolved = path.resolve(strict=True) + except OSError as exc: + raise LegacyCheckpointError(f"cannot resolve {path}: {exc}") from exc + try: + resolved.relative_to(root.resolve()) + except ValueError as exc: + raise LegacyCheckpointError(f"checkpoint source escapes root: {path}") from exc + if not resolved.is_file(): + raise LegacyCheckpointError(f"checkpoint source is not a file: {path}") + size = resolved.stat().st_size + if size > _MAX_MIGRATION_FILE_BYTES: + raise LegacyCheckpointError( + f"checkpoint source exceeds {_MAX_MIGRATION_FILE_BYTES} bytes: {path}" + ) + try: + return resolved.read_bytes() + except OSError as exc: + raise LegacyCheckpointError(f"cannot read {path}: {exc}") from exc + + +def _read_text(root: Path, path: Path) -> tuple[str, str]: + payload = _read_bytes(root, path) + try: + text = payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise LegacyCheckpointError(f"checkpoint source is not UTF-8: {path}") from exc + return text, f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def _read_json(root: Path, path: Path) -> tuple[dict[str, Any], str]: + text, digest = _read_text(root, path) + try: + value = json.loads(text) + except json.JSONDecodeError as exc: + raise LegacyCheckpointError(f"invalid JSON in {path}: {exc}") from exc + if not isinstance(value, dict): + raise LegacyCheckpointError(f"JSON root must be an object: {path}") + return value, digest + + +def _tree_source(root: Path) -> tuple[Path, str]: + for name in ("tree.json", "nodes_tree.json"): + candidate = root / name + if candidate.is_file(): + return candidate, name + candidates = sorted( + (path for path in root.glob("node_*/tree.json") if path.is_file()), + key=lambda path: (path.stat().st_mtime_ns, path.as_posix()), + reverse=True, + ) + for candidate in candidates: + if candidate.stat().st_size > 2: + return candidate, "node_*/tree.json" + raise LegacyCheckpointError(f"no legacy node tree found under {root}") + + +def load_legacy_checkpoint(path: str | Path) -> LegacyCheckpointViewV1: + """Load a digest-bound compatibility snapshot without writing any files.""" + + root = Path(path) + if not root.is_dir(): + raise LegacyCheckpointError(f"checkpoint directory not found: {root}") + digests: dict[str, str] = {} + + tree_path, layout = _tree_source(root) + tree, tree_digest = _read_json(root, tree_path) + tree_relative = tree_path.relative_to(root).as_posix() + digests[tree_relative] = tree_digest + if "nodes" not in tree: + raise LegacyCheckpointError(f"legacy tree has no nodes field: {tree_path}") + + results: dict[str, Any] = {} + results_path = root / "results.json" + if results_path.is_file(): + results, digests["results.json"] = _read_json(root, results_path) + + replay_inputs: dict[str, Any] = {} + experiment_path = root / "experiment.md" + if experiment_path.is_file(): + experiment, digests["experiment.md"] = _read_text(root, experiment_path) + replay_inputs["experiment_md"] = experiment + for name in ("launch_config.json", "settings.json"): + source = root / name + if source.is_file(): + replay_inputs[name.removesuffix(".json")], digests[name] = _read_json( + root, source + ) + workflow_path = root / "workflow.yaml" + if workflow_path.is_file(): + workflow_text, digests["workflow.yaml"] = _read_text(root, workflow_path) + try: + workflow = yaml.safe_load(workflow_text) or {} + except yaml.YAMLError as exc: + raise LegacyCheckpointError( + f"invalid YAML in {workflow_path}: {exc}" + ) from exc + if not isinstance(workflow, dict): + raise LegacyCheckpointError("legacy workflow root must be a mapping") + replay_inputs["workflow"] = workflow + + paper_source = "" + paper_relative_path = None + for relative in ( + "full_paper.tex", + "paper/full_paper.tex", + "experiment_section.tex", + ): + source = root / relative + if source.is_file(): + paper_source, digests[relative] = _read_text(root, source) + paper_relative_path = relative + break + + run_id = str(tree.get("run_id") or results.get("run_id") or root.name) + return LegacyCheckpointViewV1( + checkpoint_id=root.name, + run_id=run_id, + tree_layout=layout, + tree=tree, + results=results, + paper_source=paper_source, + paper_relative_path=paper_relative_path, + replay_inputs=replay_inputs, + source_digests=digests, + ) + + +__all__ = [ + "LEGACY_CHECKPOINT_VIEW_V1", + "LegacyCheckpointError", + "LegacyCheckpointViewV1", + "load_legacy_checkpoint", +] diff --git a/ari-core/ari/migrations/skill_manifest.py b/ari-core/ari/migrations/skill_manifest.py new file mode 100644 index 00000000..ecf90654 --- /dev/null +++ b/ari-core/ari/migrations/skill_manifest.py @@ -0,0 +1,90 @@ +"""Read-only conversion of pre-v1 Skill manifests. + +Runtime discovery must use :func:`ari.skill_manifest.load_skill_manifest` and +therefore rejects unversioned documents. This module is intentionally under the +migration namespace so old package metadata can be inspected and converted +without becoming an admission path again. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from ari.skill_manifest import ( + LEGACY_MCP_RESULT_V1, + SkillManifestError, + SkillManifestV1, + load_skill_manifest, +) + + +def _legacy_document(raw: dict, path: Path) -> dict: + package = path.parent.name + entrypoint = raw.get("entrypoint", "src/server.py") + if isinstance(entrypoint, str): + entrypoint = { + "transport": "stdio", + "command_kind": raw.get("runtime", "python"), + "module": entrypoint, + } + tools = [] + capability_prefix = package.removeprefix("ari-skill-").replace("-", ".") + for tool in raw.get("tools") or []: + if isinstance(tool, str): + tools.append( + { + "name": tool, + "capability_ref": f"ari.legacy.{capability_prefix}.{tool}", + } + ) + elif isinstance(tool, dict): + tools.append(tool) + return { + "schema_version": 1, + "name": raw.get("name") or package, + "package": package, + "version": str(raw.get("version") or "0.0.0"), + "display_name": raw.get("display_name", ""), + "description": raw.get("description", ""), + "enabled_by_default": False, + "environment_policy": "audit-pending", + "entrypoint": entrypoint, + "required_env": raw.get("required_env", raw.get("requires_env", [])) or [], + "optional_env": raw.get("optional_env", []) or [], + "tool_defaults": { + "phases": ["all"], + "side_effects": "stateful", + "determinism": "conditional", + "timeout_class": "default", + "permissions": [], + "context_requirement": "none", + "result_schema": LEGACY_MCP_RESULT_V1, + }, + "tools": tools, + } + + +def load_legacy_skill_manifest(path: str | Path) -> SkillManifestV1: + """Convert an old manifest in memory without admitting or rewriting it.""" + + manifest_path = Path(path) + try: + raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise SkillManifestError(f"cannot read {manifest_path}: {exc}") from exc + if not isinstance(raw, dict): + raise SkillManifestError(f"{manifest_path}: manifest root must be a mapping") + if "schema_version" in raw: + return load_skill_manifest(manifest_path) + try: + return SkillManifestV1.model_validate(_legacy_document(raw, manifest_path)) + except ValidationError as exc: + raise SkillManifestError( + f"cannot migrate legacy manifest {manifest_path}: {exc}" + ) from exc + + +__all__ = ["load_legacy_skill_manifest"] diff --git a/ari-core/ari/migrations/v05_to_v07/__init__.py b/ari-core/ari/migrations/v05_to_v07/__init__.py index 51e56822..3950eb44 100644 --- a/ari-core/ari/migrations/v05_to_v07/__init__.py +++ b/ari-core/ari/migrations/v05_to_v07/__init__.py @@ -6,7 +6,7 @@ reconstruction of ``node_report.json`` from the legacy tree.json dump (used by ``ari migrate node-reports``). - :mod:`ari.migrations.v05_to_v07.memory` — re-export of the v0.5 - global-memory JSONL → checkpoint-scoped Letta auto-migration. + constants used by the explicit offline JSONL → versioned-memory migration. - :mod:`ari.migrations.v05_to_v07.legacy_axes` — legacy 5-axis fallback for evaluator scores written by older runs. diff --git a/ari-core/ari/migrations/v05_to_v07/memory.py b/ari-core/ari/migrations/v05_to_v07/memory.py index 5188a54b..536a14e2 100644 --- a/ari-core/ari/migrations/v05_to_v07/memory.py +++ b/ari-core/ari/migrations/v05_to_v07/memory.py @@ -1,12 +1,8 @@ -"""v0.5 global JSONL → v0.6 checkpoint-scoped Letta migration (Phase 5). +"""Constants used only by the explicit offline v0.5 memory migrator. -The actual logic lives in :mod:`ari.memory.auto_migrate` and continues -to be invoked from there by ``ari run`` / ``ari resume`` (the canonical -hooks). This module exists so callers that already understand the -migration package layout can ``from ari.migrations.v05_to_v07.memory -import maybe_auto_migrate`` without reaching back into the memory -implementation tree, and so v1.0 has a single home for the eventual -deletion. +Runtime migration was removed: ``ari run`` and ``ari resume`` never inspect or +rename legacy memory files. Operators must invoke ``ari memory migrate`` before +launching a supported checkpoint. This is the legitimate accessor of ``~/.ari/global_memory.jsonl`` — all *other* code in ARI must avoid that path @@ -17,13 +13,10 @@ from pathlib import Path -from ari.memory.auto_migrate import maybe_auto_migrate # noqa: F401 - - # v0.5 used a single global JSONL file under ~/.ari. Recorded here as a # constant so call-sites can rely on a single, documented location and # we can grep for the path during DR4 audits. LEGACY_GLOBAL_PATH = Path.home() / ".ari" / "global_memory.jsonl" -__all__ = ["maybe_auto_migrate", "LEGACY_GLOBAL_PATH"] +__all__ = ["LEGACY_GLOBAL_PATH"] diff --git a/ari-core/ari/paper_contract.py b/ari-core/ari/paper_contract.py new file mode 100644 index 00000000..f6b65268 --- /dev/null +++ b/ari-core/ari/paper_contract.py @@ -0,0 +1,464 @@ +"""Immutable build records for evidence-grounded scientific papers.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import PurePosixPath +from typing import Any, ClassVar, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + + +PAPER_ARTIFACT_V1 = "ari.paper-artifact/v1" +PAPER_MODEL_CALL_V1 = "ari.paper-model-call/v1" +PAPER_MODEL_CALL_BATCH_V1 = "ari.paper-model-call-batch/v1" +PAPER_REVISION_V1 = "ari.paper-revision/v1" +PAPER_COMPILE_V1 = "ari.paper-compile/v1" +PAPER_REVIEW_SET_V1 = "ari.paper-review-set/v1" +PAPER_BUILD_V1 = "ari.paper-build/v1" +SHA256_PATTERN = r"^sha256:[0-9a-f]{64}$" +ZERO_DIGEST = "sha256:" + "0" * 64 +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$") + + +class PaperContractError(ValueError): + """A paper build is malformed or has lost scientific provenance.""" + + +def canonical_paper_digest(value: Any) -> str: + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +class _StrictPaperModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class _DigestBoundPaperModel(_StrictPaperModel): + digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values = dict(values) + values[cls.digest_field] = ZERO_DIGEST + return cls.model_validate(values, context={"bind_paper_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self.digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_paper_digest(self.digest_payload()) + if info.context and info.context.get("bind_paper_digest"): + object.__setattr__(self, self.digest_field, expected) + elif getattr(self, self.digest_field) != expected: + raise ValueError(f"{self.digest_field} does not match paper payload") + return self + + +PaperArtifactRole = Literal[ + "science-data", + "figure-batch", + "retrieval-records", + "ear-manifest", + "template", + "rubric", + "prompt", + "raw-model-response", + "draft-tex", + "final-tex", + "bibtex", + "pdf", + "compile-stdout", + "compile-stderr", + "claim-links", + "hard-gate", + "semantic-review", + "text-review", + "visual-review", + "code-bundle-lock", + "authoring-record", +] + + +class PaperArtifactV1(_StrictPaperModel): + schema_version: Literal["ari.paper-artifact/v1"] = PAPER_ARTIFACT_V1 + role: PaperArtifactRole + relative_path: str + digest: str = Field(pattern=SHA256_PATTERN) + media_type: str = Field(min_length=1, max_length=128) + size_bytes: int = Field(ge=0) + + @field_validator("relative_path") + @classmethod + def _relative_path(cls, value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError("paper artifact path must be safe and relative") + return value + + +class PaperModelUsageV1(_StrictPaperModel): + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + cost_usd: float | None = Field(default=None, ge=0) + cost_status: Literal["reported", "unavailable"] = "unavailable" + + @model_validator(mode="after") + def _cost_matches_status(self) -> "PaperModelUsageV1": + if (self.cost_usd is not None) != (self.cost_status == "reported"): + raise ValueError("paper model cost and status differ") + return self + + +class PaperModelCallV1(_DigestBoundPaperModel): + digest_field = "call_digest" + schema_version: Literal["ari.paper-model-call/v1"] = PAPER_MODEL_CALL_V1 + call_id: str + purpose: Literal[ + "initial-authoring", + "figure-insertion", + "reflection", + "refinement", + "text-review", + ] + model: str = Field(min_length=1, max_length=512) + model_revision: str | None = Field(default=None, max_length=512) + provider: str = Field(min_length=1, max_length=256) + prompt_digest: str = Field(pattern=SHA256_PATTERN) + prompt_artifact: PaperArtifactV1 + raw_response_artifact: PaperArtifactV1 + sampling: dict[str, Any] = Field(default_factory=dict, max_length=64) + usage: PaperModelUsageV1 = Field(default_factory=PaperModelUsageV1) + call_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("call_id") + @classmethod + def _call_id(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("paper model call_id is invalid") + return value + + @field_validator("sampling") + @classmethod + def _finite_sampling(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("paper sampling must be finite JSON") from exc + return value + + @model_validator(mode="after") + def _raw_role(self) -> "PaperModelCallV1": + if ( + self.prompt_artifact.role != "prompt" + or self.prompt_artifact.digest != self.prompt_digest + ): + raise ValueError("paper model call lacks exact prompt evidence") + if self.raw_response_artifact.role != "raw-model-response": + raise ValueError("paper model call lacks raw response evidence") + return self + + +class PaperModelCallBatchV1(_DigestBoundPaperModel): + """Ordered call provenance for one bounded paper operation.""" + + digest_field = "batch_digest" + schema_version: Literal["ari.paper-model-call-batch/v1"] = PAPER_MODEL_CALL_BATCH_V1 + operation: Literal["authoring", "refinement", "text-review"] + calls: tuple[PaperModelCallV1, ...] = Field(default_factory=tuple, max_length=100) + batch_digest: str = Field(pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _unique_calls(self) -> "PaperModelCallBatchV1": + call_ids = [call.call_id for call in self.calls] + if len(call_ids) != len(set(call_ids)): + raise ValueError("paper model call batch contains duplicate call IDs") + return self + + +class PaperRevisionV1(_DigestBoundPaperModel): + digest_field = "revision_digest" + schema_version: Literal["ari.paper-revision/v1"] = PAPER_REVISION_V1 + revision: int = Field(ge=0, le=100) + parent_revision_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + reason: Literal[ + "initial", "figure-insertion", "reflection", "refinement", "finalize" + ] + tex_artifact: PaperArtifactV1 + bib_artifact: PaperArtifactV1 | None = None + model_call_id: str | None = None + claim_anchors: tuple[str, ...] = Field(default_factory=tuple, max_length=100_000) + citation_keys: tuple[str, ...] = Field(default_factory=tuple, max_length=100_000) + figure_ids: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + math_digest: str = Field(pattern=SHA256_PATTERN) + revision_digest: str = Field(pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _coherent(self) -> "PaperRevisionV1": + if self.tex_artifact.role not in {"draft-tex", "final-tex"}: + raise ValueError("paper revision tex artifact has the wrong role") + if self.bib_artifact is not None and self.bib_artifact.role != "bibtex": + raise ValueError("paper revision bibliography has the wrong role") + if self.revision == 0 and self.parent_revision_digest is not None: + raise ValueError("initial paper revision cannot name a parent") + if self.revision > 0 and self.parent_revision_digest is None: + raise ValueError("paper revision lacks a parent digest") + for values in (self.claim_anchors, self.citation_keys, self.figure_ids): + if len(values) != len(set(values)): + raise ValueError("paper revision identities must be unique") + return self + + +class PaperCompileV1(_DigestBoundPaperModel): + digest_field = "compile_digest" + schema_version: Literal["ari.paper-compile/v1"] = PAPER_COMPILE_V1 + status: Literal["completed", "failed", "timed-out", "tool-unavailable"] + commands: tuple[tuple[str, ...], ...] = Field(min_length=1, max_length=8) + execution_identities: tuple[str, ...] = Field(default_factory=tuple, max_length=8) + log_artifacts: tuple[PaperArtifactV1, ...] = Field(min_length=2, max_length=32) + pdf_artifact: PaperArtifactV1 | None = None + environment_digest: str = Field(pattern=SHA256_PATTERN) + compile_digest: str = Field(pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _compile_status(self) -> "PaperCompileV1": + if any( + artifact.role not in {"compile-stdout", "compile-stderr"} + for artifact in self.log_artifacts + ): + raise ValueError("paper compile log artifact has the wrong role") + if self.status == "tool-unavailable": + if self.execution_identities: + raise ValueError("unavailable paper compiler cannot claim executions") + elif len(self.execution_identities) != len(self.commands): + raise ValueError("paper compile command provenance is incomplete") + if self.status == "completed": + if self.pdf_artifact is None or self.pdf_artifact.role != "pdf": + raise ValueError("completed paper compile lacks a PDF") + elif self.pdf_artifact is not None: + raise ValueError("failed paper compile cannot claim a final PDF") + return self + + +class PaperReviewSetV1(_StrictPaperModel): + schema_version: Literal["ari.paper-review-set/v1"] = PAPER_REVIEW_SET_V1 + text_review: PaperArtifactV1 | None = None + visual_review: PaperArtifactV1 | None = None + semantic_review: PaperArtifactV1 | None = None + hard_gate: PaperArtifactV1 | None = None + visual_score: float | None = Field(default=None, ge=0, le=1) + visual_passing_score: float | None = Field(default=None, ge=0, le=1) + + @model_validator(mode="after") + def _roles_match(self) -> "PaperReviewSetV1": + expected = { + "text_review": "text-review", + "visual_review": "visual-review", + "semantic_review": "semantic-review", + "hard_gate": "hard-gate", + } + for field, role in expected.items(): + artifact = getattr(self, field) + if artifact is not None and artifact.role != role: + raise ValueError(f"paper {field} artifact has the wrong role") + if (self.visual_score is None) != (self.visual_passing_score is None): + raise ValueError( + "paper visual score and passing score must be recorded together" + ) + if self.visual_score is not None and self.visual_review is None: + raise ValueError("paper visual acceptance lacks a visual review artifact") + return self + + +class PaperGateSummaryV1(_StrictPaperModel): + mode: Literal["off", "warn", "strict"] + status: Literal["pass", "blocked", "error"] + blocking_error_count: int = Field(ge=0) + report_digest: str = Field(pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _status_matches_count(self) -> "PaperGateSummaryV1": + if self.status == "pass" and self.blocking_error_count: + raise ValueError("passing paper gate reports blocking errors") + if self.status == "blocked" and self.blocking_error_count == 0: + raise ValueError("blocked paper gate lacks blocking errors") + return self + + +class PaperNumericCoverageV1(_StrictPaperModel): + result_mentions: int = Field(ge=0) + linked_mentions: int = Field(ge=0) + excluded_mentions: int = Field(ge=0) + unresolved_anchors: int = Field(ge=0) + uncovered_mentions: int = Field(ge=0) + exclusion_policy_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _coverage_is_possible(self) -> "PaperNumericCoverageV1": + if self.linked_mentions + self.excluded_mentions > self.result_mentions: + raise ValueError("paper numeric coverage exceeds detected mentions") + if self.excluded_mentions and self.exclusion_policy_digest is None: + raise ValueError("excluded paper numbers require a policy digest") + return self + + +class PaperBuildV1(_DigestBoundPaperModel): + digest_field = "build_digest" + schema_version: Literal["ari.paper-build/v1"] = PAPER_BUILD_V1 + build_id: str + run_id: str + build_revision: int = Field(ge=0, le=100) + parent_build_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + status: Literal["draft", "blocked", "compile-error", "finalized"] + input_artifacts: tuple[PaperArtifactV1, ...] = Field(min_length=4, max_length=32) + venue_id: str + venue_version: str = Field(min_length=1, max_length=128) + template_digest: str = Field(pattern=SHA256_PATTERN) + rubric_id: str + rubric_version: str = Field(min_length=1, max_length=128) + rubric_digest: str = Field(pattern=SHA256_PATTERN) + ear_digest: str = Field(pattern=SHA256_PATTERN) + revisions: tuple[PaperRevisionV1, ...] = Field(min_length=1, max_length=101) + model_calls: tuple[PaperModelCallV1, ...] = Field( + default_factory=tuple, max_length=101 + ) + compile: PaperCompileV1 | None = None + reviews: PaperReviewSetV1 = Field(default_factory=PaperReviewSetV1) + gate: PaperGateSummaryV1 | None = None + numeric_coverage: PaperNumericCoverageV1 | None = None + final_artifacts: tuple[PaperArtifactV1, ...] = Field( + default_factory=tuple, max_length=16 + ) + blocking_reasons: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + limitations: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + build_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("build_id", "run_id", "venue_id", "rubric_id") + @classmethod + def _safe_ids(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("paper build identity is invalid") + return value + + @model_validator(mode="after") + def _coherent(self) -> "PaperBuildV1": + if (self.build_revision == 0) != (self.parent_build_digest is None): + raise ValueError("paper build parent lineage is inconsistent") + input_roles = [artifact.role for artifact in self.input_artifacts] + required_inputs = { + "science-data", + "figure-batch", + "retrieval-records", + "ear-manifest", + } + if not required_inputs.issubset(input_roles) or len(input_roles) != len( + set(input_roles) + ): + raise ValueError( + "paper build input artifact set is incomplete or duplicated" + ) + revision_numbers = [revision.revision for revision in self.revisions] + if revision_numbers != list(range(len(self.revisions))): + raise ValueError("paper revisions must be contiguous and ordered") + for index, revision in enumerate(self.revisions[1:], start=1): + if ( + revision.parent_revision_digest + != self.revisions[index - 1].revision_digest + ): + raise ValueError("paper revision does not bind its direct parent") + call_ids = [call.call_id for call in self.model_calls] + if len(call_ids) != len(set(call_ids)): + raise ValueError("paper model call IDs must be unique") + if any( + revision.model_call_id is not None + and revision.model_call_id not in set(call_ids) + for revision in self.revisions + ): + raise ValueError("paper revision names an unknown model call") + if self.status == "finalized": + if self.blocking_reasons: + raise ValueError("finalized paper cannot retain blocking reasons") + if ( + self.gate is None + or self.gate.mode == "off" + or self.gate.status != "pass" + ): + raise ValueError("finalized paper lacks a passing hard gate") + if self.numeric_coverage is None or ( + self.numeric_coverage.unresolved_anchors + or self.numeric_coverage.uncovered_mentions + ): + raise ValueError("finalized paper has incomplete numeric coverage") + if self.compile is None or self.compile.status != "completed": + raise ValueError("finalized paper lacks a completed compile") + final_roles = {artifact.role for artifact in self.final_artifacts} + if not {"final-tex", "bibtex", "pdf"}.issubset(final_roles): + raise ValueError("finalized paper artifact lock is incomplete") + elif self.status in {"blocked", "compile-error"} and not self.blocking_reasons: + raise ValueError("non-final paper must explain why it is blocked") + return self + + +def parse_paper_build(value: Any) -> PaperBuildV1: + try: + if isinstance(value, str): + value = json.loads(value) + return PaperBuildV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise PaperContractError(f"invalid PaperBuildV1: {exc}") from exc + + +def parse_paper_model_call_batch(value: Any) -> PaperModelCallBatchV1: + try: + if isinstance(value, str): + value = json.loads(value) + return PaperModelCallBatchV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise PaperContractError(f"invalid PaperModelCallBatchV1: {exc}") from exc + + +__all__ = [ + "PAPER_ARTIFACT_V1", + "PAPER_BUILD_V1", + "PAPER_COMPILE_V1", + "PAPER_MODEL_CALL_BATCH_V1", + "PAPER_MODEL_CALL_V1", + "PAPER_REVIEW_SET_V1", + "PAPER_REVISION_V1", + "PaperArtifactV1", + "PaperBuildV1", + "PaperCompileV1", + "PaperContractError", + "PaperGateSummaryV1", + "PaperModelCallBatchV1", + "PaperModelCallV1", + "PaperModelUsageV1", + "PaperNumericCoverageV1", + "PaperReviewSetV1", + "PaperRevisionV1", + "canonical_paper_digest", + "parse_paper_build", + "parse_paper_model_call_batch", +] diff --git a/ari-core/ari/pipeline/claim_gate/contract.py b/ari-core/ari/pipeline/claim_gate/contract.py index 33c0c308..ee2e23cf 100644 --- a/ari-core/ari/pipeline/claim_gate/contract.py +++ b/ari-core/ari/pipeline/claim_gate/contract.py @@ -287,6 +287,11 @@ def check_contract(science_data: dict) -> list[dict]: key = mc.get("key") formula = mc.get("formula") + formula_operands = ( + mc.get("formula_operands") + if isinstance(mc.get("formula_operands"), dict) + else {} + ) ceiling_select = mc.get("ceiling_select") invs = [e for e in (mc.get("invariants") or []) if isinstance(e, str)] correctness = mc.get("correctness") if isinstance(mc.get("correctness"), dict) else None @@ -296,6 +301,10 @@ def check_contract(science_data: dict) -> list[dict]: for cid, cfg in _iter_configs(science_data): vars_: dict[str, Any] = _flatten_metrics(cfg) prov = _provenance(cfg) + for role, metric_name in formula_operands.items(): + if isinstance(role, str) and isinstance(metric_name, str): + if metric_name in vars_: + vars_[role] = vars_[metric_name] # C: declared regime — evaluate the conditional to bind the selected # ceiling. The harness only EVALUATES the declared conditional; it never diff --git a/ari-core/ari/pipeline/claim_gate/formula_eval.py b/ari-core/ari/pipeline/claim_gate/formula_eval.py index b25dc2ad..61e24279 100644 --- a/ari-core/ari/pipeline/claim_gate/formula_eval.py +++ b/ari-core/ari/pipeline/claim_gate/formula_eval.py @@ -21,6 +21,8 @@ from __future__ import annotations import ast +import hashlib +import inspect import math from typing import Any @@ -183,3 +185,16 @@ def safe_eval(expr: str, variables: dict) -> Any: return _eval(tree, variables) except Exception: return None + + +def evaluator_digest() -> str: + """Fingerprint the complete restricted evaluator implementation. + + Metric-contract expressions can influence a blocking decision just as the + named numeric formulas can. Hashing only the public grammar would let a + code change silently alter the meaning of an old gate report, so the + provenance binds the implementation source as well. + """ + + source = inspect.getsource(inspect.getmodule(safe_eval)).encode("utf-8") + return "sha256:" + hashlib.sha256(source).hexdigest() diff --git a/ari-core/ari/pipeline/claim_gate/gate.py b/ari-core/ari/pipeline/claim_gate/gate.py index 2bcd7bed..521f7b18 100644 --- a/ari-core/ari/pipeline/claim_gate/gate.py +++ b/ari-core/ari/pipeline/claim_gate/gate.py @@ -28,6 +28,30 @@ from typing import Any from ari.pipeline.claim_gate import contract, invariants, latex, numeric, policy as _pol, resolve +from ari.claim_gate_contract import ( + GateFindingV1, + GateFormulaProvenanceV1, + GateReportV1, + parse_metric_gate_contract, +) +from ari.research_contract import canonical_digest + + +def _typed_finding(raw: dict[str, Any], severity: str) -> GateFindingV1: + known = {"type", "message", "claim_id", "numeric_id", "node_id", "artifact_path"} + finding_type = str(raw.get("type") or "gate_internal_error") + return GateFindingV1( + severity=severity, + type=finding_type, + message=str(raw.get("message") or finding_type), + claim_id=(str(raw["claim_id"]) if raw.get("claim_id") is not None else None), + numeric_id=(str(raw["numeric_id"]) if raw.get("numeric_id") is not None else None), + node_id=(str(raw["node_id"]) if raw.get("node_id") is not None else None), + artifact_path=( + str(raw["artifact_path"]) if raw.get("artifact_path") is not None else None + ), + details={key: value for key, value in raw.items() if key not in known}, + ) def _flatten_numeric_assertions(science_data: dict) -> list[dict]: @@ -109,6 +133,17 @@ def run_hard_gate( block_types = block_types | {"environment_mismatch"} science_data = science_data or {} + canonical_metric_contract = None + raw_metric_contract = science_data.get("metric_contract") + strict_evidence = bool( + isinstance(raw_metric_contract, dict) + and raw_metric_contract.get("schema_version") + == "ari.metric-gate-contract/v1" + ) + if strict_evidence: + canonical_metric_contract = parse_metric_gate_contract(raw_metric_contract) + science_data = dict(science_data) + science_data["metric_contract"] = canonical_metric_contract.gate_projection() tree = resolve.load_tree(ckpt) node_by_id = resolve.index_nodes(tree) claims = science_data.get("claims", []) or [] @@ -135,6 +170,9 @@ def run_hard_gate( errors: list[dict] = [] warnings: list[dict] = [] + evidence_document_digests: set[str] = set() + conversions_used: set[str] = set() + formulas_used: set[str] = set() # ── claim existence ────────────────────────────────────────────────── grounded_claims = 0 @@ -150,19 +188,66 @@ def run_hard_gate( "message": f"claim {cid} references unknown node '{nid}'"}) ok = False elif not resolve.node_executed(node_by_id, nid): - warnings.append({"claim_id": cid, "type": "node_not_executed", - "message": f"claim {cid} node '{nid}' has no real data"}) + target = errors if strict_evidence else warnings + target.append({"claim_id": cid, "node_id": str(nid), + "type": "node_not_executed", + "message": f"claim {cid} node '{nid}' has no real data"}) ok = False for op in sb.get("results", []) or []: - val, _src = resolve.resolve_operand(ckpt, node_by_id, op.get("node_id", ""), op.get("metric_path", "")) - if val is None: - warnings.append({"claim_id": cid, "type": "result_unresolved", - "message": f"claim {cid} result {op} did not resolve"}) + if not isinstance(op, dict): + errors.append({"claim_id": cid, "type": "result_unresolved", + "message": f"claim {cid} has a malformed result reference"}) + ok = False + continue + if ( + strict_evidence and op.get("run_id") != ckpt.name + ) or ( + not strict_evidence and op.get("run_id") not in (None, ckpt.name) + ): + errors.append({"claim_id": cid, "type": "cross_run_evidence", + "node_id": str(op.get("node_id") or ""), + "message": f"claim {cid} result belongs to another run"}) + ok = False + continue + resolved = resolve.resolve_operand_evidence( + ckpt, node_by_id, op.get("node_id", ""), + op.get("metric_path", ""), strict=strict_evidence, + ) + if resolved.document_digest: + evidence_document_digests.add(resolved.document_digest) + if resolved.value is None: + target = errors if strict_evidence else warnings + finding_type = ( + resolved.error + if strict_evidence and resolved.error in { + "artifact_digest_mismatch", "artifact_missing", + "artifact_not_bound", "cross_run_or_unknown_node", + "invalid_measurement_contract", + } + else "result_unresolved" + ) + target.append({"claim_id": cid, "type": finding_type, + "node_id": str(op.get("node_id") or ""), + "message": f"claim {cid} result did not resolve: " + f"{resolved.error or 'not found'}", + "reason": resolved.error}) ok = False for art in sb.get("artifacts", []) or []: - if not resolve.artifact_exists(ckpt, art): - warnings.append({"claim_id": cid, "type": "artifact_missing", - "message": f"claim {cid} artifact '{art}' not found"}) + verified, reason = resolve.verify_artifact( + ckpt, art, strict=strict_evidence + ) + if not verified: + target = errors if strict_evidence else warnings + finding_type = ( + reason if reason in { + "cross_run_artifact", "artifact_digest_mismatch", + "artifact_reference_untyped", "invalid_artifact_reference", + } else "artifact_missing" + ) + target.append({"claim_id": cid, "type": finding_type, + "artifact_path": str(art), + "message": f"claim {cid} artifact failed verification: {reason}"}) + ok = False if c.get("status") == "supported" and not (sb.get("nodes") or sb.get("results")): errors.append({"claim_id": cid, "type": "missing_evidence", "message": f"supported claim {cid} has no supporting evidence"}) @@ -187,23 +272,96 @@ def run_hard_gate( nid = na.get("id", "?") cid = na.get("claim_id", "?") formula = na.get("formula", "") + formulas_used.add(str(formula)) operands = na.get("operands", {}) or {} tol = na.get("tolerance") or default_tol roles = numeric.required_roles(formula) values: dict[str, float] = {} + operand_units: dict[str, str] = {} unresolved_role = None + unresolved_reason = None for role in roles: op = operands.get(role, {}) or {} - v, _src = resolve.resolve_operand(ckpt, node_by_id, op.get("node_id", ""), op.get("metric_path", "")) - if v is None: + if not isinstance(op, dict): + unresolved_role = role + unresolved_reason = "malformed_operand" + break + if ( + strict_evidence and op.get("run_id") != ckpt.name + ) or ( + not strict_evidence and op.get("run_id") not in (None, ckpt.name) + ): + unresolved_role = role + unresolved_reason = "cross_run_evidence" + break + resolved = resolve.resolve_operand_evidence( + ckpt, node_by_id, op.get("node_id", ""), + op.get("metric_path", ""), strict=strict_evidence, + ) + if resolved.document_digest: + evidence_document_digests.add(resolved.document_digest) + if resolved.value is None: + unresolved_role = role + unresolved_reason = resolved.error + break + if strict_evidence and not resolved.unit: unresolved_role = role + unresolved_reason = "measurement_unit_missing" break - values[role] = v + values[role] = resolved.value + operand_units[role] = resolved.unit or "" if unresolved_role is not None or not roles: - errors.append({"claim_id": cid, "numeric_id": nid, "type": "operand_unresolved", + finding_type = ( + unresolved_reason + if strict_evidence and unresolved_reason in { + "artifact_digest_mismatch", "artifact_missing", + "artifact_not_bound", "cross_run_evidence", + "cross_run_or_unknown_node", "invalid_measurement_contract", + } + else "operand_unresolved" + ) + errors.append({"claim_id": cid, "numeric_id": nid, "type": finding_type, "message": f"{nid} operand '{unresolved_role or 'formula'}' " - f"({operands.get(unresolved_role, {})}) did not resolve"}) + f"did not resolve: {unresolved_reason or 'unknown formula'}", + "reason": unresolved_reason}) + continue + + declared_unit = str(na.get("unit") or "") + if strict_evidence and not declared_unit and canonical_metric_contract is not None: + declared_unit = canonical_metric_contract.metric_contract.unit + if strict_evidence and not declared_unit: + errors.append({"claim_id": cid, "numeric_id": nid, + "type": "unit_unresolved", + "message": f"{nid} has no declared output unit"}) continue + if "baseline" in roles and "proposed" in roles: + converted, provenance = numeric.convert_value( + values["proposed"], operand_units["proposed"], operand_units["baseline"] + ) + if converted is None: + errors.append({"claim_id": cid, "numeric_id": nid, + "type": "unit_mismatch", + "message": f"{nid} operands use incompatible units " + f"{operand_units['baseline']!r} and " + f"{operand_units['proposed']!r}"}) + continue + values["proposed"] = converted + if provenance: + conversions_used.add(provenance) + elif strict_evidence and roles == ("value",): + converted, provenance = numeric.convert_value( + values["value"], operand_units["value"], declared_unit + ) + if converted is None: + errors.append({"claim_id": cid, "numeric_id": nid, + "type": "unit_mismatch", + "message": f"{nid} operand unit " + f"{operand_units['value']!r} cannot convert " + f"to {declared_unit!r}"}) + continue + values["value"] = converted + if provenance: + conversions_used.add(provenance) recomputed = numeric.recompute(formula, values) if recomputed is None: errors.append({"claim_id": cid, "numeric_id": nid, "type": "operand_unresolved", @@ -213,14 +371,20 @@ def run_hard_gate( # quantities: a baseline/reference number stated in prose (and used here # only as an operand) is covered by its exact value (unit absolute). Sound # — propagates a real data value, never searches for a derivation. - for _ov in values.values(): - verified_values.append((_ov, "")) + for role, operand_value in values.items(): + verified_values.append((operand_value, operand_units.get(role, ""))) # same-environment check for comparison formulas. Severity is intent- # driven: a transparency WARNING by default ("any"), a blocking ERROR # only under "same_environment" intent (single-architecture studies). if "baseline" in roles and "proposed" in roles: - b_env = resolve.env_signature(ckpt, operands.get("baseline", {}).get("node_id", "")) - p_env = resolve.env_signature(ckpt, operands.get("proposed", {}).get("node_id", "")) + b_env = resolve.env_signature( + ckpt, operands.get("baseline", {}).get("node_id", ""), + strict=strict_evidence, + ) + p_env = resolve.env_signature( + ckpt, operands.get("proposed", {}).get("node_id", ""), + strict=strict_evidence, + ) if b_env.get("cpu_model") and p_env.get("cpu_model") and b_env != p_env: _envfind = {"claim_id": cid, "numeric_id": nid, "type": "environment_mismatch", "message": f"{nid} baseline/proposed differ in environment " @@ -229,15 +393,30 @@ def run_hard_gate( _rm = _reported_mention(links, mentions, nid) reported = _rm.get("value") if _rm else None if reported is not None: - if numeric.within_tolerance(reported, recomputed, tol): + reported_unit = str((_rm or {}).get("unit") or "") + comparable = float(reported) + if strict_evidence: + comparable, conversion = numeric.convert_value( + comparable, reported_unit, declared_unit + ) + if comparable is None: + errors.append({"claim_id": cid, "numeric_id": nid, + "type": "unit_mismatch", + "message": f"{nid} paper unit {reported_unit!r} " + f"cannot convert to {declared_unit!r}"}) + continue + if conversion: + conversions_used.add(conversion) + if numeric.within_tolerance(comparable, recomputed, tol): reproducible += 1 - verified_values.append((reported, (_rm or {}).get("unit", ""))) + verified_values.append((reported, reported_unit)) else: mismatch_count += 1 errors.append({"claim_id": cid, "numeric_id": nid, "type": "numeric_mismatch", "message": f"{nid}: paper value {reported} not reproducible from " f"results.json (recomputed {round(recomputed, 6)})", - "reported": reported, "recomputed": round(recomputed, 6), + "reported": reported, "reported_in_contract_unit": comparable, + "recomputed": round(recomputed, 6), "formula": formula, "tolerance": tol}) else: # no paper-linked number to compare; verify internal consistency only @@ -339,23 +518,14 @@ def run_hard_gate( status = "failed" if errors else ("warn" if warnings else "passed") always_block = _pol.always_block_on(pol) should_block = ( - phase == "final" and ( + phase == "final" and pmode != "off" and ( (pmode == "strict" and any(e.get("type") in block_types for e in errors)) # Objective-falsehood findings block at final regardless of warn/strict. or any(e.get("type") in always_block for e in errors) ) ) - report = { - "gate": "claim_evidence_hard_gate", - "phase": phase, - "policy": pmode, - "comparison_scope": cmp_scope, - "status": status, - "should_block": should_block, - "errors": errors, - "warnings": warnings, - "metrics": { + gate_metrics = { "total_claims": len(claims), "grounded_claims": grounded_claims, "execution_grounded_claim_rate": (grounded_claims / len(claims)) if claims else 0.0, @@ -369,16 +539,54 @@ def run_hard_gate( "uncovered_numeric_count": uncovered_count, "invariant_violation_count": len(invariant_violations), "contract_violation_count": len(contract_violations), - }, } + metric_contract_digest = ( + canonical_metric_contract.metric_contract.contract_digest + if canonical_metric_contract is not None + else None + ) + evidence_digest = canonical_digest( + { + "source_run_id": ckpt.name, + "tree": tree, + "science_data": science_data, + "paper_digest": canonical_digest(paper_tex or ""), + "paper_claim_links": paper_claim_links or {}, + "figures_manifest": figures_manifest, + "measurement_documents": sorted(evidence_document_digests), + } + ) + report_model = GateReportV1.create( + source_run_id=ckpt.name, + phase=phase, + policy_mode=pmode, + comparison_scope=cmp_scope, + status=status, + should_block=should_block, + policy_digest=canonical_digest(pol), + evidence_digest=evidence_digest, + formula_provenance=GateFormulaProvenanceV1( + registry_digest=numeric.formula_registry_digest(), + formulas_used=tuple(sorted(formulas_used)), + metric_contract_digest=metric_contract_digest, + unit_conversions=tuple(sorted(conversions_used)), + ), + blocking_findings=tuple(_typed_finding(item, "blocking") for item in errors), + advisory_findings=tuple(_typed_finding(item, "advisory") for item in warnings), + metrics=gate_metrics, + ) + report = report_model.model_dump(mode="json") + if write: - try: - out_dir = ckpt / "evaluation" - out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / f"claim_evidence_hard_gate_{phase}.json").write_text( - json.dumps(report, ensure_ascii=False, indent=2) - ) - except Exception as e: # pragma: no cover - defensive - report.setdefault("_write_error", str(e)) + from ari.execution import WorkspaceRefV1 + + workspace = WorkspaceRefV1(root=str(ckpt.expanduser().resolve())) + workspace.atomic_write_bytes( + f"evaluation/claim_evidence_hard_gate_{phase}.json", + ( + json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + + "\n" + ).encode("utf-8"), + ) return report diff --git a/ari-core/ari/pipeline/claim_gate/latex.py b/ari-core/ari/pipeline/claim_gate/latex.py index e6b8aaa2..0abb7c5f 100644 --- a/ari-core/ari/pipeline/claim_gate/latex.py +++ b/ari-core/ari/pipeline/claim_gate/latex.py @@ -1,203 +1,17 @@ -"""Deterministic LaTeX section + numeric-token parsing for the hard gate -(Story2Proposal Phase B, coverage check). - -This mirrors ari-skill-paper/src/claim_links.py so the gate can re-derive -numeric coverage authoritatively even when paper_claim_links is absent -(e.g. the Condition-A baseline with no anchors). When paper_claim_links IS -present the gate prefers its numeric_mentions; this module is the fallback. -""" - -from __future__ import annotations - -import re - - -ANCHOR_RE = re.compile(r"%\s*CLAIM:(C\w+):(NC\w+)") - -# Mantissa + optional exponent. Without the exponent branch, scientific -# notation (``4.44\times 10^{-16}``, ``1.2e-6``) was read as its bare mantissa -# (and the 10/16 matched as separate junk mentions), so every such paper value -# failed recompute as a false numeric_mismatch. NOTE: _strip_for_scan rewrites -# ``\times`` to `` x `` before scanning, so the multiplication sign here is -# x/× (plus \cdot, which survives the strip); requiring the ``10^{...}`` tail -# keeps plain speedup notation (``4.18 x``) out of this branch. -_NUMBER_RE = re.compile( - r"(? str: - t = title.strip().lower() - table = [ - ("introduction", "introduction"), ("related", "related_work"), - ("background", "related_work"), ("prior work", "related_work"), - ("method", "methodology"), ("approach", "methodology"), - ("design", "methodology"), ("implementation", "methodology"), - ("experiment", "experiments"), ("evaluation", "experiments"), - ("setup", "experiments"), ("result", "results"), - ("discussion", "discussion"), ("limitation", "limitations"), - ("conclusion", "conclusion"), ("summary", "conclusion"), - ("future work", "conclusion"), ("reference", "references"), - ("bibliograph", "references"), ("acknowled", "acknowledgements"), - ("abstract", "abstract"), ("appendix", "appendix"), - ] - for needle, canon in table: - if needle in t: - return canon - slug = re.sub(r"[^a-z0-9]+", "_", t).strip("_") - return slug or "body" - - -def build_section_map(tex: str) -> list[str]: - lines = tex.split("\n") - out: list[str] = [] - current = "preamble" - appendix_mode = False - sec_re = re.compile(r"\\(?:sub)*section\*?\s*\{([^}]*)\}") - for ln in lines: - s = ln.strip() - if "\\begin{abstract}" in s: - current = "abstract" - out.append(current) - continue - if "\\end{abstract}" in s: - out.append(current) - current = "body" - continue - if re.search(r"\\appendix\b", s): - appendix_mode = True - m = sec_re.search(s) - if m: - current = "appendix" if appendix_mode else _canonical_section(m.group(1)) - elif re.search(r"\\bibliography\b|\\begin\{thebibliography\}", s): - current = "references" - out.append(current) - return out - - -def section_at(section_map: list[str], line_no: int) -> str: - idx = line_no - 1 - return section_map[idx] if 0 <= idx < len(section_map) else "body" - - -def _strip_for_scan(line: str) -> str: - s = re.sub(r"(? ]", " ", s) - # \( \) are math delimiters exactly like $ — leaving them in place put a - # ")" between "\(734.8\)" and its "GB/s", defeating unit detection and the - # anchor binder (which then picked an unrelated number in the sentence). - s = s.replace("\\(", " ").replace("\\)", " ") - return s.replace("~", " ").replace("$", " ") - - -def _classify(num_str: str, has_percent: bool, before: str, after: str) -> tuple[str, bool]: - try: - value = float(num_str.replace(",", "")) - except ValueError: - return "ambiguous", False - is_int = "." not in num_str - if is_int and 1900 <= value <= 2099 and not has_percent and not _PERF_UNIT_RE.match(after): - if not _SETTING_UNIT_RE.match(after): - return "citation_year", False - if _REF_WORD_RE.search(before): - return "figure_table_ref", False - if _SETTING_UNIT_RE.match(after): - return "experimental_setting", False - if has_percent or _PERF_UNIT_RE.match(after): - return "result_claim", True - return "ambiguous", False - - -def find_anchors(tex: str) -> list[dict]: - anchors: list[dict] = [] - for i, line in enumerate(tex.split("\n"), start=1): - for m in ANCHOR_RE.finditer(line): - anchors.append({ - "anchor": f"CLAIM:{m.group(1)}:{m.group(2)}", - "claim_id": m.group(1), "numeric_id": m.group(2), "line": i, - }) - return anchors - - -def extract_numeric_mentions(tex: str, section_map: "list[str] | None" = None) -> list[dict]: - if section_map is None: - section_map = build_section_map(tex) - mentions: list[dict] = [] - for i, raw in enumerate(tex.split("\n"), start=1): - line = _strip_for_scan(raw) - for m in _NUMBER_RE.finditer(line): - int_part, frac = m.group(1), m.group(2) or "" - exp = m.group(3) or m.group(4) or "" - pct = m.group(5) or "" - num_str = int_part + frac - has_pct = pct == "%" - before = line[max(0, m.start() - 24):m.start()] - after = line[m.end():m.end() + 24] - mtype, requires = _classify(num_str, has_pct, before, after) - try: - value = float(num_str.replace(",", "")) - if exp: - value *= 10.0 ** int(exp) - except (ValueError, OverflowError): - continue - if value in (float("inf"), float("-inf")): - # 10^{4932}-style constants: a non-finite mention would poison - # the JSON report and (pre-guard) an uncaught OverflowError made - # the whole gate fail OPEN via the callers' defensive catches. - continue - if m.group(4): - # \times/\cdot 10^{exp} literals classified as result_claim - # before this branch existed too (the stripped " x " matched the - # speedup unit) — keep that, or the anchor binder starts picking - # some other number in the sentence. e-notation keeps _classify's - # verdict so settings ("1e4 iterations") stay settings. - mtype, requires = "result_claim", True - mentions.append({ - "value": value, "unit": "%" if has_pct else "", - "type": mtype, "requires_assertion": requires, - "section": section_at(section_map, i), "line": i, - }) - return mentions - - -def figure_refs(tex: str) -> list[str]: - """Figure/table labels referenced via \\ref-family commands.""" - out: list[str] = [] - for m in re.finditer(r"\\(?:ref|autoref|cref|Cref)\{([^}]*)\}", tex): - lab = m.group(1) - if lab not in out: - out.append(lab) - return out diff --git a/ari-core/ari/pipeline/claim_gate/numeric.py b/ari-core/ari/pipeline/claim_gate/numeric.py index bebccd11..ab547387 100644 --- a/ari-core/ari/pipeline/claim_gate/numeric.py +++ b/ari-core/ari/pipeline/claim_gate/numeric.py @@ -1,11 +1,8 @@ """Formula-level numeric re-computation utility (Story2Proposal Phase B2). -Canonical home of the numeric-assertion formula registry used by the -``claim_evidence_hard_gate``. The same registry is mirrored in -``ari-skill-transform/src/claims.py`` (which *declares* the assertions). Keep -the two in sync — divergence only affects the transform-declared ``value`` -(seed), because the gate verifies the **paper-reported** number against this -recomputation, not against the seed. +Canonical home of the numeric-assertion formula registry used by both the +``claim_evidence_hard_gate`` and transform producer. Skills access it through +``ari.public.science_data``; no second implementation is maintained. The documented master-plan formulas are the lower-is-better family (speedup / improvement / reduction). ``relative_gain`` / ``relative_increase_percent`` @@ -15,8 +12,13 @@ from __future__ import annotations +import json +import hashlib +import inspect from typing import Callable +from ari.pipeline.claim_gate.formula_eval import evaluator_digest + def _f_identity(o: dict) -> "float | None": return o["value"] @@ -65,6 +67,73 @@ def _f_ratio_percent(o: dict) -> "float | None": } +# Closed, dimension-preserving conversion registry. Unknown spellings are +# rejected; the evaluator never infers a unit from a metric name. +_UNIT_SCALE: dict[str, tuple[str, float]] = { + "s": ("time", 1.0), + "ms": ("time", 1e-3), + "us": ("time", 1e-6), + "µs": ("time", 1e-6), + "ns": ("time", 1e-9), + "fraction": ("fraction", 1.0), + "%": ("fraction", 1e-2), + "percent": ("fraction", 1e-2), + "B": ("bytes", 1.0), + "kB": ("bytes", 1e3), + "MB": ("bytes", 1e6), + "GB": ("bytes", 1e9), + "KiB": ("bytes", 1024.0), + "MiB": ("bytes", 1024.0**2), + "GiB": ("bytes", 1024.0**3), + "Hz": ("frequency", 1.0), + "kHz": ("frequency", 1e3), + "MHz": ("frequency", 1e6), + "GHz": ("frequency", 1e9), + "ratio": ("ratio", 1.0), + "count": ("count", 1.0), +} + + +def unit_registry_digest() -> str: + payload = json.dumps( + _UNIT_SCALE, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def convert_value( + value: float, source_unit: str, target_unit: str +) -> tuple["float | None", "str | None"]: + """Convert through the closed registry and return conversion provenance.""" + + if source_unit == target_unit: + if source_unit in _UNIT_SCALE or source_unit == "": + return float(value), None + return None, None + source = _UNIT_SCALE.get(source_unit) + target = _UNIT_SCALE.get(target_unit) + if source is None or target is None or source[0] != target[0]: + return None, None + converted = float(value) * source[1] / target[1] + return converted, f"{source_unit}->{target_unit}@{unit_registry_digest()}" + + +def formula_registry_digest() -> str: + payload = { + "declared_expression_evaluator": evaluator_digest(), + "named_formulas": { + name: { + "roles": list(roles), + "implementation": inspect.getsource(implementation).strip(), + } + for name, (roles, implementation) in sorted(FORMULAS.items()) + }, + } + return "sha256:" + hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + def required_roles(formula: str) -> tuple[str, ...]: spec = FORMULAS.get(formula) return spec[0] if spec else () diff --git a/ari-core/ari/pipeline/claim_gate/policy.py b/ari-core/ari/pipeline/claim_gate/policy.py index 7fcd97fd..c7eaa3b9 100644 --- a/ari-core/ari/pipeline/claim_gate/policy.py +++ b/ari-core/ari/pipeline/claim_gate/policy.py @@ -7,10 +7,10 @@ 3. ``{checkpoint_dir}/claim_gate_policy.json`` when present 4. env ``ARI_CLAIM_GATE_MODE`` (strict | warn | off) — the evaluation switch -``mode`` governs blocking: ``off`` never blocks; ``warn`` (MVP) reports -errors/warnings but never blocks finalize; ``strict`` (evaluation) blocks the -*final* gate when block_on errors exist, and makes uncovered result numbers in -strict sections blocking. +``mode`` governs blocking: ``off`` never blocks; ``warn`` blocks only the +objective-integrity ``always_block_on`` tier at the final phase; ``strict`` +additionally blocks configured ``block_on`` findings and uncovered result +numbers in strict sections. Draft-phase reports never block. """ from __future__ import annotations @@ -38,7 +38,24 @@ }, "numeric_match": {"default_tolerance": {"absolute": 0.0, "relative": 0.02}}, "blocking": { - "block_on": ["numeric_mismatch", "operand_unresolved", "missing_evidence"], + "block_on": [ + "numeric_mismatch", + "operand_unresolved", + "missing_evidence", + "node_not_executed", + "result_unresolved", + "unit_unresolved", + "unit_mismatch", + "cross_run_evidence", + "cross_run_or_unknown_node", + "cross_run_artifact", + "artifact_missing", + "artifact_reference_untyped", + "invalid_artifact_reference", + "artifact_digest_mismatch", + "artifact_not_bound", + "invalid_measurement_contract", + ], # Objective-falsehood findings: physically/logically impossible or # unverifiable results. Unlike block_on (which only blocks under strict), # these block the FINAL paper regardless of warn/strict — they are @@ -54,6 +71,9 @@ "invariant_violation", "correctness_failed", "correctness_uncovered", "placeholder_denominator", "recompute_mismatch", "claim_evidence_missing", "ceiling_unmeasured", + "cross_run_evidence", "cross_run_or_unknown_node", "cross_run_artifact", + "artifact_digest_mismatch", "artifact_not_bound", + "invalid_measurement_contract", ], }, } diff --git a/ari-core/ari/pipeline/claim_gate/resolve.py b/ari-core/ari/pipeline/claim_gate/resolve.py index bfd7f712..22ca8c43 100644 --- a/ari-core/ari/pipeline/claim_gate/resolve.py +++ b/ari-core/ari/pipeline/claim_gate/resolve.py @@ -16,10 +16,27 @@ from __future__ import annotations import json +import hashlib +import re +from dataclasses import dataclass from pathlib import Path from typing import Any +_SAFE_NODE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$") +_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") + + +@dataclass(frozen=True) +class ResolvedOperand: + value: float | None = None + source: str = "" + unit: str | None = None + document_digest: str | None = None + artifact_digests: tuple[str, ...] = () + error: str | None = None + + def _is_number(x: Any) -> bool: return isinstance(x, (int, float)) and not isinstance(x, bool) @@ -55,16 +72,32 @@ def index_nodes(tree: dict) -> dict[str, dict]: return out -def load_results_json(checkpoint_dir: Path, node_id: str) -> dict: +def _node_dir(checkpoint_dir: Path, node_id: str) -> Path | None: + if not _SAFE_NODE_ID.fullmatch(str(node_id)): + return None + workspace, run_id = workspace_run_id(checkpoint_dir) + candidate = (workspace / "experiments" / run_id / node_id).resolve() + root = (workspace / "experiments" / run_id).resolve() + if candidate.parent != root: + return None + return candidate + + +def load_results_json( + checkpoint_dir: Path, node_id: str, *, strict: bool = False +) -> dict: if not node_id: return {} workspace, run_id = workspace_run_id(checkpoint_dir) ckpt = Path(checkpoint_dir) - for cand in ( - workspace / "experiments" / run_id / node_id / "results.json", - workspace / "experiments" / node_id / "results.json", - ckpt / "experiments" / node_id / "results.json", - ): + canonical = _node_dir(checkpoint_dir, node_id) + candidates = (() if canonical is None else (canonical / "results.json",)) + if not strict: + candidates = (*candidates, + workspace / "experiments" / node_id / "results.json", + ckpt / "experiments" / node_id / "results.json", + ) + for cand in candidates: if cand.is_file(): try: data = json.loads(cand.read_text()) @@ -74,20 +107,29 @@ def load_results_json(checkpoint_dir: Path, node_id: str) -> dict: return {} -def load_node_report(checkpoint_dir: Path, node_id: str) -> dict: +def load_node_report( + checkpoint_dir: Path, node_id: str, *, strict: bool = False +) -> dict: if not node_id: return {} workspace, run_id = workspace_run_id(checkpoint_dir) ckpt = Path(checkpoint_dir) - for cand in ( - workspace / "experiments" / run_id / node_id / "node_report.json", - workspace / "experiments" / node_id / "node_report.json", - ckpt / "experiments" / node_id / "node_report.json", - ): + canonical = _node_dir(checkpoint_dir, node_id) + candidates = (() if canonical is None else (canonical / "node_report.json",)) + if not strict: + candidates = (*candidates, + workspace / "experiments" / node_id / "node_report.json", + ckpt / "experiments" / node_id / "node_report.json", + ) + for cand in candidates: if cand.is_file(): try: data = json.loads(cand.read_text()) - return data if isinstance(data, dict) else {} + if not isinstance(data, dict): + return {} + if strict and data.get("node_id") != node_id: + return {} + return data except Exception: return {} return {} @@ -119,42 +161,184 @@ def resolve_operand( node_by_id: dict[str, dict], node_id: str, metric_path: str, + *, + strict: bool = False, ) -> tuple["float | None", str]: """Resolve a scalar from (node_id, metric_path). Returns (value, source).""" - if not node_id or not metric_path: - return None, "" - root, _, rest = metric_path.partition(".") - rj = load_results_json(checkpoint_dir, node_id) + resolved = resolve_operand_evidence( + checkpoint_dir, + node_by_id, + node_id, + metric_path, + strict=strict, + ) + return resolved.value, resolved.source + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _verified_measurement_document( + checkpoint_dir: Path, node_id: str +) -> tuple[dict, str | None, str | None]: + node_dir = _node_dir(checkpoint_dir, node_id) + if node_dir is None: + return {}, None, "invalid_node_id" + path = node_dir / "results.json" + try: + payload = path.read_bytes() + raw = json.loads(payload) + except FileNotFoundError: + return {}, None, "missing_results" + except (OSError, json.JSONDecodeError): + return {}, None, "invalid_results" + try: + from ari.execution import parse_measurement_document + + measurement_set = parse_measurement_document(raw, allow_legacy=False) + except Exception: + return {}, None, "invalid_measurement_contract" + + report = load_node_report(checkpoint_dir, node_id, strict=True) + report_artifacts: dict[str, tuple[Path, str]] = {} + for artifact in report.get("artifacts", []) if isinstance(report, dict) else (): + if not isinstance(artifact, dict): + continue + filename = str(artifact.get("filename") or "") + raw_digest = str(artifact.get("sha256") or "") + digest = raw_digest if raw_digest.startswith("sha256:") else f"sha256:{raw_digest}" + if ( + filename + and Path(filename).name == filename + and _SHA256.fullmatch(digest) + ): + report_artifacts[digest] = (node_dir / filename, filename) + for digest in measurement_set.artifact_digests: + binding = report_artifacts.get(digest) + if binding is None: + return {}, None, "artifact_not_bound" + artifact_path, _ = binding + try: + if _sha256_file(artifact_path) != digest: + return {}, None, "artifact_digest_mismatch" + except OSError: + return {}, None, "artifact_missing" + projection = { + "params": measurement_set.parameters, + "measurements": { + item.metric_id: item.value for item in measurement_set.measurements + }, + "measurement_records": { + item.metric_id: item for item in measurement_set.measurements + }, + "predictions": measurement_set.predictions, + "scores": measurement_set.scores, + "artifact_digests": tuple(measurement_set.artifact_digests), + } + return projection, "sha256:" + hashlib.sha256(payload).hexdigest(), None + - # results.json typed containers +def _resolve_loaded_operand( + rj: dict, + node: dict, + metric_path: str, + *, + strict: bool, + document_digest: str | None, +) -> ResolvedOperand: + root, _, rest = metric_path.partition(".") if root in ("measurements", "scores", "params", "predictions"): + if strict and root != "measurements": + return ResolvedOperand(error="untyped_nonmeasurement_operand") val = _dot_get(rj.get(root, {}), rest) if _is_number(val): - return float(val), f"results.json:{metric_path}" + record = (rj.get("measurement_records") or {}).get(rest) + return ResolvedOperand( + value=float(val), + source=f"results.json:{metric_path}", + unit=getattr(record, "unit", None), + document_digest=document_digest, + artifact_digests=tuple(getattr(record, "artifact_digests", ())), + ) - # explicit node metrics path if root == "metrics": - node = node_by_id.get(str(node_id), {}) val = _dot_get(node.get("metrics", {}), rest) if _is_number(val): - return float(val), f"tree.json:metrics.{rest}" + if strict: + return ResolvedOperand(error="untyped_tree_metric") + return ResolvedOperand(value=float(val), source=f"tree.json:metrics.{rest}") - # fallback: try by trailing key across containers key = rest or root - for c in ("measurements", "scores", "predictions"): - v = (rj.get(c) or {}).get(key) - if _is_number(v): - return float(v), f"results.json:{c}.{key}" + for container in ("measurements", "scores", "predictions"): + value = (rj.get(container) or {}).get(key) + if not _is_number(value): + continue + if strict and container != "measurements": + return ResolvedOperand(error="untyped_nonmeasurement_operand") + record = (rj.get("measurement_records") or {}).get(key) + return ResolvedOperand( + value=float(value), + source=f"results.json:{container}.{key}", + unit=getattr(record, "unit", None), + document_digest=document_digest, + artifact_digests=tuple(getattr(record, "artifact_digests", ())), + ) + metric_value = (node.get("metrics") or {}).get(key) + if _is_number(metric_value): + if strict: + return ResolvedOperand(error="untyped_tree_metric") + return ResolvedOperand( + value=float(metric_value), source=f"tree.json:metrics.{key}" + ) + return ResolvedOperand(error="metric_not_found") + + +def resolve_operand_evidence( + checkpoint_dir: Path, + node_by_id: dict[str, dict], + node_id: str, + metric_path: str, + *, + strict: bool = False, +) -> ResolvedOperand: + """Resolve a scalar plus typed unit/artifact provenance.""" + + if not node_id or not metric_path: + return ResolvedOperand(error="missing_operand_binding") + if not node_exists(node_by_id, node_id): + return ResolvedOperand(error="cross_run_or_unknown_node") + if strict and not node_executed(node_by_id, node_id): + return ResolvedOperand(error="node_not_executed") + document_digest = None + if strict: + rj, document_digest, error = _verified_measurement_document( + checkpoint_dir, node_id + ) + if error: + return ResolvedOperand(error=error) + else: + rj = load_results_json(checkpoint_dir, node_id) + node = node_by_id.get(str(node_id), {}) - mv = (node.get("metrics") or {}).get(key) - if _is_number(mv): - return float(mv), f"tree.json:metrics.{key}" - return None, "" + return _resolve_loaded_operand( + rj, + node, + metric_path, + strict=strict, + document_digest=document_digest, + ) -def env_signature(checkpoint_dir: Path, node_id: str) -> dict: +def env_signature( + checkpoint_dir: Path, node_id: str, *, strict: bool = False +) -> dict: """Coarse environment signature for same-environment comparison checks.""" - rep = load_node_report(checkpoint_dir, node_id) + rep = load_node_report(checkpoint_dir, node_id, strict=strict) cpu = rep.get("cpu_info") or {} return { "executor": rep.get("executor", ""), @@ -163,13 +347,54 @@ def env_signature(checkpoint_dir: Path, node_id: str) -> dict: } -def artifact_exists(checkpoint_dir: Path, rel_path: str) -> bool: +def verify_artifact( + checkpoint_dir: Path, + artifact: Any, + *, + strict: bool = False, +) -> tuple[bool, str]: + """Verify a legacy path or a run/node/digest-bound artifact reference.""" + + if strict: + if not isinstance(artifact, dict): + return False, "artifact_reference_untyped" + workspace, run_id = workspace_run_id(checkpoint_dir) + if artifact.get("run_id") != run_id: + return False, "cross_run_artifact" + node_id = str(artifact.get("node_id") or "") + node_dir = _node_dir(checkpoint_dir, node_id) + relative = str(artifact.get("relative_path") or "") + digest = str(artifact.get("digest") or "") + if ( + node_dir is None + or not relative + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not _SHA256.fullmatch(digest) + ): + return False, "invalid_artifact_reference" + path = (node_dir / relative).resolve() + try: + path.relative_to(node_dir) + actual = _sha256_file(path) + except FileNotFoundError: + return False, "artifact_missing" + except OSError: + return False, "artifact_unreadable" + if actual != digest: + return False, "artifact_digest_mismatch" + return True, "" + rel_path = str(artifact or "") if not rel_path: - return False + return False, "artifact_missing" ckpt = Path(checkpoint_dir) if Path(rel_path).is_absolute(): - return Path(rel_path).exists() + return Path(rel_path).exists(), "" if Path(rel_path).exists() else "artifact_missing" for base in (ckpt, ckpt / "ear_published", ckpt / "ear"): if (base / rel_path).exists(): - return True - return False + return True, "" + return False, "artifact_missing" + + +def artifact_exists(checkpoint_dir: Path, rel_path: Any) -> bool: + return verify_artifact(checkpoint_dir, rel_path)[0] diff --git a/ari-core/ari/pipeline/driver.py b/ari-core/ari/pipeline/driver.py index 14649188..fa844a98 100644 --- a/ari-core/ari/pipeline/driver.py +++ b/ari-core/ari/pipeline/driver.py @@ -98,9 +98,41 @@ def run(self) -> dict[str, Any]: # Written to checkpoint as evaluation_criteria.json for downstream use _eval_criteria_path = checkpoint_dir / "evaluation_criteria.json" if not _eval_criteria_path.exists(): - _ec = {"primary_metric": "", "higher_is_better": True, "metric_rationale": ""} + _ec = { + "primary_metric": "", + "higher_is_better": True, + "metric_rationale": "", + "metric_unit": "", + "research_contract_digest": "", + } + # Typed idea contracts are authoritative and already contain the + # frozen metric vocabulary. Verify the self-digest before consulting + # legacy memory/prose projections. + try: + from ari.public.research_contract import ( + ResearchContractError, + parse_research_contract_document, + ) + + _typed_idea_path = Path(checkpoint_dir) / "idea.json" + if _typed_idea_path.is_file(): + _typed_idea = json.loads(_typed_idea_path.read_text()) + _typed_contract = parse_research_contract_document(_typed_idea) + if _typed_contract is not None: + _typed_metric = _typed_contract.metric_contract + _ec["primary_metric"] = _typed_metric.name + _ec["higher_is_better"] = _typed_metric.direction != "lower" + _ec["metric_rationale"] = _typed_metric.rationale + _ec["metric_unit"] = _typed_metric.unit + _ec["research_contract_digest"] = ( + _typed_contract.contract_digest + ) + except ResearchContractError: + raise + except Exception as _typed_exc: + log.warning("Typed research contract rejected: %s", _typed_exc) # Strategy 1: check node memory_snapshot (populated if memory.add() succeeded) - for _n in all_nodes: + for _n in (all_nodes if not _ec["primary_metric"] else []): for _snap in (_n.memory_snapshot if hasattr(_n, "memory_snapshot") else []): if isinstance(_snap, str) and "EVALUATION_CRITERIA:" in _snap: import re as _re_ec @@ -262,7 +294,15 @@ def _cap_memory_entries(entries: list[dict]) -> list[dict]: _sd_path = Path(checkpoint_dir) / "science_data.json" if _sd_path.exists(): _sd = _json.loads(_sd_path.read_text()) - _exp_ctx = _sd.get("experiment_context", {}) + if _sd.get("schema_version") == "ari.science-data/v1": + _annotation = _sd.get("interpretation") or {} + _exp_ctx = ( + _annotation.get("experiment_context", {}) + if _annotation.get("status") == "ok" + else {} + ) + else: + _exp_ctx = _sd.get("experiment_context", {}) if _exp_ctx and not _exp_ctx.get("error"): # Prioritize key_results and implementation_details at the front # so they survive truncation in downstream prompts. @@ -291,6 +331,38 @@ def _cap_memory_entries(entries: list[dict]) -> list[dict]: _idea_data = json.loads(_idea_path.read_text()) _gap = _idea_data.get("gap_analysis", "") _ideas = _idea_data.get("ideas", []) + _directive_idea_data = _idea_data + if _idea_data.get("research_contract") is not None: + from ari.public.research_contract import ( + parse_research_contract_document, + ) + + _selected_contract = parse_research_contract_document( + _idea_data + ) + if _selected_contract is not None: + _selected_idea = { + "title": _selected_contract.title, + "description": _selected_contract.hypothesis, + "hypothesis": _selected_contract.hypothesis, + "experiment_plan": _selected_contract.experiment_plan, + "candidate_id": _selected_contract.selected_candidate_id, + "falsification_conditions": list( + _selected_contract.falsification_conditions + ), + "citations": list(_selected_contract.citations), + "limitations": list(_selected_contract.limitations), + "contract_status": "admitted", + } + _alternatives = [ + item + for item in _ideas + if not isinstance(item, dict) + or item.get("candidate_id") + != _selected_contract.selected_candidate_id + ] + _ideas = [_selected_idea, *_alternatives] + _directive_idea_data = {**_idea_data, "ideas": _ideas} if _ideas: # Phase 1: auto-append plan/alternatives to checkpoint experiment.md. # Mode is read from workflow.yaml (default index_only). Idempotent — @@ -299,7 +371,9 @@ def _cap_memory_entries(entries: list[dict]) -> list[dict]: _plan_promote_mode = str(_wf_cfg.get("plan_promote", "index_only")).lower() if _plan_promote_mode in ("full", "index_only"): _did_promote = _promote_plan_to_experiment_md( - checkpoint_dir, _idea_data, mode=_plan_promote_mode + checkpoint_dir, + _directive_idea_data, + mode=_plan_promote_mode, ) if _did_promote: log.info( @@ -461,6 +535,8 @@ def _cap_memory_entries(entries: list[dict]) -> list[dict]: # Initialise the feedback slot so {{vlm_feedback}} resolves to "" on # the first pass (before any loop has injected real feedback). ctx.tpl_vars.setdefault("vlm_feedback", "") + ctx.tpl_vars.setdefault("plot_revision", 0) + ctx.tpl_vars.setdefault("previous_figure_batch", "") _stage_idx = 0 while _stage_idx < len(stages): @@ -527,8 +603,30 @@ def _cap_memory_entries(entries: list[dict]) -> list[dict]: ) else: _loop_iterations[stage_name] = _count + 1 - # Surface review feedback to downstream template vars - ctx.tpl_vars["vlm_feedback"] = _format_vlm_feedback(result) + # Preserve the exact reviewed batch and revision. Native + # visual reviews stay structured so plot feedback binds + # manifest/review digests instead of an ad-hoc prose fold. + _target_state = ctx.tpl_vars["stages"].get( + _loop_target, {} + ) + ctx.tpl_vars["previous_figure_batch"] = str( + _target_state.get("output") or "" + ) + ctx.tpl_vars["plot_revision"] = _count + 1 + if ( + isinstance(result, dict) + and result.get("schema_version") + == "ari.visual-review-batch/v1" + ): + ctx.tpl_vars["vlm_feedback"] = json.dumps( + result, + ensure_ascii=False, + sort_keys=True, + ) + else: + ctx.tpl_vars["vlm_feedback"] = _format_vlm_feedback( + result + ) # Reset state for stages [target_idx .. _stage_idx] # so they actually re-run (don't hit skip_if_exists # on their own outputs). diff --git a/ari-core/ari/pipeline/stage_runner.py b/ari-core/ari/pipeline/stage_runner.py index 8b24af3f..59247c61 100644 --- a/ari-core/ari/pipeline/stage_runner.py +++ b/ari-core/ari/pipeline/stage_runner.py @@ -385,7 +385,9 @@ def _run_stage_subprocess(tool: str, args: dict, config_path: str, skill_name: s " _cfg_path = str(_pkg_cfg) if _pkg_cfg.exists() else _cfg_path\n" "cfg = load_config(_cfg_path)\n" + _skill_filter + - "mcp = MCPClient(skills, disabled_tools=getattr(cfg, 'disabled_tools', []) or [])\n" + "_skill_lock = os.path.join(_ckpt_dir, 'SKILLS.lock') if _ckpt_dir else None\n" + "mcp = MCPClient(skills, disabled_tools=getattr(cfg, 'disabled_tools', []) or [], " + "skill_lock_path=_skill_lock, skill_lock_scope='subset')\n" "mcp.list_tools()\n" "with open(" + _apath + ") as _af:\n" " _call_args = json.load(_af)\n" diff --git a/ari-core/ari/pipeline/stages.py b/ari-core/ari/pipeline/stages.py index 0b16b721..510b2596 100644 --- a/ari-core/ari/pipeline/stages.py +++ b/ari-core/ari/pipeline/stages.py @@ -139,6 +139,17 @@ def persist( # Handle figures_manifest specially if stage_name == "generate_figures" or "figures" in stage_name: + if ( + isinstance(result, dict) + and result.get("schema_version") == "ari.figure-batch/v1" + ): + log.info( + "Stage [%s]: preserved canonical figure batch %s (%d manifests)", + stage_name, + primary_file, + len(result.get("manifests") or []), + ) + return figs = result.get("figures", {}) if isinstance(result, dict) else {} latex_snips = result.get("latex_snippets", {}) if isinstance(result, dict) else {} fig_kinds = result.get("figure_kinds", {}) if isinstance(result, dict) else {} diff --git a/ari-core/ari/public/README.md b/ari-core/ari/public/README.md index 38e062c8..5e23f1d8 100644 --- a/ari-core/ari/public/README.md +++ b/ari-core/ari/public/README.md @@ -8,14 +8,32 @@ core can refactor freely while the contract stays put. - `README.md` — this file. - `__init__.py` — exported sub-modules + rationale. -- `claim_gate.py` — re-exports five symbols from `ari.pipeline.claim_gate`: `run_hard_gate` (→ ari-skill-evaluator), `check_emission` (→ ari-skill-coding), `scan_science_data` (→ ari-skill-transform), plus `classify_concept` / `CONCEPT_INVARIANTS` (shared concept→invariant registry). +- `analysis.py` — versioned deterministic analysis requests and result contracts. +- `call_context.py` — explicit run/node/lineage models plus signed transport capability helpers. +- `claim_gate.py` — canonical deterministic gate plus versioned metric +- `clone.py` — digest-verified EAR bundle retrieval and safe extraction. - `config_schema.py` — re-export of `ari.config` models. - `container.py` — re-export of `ari.container`. - `cost_tracker.py` — re-export of `ari.cost_tracker`. +- `evaluation.py` — stable evaluator-contract surface shared by idea, +- `execution.py` — versioned workspace, bounded execution, complete-log +- `figures.py` — declarative `FigureSpecV1`, digest-bound render/batch +- `latex_claims.py` — canonical lexical LaTeX claim/number/citation/figure parser. +- `lineage.py` — TODO - `llm.py` — re-export of `ari.llm.client.LLMClient`. +- `memory.py` — content-addressed memory records, retrievals, events, and backups. +- `node_selection.py` — deterministic downstream node/source selection. +- `paper.py` — immutable paper build, revision, model-call, compile, review, and - `paths.py` — re-export of `ari.paths.PathManager`. +- `publish.py` — staged EAR publication and promotion. +- `research_contract.py` — TODO +- `result.py` — versioned `ResultEnvelopeV1`, artifact references, typed errors, - `run_env.py` — re-export of `ari.agent.run_env` capture helpers. +- `science_data.py` — canonical `ScienceDataV1` raw/derived/interpretation +- `skill_lock.py` — immutable run-level provider/schema/phase snapshot contract +- `skill_manifest.py` — canonical Skill package, entrypoint, and tool-policy - `verified_context.py` — re-export of `ari.pipeline.verified_context` (`render_grounded_block` / `write_verified_context`; used by ari-skill-paper). +- `visual_review.py` — criteria profiles, artifact-bound review findings, ## See also diff --git a/ari-core/ari/public/__init__.py b/ari-core/ari/public/__init__.py index 97325a69..ac5904f9 100644 --- a/ari-core/ari/public/__init__.py +++ b/ari-core/ari/public/__init__.py @@ -8,19 +8,49 @@ - :mod:`ari.public.container` — container runtime helpers used by ari-skill-coding's regression tests. +- :mod:`ari.public.clone` — digest-verified EAR bundle retrieval and + extraction. +- :mod:`ari.public.execution` — closed-workspace, bounded-process, + artifact-log, and typed measurement contracts. +- :mod:`ari.public.analysis` — deterministic statistical request, run + comparison, and result contracts. +- :mod:`ari.public.memory` — immutable research-memory records and + retrieval provenance. +- :mod:`ari.public.science_data` — digest-bound raw, derived, and model + interpretation sections plus the canonical numeric formula registry. +- :mod:`ari.public.figures` — declarative figure specifications, immutable + render manifests, feedback lineage, and legacy offline reader. +- :mod:`ari.public.visual_review` — criteria-versioned multimodal findings, + typed failures, cost/model provenance, and fail-closed review batches. - :mod:`ari.public.cost_tracker` — LLM cost reporting used by ari-skill-plot to log VLM/LLM call costs. - :mod:`ari.public.paths` — :class:`PathManager` for callers that need to resolve checkpoint paths without hard-coding env vars. - :mod:`ari.public.llm` — :class:`LLMClient` for callers that proxy through the ARI-side LLM client. +- :mod:`ari.public.node_selection` — deterministic downstream node/source + selection for transform and publication Skills. +- :mod:`ari.public.research_contract` — immutable survey, idea, metric, and + selected research hand-off contracts. +- :mod:`ari.public.lineage` — read-only ancestor idea/artifact catalog helpers. +- :mod:`ari.public.publish` — staged EAR publication and promotion. - :mod:`ari.public.config_schema` — Pydantic config models for callers that need typed settings access. +- :mod:`ari.public.call_context` — explicit run/node/lineage context and + signed MCP transport capability verification. +- :mod:`ari.public.skill_manifest` — canonical Skill package and tool-policy + contract used by built-in and federated MCP providers. +- :mod:`ari.public.skill_lock` — immutable run snapshot binding manifests to + live MCP schemas and phase-specific admission. +- :mod:`ari.public.result` — versioned result, artifact, error, context, + provenance, and immutable async-handle contracts for typed Skill dispatch. - :mod:`ari.public.run_env` — run-environment capture helpers (``capture_env`` / ``shell_capture_snippet``) used by ari-skill-coding and ari-skill-hpc. - :mod:`ari.public.claim_gate` — ``run_hard_gate`` (Story2Proposal deterministic claim_evidence_hard_gate) used by ari-skill-evaluator. +- :mod:`ari.public.evaluation` — digest-bound metric admission, gate report, + and advisory semantic-review contracts. - :mod:`ari.public.verified_context` — ``render_grounded_block`` / ``write_verified_context`` (artifact-grounded paper claims) used by ari-skill-paper. diff --git a/ari-core/ari/public/analysis.py b/ari-core/ari/public/analysis.py new file mode 100644 index 00000000..356eb4ff --- /dev/null +++ b/ari-core/ari/public/analysis.py @@ -0,0 +1,45 @@ +"""Stable Skill-facing deterministic analysis contracts.""" + +from ari.analysis import ( + ANALYSIS_REQUEST_V1, + ANALYSIS_RESULT_V1, + RUN_COMPARISON_REQUEST_V1, + STATISTICAL_TEST_REQUEST_V1, + AnalysisArtifactTargetV1, + AnalysisArtifactV1, + AnalysisDataSourceV1, + AnalysisObservationV1, + AnalysisRequestV1, + AnalysisResultV1, + AnalysisSummaryV1, + MetricSampleSetV1, + RunComparisonRequestV1, + RunComparisonResultV1, + RunRecordV1, + StatisticalComparisonResultV1, + StatisticalComparisonV1, + StatisticalTestRequestV1, + canonical_analysis_digest, +) + +__all__ = [ + "ANALYSIS_REQUEST_V1", + "ANALYSIS_RESULT_V1", + "RUN_COMPARISON_REQUEST_V1", + "STATISTICAL_TEST_REQUEST_V1", + "AnalysisArtifactTargetV1", + "AnalysisArtifactV1", + "AnalysisDataSourceV1", + "AnalysisObservationV1", + "AnalysisRequestV1", + "AnalysisResultV1", + "AnalysisSummaryV1", + "MetricSampleSetV1", + "RunComparisonRequestV1", + "RunComparisonResultV1", + "RunRecordV1", + "StatisticalComparisonResultV1", + "StatisticalComparisonV1", + "StatisticalTestRequestV1", + "canonical_analysis_digest", +] diff --git a/ari-core/ari/public/call_context.py b/ari-core/ari/public/call_context.py new file mode 100644 index 00000000..a94946db --- /dev/null +++ b/ari-core/ari/public/call_context.py @@ -0,0 +1,37 @@ +"""Stable public contract for explicit and authorized Skill-call context.""" + +from ari.call_context import ( # noqa: F401 + AUTHORIZED_TOOL_CONTEXT_V1, + CALL_CONTEXT_ARGUMENT, + CONTEXT_AUTHORITY_ENV, + NODE_CONTEXT_V1, + RUN_CONTEXT_V1, + AuthorizedToolContextV1, + CallContextAuthorizationError, + NodeContextV1, + RunContextV1, + ToolCallContextV1, + authorize_tool_context, + lineage_digest, + new_context_authority_key, + run_scope_digest, + verify_tool_context, +) + +__all__ = [ + "AUTHORIZED_TOOL_CONTEXT_V1", + "CALL_CONTEXT_ARGUMENT", + "CONTEXT_AUTHORITY_ENV", + "NODE_CONTEXT_V1", + "RUN_CONTEXT_V1", + "AuthorizedToolContextV1", + "CallContextAuthorizationError", + "NodeContextV1", + "RunContextV1", + "ToolCallContextV1", + "authorize_tool_context", + "lineage_digest", + "new_context_authority_key", + "run_scope_digest", + "verify_tool_context", +] diff --git a/ari-core/ari/public/claim_gate.py b/ari-core/ari/public/claim_gate.py index eeb40b13..287cdf68 100644 --- a/ari-core/ari/public/claim_gate.py +++ b/ari-core/ari/public/claim_gate.py @@ -16,6 +16,25 @@ """ from ari.pipeline.claim_gate import run_hard_gate # noqa: F401 +from ari.claim_gate_contract import ( # noqa: F401 + ClaimGateContractError, + GateFindingV1, + GateFormulaProvenanceV1, + GateReportV1, + MetricAdmissionDecisionV1, + MetricClaimV1, + MetricContractProposalV1, + MetricGateContractV1, + SemanticFindingV1, + SemanticReviewV1, + SemanticRevisionV1, + admit_metric_contract_proposal, + migrate_legacy_gate_report, + migrate_legacy_metric_gate_contract, + parse_gate_report, + parse_metric_gate_contract, + parse_semantic_review, +) from ari.pipeline.claim_gate.contract import check_emission # noqa: F401 from ari.pipeline.claim_gate.invariants import ( # noqa: F401 CONCEPT_INVARIANTS, @@ -25,5 +44,11 @@ __all__ = [ "run_hard_gate", "check_emission", "classify_concept", "scan_science_data", - "CONCEPT_INVARIANTS", + "CONCEPT_INVARIANTS", "ClaimGateContractError", "GateFindingV1", + "GateFormulaProvenanceV1", "GateReportV1", "MetricAdmissionDecisionV1", + "MetricClaimV1", "MetricContractProposalV1", "MetricGateContractV1", + "SemanticFindingV1", "SemanticReviewV1", "SemanticRevisionV1", + "admit_metric_contract_proposal", "migrate_legacy_metric_gate_contract", + "migrate_legacy_gate_report", + "parse_gate_report", "parse_metric_gate_contract", "parse_semantic_review", ] diff --git a/ari-core/ari/public/clone.py b/ari-core/ari/public/clone.py new file mode 100644 index 00000000..9ffe6c14 --- /dev/null +++ b/ari-core/ari/public/clone.py @@ -0,0 +1,5 @@ +"""Stable Skill-facing verified EAR bundle clone API.""" + +from ari.clone import CloneError, CloneResult, clone + +__all__ = ["CloneError", "CloneResult", "clone"] diff --git a/ari-core/ari/public/evaluation.py b/ari-core/ari/public/evaluation.py new file mode 100644 index 00000000..a393a41e --- /dev/null +++ b/ari-core/ari/public/evaluation.py @@ -0,0 +1,53 @@ +"""Stable evaluator contracts shared by scientific Skills.""" + +from ari.claim_gate_contract import ( + GATE_FINDING_V1, + GATE_REPORT_V1, + METRIC_ADMISSION_DECISION_V1, + METRIC_CONTRACT_PROPOSAL_V1, + METRIC_GATE_CONTRACT_V1, + SEMANTIC_REVIEW_V1, + ClaimGateContractError, + GateFindingV1, + GateFormulaProvenanceV1, + GateReportV1, + MetricAdmissionDecisionV1, + MetricClaimV1, + MetricContractProposalV1, + MetricGateContractV1, + SemanticFindingV1, + SemanticReviewV1, + SemanticRevisionV1, + admit_metric_contract_proposal, + migrate_legacy_gate_report, + migrate_legacy_metric_gate_contract, + parse_gate_report, + parse_metric_gate_contract, + parse_semantic_review, +) + +__all__ = [ + "GATE_FINDING_V1", + "GATE_REPORT_V1", + "METRIC_ADMISSION_DECISION_V1", + "METRIC_CONTRACT_PROPOSAL_V1", + "METRIC_GATE_CONTRACT_V1", + "SEMANTIC_REVIEW_V1", + "ClaimGateContractError", + "GateFindingV1", + "GateFormulaProvenanceV1", + "GateReportV1", + "MetricAdmissionDecisionV1", + "MetricClaimV1", + "MetricContractProposalV1", + "MetricGateContractV1", + "SemanticFindingV1", + "SemanticReviewV1", + "SemanticRevisionV1", + "admit_metric_contract_proposal", + "migrate_legacy_gate_report", + "migrate_legacy_metric_gate_contract", + "parse_gate_report", + "parse_metric_gate_contract", + "parse_semantic_review", +] diff --git a/ari-core/ari/public/execution.py b/ari-core/ari/public/execution.py new file mode 100644 index 00000000..f86218c5 --- /dev/null +++ b/ari-core/ari/public/execution.py @@ -0,0 +1,49 @@ +"""Stable Skill-facing execution and measurement contracts.""" + +from ari.execution import ( + ContainerIdentityV1, + EXECUTION_REQUEST_V1, + EXECUTION_RESULT_V1, + MEASUREMENT_SET_V1, + WORKSPACE_REF_V1, + ExecutionArtifactV1, + ExecutionLimitsV1, + ExecutionPolicyError, + ExecutionRequestV1, + ExecutionResultV1, + MeasurementRecordV1, + MeasurementDocumentError, + MeasurementDocumentFormat, + MeasurementSetV1, + ResourceLimitReportV1, + WorkspaceRefV1, + build_minimal_environment, + execute_local, + measurement_document_format, + parse_measurement_document, + record_completed_execution, +) + +__all__ = [ + "ContainerIdentityV1", + "EXECUTION_REQUEST_V1", + "EXECUTION_RESULT_V1", + "MEASUREMENT_SET_V1", + "WORKSPACE_REF_V1", + "ExecutionArtifactV1", + "ExecutionLimitsV1", + "ExecutionPolicyError", + "ExecutionRequestV1", + "ExecutionResultV1", + "MeasurementRecordV1", + "MeasurementDocumentError", + "MeasurementDocumentFormat", + "MeasurementSetV1", + "ResourceLimitReportV1", + "WorkspaceRefV1", + "build_minimal_environment", + "execute_local", + "measurement_document_format", + "parse_measurement_document", + "record_completed_execution", +] diff --git a/ari-core/ari/public/figures.py b/ari-core/ari/public/figures.py new file mode 100644 index 00000000..8f80da8a --- /dev/null +++ b/ari-core/ari/public/figures.py @@ -0,0 +1,51 @@ +"""Stable Skill-facing scientific figure contracts.""" + +from ari.figure_contract import ( + FIGURE_ARTIFACT_V1, + FIGURE_BATCH_V1, + FIGURE_ENVIRONMENT_V1, + FIGURE_FEEDBACK_V1, + FIGURE_MANIFEST_V1, + FIGURE_SPEC_V1, + LEGACY_FIGURE_BATCH_V0, + FigureArtifactV1, + FigureAxisV1, + FigureBatchV1, + FigureContractError, + FigureEnvironmentV1, + FigureFeedbackV1, + FigureManifestV1, + FigureSourceV1, + FigureSpecV1, + FigureUncertaintyV1, + LegacyFigureBatchV0, + canonical_figure_digest, + parse_figure_batch, + parse_figure_manifest, + read_legacy_figure_batch, +) + +__all__ = [ + "FIGURE_ARTIFACT_V1", + "FIGURE_BATCH_V1", + "FIGURE_ENVIRONMENT_V1", + "FIGURE_FEEDBACK_V1", + "FIGURE_MANIFEST_V1", + "FIGURE_SPEC_V1", + "LEGACY_FIGURE_BATCH_V0", + "FigureArtifactV1", + "FigureAxisV1", + "FigureBatchV1", + "FigureContractError", + "FigureEnvironmentV1", + "FigureFeedbackV1", + "FigureManifestV1", + "FigureSourceV1", + "FigureSpecV1", + "FigureUncertaintyV1", + "LegacyFigureBatchV0", + "canonical_figure_digest", + "parse_figure_batch", + "parse_figure_manifest", + "read_legacy_figure_batch", +] diff --git a/ari-core/ari/public/latex_claims.py b/ari-core/ari/public/latex_claims.py new file mode 100644 index 00000000..54a37e23 --- /dev/null +++ b/ari-core/ari/public/latex_claims.py @@ -0,0 +1,25 @@ +"""Stable Skill-facing LaTeX claim parsing contract.""" + +from ari.latex_claims import ( + ANCHOR_RE, + build_section_map, + claim_span_hash, + extract_numeric_mentions, + figure_references, + find_claim_anchors, + normalize_claim_sentence, + section_at, + sentence_for_anchor, +) + +__all__ = [ + "ANCHOR_RE", + "build_section_map", + "claim_span_hash", + "extract_numeric_mentions", + "figure_references", + "find_claim_anchors", + "normalize_claim_sentence", + "section_at", + "sentence_for_anchor", +] diff --git a/ari-core/ari/public/lineage.py b/ari-core/ari/public/lineage.py new file mode 100644 index 00000000..520c2988 --- /dev/null +++ b/ari-core/ari/public/lineage.py @@ -0,0 +1,13 @@ +"""Stable read-only lineage catalog API for Skills.""" + +from ari.lineage import ( + format_ancestor_pool_for_virsci, + get_idea_pool_for_ckpt, + walk_ancestor_ckpts, +) + +__all__ = [ + "format_ancestor_pool_for_virsci", + "get_idea_pool_for_ckpt", + "walk_ancestor_ckpts", +] diff --git a/ari-core/ari/public/memory.py b/ari-core/ari/public/memory.py new file mode 100644 index 00000000..2d9d9d75 --- /dev/null +++ b/ari-core/ari/public/memory.py @@ -0,0 +1,37 @@ +"""Stable Skill-facing research-memory contracts.""" + +from ari.memory_contract import ( + MEMORY_BACKUP_V1, + MEMORY_RECORD_V1, + MEMORY_RETRIEVAL_V1, + MemoryArtifactRefV1, + MemoryBackupV1, + MemoryMetricPointerV1, + MemoryNodeReportRefV1, + MemoryRecordV1, + MemoryRetrievalProvenanceV1, + MemoryRetrievalV1, + MemoryReactEntryV1, + build_memory_backup, + build_memory_react_entry, + build_memory_record, + canonical_memory_digest, +) + +__all__ = [ + "MEMORY_BACKUP_V1", + "MEMORY_RECORD_V1", + "MEMORY_RETRIEVAL_V1", + "MemoryArtifactRefV1", + "MemoryBackupV1", + "MemoryMetricPointerV1", + "MemoryNodeReportRefV1", + "MemoryRecordV1", + "MemoryRetrievalProvenanceV1", + "MemoryRetrievalV1", + "MemoryReactEntryV1", + "build_memory_backup", + "build_memory_react_entry", + "build_memory_record", + "canonical_memory_digest", +] diff --git a/ari-core/ari/public/node_selection.py b/ari-core/ari/public/node_selection.py new file mode 100644 index 00000000..6968a25e --- /dev/null +++ b/ari-core/ari/public/node_selection.py @@ -0,0 +1,23 @@ +"""Stable re-export of deterministic publication node-selection helpers.""" + +from ari.orchestrator.node_selection import ( + build_parent_chain, + collect_excluded, + contributes_code, + filter_nodes, + is_narrative_step, + is_relevant_for_synthesis, + load_selected_sources, + select_source_files_for_publication, +) + +__all__ = [ + "build_parent_chain", + "collect_excluded", + "contributes_code", + "filter_nodes", + "is_narrative_step", + "is_relevant_for_synthesis", + "load_selected_sources", + "select_source_files_for_publication", +] diff --git a/ari-core/ari/public/paper.py b/ari-core/ari/public/paper.py new file mode 100644 index 00000000..6ab13ec2 --- /dev/null +++ b/ari-core/ari/public/paper.py @@ -0,0 +1,49 @@ +"""Stable Skill-facing paper build and artifact contracts.""" + +from ari.paper_contract import ( + PAPER_ARTIFACT_V1, + PAPER_BUILD_V1, + PAPER_COMPILE_V1, + PAPER_MODEL_CALL_BATCH_V1, + PAPER_MODEL_CALL_V1, + PAPER_REVIEW_SET_V1, + PAPER_REVISION_V1, + PaperArtifactV1, + PaperBuildV1, + PaperCompileV1, + PaperContractError, + PaperGateSummaryV1, + PaperModelCallBatchV1, + PaperModelCallV1, + PaperModelUsageV1, + PaperNumericCoverageV1, + PaperReviewSetV1, + PaperRevisionV1, + canonical_paper_digest, + parse_paper_build, + parse_paper_model_call_batch, +) + +__all__ = [ + "PAPER_ARTIFACT_V1", + "PAPER_BUILD_V1", + "PAPER_COMPILE_V1", + "PAPER_MODEL_CALL_BATCH_V1", + "PAPER_MODEL_CALL_V1", + "PAPER_REVIEW_SET_V1", + "PAPER_REVISION_V1", + "PaperArtifactV1", + "PaperBuildV1", + "PaperCompileV1", + "PaperContractError", + "PaperGateSummaryV1", + "PaperModelCallBatchV1", + "PaperModelCallV1", + "PaperModelUsageV1", + "PaperNumericCoverageV1", + "PaperReviewSetV1", + "PaperRevisionV1", + "canonical_paper_digest", + "parse_paper_build", + "parse_paper_model_call_batch", +] diff --git a/ari-core/ari/public/publish.py b/ari-core/ari/public/publish.py new file mode 100644 index 00000000..0d1e2923 --- /dev/null +++ b/ari-core/ari/public/publish.py @@ -0,0 +1,5 @@ +"""Stable Skill-facing publication API.""" + +from ari.publish import PublishError, PublishRecord, promote, publish + +__all__ = ["PublishError", "PublishRecord", "promote", "publish"] diff --git a/ari-core/ari/public/research_contract.py b/ari-core/ari/public/research_contract.py new file mode 100644 index 00000000..3f55d988 --- /dev/null +++ b/ari-core/ari/public/research_contract.py @@ -0,0 +1,4 @@ +"""Stable public research hand-off contracts for Skills.""" + +from ari.research_contract import * # noqa: F403 +from ari.research_contract import __all__ # noqa: F401 diff --git a/ari-core/ari/public/result.py b/ari-core/ari/public/result.py new file mode 100644 index 00000000..b87c6d65 --- /dev/null +++ b/ari-core/ari/public/result.py @@ -0,0 +1,47 @@ +"""Stable public contract for normalized Skill results and artifacts.""" + +from ari.result import ( # noqa: F401 + ASYNC_TOOL_HANDLE_V1, + ARTIFACT_REF_V1, + DEFAULT_INLINE_RESULT_LIMIT, + RAW_RESULT_ROLE, + RESULT_ENVELOPE_V1, + SHA256_DIGEST_PATTERN, + ResultArtifactIntegrityError, + ResultArtifactV1, + ResultEnvelopeNormalizer, + ResultEnvelopeV1, + ResultErrorKind, + ResultErrorV1, + ResultProvenanceV1, + AsyncStateMapV1, + AsyncToolEndpointV1, + AsyncToolHandleV1, + NodeContextV1, + RunContextV1, + ToolCallContextV1, + utc_now_iso, +) + +__all__ = [ + "ASYNC_TOOL_HANDLE_V1", + "ARTIFACT_REF_V1", + "DEFAULT_INLINE_RESULT_LIMIT", + "RAW_RESULT_ROLE", + "RESULT_ENVELOPE_V1", + "SHA256_DIGEST_PATTERN", + "ResultArtifactIntegrityError", + "ResultArtifactV1", + "ResultEnvelopeNormalizer", + "ResultEnvelopeV1", + "ResultErrorKind", + "ResultErrorV1", + "ResultProvenanceV1", + "AsyncStateMapV1", + "AsyncToolEndpointV1", + "AsyncToolHandleV1", + "NodeContextV1", + "RunContextV1", + "ToolCallContextV1", + "utc_now_iso", +] diff --git a/ari-core/ari/public/science_data.py b/ari-core/ari/public/science_data.py new file mode 100644 index 00000000..579f55c8 --- /dev/null +++ b/ari-core/ari/public/science_data.py @@ -0,0 +1,65 @@ +"""Stable Skill-facing scientific-data and formula-registry contract.""" + +from ari.science_data_contract import ( + FORMULAS, + SCIENCE_ARTIFACT_REF_V1, + SCIENCE_DATA_V1, + SCIENCE_DERIVED_V1, + SCIENCE_INTERPRETATION_V1, + SCIENCE_PROVENANCE_V1, + SCIENCE_RAW_V1, + ScienceArtifactRefV1, + ScienceClaimEvidenceV1, + ScienceClaimV1, + ScienceConfigurationV1, + ScienceDataError, + ScienceDataV1, + ScienceDerivedV1, + ScienceEnvironmentV1, + ScienceEvidenceResultV1, + ScienceInterpretationV1, + ScienceMetricSummaryV1, + ScienceNumericAssertionV1, + ScienceOperandV1, + ScienceProvenanceV1, + ScienceRawV1, + canonical_science_digest, + formula_registry_digest, + migrate_legacy_science_data, + parse_science_data, + recompute, + required_roles, + science_data_projection, +) + +__all__ = [ + "FORMULAS", + "SCIENCE_ARTIFACT_REF_V1", + "SCIENCE_DATA_V1", + "SCIENCE_DERIVED_V1", + "SCIENCE_INTERPRETATION_V1", + "SCIENCE_PROVENANCE_V1", + "SCIENCE_RAW_V1", + "ScienceArtifactRefV1", + "ScienceClaimEvidenceV1", + "ScienceClaimV1", + "ScienceConfigurationV1", + "ScienceDataError", + "ScienceDataV1", + "ScienceDerivedV1", + "ScienceEnvironmentV1", + "ScienceEvidenceResultV1", + "ScienceInterpretationV1", + "ScienceMetricSummaryV1", + "ScienceNumericAssertionV1", + "ScienceOperandV1", + "ScienceProvenanceV1", + "ScienceRawV1", + "canonical_science_digest", + "formula_registry_digest", + "migrate_legacy_science_data", + "parse_science_data", + "recompute", + "required_roles", + "science_data_projection", +] diff --git a/ari-core/ari/public/skill_lock.py b/ari-core/ari/public/skill_lock.py new file mode 100644 index 00000000..c31a183e --- /dev/null +++ b/ari-core/ari/public/skill_lock.py @@ -0,0 +1,41 @@ +"""Stable public contract for run-level immutable Skill snapshots.""" + +from ari.skill_lock import ( # noqa: F401 + DEFAULT_RUNTIME_PHASES, + SKILLS_LOCK_FILENAME, + SKILLS_LOCK_SCHEMA_VERSION, + LockedSkillV1, + LockedCredentialScopeV1, + LockedToolV1, + SkillLockCorruptError, + SkillLockError, + SkillLockMismatchError, + SkillLockMissingError, + SkillProviderAdmissionError, + SkillsLockV1, + build_skills_lock, + load_skills_lock, + skills_lock_digest, + verify_skills_lock_subset, + write_or_verify_skills_lock, +) + +__all__ = [ + "DEFAULT_RUNTIME_PHASES", + "SKILLS_LOCK_FILENAME", + "SKILLS_LOCK_SCHEMA_VERSION", + "LockedSkillV1", + "LockedCredentialScopeV1", + "LockedToolV1", + "SkillLockCorruptError", + "SkillLockError", + "SkillLockMismatchError", + "SkillLockMissingError", + "SkillProviderAdmissionError", + "SkillsLockV1", + "build_skills_lock", + "load_skills_lock", + "skills_lock_digest", + "verify_skills_lock_subset", + "write_or_verify_skills_lock", +] diff --git a/ari-core/ari/public/skill_manifest.py b/ari-core/ari/public/skill_manifest.py new file mode 100644 index 00000000..96e3469f --- /dev/null +++ b/ari-core/ari/public/skill_manifest.py @@ -0,0 +1,43 @@ +"""Stable public contract for ARI Skill manifests.""" + +from ari.skill_manifest import ( # noqa: F401 + LEGACY_MCP_RESULT_V1, + MANIFEST_FILENAME, + RESULT_ENVELOPE_V1, + AsyncLifecycleV1, + CredentialScopeV1, + ResolvedToolManifestV1, + SkillEntrypointV1, + SkillManifestError, + SkillManifestV1, + ToolManifestV1, + ToolPolicyV1, + TimeoutBudgetV1, + legacy_mcp_document, + load_skill_manifest, + looks_like_credential_environment_name, + manifest_digest, + manifest_tool_ref, + resolve_skill_entrypoint, +) + +__all__ = [ + "LEGACY_MCP_RESULT_V1", + "MANIFEST_FILENAME", + "RESULT_ENVELOPE_V1", + "AsyncLifecycleV1", + "CredentialScopeV1", + "ResolvedToolManifestV1", + "SkillEntrypointV1", + "SkillManifestError", + "SkillManifestV1", + "ToolManifestV1", + "ToolPolicyV1", + "TimeoutBudgetV1", + "legacy_mcp_document", + "load_skill_manifest", + "looks_like_credential_environment_name", + "manifest_digest", + "manifest_tool_ref", + "resolve_skill_entrypoint", +] diff --git a/ari-core/ari/public/visual_review.py b/ari-core/ari/public/visual_review.py new file mode 100644 index 00000000..318bfda7 --- /dev/null +++ b/ari-core/ari/public/visual_review.py @@ -0,0 +1,39 @@ +"""Stable Skill-facing visual-review contracts.""" + +from ari.visual_review_contract import ( + VISUAL_ARTIFACT_REF_V1, + VISUAL_CRITERIA_PROFILE_V1, + VISUAL_REVIEW_BATCH_V1, + VISUAL_REVIEW_V1, + VisualArtifactRefV1, + VisualCriteriaProfileV1, + VisualCriterionV1, + VisualIssueV1, + VisualModelUsageV1, + VisualRegionV1, + VisualReviewBatchV1, + VisualReviewContractError, + VisualReviewV1, + canonical_visual_review_digest, + parse_visual_review, + parse_visual_review_batch, +) + +__all__ = [ + "VISUAL_ARTIFACT_REF_V1", + "VISUAL_CRITERIA_PROFILE_V1", + "VISUAL_REVIEW_BATCH_V1", + "VISUAL_REVIEW_V1", + "VisualArtifactRefV1", + "VisualCriteriaProfileV1", + "VisualCriterionV1", + "VisualIssueV1", + "VisualModelUsageV1", + "VisualRegionV1", + "VisualReviewBatchV1", + "VisualReviewContractError", + "VisualReviewV1", + "canonical_visual_review_digest", + "parse_visual_review", + "parse_visual_review_batch", +] diff --git a/ari-core/ari/publish/__init__.py b/ari-core/ari/publish/__init__.py index cef120e4..be138ff0 100644 --- a/ari-core/ari/publish/__init__.py +++ b/ari-core/ari/publish/__init__.py @@ -16,12 +16,12 @@ The registered ref + bundle_sha256 are then pushed into the paper's Code Availability section by ``inject_code_availability``. """ + from __future__ import annotations import hashlib import json import os -import shutil import tarfile import tempfile from dataclasses import dataclass, asdict @@ -52,8 +52,100 @@ class PublishRecord: def _read_manifest(curated_dir: Path) -> dict: p = curated_dir / "manifest.lock" if not p.exists(): - raise PublishError(f"manifest.lock not found in {curated_dir} — run `ari ear curate` first") - return json.loads(p.read_text(encoding="utf-8")) + raise PublishError( + f"manifest.lock not found in {curated_dir} — run `ari ear curate` first" + ) + try: + manifest = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PublishError(f"manifest.lock is unreadable: {exc}") from exc + _verify_manifest(curated_dir, manifest) + return manifest + + +def _canonical_digest(value: object, *, prefixed: bool = True) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + value = hashlib.sha256(payload).hexdigest() + return "sha256:" + value if prefixed else value + + +def _verify_manifest(curated_dir: Path, manifest: dict) -> None: + """Fail closed before any backend observes a curated bundle.""" + + version = int(manifest.get("version") or 1) + if version not in {1, 2}: + raise PublishError(f"unsupported EAR manifest version: {version}") + files = manifest.get("files") + if not isinstance(files, list): + raise PublishError("manifest files must be a list") + root = curated_dir.resolve(strict=True) + rebuilt: list[dict] = [] + declared_paths: set[str] = set() + for record in files: + if not isinstance(record, dict): + raise PublishError("manifest file record must be an object") + relative = str(record.get("path") or "") + candidate = curated_dir / relative + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise PublishError(f"manifest path escapes bundle: {relative}") from exc + if ( + relative in declared_paths + or candidate.is_symlink() + or not candidate.is_file() + ): + raise PublishError( + f"manifest path missing, duplicate, or symbolic: {relative}" + ) + declared_paths.add(relative) + actual = hashlib.sha256(candidate.read_bytes()).hexdigest() + if actual != record.get("sha256") or candidate.stat().st_size != record.get( + "size" + ): + raise PublishError(f"manifest integrity mismatch: {relative}") + rebuilt_record = { + "path": relative, + "sha256": actual, + "size": candidate.stat().st_size, + } + if version == 2: + role = record.get("role") + if not isinstance(role, str) or not role: + raise PublishError(f"v2 manifest lacks role: {relative}") + rebuilt_record["role"] = role + rebuilt.append(rebuilt_record) + actual_paths = { + path.relative_to(curated_dir).as_posix() + for path in curated_dir.rglob("*") + if path.is_file() and path.name != "manifest.lock" + } + if actual_paths != declared_paths: + raise PublishError("curated bundle contains untracked or missing files") + canonical = { + "version": version, + "files": sorted(rebuilt, key=lambda item: item["path"]), + } + if _canonical_digest(canonical, prefixed=False) != manifest.get("bundle_sha256"): + raise PublishError("manifest bundle_sha256 mismatch") + if version == 2: + deterministic_lock = { + **canonical, + "bundle_sha256": manifest.get("bundle_sha256"), + "policy_digest": manifest.get("policy_digest"), + "evidence_index_digest": manifest.get("evidence_index_digest"), + "evidence": manifest.get("evidence"), + "admission_status": manifest.get("admission_status"), + } + if _canonical_digest(deterministic_lock) != manifest.get("lock_digest"): + raise PublishError("EAR lock digest mismatch") def _build_tarball(curated_dir: Path, dest_path: Path) -> str: @@ -103,7 +195,7 @@ def publish( ckpt = Path(checkpoint).resolve() curated = ckpt / "ear_published" if not curated.is_dir(): - raise PublishError(f"ear_published/ not found — run `ari ear curate` first") + raise PublishError("ear_published/ not found — run `ari ear curate` first") manifest = _read_manifest(curated) bundle_sha256 = manifest.get("bundle_sha256", "") @@ -160,7 +252,9 @@ def promote( ckpt = Path(checkpoint).resolve() record_path = ckpt / "publish_record.json" if not record_path.exists(): - raise PublishError(f"publish_record.json not found — run `ari ear publish` first") + raise PublishError( + "publish_record.json not found — run `ari ear publish` first" + ) data = json.loads(record_path.read_text(encoding="utf-8")) backend_name = data.get("backend", "ari-registry") backend_impl = _load_backend(backend_name) @@ -178,12 +272,18 @@ def promote( {k: v for k, v in out.items() if k not in ("visibility",)} ) except Exception as e: - data["promote_failed_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + data["promote_failed_at"] = datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) data.setdefault("extra", {})["promote_error"] = f"{type(e).__name__}: {e}" - record_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + record_path.write_text( + json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8" + ) raise PublishError(f"promote failed: {e}") from e - record_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + record_path.write_text( + json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8" + ) return PublishRecord( backend=backend_name, ref=data.get("ref", ""), @@ -213,11 +313,13 @@ def promote( def _load_ari_registry_backend(): from .backends import ari_registry as backend + return backend def _load_local_tarball_backend(): from .backends import local_tarball as backend + return backend @@ -237,7 +339,9 @@ def _load_gh_backend(): return backend -_BACKEND_REGISTRY: "BaseRegistry" = BaseRegistry("publish backend", error_cls=PublishError) +_BACKEND_REGISTRY: "BaseRegistry" = BaseRegistry( + "publish backend", error_cls=PublishError +) _BACKEND_REGISTRY.register_lazy("ari-registry", _load_ari_registry_backend) _BACKEND_REGISTRY.register_lazy("local-tarball", _load_local_tarball_backend) _BACKEND_REGISTRY.register_lazy("zenodo", _load_zenodo_backend) diff --git a/ari-core/ari/research_contract.py b/ari-core/ari/research_contract.py new file mode 100644 index 00000000..90a63ac5 --- /dev/null +++ b/ari-core/ari/research_contract.py @@ -0,0 +1,725 @@ +"""Immutable literature, idea, and research-contract records. + +The models in this module form the scientific hand-off between retrieval, +ideation, evaluation, transformation, and publication. Digest-bound records +are deliberately strict: a consumer either receives the exact object selected +by the idea stage or rejects it. In particular, consumers must not silently +re-extract a metric or evidence vocabulary from prose. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime +from typing import Any, ClassVar, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + + +RETRIEVAL_RECORD_V1 = "ari.retrieval-record/v1" +SURVEY_SNAPSHOT_V1 = "ari.survey-snapshot/v1" +METRIC_CONTRACT_V1 = "ari.metric-contract/v1" +IDEA_CANDIDATE_V1 = "ari.idea-candidate/v1" +IDEA_REJECTION_V1 = "ari.idea-rejection/v1" +IDEA_GENERATION_LOCK_V1 = "ari.idea-generation-lock/v1" +IDEA_GENERATION_PROVENANCE_V1 = "ari.idea-generation-provenance/v1" +IDEA_SET_V1 = "ari.idea-set/v1" +RESEARCH_CONTRACT_V1 = "ari.research-contract/v1" +RESEARCH_ARTIFACT_REF_V1 = "ari.research-artifact-ref/v1" +CITATION_EDGE_V1 = "ari.citation-edge/v1" + +SHA256_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_ZERO_DIGEST = "sha256:" + ("0" * 64) +_SAFE_TOKEN = re.compile(r"^[a-z0-9][a-z0-9._:/@+-]{0,255}$") + + +class ResearchContractError(ValueError): + """A research hand-off is malformed, tampered with, or inconsistent.""" + + +def canonical_digest(value: Any) -> str: + """Return the stable digest of a JSON-compatible value or Pydantic model.""" + + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class _DigestBoundModel(_StrictModel): + """Base for records whose digest covers every field except itself.""" + + _digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values[cls._digest_field] = _ZERO_DIGEST + return cls.model_validate(values, context={"bind_research_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self._digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_digest(self.digest_payload()) + if info.context and info.context.get("bind_research_digest"): + object.__setattr__(self, self._digest_field, expected) + elif getattr(self, self._digest_field) != expected: + raise ValueError( + f"{self._digest_field} does not match the canonical payload" + ) + return self + + +class ResearchArtifactRefV1(_StrictModel): + """Content-addressed research artifact under a checkpoint/workspace root.""" + + schema_version: Literal["ari.research-artifact-ref/v1"] = ( + RESEARCH_ARTIFACT_REF_V1 + ) + logical_name: str = Field(min_length=1, max_length=512) + digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + media_type: str = Field(min_length=1, max_length=128) + role: str = Field(min_length=1, max_length=128) + source_run_id: str | None = Field(default=None, max_length=256) + + @field_validator("logical_name") + @classmethod + def _safe_relative_name(cls, value: str) -> str: + from pathlib import PurePosixPath + + path = PurePosixPath(value) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("logical_name must be a safe relative path") + return path.as_posix() + + +class RetrievalRecordV1(_StrictModel): + """Provider-neutral, content-addressed literature or web record.""" + + schema_version: Literal["ari.retrieval-record/v1"] = RETRIEVAL_RECORD_V1 + canonical_id: str = Field(min_length=1, max_length=256) + provider: str = Field(min_length=1, max_length=128) + provider_record_id: str | None = Field(default=None, max_length=512) + provider_version: str | None = Field(default=None, max_length=128) + query: str = Field(min_length=1, max_length=4096) + retrieved_at: datetime | None = None + title: str = Field(min_length=1, max_length=2048) + abstract: str = Field(default="", max_length=100_000) + authors: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + year: int | None = Field(default=None, ge=0, le=9999) + citation_count: int | None = Field(default=None, ge=0) + source_url: str | None = Field(default=None, max_length=8192) + payload_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + aliases: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + license: str | None = Field(default=None, max_length=512) + use_restriction: str | None = Field(default=None, max_length=2048) + + @field_validator("canonical_id", "provider") + @classmethod + def _safe_identity(cls, value: str) -> str: + normalized = value.strip().lower() + if not _SAFE_TOKEN.fullmatch(normalized): + raise ValueError("identity must be a lowercase stable token") + return normalized + + @field_validator("retrieved_at") + @classmethod + def _timezone_required(cls, value: datetime | None) -> datetime | None: + if value is not None and value.tzinfo is None: + raise ValueError("retrieved_at must include a timezone") + return value + + @field_validator("authors", "aliases") + @classmethod + def _unique_nonempty(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in values if item.strip()) + if len(normalized) != len(set(normalized)): + raise ValueError("values must be unique") + return normalized + + +class CitationEdgeV1(_StrictModel): + schema_version: Literal["ari.citation-edge/v1"] = CITATION_EDGE_V1 + source_id: str = Field(min_length=1, max_length=256) + target_id: str = Field(min_length=1, max_length=256) + relation: Literal["cites", "is-cited-by", "related"] + provider: str = Field(min_length=1, max_length=128) + + @model_validator(mode="after") + def _not_self_edge(self): + if self.source_id == self.target_id: + raise ValueError("citation self-edges are not allowed") + return self + + +SurveyMode = Literal["live", "record", "replay", "frozen", "inline"] + + +class SurveySnapshotV1(_DigestBoundModel): + """Exact retrieval input consumed by one idea-generation attempt.""" + + _digest_field = "snapshot_digest" + + schema_version: Literal["ari.survey-snapshot/v1"] = SURVEY_SNAPSHOT_V1 + snapshot_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + mode: SurveyMode + provider: str = Field(min_length=1, max_length=128) + provider_version: str | None = Field(default=None, max_length=128) + query: str = Field(min_length=1, max_length=4096) + retrieved_at: datetime | None = None + byte_reproducible: bool + records: tuple[RetrievalRecordV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + citation_edges: tuple[CitationEdgeV1, ...] = Field( + default_factory=tuple, max_length=1_000_000 + ) + artifacts: tuple[ResearchArtifactRefV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + warnings: tuple[str, ...] = Field(default_factory=tuple, max_length=1_000) + + @model_validator(mode="after") + def _snapshot_is_consistent(self): + ids = [record.canonical_id for record in self.records] + if len(ids) != len(set(ids)): + raise ValueError("snapshot canonical IDs must be unique") + id_set = set(ids) + for edge in self.citation_edges: + if edge.source_id not in id_set or edge.target_id not in id_set: + raise ValueError("citation edges must reference snapshot records") + if self.mode == "replay" and not self.byte_reproducible: + raise ValueError("replay snapshots must be byte reproducible") + if self.mode in {"live", "record"} and self.retrieved_at is None: + raise ValueError("live/record snapshots require retrieved_at") + return self + + +MetricDirection = Literal["higher", "lower", "target", "none"] +ComparisonScope = Literal[ + "same-environment", "cross-environment", "within-subject", "not-applicable" +] +NormalizationCeiling = Literal["measured", "not-applicable"] + + +class MetricToleranceV1(_StrictModel): + absolute: float = Field(ge=0) + relative: float = Field(ge=0) + + @model_validator(mode="after") + def _finite_tolerance(self): + import math + + if not math.isfinite(self.absolute) or not math.isfinite(self.relative): + raise ValueError("metric tolerance must be finite") + return self + + +class MetricFormulaProvenanceV1(_StrictModel): + source: Literal[ + "idea-generation-lock", "human-admission", "legacy-migration" + ] + source_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + model: str | None = Field(default=None, max_length=512) + prompt_digests: tuple[str, ...] = Field(default_factory=tuple, max_length=128) + + @field_validator("prompt_digests") + @classmethod + def _prompt_digests(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(values) != len(set(values)) or any( + not re.fullmatch(SHA256_DIGEST_PATTERN, value) for value in values + ): + raise ValueError("formula prompt digests must be unique SHA-256 values") + return values + + +class MetricCorrectnessV1(_StrictModel): + expr: str = Field(min_length=1, max_length=4096) + requires: tuple[str, ...] = Field(min_length=1, max_length=128) + + @field_validator("requires") + @classmethod + def _unique_requires(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in values if item.strip()) + if not normalized or len(normalized) != len(set(normalized)): + raise ValueError("correctness operands must be non-empty and unique") + return normalized + + +class MetricContractV1(_DigestBoundModel): + """Idea-owned metric vocabulary; evaluator enforcement is read-only.""" + + _digest_field = "contract_digest" + + schema_version: Literal["ari.metric-contract/v1"] = METRIC_CONTRACT_V1 + contract_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + name: str = Field(min_length=1, max_length=256) + unit: str = Field(min_length=1, max_length=128) + direction: MetricDirection + comparison_scope: ComparisonScope + rationale: str = Field(min_length=1, max_length=4096) + required_evidence: tuple[str, ...] = Field(min_length=1, max_length=128) + correctness_required: bool + normalization_ceiling: NormalizationCeiling + target_value: float | None = None + formula: str = Field(min_length=1, max_length=128) + operands: dict[str, str] = Field(min_length=1, max_length=16) + tolerance: MetricToleranceV1 + formula_provenance: MetricFormulaProvenanceV1 + required_measured: tuple[str, ...] = Field(default_factory=tuple, max_length=128) + invariants: tuple[str, ...] = Field(default_factory=tuple, max_length=128) + correctness: MetricCorrectnessV1 | None = None + confidence: float = Field(ge=0, le=1) + admission_status: Literal["admitted", "human-review-required"] + + @field_validator("unit") + @classmethod + def _known_unit(cls, value: str) -> str: + normalized = value.strip() + if normalized.lower() in {"", "?", "unknown", "unspecified", "tbd"}: + raise ValueError("metric unit must be explicit") + return normalized + + @field_validator("required_evidence") + @classmethod + def _evidence_vocabulary(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in values if item.strip()) + if not normalized: + raise ValueError("required_evidence cannot be empty") + if len(normalized) != len(set(normalized)): + raise ValueError("required_evidence must be unique") + return normalized + + @field_validator("operands") + @classmethod + def _operand_names(cls, values: dict[str, str]) -> dict[str, str]: + normalized = { + str(role).strip(): str(metric).strip() + for role, metric in values.items() + if str(role).strip() and str(metric).strip() + } + if normalized != values: + raise ValueError("formula operands must use non-empty canonical strings") + return normalized + + @field_validator("required_measured", "invariants") + @classmethod + def _unique_optional_lists(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in values if item.strip()) + if len(normalized) != len(set(normalized)): + raise ValueError("metric contract lists must contain unique values") + return normalized + + @model_validator(mode="after") + def _contract_is_consistent(self): + import ast + + from ari.pipeline.claim_gate.formula_eval import safe_eval + + if self.direction == "target" and self.target_value is None: + raise ValueError("target direction requires target_value") + if self.direction != "target" and self.target_value is not None: + raise ValueError("target_value requires target direction") + try: + formula_tree = ast.parse(self.formula, mode="eval") + except SyntaxError as exc: + raise ValueError("metric formula is not a valid expression") from exc + function_names = { + node.func.id + for node in ast.walk(formula_tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + formula_names = { + node.id for node in ast.walk(formula_tree) if isinstance(node, ast.Name) + } - function_names + if formula_names != set(self.operands): + raise ValueError("metric formula variables differ from declared operands") + if safe_eval(self.formula, {role: 1.0 for role in self.operands}) is None: + raise ValueError("metric formula is outside the safe evaluator grammar") + evidence = set(self.required_evidence) + if set(self.operands.values()) - evidence: + raise ValueError("formula operands must be named in required_evidence") + if set(self.required_measured) - evidence: + raise ValueError("required_measured must be named in required_evidence") + if self.correctness is not None and set(self.correctness.requires) - evidence: + raise ValueError("correctness operands must be named in required_evidence") + if self.correctness_required and self.correctness is None: + raise ValueError("correctness_required needs a declared correctness check") + if ( + self.confidence < 0.8 + and self.admission_status == "admitted" + and self.formula_provenance.source != "human-admission" + ): + raise ValueError("low-confidence metric contracts require human review") + return self + + +class IdeaCandidateV1(_DigestBoundModel): + _digest_field = "candidate_id" + + schema_version: Literal["ari.idea-candidate/v1"] = IDEA_CANDIDATE_V1 + candidate_id: str = Field(pattern=SHA256_DIGEST_PATTERN) + title: str = Field(min_length=1, max_length=512) + hypothesis: str = Field(min_length=1, max_length=10_000) + description: str = Field(min_length=1, max_length=20_000) + experiment_plan: str = Field(min_length=1, max_length=100_000) + falsification_conditions: tuple[str, ...] = Field(min_length=1, max_length=64) + metric_contract: MetricContractV1 + citations: tuple[str, ...] = Field(min_length=1, max_length=1_000) + artifact_references: tuple[str, ...] = Field(default_factory=tuple) + limitations: tuple[str, ...] = Field(min_length=1, max_length=64) + source_snapshot_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + generation_lock_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + generator_adapter: str = Field(min_length=1, max_length=128) + novelty_score: float | None = Field(default=None, ge=0, le=1) + feasibility_score: float | None = Field(default=None, ge=0, le=1) + overall_score: float | None = Field(default=None, ge=0, le=1) + + @field_validator( + "falsification_conditions", "citations", "artifact_references", "limitations" + ) + @classmethod + def _unique_text(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(item.strip() for item in values if item.strip()) + if len(normalized) != len(set(normalized)): + raise ValueError("candidate lists must contain unique values") + return normalized + + +class IdeaRejectionV1(_StrictModel): + schema_version: Literal["ari.idea-rejection/v1"] = IDEA_REJECTION_V1 + raw_candidate_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + title: str = Field(default="", max_length=512) + reasons: tuple[str, ...] = Field(min_length=1, max_length=64) + generator_adapter: str = Field(min_length=1, max_length=128) + + +class IdeaGenerationLockV1(_DigestBoundModel): + """Deterministic generation inputs, excluding timestamps and model output.""" + + _digest_field = "generation_lock_digest" + + schema_version: Literal["ari.idea-generation-lock/v1"] = ( + IDEA_GENERATION_LOCK_V1 + ) + generation_lock_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + adapter: str = Field(min_length=1, max_length=128) + adapter_version: str = Field(min_length=1, max_length=128) + model: str = Field(min_length=1, max_length=512) + api_base_identity: str | None = Field(default=None, max_length=2048) + prompt_digests: tuple[str, ...] = Field(min_length=1, max_length=128) + temperatures: tuple[float, ...] = Field(min_length=1, max_length=128) + seed: int | None = None + source_snapshot_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + topic_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + experiment_context_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + vendor_commit: str | None = Field(default=None, max_length=64) + vendor_license: str | None = Field(default=None, max_length=128) + model_revision: str | None = Field(default=None, max_length=256) + generation_parameters: dict[str, Any] = Field(default_factory=dict) + + @field_validator("prompt_digests") + @classmethod + def _prompt_hashes(cls, values: tuple[str, ...]) -> tuple[str, ...]: + for value in values: + if not re.fullmatch(SHA256_DIGEST_PATTERN, value): + raise ValueError("prompt digests must use canonical SHA-256") + return values + + +class IdeaGenerationProvenanceV1(_StrictModel): + schema_version: Literal["ari.idea-generation-provenance/v1"] = ( + IDEA_GENERATION_PROVENANCE_V1 + ) + lock: IdeaGenerationLockV1 + generated_at: datetime + output_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + requested_adapter: str = Field(min_length=1, max_length=128) + actual_adapter: str = Field(min_length=1, max_length=128) + fallback_reason: str | None = Field(default=None, max_length=4096) + + @field_validator("generated_at") + @classmethod + def _generated_timezone(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("generated_at must include a timezone") + return value + + +class IdeaSetV1(_DigestBoundModel): + _digest_field = "idea_set_digest" + + schema_version: Literal["ari.idea-set/v1"] = IDEA_SET_V1 + idea_set_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + topic: str = Field(min_length=1, max_length=20_000) + source_snapshot_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + generation: IdeaGenerationProvenanceV1 + candidates: tuple[IdeaCandidateV1, ...] = Field(default_factory=tuple) + rejections: tuple[IdeaRejectionV1, ...] = Field(default_factory=tuple) + selected_candidate_id: str | None = Field( + default=None, pattern=SHA256_DIGEST_PATTERN + ) + + @model_validator(mode="after") + def _set_is_consistent(self): + if self.generation.lock.source_snapshot_digest != self.source_snapshot_digest: + raise ValueError("generation lock and idea set use different snapshots") + candidate_ids = [candidate.candidate_id for candidate in self.candidates] + if len(candidate_ids) != len(set(candidate_ids)): + raise ValueError("candidate IDs must be unique") + if self.selected_candidate_id is not None and ( + self.selected_candidate_id not in candidate_ids + ): + raise ValueError("selected candidate is not in the candidate set") + return self + + +class ResearchContractV1(_DigestBoundModel): + """Mint-once scientific contract selected from an admitted idea candidate.""" + + _digest_field = "contract_digest" + + schema_version: Literal["ari.research-contract/v1"] = RESEARCH_CONTRACT_V1 + contract_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + idea_set_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + selected_candidate_id: str = Field(pattern=SHA256_DIGEST_PATTERN) + source_snapshot_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + title: str = Field(min_length=1, max_length=512) + hypothesis: str = Field(min_length=1, max_length=10_000) + experiment_plan: str = Field(min_length=1, max_length=100_000) + falsification_conditions: tuple[str, ...] = Field(min_length=1, max_length=64) + metric_contract: MetricContractV1 + citations: tuple[str, ...] = Field(min_length=1, max_length=1_000) + artifact_references: tuple[str, ...] = Field(default_factory=tuple) + limitations: tuple[str, ...] = Field(min_length=1, max_length=64) + generation_lock_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + + +def mint_research_contract( + idea_set: IdeaSetV1, + candidate_id: str | None = None, +) -> ResearchContractV1: + """Select one admitted candidate and mint its immutable run contract.""" + + selected = candidate_id or idea_set.selected_candidate_id + if selected is None: + raise ResearchContractError("idea set has no selected candidate") + candidate = next( + (item for item in idea_set.candidates if item.candidate_id == selected), None + ) + if candidate is None: + raise ResearchContractError("selected candidate is not admitted") + if candidate.metric_contract.admission_status != "admitted": + raise ResearchContractError("selected metric contract requires human review") + return ResearchContractV1.create( + idea_set_digest=idea_set.idea_set_digest, + selected_candidate_id=candidate.candidate_id, + source_snapshot_digest=candidate.source_snapshot_digest, + title=candidate.title, + hypothesis=candidate.hypothesis, + experiment_plan=candidate.experiment_plan, + falsification_conditions=candidate.falsification_conditions, + metric_contract=candidate.metric_contract, + citations=candidate.citations, + artifact_references=candidate.artifact_references, + limitations=candidate.limitations, + generation_lock_digest=candidate.generation_lock_digest, + ) + + +def validate_research_handoff( + *, + snapshot: SurveySnapshotV1, + idea_set: IdeaSetV1, + contract: ResearchContractV1 | None, +) -> None: + """Verify cross-record digests, citations, artifacts, and selection identity.""" + + if idea_set.source_snapshot_digest != snapshot.snapshot_digest: + raise ResearchContractError("idea set does not reference the supplied snapshot") + citation_ids = {record.canonical_id for record in snapshot.records} + artifact_digests = {artifact.digest for artifact in snapshot.artifacts} + for candidate in idea_set.candidates: + missing_citations = set(candidate.citations) - citation_ids + if missing_citations: + raise ResearchContractError( + f"candidate references unknown citations: {sorted(missing_citations)}" + ) + missing_artifacts = set(candidate.artifact_references) - artifact_digests + if missing_artifacts: + raise ResearchContractError( + f"candidate references unknown artifacts: {sorted(missing_artifacts)}" + ) + if contract is None: + if idea_set.selected_candidate_id is not None: + raise ResearchContractError("selected idea is missing a research contract") + return + expected = mint_research_contract(idea_set, contract.selected_candidate_id) + if contract != expected: + raise ResearchContractError("research contract differs from selected candidate") + + +def parse_survey_snapshot(document: dict[str, Any]) -> SurveySnapshotV1: + try: + return SurveySnapshotV1.model_validate(document) + except Exception as exc: + raise ResearchContractError(f"invalid survey snapshot: {exc}") from exc + + +def load_survey_snapshot_ref( + checkpoint_dir: str, + logical_name: str, + *, + max_bytes: int = 128 * 1024 * 1024, +) -> SurveySnapshotV1: + """Load a snapshot through a closed workspace and verify every artifact. + + This is the common hand-off used by retrieval, idea, and paper Skills. A + digest-valid snapshot whose referenced cassette or raw payload was changed + is still rejected before scientific consumption. + """ + + from pathlib import Path + + from ari.execution import WorkspaceRefV1 + + if not checkpoint_dir: + raise ResearchContractError("snapshot loading requires a checkpoint root") + if not logical_name: + raise ResearchContractError("snapshot loading requires a logical name") + workspace = WorkspaceRefV1(root=str(Path(checkpoint_dir).expanduser().resolve())) + payload = workspace.read_bytes(logical_name, max_bytes=max_bytes) + try: + document = json.loads(payload) + except json.JSONDecodeError as exc: + raise ResearchContractError("survey snapshot is not valid JSON") from exc + snapshot = parse_survey_snapshot(document) + for artifact in snapshot.artifacts: + artifact_payload = workspace.read_bytes( + artifact.logical_name, max_bytes=max_bytes + ) + digest = "sha256:" + hashlib.sha256(artifact_payload).hexdigest() + if digest != artifact.digest: + raise ResearchContractError( + f"survey artifact digest mismatch: {artifact.logical_name}" + ) + return snapshot + + +def parse_idea_set(document: dict[str, Any]) -> IdeaSetV1: + try: + return IdeaSetV1.model_validate(document) + except Exception as exc: + raise ResearchContractError(f"invalid idea set: {exc}") from exc + + +def parse_research_contract(document: dict[str, Any]) -> ResearchContractV1: + try: + return ResearchContractV1.model_validate(document) + except Exception as exc: + raise ResearchContractError(f"invalid research contract: {exc}") from exc + + +def parse_research_contract_document(document: dict[str, Any]) -> ResearchContractV1 | None: + """Read the typed contract embedded in ``idea.json``; never infer one. + + Legacy documents return ``None``. A document declaring the typed format but + containing no valid contract fails closed so callers cannot fall back to an + LLM-derived vocabulary for a rejected new-format idea set. + """ + + raw = document.get("research_contract") + declares_typed = document.get("typed_schema_version") == RESEARCH_CONTRACT_V1 + if raw is None: + if declares_typed: + raise ResearchContractError( + "typed idea document has no admitted research contract" + ) + return None + if not isinstance(raw, dict): + raise ResearchContractError("research_contract must be an object") + contract = parse_research_contract(raw) + advertised = document.get("research_contract_digest") + if advertised is not None and advertised != contract.contract_digest: + raise ResearchContractError("advertised research contract digest differs") + return contract + + +def metric_gate_projection(contract: ResearchContractV1) -> dict[str, Any]: + """Project one research contract through the canonical evaluator model.""" + + from ari.claim_gate_contract import MetricClaimV1, MetricGateContractV1 + + projection = MetricGateContractV1.create( + source="research-contract", + source_idea_digest=contract.selected_candidate_id, + research_contract_digest=contract.contract_digest, + metric_contract=contract.metric_contract, + claims=tuple( + MetricClaimV1( + claim=condition, + required_evidence=contract.metric_contract.required_evidence, + ) + for condition in contract.falsification_conditions + ), + ) + return projection.model_dump(mode="json") + + +__all__ = [ + "CITATION_EDGE_V1", + "IDEA_CANDIDATE_V1", + "IDEA_GENERATION_LOCK_V1", + "IDEA_GENERATION_PROVENANCE_V1", + "IDEA_REJECTION_V1", + "IDEA_SET_V1", + "METRIC_CONTRACT_V1", + "RESEARCH_ARTIFACT_REF_V1", + "RESEARCH_CONTRACT_V1", + "RETRIEVAL_RECORD_V1", + "SURVEY_SNAPSHOT_V1", + "CitationEdgeV1", + "IdeaCandidateV1", + "IdeaGenerationLockV1", + "IdeaGenerationProvenanceV1", + "IdeaRejectionV1", + "IdeaSetV1", + "MetricContractV1", + "MetricCorrectnessV1", + "MetricFormulaProvenanceV1", + "MetricToleranceV1", + "ResearchArtifactRefV1", + "ResearchContractError", + "ResearchContractV1", + "RetrievalRecordV1", + "SurveySnapshotV1", + "canonical_digest", + "load_survey_snapshot_ref", + "metric_gate_projection", + "mint_research_contract", + "parse_idea_set", + "parse_research_contract", + "parse_research_contract_document", + "parse_survey_snapshot", + "validate_research_handoff", +] diff --git a/ari-core/ari/result.py b/ari-core/ari/result.py new file mode 100644 index 00000000..edb16047 --- /dev/null +++ b/ari-core/ari/result.py @@ -0,0 +1,501 @@ +"""Versioned result contract and MCP compatibility normalization. + +The MCP transport historically exposes ``{"result": ""}`` and +``{"error": "..."}`` dictionaries to ARI callers. ``ResultEnvelopeV1`` is the +provider-neutral contract used internally and by future catalog adapters. The +normalizer keeps a lossless legacy conversion while bounding the serialized +envelope: large raw responses are stored content-addressably through the existing +``ArtifactStore`` seam. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ari.async_tools import ( + ASYNC_TOOL_HANDLE_V1, + AsyncStateMapV1, + AsyncToolEndpointV1, + AsyncToolHandleV1, +) +from ari.call_context import ( # noqa: F401 + NodeContextV1, + RunContextV1, + ToolCallContextV1, +) +from ari.protocols.stores import ArtifactStore + + +RESULT_ENVELOPE_V1 = "ari.result-envelope/v1" +ARTIFACT_REF_V1 = "ari.artifact-ref/v1" +DEFAULT_INLINE_RESULT_LIMIT = 4_000 +RAW_RESULT_ROLE = "mcp-raw-result" +SHA256_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" + +ResultErrorKind = Literal[ + "tool", + "transport", + "protocol", + "timeout", + "cancelled", + "admission", + "artifact-integrity", + "unknown", +] + + +class ResultArtifactIntegrityError(ValueError): + """Raised when a content-addressed artifact does not match its digest.""" + + +class ResultArtifactV1(BaseModel): + """Content-addressed artifact descriptor stored relative to a run store.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.artifact-ref/v1"] = ARTIFACT_REF_V1 + digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + media_type: str = Field(min_length=1) + size: int = Field(ge=0) + logical_role: str = Field(min_length=1) + logical_name: str = Field(min_length=1) + + @field_validator("digest") + @classmethod + def _valid_digest(cls, value: str) -> str: + prefix = "sha256:" + digest = value.removeprefix(prefix) + if not value.startswith(prefix) or len(digest) != 64: + raise ValueError("digest must use sha256:<64 lowercase hex> format") + try: + int(digest, 16) + except ValueError as exc: + raise ValueError("digest contains non-hexadecimal characters") from exc + if digest != digest.lower(): + raise ValueError("digest must use lowercase hexadecimal characters") + return value + + @field_validator("logical_name") + @classmethod + def _safe_logical_name(cls, value: str) -> str: + path = Path(value) + if not value or path.is_absolute() or ".." in path.parts: + raise ValueError("logical_name must be a safe relative artifact path") + return value + + +class ResultErrorV1(BaseModel): + """Typed failure information independent of provider-specific wording.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: ResultErrorKind + message: str + retryable: bool = False + details: dict[str, Any] = Field(default_factory=dict) + + +class ResultProvenanceV1(BaseModel): + """Identity and timing recorded for one normalized tool result.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + tool_ref: str = Field(min_length=1) + run_id: str = "" + node_id: str | None = None + phase: str | None = None + selection_reason: str = "" + credential_scope_ids: list[str] = Field(default_factory=list) + started_at: str = Field(min_length=1) + completed_at: str | None = None + duration_ms: int | None = Field(default=None, ge=0) + response_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + + @field_validator("response_digest") + @classmethod + def _valid_response_digest(cls, value: str) -> str: + ResultArtifactV1._valid_digest(value) + return value + + +class ResultEnvelopeV1(BaseModel): + """Bounded, provider-neutral result returned by ARI's typed dispatch API.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.result-envelope/v1"] = RESULT_ENVELOPE_V1 + status: Literal["ok", "error", "submitted", "running", "cancelled"] + content: str = "" + content_truncated: bool = False + structured_content: dict[str, Any] = Field(default_factory=dict) + async_handle: AsyncToolHandleV1 | None = None + artifacts: list[ResultArtifactV1] = Field(default_factory=list) + error: ResultErrorV1 | None = None + provenance: ResultProvenanceV1 + + @model_validator(mode="after") + def _consistent_status(self) -> "ResultEnvelopeV1": + if self.status == "error" and self.error is None: + raise ValueError("status=error requires error details") + if self.status != "error" and self.error is not None: + raise ValueError("error details require status=error") + if self.content_truncated and not any( + artifact.logical_role == RAW_RESULT_ROLE for artifact in self.artifacts + ): + raise ValueError("truncated content requires a raw-result artifact") + return self + + def raw_result_artifact(self) -> ResultArtifactV1 | None: + """Return the raw-result artifact descriptor, when externalized.""" + + return next( + ( + artifact + for artifact in self.artifacts + if artifact.logical_role == RAW_RESULT_ROLE + ), + None, + ) + + def materialize_content(self, store: ArtifactStore | None = None) -> str: + """Recover the complete raw content and verify its content digest.""" + + artifact = self.raw_result_artifact() + if artifact is None: + return self.content + if store is None: + raise ResultArtifactIntegrityError( + "artifact store is required to materialize externalized content" + ) + path = store.get(artifact.logical_name) + try: + payload = path.read_bytes() + except OSError as exc: + raise ResultArtifactIntegrityError( + f"cannot read result artifact {artifact.logical_name}: {exc}" + ) from exc + actual = _sha256(payload) + if actual != artifact.digest: + raise ResultArtifactIntegrityError( + f"result artifact digest mismatch: expected {artifact.digest}, got {actual}" + ) + if len(payload) != artifact.size: + raise ResultArtifactIntegrityError( + f"result artifact size mismatch: expected {artifact.size}, got {len(payload)}" + ) + try: + return payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise ResultArtifactIntegrityError( + f"result artifact is not valid UTF-8: {artifact.logical_name}" + ) from exc + + def to_legacy(self, store: ArtifactStore | None = None) -> dict[str, str]: + """Losslessly project back to the historical MCPClient dictionary shape.""" + + if self.error is not None and self.error.kind != "tool": + return {"error": self.error.message} + try: + content = self.materialize_content(store) + except ResultArtifactIntegrityError as exc: + return {"error": str(exc)} + if not content and self.structured_content: + content = json.dumps(self.structured_content, ensure_ascii=False) + return {"result": content} + + +class ResultEnvelopeNormalizer: + """Normalize legacy/MCP responses and externalize large raw content.""" + + def __init__( + self, + artifact_store: ArtifactStore | None = None, + *, + inline_limit: int = DEFAULT_INLINE_RESULT_LIMIT, + ) -> None: + if inline_limit <= 0: + raise ValueError("inline_limit must be positive") + self.artifact_store = artifact_store + self.inline_limit = inline_limit + + def normalize_legacy( + self, + response: dict[str, Any], + *, + tool_ref: str, + context: ToolCallContextV1 | None = None, + started_at: str | None = None, + completed_at: str | None = None, + ) -> ResultEnvelopeV1: + """Normalize one historical ``result``/``error`` response dictionary.""" + + context = context or ToolCallContextV1() + started_at = started_at or utc_now_iso() + completed_at = completed_at or utc_now_iso() + + if not isinstance(response, dict) or not ( + {"result", "error"} & response.keys() + ): + return self.error( + tool_ref=tool_ref, + kind="protocol", + message="MCP response must contain a result or error field", + retryable=True, + context=context, + started_at=started_at, + completed_at=completed_at, + ) + + if "error" in response and "result" not in response: + message = str(response.get("error") or "unknown MCP transport error") + raw_kind = str(response.get("_error_kind") or "transport") + if raw_kind not in { + "transport", + "protocol", + "timeout", + "cancelled", + "admission", + "artifact-integrity", + "unknown", + }: + raw_kind = "unknown" + return self.error( + tool_ref=tool_ref, + kind=cast(ResultErrorKind, raw_kind), + message=message, + retryable=bool(response.get("_retryable", False)), + context=context, + started_at=started_at, + completed_at=completed_at, + ) + + raw = response.get("result", "") + if not isinstance(raw, str): + raw = json.dumps(raw, ensure_ascii=False, default=str) + structured = response.get("_structured_content") + parsed = _parse_json(raw) + if structured is None: + structured = parsed + structured_dict = _structured_dict(structured) + + is_tool_error = bool(response.get("_mcp_is_error", False)) or ( + isinstance(parsed, dict) + and ( + "error" in parsed + or str(parsed.get("status", "")).strip().casefold() == "error" + ) + ) + status = _result_status(structured_dict, is_tool_error) + error = None + if is_tool_error: + error = ResultErrorV1( + kind="tool", + message=_tool_error_message(parsed, raw), + retryable=False, + ) + + payload = raw.encode("utf-8") + response_digest = _sha256(payload) + content = raw + content_truncated = False + artifacts: list[ResultArtifactV1] = [] + if len(raw) > self.inline_limit and self.artifact_store is not None: + artifact = self._put_raw_result(payload, parsed is not None) + artifacts.append(artifact) + content = _bounded_preview(raw, self.inline_limit) + content_truncated = True + # The complete structured value may be as large as the raw response; + # the content-addressed artifact remains the lossless authority. + structured_dict = {} + + provenance = _provenance( + tool_ref=tool_ref, + context=context, + started_at=started_at, + completed_at=completed_at, + response_digest=response_digest, + ) + return ResultEnvelopeV1( + status=status, + content=content, + content_truncated=content_truncated, + structured_content=structured_dict, + artifacts=artifacts, + error=error, + provenance=provenance, + ) + + def error( + self, + *, + tool_ref: str, + kind: ResultErrorKind, + message: str, + retryable: bool, + context: ToolCallContextV1 | None = None, + started_at: str | None = None, + completed_at: str | None = None, + details: dict[str, Any] | None = None, + ) -> ResultEnvelopeV1: + """Build a typed dispatch/transport failure envelope.""" + + context = context or ToolCallContextV1() + started_at = started_at or utc_now_iso() + completed_at = completed_at or utc_now_iso() + digest = _sha256(message.encode("utf-8")) + return ResultEnvelopeV1( + status="error", + content="", + error=ResultErrorV1( + kind=kind, + message=message, + retryable=retryable, + details=details or {}, + ), + provenance=_provenance( + tool_ref=tool_ref, + context=context, + started_at=started_at, + completed_at=completed_at, + response_digest=digest, + ), + ) + + def _put_raw_result(self, payload: bytes, is_json: bool) -> ResultArtifactV1: + assert self.artifact_store is not None + digest = _sha256(payload) + hex_digest = digest.removeprefix("sha256:") + suffix = "json" if is_json else "txt" + logical_name = ( + f"artifacts/mcp-results/sha256/{hex_digest[:2]}/{hex_digest}.{suffix}" + ) + if self.artifact_store.exists(logical_name): + existing = self.artifact_store.get(logical_name).read_bytes() + if existing != payload: + raise ResultArtifactIntegrityError( + f"content-address collision at {logical_name}" + ) + else: + self.artifact_store.put(logical_name, payload) + return ResultArtifactV1( + digest=digest, + media_type="application/json" if is_json else "text/plain", + size=len(payload), + logical_role=RAW_RESULT_ROLE, + logical_name=logical_name, + ) + + +def utc_now_iso() -> str: + """Return an RFC 3339 UTC timestamp with an explicit ``Z`` suffix.""" + + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse_json(raw: str) -> Any: + if not raw: + return None + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + + +def _structured_dict(value: Any) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, dict): + return value + return {"value": value} + + +def _result_status( + structured: dict[str, Any], is_tool_error: bool +) -> Literal["ok", "error", "submitted", "running", "cancelled"]: + if is_tool_error: + return "error" + value = str(structured.get("status", "")).lower() + if value in {"submitted", "running", "cancelled"}: + return value # type: ignore[return-value] + return "ok" + + +def _tool_error_message(parsed: Any, raw: str) -> str: + if isinstance(parsed, dict) and "error" in parsed: + error = parsed["error"] + return ( + error if isinstance(error, str) else json.dumps(error, ensure_ascii=False) + ) + if isinstance(parsed, dict) and parsed.get("message"): + return str(parsed["message"]) + return raw or "MCP tool reported an error" + + +def _bounded_preview(raw: str, limit: int) -> str: + suffix = "… [content externalized]" + if len(suffix) >= limit: + return suffix[:limit] + suffix = "\n" + suffix + head_length = max(0, limit - len(suffix)) + return raw[:head_length] + suffix + + +def _sha256(payload: bytes) -> str: + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def _provenance( + *, + tool_ref: str, + context: ToolCallContextV1, + started_at: str, + completed_at: str, + response_digest: str, +) -> ResultProvenanceV1: + duration_ms = None + try: + started = datetime.fromisoformat(started_at.replace("Z", "+00:00")) + completed = datetime.fromisoformat(completed_at.replace("Z", "+00:00")) + duration_ms = max(0, int((completed - started).total_seconds() * 1_000)) + except (TypeError, ValueError): + pass + return ResultProvenanceV1( + tool_ref=tool_ref, + run_id=context.run_id, + node_id=context.node_id, + phase=context.phase, + selection_reason=context.selection_reason, + credential_scope_ids=context.credential_scope_ids, + started_at=started_at, + completed_at=completed_at, + duration_ms=duration_ms, + response_digest=response_digest, + ) + + +__all__ = [ + "ASYNC_TOOL_HANDLE_V1", + "ARTIFACT_REF_V1", + "DEFAULT_INLINE_RESULT_LIMIT", + "RAW_RESULT_ROLE", + "RESULT_ENVELOPE_V1", + "SHA256_DIGEST_PATTERN", + "ResultArtifactIntegrityError", + "ResultArtifactV1", + "ResultEnvelopeNormalizer", + "ResultEnvelopeV1", + "ResultErrorKind", + "ResultErrorV1", + "ResultProvenanceV1", + "AsyncStateMapV1", + "AsyncToolEndpointV1", + "AsyncToolHandleV1", + "ToolCallContextV1", + "utc_now_iso", +] diff --git a/ari-core/ari/schemas/README.md b/ari-core/ari/schemas/README.md index 2616f23a..f5ee72e9 100644 --- a/ari-core/ari/schemas/README.md +++ b/ari-core/ari/schemas/README.md @@ -7,13 +7,45 @@ JSON Schemas shipped with ari-core, loaded by basename via - `README.md` — this file. - `__init__.py` — `load(name)` loader. +- `analysis_request_v1.schema.json` — TODO +- `analysis_result_v1.schema.json` — TODO +- `async_tool_handle_v1.schema.json` — immutable submit/status/result/cancel handle contract. +- `call_context_v1.schema.json` — explicit run, node, ordered-lineage, and call provenance context. +- `execution_request_v1.schema.json` — exact command, workspace, input digest, environment, resource, network, and container request. +- `execution_result_v1.schema.json` — attempt identity, enforcement report, bounded previews, and complete-log artifacts. +- `figure_batch_v1.schema.json` — declarative specs, render environment, +- `gate_report_v1.schema.json` — deterministic policy/evidence/formula-bound hard-gate report. +- `idea_candidate_v1.schema.json` — admitted falsifiable hypothesis candidate. +- `idea_set_v1.schema.json` — generation lock, admitted candidates, and explicit rejections. +- `measurement_set_v1.schema.json` — typed parameter/measurement/unit/execution/artifact separation. +- `memory_backup_v1.schema.json` — TODO +- `memory_record_v1.schema.json` — TODO +- `memory_retrieval_v1.schema.json` — TODO +- `metric_admission_decision_v1.schema.json` — explicit human admission/rejection record. +- `metric_contract_proposal_v1.schema.json` — provenance-bound, untrusted LLM metric proposal. +- `metric_contract_v1.schema.json` — immutable metric, unit, direction, comparison, and evidence vocabulary. +- `metric_gate_contract_v1.schema.json` — evaluator projection of one admitted metric contract. - `node_report.schema.json` — per-node report schema. +- `paper_build_v1.schema.json` — TODO +- `paper_model_call_batch_v1.schema.json` — TODO - `publish.schema.json` — publish record / manifest schema. +- `research_contract_v1.schema.json` — selected mint-once scientific hand-off consumed by evaluators. +- `result_envelope_v1.schema.json` — typed MCP result plus value-free credential-scope provenance. +- `retrieval_record_v1.schema.json` — provider-neutral literature/web record identity and payload digest. +- `run_comparison_request_v1.schema.json` — TODO +- `science_data_v1.schema.json` — separately digest-bound raw measurement, +- `semantic_review_v1.schema.json` — independent provenance-bound semantic advisory. +- `skill_manifest_v1.schema.json` — canonical Skill package, environment, and credential-scope contract. +- `skills_lock_v1.schema.json` — immutable provider/schema/phase/credential-authority snapshot. +- `statistical_test_request_v1.schema.json` — TODO +- `survey_snapshot_v1.schema.json` — digest-bound record/replay retrieval input and citation graph. +- `visual_review_batch_v1.schema.json` — criteria profiles, artifact identity, - `viz_checkpoint.schema.json` — TODO - `viz_checkpoint_summary.schema.json` — TODO - `viz_settings.schema.json` — TODO - `viz_state.schema.json` — TODO - `viz_tree_node.schema.json` — TODO +- `workspace_ref_v1.schema.json` — canonical closed workspace root. ## See also diff --git a/ari-core/ari/schemas/analysis_request_v1.schema.json b/ari-core/ari/schemas/analysis_request_v1.schema.json new file mode 100644 index 00000000..d8fadece --- /dev/null +++ b/ari-core/ari/schemas/analysis_request_v1.schema.json @@ -0,0 +1,327 @@ +{ + "$defs": { + "AnalysisArtifactTargetV1": { + "additionalProperties": false, + "description": "Optional closed-workspace destination for deterministic result artifacts.", + "properties": { + "relative_directory": { + "default": "analysis", + "title": "Relative Directory", + "type": "string" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace" + ], + "title": "AnalysisArtifactTargetV1", + "type": "object" + }, + "AnalysisDataSourceV1": { + "additionalProperties": false, + "description": "Digest-bound numeric column in a closed workspace.", + "properties": { + "backend_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Backend Id Column" + }, + "environment_digest_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Digest Column" + }, + "expected_digest": { + "title": "Expected Digest", + "type": "string" + }, + "format": { + "default": "auto", + "enum": [ + "auto", + "csv", + "json", + "npy" + ], + "title": "Format", + "type": "string" + }, + "max_bytes": { + "default": 67108864, + "maximum": 268435456, + "minimum": 1, + "title": "Max Bytes", + "type": "integer" + }, + "pair_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pair Id Column" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "replicate_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Replicate Id Column" + }, + "value_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Value Column" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace", + "relative_path", + "expected_digest" + ], + "title": "AnalysisDataSourceV1", + "type": "object" + }, + "AnalysisObservationV1": { + "additionalProperties": false, + "description": "One observed value and the identities needed to assess independence.", + "properties": { + "backend_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Backend Id" + }, + "environment_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Digest" + }, + "pair_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pair Id" + }, + "replicate_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Replicate Id" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "AnalysisObservationV1", + "type": "object" + }, + "MetricSampleSetV1": { + "additionalProperties": false, + "description": "A unit-bearing metric sample supplied inline or by immutable source.", + "properties": { + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "observations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/AnalysisObservationV1" + }, + "maxItems": 1000000, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Observations" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/AnalysisDataSourceV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "unit": { + "title": "Unit", + "type": "string" + } + }, + "required": [ + "metric_id", + "unit" + ], + "title": "MetricSampleSetV1", + "type": "object" + }, + "WorkspaceRefV1": { + "additionalProperties": false, + "description": "One canonical writable root with no caller-controlled escape path.", + "properties": { + "root": { + "title": "Root", + "type": "string" + }, + "schema_version": { + "const": "ari.workspace-ref/v1", + "default": "ari.workspace-ref/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "root" + ], + "title": "WorkspaceRefV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/analysis-request-v1.schema.json", + "additionalProperties": false, + "description": "Deterministic summary request for one or more metrics.", + "properties": { + "analysis_plan_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Analysis Plan Digest" + }, + "artifact_target": { + "anyOf": [ + { + "$ref": "#/$defs/AnalysisArtifactTargetV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "confidence_level": { + "default": 0.95, + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Confidence Level", + "type": "number" + }, + "datasets": { + "items": { + "$ref": "#/$defs/MetricSampleSetV1" + }, + "maxItems": 1024, + "minItems": 1, + "title": "Datasets", + "type": "array" + }, + "missing_policy": { + "default": "error", + "enum": [ + "error", + "drop" + ], + "title": "Missing Policy", + "type": "string" + }, + "schema_version": { + "const": "ari.analysis-request/v1", + "default": "ari.analysis-request/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "datasets" + ], + "title": "ARI Analysis Request v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/analysis_result_v1.schema.json b/ari-core/ari/schemas/analysis_result_v1.schema.json new file mode 100644 index 00000000..715bc8d1 --- /dev/null +++ b/ari-core/ari/schemas/analysis_result_v1.schema.json @@ -0,0 +1,467 @@ +{ + "$defs": { + "AnalysisArtifactV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "logical_role": { + "enum": [ + "analysis-json", + "analysis-table" + ], + "title": "Logical Role", + "type": "string" + }, + "media_type": { + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "relative_path", + "digest", + "size_bytes", + "media_type", + "logical_role" + ], + "title": "AnalysisArtifactV1", + "type": "object" + }, + "AnalysisSummaryV1": { + "additionalProperties": false, + "properties": { + "constant_data": { + "title": "Constant Data", + "type": "boolean" + }, + "count": { + "minimum": 1, + "title": "Count", + "type": "integer" + }, + "independence_status": { + "enum": [ + "verified", + "declared", + "not-established" + ], + "title": "Independence Status", + "type": "string" + }, + "maximum": { + "title": "Maximum", + "type": "number" + }, + "mean": { + "title": "Mean", + "type": "number" + }, + "mean_confidence_interval": { + "anyOf": [ + { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Mean Confidence Interval" + }, + "median": { + "title": "Median", + "type": "number" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "minimum": { + "title": "Minimum", + "type": "number" + }, + "missing_count": { + "minimum": 0, + "title": "Missing Count", + "type": "integer" + }, + "q25": { + "title": "Q25", + "type": "number" + }, + "q75": { + "title": "Q75", + "type": "number" + }, + "source_digest": { + "title": "Source Digest", + "type": "string" + }, + "std": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Std" + }, + "unit": { + "title": "Unit", + "type": "string" + }, + "variance": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Variance" + } + }, + "required": [ + "metric_id", + "unit", + "count", + "missing_count", + "mean", + "std", + "variance", + "minimum", + "q25", + "median", + "q75", + "maximum", + "mean_confidence_interval", + "constant_data", + "independence_status", + "source_digest" + ], + "title": "AnalysisSummaryV1", + "type": "object" + }, + "RunComparisonResultV1": { + "additionalProperties": false, + "properties": { + "baseline_run_id": { + "title": "Baseline Run Id", + "type": "string" + }, + "direction": { + "enum": [ + "higher", + "lower" + ], + "title": "Direction", + "type": "string" + }, + "environment_compatible": { + "title": "Environment Compatible", + "type": "boolean" + }, + "environment_groups": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Environment Groups", + "type": "array" + }, + "independence_status": { + "enum": [ + "declared", + "not-established" + ], + "title": "Independence Status", + "type": "string" + }, + "independent_replicate_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Independent Replicate Count" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "provenance_differences": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Provenance Differences", + "type": "array" + }, + "ranking": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Ranking", + "type": "array" + }, + "unit": { + "title": "Unit", + "type": "string" + } + }, + "required": [ + "metric_id", + "unit", + "direction", + "ranking", + "baseline_run_id", + "environment_compatible", + "environment_groups", + "independence_status", + "independent_replicate_count", + "provenance_differences" + ], + "title": "RunComparisonResultV1", + "type": "object" + }, + "StatisticalComparisonResultV1": { + "additionalProperties": false, + "properties": { + "adjusted_p_value": { + "title": "Adjusted P Value", + "type": "number" + }, + "alpha": { + "title": "Alpha", + "type": "number" + }, + "alternative": { + "title": "Alternative", + "type": "string" + }, + "assumptions": { + "additionalProperties": true, + "title": "Assumptions", + "type": "object" + }, + "comparison_id": { + "title": "Comparison Id", + "type": "string" + }, + "confidence_interval": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "title": "Confidence Interval", + "type": "array" + }, + "confidence_interval_name": { + "title": "Confidence Interval Name", + "type": "string" + }, + "effect_size": { + "title": "Effect Size", + "type": "number" + }, + "effect_size_name": { + "title": "Effect Size Name", + "type": "string" + }, + "input_digest": { + "title": "Input Digest", + "type": "string" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "missing_count_a": { + "minimum": 0, + "title": "Missing Count A", + "type": "integer" + }, + "missing_count_b": { + "minimum": 0, + "title": "Missing Count B", + "type": "integer" + }, + "p_value": { + "title": "P Value", + "type": "number" + }, + "pairing": { + "title": "Pairing", + "type": "string" + }, + "sample_count_a": { + "minimum": 1, + "title": "Sample Count A", + "type": "integer" + }, + "sample_count_b": { + "minimum": 1, + "title": "Sample Count B", + "type": "integer" + }, + "significant": { + "title": "Significant", + "type": "boolean" + }, + "statistic": { + "title": "Statistic", + "type": "number" + }, + "test_family": { + "title": "Test Family", + "type": "string" + }, + "unit": { + "title": "Unit", + "type": "string" + } + }, + "required": [ + "comparison_id", + "metric_id", + "unit", + "test_family", + "alternative", + "pairing", + "sample_count_a", + "sample_count_b", + "missing_count_a", + "missing_count_b", + "statistic", + "p_value", + "adjusted_p_value", + "alpha", + "significant", + "effect_size_name", + "effect_size", + "confidence_interval_name", + "confidence_interval", + "assumptions", + "input_digest" + ], + "title": "StatisticalComparisonResultV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/analysis-result-v1.schema.json", + "additionalProperties": false, + "description": "Machine-readable deterministic output shared by analysis providers.", + "properties": { + "analysis_plan_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Analysis Plan Digest" + }, + "artifacts": { + "items": { + "$ref": "#/$defs/AnalysisArtifactV1" + }, + "title": "Artifacts", + "type": "array" + }, + "comparisons": { + "items": { + "$ref": "#/$defs/StatisticalComparisonResultV1" + }, + "title": "Comparisons", + "type": "array" + }, + "input_digest": { + "title": "Input Digest", + "type": "string" + }, + "kind": { + "enum": [ + "summary", + "statistical-test", + "run-comparison" + ], + "title": "Kind", + "type": "string" + }, + "library_versions": { + "additionalProperties": { + "type": "string" + }, + "title": "Library Versions", + "type": "object" + }, + "run_comparison": { + "anyOf": [ + { + "$ref": "#/$defs/RunComparisonResultV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "ari.analysis-result/v1", + "default": "ari.analysis-result/v1", + "title": "Schema Version", + "type": "string" + }, + "summaries": { + "items": { + "$ref": "#/$defs/AnalysisSummaryV1" + }, + "title": "Summaries", + "type": "array" + } + }, + "required": [ + "kind", + "input_digest", + "library_versions" + ], + "title": "ARI Analysis Result v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/async_tool_handle_v1.schema.json b/ari-core/ari/schemas/async_tool_handle_v1.schema.json new file mode 100644 index 00000000..883d2399 --- /dev/null +++ b/ari-core/ari/schemas/async_tool_handle_v1.schema.json @@ -0,0 +1,151 @@ +{ + "$defs": { + "AsyncStateMapV1": { + "additionalProperties": false, + "description": "Case-insensitive provider state mapping used while polling a handle.", + "properties": { + "cancelled_states": { + "items": { + "type": "string" + }, + "title": "Cancelled States", + "type": "array" + }, + "failed_states": { + "items": { + "type": "string" + }, + "title": "Failed States", + "type": "array" + }, + "running_states": { + "items": { + "type": "string" + }, + "title": "Running States", + "type": "array" + }, + "submitted_states": { + "items": { + "type": "string" + }, + "title": "Submitted States", + "type": "array" + }, + "succeeded_states": { + "items": { + "type": "string" + }, + "title": "Succeeded States", + "type": "array" + } + }, + "title": "AsyncStateMapV1", + "type": "object" + }, + "AsyncToolEndpointV1": { + "additionalProperties": false, + "description": "One immutable endpoint embedded in a runtime async handle.", + "properties": { + "handle_argument": { + "title": "Handle Argument", + "type": "string" + }, + "tool_ref": { + "minLength": 1, + "title": "Tool Ref", + "type": "string" + } + }, + "required": [ + "tool_ref", + "handle_argument" + ], + "title": "AsyncToolEndpointV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/async-tool-handle-v1.schema.json", + "additionalProperties": false, + "description": "Portable handle bound to immutable lifecycle tool references.", + "properties": { + "cancel": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "handle_id": { + "maxLength": 1024, + "minLength": 1, + "title": "Handle Id", + "type": "string" + }, + "max_wait_seconds": { + "maximum": 2678400, + "minimum": 1, + "title": "Max Wait Seconds", + "type": "integer" + }, + "poll_interval_seconds": { + "maximum": 3600, + "minimum": 0.01, + "title": "Poll Interval Seconds", + "type": "number" + }, + "result": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "ari.async-tool-handle/v1", + "default": "ari.async-tool-handle/v1", + "title": "Schema Version", + "type": "string" + }, + "state_field": { + "title": "State Field", + "type": "string" + }, + "states": { + "$ref": "#/$defs/AsyncStateMapV1" + }, + "status": { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + "submission_tool_ref": { + "minLength": 1, + "title": "Submission Tool Ref", + "type": "string" + }, + "submitted_at": { + "minLength": 1, + "title": "Submitted At", + "type": "string" + } + }, + "required": [ + "handle_id", + "submission_tool_ref", + "status", + "state_field", + "states", + "poll_interval_seconds", + "max_wait_seconds", + "submitted_at" + ], + "title": "ARI Async Tool Handle v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/call_context_v1.schema.json b/ari-core/ari/schemas/call_context_v1.schema.json new file mode 100644 index 00000000..a4973a21 --- /dev/null +++ b/ari-core/ari/schemas/call_context_v1.schema.json @@ -0,0 +1,155 @@ +{ + "$defs": { + "NodeContextV1": { + "additionalProperties": false, + "description": "Immutable node identity and its ordered, run-scoped ancestry.", + "properties": { + "ancestor_node_ids": { + "items": { + "type": "string" + }, + "title": "Ancestor Node Ids", + "type": "array" + }, + "lineage_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Lineage Digest", + "type": "string" + }, + "node_id": { + "minLength": 1, + "title": "Node Id", + "type": "string" + }, + "parent_node_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Node Id" + }, + "run_id": { + "minLength": 1, + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "const": "ari.node-context/v1", + "default": "ari.node-context/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "run_id", + "node_id", + "lineage_digest" + ], + "title": "NodeContextV1", + "type": "object" + }, + "RunContextV1": { + "additionalProperties": false, + "description": "Immutable identity for one ARI run.", + "properties": { + "run_id": { + "minLength": 1, + "title": "Run Id", + "type": "string" + }, + "run_scope_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Run Scope Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.run-context/v1", + "default": "ari.run-context/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "run_id", + "run_scope_digest" + ], + "title": "RunContextV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/call-context-v1.schema.json", + "additionalProperties": false, + "description": "Explicit context supplied at a tool-call boundary.\n\n``run_id`` and ``node_id`` remain as flattened provenance fields for the\nv1 result envelope. Authorization requires the corresponding structured\ncontext, whose digests prevent accidental lineage corruption.", + "properties": { + "credential_scope_ids": { + "items": { + "type": "string" + }, + "title": "Credential Scope Ids", + "type": "array" + }, + "node_context": { + "anyOf": [ + { + "$ref": "#/$defs/NodeContextV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "node_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Node Id" + }, + "phase": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Phase" + }, + "run_context": { + "anyOf": [ + { + "$ref": "#/$defs/RunContextV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "run_id": { + "default": "", + "title": "Run Id", + "type": "string" + }, + "selection_reason": { + "default": "", + "title": "Selection Reason", + "type": "string" + } + }, + "title": "ARI Tool Call Context v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/execution_request_v1.schema.json b/ari-core/ari/schemas/execution_request_v1.schema.json new file mode 100644 index 00000000..44d690fb --- /dev/null +++ b/ari-core/ari/schemas/execution_request_v1.schema.json @@ -0,0 +1,232 @@ +{ + "$defs": { + "ContainerIdentityV1": { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Digest" + }, + "reference": { + "title": "Reference", + "type": "string" + }, + "resolution_status": { + "enum": [ + "resolved", + "unresolved" + ], + "title": "Resolution Status", + "type": "string" + }, + "runtime": { + "enum": [ + "docker", + "singularity", + "apptainer", + "unknown" + ], + "title": "Runtime", + "type": "string" + } + }, + "required": [ + "runtime", + "reference", + "resolution_status" + ], + "title": "ContainerIdentityV1", + "type": "object" + }, + "ExecutionLimitsV1": { + "additionalProperties": false, + "properties": { + "cpu_seconds": { + "anyOf": [ + { + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cpu Seconds" + }, + "max_output_bytes": { + "default": 67108864, + "maximum": 1073741824, + "minimum": 1024, + "title": "Max Output Bytes", + "type": "integer" + }, + "max_processes": { + "anyOf": [ + { + "maximum": 65536, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Processes" + }, + "memory_bytes": { + "anyOf": [ + { + "minimum": 16777216, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Memory Bytes" + } + }, + "title": "ExecutionLimitsV1", + "type": "object" + }, + "WorkspaceRefV1": { + "additionalProperties": false, + "description": "One canonical writable root with no caller-controlled escape path.", + "properties": { + "root": { + "title": "Root", + "type": "string" + }, + "schema_version": { + "const": "ari.workspace-ref/v1", + "default": "ari.workspace-ref/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "root" + ], + "title": "WorkspaceRefV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/execution-request-v1.schema.json", + "additionalProperties": false, + "properties": { + "argv": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "maxItems": 256, + "minItems": 1, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Argv" + }, + "container": { + "anyOf": [ + { + "$ref": "#/$defs/ContainerIdentityV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "environment": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 128, + "title": "Environment", + "type": "object" + }, + "input_digests": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 1024, + "title": "Input Digests", + "type": "object" + }, + "limits": { + "$ref": "#/$defs/ExecutionLimitsV1" + }, + "network": { + "default": "inherit", + "enum": [ + "inherit", + "deny" + ], + "title": "Network", + "type": "string" + }, + "request_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Request Id" + }, + "schema_version": { + "const": "ari.execution-request/v1", + "default": "ari.execution-request/v1", + "title": "Schema Version", + "type": "string" + }, + "shell_command": { + "anyOf": [ + { + "maxLength": 100000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Shell Command" + }, + "timeout_seconds": { + "default": 60, + "exclusiveMinimum": 0, + "maximum": 86400, + "title": "Timeout Seconds", + "type": "number" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace" + ], + "title": "ARI Execution Request v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/execution_result_v1.schema.json b/ari-core/ari/schemas/execution_result_v1.schema.json new file mode 100644 index 00000000..983f154c --- /dev/null +++ b/ari-core/ari/schemas/execution_result_v1.schema.json @@ -0,0 +1,357 @@ +{ + "$defs": { + "ContainerIdentityV1": { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Digest" + }, + "reference": { + "title": "Reference", + "type": "string" + }, + "resolution_status": { + "enum": [ + "resolved", + "unresolved" + ], + "title": "Resolution Status", + "type": "string" + }, + "runtime": { + "enum": [ + "docker", + "singularity", + "apptainer", + "unknown" + ], + "title": "Runtime", + "type": "string" + } + }, + "required": [ + "runtime", + "reference", + "resolution_status" + ], + "title": "ContainerIdentityV1", + "type": "object" + }, + "ExecutionArtifactV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "logical_role": { + "enum": [ + "stdout", + "stderr" + ], + "title": "Logical Role", + "type": "string" + }, + "media_type": { + "const": "text/plain; charset=utf-8", + "default": "text/plain; charset=utf-8", + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "logical_role", + "relative_path", + "digest", + "size_bytes" + ], + "title": "ExecutionArtifactV1", + "type": "object" + }, + "ExecutionLimitsV1": { + "additionalProperties": false, + "properties": { + "cpu_seconds": { + "anyOf": [ + { + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cpu Seconds" + }, + "max_output_bytes": { + "default": 67108864, + "maximum": 1073741824, + "minimum": 1024, + "title": "Max Output Bytes", + "type": "integer" + }, + "max_processes": { + "anyOf": [ + { + "maximum": 65536, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Processes" + }, + "memory_bytes": { + "anyOf": [ + { + "minimum": 16777216, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Memory Bytes" + } + }, + "title": "ExecutionLimitsV1", + "type": "object" + }, + "ResourceLimitReportV1": { + "additionalProperties": false, + "description": "What the launcher actually enforced, distinct from requested limits.", + "properties": { + "enforced": { + "items": { + "enum": [ + "cpu", + "memory", + "processes", + "output" + ], + "type": "string" + }, + "title": "Enforced", + "type": "array" + }, + "process_group": { + "enum": [ + "executor", + "external-unverified" + ], + "title": "Process Group", + "type": "string" + }, + "substrate": { + "enum": [ + "posix-kernel", + "external-unverified" + ], + "title": "Substrate", + "type": "string" + }, + "wall_time": { + "enum": [ + "executor", + "external-unverified" + ], + "title": "Wall Time", + "type": "string" + } + }, + "required": [ + "substrate", + "wall_time", + "process_group" + ], + "title": "ResourceLimitReportV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/execution-result-v1.schema.json", + "additionalProperties": false, + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/ExecutionArtifactV1" + }, + "title": "Artifacts", + "type": "array" + }, + "attempt_id": { + "title": "Attempt Id", + "type": "string" + }, + "completed_at": { + "title": "Completed At", + "type": "string" + }, + "container": { + "anyOf": [ + { + "$ref": "#/$defs/ContainerIdentityV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "environment_names": { + "items": { + "type": "string" + }, + "title": "Environment Names", + "type": "array" + }, + "execution_identity": { + "title": "Execution Identity", + "type": "string" + }, + "exit_code": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Exit Code" + }, + "input_bindings": { + "additionalProperties": { + "enum": [ + "immutable-snapshot", + "verified-at-launch", + "external-unverified" + ], + "type": "string" + }, + "title": "Input Bindings", + "type": "object" + }, + "input_digests": { + "additionalProperties": { + "type": "string" + }, + "title": "Input Digests", + "type": "object" + }, + "limit_report": { + "$ref": "#/$defs/ResourceLimitReportV1" + }, + "limits": { + "$ref": "#/$defs/ExecutionLimitsV1" + }, + "network": { + "enum": [ + "inherit", + "deny" + ], + "title": "Network", + "type": "string" + }, + "network_report": { + "enum": [ + "inherited", + "isolated", + "external-unverified" + ], + "title": "Network Report", + "type": "string" + }, + "request_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Request Id" + }, + "schema_version": { + "const": "ari.execution-result/v1", + "default": "ari.execution-result/v1", + "title": "Schema Version", + "type": "string" + }, + "started_at": { + "title": "Started At", + "type": "string" + }, + "status": { + "enum": [ + "completed", + "failed", + "timed_out", + "cancelled" + ], + "title": "Status", + "type": "string" + }, + "stderr_preview": { + "title": "Stderr Preview", + "type": "string" + }, + "stderr_truncated": { + "title": "Stderr Truncated", + "type": "boolean" + }, + "stdout_preview": { + "title": "Stdout Preview", + "type": "string" + }, + "stdout_truncated": { + "title": "Stdout Truncated", + "type": "boolean" + } + }, + "required": [ + "status", + "execution_identity", + "attempt_id", + "exit_code", + "started_at", + "completed_at", + "stdout_preview", + "stderr_preview", + "stdout_truncated", + "stderr_truncated", + "artifacts", + "environment_names", + "network", + "network_report", + "limits", + "limit_report" + ], + "title": "ARI Execution Result v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/figure_batch_v1.schema.json b/ari-core/ari/schemas/figure_batch_v1.schema.json new file mode 100644 index 00000000..19c3894e --- /dev/null +++ b/ari-core/ari/schemas/figure_batch_v1.schema.json @@ -0,0 +1,639 @@ +{ + "$defs": { + "FigureArtifactV1": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "enum": [ + "source-data", + "spec", + "png", + "pdf", + "raw-planner-response", + "prompt" + ], + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.figure-artifact/v1", + "default": "ari.figure-artifact/v1", + "title": "Schema Version", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "role", + "relative_path", + "digest", + "media_type", + "size_bytes" + ], + "title": "FigureArtifactV1", + "type": "object" + }, + "FigureAxisV1": { + "additionalProperties": false, + "properties": { + "label": { + "maxLength": 256, + "minLength": 1, + "title": "Label", + "type": "string" + }, + "scale": { + "default": "linear", + "enum": [ + "linear", + "log", + "symlog" + ], + "title": "Scale", + "type": "string" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "label", + "unit" + ], + "title": "FigureAxisV1", + "type": "object" + }, + "FigureEnvironmentV1": { + "additionalProperties": false, + "properties": { + "backend": { + "const": "agg", + "default": "agg", + "title": "Backend", + "type": "string" + }, + "container_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Container Digest" + }, + "environment_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Environment Digest", + "type": "string" + }, + "font_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Font Digest", + "type": "string" + }, + "font_family": { + "maxLength": 256, + "minLength": 1, + "title": "Font Family", + "type": "string" + }, + "matplotlib_version": { + "maxLength": 128, + "minLength": 1, + "title": "Matplotlib Version", + "type": "string" + }, + "platform": { + "maxLength": 512, + "minLength": 1, + "title": "Platform", + "type": "string" + }, + "python_version": { + "maxLength": 128, + "minLength": 1, + "title": "Python Version", + "type": "string" + }, + "renderer_version": { + "maxLength": 128, + "minLength": 1, + "title": "Renderer Version", + "type": "string" + }, + "schema_version": { + "const": "ari.figure-environment/v1", + "default": "ari.figure-environment/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "renderer_version", + "python_version", + "matplotlib_version", + "font_family", + "font_digest", + "platform", + "environment_digest" + ], + "title": "FigureEnvironmentV1", + "type": "object" + }, + "FigureFeedbackV1": { + "additionalProperties": false, + "properties": { + "feedback_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Feedback Digest", + "type": "string" + }, + "figure_id": { + "title": "Figure Id", + "type": "string" + }, + "issues": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Issues", + "type": "array" + }, + "iteration": { + "maximum": 2, + "minimum": 1, + "title": "Iteration", + "type": "integer" + }, + "review_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Review Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.figure-feedback/v1", + "default": "ari.figure-feedback/v1", + "title": "Schema Version", + "type": "string" + }, + "source_manifest_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Manifest Digest", + "type": "string" + }, + "suggestions": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Suggestions", + "type": "array" + } + }, + "required": [ + "figure_id", + "source_manifest_digest", + "review_digest", + "iteration", + "feedback_digest" + ], + "title": "FigureFeedbackV1", + "type": "object" + }, + "FigureManifestV1": { + "additionalProperties": false, + "description": "Reproducible identity of one rendered figure revision.", + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/FigureArtifactV1" + }, + "maxItems": 16, + "minItems": 4, + "title": "Artifacts", + "type": "array" + }, + "environment": { + "$ref": "#/$defs/FigureEnvironmentV1" + }, + "execution_mode": { + "const": "declarative-fixed-renderer", + "default": "declarative-fixed-renderer", + "title": "Execution Mode", + "type": "string" + }, + "feedback": { + "anyOf": [ + { + "$ref": "#/$defs/FigureFeedbackV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Limitations", + "type": "array" + }, + "manifest_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Manifest Digest", + "type": "string" + }, + "parent_manifest_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Manifest Digest" + }, + "planner": { + "default": "none", + "enum": [ + "none", + "llm-spec-only" + ], + "title": "Planner", + "type": "string" + }, + "planner_model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Planner Model" + }, + "planner_model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Planner Model Revision" + }, + "prompt_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Digest" + }, + "sampling": { + "additionalProperties": true, + "maxProperties": 128, + "title": "Sampling", + "type": "object" + }, + "schema_version": { + "const": "ari.figure-manifest/v1", + "default": "ari.figure-manifest/v1", + "title": "Schema Version", + "type": "string" + }, + "spec": { + "$ref": "#/$defs/FigureSpecV1" + } + }, + "required": [ + "spec", + "environment", + "artifacts", + "manifest_digest" + ], + "title": "FigureManifestV1", + "type": "object" + }, + "FigureSourceV1": { + "additionalProperties": false, + "properties": { + "artifact_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Artifact Digest", + "type": "string" + }, + "data_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Data Digest", + "type": "string" + }, + "node_ids": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "title": "Node Ids", + "type": "array" + }, + "record_ids": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "title": "Record Ids", + "type": "array" + } + }, + "required": [ + "artifact_digest", + "data_digest" + ], + "title": "FigureSourceV1", + "type": "object" + }, + "FigureSpecV1": { + "additionalProperties": false, + "description": "One declarative plot over an immutable, content-addressed data slice.", + "properties": { + "aggregation": { + "default": "none", + "enum": [ + "none", + "mean", + "median", + "sum" + ], + "title": "Aggregation", + "type": "string" + }, + "caption": { + "default": "", + "maxLength": 4096, + "title": "Caption", + "type": "string" + }, + "chart_type": { + "enum": [ + "bar", + "line", + "scatter", + "hist", + "errorbar", + "heatmap" + ], + "title": "Chart Type", + "type": "string" + }, + "data": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Data", + "type": "object" + }, + "figure_id": { + "title": "Figure Id", + "type": "string" + }, + "revision": { + "default": 0, + "maximum": 100, + "minimum": 0, + "title": "Revision", + "type": "integer" + }, + "schema_version": { + "const": "ari.figure-spec/v1", + "default": "ari.figure-spec/v1", + "title": "Schema Version", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FigureSourceV1" + }, + "spec_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Spec Digest", + "type": "string" + }, + "style_profile": { + "const": "ari-publication-v1", + "default": "ari-publication-v1", + "title": "Style Profile", + "type": "string" + }, + "title": { + "default": "", + "maxLength": 512, + "title": "Title", + "type": "string" + }, + "uncertainty": { + "$ref": "#/$defs/FigureUncertaintyV1" + }, + "value_unit": { + "maxLength": 128, + "minLength": 1, + "title": "Value Unit", + "type": "string" + }, + "x_axis": { + "$ref": "#/$defs/FigureAxisV1" + }, + "x_field": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "X Field" + }, + "y_axis": { + "$ref": "#/$defs/FigureAxisV1" + }, + "y_field": { + "maxLength": 256, + "minLength": 1, + "title": "Y Field", + "type": "string" + } + }, + "required": [ + "figure_id", + "chart_type", + "data", + "source", + "y_field", + "x_axis", + "y_axis", + "value_unit", + "spec_digest" + ], + "title": "FigureSpecV1", + "type": "object" + }, + "FigureUncertaintyV1": { + "additionalProperties": false, + "properties": { + "confidence": { + "anyOf": [ + { + "exclusiveMaximum": 1, + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Confidence" + }, + "field": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Field" + }, + "kind": { + "enum": [ + "none", + "standard-deviation", + "standard-error", + "confidence-interval" + ], + "title": "Kind", + "type": "string" + } + }, + "required": [ + "kind" + ], + "title": "FigureUncertaintyV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/figure-batch-v1.schema.json", + "additionalProperties": false, + "properties": { + "batch_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Batch Digest", + "type": "string" + }, + "figure_kinds": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 100, + "minProperties": 1, + "title": "Figure Kinds", + "type": "object" + }, + "figures": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 100, + "minProperties": 1, + "title": "Figures", + "type": "object" + }, + "latex_snippets": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 100, + "minProperties": 1, + "title": "Latex Snippets", + "type": "object" + }, + "manifests": { + "items": { + "$ref": "#/$defs/FigureManifestV1" + }, + "maxItems": 100, + "minItems": 1, + "title": "Manifests", + "type": "array" + }, + "revision": { + "maximum": 2, + "minimum": 0, + "title": "Revision", + "type": "integer" + }, + "schema_version": { + "const": "ari.figure-batch/v1", + "default": "ari.figure-batch/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "revision", + "manifests", + "figures", + "latex_snippets", + "figure_kinds", + "batch_digest" + ], + "title": "ARI Figure Batch v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/gate_report_v1.schema.json b/ari-core/ari/schemas/gate_report_v1.schema.json new file mode 100644 index 00000000..074fc512 --- /dev/null +++ b/ari-core/ari/schemas/gate_report_v1.schema.json @@ -0,0 +1,265 @@ +{ + "$defs": { + "GateFindingV1": { + "additionalProperties": false, + "properties": { + "artifact_path": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Artifact Path" + }, + "claim_id": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Claim Id" + }, + "details": { + "additionalProperties": true, + "title": "Details", + "type": "object" + }, + "message": { + "maxLength": 20000, + "minLength": 1, + "title": "Message", + "type": "string" + }, + "node_id": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Node Id" + }, + "numeric_id": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Numeric Id" + }, + "schema_version": { + "const": "ari.gate-finding/v1", + "default": "ari.gate-finding/v1", + "title": "Schema Version", + "type": "string" + }, + "severity": { + "enum": [ + "blocking", + "advisory" + ], + "title": "Severity", + "type": "string" + }, + "type": { + "maxLength": 128, + "minLength": 1, + "title": "Type", + "type": "string" + } + }, + "required": [ + "severity", + "type", + "message" + ], + "title": "GateFindingV1", + "type": "object" + }, + "GateFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "formulas_used": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Formulas Used", + "type": "array" + }, + "metric_contract_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Metric Contract Digest" + }, + "registry_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Registry Digest", + "type": "string" + }, + "unit_conversions": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Unit Conversions", + "type": "array" + } + }, + "required": [ + "registry_digest" + ], + "title": "GateFormulaProvenanceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/gate-report-v1.schema.json", + "additionalProperties": false, + "description": "Deterministic hard-gate result with typed finding separation.", + "properties": { + "advisory_findings": { + "items": { + "$ref": "#/$defs/GateFindingV1" + }, + "title": "Advisory Findings", + "type": "array" + }, + "blocking_findings": { + "items": { + "$ref": "#/$defs/GateFindingV1" + }, + "title": "Blocking Findings", + "type": "array" + }, + "comparison_scope": { + "enum": [ + "any", + "same_environment" + ], + "title": "Comparison Scope", + "type": "string" + }, + "evidence_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Evidence Digest", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/GateFormulaProvenanceV1" + }, + "gate": { + "const": "claim_evidence_hard_gate", + "default": "claim_evidence_hard_gate", + "title": "Gate", + "type": "string" + }, + "metrics": { + "additionalProperties": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "title": "Metrics", + "type": "object" + }, + "phase": { + "enum": [ + "draft", + "final" + ], + "title": "Phase", + "type": "string" + }, + "policy_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Policy Digest", + "type": "string" + }, + "policy_mode": { + "enum": [ + "off", + "warn", + "strict" + ], + "title": "Policy Mode", + "type": "string" + }, + "report_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Report Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.gate-report/v1", + "default": "ari.gate-report/v1", + "title": "Schema Version", + "type": "string" + }, + "should_block": { + "title": "Should Block", + "type": "boolean" + }, + "source_run_id": { + "maxLength": 512, + "minLength": 1, + "title": "Source Run Id", + "type": "string" + }, + "status": { + "enum": [ + "passed", + "warn", + "failed" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "report_digest", + "source_run_id", + "phase", + "policy_mode", + "comparison_scope", + "status", + "should_block", + "policy_digest", + "evidence_digest", + "formula_provenance", + "metrics" + ], + "title": "ARI Gate Report v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/idea_candidate_v1.schema.json b/ari-core/ari/schemas/idea_candidate_v1.schema.json new file mode 100644 index 00000000..5f57ec65 --- /dev/null +++ b/ari-core/ari/schemas/idea_candidate_v1.schema.json @@ -0,0 +1,416 @@ +{ + "$defs": { + "MetricContractV1": { + "additionalProperties": false, + "description": "Idea-owned metric vocabulary; evaluator enforcement is read-only.", + "properties": { + "admission_status": { + "enum": [ + "admitted", + "human-review-required" + ], + "title": "Admission Status", + "type": "string" + }, + "comparison_scope": { + "enum": [ + "same-environment", + "cross-environment", + "within-subject", + "not-applicable" + ], + "title": "Comparison Scope", + "type": "string" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "correctness": { + "anyOf": [ + { + "$ref": "#/$defs/MetricCorrectnessV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "correctness_required": { + "title": "Correctness Required", + "type": "boolean" + }, + "direction": { + "enum": [ + "higher", + "lower", + "target", + "none" + ], + "title": "Direction", + "type": "string" + }, + "formula": { + "maxLength": 128, + "minLength": 1, + "title": "Formula", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/MetricFormulaProvenanceV1" + }, + "invariants": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Invariants", + "type": "array" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "normalization_ceiling": { + "enum": [ + "measured", + "not-applicable" + ], + "title": "Normalization Ceiling", + "type": "string" + }, + "operands": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 16, + "minProperties": 1, + "title": "Operands", + "type": "object" + }, + "rationale": { + "maxLength": 4096, + "minLength": 1, + "title": "Rationale", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + }, + "required_measured": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Required Measured", + "type": "array" + }, + "schema_version": { + "const": "ari.metric-contract/v1", + "default": "ari.metric-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "target_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Value" + }, + "tolerance": { + "$ref": "#/$defs/MetricToleranceV1" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "contract_digest", + "name", + "unit", + "direction", + "comparison_scope", + "rationale", + "required_evidence", + "correctness_required", + "normalization_ceiling", + "formula", + "operands", + "tolerance", + "formula_provenance", + "confidence", + "admission_status" + ], + "title": "MetricContractV1", + "type": "object" + }, + "MetricCorrectnessV1": { + "additionalProperties": false, + "properties": { + "expr": { + "maxLength": 4096, + "minLength": 1, + "title": "Expr", + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Requires", + "type": "array" + } + }, + "required": [ + "expr", + "requires" + ], + "title": "MetricCorrectnessV1", + "type": "object" + }, + "MetricFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Prompt Digests", + "type": "array" + }, + "source": { + "enum": [ + "idea-generation-lock", + "human-admission", + "legacy-migration" + ], + "title": "Source", + "type": "string" + }, + "source_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Digest", + "type": "string" + } + }, + "required": [ + "source", + "source_digest" + ], + "title": "MetricFormulaProvenanceV1", + "type": "object" + }, + "MetricToleranceV1": { + "additionalProperties": false, + "properties": { + "absolute": { + "minimum": 0, + "title": "Absolute", + "type": "number" + }, + "relative": { + "minimum": 0, + "title": "Relative", + "type": "number" + } + }, + "required": [ + "absolute", + "relative" + ], + "title": "MetricToleranceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/idea-candidate-v1.schema.json", + "additionalProperties": false, + "properties": { + "artifact_references": { + "items": { + "type": "string" + }, + "title": "Artifact References", + "type": "array" + }, + "candidate_id": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Candidate Id", + "type": "string" + }, + "citations": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "minItems": 1, + "title": "Citations", + "type": "array" + }, + "description": { + "maxLength": 20000, + "minLength": 1, + "title": "Description", + "type": "string" + }, + "experiment_plan": { + "maxLength": 100000, + "minLength": 1, + "title": "Experiment Plan", + "type": "string" + }, + "falsification_conditions": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Falsification Conditions", + "type": "array" + }, + "feasibility_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Feasibility Score" + }, + "generation_lock_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Generation Lock Digest", + "type": "string" + }, + "generator_adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Generator Adapter", + "type": "string" + }, + "hypothesis": { + "maxLength": 10000, + "minLength": 1, + "title": "Hypothesis", + "type": "string" + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "metric_contract": { + "$ref": "#/$defs/MetricContractV1" + }, + "novelty_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Novelty Score" + }, + "overall_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Overall Score" + }, + "schema_version": { + "const": "ari.idea-candidate/v1", + "default": "ari.idea-candidate/v1", + "title": "Schema Version", + "type": "string" + }, + "source_snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Snapshot Digest", + "type": "string" + }, + "title": { + "maxLength": 512, + "minLength": 1, + "title": "Title", + "type": "string" + } + }, + "required": [ + "candidate_id", + "title", + "hypothesis", + "description", + "experiment_plan", + "falsification_conditions", + "metric_contract", + "citations", + "limitations", + "source_snapshot_digest", + "generation_lock_digest", + "generator_adapter" + ], + "title": "ARI Idea Candidate v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/idea_set_v1.schema.json b/ari-core/ari/schemas/idea_set_v1.schema.json new file mode 100644 index 00000000..86a1b48e --- /dev/null +++ b/ari-core/ari/schemas/idea_set_v1.schema.json @@ -0,0 +1,733 @@ +{ + "$defs": { + "IdeaCandidateV1": { + "additionalProperties": false, + "properties": { + "artifact_references": { + "items": { + "type": "string" + }, + "title": "Artifact References", + "type": "array" + }, + "candidate_id": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Candidate Id", + "type": "string" + }, + "citations": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "minItems": 1, + "title": "Citations", + "type": "array" + }, + "description": { + "maxLength": 20000, + "minLength": 1, + "title": "Description", + "type": "string" + }, + "experiment_plan": { + "maxLength": 100000, + "minLength": 1, + "title": "Experiment Plan", + "type": "string" + }, + "falsification_conditions": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Falsification Conditions", + "type": "array" + }, + "feasibility_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Feasibility Score" + }, + "generation_lock_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Generation Lock Digest", + "type": "string" + }, + "generator_adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Generator Adapter", + "type": "string" + }, + "hypothesis": { + "maxLength": 10000, + "minLength": 1, + "title": "Hypothesis", + "type": "string" + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "metric_contract": { + "$ref": "#/$defs/MetricContractV1" + }, + "novelty_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Novelty Score" + }, + "overall_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Overall Score" + }, + "schema_version": { + "const": "ari.idea-candidate/v1", + "default": "ari.idea-candidate/v1", + "title": "Schema Version", + "type": "string" + }, + "source_snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Snapshot Digest", + "type": "string" + }, + "title": { + "maxLength": 512, + "minLength": 1, + "title": "Title", + "type": "string" + } + }, + "required": [ + "candidate_id", + "title", + "hypothesis", + "description", + "experiment_plan", + "falsification_conditions", + "metric_contract", + "citations", + "limitations", + "source_snapshot_digest", + "generation_lock_digest", + "generator_adapter" + ], + "title": "IdeaCandidateV1", + "type": "object" + }, + "IdeaGenerationLockV1": { + "additionalProperties": false, + "description": "Deterministic generation inputs, excluding timestamps and model output.", + "properties": { + "adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Adapter", + "type": "string" + }, + "adapter_version": { + "maxLength": 128, + "minLength": 1, + "title": "Adapter Version", + "type": "string" + }, + "api_base_identity": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Base Identity" + }, + "experiment_context_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Experiment Context Digest", + "type": "string" + }, + "generation_lock_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Generation Lock Digest", + "type": "string" + }, + "generation_parameters": { + "additionalProperties": true, + "title": "Generation Parameters", + "type": "object" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Prompt Digests", + "type": "array" + }, + "schema_version": { + "const": "ari.idea-generation-lock/v1", + "default": "ari.idea-generation-lock/v1", + "title": "Schema Version", + "type": "string" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "source_snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Snapshot Digest", + "type": "string" + }, + "temperatures": { + "items": { + "type": "number" + }, + "maxItems": 128, + "minItems": 1, + "title": "Temperatures", + "type": "array" + }, + "topic_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Topic Digest", + "type": "string" + }, + "vendor_commit": { + "anyOf": [ + { + "maxLength": 64, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vendor Commit" + }, + "vendor_license": { + "anyOf": [ + { + "maxLength": 128, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vendor License" + } + }, + "required": [ + "generation_lock_digest", + "adapter", + "adapter_version", + "model", + "prompt_digests", + "temperatures", + "source_snapshot_digest", + "topic_digest", + "experiment_context_digest" + ], + "title": "IdeaGenerationLockV1", + "type": "object" + }, + "IdeaGenerationProvenanceV1": { + "additionalProperties": false, + "properties": { + "actual_adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Actual Adapter", + "type": "string" + }, + "fallback_reason": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fallback Reason" + }, + "generated_at": { + "format": "date-time", + "title": "Generated At", + "type": "string" + }, + "lock": { + "$ref": "#/$defs/IdeaGenerationLockV1" + }, + "output_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Output Digest", + "type": "string" + }, + "requested_adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Requested Adapter", + "type": "string" + }, + "schema_version": { + "const": "ari.idea-generation-provenance/v1", + "default": "ari.idea-generation-provenance/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "lock", + "generated_at", + "output_digest", + "requested_adapter", + "actual_adapter" + ], + "title": "IdeaGenerationProvenanceV1", + "type": "object" + }, + "IdeaRejectionV1": { + "additionalProperties": false, + "properties": { + "generator_adapter": { + "maxLength": 128, + "minLength": 1, + "title": "Generator Adapter", + "type": "string" + }, + "raw_candidate_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Raw Candidate Digest", + "type": "string" + }, + "reasons": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Reasons", + "type": "array" + }, + "schema_version": { + "const": "ari.idea-rejection/v1", + "default": "ari.idea-rejection/v1", + "title": "Schema Version", + "type": "string" + }, + "title": { + "default": "", + "maxLength": 512, + "title": "Title", + "type": "string" + } + }, + "required": [ + "raw_candidate_digest", + "reasons", + "generator_adapter" + ], + "title": "IdeaRejectionV1", + "type": "object" + }, + "MetricContractV1": { + "additionalProperties": false, + "description": "Idea-owned metric vocabulary; evaluator enforcement is read-only.", + "properties": { + "admission_status": { + "enum": [ + "admitted", + "human-review-required" + ], + "title": "Admission Status", + "type": "string" + }, + "comparison_scope": { + "enum": [ + "same-environment", + "cross-environment", + "within-subject", + "not-applicable" + ], + "title": "Comparison Scope", + "type": "string" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "correctness": { + "anyOf": [ + { + "$ref": "#/$defs/MetricCorrectnessV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "correctness_required": { + "title": "Correctness Required", + "type": "boolean" + }, + "direction": { + "enum": [ + "higher", + "lower", + "target", + "none" + ], + "title": "Direction", + "type": "string" + }, + "formula": { + "maxLength": 128, + "minLength": 1, + "title": "Formula", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/MetricFormulaProvenanceV1" + }, + "invariants": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Invariants", + "type": "array" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "normalization_ceiling": { + "enum": [ + "measured", + "not-applicable" + ], + "title": "Normalization Ceiling", + "type": "string" + }, + "operands": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 16, + "minProperties": 1, + "title": "Operands", + "type": "object" + }, + "rationale": { + "maxLength": 4096, + "minLength": 1, + "title": "Rationale", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + }, + "required_measured": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Required Measured", + "type": "array" + }, + "schema_version": { + "const": "ari.metric-contract/v1", + "default": "ari.metric-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "target_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Value" + }, + "tolerance": { + "$ref": "#/$defs/MetricToleranceV1" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "contract_digest", + "name", + "unit", + "direction", + "comparison_scope", + "rationale", + "required_evidence", + "correctness_required", + "normalization_ceiling", + "formula", + "operands", + "tolerance", + "formula_provenance", + "confidence", + "admission_status" + ], + "title": "MetricContractV1", + "type": "object" + }, + "MetricCorrectnessV1": { + "additionalProperties": false, + "properties": { + "expr": { + "maxLength": 4096, + "minLength": 1, + "title": "Expr", + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Requires", + "type": "array" + } + }, + "required": [ + "expr", + "requires" + ], + "title": "MetricCorrectnessV1", + "type": "object" + }, + "MetricFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Prompt Digests", + "type": "array" + }, + "source": { + "enum": [ + "idea-generation-lock", + "human-admission", + "legacy-migration" + ], + "title": "Source", + "type": "string" + }, + "source_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Digest", + "type": "string" + } + }, + "required": [ + "source", + "source_digest" + ], + "title": "MetricFormulaProvenanceV1", + "type": "object" + }, + "MetricToleranceV1": { + "additionalProperties": false, + "properties": { + "absolute": { + "minimum": 0, + "title": "Absolute", + "type": "number" + }, + "relative": { + "minimum": 0, + "title": "Relative", + "type": "number" + } + }, + "required": [ + "absolute", + "relative" + ], + "title": "MetricToleranceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/idea-set-v1.schema.json", + "additionalProperties": false, + "properties": { + "candidates": { + "items": { + "$ref": "#/$defs/IdeaCandidateV1" + }, + "title": "Candidates", + "type": "array" + }, + "generation": { + "$ref": "#/$defs/IdeaGenerationProvenanceV1" + }, + "idea_set_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Idea Set Digest", + "type": "string" + }, + "rejections": { + "items": { + "$ref": "#/$defs/IdeaRejectionV1" + }, + "title": "Rejections", + "type": "array" + }, + "schema_version": { + "const": "ari.idea-set/v1", + "default": "ari.idea-set/v1", + "title": "Schema Version", + "type": "string" + }, + "selected_candidate_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Selected Candidate Id" + }, + "source_snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Snapshot Digest", + "type": "string" + }, + "topic": { + "maxLength": 20000, + "minLength": 1, + "title": "Topic", + "type": "string" + } + }, + "required": [ + "idea_set_digest", + "topic", + "source_snapshot_digest", + "generation" + ], + "title": "ARI Idea Set v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/measurement_set_v1.schema.json b/ari-core/ari/schemas/measurement_set_v1.schema.json new file mode 100644 index 00000000..de1c24ed --- /dev/null +++ b/ari-core/ari/schemas/measurement_set_v1.schema.json @@ -0,0 +1,165 @@ +{ + "$defs": { + "MeasurementRecordV1": { + "additionalProperties": false, + "properties": { + "artifact_digests": { + "items": { + "type": "string" + }, + "title": "Artifact Digests", + "type": "array" + }, + "execution_attempt_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Attempt Id" + }, + "execution_identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Identity" + }, + "execution_status": { + "default": "unreported", + "enum": [ + "completed", + "failed", + "timed_out", + "cancelled", + "unreported" + ], + "title": "Execution Status", + "type": "string" + }, + "exit_code": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exit Code" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "provenance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provenance" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unit" + }, + "unit_status": { + "enum": [ + "declared", + "missing" + ], + "title": "Unit Status", + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "title": "Value" + } + }, + "required": [ + "metric_id", + "value", + "unit_status" + ], + "title": "MeasurementRecordV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/measurement-set-v1.schema.json", + "additionalProperties": false, + "description": "Canonical typed scientific-measurement document.", + "properties": { + "artifact_digests": { + "items": { + "type": "string" + }, + "title": "Artifact Digests", + "type": "array" + }, + "measurements": { + "items": { + "$ref": "#/$defs/MeasurementRecordV1" + }, + "title": "Measurements", + "type": "array" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "predictions": { + "additionalProperties": true, + "title": "Predictions", + "type": "object" + }, + "schema_version": { + "const": "ari.measurement-set/v1", + "default": "ari.measurement-set/v1", + "title": "Schema Version", + "type": "string" + }, + "scores": { + "additionalProperties": true, + "title": "Scores", + "type": "object" + } + }, + "title": "ARI Measurement Set v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/memory_backup_v1.schema.json b/ari-core/ari/schemas/memory_backup_v1.schema.json new file mode 100644 index 00000000..10c99006 --- /dev/null +++ b/ari-core/ari/schemas/memory_backup_v1.schema.json @@ -0,0 +1,340 @@ +{ + "$defs": { + "MemoryArtifactRefV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "integrity_status": { + "enum": [ + "verified", + "unverified" + ], + "title": "Integrity Status", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "maxLength": 128, + "minLength": 1, + "title": "Role", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "relative_path", + "digest", + "size_bytes", + "role", + "integrity_status" + ], + "title": "MemoryArtifactRefV1", + "type": "object" + }, + "MemoryMetricPointerV1": { + "additionalProperties": false, + "properties": { + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "name", + "value", + "unit" + ], + "title": "MemoryMetricPointerV1", + "type": "object" + }, + "MemoryNodeReportRefV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "node_id": { + "title": "Node Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "node_id", + "digest" + ], + "title": "MemoryNodeReportRefV1", + "type": "object" + }, + "MemoryReactEntryV1": { + "additionalProperties": false, + "properties": { + "content": { + "maxLength": 1000000, + "title": "Content", + "type": "string" + }, + "entry_digest": { + "title": "Entry Digest", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "maxProperties": 256, + "title": "Metadata", + "type": "object" + }, + "ts": { + "title": "Ts", + "type": "number" + } + }, + "required": [ + "content", + "ts", + "entry_digest" + ], + "title": "MemoryReactEntryV1", + "type": "object" + }, + "MemoryRecordV1": { + "additionalProperties": false, + "description": "Immutable index record; artifacts, not text, remain evidence.", + "properties": { + "ancestor_node_ids": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Ancestor Node Ids", + "type": "array" + }, + "artifact_refs": { + "items": { + "$ref": "#/$defs/MemoryArtifactRefV1" + }, + "maxItems": 1024, + "title": "Artifact Refs", + "type": "array" + }, + "attributes": { + "additionalProperties": true, + "maxProperties": 256, + "title": "Attributes", + "type": "object" + }, + "confidence": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Confidence" + }, + "created_by_tool_ref": { + "title": "Created By Tool Ref", + "type": "string" + }, + "kind": { + "enum": [ + "observation", + "experiment_result", + "failure_case", + "procedure", + "reflection", + "artifact_summary", + "paper_claim", + "reproducibility_event" + ], + "title": "Kind", + "type": "string" + }, + "metric_ptr": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryMetricPointerV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "node_report_ref": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryNodeReportRefV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "record_digest": { + "title": "Record Digest", + "type": "string" + }, + "record_id": { + "title": "Record Id", + "type": "string" + }, + "repro_status": { + "anyOf": [ + { + "enum": [ + "unverified", + "rerun_passed", + "rerun_failed", + "paper_only_reproduced" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repro Status" + }, + "repro_target_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repro Target Id" + }, + "schema_version": { + "const": "ari.memory-record/v1", + "default": "ari.memory-record/v1", + "title": "Schema Version", + "type": "string" + }, + "source_node_id": { + "title": "Source Node Id", + "type": "string" + }, + "source_run_id": { + "title": "Source Run Id", + "type": "string" + }, + "text": { + "maxLength": 100000, + "minLength": 1, + "title": "Text", + "type": "string" + } + }, + "required": [ + "kind", + "text", + "source_run_id", + "source_node_id", + "created_by_tool_ref", + "record_id", + "record_digest" + ], + "title": "MemoryRecordV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/memory-backup-v1.schema.json", + "additionalProperties": false, + "properties": { + "backup_digest": { + "title": "Backup Digest", + "type": "string" + }, + "core_context": { + "additionalProperties": true, + "maxProperties": 256, + "title": "Core Context", + "type": "object" + }, + "react_entries": { + "items": { + "$ref": "#/$defs/MemoryReactEntryV1" + }, + "maxItems": 1000000, + "title": "React Entries", + "type": "array" + }, + "record_digests": { + "items": { + "type": "string" + }, + "maxItems": 1000000, + "title": "Record Digests", + "type": "array" + }, + "record_order": { + "items": { + "type": "string" + }, + "maxItems": 1000000, + "title": "Record Order", + "type": "array" + }, + "records": { + "items": { + "$ref": "#/$defs/MemoryRecordV1" + }, + "maxItems": 1000000, + "title": "Records", + "type": "array" + }, + "schema_version": { + "const": "ari.memory-backup/v1", + "default": "ari.memory-backup/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "records", + "react_entries", + "record_digests", + "record_order", + "backup_digest" + ], + "title": "ARI Memory Backup v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/memory_record_v1.schema.json b/ari-core/ari/schemas/memory_record_v1.schema.json new file mode 100644 index 00000000..015274d7 --- /dev/null +++ b/ari-core/ari/schemas/memory_record_v1.schema.json @@ -0,0 +1,247 @@ +{ + "$defs": { + "MemoryArtifactRefV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "integrity_status": { + "enum": [ + "verified", + "unverified" + ], + "title": "Integrity Status", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "maxLength": 128, + "minLength": 1, + "title": "Role", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "relative_path", + "digest", + "size_bytes", + "role", + "integrity_status" + ], + "title": "MemoryArtifactRefV1", + "type": "object" + }, + "MemoryMetricPointerV1": { + "additionalProperties": false, + "properties": { + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "name", + "value", + "unit" + ], + "title": "MemoryMetricPointerV1", + "type": "object" + }, + "MemoryNodeReportRefV1": { + "additionalProperties": false, + "properties": { + "digest": { + "title": "Digest", + "type": "string" + }, + "node_id": { + "title": "Node Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "node_id", + "digest" + ], + "title": "MemoryNodeReportRefV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/memory-record-v1.schema.json", + "additionalProperties": false, + "description": "Immutable index record; artifacts, not text, remain evidence.", + "properties": { + "ancestor_node_ids": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Ancestor Node Ids", + "type": "array" + }, + "artifact_refs": { + "items": { + "$ref": "#/$defs/MemoryArtifactRefV1" + }, + "maxItems": 1024, + "title": "Artifact Refs", + "type": "array" + }, + "attributes": { + "additionalProperties": true, + "maxProperties": 256, + "title": "Attributes", + "type": "object" + }, + "confidence": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Confidence" + }, + "created_by_tool_ref": { + "title": "Created By Tool Ref", + "type": "string" + }, + "kind": { + "enum": [ + "observation", + "experiment_result", + "failure_case", + "procedure", + "reflection", + "artifact_summary", + "paper_claim", + "reproducibility_event" + ], + "title": "Kind", + "type": "string" + }, + "metric_ptr": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryMetricPointerV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "node_report_ref": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryNodeReportRefV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "record_digest": { + "title": "Record Digest", + "type": "string" + }, + "record_id": { + "title": "Record Id", + "type": "string" + }, + "repro_status": { + "anyOf": [ + { + "enum": [ + "unverified", + "rerun_passed", + "rerun_failed", + "paper_only_reproduced" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repro Status" + }, + "repro_target_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repro Target Id" + }, + "schema_version": { + "const": "ari.memory-record/v1", + "default": "ari.memory-record/v1", + "title": "Schema Version", + "type": "string" + }, + "source_node_id": { + "title": "Source Node Id", + "type": "string" + }, + "source_run_id": { + "title": "Source Run Id", + "type": "string" + }, + "text": { + "maxLength": 100000, + "minLength": 1, + "title": "Text", + "type": "string" + } + }, + "required": [ + "kind", + "text", + "source_run_id", + "source_node_id", + "created_by_tool_ref", + "record_id", + "record_digest" + ], + "title": "ARI Memory Record v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/memory_retrieval_v1.schema.json b/ari-core/ari/schemas/memory_retrieval_v1.schema.json new file mode 100644 index 00000000..06f93f0a --- /dev/null +++ b/ari-core/ari/schemas/memory_retrieval_v1.schema.json @@ -0,0 +1,118 @@ +{ + "$defs": { + "MemoryRetrievalProvenanceV1": { + "additionalProperties": false, + "properties": { + "backend": { + "maxLength": 256, + "minLength": 1, + "title": "Backend", + "type": "string" + }, + "backend_version": { + "maxLength": 256, + "minLength": 1, + "title": "Backend Version", + "type": "string" + }, + "candidate_count": { + "minimum": 0, + "title": "Candidate Count", + "type": "integer" + }, + "deterministic": { + "title": "Deterministic", + "type": "boolean" + }, + "filter_evidence": { + "additionalProperties": true, + "title": "Filter Evidence", + "type": "object" + }, + "limit": { + "maximum": 1000, + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_version": { + "maxLength": 256, + "minLength": 1, + "title": "Model Version", + "type": "string" + }, + "query_digest": { + "title": "Query Digest", + "type": "string" + }, + "ranking": { + "maxLength": 1024, + "minLength": 1, + "title": "Ranking", + "type": "string" + }, + "returned_count": { + "minimum": 0, + "title": "Returned Count", + "type": "integer" + }, + "server_version": { + "maxLength": 256, + "minLength": 1, + "title": "Server Version", + "type": "string" + } + }, + "required": [ + "backend", + "backend_version", + "server_version", + "model", + "model_version", + "ranking", + "deterministic", + "query_digest", + "candidate_count", + "returned_count", + "limit", + "filter_evidence" + ], + "title": "MemoryRetrievalProvenanceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/memory-retrieval-v1.schema.json", + "additionalProperties": false, + "properties": { + "provenance": { + "$ref": "#/$defs/MemoryRetrievalProvenanceV1" + }, + "results": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "maxItems": 1000, + "title": "Results", + "type": "array" + }, + "schema_version": { + "const": "ari.memory-retrieval/v1", + "default": "ari.memory-retrieval/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "results", + "provenance" + ], + "title": "ARI Memory Retrieval v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/metric_admission_decision_v1.schema.json b/ari-core/ari/schemas/metric_admission_decision_v1.schema.json new file mode 100644 index 00000000..471654c1 --- /dev/null +++ b/ari-core/ari/schemas/metric_admission_decision_v1.schema.json @@ -0,0 +1,74 @@ +{ + "$id": "https://ari.dev/schemas/metric-admission-decision-v1.schema.json", + "additionalProperties": false, + "properties": { + "admitted_contract_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Admitted Contract Digest" + }, + "decision": { + "enum": [ + "admitted", + "review-required", + "rejected" + ], + "title": "Decision", + "type": "string" + }, + "decision_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Decision Digest", + "type": "string" + }, + "proposal_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Proposal Digest", + "type": "string" + }, + "reasons": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Reasons", + "type": "array" + }, + "reviewer": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reviewer" + }, + "schema_version": { + "const": "ari.metric-admission-decision/v1", + "default": "ari.metric-admission-decision/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "decision_digest", + "proposal_digest", + "decision", + "reasons" + ], + "title": "ARI Metric Admission Decision v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/metric_contract_proposal_v1.schema.json b/ari-core/ari/schemas/metric_contract_proposal_v1.schema.json new file mode 100644 index 00000000..bf20a66f --- /dev/null +++ b/ari-core/ari/schemas/metric_contract_proposal_v1.schema.json @@ -0,0 +1,80 @@ +{ + "$id": "https://ari.dev/schemas/metric-contract-proposal-v1.schema.json", + "additionalProperties": false, + "description": "Untrusted LLM suggestion; never an admitted scientific contract.", + "properties": { + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "evidence_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Evidence Digest", + "type": "string" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Prompt Digest", + "type": "string" + }, + "proposal_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Proposal Digest", + "type": "string" + }, + "proposed_contract": { + "additionalProperties": true, + "title": "Proposed Contract", + "type": "object" + }, + "requires_human_review": { + "const": true, + "default": true, + "title": "Requires Human Review", + "type": "boolean" + }, + "schema_version": { + "const": "ari.metric-contract-proposal/v1", + "default": "ari.metric-contract-proposal/v1", + "title": "Schema Version", + "type": "string" + }, + "source_idea_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Idea Digest", + "type": "string" + } + }, + "required": [ + "proposal_digest", + "source_idea_digest", + "evidence_digest", + "model", + "prompt_digest", + "proposed_contract", + "confidence" + ], + "title": "ARI Metric Contract Proposal v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/metric_contract_v1.schema.json b/ari-core/ari/schemas/metric_contract_v1.schema.json new file mode 100644 index 00000000..d6d28b64 --- /dev/null +++ b/ari-core/ari/schemas/metric_contract_v1.schema.json @@ -0,0 +1,265 @@ +{ + "$defs": { + "MetricCorrectnessV1": { + "additionalProperties": false, + "properties": { + "expr": { + "maxLength": 4096, + "minLength": 1, + "title": "Expr", + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Requires", + "type": "array" + } + }, + "required": [ + "expr", + "requires" + ], + "title": "MetricCorrectnessV1", + "type": "object" + }, + "MetricFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Prompt Digests", + "type": "array" + }, + "source": { + "enum": [ + "idea-generation-lock", + "human-admission", + "legacy-migration" + ], + "title": "Source", + "type": "string" + }, + "source_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Digest", + "type": "string" + } + }, + "required": [ + "source", + "source_digest" + ], + "title": "MetricFormulaProvenanceV1", + "type": "object" + }, + "MetricToleranceV1": { + "additionalProperties": false, + "properties": { + "absolute": { + "minimum": 0, + "title": "Absolute", + "type": "number" + }, + "relative": { + "minimum": 0, + "title": "Relative", + "type": "number" + } + }, + "required": [ + "absolute", + "relative" + ], + "title": "MetricToleranceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/metric-contract-v1.schema.json", + "additionalProperties": false, + "description": "Idea-owned metric vocabulary; evaluator enforcement is read-only.", + "properties": { + "admission_status": { + "enum": [ + "admitted", + "human-review-required" + ], + "title": "Admission Status", + "type": "string" + }, + "comparison_scope": { + "enum": [ + "same-environment", + "cross-environment", + "within-subject", + "not-applicable" + ], + "title": "Comparison Scope", + "type": "string" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "correctness": { + "anyOf": [ + { + "$ref": "#/$defs/MetricCorrectnessV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "correctness_required": { + "title": "Correctness Required", + "type": "boolean" + }, + "direction": { + "enum": [ + "higher", + "lower", + "target", + "none" + ], + "title": "Direction", + "type": "string" + }, + "formula": { + "maxLength": 128, + "minLength": 1, + "title": "Formula", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/MetricFormulaProvenanceV1" + }, + "invariants": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Invariants", + "type": "array" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "normalization_ceiling": { + "enum": [ + "measured", + "not-applicable" + ], + "title": "Normalization Ceiling", + "type": "string" + }, + "operands": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 16, + "minProperties": 1, + "title": "Operands", + "type": "object" + }, + "rationale": { + "maxLength": 4096, + "minLength": 1, + "title": "Rationale", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + }, + "required_measured": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Required Measured", + "type": "array" + }, + "schema_version": { + "const": "ari.metric-contract/v1", + "default": "ari.metric-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "target_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Value" + }, + "tolerance": { + "$ref": "#/$defs/MetricToleranceV1" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "contract_digest", + "name", + "unit", + "direction", + "comparison_scope", + "rationale", + "required_evidence", + "correctness_required", + "normalization_ceiling", + "formula", + "operands", + "tolerance", + "formula_provenance", + "confidence", + "admission_status" + ], + "title": "ARI Metric Contract v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/metric_gate_contract_v1.schema.json b/ari-core/ari/schemas/metric_gate_contract_v1.schema.json new file mode 100644 index 00000000..2d223128 --- /dev/null +++ b/ari-core/ari/schemas/metric_gate_contract_v1.schema.json @@ -0,0 +1,354 @@ +{ + "$defs": { + "MetricClaimV1": { + "additionalProperties": false, + "properties": { + "claim": { + "maxLength": 10000, + "minLength": 1, + "title": "Claim", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + } + }, + "required": [ + "claim", + "required_evidence" + ], + "title": "MetricClaimV1", + "type": "object" + }, + "MetricContractV1": { + "additionalProperties": false, + "description": "Idea-owned metric vocabulary; evaluator enforcement is read-only.", + "properties": { + "admission_status": { + "enum": [ + "admitted", + "human-review-required" + ], + "title": "Admission Status", + "type": "string" + }, + "comparison_scope": { + "enum": [ + "same-environment", + "cross-environment", + "within-subject", + "not-applicable" + ], + "title": "Comparison Scope", + "type": "string" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "correctness": { + "anyOf": [ + { + "$ref": "#/$defs/MetricCorrectnessV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "correctness_required": { + "title": "Correctness Required", + "type": "boolean" + }, + "direction": { + "enum": [ + "higher", + "lower", + "target", + "none" + ], + "title": "Direction", + "type": "string" + }, + "formula": { + "maxLength": 128, + "minLength": 1, + "title": "Formula", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/MetricFormulaProvenanceV1" + }, + "invariants": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Invariants", + "type": "array" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "normalization_ceiling": { + "enum": [ + "measured", + "not-applicable" + ], + "title": "Normalization Ceiling", + "type": "string" + }, + "operands": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 16, + "minProperties": 1, + "title": "Operands", + "type": "object" + }, + "rationale": { + "maxLength": 4096, + "minLength": 1, + "title": "Rationale", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + }, + "required_measured": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Required Measured", + "type": "array" + }, + "schema_version": { + "const": "ari.metric-contract/v1", + "default": "ari.metric-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "target_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Value" + }, + "tolerance": { + "$ref": "#/$defs/MetricToleranceV1" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "contract_digest", + "name", + "unit", + "direction", + "comparison_scope", + "rationale", + "required_evidence", + "correctness_required", + "normalization_ceiling", + "formula", + "operands", + "tolerance", + "formula_provenance", + "confidence", + "admission_status" + ], + "title": "MetricContractV1", + "type": "object" + }, + "MetricCorrectnessV1": { + "additionalProperties": false, + "properties": { + "expr": { + "maxLength": 4096, + "minLength": 1, + "title": "Expr", + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Requires", + "type": "array" + } + }, + "required": [ + "expr", + "requires" + ], + "title": "MetricCorrectnessV1", + "type": "object" + }, + "MetricFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Prompt Digests", + "type": "array" + }, + "source": { + "enum": [ + "idea-generation-lock", + "human-admission", + "legacy-migration" + ], + "title": "Source", + "type": "string" + }, + "source_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Digest", + "type": "string" + } + }, + "required": [ + "source", + "source_digest" + ], + "title": "MetricFormulaProvenanceV1", + "type": "object" + }, + "MetricToleranceV1": { + "additionalProperties": false, + "properties": { + "absolute": { + "minimum": 0, + "title": "Absolute", + "type": "number" + }, + "relative": { + "minimum": 0, + "title": "Relative", + "type": "number" + } + }, + "required": [ + "absolute", + "relative" + ], + "title": "MetricToleranceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/metric-gate-contract-v1.schema.json", + "additionalProperties": false, + "description": "Evaluator projection of one immutable idea-owned metric contract.", + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/MetricClaimV1" + }, + "maxItems": 128, + "title": "Claims", + "type": "array" + }, + "metric_contract": { + "$ref": "#/$defs/MetricContractV1" + }, + "projection_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Projection Digest", + "type": "string" + }, + "research_contract_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Research Contract Digest" + }, + "schema_version": { + "const": "ari.metric-gate-contract/v1", + "default": "ari.metric-gate-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "source": { + "enum": [ + "research-contract", + "human-admitted", + "legacy-migrated" + ], + "title": "Source", + "type": "string" + }, + "source_idea_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Idea Digest", + "type": "string" + } + }, + "required": [ + "projection_digest", + "source", + "source_idea_digest", + "metric_contract" + ], + "title": "ARI Metric Gate Contract v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/paper_build_v1.schema.json b/ari-core/ari/schemas/paper_build_v1.schema.json new file mode 100644 index 00000000..d84f90bb --- /dev/null +++ b/ari-core/ari/schemas/paper_build_v1.schema.json @@ -0,0 +1,773 @@ +{ + "$defs": { + "PaperArtifactV1": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "enum": [ + "science-data", + "figure-batch", + "retrieval-records", + "ear-manifest", + "template", + "rubric", + "prompt", + "raw-model-response", + "draft-tex", + "final-tex", + "bibtex", + "pdf", + "compile-stdout", + "compile-stderr", + "claim-links", + "hard-gate", + "semantic-review", + "text-review", + "visual-review", + "code-bundle-lock", + "authoring-record" + ], + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.paper-artifact/v1", + "default": "ari.paper-artifact/v1", + "title": "Schema Version", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "role", + "relative_path", + "digest", + "media_type", + "size_bytes" + ], + "title": "PaperArtifactV1", + "type": "object" + }, + "PaperCompileV1": { + "additionalProperties": false, + "properties": { + "commands": { + "items": { + "items": { + "type": "string" + }, + "type": "array" + }, + "maxItems": 8, + "minItems": 1, + "title": "Commands", + "type": "array" + }, + "compile_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Compile Digest", + "type": "string" + }, + "environment_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Environment Digest", + "type": "string" + }, + "execution_identities": { + "items": { + "type": "string" + }, + "maxItems": 8, + "title": "Execution Identities", + "type": "array" + }, + "log_artifacts": { + "items": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "maxItems": 32, + "minItems": 2, + "title": "Log Artifacts", + "type": "array" + }, + "pdf_artifact": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "ari.paper-compile/v1", + "default": "ari.paper-compile/v1", + "title": "Schema Version", + "type": "string" + }, + "status": { + "enum": [ + "completed", + "failed", + "timed-out", + "tool-unavailable" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "commands", + "log_artifacts", + "environment_digest", + "compile_digest" + ], + "title": "PaperCompileV1", + "type": "object" + }, + "PaperGateSummaryV1": { + "additionalProperties": false, + "properties": { + "blocking_error_count": { + "minimum": 0, + "title": "Blocking Error Count", + "type": "integer" + }, + "mode": { + "enum": [ + "off", + "warn", + "strict" + ], + "title": "Mode", + "type": "string" + }, + "report_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Report Digest", + "type": "string" + }, + "status": { + "enum": [ + "pass", + "blocked", + "error" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "mode", + "status", + "blocking_error_count", + "report_digest" + ], + "title": "PaperGateSummaryV1", + "type": "object" + }, + "PaperModelCallV1": { + "additionalProperties": false, + "properties": { + "call_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Call Digest", + "type": "string" + }, + "call_id": { + "title": "Call Id", + "type": "string" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_artifact": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "prompt_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Prompt Digest", + "type": "string" + }, + "provider": { + "maxLength": 256, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "purpose": { + "enum": [ + "initial-authoring", + "figure-insertion", + "reflection", + "refinement", + "text-review" + ], + "title": "Purpose", + "type": "string" + }, + "raw_response_artifact": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "sampling": { + "additionalProperties": true, + "maxProperties": 64, + "title": "Sampling", + "type": "object" + }, + "schema_version": { + "const": "ari.paper-model-call/v1", + "default": "ari.paper-model-call/v1", + "title": "Schema Version", + "type": "string" + }, + "usage": { + "$ref": "#/$defs/PaperModelUsageV1" + } + }, + "required": [ + "call_id", + "purpose", + "model", + "provider", + "prompt_digest", + "prompt_artifact", + "raw_response_artifact", + "call_digest" + ], + "title": "PaperModelCallV1", + "type": "object" + }, + "PaperModelUsageV1": { + "additionalProperties": false, + "properties": { + "cost_status": { + "default": "unavailable", + "enum": [ + "reported", + "unavailable" + ], + "title": "Cost Status", + "type": "string" + }, + "cost_usd": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cost Usd" + }, + "input_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Tokens" + }, + "output_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output Tokens" + } + }, + "title": "PaperModelUsageV1", + "type": "object" + }, + "PaperNumericCoverageV1": { + "additionalProperties": false, + "properties": { + "excluded_mentions": { + "minimum": 0, + "title": "Excluded Mentions", + "type": "integer" + }, + "exclusion_policy_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exclusion Policy Digest" + }, + "linked_mentions": { + "minimum": 0, + "title": "Linked Mentions", + "type": "integer" + }, + "result_mentions": { + "minimum": 0, + "title": "Result Mentions", + "type": "integer" + }, + "uncovered_mentions": { + "minimum": 0, + "title": "Uncovered Mentions", + "type": "integer" + }, + "unresolved_anchors": { + "minimum": 0, + "title": "Unresolved Anchors", + "type": "integer" + } + }, + "required": [ + "result_mentions", + "linked_mentions", + "excluded_mentions", + "unresolved_anchors", + "uncovered_mentions" + ], + "title": "PaperNumericCoverageV1", + "type": "object" + }, + "PaperReviewSetV1": { + "additionalProperties": false, + "properties": { + "hard_gate": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "ari.paper-review-set/v1", + "default": "ari.paper-review-set/v1", + "title": "Schema Version", + "type": "string" + }, + "semantic_review": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "text_review": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "visual_passing_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Visual Passing Score" + }, + "visual_review": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "visual_score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Visual Score" + } + }, + "title": "PaperReviewSetV1", + "type": "object" + }, + "PaperRevisionV1": { + "additionalProperties": false, + "properties": { + "bib_artifact": { + "anyOf": [ + { + "$ref": "#/$defs/PaperArtifactV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "citation_keys": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "title": "Citation Keys", + "type": "array" + }, + "claim_anchors": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "title": "Claim Anchors", + "type": "array" + }, + "figure_ids": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Figure Ids", + "type": "array" + }, + "math_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Math Digest", + "type": "string" + }, + "model_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Call Id" + }, + "parent_revision_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Revision Digest" + }, + "reason": { + "enum": [ + "initial", + "figure-insertion", + "reflection", + "refinement", + "finalize" + ], + "title": "Reason", + "type": "string" + }, + "revision": { + "maximum": 100, + "minimum": 0, + "title": "Revision", + "type": "integer" + }, + "revision_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Revision Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.paper-revision/v1", + "default": "ari.paper-revision/v1", + "title": "Schema Version", + "type": "string" + }, + "tex_artifact": { + "$ref": "#/$defs/PaperArtifactV1" + } + }, + "required": [ + "revision", + "reason", + "tex_artifact", + "math_digest", + "revision_digest" + ], + "title": "PaperRevisionV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/paper-build-v1.schema.json", + "additionalProperties": false, + "properties": { + "blocking_reasons": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Blocking Reasons", + "type": "array" + }, + "build_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Build Digest", + "type": "string" + }, + "build_id": { + "title": "Build Id", + "type": "string" + }, + "build_revision": { + "maximum": 100, + "minimum": 0, + "title": "Build Revision", + "type": "integer" + }, + "compile": { + "anyOf": [ + { + "$ref": "#/$defs/PaperCompileV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "ear_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Ear Digest", + "type": "string" + }, + "final_artifacts": { + "items": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "maxItems": 16, + "title": "Final Artifacts", + "type": "array" + }, + "gate": { + "anyOf": [ + { + "$ref": "#/$defs/PaperGateSummaryV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "input_artifacts": { + "items": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "maxItems": 32, + "minItems": 4, + "title": "Input Artifacts", + "type": "array" + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Limitations", + "type": "array" + }, + "model_calls": { + "items": { + "$ref": "#/$defs/PaperModelCallV1" + }, + "maxItems": 101, + "title": "Model Calls", + "type": "array" + }, + "numeric_coverage": { + "anyOf": [ + { + "$ref": "#/$defs/PaperNumericCoverageV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "parent_build_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Build Digest" + }, + "reviews": { + "$ref": "#/$defs/PaperReviewSetV1" + }, + "revisions": { + "items": { + "$ref": "#/$defs/PaperRevisionV1" + }, + "maxItems": 101, + "minItems": 1, + "title": "Revisions", + "type": "array" + }, + "rubric_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Rubric Digest", + "type": "string" + }, + "rubric_id": { + "title": "Rubric Id", + "type": "string" + }, + "rubric_version": { + "maxLength": 128, + "minLength": 1, + "title": "Rubric Version", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "const": "ari.paper-build/v1", + "default": "ari.paper-build/v1", + "title": "Schema Version", + "type": "string" + }, + "status": { + "enum": [ + "draft", + "blocked", + "compile-error", + "finalized" + ], + "title": "Status", + "type": "string" + }, + "template_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Template Digest", + "type": "string" + }, + "venue_id": { + "title": "Venue Id", + "type": "string" + }, + "venue_version": { + "maxLength": 128, + "minLength": 1, + "title": "Venue Version", + "type": "string" + } + }, + "required": [ + "build_id", + "run_id", + "build_revision", + "status", + "input_artifacts", + "venue_id", + "venue_version", + "template_digest", + "rubric_id", + "rubric_version", + "rubric_digest", + "ear_digest", + "revisions", + "build_digest" + ], + "title": "ARI Paper Build v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/paper_model_call_batch_v1.schema.json b/ari-core/ari/schemas/paper_model_call_batch_v1.schema.json new file mode 100644 index 00000000..c0110861 --- /dev/null +++ b/ari-core/ari/schemas/paper_model_call_batch_v1.schema.json @@ -0,0 +1,253 @@ +{ + "$defs": { + "PaperArtifactV1": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "enum": [ + "science-data", + "figure-batch", + "retrieval-records", + "ear-manifest", + "template", + "rubric", + "prompt", + "raw-model-response", + "draft-tex", + "final-tex", + "bibtex", + "pdf", + "compile-stdout", + "compile-stderr", + "claim-links", + "hard-gate", + "semantic-review", + "text-review", + "visual-review", + "code-bundle-lock", + "authoring-record" + ], + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.paper-artifact/v1", + "default": "ari.paper-artifact/v1", + "title": "Schema Version", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "role", + "relative_path", + "digest", + "media_type", + "size_bytes" + ], + "title": "PaperArtifactV1", + "type": "object" + }, + "PaperModelCallV1": { + "additionalProperties": false, + "properties": { + "call_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Call Digest", + "type": "string" + }, + "call_id": { + "title": "Call Id", + "type": "string" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_artifact": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "prompt_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Prompt Digest", + "type": "string" + }, + "provider": { + "maxLength": 256, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "purpose": { + "enum": [ + "initial-authoring", + "figure-insertion", + "reflection", + "refinement", + "text-review" + ], + "title": "Purpose", + "type": "string" + }, + "raw_response_artifact": { + "$ref": "#/$defs/PaperArtifactV1" + }, + "sampling": { + "additionalProperties": true, + "maxProperties": 64, + "title": "Sampling", + "type": "object" + }, + "schema_version": { + "const": "ari.paper-model-call/v1", + "default": "ari.paper-model-call/v1", + "title": "Schema Version", + "type": "string" + }, + "usage": { + "$ref": "#/$defs/PaperModelUsageV1" + } + }, + "required": [ + "call_id", + "purpose", + "model", + "provider", + "prompt_digest", + "prompt_artifact", + "raw_response_artifact", + "call_digest" + ], + "title": "PaperModelCallV1", + "type": "object" + }, + "PaperModelUsageV1": { + "additionalProperties": false, + "properties": { + "cost_status": { + "default": "unavailable", + "enum": [ + "reported", + "unavailable" + ], + "title": "Cost Status", + "type": "string" + }, + "cost_usd": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cost Usd" + }, + "input_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Tokens" + }, + "output_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output Tokens" + } + }, + "title": "PaperModelUsageV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/paper-model-call-batch-v1.schema.json", + "additionalProperties": false, + "description": "Ordered call provenance for one bounded paper operation.", + "properties": { + "batch_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Batch Digest", + "type": "string" + }, + "calls": { + "items": { + "$ref": "#/$defs/PaperModelCallV1" + }, + "maxItems": 100, + "title": "Calls", + "type": "array" + }, + "operation": { + "enum": [ + "authoring", + "refinement", + "text-review" + ], + "title": "Operation", + "type": "string" + }, + "schema_version": { + "const": "ari.paper-model-call-batch/v1", + "default": "ari.paper-model-call-batch/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "operation", + "batch_digest" + ], + "title": "ARI Paper Model Call Batch v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/research_contract_v1.schema.json b/ari-core/ari/schemas/research_contract_v1.schema.json new file mode 100644 index 00000000..bade516e --- /dev/null +++ b/ari-core/ari/schemas/research_contract_v1.schema.json @@ -0,0 +1,373 @@ +{ + "$defs": { + "MetricContractV1": { + "additionalProperties": false, + "description": "Idea-owned metric vocabulary; evaluator enforcement is read-only.", + "properties": { + "admission_status": { + "enum": [ + "admitted", + "human-review-required" + ], + "title": "Admission Status", + "type": "string" + }, + "comparison_scope": { + "enum": [ + "same-environment", + "cross-environment", + "within-subject", + "not-applicable" + ], + "title": "Comparison Scope", + "type": "string" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "title": "Confidence", + "type": "number" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "correctness": { + "anyOf": [ + { + "$ref": "#/$defs/MetricCorrectnessV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "correctness_required": { + "title": "Correctness Required", + "type": "boolean" + }, + "direction": { + "enum": [ + "higher", + "lower", + "target", + "none" + ], + "title": "Direction", + "type": "string" + }, + "formula": { + "maxLength": 128, + "minLength": 1, + "title": "Formula", + "type": "string" + }, + "formula_provenance": { + "$ref": "#/$defs/MetricFormulaProvenanceV1" + }, + "invariants": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Invariants", + "type": "array" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "normalization_ceiling": { + "enum": [ + "measured", + "not-applicable" + ], + "title": "Normalization Ceiling", + "type": "string" + }, + "operands": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 16, + "minProperties": 1, + "title": "Operands", + "type": "object" + }, + "rationale": { + "maxLength": 4096, + "minLength": 1, + "title": "Rationale", + "type": "string" + }, + "required_evidence": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Required Evidence", + "type": "array" + }, + "required_measured": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Required Measured", + "type": "array" + }, + "schema_version": { + "const": "ari.metric-contract/v1", + "default": "ari.metric-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "target_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Value" + }, + "tolerance": { + "$ref": "#/$defs/MetricToleranceV1" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "contract_digest", + "name", + "unit", + "direction", + "comparison_scope", + "rationale", + "required_evidence", + "correctness_required", + "normalization_ceiling", + "formula", + "operands", + "tolerance", + "formula_provenance", + "confidence", + "admission_status" + ], + "title": "MetricContractV1", + "type": "object" + }, + "MetricCorrectnessV1": { + "additionalProperties": false, + "properties": { + "expr": { + "maxLength": 4096, + "minLength": 1, + "title": "Expr", + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "maxItems": 128, + "minItems": 1, + "title": "Requires", + "type": "array" + } + }, + "required": [ + "expr", + "requires" + ], + "title": "MetricCorrectnessV1", + "type": "object" + }, + "MetricFormulaProvenanceV1": { + "additionalProperties": false, + "properties": { + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "prompt_digests": { + "items": { + "type": "string" + }, + "maxItems": 128, + "title": "Prompt Digests", + "type": "array" + }, + "source": { + "enum": [ + "idea-generation-lock", + "human-admission", + "legacy-migration" + ], + "title": "Source", + "type": "string" + }, + "source_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Digest", + "type": "string" + } + }, + "required": [ + "source", + "source_digest" + ], + "title": "MetricFormulaProvenanceV1", + "type": "object" + }, + "MetricToleranceV1": { + "additionalProperties": false, + "properties": { + "absolute": { + "minimum": 0, + "title": "Absolute", + "type": "number" + }, + "relative": { + "minimum": 0, + "title": "Relative", + "type": "number" + } + }, + "required": [ + "absolute", + "relative" + ], + "title": "MetricToleranceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/research-contract-v1.schema.json", + "additionalProperties": false, + "description": "Mint-once scientific contract selected from an admitted idea candidate.", + "properties": { + "artifact_references": { + "items": { + "type": "string" + }, + "title": "Artifact References", + "type": "array" + }, + "citations": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "minItems": 1, + "title": "Citations", + "type": "array" + }, + "contract_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Contract Digest", + "type": "string" + }, + "experiment_plan": { + "maxLength": 100000, + "minLength": 1, + "title": "Experiment Plan", + "type": "string" + }, + "falsification_conditions": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Falsification Conditions", + "type": "array" + }, + "generation_lock_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Generation Lock Digest", + "type": "string" + }, + "hypothesis": { + "maxLength": 10000, + "minLength": 1, + "title": "Hypothesis", + "type": "string" + }, + "idea_set_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Idea Set Digest", + "type": "string" + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "metric_contract": { + "$ref": "#/$defs/MetricContractV1" + }, + "schema_version": { + "const": "ari.research-contract/v1", + "default": "ari.research-contract/v1", + "title": "Schema Version", + "type": "string" + }, + "selected_candidate_id": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Selected Candidate Id", + "type": "string" + }, + "source_snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Snapshot Digest", + "type": "string" + }, + "title": { + "maxLength": 512, + "minLength": 1, + "title": "Title", + "type": "string" + } + }, + "required": [ + "contract_digest", + "idea_set_digest", + "selected_candidate_id", + "source_snapshot_digest", + "title", + "hypothesis", + "experiment_plan", + "falsification_conditions", + "metric_contract", + "citations", + "limitations", + "generation_lock_digest" + ], + "title": "ARI Research Contract v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/result_envelope_v1.schema.json b/ari-core/ari/schemas/result_envelope_v1.schema.json new file mode 100644 index 00000000..0d33cd66 --- /dev/null +++ b/ari-core/ari/schemas/result_envelope_v1.schema.json @@ -0,0 +1,407 @@ +{ + "$defs": { + "AsyncStateMapV1": { + "additionalProperties": false, + "description": "Case-insensitive provider state mapping used while polling a handle.", + "properties": { + "cancelled_states": { + "items": { + "type": "string" + }, + "title": "Cancelled States", + "type": "array" + }, + "failed_states": { + "items": { + "type": "string" + }, + "title": "Failed States", + "type": "array" + }, + "running_states": { + "items": { + "type": "string" + }, + "title": "Running States", + "type": "array" + }, + "submitted_states": { + "items": { + "type": "string" + }, + "title": "Submitted States", + "type": "array" + }, + "succeeded_states": { + "items": { + "type": "string" + }, + "title": "Succeeded States", + "type": "array" + } + }, + "title": "AsyncStateMapV1", + "type": "object" + }, + "AsyncToolEndpointV1": { + "additionalProperties": false, + "description": "One immutable endpoint embedded in a runtime async handle.", + "properties": { + "handle_argument": { + "title": "Handle Argument", + "type": "string" + }, + "tool_ref": { + "minLength": 1, + "title": "Tool Ref", + "type": "string" + } + }, + "required": [ + "tool_ref", + "handle_argument" + ], + "title": "AsyncToolEndpointV1", + "type": "object" + }, + "AsyncToolHandleV1": { + "additionalProperties": false, + "description": "Portable handle bound to immutable lifecycle tool references.", + "properties": { + "cancel": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "handle_id": { + "maxLength": 1024, + "minLength": 1, + "title": "Handle Id", + "type": "string" + }, + "max_wait_seconds": { + "maximum": 2678400, + "minimum": 1, + "title": "Max Wait Seconds", + "type": "integer" + }, + "poll_interval_seconds": { + "maximum": 3600, + "minimum": 0.01, + "title": "Poll Interval Seconds", + "type": "number" + }, + "result": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "ari.async-tool-handle/v1", + "default": "ari.async-tool-handle/v1", + "title": "Schema Version", + "type": "string" + }, + "state_field": { + "title": "State Field", + "type": "string" + }, + "states": { + "$ref": "#/$defs/AsyncStateMapV1" + }, + "status": { + "$ref": "#/$defs/AsyncToolEndpointV1" + }, + "submission_tool_ref": { + "minLength": 1, + "title": "Submission Tool Ref", + "type": "string" + }, + "submitted_at": { + "minLength": 1, + "title": "Submitted At", + "type": "string" + } + }, + "required": [ + "handle_id", + "submission_tool_ref", + "status", + "state_field", + "states", + "poll_interval_seconds", + "max_wait_seconds", + "submitted_at" + ], + "title": "AsyncToolHandleV1", + "type": "object" + }, + "ResultArtifactV1": { + "additionalProperties": false, + "description": "Content-addressed artifact descriptor stored relative to a run store.", + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "logical_name": { + "minLength": 1, + "title": "Logical Name", + "type": "string" + }, + "logical_role": { + "minLength": 1, + "title": "Logical Role", + "type": "string" + }, + "media_type": { + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "schema_version": { + "const": "ari.artifact-ref/v1", + "default": "ari.artifact-ref/v1", + "title": "Schema Version", + "type": "string" + }, + "size": { + "minimum": 0, + "title": "Size", + "type": "integer" + } + }, + "required": [ + "digest", + "media_type", + "size", + "logical_role", + "logical_name" + ], + "title": "ResultArtifactV1", + "type": "object" + }, + "ResultErrorV1": { + "additionalProperties": false, + "description": "Typed failure information independent of provider-specific wording.", + "properties": { + "details": { + "additionalProperties": true, + "title": "Details", + "type": "object" + }, + "kind": { + "enum": [ + "tool", + "transport", + "protocol", + "timeout", + "cancelled", + "admission", + "artifact-integrity", + "unknown" + ], + "title": "Kind", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "retryable": { + "default": false, + "title": "Retryable", + "type": "boolean" + } + }, + "required": [ + "kind", + "message" + ], + "title": "ResultErrorV1", + "type": "object" + }, + "ResultProvenanceV1": { + "additionalProperties": false, + "description": "Identity and timing recorded for one normalized tool result.", + "properties": { + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Completed At" + }, + "credential_scope_ids": { + "items": { + "type": "string" + }, + "title": "Credential Scope Ids", + "type": "array" + }, + "duration_ms": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + }, + "node_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Node Id" + }, + "phase": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Phase" + }, + "response_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Response Digest", + "type": "string" + }, + "run_id": { + "default": "", + "title": "Run Id", + "type": "string" + }, + "selection_reason": { + "default": "", + "title": "Selection Reason", + "type": "string" + }, + "started_at": { + "minLength": 1, + "title": "Started At", + "type": "string" + }, + "tool_ref": { + "minLength": 1, + "title": "Tool Ref", + "type": "string" + } + }, + "required": [ + "tool_ref", + "started_at", + "response_digest" + ], + "title": "ResultProvenanceV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/result-envelope-v1.schema.json", + "additionalProperties": false, + "description": "Bounded, provider-neutral result returned by ARI's typed dispatch API.", + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/ResultArtifactV1" + }, + "title": "Artifacts", + "type": "array" + }, + "async_handle": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncToolHandleV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "content": { + "default": "", + "title": "Content", + "type": "string" + }, + "content_truncated": { + "default": false, + "title": "Content Truncated", + "type": "boolean" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ResultErrorV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "provenance": { + "$ref": "#/$defs/ResultProvenanceV1" + }, + "schema_version": { + "const": "ari.result-envelope/v1", + "default": "ari.result-envelope/v1", + "title": "Schema Version", + "type": "string" + }, + "status": { + "enum": [ + "ok", + "error", + "submitted", + "running", + "cancelled" + ], + "title": "Status", + "type": "string" + }, + "structured_content": { + "additionalProperties": true, + "title": "Structured Content", + "type": "object" + } + }, + "required": [ + "status", + "provenance" + ], + "title": "ARI Result Envelope v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/retrieval_record_v1.schema.json b/ari-core/ari/schemas/retrieval_record_v1.schema.json new file mode 100644 index 00000000..5c9ef686 --- /dev/null +++ b/ari-core/ari/schemas/retrieval_record_v1.schema.json @@ -0,0 +1,178 @@ +{ + "$id": "https://ari.dev/schemas/retrieval-record-v1.schema.json", + "additionalProperties": false, + "description": "Provider-neutral, content-addressed literature or web record.", + "properties": { + "abstract": { + "default": "", + "maxLength": 100000, + "title": "Abstract", + "type": "string" + }, + "aliases": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Aliases", + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Authors", + "type": "array" + }, + "canonical_id": { + "maxLength": 256, + "minLength": 1, + "title": "Canonical Id", + "type": "string" + }, + "citation_count": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Citation Count" + }, + "license": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "License" + }, + "payload_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Payload Digest", + "type": "string" + }, + "provider": { + "maxLength": 128, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "provider_record_id": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider Record Id" + }, + "provider_version": { + "anyOf": [ + { + "maxLength": 128, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider Version" + }, + "query": { + "maxLength": 4096, + "minLength": 1, + "title": "Query", + "type": "string" + }, + "retrieved_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retrieved At" + }, + "schema_version": { + "const": "ari.retrieval-record/v1", + "default": "ari.retrieval-record/v1", + "title": "Schema Version", + "type": "string" + }, + "source_url": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Url" + }, + "title": { + "maxLength": 2048, + "minLength": 1, + "title": "Title", + "type": "string" + }, + "use_restriction": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use Restriction" + }, + "year": { + "anyOf": [ + { + "maximum": 9999, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Year" + } + }, + "required": [ + "canonical_id", + "provider", + "query", + "title", + "payload_digest" + ], + "title": "ARI Retrieval Record v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/run_comparison_request_v1.schema.json b/ari-core/ari/schemas/run_comparison_request_v1.schema.json new file mode 100644 index 00000000..41283a25 --- /dev/null +++ b/ari-core/ari/schemas/run_comparison_request_v1.schema.json @@ -0,0 +1,214 @@ +{ + "$defs": { + "AnalysisArtifactTargetV1": { + "additionalProperties": false, + "description": "Optional closed-workspace destination for deterministic result artifacts.", + "properties": { + "relative_directory": { + "default": "analysis", + "title": "Relative Directory", + "type": "string" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace" + ], + "title": "AnalysisArtifactTargetV1", + "type": "object" + }, + "RunRecordV1": { + "additionalProperties": false, + "description": "One scalar run outcome with environment and provenance identity.", + "properties": { + "backend_id": { + "title": "Backend Id", + "type": "string" + }, + "environment_digest": { + "title": "Environment Digest", + "type": "string" + }, + "execution_attempt_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Attempt Id" + }, + "input_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Digest" + }, + "library_versions": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 256, + "title": "Library Versions", + "type": "object" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "provenance_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provenance Digest" + }, + "replicate_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Replicate Id" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "unit": { + "title": "Unit", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "run_id", + "metric_id", + "unit", + "value", + "backend_id", + "environment_digest" + ], + "title": "RunRecordV1", + "type": "object" + }, + "WorkspaceRefV1": { + "additionalProperties": false, + "description": "One canonical writable root with no caller-controlled escape path.", + "properties": { + "root": { + "title": "Root", + "type": "string" + }, + "schema_version": { + "const": "ari.workspace-ref/v1", + "default": "ari.workspace-ref/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "root" + ], + "title": "WorkspaceRefV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/run-comparison-request-v1.schema.json", + "additionalProperties": false, + "description": "Rank runs while retaining compatibility and independence caveats.", + "properties": { + "analysis_plan_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Analysis Plan Digest" + }, + "artifact_target": { + "anyOf": [ + { + "$ref": "#/$defs/AnalysisArtifactTargetV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "baseline_run_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Run Id" + }, + "direction": { + "enum": [ + "higher", + "lower" + ], + "title": "Direction", + "type": "string" + }, + "require_compatible_environment": { + "default": true, + "title": "Require Compatible Environment", + "type": "boolean" + }, + "runs": { + "items": { + "$ref": "#/$defs/RunRecordV1" + }, + "maxItems": 100000, + "minItems": 2, + "title": "Runs", + "type": "array" + }, + "schema_version": { + "const": "ari.run-comparison-request/v1", + "default": "ari.run-comparison-request/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "runs", + "direction" + ], + "title": "ARI Run Comparison Request v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/science_data_v1.schema.json b/ari-core/ari/schemas/science_data_v1.schema.json new file mode 100644 index 00000000..b4288bf3 --- /dev/null +++ b/ari-core/ari/schemas/science_data_v1.schema.json @@ -0,0 +1,1090 @@ +{ + "$defs": { + "MeasurementRecordV1": { + "additionalProperties": false, + "properties": { + "artifact_digests": { + "items": { + "type": "string" + }, + "title": "Artifact Digests", + "type": "array" + }, + "execution_attempt_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Attempt Id" + }, + "execution_identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Identity" + }, + "execution_status": { + "default": "unreported", + "enum": [ + "completed", + "failed", + "timed_out", + "cancelled", + "unreported" + ], + "title": "Execution Status", + "type": "string" + }, + "exit_code": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exit Code" + }, + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "provenance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provenance" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unit" + }, + "unit_status": { + "enum": [ + "declared", + "missing" + ], + "title": "Unit Status", + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "title": "Value" + } + }, + "required": [ + "metric_id", + "value", + "unit_status" + ], + "title": "MeasurementRecordV1", + "type": "object" + }, + "ScienceArtifactRefV1": { + "additionalProperties": false, + "description": "A content-addressed input or output under the run checkpoint.", + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "maxLength": 128, + "minLength": 1, + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.science-artifact-ref/v1", + "default": "ari.science-artifact-ref/v1", + "title": "Schema Version", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "relative_path", + "digest", + "media_type", + "role", + "size_bytes" + ], + "title": "ScienceArtifactRefV1", + "type": "object" + }, + "ScienceClaimEvidenceV1": { + "additionalProperties": false, + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + "maxItems": 10000, + "title": "Artifacts", + "type": "array" + }, + "figures": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Figures", + "type": "array" + }, + "nodes": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "title": "Nodes", + "type": "array" + }, + "results": { + "items": { + "$ref": "#/$defs/ScienceEvidenceResultV1" + }, + "maxItems": 100000, + "title": "Results", + "type": "array" + } + }, + "title": "ScienceClaimEvidenceV1", + "type": "object" + }, + "ScienceClaimV1": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "numeric_assertions": { + "items": { + "$ref": "#/$defs/ScienceNumericAssertionV1" + }, + "maxItems": 10000, + "title": "Numeric Assertions", + "type": "array" + }, + "risk": { + "default": "", + "maxLength": 10000, + "title": "Risk", + "type": "string" + }, + "section": { + "maxLength": 128, + "minLength": 1, + "title": "Section", + "type": "string" + }, + "status": { + "enum": [ + "draft", + "supported", + "unsupported", + "rejected" + ], + "title": "Status", + "type": "string" + }, + "supported_by": { + "$ref": "#/$defs/ScienceClaimEvidenceV1" + }, + "text": { + "maxLength": 100000, + "minLength": 1, + "title": "Text", + "type": "string" + } + }, + "required": [ + "id", + "text", + "section", + "status", + "supported_by" + ], + "title": "ScienceClaimV1", + "type": "object" + }, + "ScienceConfigurationV1": { + "additionalProperties": false, + "description": "One executed configuration and its exact evidence sources.", + "properties": { + "claim_eligible": { + "title": "Claim Eligible", + "type": "boolean" + }, + "config_id": { + "title": "Config Id", + "type": "string" + }, + "environment": { + "$ref": "#/$defs/ScienceEnvironmentV1" + }, + "label": { + "default": "", + "maxLength": 256, + "title": "Label", + "type": "string" + }, + "legacy_metrics": { + "additionalProperties": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "maxProperties": 10000, + "title": "Legacy Metrics", + "type": "object" + }, + "measurement_records": { + "items": { + "$ref": "#/$defs/MeasurementRecordV1" + }, + "maxItems": 10000, + "title": "Measurement Records", + "type": "array" + }, + "measurements": { + "additionalProperties": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "maxProperties": 10000, + "title": "Measurements", + "type": "object" + }, + "node_id": { + "title": "Node Id", + "type": "string" + }, + "parameters": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Parameters", + "type": "object" + }, + "predictions": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Predictions", + "type": "object" + }, + "provenance_labels": { + "additionalProperties": { + "type": "string" + }, + "maxProperties": 10000, + "title": "Provenance Labels", + "type": "object" + }, + "rank": { + "minimum": 1, + "title": "Rank", + "type": "integer" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "scores": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Scores", + "type": "object" + }, + "source_artifacts": { + "items": { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + "maxItems": 10000, + "title": "Source Artifacts", + "type": "array" + }, + "source_kind": { + "enum": [ + "typed-measurement", + "legacy-untyped" + ], + "title": "Source Kind", + "type": "string" + } + }, + "required": [ + "config_id", + "run_id", + "node_id", + "rank", + "source_kind", + "claim_eligible" + ], + "title": "ScienceConfigurationV1", + "type": "object" + }, + "ScienceDerivedV1": { + "additionalProperties": false, + "properties": { + "anomalies": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "maxItems": 100000, + "title": "Anomalies", + "type": "array" + }, + "claims": { + "items": { + "$ref": "#/$defs/ScienceClaimV1" + }, + "maxItems": 100000, + "title": "Claims", + "type": "array" + }, + "derived_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Derived Digest", + "type": "string" + }, + "formula_registry_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Formula Registry Digest", + "type": "string" + }, + "metric_summaries": { + "items": { + "$ref": "#/$defs/ScienceMetricSummaryV1" + }, + "maxItems": 10000, + "title": "Metric Summaries", + "type": "array" + }, + "numeric_assertions": { + "items": { + "$ref": "#/$defs/ScienceNumericAssertionV1" + }, + "maxItems": 100000, + "title": "Numeric Assertions", + "type": "array" + }, + "schema_version": { + "const": "ari.science-derived/v1", + "default": "ari.science-derived/v1", + "title": "Schema Version", + "type": "string" + }, + "summary_stats": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Summary Stats", + "type": "object" + } + }, + "required": [ + "derived_digest", + "formula_registry_digest" + ], + "title": "ScienceDerivedV1", + "type": "object" + }, + "ScienceEnvironmentV1": { + "additionalProperties": false, + "properties": { + "arch": { + "default": "", + "maxLength": 256, + "title": "Arch", + "type": "string" + }, + "cpu_model": { + "default": "", + "maxLength": 1024, + "title": "Cpu Model", + "type": "string" + }, + "environment_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Digest" + }, + "executor": { + "default": "", + "maxLength": 256, + "title": "Executor", + "type": "string" + }, + "hostname": { + "default": "", + "maxLength": 256, + "title": "Hostname", + "type": "string" + }, + "scheduler_job_id": { + "default": "", + "maxLength": 256, + "title": "Scheduler Job Id", + "type": "string" + }, + "scheduler_partition": { + "default": "", + "maxLength": 256, + "title": "Scheduler Partition", + "type": "string" + } + }, + "title": "ScienceEnvironmentV1", + "type": "object" + }, + "ScienceEvidenceResultV1": { + "additionalProperties": false, + "properties": { + "metric_path": { + "maxLength": 1024, + "minLength": 1, + "title": "Metric Path", + "type": "string" + }, + "node_id": { + "title": "Node Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "node_id", + "metric_path" + ], + "title": "ScienceEvidenceResultV1", + "type": "object" + }, + "ScienceInterpretationV1": { + "additionalProperties": false, + "description": "Non-authoritative model annotation; never a numeric evidence source.", + "properties": { + "claim_eligible": { + "const": false, + "default": false, + "title": "Claim Eligible", + "type": "boolean" + }, + "error_kind": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error Kind" + }, + "error_message": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error Message" + }, + "evaluation_protocol": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Evaluation Protocol", + "type": "object" + }, + "experiment_context": { + "additionalProperties": true, + "maxProperties": 10000, + "title": "Experiment Context", + "type": "object" + }, + "implementation_overview": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Implementation Overview" + }, + "input_raw_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Input Raw Digest", + "type": "string" + }, + "interpretation_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Interpretation Digest", + "type": "string" + }, + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Digest" + }, + "provider": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "raw_response_artifact": { + "anyOf": [ + { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "sampling": { + "additionalProperties": true, + "maxProperties": 128, + "title": "Sampling", + "type": "object" + }, + "schema_version": { + "const": "ari.science-interpretation/v1", + "default": "ari.science-interpretation/v1", + "title": "Schema Version", + "type": "string" + }, + "status": { + "enum": [ + "ok", + "unavailable", + "invalid", + "legacy-migrated" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "interpretation_digest", + "status", + "input_raw_digest" + ], + "title": "ScienceInterpretationV1", + "type": "object" + }, + "ScienceMetricSummaryV1": { + "additionalProperties": false, + "properties": { + "best_value": { + "title": "Best Value", + "type": "number" + }, + "count": { + "minimum": 1, + "title": "Count", + "type": "integer" + }, + "direction": { + "default": "unspecified", + "enum": [ + "higher", + "lower", + "unspecified" + ], + "title": "Direction", + "type": "string" + }, + "maximum": { + "title": "Maximum", + "type": "number" + }, + "metric_id": { + "maxLength": 256, + "minLength": 1, + "title": "Metric Id", + "type": "string" + }, + "minimum": { + "title": "Minimum", + "type": "number" + }, + "source_config_ids": { + "items": { + "type": "string" + }, + "maxItems": 100000, + "minItems": 1, + "title": "Source Config Ids", + "type": "array" + }, + "unit": { + "default": "unknown", + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + } + }, + "required": [ + "metric_id", + "minimum", + "maximum", + "best_value", + "count", + "source_config_ids" + ], + "title": "ScienceMetricSummaryV1", + "type": "object" + }, + "ScienceNumericAssertionV1": { + "additionalProperties": false, + "properties": { + "aggregation": { + "additionalProperties": true, + "maxProperties": 128, + "title": "Aggregation", + "type": "object" + }, + "claim_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Claim Id" + }, + "cross_environment": { + "default": false, + "title": "Cross Environment", + "type": "boolean" + }, + "formula": { + "title": "Formula", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metric": { + "maxLength": 256, + "minLength": 1, + "title": "Metric", + "type": "string" + }, + "operands": { + "additionalProperties": { + "$ref": "#/$defs/ScienceOperandV1" + }, + "title": "Operands", + "type": "object" + }, + "text_span": { + "default": "", + "maxLength": 4096, + "title": "Text Span", + "type": "string" + }, + "tolerance": { + "additionalProperties": { + "type": "number" + }, + "title": "Tolerance", + "type": "object" + }, + "unit": { + "maxLength": 128, + "minLength": 1, + "title": "Unit", + "type": "string" + }, + "value": { + "title": "Value", + "type": "number" + } + }, + "required": [ + "id", + "metric", + "value", + "unit", + "formula", + "operands" + ], + "title": "ScienceNumericAssertionV1", + "type": "object" + }, + "ScienceOperandV1": { + "additionalProperties": false, + "properties": { + "environment": { + "$ref": "#/$defs/ScienceEnvironmentV1" + }, + "metric_path": { + "maxLength": 1024, + "minLength": 1, + "title": "Metric Path", + "type": "string" + }, + "node_id": { + "title": "Node Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "node_id", + "metric_path" + ], + "title": "ScienceOperandV1", + "type": "object" + }, + "ScienceProvenanceV1": { + "additionalProperties": false, + "properties": { + "admission_artifacts": { + "items": { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + "maxItems": 100000, + "title": "Admission Artifacts", + "type": "array" + }, + "cassette_artifacts": { + "items": { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + "maxItems": 100000, + "title": "Cassette Artifacts", + "type": "array" + }, + "catalog_lock": { + "anyOf": [ + { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "input_artifacts": { + "items": { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + "maxItems": 100000, + "minItems": 1, + "title": "Input Artifacts", + "type": "array" + }, + "producer_tool_ref": { + "title": "Producer Tool Ref", + "type": "string" + }, + "producer_version": { + "maxLength": 256, + "minLength": 1, + "title": "Producer Version", + "type": "string" + }, + "schema_version": { + "const": "ari.science-provenance/v1", + "default": "ari.science-provenance/v1", + "title": "Schema Version", + "type": "string" + }, + "skills_lock": { + "anyOf": [ + { + "$ref": "#/$defs/ScienceArtifactRefV1" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "producer_tool_ref", + "producer_version", + "input_artifacts" + ], + "title": "ScienceProvenanceV1", + "type": "object" + }, + "ScienceRawV1": { + "additionalProperties": false, + "properties": { + "configurations": { + "items": { + "$ref": "#/$defs/ScienceConfigurationV1" + }, + "maxItems": 100000, + "minItems": 1, + "title": "Configurations", + "type": "array" + }, + "measurement_status": { + "enum": [ + "complete", + "partial", + "missing", + "legacy" + ], + "title": "Measurement Status", + "type": "string" + }, + "node_report_status": { + "enum": [ + "complete", + "partial", + "missing", + "legacy" + ], + "title": "Node Report Status", + "type": "string" + }, + "raw_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Raw Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.science-raw/v1", + "default": "ari.science-raw/v1", + "title": "Schema Version", + "type": "string" + }, + "tree_artifact": { + "$ref": "#/$defs/ScienceArtifactRefV1" + } + }, + "required": [ + "raw_digest", + "tree_artifact", + "configurations", + "node_report_status", + "measurement_status" + ], + "title": "ScienceRawV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/science-data-v1.schema.json", + "additionalProperties": false, + "description": "Immutable hand-off from executed experiments to figures and papers.", + "properties": { + "derived": { + "$ref": "#/$defs/ScienceDerivedV1" + }, + "deterministic_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Deterministic Digest", + "type": "string" + }, + "interpretation": { + "$ref": "#/$defs/ScienceInterpretationV1" + }, + "limitations": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Limitations", + "type": "array" + }, + "metric_contract": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Metric Contract" + }, + "migration_status": { + "default": "native-v1", + "enum": [ + "native-v1", + "legacy-explicit" + ], + "title": "Migration Status", + "type": "string" + }, + "provenance": { + "$ref": "#/$defs/ScienceProvenanceV1" + }, + "raw": { + "$ref": "#/$defs/ScienceRawV1" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "const": "ari.science-data/v1", + "default": "ari.science-data/v1", + "title": "Schema Version", + "type": "string" + }, + "science_data_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Science Data Digest", + "type": "string" + } + }, + "required": [ + "run_id", + "raw", + "derived", + "interpretation", + "provenance", + "deterministic_digest", + "science_data_digest" + ], + "title": "ARI Science Data v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/semantic_review_v1.schema.json b/ari-core/ari/schemas/semantic_review_v1.schema.json new file mode 100644 index 00000000..cef95958 --- /dev/null +++ b/ari-core/ari/schemas/semantic_review_v1.schema.json @@ -0,0 +1,235 @@ +{ + "$defs": { + "SemanticFindingV1": { + "additionalProperties": false, + "properties": { + "message": { + "maxLength": 20000, + "minLength": 1, + "title": "Message", + "type": "string" + }, + "section": { + "maxLength": 256, + "minLength": 1, + "title": "Section", + "type": "string" + }, + "type": { + "enum": [ + "overclaim", + "overgeneralization", + "unsupported_claim", + "interpretation", + "visual_semantics" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "section", + "message" + ], + "title": "SemanticFindingV1", + "type": "object" + }, + "SemanticRevisionV1": { + "additionalProperties": false, + "properties": { + "instruction": { + "maxLength": 20000, + "minLength": 1, + "title": "Instruction", + "type": "string" + }, + "section": { + "maxLength": 256, + "minLength": 1, + "title": "Section", + "type": "string" + } + }, + "required": [ + "section", + "instruction" + ], + "title": "SemanticRevisionV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/semantic-review-v1.schema.json", + "additionalProperties": false, + "description": "Advisory review whose provenance cannot mutate the hard-gate result.", + "properties": { + "detected_overclaim_count": { + "minimum": 0, + "title": "Detected Overclaim Count", + "type": "integer" + }, + "evidence_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Evidence Digest", + "type": "string" + }, + "findings": { + "items": { + "$ref": "#/$defs/SemanticFindingV1" + }, + "title": "Findings", + "type": "array" + }, + "hard_gate_report_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hard Gate Report Digest" + }, + "human_verified_overclaim_precision": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Human Verified Overclaim Precision" + }, + "model": { + "maxLength": 512, + "minLength": 1, + "title": "Model", + "type": "string" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "note": { + "anyOf": [ + { + "maxLength": 20000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Note" + }, + "phase": { + "maxLength": 128, + "minLength": 1, + "title": "Phase", + "type": "string" + }, + "previous_review_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Previous Review Digest" + }, + "prompt_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Prompt Digest", + "type": "string" + }, + "resolved_overclaim_count": { + "default": 0, + "title": "Resolved Overclaim Count", + "type": "integer" + }, + "review_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Review Digest", + "type": "string" + }, + "schema_version": { + "const": "ari.semantic-review/v1", + "default": "ari.semantic-review/v1", + "title": "Schema Version", + "type": "string" + }, + "score_delta": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Score Delta" + }, + "scores": { + "additionalProperties": { + "type": "number" + }, + "title": "Scores", + "type": "object" + }, + "stage": { + "const": "evidence_grounded_semantic_review", + "default": "evidence_grounded_semantic_review", + "title": "Stage", + "type": "string" + }, + "status": { + "enum": [ + "ok", + "revise", + "unavailable" + ], + "title": "Status", + "type": "string" + }, + "suggested_revisions": { + "items": { + "$ref": "#/$defs/SemanticRevisionV1" + }, + "title": "Suggested Revisions", + "type": "array" + } + }, + "required": [ + "review_digest", + "phase", + "status", + "model", + "prompt_digest", + "evidence_digest", + "detected_overclaim_count" + ], + "title": "ARI Semantic Review v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/skill_manifest_v1.schema.json b/ari-core/ari/schemas/skill_manifest_v1.schema.json new file mode 100644 index 00000000..650014b2 --- /dev/null +++ b/ari-core/ari/schemas/skill_manifest_v1.schema.json @@ -0,0 +1,566 @@ +{ + "$defs": { + "AsyncLifecycleV1": { + "additionalProperties": false, + "description": "Manifest-declared submit/status/result/cancel protocol.", + "properties": { + "cancel": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncOperationV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "handle_field": { + "default": "handle_id", + "title": "Handle Field", + "type": "string" + }, + "max_wait_seconds": { + "default": 86400, + "maximum": 2678400, + "minimum": 1, + "title": "Max Wait Seconds", + "type": "integer" + }, + "poll_interval_seconds": { + "default": 5.0, + "maximum": 3600, + "minimum": 0.01, + "title": "Poll Interval Seconds", + "type": "number" + }, + "result": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncOperationV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "state_field": { + "default": "status", + "title": "State Field", + "type": "string" + }, + "states": { + "$ref": "#/$defs/AsyncStateMapV1" + }, + "status": { + "$ref": "#/$defs/AsyncOperationV1" + } + }, + "required": [ + "status" + ], + "title": "AsyncLifecycleV1", + "type": "object" + }, + "AsyncOperationV1": { + "additionalProperties": false, + "description": "Manifest reference to one operation in an async lifecycle.", + "properties": { + "capability_ref": { + "title": "Capability Ref", + "type": "string" + }, + "handle_argument": { + "default": "handle_id", + "title": "Handle Argument", + "type": "string" + } + }, + "required": [ + "capability_ref" + ], + "title": "AsyncOperationV1", + "type": "object" + }, + "AsyncStateMapV1": { + "additionalProperties": false, + "description": "Case-insensitive provider state mapping used while polling a handle.", + "properties": { + "cancelled_states": { + "items": { + "type": "string" + }, + "title": "Cancelled States", + "type": "array" + }, + "failed_states": { + "items": { + "type": "string" + }, + "title": "Failed States", + "type": "array" + }, + "running_states": { + "items": { + "type": "string" + }, + "title": "Running States", + "type": "array" + }, + "submitted_states": { + "items": { + "type": "string" + }, + "title": "Submitted States", + "type": "array" + }, + "succeeded_states": { + "items": { + "type": "string" + }, + "title": "Succeeded States", + "type": "array" + } + }, + "title": "AsyncStateMapV1", + "type": "object" + }, + "CredentialScopeV1": { + "additionalProperties": false, + "description": "Named credential authority whose values are never serialized to locks.", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "optional_env": { + "items": { + "type": "string" + }, + "title": "Optional Env", + "type": "array" + }, + "required_env": { + "items": { + "type": "string" + }, + "title": "Required Env", + "type": "array" + } + }, + "required": [ + "id" + ], + "title": "CredentialScopeV1", + "type": "object" + }, + "SkillEntrypointV1": { + "additionalProperties": false, + "description": "How ari-core launches one Skill server.", + "properties": { + "command_kind": { + "const": "python", + "default": "python", + "title": "Command Kind", + "type": "string" + }, + "module": { + "default": "src/server.py", + "title": "Module", + "type": "string" + }, + "transport": { + "const": "stdio", + "default": "stdio", + "title": "Transport", + "type": "string" + } + }, + "title": "SkillEntrypointV1", + "type": "object" + }, + "TimeoutBudgetV1": { + "additionalProperties": false, + "description": "A caller-controlled timeout argument explicitly admitted by a manifest.", + "properties": { + "argument": { + "title": "Argument", + "type": "string" + }, + "maximum_seconds": { + "exclusiveMinimum": 0, + "maximum": 2678400, + "title": "Maximum Seconds", + "type": "integer" + }, + "overhead_seconds": { + "default": 0, + "maximum": 2678400, + "minimum": 0, + "title": "Overhead Seconds", + "type": "integer" + }, + "unit": { + "default": "seconds", + "enum": [ + "seconds", + "minutes", + "hours" + ], + "title": "Unit", + "type": "string" + } + }, + "required": [ + "argument", + "maximum_seconds" + ], + "title": "TimeoutBudgetV1", + "type": "object" + }, + "ToolManifestV1": { + "additionalProperties": false, + "description": "One tool declaration; omitted policy fields inherit ``tool_defaults``.", + "properties": { + "async_lifecycle": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncLifecycleV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "capability_ref": { + "title": "Capability Ref", + "type": "string" + }, + "context_requirement": { + "anyOf": [ + { + "enum": [ + "none", + "run", + "node" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Context Requirement" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "determinism": { + "anyOf": [ + { + "enum": [ + "deterministic", + "conditional", + "stochastic", + "live-data" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Determinism" + }, + "name": { + "title": "Name", + "type": "string" + }, + "permissions": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Permissions" + }, + "phases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Phases" + }, + "result_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Result Schema" + }, + "side_effects": { + "anyOf": [ + { + "enum": [ + "read-only", + "workspace-write", + "stateful", + "destructive" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Side Effects" + }, + "timeout_budget": { + "anyOf": [ + { + "$ref": "#/$defs/TimeoutBudgetV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "timeout_class": { + "anyOf": [ + { + "enum": [ + "default", + "bounded", + "slow", + "very-slow", + "async" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timeout Class" + } + }, + "required": [ + "name", + "capability_ref" + ], + "title": "ToolManifestV1", + "type": "object" + }, + "ToolPolicyV1": { + "additionalProperties": false, + "description": "Policy inherited by tools that do not declare an override.", + "properties": { + "async_lifecycle": { + "anyOf": [ + { + "$ref": "#/$defs/AsyncLifecycleV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "context_requirement": { + "default": "none", + "enum": [ + "none", + "run", + "node" + ], + "title": "Context Requirement", + "type": "string" + }, + "determinism": { + "default": "conditional", + "enum": [ + "deterministic", + "conditional", + "stochastic", + "live-data" + ], + "title": "Determinism", + "type": "string" + }, + "permissions": { + "items": { + "type": "string" + }, + "title": "Permissions", + "type": "array" + }, + "phases": { + "items": { + "type": "string" + }, + "title": "Phases", + "type": "array" + }, + "result_schema": { + "default": "ari.result-envelope/v1", + "title": "Result Schema", + "type": "string" + }, + "side_effects": { + "default": "read-only", + "enum": [ + "read-only", + "workspace-write", + "stateful", + "destructive" + ], + "title": "Side Effects", + "type": "string" + }, + "timeout_budget": { + "anyOf": [ + { + "$ref": "#/$defs/TimeoutBudgetV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "timeout_class": { + "default": "default", + "enum": [ + "default", + "bounded", + "slow", + "very-slow", + "async" + ], + "title": "Timeout Class", + "type": "string" + } + }, + "title": "ToolPolicyV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/skill-manifest-v1.schema.json", + "additionalProperties": false, + "description": "Canonical ARI Skill package manifest, schema version 1.", + "properties": { + "credential_scopes": { + "items": { + "$ref": "#/$defs/CredentialScopeV1" + }, + "title": "Credential Scopes", + "type": "array" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "display_name": { + "default": "", + "title": "Display Name", + "type": "string" + }, + "enabled_by_default": { + "default": true, + "title": "Enabled By Default", + "type": "boolean" + }, + "entrypoint": { + "$ref": "#/$defs/SkillEntrypointV1" + }, + "environment_policy": { + "default": "audit-pending", + "enum": [ + "audit-pending", + "complete" + ], + "title": "Environment Policy", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "optional_env": { + "items": { + "type": "string" + }, + "title": "Optional Env", + "type": "array" + }, + "package": { + "title": "Package", + "type": "string" + }, + "required_env": { + "items": { + "type": "string" + }, + "title": "Required Env", + "type": "array" + }, + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "tool_defaults": { + "$ref": "#/$defs/ToolPolicyV1" + }, + "tools": { + "items": { + "$ref": "#/$defs/ToolManifestV1" + }, + "title": "Tools", + "type": "array" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "schema_version", + "name", + "package", + "version", + "tools" + ], + "title": "ARI Skill Manifest v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/skills_lock_v1.schema.json b/ari-core/ari/schemas/skills_lock_v1.schema.json new file mode 100644 index 00000000..32e7e8d0 --- /dev/null +++ b/ari-core/ari/schemas/skills_lock_v1.schema.json @@ -0,0 +1,244 @@ +{ + "$defs": { + "LockedCredentialScopeV1": { + "additionalProperties": false, + "description": "Value-free identity of credential authority available to one provider.", + "properties": { + "declared_env": { + "items": { + "type": "string" + }, + "title": "Declared Env", + "type": "array" + }, + "identity_digest": { + "title": "Identity Digest", + "type": "string" + }, + "present_env": { + "items": { + "type": "string" + }, + "title": "Present Env", + "type": "array" + }, + "scope_id": { + "title": "Scope Id", + "type": "string" + } + }, + "required": [ + "scope_id", + "identity_digest" + ], + "title": "LockedCredentialScopeV1", + "type": "object" + }, + "LockedSkillV1": { + "additionalProperties": false, + "description": "One configured provider and the digest of its admitted live surface.", + "properties": { + "configured_phases": { + "items": { + "type": "string" + }, + "title": "Configured Phases", + "type": "array" + }, + "credential_scopes": { + "items": { + "$ref": "#/$defs/LockedCredentialScopeV1" + }, + "title": "Credential Scopes", + "type": "array" + }, + "entrypoint": { + "title": "Entrypoint", + "type": "string" + }, + "environment_policy": { + "enum": [ + "audit-pending", + "complete" + ], + "title": "Environment Policy", + "type": "string" + }, + "manifest_digest": { + "title": "Manifest Digest", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "optional_env": { + "items": { + "type": "string" + }, + "title": "Optional Env", + "type": "array" + }, + "package": { + "title": "Package", + "type": "string" + }, + "provider_digest": { + "title": "Provider Digest", + "type": "string" + }, + "required_env": { + "items": { + "type": "string" + }, + "title": "Required Env", + "type": "array" + }, + "tool_refs": { + "items": { + "type": "string" + }, + "title": "Tool Refs", + "type": "array" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "package", + "version", + "entrypoint", + "manifest_digest", + "provider_digest", + "configured_phases", + "environment_policy" + ], + "title": "LockedSkillV1", + "type": "object" + }, + "LockedToolV1": { + "additionalProperties": false, + "description": "One live MCP tool bound to its provider, policy, and exact JSON Schemas.", + "properties": { + "capability_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Capability Ref" + }, + "input_schema": { + "additionalProperties": true, + "title": "Input Schema", + "type": "object" + }, + "input_schema_digest": { + "title": "Input Schema Digest", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "output_schema": { + "additionalProperties": true, + "title": "Output Schema", + "type": "object" + }, + "output_schema_digest": { + "title": "Output Schema Digest", + "type": "string" + }, + "policy": { + "additionalProperties": true, + "title": "Policy", + "type": "object" + }, + "skill_name": { + "title": "Skill Name", + "type": "string" + }, + "tool_ref": { + "title": "Tool Ref", + "type": "string" + } + }, + "required": [ + "tool_ref", + "name", + "skill_name", + "input_schema_digest", + "output_schema_digest" + ], + "title": "LockedToolV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/skills-lock-v1.schema.json", + "additionalProperties": false, + "description": "Canonical, deterministic run snapshot persisted as ``SKILLS.lock``.", + "properties": { + "disabled_tools": { + "items": { + "type": "string" + }, + "title": "Disabled Tools", + "type": "array" + }, + "phase_active_tools": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "title": "Phase Active Tools", + "type": "object" + }, + "registry_digest": { + "title": "Registry Digest", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "const": "ari.skills-lock/v1", + "default": "ari.skills-lock/v1", + "title": "Schema Version", + "type": "string" + }, + "skills": { + "items": { + "$ref": "#/$defs/LockedSkillV1" + }, + "title": "Skills", + "type": "array" + }, + "tools": { + "items": { + "$ref": "#/$defs/LockedToolV1" + }, + "title": "Tools", + "type": "array" + } + }, + "required": [ + "run_id", + "registry_digest", + "skills", + "tools", + "phase_active_tools" + ], + "title": "ARI Skills Lock v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/statistical_test_request_v1.schema.json b/ari-core/ari/schemas/statistical_test_request_v1.schema.json new file mode 100644 index 00000000..542fb23f --- /dev/null +++ b/ari-core/ari/schemas/statistical_test_request_v1.schema.json @@ -0,0 +1,401 @@ +{ + "$defs": { + "AnalysisArtifactTargetV1": { + "additionalProperties": false, + "description": "Optional closed-workspace destination for deterministic result artifacts.", + "properties": { + "relative_directory": { + "default": "analysis", + "title": "Relative Directory", + "type": "string" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace" + ], + "title": "AnalysisArtifactTargetV1", + "type": "object" + }, + "AnalysisDataSourceV1": { + "additionalProperties": false, + "description": "Digest-bound numeric column in a closed workspace.", + "properties": { + "backend_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Backend Id Column" + }, + "environment_digest_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Digest Column" + }, + "expected_digest": { + "title": "Expected Digest", + "type": "string" + }, + "format": { + "default": "auto", + "enum": [ + "auto", + "csv", + "json", + "npy" + ], + "title": "Format", + "type": "string" + }, + "max_bytes": { + "default": 67108864, + "maximum": 268435456, + "minimum": 1, + "title": "Max Bytes", + "type": "integer" + }, + "pair_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pair Id Column" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "replicate_id_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Replicate Id Column" + }, + "value_column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Value Column" + }, + "workspace": { + "$ref": "#/$defs/WorkspaceRefV1" + } + }, + "required": [ + "workspace", + "relative_path", + "expected_digest" + ], + "title": "AnalysisDataSourceV1", + "type": "object" + }, + "AnalysisObservationV1": { + "additionalProperties": false, + "description": "One observed value and the identities needed to assess independence.", + "properties": { + "backend_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Backend Id" + }, + "environment_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Digest" + }, + "pair_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pair Id" + }, + "replicate_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Replicate Id" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "AnalysisObservationV1", + "type": "object" + }, + "MetricSampleSetV1": { + "additionalProperties": false, + "description": "A unit-bearing metric sample supplied inline or by immutable source.", + "properties": { + "metric_id": { + "title": "Metric Id", + "type": "string" + }, + "observations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/AnalysisObservationV1" + }, + "maxItems": 1000000, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Observations" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/AnalysisDataSourceV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "unit": { + "title": "Unit", + "type": "string" + } + }, + "required": [ + "metric_id", + "unit" + ], + "title": "MetricSampleSetV1", + "type": "object" + }, + "StatisticalComparisonV1": { + "additionalProperties": false, + "description": "One pre-declared two-sample or paired comparison.", + "properties": { + "alpha": { + "default": 0.05, + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Alpha", + "type": "number" + }, + "alternative": { + "default": "two-sided", + "enum": [ + "two-sided", + "less", + "greater" + ], + "title": "Alternative", + "type": "string" + }, + "comparison_id": { + "title": "Comparison Id", + "type": "string" + }, + "confidence_level": { + "default": 0.95, + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Confidence Level", + "type": "number" + }, + "group_a": { + "$ref": "#/$defs/MetricSampleSetV1" + }, + "group_b": { + "$ref": "#/$defs/MetricSampleSetV1" + }, + "pairing": { + "default": "unpaired", + "enum": [ + "unpaired", + "ordered", + "pair_id" + ], + "title": "Pairing", + "type": "string" + }, + "test_family": { + "default": "auto", + "enum": [ + "auto", + "welch_t", + "student_t", + "paired_t", + "mann_whitney", + "wilcoxon" + ], + "title": "Test Family", + "type": "string" + } + }, + "required": [ + "comparison_id", + "group_a", + "group_b" + ], + "title": "StatisticalComparisonV1", + "type": "object" + }, + "WorkspaceRefV1": { + "additionalProperties": false, + "description": "One canonical writable root with no caller-controlled escape path.", + "properties": { + "root": { + "title": "Root", + "type": "string" + }, + "schema_version": { + "const": "ari.workspace-ref/v1", + "default": "ari.workspace-ref/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "root" + ], + "title": "WorkspaceRefV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/statistical-test-request-v1.schema.json", + "additionalProperties": false, + "description": "A family of comparisons with an explicit multiplicity policy.", + "properties": { + "analysis_plan_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Analysis Plan Digest" + }, + "artifact_target": { + "anyOf": [ + { + "$ref": "#/$defs/AnalysisArtifactTargetV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "comparisons": { + "items": { + "$ref": "#/$defs/StatisticalComparisonV1" + }, + "maxItems": 10000, + "minItems": 1, + "title": "Comparisons", + "type": "array" + }, + "correction": { + "default": "none", + "enum": [ + "none", + "bonferroni", + "holm", + "benjamini_hochberg" + ], + "title": "Correction", + "type": "string" + }, + "missing_policy": { + "default": "error", + "enum": [ + "error", + "drop" + ], + "title": "Missing Policy", + "type": "string" + }, + "schema_version": { + "const": "ari.statistical-test-request/v1", + "default": "ari.statistical-test-request/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "comparisons" + ], + "title": "ARI Statistical Test Request v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/survey_snapshot_v1.schema.json b/ari-core/ari/schemas/survey_snapshot_v1.schema.json new file mode 100644 index 00000000..4c80a46b --- /dev/null +++ b/ari-core/ari/schemas/survey_snapshot_v1.schema.json @@ -0,0 +1,393 @@ +{ + "$defs": { + "CitationEdgeV1": { + "additionalProperties": false, + "properties": { + "provider": { + "maxLength": 128, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "relation": { + "enum": [ + "cites", + "is-cited-by", + "related" + ], + "title": "Relation", + "type": "string" + }, + "schema_version": { + "const": "ari.citation-edge/v1", + "default": "ari.citation-edge/v1", + "title": "Schema Version", + "type": "string" + }, + "source_id": { + "maxLength": 256, + "minLength": 1, + "title": "Source Id", + "type": "string" + }, + "target_id": { + "maxLength": 256, + "minLength": 1, + "title": "Target Id", + "type": "string" + } + }, + "required": [ + "source_id", + "target_id", + "relation", + "provider" + ], + "title": "CitationEdgeV1", + "type": "object" + }, + "ResearchArtifactRefV1": { + "additionalProperties": false, + "description": "Content-addressed research artifact under a checkpoint/workspace root.", + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "logical_name": { + "maxLength": 512, + "minLength": 1, + "title": "Logical Name", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "role": { + "maxLength": 128, + "minLength": 1, + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.research-artifact-ref/v1", + "default": "ari.research-artifact-ref/v1", + "title": "Schema Version", + "type": "string" + }, + "source_run_id": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Run Id" + } + }, + "required": [ + "logical_name", + "digest", + "media_type", + "role" + ], + "title": "ResearchArtifactRefV1", + "type": "object" + }, + "RetrievalRecordV1": { + "additionalProperties": false, + "description": "Provider-neutral, content-addressed literature or web record.", + "properties": { + "abstract": { + "default": "", + "maxLength": 100000, + "title": "Abstract", + "type": "string" + }, + "aliases": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Aliases", + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Authors", + "type": "array" + }, + "canonical_id": { + "maxLength": 256, + "minLength": 1, + "title": "Canonical Id", + "type": "string" + }, + "citation_count": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Citation Count" + }, + "license": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "License" + }, + "payload_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Payload Digest", + "type": "string" + }, + "provider": { + "maxLength": 128, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "provider_record_id": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider Record Id" + }, + "provider_version": { + "anyOf": [ + { + "maxLength": 128, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider Version" + }, + "query": { + "maxLength": 4096, + "minLength": 1, + "title": "Query", + "type": "string" + }, + "retrieved_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retrieved At" + }, + "schema_version": { + "const": "ari.retrieval-record/v1", + "default": "ari.retrieval-record/v1", + "title": "Schema Version", + "type": "string" + }, + "source_url": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Url" + }, + "title": { + "maxLength": 2048, + "minLength": 1, + "title": "Title", + "type": "string" + }, + "use_restriction": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use Restriction" + }, + "year": { + "anyOf": [ + { + "maximum": 9999, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Year" + } + }, + "required": [ + "canonical_id", + "provider", + "query", + "title", + "payload_digest" + ], + "title": "RetrievalRecordV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/survey-snapshot-v1.schema.json", + "additionalProperties": false, + "description": "Exact retrieval input consumed by one idea-generation attempt.", + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/ResearchArtifactRefV1" + }, + "maxItems": 10000, + "title": "Artifacts", + "type": "array" + }, + "byte_reproducible": { + "title": "Byte Reproducible", + "type": "boolean" + }, + "citation_edges": { + "items": { + "$ref": "#/$defs/CitationEdgeV1" + }, + "maxItems": 1000000, + "title": "Citation Edges", + "type": "array" + }, + "mode": { + "enum": [ + "live", + "record", + "replay", + "frozen", + "inline" + ], + "title": "Mode", + "type": "string" + }, + "provider": { + "maxLength": 128, + "minLength": 1, + "title": "Provider", + "type": "string" + }, + "provider_version": { + "anyOf": [ + { + "maxLength": 128, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider Version" + }, + "query": { + "maxLength": 4096, + "minLength": 1, + "title": "Query", + "type": "string" + }, + "records": { + "items": { + "$ref": "#/$defs/RetrievalRecordV1" + }, + "maxItems": 100000, + "title": "Records", + "type": "array" + }, + "retrieved_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retrieved At" + }, + "schema_version": { + "const": "ari.survey-snapshot/v1", + "default": "ari.survey-snapshot/v1", + "title": "Schema Version", + "type": "string" + }, + "snapshot_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Snapshot Digest", + "type": "string" + }, + "warnings": { + "items": { + "type": "string" + }, + "maxItems": 1000, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "snapshot_digest", + "mode", + "provider", + "query", + "byte_reproducible" + ], + "title": "ARI Survey Snapshot v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/visual_review_batch_v1.schema.json b/ari-core/ari/schemas/visual_review_batch_v1.schema.json new file mode 100644 index 00000000..6e88363c --- /dev/null +++ b/ari-core/ari/schemas/visual_review_batch_v1.schema.json @@ -0,0 +1,531 @@ +{ + "$defs": { + "VisualArtifactRefV1": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Digest", + "type": "string" + }, + "media_type": { + "maxLength": 128, + "minLength": 1, + "title": "Media Type", + "type": "string" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "role": { + "enum": [ + "review-target", + "raw-model-response", + "table-source" + ], + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "ari.visual-artifact-ref/v1", + "default": "ari.visual-artifact-ref/v1", + "title": "Schema Version", + "type": "string" + }, + "size_bytes": { + "minimum": 0, + "title": "Size Bytes", + "type": "integer" + } + }, + "required": [ + "role", + "relative_path", + "digest", + "media_type", + "size_bytes" + ], + "title": "VisualArtifactRefV1", + "type": "object" + }, + "VisualIssueV1": { + "additionalProperties": false, + "properties": { + "criterion_id": { + "maxLength": 128, + "minLength": 1, + "title": "Criterion Id", + "type": "string" + }, + "evidence": { + "default": "", + "maxLength": 4096, + "title": "Evidence", + "type": "string" + }, + "issue_id": { + "title": "Issue Id", + "type": "string" + }, + "message": { + "maxLength": 4096, + "minLength": 1, + "title": "Message", + "type": "string" + }, + "page": { + "anyOf": [ + { + "maximum": 100000, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Page" + }, + "region": { + "anyOf": [ + { + "$ref": "#/$defs/VisualRegionV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "severity": { + "enum": [ + "info", + "minor", + "major", + "blocking" + ], + "title": "Severity", + "type": "string" + }, + "suggestion": { + "default": "", + "maxLength": 4096, + "title": "Suggestion", + "type": "string" + } + }, + "required": [ + "issue_id", + "criterion_id", + "severity", + "message" + ], + "title": "VisualIssueV1", + "type": "object" + }, + "VisualModelUsageV1": { + "additionalProperties": false, + "properties": { + "cost_status": { + "default": "unavailable", + "enum": [ + "reported", + "unavailable" + ], + "title": "Cost Status", + "type": "string" + }, + "cost_usd": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cost Usd" + }, + "input_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Tokens" + }, + "output_tokens": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output Tokens" + } + }, + "title": "VisualModelUsageV1", + "type": "object" + }, + "VisualRegionV1": { + "additionalProperties": false, + "properties": { + "height": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Height", + "type": "number" + }, + "width": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Width", + "type": "number" + }, + "x": { + "maximum": 1, + "minimum": 0, + "title": "X", + "type": "number" + }, + "y": { + "maximum": 1, + "minimum": 0, + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "title": "VisualRegionV1", + "type": "object" + }, + "VisualReviewV1": { + "additionalProperties": false, + "properties": { + "context_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Context Digest", + "type": "string" + }, + "criteria_profile_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Criteria Profile Digest", + "type": "string" + }, + "criteria_profile_id": { + "title": "Criteria Profile Id", + "type": "string" + }, + "error_kind": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error Kind" + }, + "error_message": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error Message" + }, + "figure_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Figure Id" + }, + "issues": { + "items": { + "$ref": "#/$defs/VisualIssueV1" + }, + "maxItems": 1000, + "title": "Issues", + "type": "array" + }, + "iteration": { + "default": 0, + "maximum": 2, + "minimum": 0, + "title": "Iteration", + "type": "integer" + }, + "model": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model" + }, + "model_revision": { + "anyOf": [ + { + "maxLength": 512, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Revision" + }, + "prompt_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Digest" + }, + "provider": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "raw_response_artifact": { + "anyOf": [ + { + "$ref": "#/$defs/VisualArtifactRefV1" + }, + { + "type": "null" + } + ], + "default": null + }, + "review_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Review Digest", + "type": "string" + }, + "sampling": { + "additionalProperties": true, + "maxProperties": 64, + "title": "Sampling", + "type": "object" + }, + "schema_version": { + "const": "ari.visual-review/v1", + "default": "ari.visual-review/v1", + "title": "Schema Version", + "type": "string" + }, + "score": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Score" + }, + "source_manifest_digest": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Manifest Digest" + }, + "status": { + "enum": [ + "completed", + "artifact-error", + "limit-error", + "model-error", + "schema-error" + ], + "title": "Status", + "type": "string" + }, + "summary": { + "default": "", + "maxLength": 8192, + "title": "Summary", + "type": "string" + }, + "target_artifact": { + "$ref": "#/$defs/VisualArtifactRefV1" + }, + "target_id": { + "title": "Target Id", + "type": "string" + }, + "target_kind": { + "enum": [ + "figure", + "table" + ], + "title": "Target Kind", + "type": "string" + }, + "usage": { + "$ref": "#/$defs/VisualModelUsageV1" + } + }, + "required": [ + "target_kind", + "target_id", + "target_artifact", + "context_digest", + "criteria_profile_id", + "criteria_profile_digest", + "status", + "review_digest" + ], + "title": "VisualReviewV1", + "type": "object" + } + }, + "$id": "https://ari.dev/schemas/visual-review-batch-v1.schema.json", + "additionalProperties": false, + "properties": { + "aggregation": { + "const": "minimum-fail-closed", + "default": "minimum-fail-closed", + "title": "Aggregation", + "type": "string" + }, + "batch_review_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Batch Review Digest", + "type": "string" + }, + "failure_count": { + "minimum": 0, + "title": "Failure Count", + "type": "integer" + }, + "issues": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Issues", + "type": "array" + }, + "iteration": { + "maximum": 2, + "minimum": 0, + "title": "Iteration", + "type": "integer" + }, + "review_text": { + "default": "", + "maxLength": 100000, + "title": "Review Text", + "type": "string" + }, + "reviews": { + "items": { + "$ref": "#/$defs/VisualReviewV1" + }, + "maxItems": 100, + "minItems": 1, + "title": "Reviews", + "type": "array" + }, + "schema_version": { + "const": "ari.visual-review-batch/v1", + "default": "ari.visual-review-batch/v1", + "title": "Schema Version", + "type": "string" + }, + "score": { + "maximum": 1, + "minimum": 0, + "title": "Score", + "type": "number" + }, + "source_batch_digest": { + "pattern": "^sha256:[0-9a-f]{64}$", + "title": "Source Batch Digest", + "type": "string" + }, + "suggestions": { + "items": { + "type": "string" + }, + "maxItems": 10000, + "title": "Suggestions", + "type": "array" + } + }, + "required": [ + "source_batch_digest", + "iteration", + "reviews", + "score", + "failure_count", + "batch_review_digest" + ], + "title": "ARI Visual Review Batch v1", + "type": "object" +} diff --git a/ari-core/ari/schemas/workspace_ref_v1.schema.json b/ari-core/ari/schemas/workspace_ref_v1.schema.json new file mode 100644 index 00000000..1780686b --- /dev/null +++ b/ari-core/ari/schemas/workspace_ref_v1.schema.json @@ -0,0 +1,22 @@ +{ + "$id": "https://ari.dev/schemas/workspace-ref-v1.schema.json", + "additionalProperties": false, + "description": "One canonical writable root with no caller-controlled escape path.", + "properties": { + "root": { + "title": "Root", + "type": "string" + }, + "schema_version": { + "const": "ari.workspace-ref/v1", + "default": "ari.workspace-ref/v1", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "root" + ], + "title": "ARI Workspace Reference v1", + "type": "object" +} diff --git a/ari-core/ari/science_data_base.py b/ari-core/ari/science_data_base.py new file mode 100644 index 00000000..0b3ab6d9 --- /dev/null +++ b/ari-core/ari/science_data_base.py @@ -0,0 +1,144 @@ +"""Shared primitives for versioned scientific-data contracts.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import PurePosixPath +from typing import Any, ClassVar, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + + +SCIENCE_DATA_V1 = "ari.science-data/v1" +SCIENCE_RAW_V1 = "ari.science-raw/v1" +SCIENCE_DERIVED_V1 = "ari.science-derived/v1" +SCIENCE_INTERPRETATION_V1 = "ari.science-interpretation/v1" +SCIENCE_PROVENANCE_V1 = "ari.science-provenance/v1" +SCIENCE_ARTIFACT_REF_V1 = "ari.science-artifact-ref/v1" + +SHA256_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,255}$") +ZERO_DIGEST = "sha256:" + "0" * 64 + + +class ScienceDataError(ValueError): + """The scientific-data record is malformed, unbound, or tampered with.""" + + +def canonical_science_digest(value: Any) -> str: + """Return a stable SHA-256 identity for finite JSON-compatible data.""" + + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def finite_json(value: Any, field: str) -> Any: + try: + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be finite JSON") from exc + return value + + +def safe_id(value: str, field: str) -> str: + if not _SAFE_ID_RE.fullmatch(value): + raise ValueError(f"{field} contains unsupported characters") + return value + + +class StrictScienceModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class DigestBoundScienceModel(StrictScienceModel): + digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values = dict(values) + values[cls.digest_field] = ZERO_DIGEST + return cls.model_validate(values, context={"bind_science_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self.digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_science_digest(self.digest_payload()) + if info.context and info.context.get("bind_science_digest"): + object.__setattr__(self, self.digest_field, expected) + elif getattr(self, self.digest_field) != expected: + raise ValueError( + f"{self.digest_field} does not match the canonical payload" + ) + return self + + +class ScienceArtifactRefV1(StrictScienceModel): + """A content-addressed input or output under the run checkpoint.""" + + schema_version: Literal["ari.science-artifact-ref/v1"] = SCIENCE_ARTIFACT_REF_V1 + relative_path: str + digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + media_type: str = Field(min_length=1, max_length=128) + role: str = Field(min_length=1, max_length=128) + size_bytes: int = Field(ge=0) + + @field_validator("relative_path") + @classmethod + def _relative_path(cls, value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError("science artifact path must be safe and relative") + return path.as_posix() + + +class ScienceEnvironmentV1(StrictScienceModel): + executor: str = Field(default="", max_length=256) + hostname: str = Field(default="", max_length=256) + cpu_model: str = Field(default="", max_length=1024) + arch: str = Field(default="", max_length=256) + scheduler_job_id: str = Field(default="", max_length=256) + scheduler_partition: str = Field(default="", max_length=256) + environment_digest: str | None = Field(default=None, pattern=SHA256_DIGEST_PATTERN) + + +__all__ = [ + "DigestBoundScienceModel", + "SCIENCE_ARTIFACT_REF_V1", + "SCIENCE_DATA_V1", + "SCIENCE_DERIVED_V1", + "SCIENCE_INTERPRETATION_V1", + "SCIENCE_PROVENANCE_V1", + "SCIENCE_RAW_V1", + "SHA256_DIGEST_PATTERN", + "ZERO_DIGEST", + "ScienceArtifactRefV1", + "ScienceDataError", + "ScienceEnvironmentV1", + "StrictScienceModel", + "canonical_science_digest", + "finite_json", + "safe_id", +] diff --git a/ari-core/ari/science_data_contract.py b/ari-core/ari/science_data_contract.py new file mode 100644 index 00000000..9a20829f --- /dev/null +++ b/ari-core/ari/science_data_contract.py @@ -0,0 +1,419 @@ +"""Canonical scientific-data hand-off contracts. + +``ScienceDataV1`` deliberately keeps executed facts, deterministic derivations, +and stochastic interpretation in separate digest-bound sections. A model +annotation can therefore be replaced, rejected, or replayed without changing +the identity of the measurements and claims that a paper is allowed to cite. + +The legacy flat ``science_data.json`` shape is supported only by the explicit +``migrate_legacy_science_data`` reader. New producers must emit v1 directly. +""" + +from __future__ import annotations + +import json +import math +from typing import Any, Literal + +from pydantic import ( + Field, + ValidationInfo, + field_validator, + model_validator, +) + +from ari.claim_gate_contract import parse_metric_gate_contract +from ari.execution import MeasurementRecordV1 +from ari.pipeline.claim_gate.numeric import ( + FORMULAS, + formula_registry_digest, + recompute, + required_roles, +) +from ari.science_data_base import ( + DigestBoundScienceModel as _DigestBoundModel, + SCIENCE_ARTIFACT_REF_V1, + SCIENCE_DATA_V1, + SCIENCE_DERIVED_V1, + SCIENCE_INTERPRETATION_V1, + SCIENCE_PROVENANCE_V1, + SCIENCE_RAW_V1, + SHA256_DIGEST_PATTERN, + ZERO_DIGEST as _ZERO_DIGEST, + ScienceArtifactRefV1, + ScienceDataError, + ScienceEnvironmentV1, + StrictScienceModel as _StrictModel, + canonical_science_digest, + finite_json as _finite_json, + safe_id as _safe_id, +) +from ari.science_data_derived import ( + ScienceClaimEvidenceV1, + ScienceClaimV1, + ScienceDerivedV1, + ScienceEvidenceResultV1, + ScienceInterpretationV1, + ScienceMetricSummaryV1, + ScienceNumericAssertionV1, + ScienceOperandV1, + ScienceProvenanceV1, +) + + +ScienceSourceKind = Literal["typed-measurement", "legacy-untyped"] + + +class ScienceConfigurationV1(_StrictModel): + """One executed configuration and its exact evidence sources.""" + + config_id: str + run_id: str + node_id: str + rank: int = Field(ge=1) + label: str = Field(default="", max_length=256) + source_kind: ScienceSourceKind + claim_eligible: bool + parameters: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + measurements: dict[str, int | float] = Field( + default_factory=dict, max_length=10_000 + ) + measurement_records: tuple[MeasurementRecordV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + predictions: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + scores: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + legacy_metrics: dict[str, int | float] = Field( + default_factory=dict, max_length=10_000 + ) + environment: ScienceEnvironmentV1 = Field(default_factory=ScienceEnvironmentV1) + source_artifacts: tuple[ScienceArtifactRefV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + provenance_labels: dict[str, str] = Field(default_factory=dict, max_length=10_000) + + @field_validator("config_id", "run_id", "node_id") + @classmethod + def _ids(cls, value: str, info: ValidationInfo) -> str: + return _safe_id(value, info.field_name) + + @field_validator("parameters", "predictions", "scores") + @classmethod + def _json_fields( + cls, value: dict[str, Any], info: ValidationInfo + ) -> dict[str, Any]: + return _finite_json(value, info.field_name) + + @field_validator("measurements", "legacy_metrics") + @classmethod + def _finite_metrics( + cls, value: dict[str, int | float], info: ValidationInfo + ) -> dict[str, int | float]: + for metric, number in value.items(): + if ( + not metric + or metric.startswith("_") + or isinstance(number, bool) + or not isinstance(number, (int, float)) + or not math.isfinite(float(number)) + ): + raise ValueError(f"{info.field_name} contains an invalid metric") + return value + + @model_validator(mode="after") + def _source_consistent(self) -> "ScienceConfigurationV1": + if self.source_kind == "typed-measurement": + if not self.measurement_records: + raise ValueError( + "typed measurement source requires measurement records" + ) + metric_ids = [record.metric_id for record in self.measurement_records] + if len(metric_ids) != len(set(metric_ids)): + raise ValueError( + "typed measurement records must have unique metric IDs" + ) + if any( + record.execution_status != "completed" + for record in self.measurement_records + ): + raise ValueError( + "claim-eligible measurements must have completed execution" + ) + record_values = { + record.metric_id: float(record.value) + for record in self.measurement_records + } + if set(record_values) != set(self.measurements) or any( + not math.isclose( + record_values[key], + float(self.measurements[key]), + rel_tol=0, + abs_tol=0, + ) + for key in record_values + ): + raise ValueError("measurement projection differs from typed records") + if not self.claim_eligible: + raise ValueError("validated typed measurements must be claim eligible") + if not any( + source.role == "measurement-set" for source in self.source_artifacts + ): + raise ValueError( + "typed measurements require a measurement-set artifact" + ) + elif self.claim_eligible: + raise ValueError("legacy-untyped metrics cannot be claim eligible") + if self.claim_eligible and not self.source_artifacts: + raise ValueError( + "claim-eligible measurements require content-addressed sources" + ) + return self + + +class ScienceRawV1(_DigestBoundModel): + digest_field = "raw_digest" + + schema_version: Literal["ari.science-raw/v1"] = SCIENCE_RAW_V1 + raw_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + tree_artifact: ScienceArtifactRefV1 + configurations: tuple[ScienceConfigurationV1, ...] = Field( + min_length=1, max_length=100_000 + ) + node_report_status: Literal["complete", "partial", "missing", "legacy"] + measurement_status: Literal["complete", "partial", "missing", "legacy"] + + @model_validator(mode="after") + def _unique_configurations(self) -> "ScienceRawV1": + config_ids = [item.config_id for item in self.configurations] + node_ids = [item.node_id for item in self.configurations] + if len(config_ids) != len(set(config_ids)): + raise ValueError("science configuration IDs must be unique") + if len(node_ids) != len(set(node_ids)): + raise ValueError("science node IDs must be unique") + return self + + +class ScienceDataV1(_StrictModel): + """Immutable hand-off from executed experiments to figures and papers.""" + + schema_version: Literal["ari.science-data/v1"] = SCIENCE_DATA_V1 + run_id: str + raw: ScienceRawV1 + derived: ScienceDerivedV1 + interpretation: ScienceInterpretationV1 + metric_contract: dict[str, Any] | None = None + limitations: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + provenance: ScienceProvenanceV1 + migration_status: Literal["native-v1", "legacy-explicit"] = "native-v1" + deterministic_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + science_data_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + + @field_validator("run_id") + @classmethod + def _run_id(cls, value: str) -> str: + return _safe_id(value, "run_id") + + @field_validator("metric_contract") + @classmethod + def _metric_contract(cls, value: dict[str, Any] | None) -> dict[str, Any] | None: + if value is None: + return None + _finite_json(value, "metric_contract") + try: + contract = parse_metric_gate_contract(value) + except ValueError as exc: + raise ValueError( + "metric_contract is not a canonical digest-bound contract" + ) from exc + return contract.model_dump(mode="json") + + @classmethod + def create(cls, **values: Any) -> "ScienceDataV1": + values = dict(values) + values.setdefault("schema_version", SCIENCE_DATA_V1) + values["deterministic_digest"] = _ZERO_DIGEST + values["science_data_digest"] = _ZERO_DIGEST + provisional = cls.model_validate(values, context={"bind_science_data": True}) + return provisional + + def deterministic_payload(self) -> dict[str, Any]: + return self.model_dump( + mode="json", + exclude={"interpretation", "deterministic_digest", "science_data_digest"}, + ) + + def complete_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={"science_data_digest"}) + + @model_validator(mode="after") + def _digests_match(self, info: ValidationInfo) -> "ScienceDataV1": + expected_deterministic = canonical_science_digest(self.deterministic_payload()) + if info.context and info.context.get("bind_science_data"): + object.__setattr__(self, "deterministic_digest", expected_deterministic) + expected_complete = canonical_science_digest(self.complete_payload()) + object.__setattr__(self, "science_data_digest", expected_complete) + else: + if self.deterministic_digest != expected_deterministic: + raise ValueError("deterministic science-data digest mismatch") + expected_complete = canonical_science_digest(self.complete_payload()) + if self.science_data_digest != expected_complete: + raise ValueError("complete science-data digest mismatch") + if self.interpretation.input_raw_digest != self.raw.raw_digest: + raise ValueError("interpretation is not bound to this raw section") + if any(config.run_id != self.run_id for config in self.raw.configurations): + raise ValueError("configuration run identity differs from science-data run") + from ari.science_data_validation import validate_science_data_links + + validate_science_data_links(self) + return self + + +def parse_science_data(value: Any) -> ScienceDataV1: + """Parse native v1 only; legacy input is intentionally not auto-migrated.""" + + if isinstance(value, ScienceDataV1): + return ScienceDataV1.model_validate(value.model_dump(mode="json")) + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ScienceDataError("science data is not valid JSON") from exc + if not isinstance(value, dict) or value.get("schema_version") != SCIENCE_DATA_V1: + raise ScienceDataError( + "science data is not native ari.science-data/v1; use the explicit legacy reader" + ) + try: + return ScienceDataV1.model_validate(value) + except ValueError as exc: + raise ScienceDataError(str(exc)) from exc + + +def science_data_projection( + value: ScienceDataV1 | dict[str, Any] | str, +) -> dict[str, Any]: + """Return the old flat view for gates during the bounded migration window. + + The projection is generated from canonical sections. It never promotes the + interpretation into measurements, summaries, claims, or numeric assertions. + """ + + science = parse_science_data(value) + configurations: list[dict[str, Any]] = [] + config_nodes: dict[str, dict[str, Any]] = {} + for config in science.raw.configurations: + metrics = dict(config.legacy_metrics) + metrics.update(config.measurements) + item: dict[str, Any] = { + "rank": config.rank, + "config_id": config.config_id, + "parameters": config.parameters, + "metrics": metrics, + "label": config.label, + "environment": config.environment.model_dump(mode="json"), + "_source_kind": config.source_kind, + "_claim_eligible": config.claim_eligible, + } + if config.measurements: + item["measurements"] = config.measurements + if config.measurement_records: + item["measurement_records"] = [ + record.model_dump(mode="json") for record in config.measurement_records + ] + if config.predictions: + item["predictions"] = config.predictions + if config.scores: + item["scores"] = config.scores + if config.provenance_labels: + item["_provenance"] = config.provenance_labels + configurations.append(item) + config_nodes[config.config_id] = { + "node_id": config.node_id, + "run_id": config.run_id, + "environment": config.environment.model_dump(mode="json"), + "metrics": metrics, + "claim_eligible": config.claim_eligible, + } + + per_key_summary = { + summary.metric_id: { + "best_value": summary.best_value, + "min": summary.minimum, + "max": summary.maximum, + "n": summary.count, + "unit": summary.unit, + "source_config_ids": list(summary.source_config_ids), + } + for summary in science.derived.metric_summaries + } + claims = [claim.model_dump(mode="json") for claim in science.derived.claims] + numeric = [ + assertion.model_dump(mode="json") + for assertion in science.derived.numeric_assertions + ] + projection: dict[str, Any] = { + "schema_version": science.schema_version, + "science_data_digest": science.science_data_digest, + "deterministic_digest": science.deterministic_digest, + "configurations": configurations, + "per_key_summary": per_key_summary, + "summary_stats": science.derived.summary_stats, + "claims": claims, + "numeric_assertions": numeric, + "_config_nodes": config_nodes, + "_anomalies": list(science.derived.anomalies), + "limitations": list(science.limitations), + "metric_contract": science.metric_contract, + "interpretation": science.interpretation.model_dump(mode="json"), + "migration_status": science.migration_status, + } + return projection + + +def migrate_legacy_science_data( + value: dict[str, Any] | str, + *, + run_id: str, + logical_name: str = "science_data.legacy.json", +) -> ScienceDataV1: + """Explicit offline conversion of the pre-v1 flat checkpoint format.""" + + from ari.science_data_migration import migrate_legacy_document + + return migrate_legacy_document( + value, + run_id=run_id, + logical_name=logical_name, + ) + + +__all__ = [ + "FORMULAS", + "SCIENCE_ARTIFACT_REF_V1", + "SCIENCE_DATA_V1", + "SCIENCE_DERIVED_V1", + "SCIENCE_INTERPRETATION_V1", + "SCIENCE_PROVENANCE_V1", + "SCIENCE_RAW_V1", + "ScienceArtifactRefV1", + "ScienceClaimEvidenceV1", + "ScienceClaimV1", + "ScienceConfigurationV1", + "ScienceDataError", + "ScienceDataV1", + "ScienceDerivedV1", + "ScienceEnvironmentV1", + "ScienceEvidenceResultV1", + "ScienceInterpretationV1", + "ScienceMetricSummaryV1", + "ScienceNumericAssertionV1", + "ScienceOperandV1", + "ScienceProvenanceV1", + "ScienceRawV1", + "canonical_science_digest", + "formula_registry_digest", + "migrate_legacy_science_data", + "parse_science_data", + "recompute", + "required_roles", + "science_data_projection", +] diff --git a/ari-core/ari/science_data_derived.py b/ari-core/ari/science_data_derived.py new file mode 100644 index 00000000..8be769dc --- /dev/null +++ b/ari-core/ari/science_data_derived.py @@ -0,0 +1,322 @@ +"""Derived, interpretation, and provenance sections of ``ScienceDataV1``.""" + +from __future__ import annotations + +import math +from typing import Any, Literal + +from pydantic import Field, ValidationInfo, field_validator, model_validator + +from ari.pipeline.claim_gate.numeric import ( + FORMULAS, + formula_registry_digest, + required_roles, +) +from ari.science_data_base import ( + DigestBoundScienceModel, + SCIENCE_DERIVED_V1, + SCIENCE_INTERPRETATION_V1, + SCIENCE_PROVENANCE_V1, + SHA256_DIGEST_PATTERN, + ScienceArtifactRefV1, + ScienceEnvironmentV1, + StrictScienceModel, + finite_json, + safe_id, +) + + +class ScienceMetricSummaryV1(StrictScienceModel): + metric_id: str = Field(min_length=1, max_length=256) + unit: str = Field(default="unknown", min_length=1, max_length=128) + minimum: float + maximum: float + best_value: float + count: int = Field(ge=1) + direction: Literal["higher", "lower", "unspecified"] = "unspecified" + source_config_ids: tuple[str, ...] = Field(min_length=1, max_length=100_000) + + @field_validator("minimum", "maximum", "best_value") + @classmethod + def _finite(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("derived summary values must be finite") + return value + + @model_validator(mode="after") + def _range(self) -> "ScienceMetricSummaryV1": + if self.minimum > self.maximum: + raise ValueError("metric summary minimum exceeds maximum") + if not self.minimum <= self.best_value <= self.maximum: + raise ValueError("metric summary best value lies outside its range") + if self.count != len(self.source_config_ids): + raise ValueError("metric summary count differs from its source set") + if len(self.source_config_ids) != len(set(self.source_config_ids)): + raise ValueError("metric summary sources must be unique") + return self + + +class ScienceOperandV1(StrictScienceModel): + run_id: str + node_id: str + metric_path: str = Field(min_length=1, max_length=1024) + environment: ScienceEnvironmentV1 = Field(default_factory=ScienceEnvironmentV1) + + @field_validator("run_id", "node_id") + @classmethod + def _ids(cls, value: str, info: ValidationInfo) -> str: + return safe_id(value, info.field_name) + + +class ScienceNumericAssertionV1(StrictScienceModel): + id: str + claim_id: str | None = None + text_span: str = Field(default="", max_length=4096) + metric: str = Field(min_length=1, max_length=256) + value: float + unit: str = Field(min_length=1, max_length=128) + formula: str + operands: dict[str, ScienceOperandV1] + cross_environment: bool = False + aggregation: dict[str, Any] = Field(default_factory=dict, max_length=128) + tolerance: dict[str, float] = Field(default_factory=dict) + + @field_validator("id", "claim_id") + @classmethod + def _ids(cls, value: str | None, info: ValidationInfo) -> str | None: + return safe_id(value, info.field_name) if value is not None else None + + @field_validator("value") + @classmethod + def _finite_value(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("numeric assertion value must be finite") + return value + + @field_validator("formula") + @classmethod + def _formula(cls, value: str) -> str: + if value not in FORMULAS: + raise ValueError("numeric assertion formula is not registered") + return value + + @field_validator("aggregation") + @classmethod + def _aggregation(cls, value: dict[str, Any]) -> dict[str, Any]: + return finite_json(value, "aggregation") + + @field_validator("tolerance") + @classmethod + def _tolerance(cls, value: dict[str, float]) -> dict[str, float]: + if set(value) - {"absolute", "relative"} or any( + isinstance(number, bool) + or not isinstance(number, (int, float)) + or not math.isfinite(float(number)) + or number < 0 + for number in value.values() + ): + raise ValueError("numeric assertion tolerance is invalid") + return {key: float(number) for key, number in value.items()} + + @model_validator(mode="after") + def _roles(self) -> "ScienceNumericAssertionV1": + if set(self.operands) != set(required_roles(self.formula)): + raise ValueError("numeric assertion operand roles differ from formula") + return self + + +class ScienceEvidenceResultV1(StrictScienceModel): + run_id: str + node_id: str + metric_path: str = Field(min_length=1, max_length=1024) + + @field_validator("run_id", "node_id") + @classmethod + def _ids(cls, value: str, info: ValidationInfo) -> str: + return safe_id(value, info.field_name) + + +class ScienceClaimEvidenceV1(StrictScienceModel): + nodes: tuple[str, ...] = Field(default_factory=tuple, max_length=100_000) + results: tuple[ScienceEvidenceResultV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + figures: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + artifacts: tuple[ScienceArtifactRefV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + + @field_validator("nodes") + @classmethod + def _nodes(cls, value: tuple[str, ...]) -> tuple[str, ...]: + for node_id in value: + safe_id(node_id, "nodes") + if len(value) != len(set(value)): + raise ValueError("claim evidence nodes must be unique") + return value + + +class ScienceClaimV1(StrictScienceModel): + id: str + text: str = Field(min_length=1, max_length=100_000) + section: str = Field(min_length=1, max_length=128) + status: Literal["draft", "supported", "unsupported", "rejected"] + supported_by: ScienceClaimEvidenceV1 + numeric_assertions: tuple[ScienceNumericAssertionV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + risk: str = Field(default="", max_length=10_000) + + @field_validator("id") + @classmethod + def _id(cls, value: str) -> str: + return safe_id(value, "claim id") + + +class ScienceDerivedV1(DigestBoundScienceModel): + digest_field = "derived_digest" + + schema_version: Literal["ari.science-derived/v1"] = SCIENCE_DERIVED_V1 + derived_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + formula_registry_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + metric_summaries: tuple[ScienceMetricSummaryV1, ...] = Field( + default_factory=tuple, max_length=10_000 + ) + summary_stats: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + claims: tuple[ScienceClaimV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + numeric_assertions: tuple[ScienceNumericAssertionV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + anomalies: tuple[dict[str, Any], ...] = Field( + default_factory=tuple, max_length=100_000 + ) + + @field_validator("summary_stats", "anomalies") + @classmethod + def _json_fields(cls, value: Any, info: ValidationInfo) -> Any: + return finite_json(value, info.field_name) + + @model_validator(mode="after") + def _claim_links(self) -> "ScienceDerivedV1": + if self.formula_registry_digest != formula_registry_digest(): + raise ValueError("derived section uses a different formula registry") + claim_ids = [claim.id for claim in self.claims] + if len(claim_ids) != len(set(claim_ids)): + raise ValueError("science claim IDs must be unique") + assertion_ids = [assertion.id for assertion in self.numeric_assertions] + if len(assertion_ids) != len(set(assertion_ids)): + raise ValueError("science numeric assertion IDs must be unique") + known_claims = set(claim_ids) + if any( + assertion.claim_id is None or assertion.claim_id not in known_claims + for assertion in self.numeric_assertions + ): + raise ValueError("flattened assertions must reference a known claim") + nested = { + assertion.id + for claim in self.claims + for assertion in claim.numeric_assertions + } + if nested != set(assertion_ids): + raise ValueError("nested and flattened numeric assertion sets differ") + flattened = {assertion.id: assertion for assertion in self.numeric_assertions} + for claim in self.claims: + for assertion in claim.numeric_assertions: + flat = flattened[assertion.id] + if flat.claim_id != claim.id or flat.model_dump( + mode="json", exclude={"claim_id"} + ) != assertion.model_dump(mode="json", exclude={"claim_id"}): + raise ValueError("nested and flattened assertions differ") + return self + + +class ScienceInterpretationV1(DigestBoundScienceModel): + """Non-authoritative model annotation; never a numeric evidence source.""" + + digest_field = "interpretation_digest" + schema_version: Literal["ari.science-interpretation/v1"] = SCIENCE_INTERPRETATION_V1 + interpretation_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + status: Literal["ok", "unavailable", "invalid", "legacy-migrated"] + claim_eligible: Literal[False] = False + input_raw_digest: str = Field(pattern=SHA256_DIGEST_PATTERN) + prompt_digest: str | None = Field(default=None, pattern=SHA256_DIGEST_PATTERN) + model: str | None = Field(default=None, max_length=512) + model_revision: str | None = Field(default=None, max_length=512) + provider: str | None = Field(default=None, max_length=256) + sampling: dict[str, Any] = Field(default_factory=dict, max_length=128) + evaluation_protocol: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + experiment_context: dict[str, Any] = Field(default_factory=dict, max_length=10_000) + implementation_overview: dict[str, Any] | None = None + raw_response_artifact: ScienceArtifactRefV1 | None = None + error_kind: str | None = Field(default=None, max_length=256) + error_message: str | None = Field(default=None, max_length=4096) + + @field_validator( + "sampling", + "evaluation_protocol", + "experiment_context", + "implementation_overview", + ) + @classmethod + def _json_fields(cls, value: Any, info: ValidationInfo) -> Any: + if value is not None: + finite_json(value, info.field_name) + return value + + @model_validator(mode="after") + def _status_consistent(self) -> "ScienceInterpretationV1": + if self.status == "ok": + if self.raw_response_artifact is None or self.error_kind is not None: + raise ValueError( + "successful interpretation requires raw artifact and no error" + ) + elif self.status in {"invalid", "unavailable"} and self.error_kind is None: + raise ValueError("failed interpretation requires an error kind") + return self + + +class ScienceProvenanceV1(StrictScienceModel): + schema_version: Literal["ari.science-provenance/v1"] = SCIENCE_PROVENANCE_V1 + producer_tool_ref: str + producer_version: str = Field(min_length=1, max_length=256) + input_artifacts: tuple[ScienceArtifactRefV1, ...] = Field( + min_length=1, max_length=100_000 + ) + skills_lock: ScienceArtifactRefV1 | None = None + catalog_lock: ScienceArtifactRefV1 | None = None + admission_artifacts: tuple[ScienceArtifactRefV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + cassette_artifacts: tuple[ScienceArtifactRefV1, ...] = Field( + default_factory=tuple, max_length=100_000 + ) + + @field_validator("producer_tool_ref") + @classmethod + def _tool_ref(cls, value: str) -> str: + return safe_id(value, "producer_tool_ref") + + @model_validator(mode="after") + def _unique_artifacts(self) -> "ScienceProvenanceV1": + identities = [ + (artifact.relative_path, artifact.digest) + for artifact in self.input_artifacts + ] + if len(identities) != len(set(identities)): + raise ValueError("science provenance input artifacts must be unique") + return self + + +__all__ = [ + "ScienceClaimEvidenceV1", + "ScienceClaimV1", + "ScienceDerivedV1", + "ScienceEvidenceResultV1", + "ScienceInterpretationV1", + "ScienceMetricSummaryV1", + "ScienceNumericAssertionV1", + "ScienceOperandV1", + "ScienceProvenanceV1", +] diff --git a/ari-core/ari/science_data_migration.py b/ari-core/ari/science_data_migration.py new file mode 100644 index 00000000..a6dff491 --- /dev/null +++ b/ari-core/ari/science_data_migration.py @@ -0,0 +1,208 @@ +"""Offline-only migration for pre-v1 scientific-data checkpoints. + +The native runtime parser deliberately does not import old flat formats. This +module is reached only through the explicit migration entrypoint retained by +``ari.science_data_contract``. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from typing import Any + +from ari.claim_gate_contract import ( + migrate_legacy_metric_gate_contract, + parse_metric_gate_contract, +) +from ari.science_data_contract import ( + SCIENCE_DATA_V1, + ScienceArtifactRefV1, + ScienceConfigurationV1, + ScienceDataError, + ScienceDataV1, + ScienceDerivedV1, + ScienceInterpretationV1, + ScienceMetricSummaryV1, + ScienceProvenanceV1, + ScienceRawV1, + formula_registry_digest, + parse_science_data, +) + + +def _finite_metrics(value: Any) -> dict[str, int | float]: + if not isinstance(value, dict): + return {} + return { + str(key): number + for key, number in value.items() + if isinstance(number, (int, float)) + and not isinstance(number, bool) + and math.isfinite(float(number)) + and not str(key).startswith("_") + } + + +def _migrate_metric_contract(value: Any) -> tuple[dict[str, Any] | None, bool]: + if not isinstance(value, dict): + return None, False + try: + if value.get("schema_version") == "ari.metric-gate-contract/v1": + contract = parse_metric_gate_contract(value) + else: + contract = migrate_legacy_metric_gate_contract(value) + except ValueError: + return None, True + return contract.model_dump(mode="json"), False + + +def migrate_legacy_document( + value: dict[str, Any] | str, + *, + run_id: str, + logical_name: str, +) -> ScienceDataV1: + """Conservatively bind a legacy flat document without admitting claims.""" + + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ScienceDataError("legacy science data is not valid JSON") from exc + if not isinstance(value, dict): + raise ScienceDataError("legacy science data must be an object") + if value.get("schema_version") == SCIENCE_DATA_V1: + return parse_science_data(value) + source_payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + source = ScienceArtifactRefV1( + relative_path=logical_name, + digest="sha256:" + hashlib.sha256(source_payload).hexdigest(), + media_type="application/json", + role="legacy-science-data", + size_bytes=len(source_payload), + ) + config_nodes = value.get("_config_nodes") or {} + configs: list[ScienceConfigurationV1] = [] + for index, raw_config in enumerate(value.get("configurations") or []): + if not isinstance(raw_config, dict): + continue + config_id = str(raw_config.get("config_id") or f"cfg{index + 1}") + node_info = ( + config_nodes.get(config_id) if isinstance(config_nodes, dict) else {} + ) + node_info = node_info if isinstance(node_info, dict) else {} + configs.append( + ScienceConfigurationV1( + config_id=config_id, + run_id=run_id, + node_id=str(node_info.get("node_id") or f"legacy-node-{index + 1}"), + rank=int(raw_config.get("rank") or index + 1), + label=str(raw_config.get("label") or "legacy")[:256], + source_kind="legacy-untyped", + claim_eligible=False, + parameters=dict(raw_config.get("parameters") or {}), + measurements={}, + measurement_records=(), + predictions=dict(raw_config.get("predictions") or {}), + scores=dict(raw_config.get("scores") or {}), + legacy_metrics={ + **_finite_metrics(raw_config.get("metrics")), + **_finite_metrics(raw_config.get("measurements")), + }, + source_artifacts=(source,), + ) + ) + if not configs: + raise ScienceDataError("legacy science data contains no configurations") + raw = ScienceRawV1.create( + tree_artifact=source, + configurations=tuple(configs), + node_report_status="legacy", + measurement_status="legacy", + ) + summaries: list[ScienceMetricSummaryV1] = [] + summary_source = value.get("per_key_summary") or {} + if not isinstance(summary_source, dict): + summary_source = {} + for metric, record in summary_source.items(): + if not isinstance(record, dict): + continue + numbers = [ + (config.config_id, config.legacy_metrics.get(metric)) for config in configs + ] + numbers = [ + (config_id, number) for config_id, number in numbers if number is not None + ] + if not numbers: + continue + numeric_values = [float(number) for _, number in numbers] + summaries.append( + ScienceMetricSummaryV1( + metric_id=str(metric), + unit=str(record.get("unit") or "unknown"), + minimum=min(numeric_values), + maximum=max(numeric_values), + best_value=float(record.get("best_value", max(numeric_values))), + count=len(numeric_values), + direction="unspecified", + source_config_ids=tuple(config_id for config_id, _ in numbers), + ) + ) + derived = ScienceDerivedV1.create( + formula_registry_digest=formula_registry_digest(), + metric_summaries=tuple(summaries), + summary_stats=dict(value.get("summary_stats") or {}), + claims=(), + numeric_assertions=(), + anomalies=tuple(value.get("_anomalies") or ()), + ) + interpretation = ScienceInterpretationV1.create( + status="legacy-migrated", + input_raw_digest=raw.raw_digest, + evaluation_protocol={}, + experiment_context=dict(value.get("experiment_context") or {}), + implementation_overview=( + dict(value["implementation_overview"]) + if isinstance(value.get("implementation_overview"), dict) + else None + ), + error_kind="legacy-unverified", + error_message="Imported explicitly from a pre-v1 science_data artifact.", + ) + provenance = ScienceProvenanceV1( + producer_tool_ref="transform-skill/migrate-science-data@v1", + producer_version="1", + input_artifacts=(source,), + ) + metric_contract, contract_dropped = _migrate_metric_contract( + value.get("metric_contract") + ) + limitations = [ + "Legacy metrics are untyped and are not eligible as paper evidence.", + "Legacy claims were not carried forward without exact artifact binding.", + ] + if contract_dropped: + limitations.append( + "The legacy metric contract was invalid and was not carried forward." + ) + return ScienceDataV1.create( + run_id=run_id, + raw=raw, + derived=derived, + interpretation=interpretation, + metric_contract=metric_contract, + limitations=tuple(limitations), + provenance=provenance, + migration_status="legacy-explicit", + ) + + +__all__ = ["migrate_legacy_document"] diff --git a/ari-core/ari/science_data_validation.py b/ari-core/ari/science_data_validation.py new file mode 100644 index 00000000..fac3071e --- /dev/null +++ b/ari-core/ari/science_data_validation.py @@ -0,0 +1,170 @@ +"""Cross-section validation for canonical scientific-data documents.""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING, Any + +from ari.pipeline.claim_gate.numeric import recompute + +if TYPE_CHECKING: + from ari.science_data_contract import ( + ScienceConfigurationV1, + ScienceDataV1, + ScienceNumericAssertionV1, + ) + + +def _environment_key(value: Any) -> tuple[str, str, str]: + return (value.executor, value.cpu_model, value.arch) + + +def _measurement_value(config: "ScienceConfigurationV1", path: str) -> float: + namespace, separator, metric = path.partition(".") + if ( + namespace != "measurements" + or not separator + or metric not in config.measurements + ): + raise ValueError( + f"claim pointer does not resolve to a typed measurement: {path}" + ) + return float(config.measurements[metric]) + + +def _validate_assertion( + science: "ScienceDataV1", + assertion: "ScienceNumericAssertionV1", + by_node: dict[str, "ScienceConfigurationV1"], +) -> None: + operand_values: dict[str, float] = {} + environments: set[tuple[str, str, str]] = set() + for role, operand in assertion.operands.items(): + if operand.run_id != science.run_id or operand.node_id not in by_node: + raise ValueError("numeric assertion operand has an unknown run or node") + config = by_node[operand.node_id] + if not config.claim_eligible: + raise ValueError("numeric assertion references non-claimable data") + if _environment_key(operand.environment) != _environment_key( + config.environment + ): + raise ValueError( + "numeric assertion environment differs from raw provenance" + ) + operand_values[role] = _measurement_value(config, operand.metric_path) + environments.add(_environment_key(config.environment)) + if assertion.cross_environment != (len(environments) > 1): + raise ValueError("numeric assertion cross-environment flag is incorrect") + computed = recompute(assertion.formula, operand_values) + if computed is None: + raise ValueError("numeric assertion formula is undefined for its operands") + absolute = float(assertion.tolerance.get("absolute", 0.0)) + relative = float(assertion.tolerance.get("relative", 0.0)) + allowed = max(absolute, relative * abs(float(computed))) + if not math.isclose( + float(assertion.value), float(computed), rel_tol=0, abs_tol=allowed + ): + raise ValueError("numeric assertion does not recompute from raw measurements") + + +def _validate_migration_status(science: "ScienceDataV1") -> bool: + if science.migration_status != "legacy-explicit": + if "legacy" in { + science.raw.node_report_status, + science.raw.measurement_status, + }: + raise ValueError("native science data cannot use legacy raw status") + if science.interpretation.status == "legacy-migrated": + raise ValueError("native science data cannot contain legacy interpretation") + return False + if ( + science.derived.claims + or science.derived.numeric_assertions + or any(config.claim_eligible for config in science.raw.configurations) + ): + raise ValueError("legacy migration cannot admit scientific claims") + if science.interpretation.status != "legacy-migrated" or { + science.raw.node_report_status, + science.raw.measurement_status, + } != {"legacy"}: + raise ValueError("legacy migration status is inconsistent") + return True + + +def _validate_native_raw_status(science: "ScienceDataV1") -> None: + count = len(science.raw.configurations) + report_count = sum( + any(source.role == "node-report" for source in config.source_artifacts) + for config in science.raw.configurations + ) + typed_count = sum(config.claim_eligible for config in science.raw.configurations) + expected_reports = ( + "complete" + if report_count == count + else "partial" + if report_count + else "missing" + ) + expected_measurements = ( + "complete" if typed_count == count else "partial" if typed_count else "missing" + ) + if science.raw.node_report_status != expected_reports: + raise ValueError("node-report status differs from raw source artifacts") + if science.raw.measurement_status != expected_measurements: + raise ValueError("measurement status differs from claim eligibility") + + +def _validate_derived_links( + science: "ScienceDataV1", + by_node: dict[str, "ScienceConfigurationV1"], + by_config: dict[str, "ScienceConfigurationV1"], +) -> None: + for summary in science.derived.metric_summaries: + for config_id in summary.source_config_ids: + config = by_config.get(config_id) + if config is None or not config.claim_eligible: + raise ValueError( + "metric summary references non-claimable configuration" + ) + _measurement_value(config, f"measurements.{summary.metric_id}") + for claim in science.derived.claims: + if any( + node_id not in by_node or not by_node[node_id].claim_eligible + for node_id in claim.supported_by.nodes + ): + raise ValueError("claim references an unknown or non-claimable node") + for result in claim.supported_by.results: + if result.run_id != science.run_id or result.node_id not in by_node: + raise ValueError("claim result pointer has an unknown run or node") + _measurement_value(by_node[result.node_id], result.metric_path) + for assertion in science.derived.numeric_assertions: + _validate_assertion(science, assertion, by_node) + + +def validate_science_data_links(science: "ScienceDataV1") -> None: + """Reject cross-section pointers that cannot be replayed from raw facts.""" + + by_node = {config.node_id: config for config in science.raw.configurations} + by_config = {config.config_id: config for config in science.raw.configurations} + provenance_inputs = { + (artifact.relative_path, artifact.digest) + for artifact in science.provenance.input_artifacts + } + required_sources = { + (science.raw.tree_artifact.relative_path, science.raw.tree_artifact.digest), + *( + (artifact.relative_path, artifact.digest) + for config in science.raw.configurations + for artifact in config.source_artifacts + ), + } + if not required_sources <= provenance_inputs: + raise ValueError("raw source artifact is absent from science provenance") + + if _validate_migration_status(science): + return + _validate_native_raw_status(science) + _validate_derived_links(science, by_node, by_config) + + +__all__ = ["validate_science_data_links"] diff --git a/ari-core/ari/skill_lock.py b/ari-core/ari/skill_lock.py new file mode 100644 index 00000000..eea576d4 --- /dev/null +++ b/ari-core/ari/skill_lock.py @@ -0,0 +1,454 @@ +"""Run-level immutable snapshot for admitted MCP Skills and live tool schemas. + +``skill.yaml`` describes a provider before launch. ``SKILLS.lock`` binds that +declaration to the schemas returned by the live MCP ``tools/list`` handshake and +to the phase-specific set that ARI actually admits for one run. A checkpoint +therefore cannot silently resume with a different provider, policy, or schema. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +from ari.config import SkillConfig +from ari.mcp.dispatch_support import normalize_phases, phase_is_disabled, phase_matches + + +SKILLS_LOCK_FILENAME = "SKILLS.lock" +SKILLS_LOCK_SCHEMA_VERSION = "ari.skills-lock/v1" +DEFAULT_RUNTIME_PHASES = ( + "bfts", + "control", + "evaluation", + "paper", + "pipeline", + "reproduce", +) + + +class SkillLockError(RuntimeError): + """Base class for lock creation, validation, and reconciliation failures.""" + + +class SkillLockCorruptError(SkillLockError): + """Raised when an existing ``SKILLS.lock`` is malformed or self-inconsistent.""" + + +class SkillLockMismatchError(SkillLockError): + """Raised when live admission differs from the run's immutable snapshot.""" + + +class SkillLockMissingError(SkillLockError): + """Raised when a subset worker requires a run lock that does not exist.""" + + +class SkillProviderAdmissionError(SkillLockError): + """Raised when a provider required by a locked run cannot be discovered.""" + + +class LockedToolV1(BaseModel): + """One live MCP tool bound to its provider, policy, and exact JSON Schemas.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + tool_ref: str + name: str + skill_name: str + capability_ref: str | None = None + input_schema: dict[str, Any] = Field(default_factory=dict) + output_schema: dict[str, Any] = Field(default_factory=dict) + input_schema_digest: str + output_schema_digest: str + policy: dict[str, Any] = Field(default_factory=dict) + + +class LockedCredentialScopeV1(BaseModel): + """Value-free identity of credential authority available to one provider.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + declared_env: list[str] = Field(default_factory=list) + present_env: list[str] = Field(default_factory=list) + identity_digest: str + + +class LockedSkillV1(BaseModel): + """One configured provider and the digest of its admitted live surface.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + name: str + package: str + version: str + entrypoint: str + manifest_digest: str + provider_digest: str + configured_phases: list[str] + environment_policy: Literal["audit-pending", "complete"] + required_env: list[str] = Field(default_factory=list) + optional_env: list[str] = Field(default_factory=list) + credential_scopes: list[LockedCredentialScopeV1] = Field(default_factory=list) + tool_refs: list[str] = Field(default_factory=list) + + +class SkillsLockV1(BaseModel): + """Canonical, deterministic run snapshot persisted as ``SKILLS.lock``.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["ari.skills-lock/v1"] = SKILLS_LOCK_SCHEMA_VERSION + run_id: str + registry_digest: str + skills: list[LockedSkillV1] + tools: list[LockedToolV1] + disabled_tools: list[str] = Field(default_factory=list) + phase_active_tools: dict[str, list[str]] + + @model_validator(mode="after") + def _internally_consistent(self) -> "SkillsLockV1": + tool_refs = [tool.tool_ref for tool in self.tools] + if len(tool_refs) != len(set(tool_refs)): + raise ValueError("SKILLS.lock contains duplicate tool_ref values") + known = set(tool_refs) + for phase, refs in self.phase_active_tools.items(): + unknown = sorted(set(refs) - known) + if unknown: + raise ValueError( + f"phase {phase!r} refers to unknown tool_ref values: {unknown}" + ) + return self + + +def _json_digest(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _registry_payload(lock: SkillsLockV1 | dict[str, Any]) -> dict[str, Any]: + if isinstance(lock, SkillsLockV1): + payload = lock.model_dump(mode="json") + else: + payload = dict(lock) + payload.pop("registry_digest", None) + return payload + + +def skills_lock_digest(lock: SkillsLockV1) -> str: + """Recompute the self-authenticating digest for a lock document.""" + + return _json_digest(_registry_payload(lock)) + + +def _normalized_tool(raw: dict[str, Any]) -> dict[str, Any]: + """Copy only stable live-discovery fields into a lock-safe structure.""" + + return { + "tool_ref": str(raw["tool_ref"]), + "name": str(raw["name"]), + "skill_name": str(raw["skill_name"]), + "capability_ref": ( + str(raw["capability_ref"]) if raw.get("capability_ref") else None + ), + "input_schema": raw.get("inputSchema") or {}, + "output_schema": raw.get("outputSchema") or {}, + "policy": raw.get("policy") or {}, + } + + +def build_skills_lock( + *, + run_id: str, + skills: list[SkillConfig], + tools: list[dict[str, Any]], + disabled_tools: set[str] | list[str] | tuple[str, ...] = (), + runtime_phases: tuple[str, ...] = DEFAULT_RUNTIME_PHASES, +) -> SkillsLockV1: + """Build a byte-stable lock from configured providers and live discovery.""" + + normalized = [_normalized_tool(tool) for tool in tools] + locked_tools = [ + LockedToolV1( + **tool, + input_schema_digest=_json_digest(tool["input_schema"]), + output_schema_digest=_json_digest(tool["output_schema"]), + ) + for tool in normalized + ] + locked_tools.sort(key=lambda item: item.tool_ref) + + tools_by_skill: dict[str, list[LockedToolV1]] = {} + for tool in locked_tools: + tools_by_skill.setdefault(tool.skill_name, []).append(tool) + + enabled_skills = [ + skill + for skill in skills + if not phase_is_disabled(getattr(skill, "phase", "all")) + ] + locked_skills: list[LockedSkillV1] = [] + for skill in enabled_skills: + owned = tools_by_skill.get(skill.name, []) + credential_scopes = [ + LockedCredentialScopeV1.model_validate(identity) + for identity in skill.credential_scope_identities + ] + credential_scopes.sort(key=lambda item: item.scope_id) + provider_payload = { + "name": skill.name, + "package": skill.package or skill.name, + "version": skill.version, + "entrypoint": skill.entrypoint, + "manifest_digest": skill.manifest_digest, + "configured_phases": sorted(normalize_phases(skill.phase)), + "credential_scopes": [ + scope.model_dump(mode="json") for scope in credential_scopes + ], + "tools": [ + { + "tool_ref": tool.tool_ref, + "input_schema_digest": tool.input_schema_digest, + "output_schema_digest": tool.output_schema_digest, + "policy": tool.policy, + } + for tool in owned + ], + } + locked_skills.append( + LockedSkillV1( + name=skill.name, + package=skill.package or skill.name, + version=skill.version, + entrypoint=skill.entrypoint, + manifest_digest=skill.manifest_digest, + provider_digest=_json_digest(provider_payload), + configured_phases=sorted(normalize_phases(skill.phase)), + environment_policy=skill.environment_policy, + required_env=sorted(skill.required_env), + optional_env=sorted(skill.optional_env), + credential_scopes=credential_scopes, + tool_refs=sorted(tool.tool_ref for tool in owned), + ) + ) + locked_skills.sort(key=lambda item: (item.package, item.name)) + + disabled = set(disabled_tools) + phase_names = set(runtime_phases) + for skill in enabled_skills: + phase_names.update(normalize_phases(skill.phase)) + for tool in locked_tools: + policy_phases = tool.policy.get("phases", ["all"]) + phase_names.update(normalize_phases(policy_phases)) + phase_names.difference_update({"", "all", "none"}) + + skill_by_name = {skill.name: skill for skill in enabled_skills} + active: dict[str, list[str]] = {} + for phase in sorted(phase_names): + admitted: list[str] = [] + for tool in locked_tools: + skill = skill_by_name.get(tool.skill_name) + if skill is None or not phase_matches(skill.phase, phase): + continue + tool_phases = tool.policy.get("phases", ["all"]) + if not phase_matches(normalize_phases(tool_phases), phase): + continue + if tool.name in disabled or tool.tool_ref in disabled: + continue + admitted.append(tool.tool_ref) + active[phase] = sorted(admitted) + + provisional: dict[str, Any] = { + "schema_version": SKILLS_LOCK_SCHEMA_VERSION, + "run_id": run_id, + "registry_digest": "", + "skills": [skill.model_dump(mode="json") for skill in locked_skills], + "tools": [tool.model_dump(mode="json") for tool in locked_tools], + "disabled_tools": sorted(disabled), + "phase_active_tools": active, + } + provisional["registry_digest"] = _json_digest(_registry_payload(provisional)) + return SkillsLockV1.model_validate(provisional) + + +def load_skills_lock(path: str | Path) -> SkillsLockV1: + """Read and fully validate an existing lock, including its digest.""" + + lock_path = Path(path) + if lock_path.is_symlink(): + raise SkillLockCorruptError(f"invalid {lock_path}: symbolic links are refused") + try: + raw = json.loads(lock_path.read_text(encoding="utf-8")) + lock = SkillsLockV1.model_validate(raw) + except (OSError, json.JSONDecodeError, ValidationError, ValueError) as exc: + raise SkillLockCorruptError(f"invalid {lock_path}: {exc}") from exc + actual = skills_lock_digest(lock) + if actual != lock.registry_digest: + raise SkillLockCorruptError( + f"invalid {lock_path}: registry_digest is {lock.registry_digest}, " + f"recomputed {actual}" + ) + return lock + + +def _mismatch_summary(expected: SkillsLockV1, current: SkillsLockV1) -> str: + expected_refs = {tool.tool_ref for tool in expected.tools} + current_refs = {tool.tool_ref for tool in current.tools} + parts = [ + f"locked digest {expected.registry_digest}", + f"live digest {current.registry_digest}", + ] + added = sorted(current_refs - expected_refs) + removed = sorted(expected_refs - current_refs) + if added: + parts.append(f"added tool refs: {added}") + if removed: + parts.append(f"removed tool refs: {removed}") + if expected.disabled_tools != current.disabled_tools: + parts.append("disabled tool policy changed") + if expected.phase_active_tools != current.phase_active_tools: + parts.append("phase active sets changed") + return "; ".join(parts) + + +def write_or_verify_skills_lock( + path: str | Path, + current: SkillsLockV1, +) -> SkillsLockV1: + """Atomically create a run lock, or require exact equality when it exists.""" + + lock_path = Path(path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + rendered = json.dumps( + current.model_dump(mode="json"), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + "\n" + fd, temporary_name = tempfile.mkstemp( + prefix=f".{SKILLS_LOCK_FILENAME}.", + dir=lock_path.parent, + text=True, + ) + temporary_path = Path(temporary_name) + descriptor_owned = True + try: + os.fchmod(fd, 0o600) + handle = os.fdopen(fd, "w", encoding="utf-8") + descriptor_owned = False # ``handle`` now owns and closes the descriptor. + with handle: + handle.write(rendered) + handle.flush() + os.fsync(handle.fileno()) + try: + # A hard link publishes a fully-written inode without replacing an + # existing snapshot. Concurrent creators either win this operation + # or verify the winner; no reader can observe a partial document. + os.link(temporary_path, lock_path) + except FileExistsError: + existing = load_skills_lock(lock_path) + if existing != current: + raise SkillLockMismatchError( + f"live MCP registry does not match immutable {lock_path}: " + f"{_mismatch_summary(existing, current)}" + ) + return existing + finally: + if descriptor_owned: + try: + os.close(fd) + except OSError: + pass + try: + temporary_path.unlink() + except OSError: + pass + return current + + +def verify_skills_lock_subset( + path: str | Path, + current: SkillsLockV1, +) -> SkillsLockV1: + """Verify a single-provider worker against an already-created full run lock. + + Pipeline stages intentionally launch only their owning Skill. Such a worker + may validate its provider and tools as an exact subset, but it may never mint + the authoritative run snapshot or change run-wide disabled/phase policy. + """ + + lock_path = Path(path) + if not lock_path.is_file(): + raise SkillLockMissingError( + f"subset MCP worker requires existing immutable {lock_path}" + ) + if not current.skills: + raise SkillLockMismatchError( + f"subset MCP worker discovered no configured provider for {lock_path}" + ) + expected = load_skills_lock(lock_path) + if current.run_id != expected.run_id: + raise SkillLockMismatchError( + f"subset MCP registry run_id {current.run_id!r} does not match " + f"{expected.run_id!r} in {lock_path}" + ) + if current.disabled_tools != expected.disabled_tools: + raise SkillLockMismatchError( + f"subset MCP registry disabled-tool policy does not match {lock_path}" + ) + + expected_skills = {skill.name: skill for skill in expected.skills} + for skill in current.skills: + if expected_skills.get(skill.name) != skill: + raise SkillLockMismatchError( + f"subset MCP provider {skill.name!r} does not match {lock_path}" + ) + expected_tools = {tool.tool_ref: tool for tool in expected.tools} + for tool in current.tools: + if expected_tools.get(tool.tool_ref) != tool: + raise SkillLockMismatchError( + f"subset MCP tool {tool.tool_ref!r} does not match {lock_path}" + ) + + current_refs = {tool.tool_ref for tool in current.tools} + for phase, refs in current.phase_active_tools.items(): + locked_subset = sorted( + ref for ref in expected.phase_active_tools.get(phase, []) if ref in current_refs + ) + if refs != locked_subset: + raise SkillLockMismatchError( + f"subset MCP phase {phase!r} active set does not match {lock_path}" + ) + return expected + + +__all__ = [ + "DEFAULT_RUNTIME_PHASES", + "SKILLS_LOCK_FILENAME", + "SKILLS_LOCK_SCHEMA_VERSION", + "LockedSkillV1", + "LockedCredentialScopeV1", + "LockedToolV1", + "SkillLockCorruptError", + "SkillLockError", + "SkillLockMismatchError", + "SkillLockMissingError", + "SkillProviderAdmissionError", + "SkillsLockV1", + "build_skills_lock", + "load_skills_lock", + "skills_lock_digest", + "verify_skills_lock_subset", + "write_or_verify_skills_lock", +] diff --git a/ari-core/ari/skill_manifest.py b/ari-core/ari/skill_manifest.py new file mode 100644 index 00000000..2b04c6c4 --- /dev/null +++ b/ari-core/ari/skill_manifest.py @@ -0,0 +1,470 @@ +"""Versioned contract for ARI MCP Skill packages. + +``skill.yaml`` is the canonical source for package identity, process startup, +tool policy, and compatibility metadata. This module intentionally has no MCP +runtime dependency, so manifests can be validated by packaging and CI jobs in a +clean interpreter. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path, PurePosixPath +from typing import Literal + +import yaml +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) + +from ari.async_tools import AsyncLifecycleV1, TimeoutBudgetV1 +from ari.credential_scope import ( + ENVIRONMENT_NAME_RE, + CredentialScopeV1, + looks_like_credential_environment_name, +) + + +MANIFEST_FILENAME = "skill.yaml" +LEGACY_MCP_RESULT_V1 = "ari.legacy-mcp-result/v1" +RESULT_ENVELOPE_V1 = "ari.result-envelope/v1" + +_KEBAB_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +_TOOL_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_REF_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$") + + +class SkillManifestError(ValueError): + """Raised when a canonical Skill manifest cannot be loaded or validated.""" + + +class SkillEntrypointV1(BaseModel): + """How ari-core launches one Skill server.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + transport: Literal["stdio"] = "stdio" + command_kind: Literal["python"] = "python" + module: str = "src/server.py" + + @field_validator("module") + @classmethod + def _relative_module(cls, value: str) -> str: + value = value.strip() + path = PurePosixPath(value) + if not value or "\\" in value or path.is_absolute() or ".." in path.parts: + raise ValueError("entrypoint.module must be a safe POSIX-relative path") + if path.suffix != ".py": + raise ValueError("python entrypoint.module must end in .py") + return value + + +class ToolPolicyV1(BaseModel): + """Policy inherited by tools that do not declare an override.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + phases: list[str] = Field(default_factory=lambda: ["all"]) + side_effects: Literal["read-only", "workspace-write", "stateful", "destructive"] = ( + "read-only" + ) + determinism: Literal["deterministic", "conditional", "stochastic", "live-data"] = ( + "conditional" + ) + timeout_class: Literal["default", "bounded", "slow", "very-slow", "async"] = ( + "default" + ) + timeout_budget: TimeoutBudgetV1 | None = None + async_lifecycle: AsyncLifecycleV1 | None = None + permissions: list[str] = Field(default_factory=list) + context_requirement: Literal["none", "run", "node"] = "none" + result_schema: str = RESULT_ENVELOPE_V1 + + @field_validator("phases") + @classmethod + def _valid_phases(cls, values: list[str]) -> list[str]: + if not values: + raise ValueError("phases must contain at least one phase") + return _validated_tokens(values, "phase") + + @field_validator("permissions") + @classmethod + def _valid_permissions(cls, values: list[str]) -> list[str]: + return _validated_tokens(values, "permission") + + @field_validator("result_schema") + @classmethod + def _valid_result_schema(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("result_schema cannot be empty") + return value + + +class ToolManifestV1(BaseModel): + """One tool declaration; omitted policy fields inherit ``tool_defaults``.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + name: str + capability_ref: str + description: str = "" + phases: list[str] | None = None + side_effects: ( + Literal["read-only", "workspace-write", "stateful", "destructive"] | None + ) = None + determinism: ( + Literal["deterministic", "conditional", "stochastic", "live-data"] | None + ) = None + timeout_class: ( + Literal["default", "bounded", "slow", "very-slow", "async"] | None + ) = None + timeout_budget: TimeoutBudgetV1 | None = None + async_lifecycle: AsyncLifecycleV1 | None = None + permissions: list[str] | None = None + context_requirement: Literal["none", "run", "node"] | None = None + result_schema: str | None = None + + @field_validator("name") + @classmethod + def _valid_name(cls, value: str) -> str: + if not _TOOL_RE.fullmatch(value): + raise ValueError("tool name must be a Python-style identifier") + return value + + @field_validator("capability_ref") + @classmethod + def _valid_capability(cls, value: str) -> str: + value = value.strip() + if not _REF_RE.fullmatch(value): + raise ValueError("capability_ref must be a lowercase dotted identifier") + return value + + @field_validator("phases") + @classmethod + def _valid_optional_phases(cls, values: list[str] | None) -> list[str] | None: + if values is None: + return None + if not values: + raise ValueError("phases override cannot be empty") + return _validated_tokens(values, "phase") + + @field_validator("permissions") + @classmethod + def _valid_optional_permissions(cls, values: list[str] | None) -> list[str] | None: + if values is None: + return None + return _validated_tokens(values, "permission") + + @field_validator("result_schema") + @classmethod + def _valid_optional_result_schema(cls, value: str | None) -> str | None: + if value is not None and not value.strip(): + raise ValueError("result_schema override cannot be empty") + return value.strip() if value is not None else None + + def resolve(self, defaults: ToolPolicyV1) -> "ResolvedToolManifestV1": + """Return a fully populated tool policy.""" + + policy = defaults.model_dump() + for field_name in ( + "phases", + "side_effects", + "determinism", + "timeout_class", + "timeout_budget", + "async_lifecycle", + "permissions", + "context_requirement", + "result_schema", + ): + value = getattr(self, field_name) + if value is not None: + policy[field_name] = value + return ResolvedToolManifestV1( + name=self.name, + capability_ref=self.capability_ref, + description=self.description, + **policy, + ) + + +class ResolvedToolManifestV1(ToolPolicyV1): + """A tool declaration after package defaults have been applied.""" + + name: str + capability_ref: str + description: str = "" + + @model_validator(mode="after") + def _valid_execution_mode(self) -> "ResolvedToolManifestV1": + if self.timeout_class == "async" and self.async_lifecycle is None: + raise ValueError("timeout_class=async requires async_lifecycle") + if self.timeout_class != "async" and self.async_lifecycle is not None: + raise ValueError("async_lifecycle requires timeout_class=async") + return self + + +class SkillManifestV1(BaseModel): + """Canonical ARI Skill package manifest, schema version 1.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal[1] + name: str + package: str + version: str + display_name: str = "" + description: str = "" + enabled_by_default: bool = True + environment_policy: Literal["audit-pending", "complete"] = "audit-pending" + entrypoint: SkillEntrypointV1 = Field(default_factory=SkillEntrypointV1) + required_env: list[str] = Field(default_factory=list) + optional_env: list[str] = Field(default_factory=list) + credential_scopes: list[CredentialScopeV1] = Field(default_factory=list) + tool_defaults: ToolPolicyV1 = Field(default_factory=ToolPolicyV1) + tools: list[ToolManifestV1] + + @field_validator("name", "package") + @classmethod + def _valid_kebab_identifier(cls, value: str) -> str: + if not _KEBAB_RE.fullmatch(value): + raise ValueError("name and package must be lowercase kebab-case") + return value + + @field_validator("version") + @classmethod + def _valid_version(cls, value: str) -> str: + value = value.strip() + if not _VERSION_RE.fullmatch(value): + raise ValueError("version must be SemVer-compatible (for example 1.2.3)") + return value + + @field_validator("required_env", "optional_env") + @classmethod + def _valid_env_names(cls, values: list[str]) -> list[str]: + if len(values) != len(set(values)): + raise ValueError("environment variable names must be unique") + invalid = [ + value for value in values if not ENVIRONMENT_NAME_RE.fullmatch(value) + ] + if invalid: + raise ValueError(f"invalid environment variable names: {invalid}") + return values + + @model_validator(mode="after") + def _unique_contract(self) -> "SkillManifestV1": + names = [tool.name for tool in self.tools] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate tool declarations: {duplicates}") + overlap = sorted(set(self.required_env) & set(self.optional_env)) + if overlap: + raise ValueError( + f"environment variables cannot be required and optional: {overlap}" + ) + scope_ids = [scope.id for scope in self.credential_scopes] + duplicate_scopes = sorted( + {scope_id for scope_id in scope_ids if scope_ids.count(scope_id) > 1} + ) + if duplicate_scopes: + raise ValueError(f"duplicate credential scopes: {duplicate_scopes}") + ordinary = set(self.required_env) | set(self.optional_env) + credential_names = [ + name + for scope in self.credential_scopes + for name in scope.environment_names() + ] + duplicate_credentials = sorted( + {name for name in credential_names if credential_names.count(name) > 1} + ) + if duplicate_credentials: + raise ValueError( + "credential environment variables must belong to one scope: " + f"{duplicate_credentials}" + ) + classified_overlap = sorted(ordinary & set(credential_names)) + if classified_overlap: + raise ValueError( + "credential environment variables cannot also be ordinary env: " + f"{classified_overlap}" + ) + if self.environment_policy == "complete": + unclassified = sorted( + name + for name in ordinary + if looks_like_credential_environment_name(name) + ) + if unclassified: + raise ValueError( + "credential-like environment variables require a credential " + f"scope: {unclassified}" + ) + capabilities: dict[str, list[str]] = {} + for tool in self.tools: + capabilities.setdefault(tool.capability_ref, []).append(tool.name) + for tool in self.resolved_tools(): + lifecycle = tool.async_lifecycle + if lifecycle is None: + continue + for operation_name in ("status", "result", "cancel"): + operation = getattr(lifecycle, operation_name) + if operation is None: + continue + targets = capabilities.get(operation.capability_ref, []) + if len(targets) != 1: + raise ValueError( + f"tool {tool.name!r} async {operation_name} capability " + f"{operation.capability_ref!r} must resolve to exactly one " + f"tool in the package; found {targets}" + ) + return self + + def environment_names(self) -> tuple[str, ...]: + """Return every ordinary and credential environment name declared.""" + + credential_names = [ + name + for scope in self.credential_scopes + for name in scope.environment_names() + ] + return tuple(self.required_env + self.optional_env + credential_names) + + def resolved_tools(self) -> tuple[ResolvedToolManifestV1, ...]: + """Return tools with package defaults applied, preserving manifest order.""" + + return tuple(tool.resolve(self.tool_defaults) for tool in self.tools) + + def tool(self, name: str) -> ResolvedToolManifestV1 | None: + """Return one resolved declaration by runtime tool name.""" + + return next((tool for tool in self.resolved_tools() if tool.name == name), None) + + +def _validated_tokens(values: list[str], label: str) -> list[str]: + if len(values) != len(set(values)): + raise ValueError(f"{label} values must be unique") + invalid = [value for value in values if not _KEBAB_RE.fullmatch(value)] + if invalid: + raise ValueError(f"invalid {label} values: {invalid}") + return values + + +def load_skill_manifest(path: str | Path) -> SkillManifestV1: + """Load and validate one manifest. + + This production loader accepts canonical v1 documents only. Unversioned + conversion is isolated under :mod:`ari.migrations.skill_manifest` and is + never used by discovery or runtime admission. + """ + + manifest_path = Path(path) + try: + raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise SkillManifestError(f"cannot read {manifest_path}: {exc}") from exc + if not isinstance(raw, dict): + raise SkillManifestError(f"{manifest_path}: manifest root must be a mapping") + if "schema_version" not in raw: + raise SkillManifestError( + f"{manifest_path}: schema_version is required for a canonical manifest" + ) + try: + return SkillManifestV1.model_validate(raw) + except ValidationError as exc: + raise SkillManifestError(f"{manifest_path}: {exc}") from exc + + +def resolve_skill_entrypoint( + skill_dir: str | Path, + manifest: SkillManifestV1, + *, + require_exists: bool = True, +) -> Path: + """Resolve a manifest entrypoint while preventing package-root escape.""" + + root = Path(skill_dir).resolve() + target = (root / manifest.entrypoint.module).resolve() + try: + target.relative_to(root) + except ValueError as exc: + raise SkillManifestError( + f"entrypoint {manifest.entrypoint.module!r} escapes package {root}" + ) from exc + if require_exists and not target.is_file(): + raise SkillManifestError(f"entrypoint does not exist: {target}") + return target + + +def manifest_digest(manifest: SkillManifestV1) -> str: + """Return a stable SHA-256 identity for the normalized manifest.""" + + payload = json.dumps( + manifest.model_dump(mode="json"), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def manifest_tool_ref(manifest: SkillManifestV1, tool_name: str) -> str: + """Return the declared opaque identity for one manifest tool. + + The whole normalized manifest participates in the digest so package version, + launcher, environment policy, and tool policy changes invalidate the + identity. Runtime input/output-schema identity is layered on by the run + registry after ``tools/list``. + """ + + if manifest.tool(tool_name) is None: + raise SkillManifestError( + f"tool {tool_name!r} is not declared by {manifest.package}" + ) + return f"{manifest.package}/{tool_name}@sha256:{manifest_digest(manifest)}" + + +def legacy_mcp_document(manifest: SkillManifestV1) -> dict: + """Render the read-only ``mcp.json`` compatibility view.""" + + return { + "schema_version": 1, + "generated_from": MANIFEST_FILENAME, + "name": manifest.name, + "package": manifest.package, + "version": manifest.version, + "description": manifest.description, + "tools": [tool.name for tool in manifest.tools], + "runtime": manifest.entrypoint.command_kind, + "entrypoint": manifest.entrypoint.module, + } + + +__all__ = [ + "MANIFEST_FILENAME", + "LEGACY_MCP_RESULT_V1", + "RESULT_ENVELOPE_V1", + "AsyncLifecycleV1", + "TimeoutBudgetV1", + "CredentialScopeV1", + "ResolvedToolManifestV1", + "SkillEntrypointV1", + "SkillManifestError", + "SkillManifestV1", + "ToolManifestV1", + "ToolPolicyV1", + "legacy_mcp_document", + "load_skill_manifest", + "looks_like_credential_environment_name", + "manifest_digest", + "manifest_tool_ref", + "resolve_skill_entrypoint", +] diff --git a/ari-core/ari/visual_review_contract.py b/ari-core/ari/visual_review_contract.py new file mode 100644 index 00000000..babff673 --- /dev/null +++ b/ari-core/ari/visual_review_contract.py @@ -0,0 +1,320 @@ +"""Versioned, digest-bound contracts for visual scientific review.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import PurePosixPath +from typing import Any, ClassVar, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + + +VISUAL_REVIEW_V1 = "ari.visual-review/v1" +VISUAL_REVIEW_BATCH_V1 = "ari.visual-review-batch/v1" +VISUAL_CRITERIA_PROFILE_V1 = "ari.visual-criteria-profile/v1" +VISUAL_ARTIFACT_REF_V1 = "ari.visual-artifact-ref/v1" +SHA256_PATTERN = r"^sha256:[0-9a-f]{64}$" +ZERO_DIGEST = "sha256:" + "0" * 64 +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$") + + +class VisualReviewContractError(ValueError): + """A visual-review document is invalid or has lost provenance.""" + + +def canonical_visual_review_digest(value: Any) -> str: + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +class _StrictReviewModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class _DigestBoundReviewModel(_StrictReviewModel): + digest_field: ClassVar[str] + + @classmethod + def create(cls, **values: Any): + values = dict(values) + values[cls.digest_field] = ZERO_DIGEST + return cls.model_validate(values, context={"bind_review_digest": True}) + + def digest_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude={self.digest_field}) + + @model_validator(mode="after") + def _digest_matches(self, info: ValidationInfo): + expected = canonical_visual_review_digest(self.digest_payload()) + if info.context and info.context.get("bind_review_digest"): + object.__setattr__(self, self.digest_field, expected) + elif getattr(self, self.digest_field) != expected: + raise ValueError(f"{self.digest_field} does not match review payload") + return self + + +class VisualArtifactRefV1(_StrictReviewModel): + schema_version: Literal["ari.visual-artifact-ref/v1"] = VISUAL_ARTIFACT_REF_V1 + role: Literal["review-target", "raw-model-response", "table-source"] + relative_path: str + digest: str = Field(pattern=SHA256_PATTERN) + media_type: str = Field(min_length=1, max_length=128) + size_bytes: int = Field(ge=0) + + @field_validator("relative_path") + @classmethod + def _relative_path(cls, value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError("visual artifact path must be safe and relative") + return value + + +class VisualCriterionV1(_StrictReviewModel): + criterion_id: str = Field(min_length=1, max_length=128) + description: str = Field(min_length=1, max_length=1_024) + required: bool = True + + +class VisualCriteriaProfileV1(_DigestBoundReviewModel): + digest_field = "profile_digest" + schema_version: Literal["ari.visual-criteria-profile/v1"] = ( + VISUAL_CRITERIA_PROFILE_V1 + ) + profile_id: str + target_kind: Literal["figure", "table"] + version: str = Field(min_length=1, max_length=64) + criteria: tuple[VisualCriterionV1, ...] = Field(min_length=1, max_length=64) + passing_score: float = Field(ge=0, le=1) + profile_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("profile_id") + @classmethod + def _id(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("visual criteria profile_id is invalid") + return value + + @model_validator(mode="after") + def _unique_criteria(self) -> "VisualCriteriaProfileV1": + ids = [item.criterion_id for item in self.criteria] + if len(ids) != len(set(ids)): + raise ValueError("visual criteria IDs must be unique") + return self + + +class VisualRegionV1(_StrictReviewModel): + x: float = Field(ge=0, le=1) + y: float = Field(ge=0, le=1) + width: float = Field(gt=0, le=1) + height: float = Field(gt=0, le=1) + + @model_validator(mode="after") + def _inside(self) -> "VisualRegionV1": + if self.x + self.width > 1 or self.y + self.height > 1: + raise ValueError("visual issue region lies outside the target") + return self + + +class VisualIssueV1(_StrictReviewModel): + issue_id: str + criterion_id: str = Field(min_length=1, max_length=128) + severity: Literal["info", "minor", "major", "blocking"] + message: str = Field(min_length=1, max_length=4_096) + suggestion: str = Field(default="", max_length=4_096) + evidence: str = Field(default="", max_length=4_096) + region: VisualRegionV1 | None = None + page: int | None = Field(default=None, ge=1, le=100_000) + + @field_validator("issue_id") + @classmethod + def _id(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("visual issue_id is invalid") + return value + + +class VisualModelUsageV1(_StrictReviewModel): + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + cost_usd: float | None = Field(default=None, ge=0) + cost_status: Literal["reported", "unavailable"] = "unavailable" + + @model_validator(mode="after") + def _cost_status(self) -> "VisualModelUsageV1": + if (self.cost_usd is not None) != (self.cost_status == "reported"): + raise ValueError("visual review cost status and value differ") + return self + + +class VisualReviewV1(_DigestBoundReviewModel): + digest_field = "review_digest" + schema_version: Literal["ari.visual-review/v1"] = VISUAL_REVIEW_V1 + target_kind: Literal["figure", "table"] + target_id: str + figure_id: str | None = None + source_manifest_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + target_artifact: VisualArtifactRefV1 + context_digest: str = Field(pattern=SHA256_PATTERN) + criteria_profile_id: str + criteria_profile_digest: str = Field(pattern=SHA256_PATTERN) + iteration: int = Field(default=0, ge=0, le=2) + status: Literal[ + "completed", + "artifact-error", + "limit-error", + "model-error", + "schema-error", + ] + score: float | None = Field(default=None, ge=0, le=1) + issues: tuple[VisualIssueV1, ...] = Field(default_factory=tuple, max_length=1_000) + summary: str = Field(default="", max_length=8_192) + model: str | None = Field(default=None, max_length=512) + model_revision: str | None = Field(default=None, max_length=512) + provider: str | None = Field(default=None, max_length=256) + prompt_digest: str | None = Field(default=None, pattern=SHA256_PATTERN) + sampling: dict[str, Any] = Field(default_factory=dict, max_length=64) + usage: VisualModelUsageV1 = Field(default_factory=VisualModelUsageV1) + raw_response_artifact: VisualArtifactRefV1 | None = None + error_kind: str | None = Field(default=None, max_length=256) + error_message: str | None = Field(default=None, max_length=4_096) + review_digest: str = Field(pattern=SHA256_PATTERN) + + @field_validator("target_id", "criteria_profile_id") + @classmethod + def _ids(cls, value: str) -> str: + if not _SAFE_ID.fullmatch(value): + raise ValueError("visual review identity is invalid") + return value + + @field_validator("sampling") + @classmethod + def _sampling(cls, value: dict[str, Any]) -> dict[str, Any]: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as exc: + raise ValueError("visual review sampling must be finite JSON") from exc + return value + + @model_validator(mode="after") + def _status_coherent(self) -> "VisualReviewV1": + if self.target_kind == "figure": + if self.figure_id != self.target_id or self.source_manifest_digest is None: + raise ValueError("figure review lacks manifest/figure identity") + elif self.figure_id is not None or self.source_manifest_digest is not None: + raise ValueError("table review cannot claim figure manifest identity") + if self.status == "completed": + if ( + self.score is None + or not self.model + or not self.prompt_digest + or self.raw_response_artifact is None + or self.error_kind is not None + ): + raise ValueError("completed visual review lacks model evidence") + elif self.score is not None or self.error_kind is None: + raise ValueError("failed visual review must retain an explicit typed error") + if self.raw_response_artifact is not None and ( + self.raw_response_artifact.role != "raw-model-response" + ): + raise ValueError("visual raw response artifact has the wrong role") + criteria = [item.criterion_id for item in self.issues] + issue_ids = [item.issue_id for item in self.issues] + if len(issue_ids) != len(set(issue_ids)) or any(not item for item in criteria): + raise ValueError("visual issues must have unique identities") + return self + + +class VisualReviewBatchV1(_DigestBoundReviewModel): + digest_field = "batch_review_digest" + schema_version: Literal["ari.visual-review-batch/v1"] = VISUAL_REVIEW_BATCH_V1 + source_batch_digest: str = Field(pattern=SHA256_PATTERN) + iteration: int = Field(ge=0, le=2) + reviews: tuple[VisualReviewV1, ...] = Field(min_length=1, max_length=100) + aggregation: Literal["minimum-fail-closed"] = "minimum-fail-closed" + score: float = Field(ge=0, le=1) + failure_count: int = Field(ge=0) + issues: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + suggestions: tuple[str, ...] = Field(default_factory=tuple, max_length=10_000) + review_text: str = Field(default="", max_length=100_000) + batch_review_digest: str = Field(pattern=SHA256_PATTERN) + + @model_validator(mode="after") + def _aggregate_matches(self) -> "VisualReviewBatchV1": + ids = [item.target_id for item in self.reviews] + if len(ids) != len(set(ids)): + raise ValueError("visual batch target IDs must be unique") + if any(item.iteration != self.iteration for item in self.reviews): + raise ValueError("visual batch mixes figure iterations") + failures = sum(item.status != "completed" for item in self.reviews) + if self.failure_count != failures: + raise ValueError("visual batch failure count is inconsistent") + expected = ( + 0.0 + if failures + else min(float(item.score) for item in self.reviews if item.score is not None) + ) + if self.score != expected: + raise ValueError("visual batch score is not minimum/fail-closed") + return self + + +def parse_visual_review(value: Any) -> VisualReviewV1: + try: + if isinstance(value, str): + value = json.loads(value) + return VisualReviewV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise VisualReviewContractError(f"invalid VisualReviewV1: {exc}") from exc + + +def parse_visual_review_batch(value: Any) -> VisualReviewBatchV1: + try: + if isinstance(value, str): + value = json.loads(value) + return VisualReviewBatchV1.model_validate(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise VisualReviewContractError( + f"invalid VisualReviewBatchV1: {exc}" + ) from exc + + +__all__ = [ + "VISUAL_ARTIFACT_REF_V1", + "VISUAL_CRITERIA_PROFILE_V1", + "VISUAL_REVIEW_BATCH_V1", + "VISUAL_REVIEW_V1", + "VisualArtifactRefV1", + "VisualCriteriaProfileV1", + "VisualCriterionV1", + "VisualIssueV1", + "VisualModelUsageV1", + "VisualRegionV1", + "VisualReviewBatchV1", + "VisualReviewContractError", + "VisualReviewV1", + "canonical_visual_review_digest", + "parse_visual_review", + "parse_visual_review_batch", +] diff --git a/ari-core/ari/viz/api_experiment.py b/ari-core/ari/viz/api_experiment.py index db155523..c9c82b7a 100644 --- a/ari-core/ari/viz/api_experiment.py +++ b/ari-core/ari/viz/api_experiment.py @@ -1,17 +1,16 @@ -from __future__ import annotations """ARI viz: api_experiment — launch, run stages, log streaming.""" +from __future__ import annotations + import json +import logging import os import re import subprocess -import threading -import time from pathlib import Path from . import state as _st -import logging log = logging.getLogger(__name__) @@ -137,6 +136,34 @@ def _api_launch(body: bytes) -> dict: data = json.loads(body) except (json.JSONDecodeError, TypeError, ValueError) as e: return {"ok": False, "error": f"Invalid request body: {e}"} + requested_retrieval = data.get("retrieval_backend") + if requested_retrieval and requested_retrieval not in { + "semantic_scholar", + "arxiv", + "alphaxiv", + }: + return { + "ok": False, + "error": "retrieval_backend must select one pinned provider", + } + if not requested_retrieval: + settings_path = _st._settings_path + if settings_path is not None and settings_path.is_file(): + try: + saved_retrieval = json.loads(settings_path.read_text()).get( + "retrieval_backend" + ) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + return {"ok": False, "error": f"Invalid project settings: {exc}"} + if saved_retrieval and saved_retrieval not in { + "semantic_scholar", + "arxiv", + "alphaxiv", + }: + return { + "ok": False, + "error": "saved retrieval_backend must select one pinned provider", + } profile = data.get("profile", "") experiment_md = data.get("experiment_md", "") # ── Trace: log received experiment_md from GUI ────────────────── @@ -298,6 +325,12 @@ def _api_launch(body: bytes) -> dict: val = saved.get(f"model_{skill}", "") if val: proc_env[f"ARI_MODEL_{skill.upper()}"] = val + if skill == "eval": + # The GUI keeps one evaluator-model field for + # compatibility, but the evaluator Skill has two + # independently addressable LLM operations. + proc_env["ARI_MODEL_METRIC_PROPOSAL"] = val + proc_env["ARI_MODEL_SEMANTIC_REVIEW"] = val # VLM review model from settings _vlm_model = saved.get("vlm_review_model", "") if _vlm_model: @@ -499,8 +532,6 @@ def _api_launch(body: bytes) -> dict: proc_env["ARI_RUBRIC_GEN_TARGET_LEAVES"] = str(int(wiz_ors["rubric_gen_target_leaves"])) if wiz_ors.get("rubric_gen_temperature") is not None: proc_env["ARI_RUBRIC_GEN_TEMPERATURE"] = str(float(wiz_ors["rubric_gen_temperature"])) - if wiz_ors.get("rubric_gen_two_stage") is not None: - proc_env["ARI_RUBRIC_GEN_TWO_STAGE"] = "1" if wiz_ors["rubric_gen_two_stage"] else "0" if wiz_ors.get("judge_n_runs") is not None: proc_env["ARI_JUDGE_N_RUNS"] = str(int(wiz_ors["judge_n_runs"])) # Replicator agent (v0.7+) — wall-clock budget and BasicAgent vs @@ -625,7 +656,8 @@ def _api_launch(body: bytes) -> dict: if isinstance(wiz_ors, dict) and wiz_ors: _launch_cfg["ors"] = {k: v for k, v in wiz_ors.items()} _st._launch_config = _launch_cfg - import time, shutil + import shutil + import time # Write log and launch_config.json inside pre-created checkpoint log_path = _pre_ckpt / f"ari_run_{int(time.time())}.log" _st._last_log_path = log_path @@ -910,5 +942,3 @@ def _emit(payload: dict) -> bool: break time.sleep(1) _emit({"msg": "[end of log]"}) - - diff --git a/ari-core/ari/viz/api_paperbench.py b/ari-core/ari/viz/api_paperbench.py index aa429225..2480d860 100644 --- a/ari-core/ari/viz/api_paperbench.py +++ b/ari-core/ari/viz/api_paperbench.py @@ -51,7 +51,6 @@ import re import shutil import threading -import time import urllib.error import urllib.parse import urllib.request @@ -572,9 +571,8 @@ def _estimate_cost(rubric_config: dict, reproduce_config: dict, judge_config: di tokens out per rubric pass; ~$1/M input + $5/M output). Tune as needed — these are advisory. """ - rubric_two_stage = bool(rubric_config.get("two_stage", True)) - rubric_walltime = 300 if rubric_two_stage else 180 # sec - rubric_cost_usd = 0.45 if rubric_two_stage else 0.20 + rubric_walltime = 300 # calibrated hierarchical skeleton + subtree generation + rubric_cost_usd = 0.45 reproduce_walltime = int(reproduce_config.get("time_limit_sec") or 12 * 3600) reproduce_cost_usd = 2.0 # ballpark for a 12 h BasicAgent rollout @@ -594,6 +592,28 @@ def _estimate_cost(rubric_config: dict, reproduce_config: dict, judge_config: di } +_RUBRIC_CONFIG_KEYS = frozenset( + { + "model", + "target_leaf_count", + "temperature", + "seed", + "paperbench_rubric_id", + "max_model_calls", + "subtree_concurrency", + "provider", + "model_revision", + } +) + + +def _rubric_config_error(config: dict) -> str | None: + unknown = sorted(set(config) - _RUBRIC_CONFIG_KEYS) + if unknown: + return "unknown rubric_config fields: " + ", ".join(unknown) + return None + + def _api_launch_run(body: dict) -> dict: """Enqueue PaperBench runs for the supplied paper_ids. @@ -621,6 +641,8 @@ def _api_launch_run(body: dict) -> dict: rubric_config = dict(body.get("rubric_config") or {}) reproduce_config = dict(body.get("reproduce_config") or {}) judge_config = dict(body.get("judge_config") or {}) + if error := _rubric_config_error(rubric_config): + return {"error": error} dry_run = bool(body.get("dry_run")) est = _estimate_cost(rubric_config, reproduce_config, judge_config) @@ -694,8 +716,11 @@ def _api_run_results(job_id: str) -> dict: def _api_cost_estimate(query: dict) -> dict: """Same shape as ``_api_launch_run`` with ``dry_run=True``, but exposed as GET for the wizard's live-update path.""" + rubric_config = dict(query.get("rubric_config") or {}) + if error := _rubric_config_error(rubric_config): + return {"error": error} return _estimate_cost( - rubric_config=query.get("rubric_config") or {}, + rubric_config=rubric_config, reproduce_config=query.get("reproduce_config") or {}, judge_config=query.get("judge_config") or {}, ) diff --git a/ari-core/ari/viz/api_paperbench_worker.py b/ari-core/ari/viz/api_paperbench_worker.py index c21548af..4462c875 100644 --- a/ari-core/ari/viz/api_paperbench_worker.py +++ b/ari-core/ari/viz/api_paperbench_worker.py @@ -94,16 +94,19 @@ def _generate_rubric_args(paper_pdf: Path, rubric_path: Path, cfg: dict) -> dict "paper_path": str(paper_pdf), "output_path": str(rubric_path), "model": cfg.get("model") or "", - "two_stage": bool(cfg.get("two_stage", True)), "target_leaf_count": int(cfg.get("target_leaf_count") or 0), "temperature": float(cfg.get("temperature") or 0.0), + "seed": int(cfg.get("seed") or 0), + "paperbench_rubric_id": str(cfg.get("paperbench_rubric_id") or ""), + "max_model_calls": int(cfg.get("max_model_calls") or 64), + "subtree_concurrency": int(cfg.get("subtree_concurrency") or 4), + "provider": str(cfg.get("provider") or ""), + "model_revision": str(cfg.get("model_revision") or ""), } def _build_reproduce_args(paper_pdf: Path, rubric_path: Path, repro_dir: Path, cfg: dict) -> dict: - # ``container_image`` is the wizard's unified field; ``apptainer_image`` is - # the legacy Stage 1-only alias still accepted for back-compat. - img = str(cfg.get("container_image") or cfg.get("apptainer_image") or "") + img = str(cfg.get("container_image") or "") return { "paper_path": str(paper_pdf), "rubric_path": str(rubric_path), @@ -113,7 +116,6 @@ def _build_reproduce_args(paper_pdf: Path, rubric_path: Path, repro_dir: Path, c "iterative_agent": bool(cfg.get("iterative_agent", False)), "sandbox_kind": str(cfg.get("sandbox_kind") or "auto"), "container_image": img, - "apptainer_image": img, # back-compat for callers still reading this key "max_steps": int(cfg.get("max_steps") or 0), } diff --git a/ari-core/ari/viz/api_settings.py b/ari-core/ari/viz/api_settings.py index f8142ff5..c0f95fbc 100644 --- a/ari-core/ari/viz/api_settings.py +++ b/ari-core/ari/viz/api_settings.py @@ -1,40 +1,18 @@ -from __future__ import annotations """ARI viz: api_settings — env keys, settings, workflow, skills, profiles.""" +from __future__ import annotations + import json import logging import os -import re from pathlib import Path from . import state as _st log = logging.getLogger(__name__) - - -def _extract_tools_from_server(skill_dir: Path) -> list[str]: - """Extract MCP tool names from server.py when mcp.json has no tools. - - Looks for two patterns: - - ``@mcp.tool()`` decorator followed by ``async def (`` or ``def (`` - - ``Tool(name=""`` in ``list_tools()`` style registration - """ - server_py = skill_dir / "src" / "server.py" - if not server_py.exists(): - return [] - try: - src = server_py.read_text() - except Exception: - return [] - tools: list[str] = [] - # Pattern 1: @mcp.tool() decorator - for m in re.finditer(r"@mcp\.tool\(\)\s*\n\s*(?:async\s+)?def\s+(\w+)\s*\(", src): - tools.append(m.group(1)) - # Pattern 2: Tool(name="...") - for m in re.finditer(r'Tool\(\s*name\s*=\s*"(\w+)"', src): - if m.group(1) not in tools: - tools.append(m.group(1)) - return tools +_PINNED_RETRIEVAL_BACKENDS = frozenset( + {"semantic_scholar", "arxiv", "alphaxiv"} +) def _api_get_env_keys() -> dict: @@ -60,7 +38,8 @@ def _api_get_env_keys() -> dict: continue if "=" in line: k, _, v = line.partition("=") - k = k.strip(); v = v.strip().strip('"').strip("'") + k = k.strip() + v = v.strip().strip('"').strip("'") if any(x in k.upper() for x in ["API_KEY", "SECRET", "TOKEN"]): if k not in keys: keys[k] = v @@ -172,7 +151,6 @@ def _api_get_settings() -> dict: "judge_model": os.environ.get("ARI_MODEL_JUDGE", "gpt-4o-2024-11-20"), "rubric_gen_temperature": 0.0, "rubric_gen_target_leaves": 0, - "rubric_gen_two_stage": True, "judge_n_runs": 3, "phase1_max_runtime_sec": 21600, "phase1_sandbox_kind": os.environ.get("ARI_PHASE1_SANDBOX", "auto"), @@ -201,6 +179,16 @@ def _api_get_settings() -> dict: def _api_save_settings(body: bytes) -> dict: data = json.loads(body) + retrieval_backend = data.get("retrieval_backend") + if ( + retrieval_backend is not None + and retrieval_backend not in _PINNED_RETRIEVAL_BACKENDS + ): + return { + "ok": False, + "error": "retrieval_backend must select one pinned provider", + "_status": 400, + } # Extract API key — write to .env instead of settings.json _raw_key = data.pop("api_key", "") or data.pop("llm_api_key", "") or "" # Also remove from the dict so it's never persisted in settings.json @@ -245,41 +233,33 @@ def _api_get_workflow() -> dict: if wf.exists(): try: data = yaml.safe_load(wf.read_text()) - # Load MCP tool metadata from each skill directory + # Load dashboard metadata from the same canonical manifests as + # runtime admission. Generated mcp.json and source scraping are + # deliberately not dashboard authorities. + from ari.skill_manifest import load_skill_manifest, manifest_digest + ari_root = wf.parent.parent.parent skill_mcp: dict = {} - # Build dir-name → mcp data mapping first dir_mcp: dict[str, dict] = {} - for skill_dir in sorted(ari_root.glob("ari-skill-*")): - mcp_file = skill_dir / "mcp.json" - tools: list = [] - mcp_name = skill_dir.name - mcp_desc = "" - mcp_ver = "" - if mcp_file.exists(): - try: - mcp_data = json.loads(mcp_file.read_text()) - mcp_name = mcp_data.get("name") or skill_dir.name - mcp_desc = mcp_data.get("description", "") - tools = mcp_data.get("tools", []) - mcp_ver = mcp_data.get("version", "") - except Exception: - log.debug("skill metadata read error", exc_info=True) - # Fallback: extract tool names from server.py if - # mcp.json has no tools listed - if not tools: - tools = _extract_tools_from_server(skill_dir) + for manifest_path in sorted(ari_root.glob("ari-skill-*/skill.yaml")): + skill_dir = manifest_path.parent + manifest = load_skill_manifest(manifest_path) + resolved_tools = manifest.resolved_tools() entry = { - "name": mcp_name, - "description": mcp_desc, - "tools": tools, - "version": mcp_ver, + "name": manifest.name, + "description": manifest.description, + "tools": [tool.name for tool in resolved_tools], + "version": manifest.version, "dir": skill_dir.name, + "manifest_digest": manifest_digest(manifest), + "capabilities": { + tool.name: tool.capability_ref for tool in resolved_tools + }, } dir_mcp[skill_dir.name] = entry skill_mcp[entry["name"]] = entry - # Resolve workflow.yaml skills section: map workflow skill - # names to their mcp.json tools via the path field + # Resolve workflow aliases to canonical manifest entries via + # the configured package path. for sk in data.get("skills", []): sk_name = sk.get("name", "") sk_path = sk.get("path", "") @@ -287,7 +267,7 @@ def _api_get_workflow() -> dict: resolved = sk_path.replace("{{ari_root}}", str(ari_root)) dir_name = Path(resolved).name if resolved else "" if dir_name and dir_name in dir_mcp: - # Merge mcp.json data under the workflow skill name + # Merge canonical data under the workflow skill name. src = dir_mcp[dir_name] entry = { "name": sk_name, @@ -295,12 +275,14 @@ def _api_get_workflow() -> dict: "tools": src["tools"], "version": src["version"], "dir": src["dir"], + "manifest_digest": src["manifest_digest"], + "capabilities": src["capabilities"], } # Read phase directly from workflow.yaml skills entry if sk.get("phase"): entry["phase"] = sk["phase"] skill_mcp[sk_name] = entry - # Remove the mcp.json alias if it differs from + # Remove the canonical alias if it differs from # the workflow name (e.g. vlm-review-skill vs # vlm-skill) to avoid duplicate entries mcp_alias = src["name"] @@ -348,31 +330,19 @@ def _api_get_workflow() -> dict: elif sk_name in paper_skills: entry["phase"] = "pipeline" - # Determine usage: stage / active / registered - # Scan core source for tool name references - core_dir = ari_root / "ari-core" / "ari" - _core_src = "" - if core_dir.is_dir(): - for py in core_dir.rglob("*.py"): - if "viz/" in str(py) or "__pycache__" in str(py): - continue - try: - _core_src += py.read_text(errors="ignore") - except Exception: - pass + # Usage is declarative: pipeline-owned, configured/active, or + # manifest-only/registered. Source-text references are not an + # execution contract. + configured_skills = { + str(skill.get("name") or "") for skill in data.get("skills", []) + } for sk_name, entry in skill_mcp.items(): if sk_name in bfts_skills or sk_name in paper_skills: entry["usage"] = "stage" + elif sk_name in configured_skills: + entry["usage"] = "active" else: - tool_names = [ - t if isinstance(t, str) else t.get("name", "") - for t in entry.get("tools", []) - ] - called = any( - f'"{tn}"' in _core_src or f"'{tn}'" in _core_src - for tn in tool_names if tn - ) - entry["usage"] = "active" if called else "registered" + entry["usage"] = "registered" # Read BFTS and paper pipelines from YAML (no hardcoded stages) bfts_pipeline = data.get("bfts_pipeline") or [] @@ -427,7 +397,6 @@ def _api_save_workflow(body: bytes) -> dict: def _api_skill_detail(name: str) -> dict: """Return skill source files and README.""" - import yaml as _yaml ari_root = Path(__file__).parent.parent.parent.parent skill_dir = ari_root / ("ari-skill-" + name.replace("ari-skill-", "").replace("-skill", "") + "-skill" if not name.startswith("ari-") else name) # Try multiple candidate names @@ -489,7 +458,9 @@ def _api_skills() -> list: data.setdefault("name", d.name) data.setdefault("display_name", d.name) data.setdefault("description", "") - data.setdefault("requires_env", []) + # Frontend compatibility while canonical manifests use the + # grammatically explicit required_env field. + data.setdefault("requires_env", data.get("required_env", [])) skills.append(data) except Exception: skills.append({"name": d.name, "display_name": d.name, "description": "", "requires_env": []}) @@ -549,5 +520,3 @@ def _api_rubrics() -> list: except Exception: continue return out - - diff --git a/ari-core/ari/viz/frontend/src/components/PaperBench/PaperBenchWizard.tsx b/ari-core/ari/viz/frontend/src/components/PaperBench/PaperBenchWizard.tsx index a837aa47..2f798ba7 100644 --- a/ari-core/ari/viz/frontend/src/components/PaperBench/PaperBenchWizard.tsx +++ b/ari-core/ari/viz/frontend/src/components/PaperBench/PaperBenchWizard.tsx @@ -5,7 +5,6 @@ import { ErrorState } from '../common'; interface RubricConfig { model: string; - two_stage: boolean; target_leaf_count: number; temperature: number; } @@ -63,7 +62,6 @@ export function PaperBenchWizard() { const [selectedIds, setSelectedIds] = useState>(new Set()); const [rubric, setRubric] = useState({ model: 'gemini/gemini-2.5-pro', - two_stage: true, target_leaf_count: 0, temperature: 0.0, }); @@ -182,14 +180,6 @@ export function PaperBenchWizard() { {t('pb_model')} setRubric({ ...rubric, model: e.target.value })} /> {' '} - {' '}