From 52d80d802bbaeeedfd56ba023171d1e33123713a Mon Sep 17 00:00:00 2001 From: overseek944 Date: Fri, 20 Mar 2026 00:46:00 +0530 Subject: [PATCH] feat: enhance Geval to support multiple contracts in checks - Updated the `geval check` command to accept multiple contract files, allowing users to evaluate several contracts against the same signals in a single PR. - Revised the decision artifact format to accommodate multi-contract evaluations, including overall decision reporting and individual contract results. - Enhanced documentation across README, examples, and CLI commands to reflect the new multi-contract functionality and provide clearer usage instructions. - Improved the explanation output to detail results for each contract and the overall decision, enhancing user understanding of the evaluation process. --- .github/workflows/geval.yml | 1 + geval/README.md | 10 +- geval/docs/architecture.md | 15 +- geval/docs/auditing.md | 21 +- geval/docs/extending.md | 4 +- geval/docs/github-actions.md | 10 +- geval/docs/versioning.md | 9 +- geval/examples/README.md | 10 +- geval/examples/contract-b.yaml | 6 + geval/src/artifact/mod.rs | 5 +- geval/src/artifact/writer.rs | 281 +++++++++++-- geval/src/cli/commands.rs | 158 +++++--- geval/src/cli/init.rs | 9 +- geval/src/contract/mod.rs | 4 +- geval/src/contract/runner.rs | 677 ++++++++++++++++++++++++++++++- geval/src/explanation/explain.rs | 128 +++++- geval/src/explanation/mod.rs | 4 +- geval/src/hashing/mod.rs | 2 +- geval/src/hashing/sha.rs | 29 ++ geval/src/lib.rs | 8 +- 20 files changed, 1251 insertions(+), 140 deletions(-) create mode 100644 geval/examples/contract-b.yaml diff --git a/.github/workflows/geval.yml b/.github/workflows/geval.yml index 6bc40cd..2330ee9 100644 --- a/.github/workflows/geval.yml +++ b/.github/workflows/geval.yml @@ -39,6 +39,7 @@ jobs: run: | ./geval/target/release/geval check \ --contract geval/examples/contract.yaml \ + --contract geval/examples/contract-b.yaml \ --signals signals.json \ --env prod continue-on-error: true diff --git a/geval/README.md b/geval/README.md index 4b603ea..c38e577 100644 --- a/geval/README.md +++ b/geval/README.md @@ -9,7 +9,7 @@ Geval consumes **signals** (JSON) and **policy** (YAML), evaluates rules in prio Geval is **not** an npm or pip package. It is a **single static binary** you run locally or in CI. - **Install:** Download a [release binary](https://github.com/geval/geval/releases) for your OS, or build from source with `cargo build --release`. -- **Integrate:** Run `geval check --signals signals.json --policy policy.yaml` in your repo; use exit codes (0/1/2) in CI or scripts. Your pipeline produces `signals.json` (e.g. via Node or Python); Geval only reads files and writes artifacts. +- **Integrate:** Run `geval check --contract contract.yaml --signals signals.json` (repeat `--contract` for multiple gates on one PR); use exit codes (0/1/2) in CI or scripts. Your pipeline produces `signals.json`; Geval only reads files and writes artifacts. See **[Installation](docs/installation.md)** for download links, build-from-source steps, and local/CI integration. @@ -31,7 +31,7 @@ cargo build --release **If you have a release binary:** ensure `geval` is on your PATH, then: ```bash -geval check --signals signals.json --policy policy.yaml --env prod +geval check --contract contract.yaml --signals signals.json --env prod ``` Exit codes: `0` = PASS, `1` = REQUIRE_APPROVAL, `2` = BLOCK. @@ -40,15 +40,15 @@ Exit codes: `0` = PASS, `1` = REQUIRE_APPROVAL, `2` = BLOCK. | Command | Description | |--------|-------------| -| `geval check` | Evaluate signals against policy; exit 0/1/2 | +| `geval check` | Evaluate signals against one or more contracts; exit 0/1/2 | | `geval approve` | Record human approval (e.g. for REQUIRE_APPROVAL) | | `geval reject` | Record human rejection | | `geval explain` | Print human-readable decision report | -| `geval validate-policy` | Validate policy file syntax | +| `geval validate-contract` | Validate contract file(s) and referenced policies | ## Artifacts -- **Decisions:** `.geval/decisions/.json` (policy_hash, signals_hash, decision, matched_rule) +- **Decisions:** `.geval/decisions/.json` (v3: per-contract results, `bundle_hash`, overall decision, signals_hash) - **Approval:** e.g. `.geval/approval.json` (approved_by, reason, timestamp) ## Docs diff --git a/geval/docs/architecture.md b/geval/docs/architecture.md index 58c6d4d..c991e0c 100644 --- a/geval/docs/architecture.md +++ b/geval/docs/architecture.md @@ -16,8 +16,9 @@ Geval is **contract-centric**: a **contract** is a named, versioned set of **pol 1. **Load contract** – Parse contract YAML; resolve policy paths relative to the contract file; load each policy. 2. **Load signals** – Parse signals JSON; build an in-memory signal graph (metric → value lookup). 3. **Evaluate each policy** – For each policy, evaluate rules in priority order; first matching rule gives that policy’s outcome (PASS / REQUIRE_APPROVAL / BLOCK). -4. **Combine** – Apply the contract’s combination rule to the list of policy outcomes → single combined decision. -5. **Artifact** – Write `.geval/decisions/.json` with contract identity, per-policy results, combined decision, and hashes. +4. **Combine (policies)** – Apply the contract’s combination rule to the list of policy outcomes → one combined decision **per contract**. +5. **Combine (contracts)** – If multiple contract files are passed (`geval check -c a.yaml -c b.yaml`), apply **`--combine-contracts`** to each contract’s combined outcome → one **overall** PR-level decision (same rule vocabulary: `all_pass`, `any_block_blocks`). +6. **Artifact** – Write `.geval/decisions/.json` (v3) with `bundle_hash`, each contract block, `contracts_combine_rule`, and overall outcome + hashes. ## Module layout @@ -27,7 +28,7 @@ geval/src/ model.rs # ContractDef, PolicyRef combine.rs # CombineRule (all_pass, any_block_blocks), apply_combine_rule loader.rs # load_contract, load_contract_and_policies, parse_contract_str - runner.rs # run_contract → ContractResult (per-policy + combined) + runner.rs # run_contract, load_run_contracts → ContractResult / MultiContractRun policy/ # Single policy model and parser model.rs # Policy, Rule, RuleCondition, RuleConsequence, Action, Operator parser.rs # parse_policy, parse_policy_str @@ -35,9 +36,9 @@ geval/src/ engine.rs # evaluate(policy, graph) → Decision; evaluate_with_trace signal_graph/ # Build lookup from signals for rule matching signals/ # Load signals JSON (name, version, signals array) - hashing/ # SHA256 for contract, policy, signals (audit) - artifact/ # Write decision artifact (contract + per-policy + combined) - explanation/ # Human-readable report (explain_contract_result, explain_decision) + hashing/ # SHA256 for contract, policy, signals, contract bundle (audit) + artifact/ # write_multi_contract_artifact (v3: multi-contract + overall) + explanation/ # explain_contract_result, explain_multi_contract_result, explain_decision approval/ # Approval/rejection artifact (versioned) cli/ # Commands: check, init, demo, explain, validate-contract, approve, reject ``` @@ -45,7 +46,7 @@ geval/src/ ## Invariants - **Nothing unversioned** – Contract, policies, and signals have name/version; artifact records them and hashes. -- **Deterministic** – Same contract + same signals → same combined decision. +- **Deterministic** – Same contract set (order) + same signals → same overall decision. - **No remote calls** – All inputs and outputs are local files. ## Adding a new combination rule diff --git a/geval/docs/auditing.md b/geval/docs/auditing.md index a88bda2..285e3c7 100644 --- a/geval/docs/auditing.md +++ b/geval/docs/auditing.md @@ -4,7 +4,7 @@ Geval is designed so **nothing is unversioned**: every decision and every action - **Why was this deployed?** – Decision report and matched rule (and optional approval reason). - **Who approved it?** – `geval approve` writes an artifact with `approved_by`, `reason`, and artifact `version`. -- **What policy (contract) was used?** – Artifact stores `policy_name`, `policy_version`, and `policy_hash` (SHA256). +- **What policy (contract) was used?** – Artifact stores each contract’s identity, `contract_hash`, per-policy hashes, and (v3) `bundle_hash` for the ordered set of contracts. - **What signals were used?** – Artifact stores `signals_name`, `signals_version`, and `signals_hash` (SHA256). - **Which Geval binary?** – Artifact stores `geval_version`. @@ -17,18 +17,23 @@ Geval is designed so **nothing is unversioned**: every decision and every action Each `geval check` run writes: - **Path:** `.geval/decisions/.json` -- **Contents (artifact_version 2, contract-centric):** - - `artifact_version` – schema version (e.g. `"2"`) +- **Contents (artifact_version 3, multi-contract):** + - `artifact_version` – schema version (`"3"`) - `geval_version` – binary version that produced the decision - - `contract_name`, `contract_version`, `contract_hash` – contract identity and content hash + - `bundle_hash` – SHA256 over the ordered list of `(contract_path, contract_hash)` (audit the exact contract set) + - `contracts_combine_rule` – how each contract’s **combined** outcome was merged (e.g. `all_pass`, `any_block_blocks`) + - `contracts` – array of blocks, each with: + - `contract_path`, `contract_name`, `contract_version`, `contract_hash` + - `combine_rule` (policies within that contract) + - `policy_results` – `{ policy_path, policy_name?, policy_version?, policy_hash, outcome, matched_rule? }[]` + - `combined_decision`, `combined_matched_rule`, `combined_reason` (outcome for that contract) + - `overall_combined_decision`, `overall_matched_rule`, `overall_reason` – PR-level outcome after `contracts_combine_rule` - `signals_name`, `signals_version`, `signals_hash` – signals identity and content hash - - `combine_rule` – how policy outcomes were merged (e.g. `all_pass`, `any_block_blocks`) - - `policy_results` – array of `{ policy_path, policy_name?, policy_version?, policy_hash, outcome, matched_rule? }` for each policy - - `combined_decision` – final outcome (PASS | REQUIRE_APPROVAL | BLOCK) - - `combined_matched_rule` – first non-PASS policy and rule (if any) - `timestamp` – ISO8601 - `approval` – optional; set when an approval is recorded for this decision +Older tooling may still reference **artifact_version 2** (single flat contract); Geval now writes v3 only. + ### Approval artifact `geval approve` / `geval reject` write: diff --git a/geval/docs/extending.md b/geval/docs/extending.md index 407de44..4d318f7 100644 --- a/geval/docs/extending.md +++ b/geval/docs/extending.md @@ -22,7 +22,7 @@ This document describes how to change or extend Geval in a consistent, testable - **Contract** – `contract/model.rs`, `contract/loader.rs`, `contract/runner.rs`, `contract/combine.rs`. - **Policy** – `policy/model.rs`, `policy/parser.rs`; then `evaluator/engine.rs` if rule semantics change. - **Signals** – `signals/loader.rs`, `signal_graph/` if lookup behavior changes. -- **Artifact** – `artifact/writer.rs`; bump `DECISION_ARTIFACT_VERSION` if the JSON shape changes. +- **Artifact** – `artifact/writer.rs` (`write_multi_contract_artifact`); bump `DECISION_ARTIFACT_VERSION` if the JSON shape changes. - **CLI** – `cli/commands.rs`; add or update subcommands/args. Keep functions small and pure where possible; use `anyhow::Result` and `Context` for errors. @@ -59,7 +59,7 @@ Run: `cargo test --manifest-path geval/Cargo.toml`. 1. **Model** – Add the field to `ContractDef` or `Policy` in the appropriate `model.rs`; use `Option` and `#[serde(default)]` for backward compatibility if we still support old files. 2. **Parser** – If the field comes from YAML, ensure the parser (contract loader or policy parser) reads it and fills the model. -3. **Artifact** – If the field should be audited, add it to `DecisionArtifact` (and to the code that builds the artifact from `ContractResult` and versions). +3. **Artifact** – If the field should be audited, add it to `DecisionArtifactV3` / `ContractDecisionBlock` in `artifact/writer.rs` (and the code that builds from `MultiContractRun`). 4. **Tests** – Parse a sample YAML with the new field and assert it’s present; if the field affects evaluation, add an evaluator or runner test. ## Adding a new CLI command diff --git a/geval/docs/github-actions.md b/geval/docs/github-actions.md index 9b7a895..939d7df 100644 --- a/geval/docs/github-actions.md +++ b/geval/docs/github-actions.md @@ -35,10 +35,13 @@ jobs: run: | ./geval/target/release/geval check \ --contract contract.yaml \ + --contract other-team/contract.yaml \ --signals signals.json \ --env prod ``` +Repeat `--contract` for each gate YAML attached to the PR. Optional: `--combine-contracts all_pass` (default) or `any_block_blocks`. + ## Option B: Download released binary Use when you rely on an official Geval release: @@ -57,6 +60,7 @@ Use when you rely on an official Geval release: run: | ./geval check \ --contract contract.yaml \ + --contract other-team/contract.yaml \ --signals signals.json ``` @@ -72,7 +76,7 @@ Use these in a later step to fail the job on BLOCK or REQUIRE_APPROVAL if desire - name: Run Geval id: geval run: | - ./geval check --contract contract.yaml --signals signals.json --env prod + ./geval check --contract contract.yaml --contract other-team/contract.yaml --signals signals.json --env prod echo "exitcode=$?" >> $GITHUB_OUTPUT ``` @@ -81,13 +85,13 @@ Then `if: steps.geval.outputs.exitcode == '0'` for merge gates. ## Post result to PR (GitHub CLI) ```bash -RESULT=$(./geval check --contract contract.yaml --signals signals.json) +RESULT=$(./geval check --contract contract.yaml --contract other-team/contract.yaml --signals signals.json) gh pr comment $PR_NUMBER --body "$RESULT" ``` Or capture the explain output: ```bash -RESULT=$(./geval explain --contract contract.yaml --signals signals.json) +RESULT=$(./geval explain --contract contract.yaml --contract other-team/contract.yaml --signals signals.json) gh pr comment $PR_NUMBER --body "$RESULT" ``` diff --git a/geval/docs/versioning.md b/geval/docs/versioning.md index 58ce01c..06557cd 100644 --- a/geval/docs/versioning.md +++ b/geval/docs/versioning.md @@ -57,13 +57,14 @@ The decision artifact records `signals_name` and `signals_version` when present. Every `geval check` writes a versioned artifact to `.geval/decisions/.json`: -- **artifact_version** – Schema version of the artifact format. +- **artifact_version** – Schema version (current: **3** — multi-contract). - **geval_version** – Geval binary version that produced the decision. -- **policy_name**, **policy_version** – From the policy (contract) file. +- **contracts** – Each contract’s `contract_name`, `contract_version`, `contract_hash`, and per-policy `policy_name` / `policy_version` / `policy_hash` when present. +- **bundle_hash** – Hash of the ordered contract set (paths + content hashes). - **signals_name**, **signals_version** – From the signals file. -- **policy_hash**, **signals_hash** – Content hashes (SHA256) for integrity. +- **signals_hash** – Content hash (SHA256) for integrity. -So every decision is fully traceable: which contract version, which signals version, which binary. +So every decision is fully traceable: which contract versions (one or many), which signals version, which binary. ## Approval artifact diff --git a/geval/examples/README.md b/geval/examples/README.md index 16dc74c..e436069 100644 --- a/geval/examples/README.md +++ b/geval/examples/README.md @@ -5,6 +5,7 @@ Example contract, policies, and signals for Geval (decision orchestration and re ## Files - **contract.yaml** – Contract: name, version, combine rule, and list of policy paths. This example references a single policy. +- **contract-b.yaml** – Second contract (distinct `name`), same `policy.yaml` — used to demo **multiple contracts** on one PR. - **policy.yaml** – One policy with priority-ordered rules: business block, hallucination guard, retrieval quality. - **signals.json** – Example signals (eval metrics, A/B metrics, component-level). @@ -16,11 +17,18 @@ cargo build --release --manifest-path geval/Cargo.toml # Check: evaluate signals against contract (exit 0=PASS, 1=REQUIRE_APPROVAL, 2=BLOCK) ./geval/target/release/geval check --contract geval/examples/contract.yaml --signals geval/examples/signals.json --env prod +# Multiple contracts (same signals): repeat --contract / -c +./geval/target/release/geval check \ + --contract geval/examples/contract.yaml \ + --contract geval/examples/contract-b.yaml \ + --signals geval/examples/signals.json --env prod + # Explain: human-readable report (per-policy + combined) ./geval/target/release/geval explain --contract geval/examples/contract.yaml --signals geval/examples/signals.json --env prod -# Validate contract and all referenced policies +# Validate one or more contract files ./geval/target/release/geval validate-contract geval/examples/contract.yaml +./geval/target/release/geval validate-contract geval/examples/contract.yaml geval/examples/contract-b.yaml ``` With the example data, the policy matches `business_block`: `engagement_drop` 0.03 > 0, so the decision is **BLOCK**. diff --git a/geval/examples/contract-b.yaml b/geval/examples/contract-b.yaml new file mode 100644 index 0000000..7f2210c --- /dev/null +++ b/geval/examples/contract-b.yaml @@ -0,0 +1,6 @@ +# Second example contract: same policy file, distinct contract identity (multi-contract PR demo). +name: demo-secondary +version: "1.0.0" +combine: all_pass +policies: + - path: policy.yaml diff --git a/geval/src/artifact/mod.rs b/geval/src/artifact/mod.rs index 08487f6..39ae22b 100644 --- a/geval/src/artifact/mod.rs +++ b/geval/src/artifact/mod.rs @@ -1,3 +1,6 @@ mod writer; -pub use writer::{write_decision_artifact, DECISION_ARTIFACT_VERSION}; +pub use writer::{ + write_multi_contract_artifact, ApprovalPayload, ContractDecisionBlock, DecisionArtifactV3, + PolicyResultRecord, DECISION_ARTIFACT_VERSION, +}; diff --git a/geval/src/artifact/writer.rs b/geval/src/artifact/writer.rs index 32e9233..6329072 100644 --- a/geval/src/artifact/writer.rs +++ b/geval/src/artifact/writer.rs @@ -1,17 +1,19 @@ //! Write decision artifacts to .geval/decisions/.json //! -//! Contract-centric: every artifact records the contract (name, version, combine rule), -//! per-policy results, and the combined decision. Nothing is unversioned. +//! Multi-contract: artifact v3 records each contract (path, hashes, per-policy results) plus +//! `contracts_combine_rule`, `bundle_hash`, and overall PR-level decision. -use crate::contract::ContractResult; +use crate::contract::MultiContractRun; use crate::evaluator::DecisionOutcome; use anyhow::{Context, Result}; use chrono::Utc; use serde::Serialize; use std::path::Path; +use crate::hashing::hash_contract_bundle; + /// Schema version of the decision artifact format. Bump when the artifact shape changes. -pub const DECISION_ARTIFACT_VERSION: &str = "2"; +pub const DECISION_ARTIFACT_VERSION: &str = "3"; /// Per-policy result as stored in the artifact. #[derive(Debug, Serialize)] @@ -27,24 +29,40 @@ pub struct PolicyResultRecord { pub matched_rule: Option, } -/// Artifact written per run. Contract-centric; all versioned. +/// One contract’s slice of the artifact (mirrors former v2 single-contract payload, nested). #[derive(Debug, Serialize)] -pub struct DecisionArtifact { - pub artifact_version: String, - pub geval_version: String, +pub struct ContractDecisionBlock { + pub contract_path: String, pub contract_name: String, pub contract_version: String, pub contract_hash: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub signals_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub signals_version: Option, - pub signals_hash: String, pub combine_rule: String, pub policy_results: Vec, pub combined_decision: String, #[serde(skip_serializing_if = "Option::is_none")] pub combined_matched_rule: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub combined_reason: Option, +} + +/// Multi-contract decision artifact (v3). +#[derive(Debug, Serialize)] +pub struct DecisionArtifactV3 { + pub artifact_version: String, + pub geval_version: String, + pub bundle_hash: String, + pub contracts_combine_rule: String, + pub contracts: Vec, + pub overall_combined_decision: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub overall_matched_rule: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub overall_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signals_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signals_version: Option, + pub signals_hash: String, pub timestamp: String, pub approval: Option, } @@ -56,12 +74,45 @@ pub struct ApprovalPayload { pub timestamp: String, } -/// Write artifact to .geval/decisions/.json -pub fn write_decision_artifact( +fn policy_records_for_contract( + entry: &crate::contract::ContractRunEntry, +) -> Vec { + entry + .result + .policy_results + .iter() + .zip(entry.policy_hashes.iter()) + .map(|(r, hash)| PolicyResultRecord { + policy_path: r.policy_path.clone(), + policy_name: r.policy_name.clone(), + policy_version: r.policy_version.clone(), + policy_hash: hash.clone(), + outcome: outcome_str(r.outcome).to_string(), + matched_rule: r.matched_rule.clone(), + }) + .collect() +} + +fn contract_block(entry: &crate::contract::ContractRunEntry) -> ContractDecisionBlock { + let result = &entry.result; + let combined_decision_str = outcome_str(result.combined_decision.outcome).to_string(); + ContractDecisionBlock { + contract_path: entry.contract_path.display().to_string(), + contract_name: result.contract_name.clone(), + contract_version: result.contract_version.clone(), + contract_hash: entry.contract_hash.clone(), + combine_rule: result.combine_rule.to_string(), + policy_results: policy_records_for_contract(entry), + combined_decision: combined_decision_str, + combined_matched_rule: result.combined_decision.matched_rule.clone(), + combined_reason: result.combined_decision.reason.clone(), + } +} + +/// Write multi-contract artifact to `.geval/decisions/.json`. +pub fn write_multi_contract_artifact( dir: &Path, - result: &ContractResult, - contract_hash: &str, - policy_hashes: &[String], + run: &MultiContractRun, signals_hash: &str, signals_name: Option<&str>, signals_version: Option<&str>, @@ -72,44 +123,33 @@ pub fn write_decision_artifact( .with_context(|| format!("create {}", decisions_dir.display()))?; let now = Utc::now(); let ts_iso = now.format("%Y-%m-%dT%H:%M:%SZ").to_string(); - // Windows does not allow ':' in filenames; use dashes in time part for the filename only. let ts_filename = now.format("%Y-%m-%dT%H-%M-%SZ"); let filename = format!("{}.json", ts_filename); let path = decisions_dir.join(&filename); - let policy_results: Vec = result - .policy_results + let bundle_pairs: Vec<(std::path::PathBuf, &str)> = run + .entries .iter() - .zip(policy_hashes.iter()) - .map(|(r, hash)| PolicyResultRecord { - policy_path: r.policy_path.clone(), - policy_name: r.policy_name.clone(), - policy_version: r.policy_version.clone(), - policy_hash: hash.clone(), - outcome: outcome_str(r.outcome).to_string(), - matched_rule: r.matched_rule.clone(), - }) + .map(|e| (e.contract_path.clone(), e.contract_hash.as_str())) .collect(); + let bundle_hash = hash_contract_bundle(&bundle_pairs); - let combined_decision_str = match result.combined_decision.outcome { - DecisionOutcome::Pass => "PASS", - DecisionOutcome::RequireApproval => "REQUIRE_APPROVAL", - DecisionOutcome::Block => "BLOCK", - }; + let contracts: Vec = run.entries.iter().map(contract_block).collect(); - let artifact = DecisionArtifact { + let overall_combined_decision = outcome_str(run.overall.outcome).to_string(); + + let artifact = DecisionArtifactV3 { artifact_version: DECISION_ARTIFACT_VERSION.to_string(), geval_version: crate::GEVAL_VERSION.to_string(), - contract_name: result.contract_name.clone(), - contract_version: result.contract_version.clone(), - contract_hash: contract_hash.to_string(), + bundle_hash, + contracts_combine_rule: run.contracts_combine.to_string(), + contracts, + overall_combined_decision, + overall_matched_rule: run.overall.matched_rule.clone(), + overall_reason: run.overall.reason.clone(), signals_name: signals_name.map(String::from), signals_version: signals_version.map(String::from), signals_hash: signals_hash.to_string(), - combine_rule: result.combine_rule.to_string(), - policy_results, - combined_decision: combined_decision_str.to_string(), - combined_matched_rule: result.combined_decision.matched_rule.clone(), timestamp: ts_iso, approval, }; @@ -126,3 +166,158 @@ fn outcome_str(o: DecisionOutcome) -> &'static str { DecisionOutcome::Block => "BLOCK", } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::contract::load_run_contracts; + use crate::contract::CombineRule; + use crate::signal_graph::SignalGraph; + use crate::signals::{Signal, SignalSet}; + + fn sig(metric: &str, value: f64) -> Signal { + Signal { + system: None, + agent: None, + component: None, + step: None, + metric: Some(metric.to_string()), + value: Some(serde_json::json!(value)), + r#type: None, + } + } + + fn minimal_pass_contract(dir: &std::path::Path, name: &str, contract_file: &str, policy_file: &str) { + let p = dir.join(policy_file); + std::fs::write( + &p, + r#"rules: [{ priority: 1, name: ok, when: { metric: x, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + let c = dir.join(contract_file); + std::fs::write( + &c, + format!( + r#"name: {} +version: "1.0.0" +combine: all_pass +policies: + - path: {} +"#, + name, policy_file + ), + ) + .unwrap(); + } + + #[test] + fn write_multi_contract_artifact_is_valid_v3_json() { + let dir = tempfile::tempdir().unwrap(); + minimal_pass_contract(dir.path(), "a", "ca.yaml", "pa.yaml"); + minimal_pass_contract(dir.path(), "b", "cb.yaml", "pb.yaml"); + let c1 = dir.path().join("ca.yaml"); + let c2 = dir.path().join("cb.yaml"); + + let signals = SignalSet::new(vec![sig("x", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1, c2], &graph, CombineRule::AllPass).unwrap(); + + let out_dir = tempfile::tempdir().unwrap(); + let path = write_multi_contract_artifact( + out_dir.path(), + &run, + "deadbeefsignals", + Some("ci-signals"), + Some("1.0.0"), + None, + ) + .unwrap(); + + let json = std::fs::read_to_string(&path).unwrap(); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(v["artifact_version"], "3"); + assert_eq!(v["contracts_combine_rule"], "all_pass"); + assert_eq!(v["overall_combined_decision"], "PASS"); + assert!(v["bundle_hash"].as_str().unwrap().len() == 64); + assert_eq!(v["contracts"].as_array().unwrap().len(), 2); + assert_eq!(v["signals_hash"], "deadbeefsignals"); + assert_eq!(v["signals_name"], "ci-signals"); + assert!(v["timestamp"].as_str().unwrap().contains('T')); + } + + #[test] + fn write_multi_contract_artifact_nested_policy_results() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("policy.yaml"); + std::fs::write( + &p, + r#"rules: + - priority: 1 + name: block_x + when: + metric: x + operator: ">" + threshold: 10 + then: + action: block +"#, + ) + .unwrap(); + let c = dir.path().join("contract.yaml"); + std::fs::write( + &c, + r#"name: solo +version: "1.0.0" +combine: all_pass +policies: + - path: policy.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig("x", 99.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c], &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.entries.len(), 1); + + let out_dir = tempfile::tempdir().unwrap(); + let path = write_multi_contract_artifact(out_dir.path(), &run, "hash", None, None, None).unwrap(); + let json = std::fs::read_to_string(&path).unwrap(); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + let contracts = v["contracts"].as_array().unwrap(); + assert_eq!(contracts.len(), 1); + let pr = contracts[0]["policy_results"].as_array().unwrap(); + assert_eq!(pr.len(), 1); + assert_eq!(pr[0]["outcome"], "BLOCK"); + assert_eq!(v["overall_combined_decision"], "BLOCK"); + } + + #[test] + fn approval_payload_round_trips_in_json() { + let dir = tempfile::tempdir().unwrap(); + minimal_pass_contract(dir.path(), "x", "c.yaml", "p.yaml"); + let c = dir.path().join("c.yaml"); + let signals = SignalSet::new(vec![sig("x", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c], &graph, CombineRule::AllPass).unwrap(); + let out_dir = tempfile::tempdir().unwrap(); + let approval = ApprovalPayload { + approved_by: "alice".to_string(), + reason: "lgtm".to_string(), + timestamp: "2020-01-01T00:00:00Z".to_string(), + }; + let path = write_multi_contract_artifact( + out_dir.path(), + &run, + "h", + None, + None, + Some(approval), + ) + .unwrap(); + let json = std::fs::read_to_string(&path).unwrap(); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(v["approval"]["approved_by"], "alice"); + assert_eq!(v["approval"]["reason"], "lgtm"); + } +} diff --git a/geval/src/cli/commands.rs b/geval/src/cli/commands.rs index cc4575d..4b10398 100644 --- a/geval/src/cli/commands.rs +++ b/geval/src/cli/commands.rs @@ -5,14 +5,15 @@ use clap::{Parser, Subcommand}; use std::path::PathBuf; use crate::approval::write_approval; -use crate::artifact::write_decision_artifact; +use crate::artifact::write_multi_contract_artifact; use crate::cli::{demo_ui::print_demo_report, init::run_init as do_init}; use crate::contract::{ - load_contract_and_policies, run_contract, CombineRule, ContractDef, PolicyRef, + load_contract_and_policies, load_run_contracts, run_contract, CombineRule, ContractDef, + PolicyRef, }; use crate::evaluator::{evaluate_with_trace, DecisionOutcome}; -use crate::explanation::explain_contract_result; -use crate::hashing::{hash_contract_content, hash_policy, hash_signals}; +use crate::explanation::explain_multi_contract_result; +use crate::hashing::{hash_contract_bundle, hash_signals}; use crate::policy::parse_policy_str; use crate::signal_graph::SignalGraph; use crate::signals::load_signals_from_reader; @@ -30,7 +31,7 @@ pub struct Commands { #[derive(Subcommand)] pub enum Sub { - /// Evaluate signals against a contract (multiple policies); exit 0=PASS, 1=REQUIRE_APPROVAL, 2=BLOCK. + /// Evaluate signals against one or more contracts; exit 0=PASS, 1=REQUIRE_APPROVAL, 2=BLOCK. Check(CheckOpts), /// Create a template folder with contract and policies. Edit and run. Init(InitOpts), @@ -40,9 +41,9 @@ pub enum Sub { Approve(ApproveOpts), /// Record human rejection. Reject(RejectOpts), - /// Print human-readable decision report (contract + per-policy + combined). + /// Print human-readable decision report (multi-contract + overall). Explain(ExplainOpts), - /// Validate contract file and all referenced policies. + /// Validate one or more contract files and all referenced policies. ValidateContract(ValidateContractOpts), } @@ -106,12 +107,24 @@ policy: action: require_approval "#; +fn parse_combine_rule(s: &str) -> Result { + s.parse() +} + #[derive(clap::Args)] pub struct CheckOpts { #[arg(long, short = 's')] pub signals: PathBuf, - #[arg(long, short = 'c')] - pub contract: PathBuf, + /// Contract YAML file(s); repeat for multiple contracts on one PR. + #[arg(long, short = 'c', action = clap::ArgAction::Append, required = true)] + pub contract: Vec, + /// How to merge each contract’s combined outcome (default: all must pass). + #[arg( + long = "combine-contracts", + default_value = "all_pass", + value_parser = parse_combine_rule + )] + pub combine_contracts: CombineRule, #[arg(long, short = 'e', env = "GEVAL_ENV")] pub env: Option, #[arg(long)] @@ -142,15 +155,23 @@ pub struct RejectOpts { pub struct ExplainOpts { #[arg(long, short = 's')] pub signals: PathBuf, - #[arg(long, short = 'c')] - pub contract: PathBuf, + #[arg(long, short = 'c', action = clap::ArgAction::Append, required = true)] + pub contract: Vec, + #[arg( + long = "combine-contracts", + default_value = "all_pass", + value_parser = parse_combine_rule + )] + pub combine_contracts: CombineRule, #[arg(long, short = 'e', env = "GEVAL_ENV")] pub env: Option, } #[derive(clap::Args)] pub struct ValidateContractOpts { - pub contract: PathBuf, + /// One or more contract YAML files to validate. + #[arg(required = true)] + pub contract: Vec, #[arg(long)] pub json: bool, } @@ -194,6 +215,10 @@ fn run_init(opts: &InitOpts) -> Result<()> { opts.directory.display(), opts.directory.display() ); + println!( + "Multiple contracts: geval check -c path/a.yaml -c path/b.yaml --signals {}/signals.json", + opts.directory.display() + ); Ok(()) } @@ -237,23 +262,24 @@ fn run_demo(opts: &DemoOpts) -> Result<()> { } fn run_check(opts: &CheckOpts) -> Result<()> { - let (contract, policies) = - load_contract_and_policies(&opts.contract).context("load contract and policies")?; let signals = crate::signals::load_signals(&opts.signals).context("load signals")?; let graph = SignalGraph::build(&signals.signals); - let result = run_contract(&contract, &policies, &graph).context("run contract")?; + let run = load_run_contracts(&opts.contract, &graph, opts.combine_contracts) + .context("run contracts")?; - let contract_hash = hash_contract_content(&contract); - let policy_hashes: Vec = policies.iter().map(hash_policy).collect(); let signals_hash = hash_signals(&signals); + let bundle_pairs: Vec<(PathBuf, &str)> = run + .entries + .iter() + .map(|e| (e.contract_path.clone(), e.contract_hash.as_str())) + .collect(); + let bundle_hash = hash_contract_bundle(&bundle_pairs); let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - let _ = write_decision_artifact( + let _ = write_multi_contract_artifact( &cwd, - &result, - &contract_hash, - &policy_hashes, + &run, &signals_hash, signals.name.as_deref(), signals.version.as_deref(), @@ -263,24 +289,34 @@ fn run_check(opts: &CheckOpts) -> Result<()> { if opts.json { let out = serde_json::json!({ - "contract": result.contract_name, - "combined_decision": outcome_str(result.combined_decision.outcome), - "combine_rule": result.combine_rule.to_string(), - "policy_results": result.policy_results.iter().map(|r| serde_json::json!({ - "policy_path": r.policy_path, - "outcome": outcome_str(r.outcome), - "matched_rule": r.matched_rule, + "contracts_combine_rule": run.contracts_combine.to_string(), + "bundle_hash": bundle_hash, + "contracts": run.entries.iter().map(|e| serde_json::json!({ + "contract_path": e.contract_path, + "contract_name": e.result.contract_name, + "contract_version": e.result.contract_version, + "contract_hash": e.contract_hash, + "combine_rule": e.result.combine_rule.to_string(), + "policy_results": e.result.policy_results.iter().map(|r| serde_json::json!({ + "policy_path": r.policy_path, + "outcome": outcome_str(r.outcome), + "matched_rule": r.matched_rule, + })).collect::>(), + "combined_decision": outcome_str(e.result.combined_decision.outcome), })).collect::>(), + "overall_combined_decision": outcome_str(run.overall.outcome), + "overall_matched_rule": run.overall.matched_rule, + "overall_reason": run.overall.reason, }); println!("{}", serde_json::to_string_pretty(&out)?); } else { println!( "{}", - explain_contract_result(&result, &graph, opts.env.as_deref()) + explain_multi_contract_result(&run, &graph, opts.env.as_deref()) ); } - let code = match result.combined_decision.outcome { + let code = match run.overall.outcome { DecisionOutcome::Pass => 0, DecisionOutcome::RequireApproval => 1, DecisionOutcome::Block => 2, @@ -311,45 +347,57 @@ fn run_reject(opts: &RejectOpts) -> Result<()> { } fn run_explain(opts: &ExplainOpts) -> Result<()> { - let (contract, policies) = - load_contract_and_policies(&opts.contract).context("load contract and policies")?; let signals = crate::signals::load_signals(&opts.signals).context("load signals")?; let graph = SignalGraph::build(&signals.signals); - let result = run_contract(&contract, &policies, &graph).context("run contract")?; + let run = load_run_contracts(&opts.contract, &graph, opts.combine_contracts) + .context("run contracts")?; println!( "{}", - explain_contract_result(&result, &graph, opts.env.as_deref()) + explain_multi_contract_result(&run, &graph, opts.env.as_deref()) ); Ok(()) } fn run_validate_contract(opts: &ValidateContractOpts) -> Result<()> { - let (contract, policies) = - load_contract_and_policies(&opts.contract).context("load contract and policies")?; + let mut summaries = Vec::new(); + for path in &opts.contract { + let (contract, policies) = + load_contract_and_policies(path).with_context(|| format!("{}", path.display()))?; + summaries.push((path.clone(), contract, policies)); + } if opts.json { - let out = serde_json::json!({ - "name": contract.name, - "version": contract.version, - "combine": contract.combine.to_string(), - "policies": contract.policies.iter().map(|p| &p.path).collect::>(), - "policy_count": policies.len(), - }); + let out: Vec = summaries + .iter() + .map(|(path, contract, policies)| { + serde_json::json!({ + "contract_path": path, + "name": contract.name, + "version": contract.version, + "combine": contract.combine.to_string(), + "policies": contract.policies.iter().map(|p| &p.path).collect::>(), + "policy_count": policies.len(), + }) + }) + .collect(); println!("{}", serde_json::to_string_pretty(&out)?); } else { - println!( - "Contract valid: {} (version {}), {} policy/policies, combine={}", - contract.name, - contract.version, - policies.len(), - contract.combine - ); - for (i, (pref, policy)) in contract.policies.iter().zip(policies.iter()).enumerate() { + for (path, contract, policies) in &summaries { println!( - " {}: {} ({} rule(s))", - i + 1, - pref.path, - policy.rules.len() + "Contract valid: {} — {} (version {}), {} policy/policies, combine={}", + path.display(), + contract.name, + contract.version, + policies.len(), + contract.combine ); + for (i, (pref, policy)) in contract.policies.iter().zip(policies.iter()).enumerate() { + println!( + " {}: {} ({} rule(s))", + i + 1, + pref.path, + policy.rules.len() + ); + } } } Ok(()) diff --git a/geval/src/cli/init.rs b/geval/src/cli/init.rs index 82ecbfb..87a4c37 100644 --- a/geval/src/cli/init.rs +++ b/geval/src/cli/init.rs @@ -133,6 +133,12 @@ From the **project root**: geval check --contract {}/contract.yaml --signals {}/signals.json ``` +Multiple contracts (one PR, same signals): + +```bash +geval check -c {}/contract.yaml -c path/to/other-contract.yaml --signals {}/signals.json +``` + Explain: ```bash @@ -143,6 +149,7 @@ Validate contract and all policies: ```bash geval validate-contract {}/contract.yaml +geval validate-contract {}/contract.yaml path/to/other-contract.yaml ``` ## Approve / reject @@ -156,7 +163,7 @@ geval reject --reason "Needs more testing" --output {}/rejection.json Add these files to version control to share the contract with your team. "#, - dir_str, dir_str, dir_str, dir_str, dir_str, dir_str, dir_str + dir_str, dir_str, dir_str, dir_str, dir_str, dir_str, dir_str, dir_str, dir_str, dir_str ) } diff --git a/geval/src/contract/mod.rs b/geval/src/contract/mod.rs index 1a33454..8f09426 100644 --- a/geval/src/contract/mod.rs +++ b/geval/src/contract/mod.rs @@ -12,4 +12,6 @@ mod runner; pub use combine::{apply_combine_rule, CombineRule}; pub use loader::{load_contract, load_contract_and_policies, parse_contract_str, resolve_policy_path}; pub use model::{ContractDef, PolicyRef}; -pub use runner::{run_contract, ContractResult, PolicyResult}; +pub use runner::{ + load_run_contracts, run_contract, ContractResult, ContractRunEntry, MultiContractRun, PolicyResult, +}; diff --git a/geval/src/contract/runner.rs b/geval/src/contract/runner.rs index 74d0123..ddb44bf 100644 --- a/geval/src/contract/runner.rs +++ b/geval/src/contract/runner.rs @@ -1,9 +1,13 @@ //! Run a contract: evaluate each policy against signals, then combine outcomes. +//! +//! Multiple contracts share one signal graph; outcomes are merged with `contracts_combine`. use anyhow::Result; +use std::path::PathBuf; -use crate::contract::{apply_combine_rule, ContractDef}; +use crate::contract::{apply_combine_rule, load_contract_and_policies, ContractDef, CombineRule}; use crate::evaluator::{evaluate, Decision, DecisionOutcome}; +use crate::hashing::{hash_contract_content, hash_policy}; use crate::policy::Policy; use crate::signal_graph::SignalGraph; @@ -31,7 +35,24 @@ pub struct ContractResult { pub contract_version: String, pub policy_results: Vec, pub combined_decision: Decision, - pub combine_rule: crate::contract::CombineRule, + pub combine_rule: CombineRule, +} + +/// One evaluated contract file plus hashes for audit. +#[derive(Debug, Clone)] +pub struct ContractRunEntry { + pub contract_path: PathBuf, + pub result: ContractResult, + pub contract_hash: String, + pub policy_hashes: Vec, +} + +/// Multiple contracts evaluated against the same signals; overall decision from `contracts_combine`. +#[derive(Debug, Clone)] +pub struct MultiContractRun { + pub entries: Vec, + pub contracts_combine: CombineRule, + pub overall: Decision, } /// Evaluate the contract: run each policy against the graph, then combine. @@ -74,6 +95,41 @@ pub fn run_contract( }) } +/// Load each contract path, run it against `graph`, then combine contract-level outcomes with `contracts_combine`. +pub fn load_run_contracts( + paths: &[PathBuf], + graph: &SignalGraph, + contracts_combine: CombineRule, +) -> Result { + if paths.is_empty() { + anyhow::bail!("at least one contract path is required"); + } + let mut entries = Vec::with_capacity(paths.len()); + for path in paths { + let (contract, policies) = load_contract_and_policies(path)?; + let contract_hash = hash_contract_content(&contract); + let policy_hashes: Vec = policies.iter().map(hash_policy).collect(); + let result = run_contract(&contract, &policies, graph)?; + entries.push(ContractRunEntry { + contract_path: path.clone(), + result, + contract_hash, + policy_hashes, + }); + } + let outcomes: Vec = entries + .iter() + .map(|e| e.result.combined_decision.outcome) + .collect(); + let overall_outcome = apply_combine_rule(contracts_combine, &outcomes); + let overall = overall_decision_from_contracts(&entries, overall_outcome); + Ok(MultiContractRun { + entries, + contracts_combine, + overall, + }) +} + /// Build the combined Decision (outcome + a representative matched_rule/reason from the first non-PASS policy). fn combined_decision_from_results( results: &[PolicyResult], @@ -101,6 +157,41 @@ fn combined_decision_from_results( } } +/// Overall PR-level decision from contract-level outcomes (first failing contract supplies rule/reason). +fn overall_decision_from_contracts( + entries: &[ContractRunEntry], + outcome: DecisionOutcome, +) -> Decision { + if outcome == DecisionOutcome::Pass { + return Decision { + outcome: DecisionOutcome::Pass, + matched_rule: None, + reason: None, + }; + } + let first_non_pass = entries + .iter() + .find(|e| e.result.combined_decision.outcome != DecisionOutcome::Pass); + match first_non_pass { + Some(e) => { + let d = &e.result.combined_decision; + let matched_rule = d.matched_rule.as_ref().map(|m| { + format!("{}:{}", e.contract_path.display(), m) + }); + Decision { + outcome, + matched_rule, + reason: d.reason.clone(), + } + } + None => Decision { + outcome, + matched_rule: None, + reason: None, + }, + } +} + #[cfg(test)] mod tests { use super::*; @@ -240,4 +331,586 @@ rules: assert_eq!(result_block.policy_results[0].outcome, DecisionOutcome::Block); assert_eq!(result_block.combined_decision.outcome, DecisionOutcome::Block); } + + #[test] + fn load_run_contracts_two_all_pass_overall_pass() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("p1.yaml"); + let p2 = dir.path().join("p2.yaml"); + std::fs::write( + &p1, + r#"rules: + - priority: 1 + name: ok + when: + metric: x + operator: ">=" + threshold: 0 + then: + action: pass +"#, + ) + .unwrap(); + std::fs::write( + &p2, + r#"rules: + - priority: 1 + name: ok2 + when: + metric: y + operator: ">=" + threshold: 0 + then: + action: pass +"#, + ) + .unwrap(); + let c1 = dir.path().join("contract1.yaml"); + let c2 = dir.path().join("contract2.yaml"); + std::fs::write( + &c1, + r#" +name: c1 +version: "1.0.0" +combine: all_pass +policies: + - path: p1.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#" +name: c2 +version: "1.0.0" +combine: all_pass +policies: + - path: p2.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0), sig(None, "y", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts( + &[c1, c2], + &graph, + CombineRule::AllPass, + ) + .unwrap(); + assert_eq!(run.entries.len(), 2); + assert_eq!(run.overall.outcome, DecisionOutcome::Pass); + } + + #[test] + fn load_run_contracts_one_block_overall_block() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("p1.yaml"); + let p2 = dir.path().join("p2.yaml"); + std::fs::write( + &p1, + r#"rules: + - priority: 1 + name: block_x + when: + metric: x + operator: ">" + threshold: 100 + then: + action: block +"#, + ) + .unwrap(); + std::fs::write( + &p2, + r#"rules: + - priority: 1 + name: ok + when: + metric: y + operator: ">=" + threshold: 0 + then: + action: pass +"#, + ) + .unwrap(); + let c1 = dir.path().join("contract1.yaml"); + let c2 = dir.path().join("contract2.yaml"); + std::fs::write( + &c1, + r#" +name: c1 +version: "1.0.0" +combine: all_pass +policies: + - path: p1.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#" +name: c2 +version: "1.0.0" +combine: all_pass +policies: + - path: p2.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 150.0), sig(None, "y", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts( + &[c1.clone(), c2], + &graph, + CombineRule::AllPass, + ) + .unwrap(); + assert_eq!(run.overall.outcome, DecisionOutcome::Block); + assert!(run.overall.matched_rule.unwrap().contains(c1.to_str().unwrap())); + } + + #[test] + fn load_run_contracts_any_block_blocks_across_contracts() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("p1.yaml"); + let p2 = dir.path().join("p2.yaml"); + std::fs::write( + &p1, + r#"rules: [{ priority: 1, name: b, when: { metric: x, operator: ">", threshold: 10 }, then: { action: block } }]"#, + ) + .unwrap(); + std::fs::write( + &p2, + r#"rules: [{ priority: 1, name: p, when: { metric: y, operator: ">", threshold: 10 }, then: { action: pass } }]"#, + ) + .unwrap(); + let c1 = dir.path().join("contract1.yaml"); + let c2 = dir.path().join("contract2.yaml"); + std::fs::write( + &c1, + r#" +name: c1 +version: "1.0.0" +combine: all_pass +policies: + - path: p1.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#" +name: c2 +version: "1.0.0" +combine: all_pass +policies: + - path: p2.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0), sig(None, "y", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1, c2], &graph, CombineRule::AnyBlockBlocks).unwrap(); + assert_eq!(run.overall.outcome, DecisionOutcome::Pass); + + let signals_block = SignalSet::new(vec![sig(None, "x", 20.0), sig(None, "y", 1.0)]); + let graph_b = SignalGraph::build(&signals_block.signals); + let c1b = dir.path().join("contract1.yaml"); + let c2b = dir.path().join("contract2.yaml"); + let run_b = + load_run_contracts(&[c1b, c2b], &graph_b, CombineRule::AnyBlockBlocks).unwrap(); + assert_eq!(run_b.overall.outcome, DecisionOutcome::Block); + } + + #[test] + fn load_run_contracts_empty_paths_errors() { + let signals = SignalSet::new(vec![sig(None, "x", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let err = load_run_contracts(&[], &graph, CombineRule::AllPass).unwrap_err(); + assert!(err.to_string().contains("at least one contract")); + } + + /// all_pass across contracts: PASS + REQUIRE_APPROVAL → overall REQUIRE_APPROVAL (no BLOCK). + #[test] + fn load_run_contracts_all_pass_pass_and_require_approval_overall_require_approval() { + let dir = tempfile::tempdir().unwrap(); + let p_ok = dir.path().join("ok.yaml"); + let p_appr = dir.path().join("appr.yaml"); + std::fs::write( + &p_ok, + r#"rules: + - priority: 1 + name: always_pass + when: + metric: x + operator: ">=" + threshold: 0 + then: + action: pass +"#, + ) + .unwrap(); + std::fs::write( + &p_appr, + r#"rules: + - priority: 1 + name: need_signoff + when: + metric: z + operator: ">" + threshold: 0 + then: + action: require_approval +"#, + ) + .unwrap(); + let c1 = dir.path().join("c_pass.yaml"); + let c2 = dir.path().join("c_appr.yaml"); + std::fs::write( + &c1, + r#"name: gate-a +version: "1.0.0" +combine: all_pass +policies: + - path: ok.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#"name: gate-b +version: "1.0.0" +combine: all_pass +policies: + - path: appr.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0), sig(None, "z", 0.5)]); + let graph = SignalGraph::build(&signals.signals); + let c2p = c2.clone(); + let run = load_run_contracts(&[c1, c2], &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.entries[0].result.combined_decision.outcome, DecisionOutcome::Pass); + assert_eq!( + run.entries[1].result.combined_decision.outcome, + DecisionOutcome::RequireApproval + ); + assert_eq!(run.overall.outcome, DecisionOutcome::RequireApproval); + assert!( + run + .overall + .matched_rule + .unwrap() + .contains(c2p.to_str().unwrap()), + "first non-PASS contract should be second" + ); + } + + /// Second contract in CLI order blocks; overall BLOCK attributes to that contract path. + #[test] + fn load_run_contracts_second_contract_blocks_first_passes() { + let dir = tempfile::tempdir().unwrap(); + let p_ok = dir.path().join("ok.yaml"); + let p_block = dir.path().join("blk.yaml"); + std::fs::write( + &p_ok, + r#"rules: [{ priority: 1, name: p, when: { metric: x, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + std::fs::write( + &p_block, + r#"rules: [{ priority: 1, name: b, when: { metric: w, operator: ">", threshold: 0.5 }, then: { action: block } }]"#, + ) + .unwrap(); + let c1 = dir.path().join("first.yaml"); + let c2 = dir.path().join("second.yaml"); + std::fs::write( + &c1, + r#"name: first +version: "1.0.0" +combine: all_pass +policies: + - path: ok.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#"name: second +version: "1.0.0" +combine: all_pass +policies: + - path: blk.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0), sig(None, "w", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1, c2.clone()], &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.overall.outcome, DecisionOutcome::Block); + assert!(run.overall.matched_rule.unwrap().contains(c2.to_str().unwrap())); + } + + /// BLOCK in first contract wins over REQUIRE_APPROVAL in second under all_pass. + #[test] + fn load_run_contracts_all_pass_block_before_require_approval_second_contract() { + let dir = tempfile::tempdir().unwrap(); + let p_block = dir.path().join("blk.yaml"); + let p_appr = dir.path().join("appr.yaml"); + std::fs::write( + &p_block, + r#"rules: [{ priority: 1, name: b, when: { metric: x, operator: ">", threshold: 10 }, then: { action: block } }]"#, + ) + .unwrap(); + std::fs::write( + &p_appr, + r#"rules: [{ priority: 1, name: a, when: { metric: z, operator: ">", threshold: 0 }, then: { action: require_approval } }]"#, + ) + .unwrap(); + let c1 = dir.path().join("blocks.yaml"); + let c2 = dir.path().join("needs_appr.yaml"); + std::fs::write( + &c1, + r#"name: blocks-first +version: "1.0.0" +combine: all_pass +policies: + - path: blk.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#"name: appr-second +version: "1.0.0" +combine: all_pass +policies: + - path: appr.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 20.0), sig(None, "z", 0.1)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1.clone(), c2], &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.overall.outcome, DecisionOutcome::Block); + assert!(run.overall.matched_rule.unwrap().contains(c1.to_str().unwrap())); + } + + /// any_block_blocks: no BLOCK anywhere → PASS + REQUIRE_APPROVAL → REQUIRE_APPROVAL. + #[test] + fn load_run_contracts_any_block_blocks_pass_and_require_approval() { + let dir = tempfile::tempdir().unwrap(); + let p_ok = dir.path().join("ok.yaml"); + let p_appr = dir.path().join("appr.yaml"); + std::fs::write( + &p_ok, + r#"rules: [{ priority: 1, name: p, when: { metric: x, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + std::fs::write( + &p_appr, + r#"rules: [{ priority: 1, name: a, when: { metric: z, operator: ">", threshold: 0 }, then: { action: require_approval } }]"#, + ) + .unwrap(); + let c1 = dir.path().join("c1.yaml"); + let c2 = dir.path().join("c2.yaml"); + std::fs::write( + &c1, + r#"name: a +version: "1.0.0" +combine: all_pass +policies: + - path: ok.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c2, + r#"name: b +version: "1.0.0" +combine: all_pass +policies: + - path: appr.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0), sig(None, "z", 0.1)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1, c2], &graph, CombineRule::AnyBlockBlocks).unwrap(); + assert_eq!(run.overall.outcome, DecisionOutcome::RequireApproval); + } + + #[test] + fn load_run_contracts_three_contracts_all_pass() { + let dir = tempfile::tempdir().unwrap(); + for i in 1..=3 { + let p = dir.path().join(format!("p{}.yaml", i)); + std::fs::write( + &p, + format!( + r#"rules: + - priority: 1 + name: ok{} + when: + metric: m{} + operator: ">=" + threshold: 0 + then: + action: pass +"#, + i, i + ), + ) + .unwrap(); + } + let mut paths = Vec::new(); + for i in 1..=3 { + let c = dir.path().join(format!("c{}.yaml", i)); + std::fs::write( + &c, + format!( + r#"name: c{} +version: "1.0.0" +combine: all_pass +policies: + - path: p{}.yaml +"#, + i, i + ), + ) + .unwrap(); + paths.push(c); + } + let signals = SignalSet::new(vec![ + sig(None, "m1", 1.0), + sig(None, "m2", 1.0), + sig(None, "m3", 1.0), + ]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&paths, &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.entries.len(), 3); + assert_eq!(run.overall.outcome, DecisionOutcome::Pass); + } + + /// One contract with two policies (internal all_pass); partner contract passes — overall pass. + #[test] + fn load_run_contracts_partner_passes_when_first_has_two_policies_internal_combine() { + let dir = tempfile::tempdir().unwrap(); + let p_a = dir.path().join("pa.yaml"); + let p_b = dir.path().join("pb.yaml"); + let p_partner = dir.path().join("partner.yaml"); + std::fs::write( + &p_a, + r#"rules: [{ priority: 1, name: p, when: { metric: x, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + std::fs::write( + &p_b, + r#"rules: [{ priority: 1, name: q, when: { metric: y, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + std::fs::write( + &p_partner, + r#"rules: [{ priority: 1, name: r, when: { metric: z, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + let c_multi = dir.path().join("multi_policy_contract.yaml"); + let c_single = dir.path().join("partner_contract.yaml"); + std::fs::write( + &c_multi, + r#"name: dual-policy-gate +version: "1.0.0" +combine: all_pass +policies: + - path: pa.yaml + - path: pb.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c_single, + r#"name: partner +version: "1.0.0" +combine: all_pass +policies: + - path: partner.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![ + sig(None, "x", 1.0), + sig(None, "y", 1.0), + sig(None, "z", 1.0), + ]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c_multi, c_single], &graph, CombineRule::AllPass).unwrap(); + assert_eq!(run.entries[0].result.policy_results.len(), 2); + assert_eq!(run.overall.outcome, DecisionOutcome::Pass); + } + + /// Both contracts BLOCK on the same signal; overall `matched_rule` prefixes the **first** failing contract path (CLI order). + #[test] + fn load_run_contracts_order_first_block_wins_for_matched_rule_prefix() { + let dir = tempfile::tempdir().unwrap(); + let p_blk = dir.path().join("blk.yaml"); + std::fs::write( + &p_blk, + r#"rules: [{ priority: 1, name: stop, when: { metric: x, operator: ">", threshold: 0 }, then: { action: block } }]"#, + ) + .unwrap(); + let c_first = dir.path().join("contract_alpha.yaml"); + let c_second = dir.path().join("contract_beta.yaml"); + std::fs::write( + &c_first, + r#"name: alpha +version: "1.0.0" +combine: all_pass +policies: + - path: blk.yaml +"#, + ) + .unwrap(); + std::fs::write( + &c_second, + r#"name: beta +version: "1.0.0" +combine: all_pass +policies: + - path: blk.yaml +"#, + ) + .unwrap(); + + let signals = SignalSet::new(vec![sig(None, "x", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run_a_then_b = load_run_contracts( + &[c_first.clone(), c_second.clone()], + &graph, + CombineRule::AllPass, + ) + .unwrap(); + let rule_ab = run_a_then_b.overall.matched_rule.unwrap(); + assert!( + rule_ab.contains("contract_alpha"), + "expected alpha path first: {}", + rule_ab + ); + + let run_b_then_a = load_run_contracts(&[c_second, c_first], &graph, CombineRule::AllPass).unwrap(); + let rule_ba = run_b_then_a.overall.matched_rule.unwrap(); + assert!( + rule_ba.contains("contract_beta"), + "expected beta path first: {}", + rule_ba + ); + } } diff --git a/geval/src/explanation/explain.rs b/geval/src/explanation/explain.rs index e55d21e..3d26ca9 100644 --- a/geval/src/explanation/explain.rs +++ b/geval/src/explanation/explain.rs @@ -1,6 +1,6 @@ //! Human-readable explanation of the decision (GEVAL DECISION REPORT). -use crate::contract::ContractResult; +use crate::contract::{ContractResult, MultiContractRun}; use crate::evaluator::{Decision, DecisionOutcome}; use crate::policy::Policy; use crate::signal_graph::SignalGraph; @@ -51,6 +51,74 @@ pub fn explain_contract_result( out } +/// Multi-contract report: signals once, then each contract, then overall PR-level decision. +pub fn explain_multi_contract_result( + run: &MultiContractRun, + graph: &SignalGraph, + _environment: Option<&str>, +) -> String { + let mut out = String::new(); + out.push_str("GEVAL DECISION REPORT (MULTI-CONTRACT)\n"); + out.push_str("========================================\n"); + let _ = writeln!( + out, + "Combine contracts rule: {}", + run.contracts_combine + ); + out.push_str("\nSignals:\n"); + for s in &graph.signals { + let label = signal_label(s); + let value_str = value_str(s); + let _ = writeln!(out, " {} = {}", label, value_str); + } + for entry in &run.entries { + out.push_str("\n---\n"); + let _ = writeln!( + out, + "Contract file: {}", + entry.contract_path.display() + ); + let result = &entry.result; + let _ = writeln!(out, "Contract: {} @ {}", result.contract_name, result.contract_version); + let _ = writeln!(out, "Combine rule (policies): {}", result.combine_rule); + out.push_str("Per-policy results:\n"); + for r in &result.policy_results { + let match_info = r + .matched_rule + .as_ref() + .map(|m| format!(" (matched: {})", m)) + .unwrap_or_default(); + let _ = writeln!( + out, + " {}: {}{}", + r.policy_path, + outcome_str(r.outcome), + match_info + ); + } + out.push_str("Combined decision (this contract):\n"); + let _ = writeln!(out, "{}", outcome_str(result.combined_decision.outcome)); + if let Some(ref rule) = result.combined_decision.matched_rule { + let _ = writeln!(out, "First non-PASS: {}", rule); + } + if let Some(ref reason) = result.combined_decision.reason { + out.push_str("Reason:\n"); + let _ = writeln!(out, "{}", reason); + } + } + out.push_str("\n======== OVERALL (PR) ========\n"); + let _ = writeln!(out, "{}", outcome_str(run.overall.outcome)); + if let Some(ref rule) = run.overall.matched_rule { + let _ = writeln!(out, "{}", rule); + } + if let Some(ref reason) = run.overall.reason { + out.push_str("Reason:\n"); + let _ = writeln!(out, "{}", reason); + } + out.push_str("========================================\n"); + out +} + /// Produce a text report suitable for CLI output (single policy). pub fn explain_decision( _policy: &Policy, @@ -126,3 +194,61 @@ fn outcome_str(o: DecisionOutcome) -> &'static str { DecisionOutcome::Block => "BLOCK", } } + +#[cfg(test)] +mod tests { + use super::explain_multi_contract_result; + use crate::contract::load_run_contracts; + use crate::contract::CombineRule; + use crate::signal_graph::SignalGraph; + use crate::signals::{Signal, SignalSet}; + + fn sig(metric: &str, value: f64) -> Signal { + Signal { + system: None, + agent: None, + component: None, + step: None, + metric: Some(metric.to_string()), + value: Some(serde_json::json!(value)), + r#type: None, + } + } + + #[test] + fn explain_multi_contract_contains_sections() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("p.yaml"); + std::fs::write( + &p, + r#"rules: [{ priority: 1, name: ok, when: { metric: x, operator: ">=", threshold: 0 }, then: { action: pass } }]"#, + ) + .unwrap(); + let c1 = dir.path().join("c1.yaml"); + let c2 = dir.path().join("c2.yaml"); + for (name, path) in [("one", &c1), ("two", &c2)] { + std::fs::write( + path, + format!( + r#"name: {} +version: "1.0.0" +combine: all_pass +policies: + - path: p.yaml +"#, + name + ), + ) + .unwrap(); + } + let signals = SignalSet::new(vec![sig("x", 1.0)]); + let graph = SignalGraph::build(&signals.signals); + let run = load_run_contracts(&[c1, c2], &graph, CombineRule::AllPass).unwrap(); + let text = explain_multi_contract_result(&run, &graph, None); + assert!(text.contains("MULTI-CONTRACT")); + assert!(text.contains("Combine contracts rule")); + assert!(text.contains("OVERALL (PR)")); + assert!(text.contains("PASS")); + assert!(text.contains("Contract file:")); + } +} diff --git a/geval/src/explanation/mod.rs b/geval/src/explanation/mod.rs index 9db893f..e68e8b3 100644 --- a/geval/src/explanation/mod.rs +++ b/geval/src/explanation/mod.rs @@ -1,3 +1,5 @@ mod explain; -pub use explain::{explain_contract_result, explain_decision}; +pub use explain::{ + explain_contract_result, explain_decision, explain_multi_contract_result, +}; diff --git a/geval/src/hashing/mod.rs b/geval/src/hashing/mod.rs index d05f29b..2953c07 100644 --- a/geval/src/hashing/mod.rs +++ b/geval/src/hashing/mod.rs @@ -1,3 +1,3 @@ mod sha; -pub use sha::{hash_contract_content, hash_policy, hash_signals}; +pub use sha::{hash_contract_bundle, hash_contract_content, hash_policy, hash_signals}; diff --git a/geval/src/hashing/sha.rs b/geval/src/hashing/sha.rs index 97dd31a..9040498 100644 --- a/geval/src/hashing/sha.rs +++ b/geval/src/hashing/sha.rs @@ -26,3 +26,32 @@ pub fn hash_signals(signals: &crate::signals::SignalSet) -> String { hasher.update(bytes.as_bytes()); format!("{:x}", hasher.finalize()) } + +/// Deterministic digest of an ordered list of contract paths and their content hashes (audit bundle). +pub fn hash_contract_bundle(entries: &[(std::path::PathBuf, &str)]) -> String { + let mut buf = String::new(); + for (path, contract_hash) in entries { + use std::fmt::Write; + let _ = writeln!(buf, "{}\t{}", path.display(), contract_hash); + } + let mut hasher = Sha256::new(); + hasher.update(buf.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn bundle_hash_stable_and_order_dependent() { + let a = PathBuf::from("/a/c1.yaml"); + let b = PathBuf::from("/b/c2.yaml"); + let h1 = hash_contract_bundle(&[(a.clone(), "aaa"), (b.clone(), "bbb")]); + let h2 = hash_contract_bundle(&[(a.clone(), "aaa"), (b.clone(), "bbb")]); + assert_eq!(h1, h2); + let h3 = hash_contract_bundle(&[(b, "bbb"), (a, "aaa")]); + assert_ne!(h1, h3); + } +} diff --git a/geval/src/lib.rs b/geval/src/lib.rs index 51862c7..3d11e2b 100644 --- a/geval/src/lib.rs +++ b/geval/src/lib.rs @@ -19,14 +19,14 @@ pub mod signal_graph; pub mod signals; pub use approval::{ApprovalArtifact, ApprovalOutcome, read_approval, write_approval}; -pub use artifact::{write_decision_artifact, DECISION_ARTIFACT_VERSION}; +pub use artifact::{write_multi_contract_artifact, DECISION_ARTIFACT_VERSION}; pub use contract::{ - load_contract, load_contract_and_policies, run_contract, CombineRule, ContractDef, ContractResult, - PolicyRef, PolicyResult, + load_contract, load_contract_and_policies, load_run_contracts, run_contract, CombineRule, + ContractDef, ContractResult, ContractRunEntry, MultiContractRun, PolicyRef, PolicyResult, }; pub use evaluator::{evaluate, Decision, DecisionOutcome}; pub use explanation::explain_decision; -pub use hashing::{hash_contract_content, hash_policy, hash_signals}; +pub use hashing::{hash_contract_bundle, hash_contract_content, hash_policy, hash_signals}; pub use policy::{parse_policy, parse_policy_str, Policy, Rule}; pub use signal_graph::SignalGraph; pub use signals::{load_signals, Signal, SignalSet};