diff --git a/CLAUDE.md b/CLAUDE.md index b2a12d6..cb39450 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,6 +101,8 @@ cargo fmt --check # Run CLI cargo run -- scan tests/fixtures/mcp_servers/vuln_cmd_inject cargo run -- scan . --ignore-tests --format html --output report.html +cargo run -- scan . --experimental-risk +cargo run -- scan . --format json --experimental-risk cargo run -- scan . --write-baseline baseline.json cargo run -- scan . --baseline baseline.json cargo run -- discover --no-default-paths --root . diff --git a/README.md b/README.md index a810d57..e80b81a 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,7 @@ AgentShield runs all matching adapters in a repository instead of stopping at th |---------|---------| | `agentshield discover` | Read allowlisted local client configuration paths without execution, network access, or automatic scanning; use `--no-default-paths`, repeated `--root`, `--format console\|json`, and `--explain` to control consent and output. | | `agentshield scan [path]` | Scan an agent extension directory and emit console, JSON, SARIF, or HTML output. | +| `agentshield scan [path] --experimental-risk` | Add an informational, versioned risk index to console or JSON output without changing policy or exit status. | | `agentshield scan [path] --explain` | Print a console-only gate, coverage, confidence, grouped findings, next-actions, and limits summary. | | `agentshield quickstart [path]` | Create first-run config, suggest CI setup, run the first scan, and explain the result. | | `agentshield ci install` | Generate a GitHub Actions workflow for AgentShield. | @@ -267,7 +268,7 @@ AgentShield runs all matching adapters in a repository instead of stopping at th | `agentshield guard --stdin` | Evaluate one runtime event JSON document when built with the `runtime-guard` feature. | | `agentshield guard --mcp-proxy [-- ]` | EXPERIMENTAL: evaluate line-delimited MCP JSON-RPC `tools/call` messages, block unsafe calls, and either emit forward markers or bridge stdio to a spawned downstream MCP server when built with the `runtime-guard` feature. | -Useful `scan` options include `--config`, `--format`, `--fail-on`, `--output`, `--ignore-tests`, `--explain`, `--baseline`, `--write-baseline`, and `--emit-egress-policy`. +Useful `scan` options include `--config`, `--format`, `--fail-on`, `--output`, `--ignore-tests`, `--explain`, `--experimental-risk`, `--baseline`, `--write-baseline`, and `--emit-egress-policy`. Configured `[scan] include` and `[scan] exclude` filters scope source and metadata-derived findings before detectors run. @@ -284,6 +285,11 @@ agentshield ci install --baseline .agentshield-baseline.json SARIF, or HTML output. Explain output includes the scan root, metadata root when different, and hotspot summaries for concentrated blocking findings. +`--experimental-risk` is a separate console/JSON-only mode and cannot be +combined with `--explain`. Its versioned index is informational: it does not +change findings, PASS/FAIL, policy thresholds, baselines, suppressions, or the +process exit status. SARIF, HTML, and DSSE remain unchanged. + --- ## Detection Rules diff --git a/docs/specs/risk-model-v1.md b/docs/specs/risk-model-v1.md index c4627b5..0e624f5 100644 --- a/docs/specs/risk-model-v1.md +++ b/docs/specs/risk-model-v1.md @@ -1,14 +1,15 @@ # AgentShield risk model v1 -Status: internal golden model +Status: experimental opt-in output Model identifier: `agentshield-risk-v1` This document freezes the E.1 model selected under [`explainable-risk-score.md`](explainable-risk-score.md). The model is -crate-private and is not emitted by any default or opt-in output in E.1. -Findings remain the security facts; policy verdict and process exit status -remain the only enforcement contract. +emitted only when `scan --experimental-risk` is explicitly selected with +console or JSON output. Default output remains unchanged. Findings remain the +security facts; policy verdict and process exit status remain the only +enforcement contract. ## Inputs @@ -74,4 +75,12 @@ contexts; E.1 does not expose a comparison UI. Fingerprint deduplication removes exact duplicates but does not prove semantic independence across different rules. Golden tests validate deterministic mechanics, not breach probability or real-world calibration. E.2 output remains -blocked if representative correlated findings make the ranking misleading. +experimental and must be removed or revised if representative correlated +findings make the ranking misleading. + +## Experimental output boundary + +`--experimental-risk` supports console and JSON only. It is incompatible with +`--explain`; it does not modify SARIF, HTML or DSSE. At most 50 ordered +contributions are emitted, with an explicit omitted count. The flag never +changes findings, policy, verdict, baseline behavior or process exit status. diff --git a/src/bin/cli.rs b/src/bin/cli.rs index 8748bfd..5a2a25b 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -108,6 +108,10 @@ enum Commands { /// This is console-only so JSON, SARIF, and HTML output contracts stay stable. #[arg(long)] explain: bool, + + /// Add an experimental informational risk index (console or JSON only) + #[arg(long)] + experimental_risk: bool, }, /// First-run setup: create config, inspect coverage, and run an explained scan @@ -323,6 +327,7 @@ fn main() { write_baseline, emit_egress_policy, explain, + experimental_risk, } => cmd_scan(ScanArgs { path, config, @@ -334,6 +339,7 @@ fn main() { write_baseline_path: write_baseline, emit_egress_policy_path: emit_egress_policy, explain, + experimental_risk, }), Commands::Quickstart { path, diff --git a/src/bin/cli/scan.rs b/src/bin/cli/scan.rs index 97eeee7..d5ed1af 100644 --- a/src/bin/cli/scan.rs +++ b/src/bin/cli/scan.rs @@ -19,6 +19,7 @@ pub(super) struct ScanArgs { pub(super) write_baseline_path: Option, pub(super) emit_egress_policy_path: Option, pub(super) explain: bool, + pub(super) experimental_risk: bool, } pub(super) fn cmd_scan(args: ScanArgs) -> Result { @@ -33,6 +34,7 @@ pub(super) fn cmd_scan(args: ScanArgs) -> Result Result Result Result Result { + if !matches!(format, OutputFormat::Console | OutputFormat::Json) { + return Err(error::ShieldError::Config( + "`--experimental-risk` supports only console and JSON output".to_owned(), + )); + } + + let base_report = render_report(report, format)?; + let coverage = risk::CoverageDescriptor::current(); + let assessment = risk::assess(&report.findings, &report.scan_root, &coverage) + .map_err(|error| error::ShieldError::Internal(error.to_string()))?; + risk::render_experimental(&base_report, &assessment, format) +} + #[cfg(test)] mod integration_tests { use super::*; diff --git a/src/risk.rs b/src/risk.rs index 6aba548..f909515 100644 --- a/src/risk.rs +++ b/src/risk.rs @@ -3,22 +3,21 @@ //! Findings remain the security facts and policy remains the enforcement //! boundary. This module is intentionally not part of the public API. //! -//! E.1 deliberately has no production caller: integration is the separate E.2 -//! review gate. Remove this allowance when that gate adds an explicit consumer. -#![cfg_attr(not(test), allow(dead_code))] - use std::collections::BTreeMap; use std::path::Path; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use crate::error::{Result as ShieldResult, ShieldError}; +use crate::output::OutputFormat; use crate::rules::{Confidence, Finding, RuleEngine, RuleMetadata, Severity}; pub(crate) const MODEL_VERSION: &str = "agentshield-risk-v1"; const COVERAGE_SCHEMA: &str = "agentshield-coverage-v1"; const SATURATION_CONSTANT: u64 = 30; const MAX_EMITTED_SCORE: u8 = 99; +const MAX_OUTPUT_CONTRIBUTIONS: usize = 50; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct RiskAssessment { @@ -31,6 +30,7 @@ pub(crate) struct RiskAssessment { } impl RiskAssessment { + #[cfg(test)] pub(crate) fn is_comparable_to(&self, other: &Self) -> bool { self.model_version == other.model_version && self.coverage_id == other.coverage_id } @@ -52,6 +52,19 @@ pub(crate) struct RiskSummary { pub(crate) duplicate_findings: usize, } +#[derive(Serialize)] +struct ExperimentalRiskOutput<'a> { + status: &'static str, + score: u8, + model_version: &'a str, + coverage_id: &'a str, + raw_points: u64, + contributions: &'a [RiskContribution], + contributions_truncated: usize, + summary: &'a RiskSummary, + interpretation: &'static str, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct CoverageDescriptor { scanner_version: String, @@ -182,6 +195,80 @@ pub(crate) fn assess( }) } +pub(crate) fn render_experimental( + base_report: &str, + assessment: &RiskAssessment, + format: OutputFormat, +) -> ShieldResult { + let displayed = assessment.contributions.len().min(MAX_OUTPUT_CONTRIBUTIONS); + let output = ExperimentalRiskOutput { + status: "informational", + score: assessment.score, + model_version: &assessment.model_version, + coverage_id: &assessment.coverage_id, + raw_points: assessment.raw_points, + contributions: &assessment.contributions[..displayed], + contributions_truncated: assessment.contributions.len() - displayed, + summary: &assessment.summary, + interpretation: + "Prioritization index only; not a probability, percentage, grade, or policy verdict.", + }; + + match format { + OutputFormat::Console => Ok(render_experimental_console(base_report, &output)), + OutputFormat::Json => render_experimental_json(base_report, &output), + OutputFormat::Sarif | OutputFormat::Html => Err(ShieldError::Config( + "`--experimental-risk` supports only console and JSON output".to_owned(), + )), + } +} + +fn render_experimental_console(base_report: &str, output: &ExperimentalRiskOutput<'_>) -> String { + use std::fmt::Write; + + let mut rendered = base_report.to_owned(); + if !rendered.ends_with('\n') { + rendered.push('\n'); + } + rendered.push('\n'); + rendered.push_str("Experimental risk assessment (informational)\n"); + let _ = writeln!(rendered, "Score: {}", output.score); + let _ = writeln!(rendered, "Model: {}", output.model_version); + let _ = writeln!(rendered, "Coverage: {}", output.coverage_id); + let _ = writeln!(rendered, "Raw points: {}", output.raw_points); + let _ = writeln!( + rendered, + "Contributions: {} shown, {} omitted", + output.contributions.len(), + output.contributions_truncated + ); + for contribution in output.contributions { + let _ = writeln!( + rendered, + "- {} {} {} {}: {} point(s)", + contribution.fingerprint, + contribution.rule_id, + contribution.effective_severity, + contribution.confidence, + contribution.points + ); + } + let _ = writeln!(rendered, "Interpretation: {}", output.interpretation); + rendered +} + +fn render_experimental_json( + base_report: &str, + output: &ExperimentalRiskOutput<'_>, +) -> ShieldResult { + let mut report: serde_json::Value = serde_json::from_str(base_report)?; + let object = report + .as_object_mut() + .ok_or_else(|| ShieldError::Output("default JSON report was not an object".to_owned()))?; + object.insert("risk_assessment".to_owned(), serde_json::to_value(output)?); + Ok(serde_json::to_string_pretty(&report)?) +} + fn contribution_points(severity: Severity, confidence: Confidence) -> u64 { severity_weight(severity) * confidence_multiplier(confidence) } @@ -556,6 +643,38 @@ mod tests { assert!(!base.is_comparable_to(&other)); } + #[test] + fn experimental_output_bounds_contributions_and_reports_omissions() { + let findings = (0..51) + .map(|index| { + finding( + &format!("SHIELD-GOLDEN-{index:02}"), + Severity::Low, + Confidence::Low, + &format!("evidence {index}"), + ) + }) + .collect::>(); + let assessment = assess(&findings, Path::new("/scan"), &coverage()).expect("assessment"); + let rendered = render_experimental( + r#"{"findings":[],"verdict":{"pass":true}}"#, + &assessment, + OutputFormat::Json, + ) + .expect("render experimental JSON"); + let report: serde_json::Value = + serde_json::from_str(&rendered).expect("parse experimental JSON"); + + assert_eq!( + report["risk_assessment"]["contributions"] + .as_array() + .expect("contributions") + .len(), + 50 + ); + assert_eq!(report["risk_assessment"]["contributions_truncated"], 1); + } + #[test] fn arithmetic_overflow_is_explicit() { assert_eq!( diff --git a/tests/experimental_risk_cli.rs b/tests/experimental_risk_cli.rs new file mode 100644 index 0000000..66191b1 --- /dev/null +++ b/tests/experimental_risk_cli.rs @@ -0,0 +1,99 @@ +use std::process::Command; + +fn scan(arguments: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_agentshield")) + .args(arguments) + .output() + .expect("run agentshield") +} + +#[test] +fn default_json_contract_has_no_risk_assessment() { + let output = scan(&[ + "scan", + "tests/fixtures/mcp_servers/safe_calculator", + "--format", + "json", + ]); + assert!(output.status.success()); + + let report: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("parse default JSON"); + assert!(report.get("risk_assessment").is_none()); +} + +#[test] +fn opt_in_json_is_informational_bounded_and_policy_neutral() { + let output = scan(&[ + "scan", + "tests/fixtures/mcp_servers/vuln_cmd_inject", + "--format", + "json", + "--experimental-risk", + ]); + assert_eq!(output.status.code(), Some(1)); + + let report: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("parse experimental JSON"); + assert_eq!(report["verdict"]["pass"], false); + assert_eq!(report["risk_assessment"]["status"], "informational"); + assert_eq!( + report["risk_assessment"]["model_version"], + "agentshield-risk-v1" + ); + assert!(report["risk_assessment"]["score"].as_u64().is_some()); + assert!( + report["risk_assessment"]["contributions"] + .as_array() + .expect("contributions array") + .len() + <= 50 + ); + assert_eq!( + report["risk_assessment"]["interpretation"], + "Prioritization index only; not a probability, percentage, grade, or policy verdict." + ); +} + +#[test] +fn opt_in_console_keeps_verdict_primary_and_labels_risk_informational() { + let output = scan(&[ + "scan", + "tests/fixtures/mcp_servers/safe_calculator", + "--experimental-risk", + ]); + assert!(output.status.success()); + + let rendered = String::from_utf8_lossy(&output.stdout); + let verdict_position = rendered.find("PASS").expect("console verdict"); + let risk_position = rendered + .find("Experimental risk assessment (informational)") + .expect("experimental risk heading"); + assert!(verdict_position < risk_position); + assert!(rendered.contains("Model: agentshield-risk-v1")); + assert!(rendered.contains("Interpretation: Prioritization index only")); +} + +#[test] +fn unsupported_and_conflicting_output_modes_are_rejected() { + let sarif = scan(&[ + "scan", + "tests/fixtures/mcp_servers/safe_calculator", + "--format", + "sarif", + "--experimental-risk", + ]); + assert_eq!(sarif.status.code(), Some(2)); + assert!( + String::from_utf8_lossy(&sarif.stderr).contains("supports only console and JSON output") + ); + + let explain = scan(&[ + "scan", + "tests/fixtures/mcp_servers/safe_calculator", + "--explain", + "--experimental-risk", + ]); + assert_eq!(explain.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&explain.stderr).contains("separate output modes")); +}