From 0221649eab104afda81a3f2aad608214056060c1 Mon Sep 17 00:00:00 2001 From: Drake Date: Mon, 21 Sep 2026 16:42:27 -0500 Subject: [PATCH] feat(probe): integrate sovereign probe evaluation into inquisitor diff audits --- Cargo.toml | 1 + action.yml | 7 +- src/lib.rs | 3 + src/main.rs | 92 ++++++++++++ src/probe_audit.rs | 288 +++++++++++++++++++++++++++++++++++++ tests/probe_audit_tests.rs | 116 +++++++++++++++ 6 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 src/probe_audit.rs create mode 100644 tests/probe_audit_tests.rs diff --git a/Cargo.toml b/Cargo.toml index f977274..9cc9d5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ tokio = { version = "1.40", features = ["full"] } p256 = { version = "0.14.0", features = ["ecdsa"] } aien-protocol-types = { path = "../aien-protocols/crates/aien-protocol-types" } aien-evaluation-protocol = { path = "../aien-protocols/crates/aien-evaluation-protocol" } +aien-probe = { path = "../aien-protocols/crates/aien-probe" } [profile.release] opt-level = 3 diff --git a/action.yml b/action.yml index eeff366..d40dd61 100644 --- a/action.yml +++ b/action.yml @@ -4,7 +4,7 @@ author: "Drake Stapleton & AIEN " inputs: command: - description: "Inquisitor command: review, triage-issue, interview, audit, evaluate, or doctor" + description: "Inquisitor command: review, triage-issue, interview, audit, evaluate, probe, or doctor" required: true default: "review" diff: @@ -42,6 +42,11 @@ runs: doctor) spark-inquisitor doctor ;; + probe) + spark-inquisitor probe \ + --diff "${{ inputs.diff }}" \ + --output "${{ inputs.output-comment }}" + ;; review) spark-inquisitor review \ --diff "${{ inputs.diff }}" \ diff --git a/src/lib.rs b/src/lib.rs index 7d72773..e8c7690 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,3 +17,6 @@ pub use triage::{triage_issue, IssueCategory, TriageResult}; pub mod protocol_evaluator; pub use protocol_evaluator::{InquisitorConstitutionalEvaluator, InquisitorDiffEvaluator}; + +pub mod probe_audit; +pub use probe_audit::{run_probe_audit, InquisitorProbeEvaluator, ProbeAuditReport}; diff --git a/src/main.rs b/src/main.rs index 88d94fc..fadc55a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -118,6 +118,35 @@ enum Commands { output_labels: Option, }, + /// Audit a git diff with the sovereign native probe evaluation engine + Probe { + #[arg(long, help = "Path to diff file")] + diff: PathBuf, + + #[arg( + long, + default_value = "reference", + help = "Probe backend: reference or http" + )] + backend: String, + + #[arg(long, help = "Endpoint for the http backend")] + endpoint: Option, + + #[arg( + long, + default_value_t = 0.95, + help = "Minimum telemetry safety probability" + )] + min_telemetry: f64, + + #[arg(long, default_value_t = 2.0, help = "Minimum quality score")] + min_quality: f64, + + #[arg(long, help = "Path to write the probe report markdown")] + output: Option, + }, + /// Run internal self-diagnostics Doctor, } @@ -375,6 +404,69 @@ fn main() { std::process::exit(1); } } + Commands::Probe { + diff, + backend, + endpoint, + min_telemetry, + min_quality, + output, + } => { + let content = match fs::read_to_string(&diff) { + Ok(c) => c, + Err(e) => { + eprintln!( + "{} Failed to read diff file {}: {}", + "error:".red().bold(), + diff.display(), + e + ); + std::process::exit(1); + } + }; + + let runtime = match tokio::runtime::Runtime::new() { + Ok(runtime) => runtime, + Err(e) => { + eprintln!( + "{} Failed to initialize async runtime: {}", + "error:".red().bold(), + e + ); + std::process::exit(1); + } + }; + + match runtime.block_on(spark_inquisitor::run_probe_audit( + &content, + &backend, + endpoint, + min_telemetry, + min_quality, + )) { + Ok(report) => { + if let Some(path) = &output { + if let Err(e) = fs::write(path, &report.markdown) { + eprintln!( + "{} Failed to write probe report to {}: {}", + "error:".red().bold(), + path.display(), + e + ); + std::process::exit(1); + } + } + println!("{}", report.markdown); + if !report.clean { + std::process::exit(1); + } + } + Err(e) => { + eprintln!("{} Probe audit failed: {}", "error:".red().bold(), e); + std::process::exit(2); + } + } + } Commands::Doctor => { println!("⚖️ Sovereign Inquisitor Diagnostics: OK"); println!(" - Invariant rules: Active"); diff --git a/src/probe_audit.rs b/src/probe_audit.rs new file mode 100644 index 0000000..3986c92 --- /dev/null +++ b/src/probe_audit.rs @@ -0,0 +1,288 @@ +use aien_evaluation_protocol::{ + EvaluationError, EvaluationPlan, Evaluator, EvaluatorDescriptor, EvaluatorIdentity, + EvaluatorOutcome, EvaluatorStatus, Finding, Measurement, +}; +use aien_probe::{ + Choice, ChoiceOption, DeterministicReferenceBackend, HttpProbeBackend, Noul, Probe, + ProbeBackend, ProbeEngine, ProbeSet, Score, +}; +use aien_protocol_types::{ArtifactRef, Digest32}; +use serde_json::json; + +/// The sovereign probe audit report for one git diff. +#[derive(Clone, Debug)] +pub struct ProbeAuditReport { + pub markdown: String, + pub clean: bool, + pub telemetry_probability: f64, + pub unslop_probability: f64, + pub quality_score: f64, + pub classification: String, + pub backend: String, +} + +/// The standard probe set used for diff audits. +pub fn diff_probe_set() -> ProbeSet { + ProbeSet::new() + .with( + "telemetry", + Probe::Noul(Noul::new( + "Is this diff free of covert network calls, analytics, trackers, and surveillance telemetry?", + )), + ) + .with( + "unslop", + Probe::Noul(Noul::new( + "Does this diff use affirmative, direct technical voice with zero formulaic tropes and zero em or en dashes?", + )), + ) + .with( + "quality", + Probe::Score(Score::new( + "Rate the overall code quality and architectural safety of this diff.", + ["critical", "weak", "acceptable", "exemplary"], + )), + ) + .with( + "classification", + Probe::Choice(Choice::new( + "Classify the change.", + [ + ChoiceOption::new("clean").with_description("Verified clean sovereign contribution"), + ChoiceOption::new("advisory").with_description("Minor non-blocking recommendations"), + ChoiceOption::new("suspicious").with_description("Questionable logic requiring manual audit"), + ChoiceOption::new("critical").with_description("Critical invariant violation or covert telemetry"), + ], + )), + ) +} + +/// Evaluates a diff using any provided sovereign probe engine. +pub async fn audit_diff_with_probes( + engine: &ProbeEngine, + diff: &str, + min_telemetry: f64, + min_quality: f64, +) -> Result { + let state = json!({ + "diff": diff, + "line_count": diff.lines().count(), + }); + + let probes = diff_probe_set(); + let response = engine + .evaluate(&state, &probes) + .await + .map_err(|e| format!("Probe engine execution failed: {e}"))?; + + let telemetry = response.noul("telemetry").map(|a| a.noul).unwrap_or(0.0); + let unslop = response.noul("unslop").map(|a| a.noul).unwrap_or(0.0); + let quality = response.score("quality").map(|a| a.score).unwrap_or(0.0); + let classification = response + .choice("classification") + .map(|a| a.choice.clone()) + .unwrap_or_else(|| "unclassified".to_string()); + + let telemetry_ok = telemetry >= min_telemetry; + let quality_ok = quality >= min_quality; + let classification_ok = classification == "clean" || classification == "advisory"; + let clean = telemetry_ok && quality_ok && classification_ok; + + let status_str = if clean { "PASS" } else { "FAIL" }; + + let markdown = format!( + "### Sovereign Probe Audit: {}\n\n\ + - **Backend**: `{}`\n\ + - **Classification**: `{}`\n\ + - **Telemetry Safety Probability**: `{:.4}` (Threshold: `>= {:.4}`)\n\ + - **Unslop Alignment Probability**: `{:.4}`\n\ + - **Architectural Quality Score**: `{:.2} / 3.00` (Threshold: `>= {:.2}`)\n\ + - **Verdict**: `{}`\n", + status_str, + response.model, + classification, + telemetry, + min_telemetry, + unslop, + quality, + min_quality, + status_str + ); + + Ok(ProbeAuditReport { + markdown, + clean, + telemetry_probability: telemetry, + unslop_probability: unslop, + quality_score: quality, + classification, + backend: response.model, + }) +} + +/// Runs probe audit against a diff string using the specified backend. +pub async fn run_probe_audit( + diff: &str, + backend: &str, + endpoint: Option, + min_telemetry: f64, + min_quality: f64, +) -> Result { + match backend { + "reference" | "oracle" => { + let engine = ProbeEngine::new(DeterministicReferenceBackend::new()); + audit_diff_with_probes(&engine, diff, min_telemetry, min_quality).await + } + "http" | "gb10" => { + let ep = endpoint.unwrap_or_else(|| "http://127.0.0.1:18006".to_string()); + let backend_client = HttpProbeBackend::with_endpoint(ep, "atlas-lightning-omni"); + let engine = ProbeEngine::new(backend_client); + audit_diff_with_probes(&engine, diff, min_telemetry, min_quality).await + } + other => Err(format!( + "Unknown probe backend '{}'; expected 'reference' or 'http'", + other + )), + } +} + +/// Evaluator adapter plugging probe audits into the CanaryRollbackHarness. +pub struct InquisitorProbeEvaluator { + diff_payload: Option, + min_telemetry: f64, + min_quality: f64, +} + +impl InquisitorProbeEvaluator { + pub fn new() -> Self { + Self { + diff_payload: None, + min_telemetry: 0.95, + min_quality: 2.0, + } + } + + pub fn with_diff(diff: impl Into) -> Self { + Self { + diff_payload: Some(diff.into()), + min_telemetry: 0.95, + min_quality: 2.0, + } + } + + pub fn with_thresholds(mut self, min_telemetry: f64, min_quality: f64) -> Self { + self.min_telemetry = min_telemetry; + self.min_quality = min_quality; + self + } +} + +impl Default for InquisitorProbeEvaluator { + fn default() -> Self { + Self::new() + } +} + +#[async_trait::async_trait] +impl Evaluator for InquisitorProbeEvaluator { + fn descriptor(&self) -> EvaluatorDescriptor { + EvaluatorDescriptor { + evaluator_id: "spark-inquisitor-probe".to_string(), + version: "0.2.0".to_string(), + required: true, + } + } + + async fn evaluate( + &self, + _plan: &EvaluationPlan, + _subject: &ArtifactRef, + ) -> Result { + let diff = self.diff_payload.as_deref().unwrap_or(""); + let report = run_probe_audit( + diff, + "reference", + None, + self.min_telemetry, + self.min_quality, + ) + .await + .map_err(EvaluationError::ExecutionFailed)?; + + let mut findings = Vec::new(); + if report.telemetry_probability < self.min_telemetry { + findings.push(Finding { + severity: "Fatal".to_string(), + rule_id: "PROBE_TELEMETRY_THRESHOLD".to_string(), + message: format!( + "Telemetry safety probability {:.4} failed threshold {:.4}", + report.telemetry_probability, self.min_telemetry + ), + location: None, + }); + } + if report.quality_score < self.min_quality { + findings.push(Finding { + severity: "Fatal".to_string(), + rule_id: "PROBE_QUALITY_THRESHOLD".to_string(), + message: format!( + "Architectural quality score {:.2} failed threshold {:.2}", + report.quality_score, self.min_quality + ), + location: None, + }); + } + if report.classification == "suspicious" || report.classification == "critical" { + findings.push(Finding { + severity: "Fatal".to_string(), + rule_id: "PROBE_CLASSIFICATION_REJECTED".to_string(), + message: format!( + "Diff classified as '{}', requiring quarantine", + report.classification + ), + location: None, + }); + } + + let status = if report.clean { + EvaluatorStatus::Passed + } else { + EvaluatorStatus::Failed + }; + + let measurements = vec![ + Measurement { + metric: "probe_telemetry_probability".to_string(), + value: report.telemetry_probability, + unit: "ratio".to_string(), + }, + Measurement { + metric: "probe_unslop_probability".to_string(), + value: report.unslop_probability, + unit: "ratio".to_string(), + }, + Measurement { + metric: "probe_quality_score".to_string(), + value: report.quality_score, + unit: "points".to_string(), + }, + ]; + + let mut outcome = EvaluatorOutcome { + evaluator: EvaluatorIdentity { + evaluator_id: "spark-inquisitor-probe".to_string(), + version: "0.2.0".to_string(), + binary_digest: Digest32([0xee; 32]), + config_digest: Digest32([0xff; 32]), + }, + status, + findings, + measurements, + evidence: vec![], + execution_digest: Digest32::ZERO, + }; + + outcome.execution_digest = outcome.compute_execution_digest(); + Ok(outcome) + } +} diff --git a/tests/probe_audit_tests.rs b/tests/probe_audit_tests.rs new file mode 100644 index 0000000..7feba9e --- /dev/null +++ b/tests/probe_audit_tests.rs @@ -0,0 +1,116 @@ +use aien_evaluation_protocol::{ + CanaryRollbackHarness, EvaluationPlan, Evaluator, EvaluatorStatus, SoftwareP256Signer, Verdict, + VerifierIdentity, VerifierSigner, +}; +use aien_protocol_types::{ArtifactRef, Digest32, EvaluationId, Timestamp}; +use p256::ecdsa::SigningKey; +use spark_inquisitor::probe_audit::{run_probe_audit, InquisitorProbeEvaluator}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +#[tokio::test] +async fn test_run_probe_audit_reference_is_deterministic() { + let diff = + "diff --git a/src/lib.rs b/src/lib.rs\n+ pub fn hello() -> &'static str { \"hello\" }"; + let r1 = run_probe_audit(diff, "reference", None, 0.0, 0.0) + .await + .unwrap(); + let r2 = run_probe_audit(diff, "reference", None, 0.0, 0.0) + .await + .unwrap(); + + assert_eq!(r1.telemetry_probability, r2.telemetry_probability); + assert_eq!(r1.unslop_probability, r2.unslop_probability); + assert_eq!(r1.quality_score, r2.quality_score); + assert_eq!(r1.classification, r2.classification); + assert_eq!(r1.backend, "deterministic-reference-oracle"); +} + +#[tokio::test] +async fn test_inquisitor_probe_evaluator_outcome() { + let diff = "diff --git a/src/lib.rs b/src/lib.rs\n+ pub fn compute() -> i32 { 100 }"; + let evaluator = InquisitorProbeEvaluator::with_diff(diff).with_thresholds(0.0, 0.0); + + let subject = ArtifactRef { + artifact_id: uuid::Uuid::new_v4(), + digest: Digest32([0x12; 32]), + media_type: "text/x-diff".to_string(), + byte_size: 256, + }; + let mut plan = EvaluationPlan { + evaluation_id: EvaluationId::new_v4(), + subject: subject.clone(), + profile: "probe-audit".to_string(), + evaluators: vec![evaluator.descriptor()], + baseline: None, + sandbox_profile: "airgap".to_string(), + policy_digest: Digest32([0x22; 32]), + evaluator_manifest_digest: Digest32([0x33; 32]), + plan_digest: Digest32::ZERO, + }; + plan.plan_digest = plan.compute_plan_digest(); + + let outcome = evaluator.evaluate(&plan, &subject).await.unwrap(); + assert_eq!(outcome.status, EvaluatorStatus::Passed); + assert_eq!(outcome.measurements.len(), 3); +} + +#[tokio::test] +async fn test_inquisitor_probe_evaluator_canary_rollback_integration() { + let diff = "diff --git a/bad.rs b/bad.rs\n+ // test diff"; + // Threshold set impossibly high so it triggers failure and rollback + let evaluator = InquisitorProbeEvaluator::with_diff(diff).with_thresholds(0.99999, 3.99999); + let descriptor = evaluator.descriptor(); + + let signing_key = SigningKey::from_slice(&[42u8; 32]).unwrap(); + let signer = Arc::new(SoftwareP256Signer::new(signing_key)); + let verifier = VerifierIdentity { + principal_id: "inquisitor-daemon".to_string(), + key_id: signer.key_fingerprint(), + trust_epoch: 1, + trusted_build_digest: Digest32([0x44; 32]), + policy_bundle_digest: Digest32([0x55; 32]), + }; + let harness = CanaryRollbackHarness::new(verifier, signer); + + let subject = ArtifactRef { + artifact_id: uuid::Uuid::new_v4(), + digest: Digest32([0x66; 32]), + media_type: "text/x-diff".to_string(), + byte_size: 128, + }; + let mut plan = EvaluationPlan { + evaluation_id: EvaluationId::new_v4(), + subject: subject.clone(), + profile: "probe-canary".to_string(), + evaluators: vec![descriptor], + baseline: None, + sandbox_profile: "airgap".to_string(), + policy_digest: Digest32([0x77; 32]), + evaluator_manifest_digest: Digest32([0x88; 32]), + plan_digest: Digest32::ZERO, + }; + plan.plan_digest = plan.compute_plan_digest(); + + let rollback_executed = Arc::new(AtomicBool::new(false)); + let rollback_flag = rollback_executed.clone(); + let evaluators: Vec> = vec![Box::new(evaluator)]; + + let (receipt, outcomes) = harness + .evaluate_and_enforce( + &plan, + &subject, + &evaluators, + Timestamp(1000), + || async move { + rollback_flag.store(true, Ordering::SeqCst); + Ok(()) + }, + ) + .await + .unwrap(); + + assert_eq!(receipt.receipt.verdict, Verdict::Fail); + assert_eq!(outcomes[0].status, EvaluatorStatus::Failed); + assert!(rollback_executed.load(Ordering::SeqCst)); +}