Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "postil-cli"
version = "0.4.3"
version = "0.4.4"
edition = "2024"
description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk."
license = "Apache-2.0"
Expand Down
5 changes: 3 additions & 2 deletions src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ pub struct Config {
pub gate_fail_on: GateLevel,
/// Gate behavior on operational error. Default: fail closed.
pub gate_on_error: OnError,
/// Finding kinds that block the gate regardless of severity. Default: [HumanEscalation].
/// Finding kinds that block regardless of severity. Human escalations must
/// also meet the calibrated 0.30 gate confidence floor. Default: [HumanEscalation].
pub block_on_kinds: Vec<Kind>,
pub model: String,
pub cascade: Vec<String>,
Expand Down Expand Up @@ -499,7 +500,7 @@ review:

gate:
failOn: error # the postil/gate check fails at/above: info | warn | error | never
# blockOnKinds: # finding kinds that block regardless of severity (default: [humanEscalation])
# blockOnKinds: # kinds that block regardless of severity; humanEscalation also requires confidence >= 0.30
# - humanEscalation
# onError: block # block (default, fail closed) | advisory — gate outcome when
# # the review itself errors (model outage). advisory keeps an
Expand Down
59 changes: 59 additions & 0 deletions src/envelope.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ pub enum Kind {
ContentPolicy,
}

/// Minimum confidence at which a human-escalation finding can block a merge.
///
/// Escalations are intentionally kept visible below this floor so reviewers can
/// inspect weak signals, but a very weak escalation must not turn into a hard
/// gate merely because of its kind or severity. A 0.30 floor is conservative:
/// it filters the near-zero boilerplate observed in production while retaining
/// uncertain but concrete concerns for human attention.
pub const HUMAN_ESCALATION_GATE_MIN_CONFIDENCE: f64 = 0.30;

impl Kind {
pub fn parse(s: &str) -> Option<Self> {
match s.trim().to_ascii_lowercase().as_str() {
Expand Down Expand Up @@ -192,6 +201,33 @@ impl Envelope {
}
}

/// Apply the serialized gate contract to one finding.
///
/// Keeping this beside the envelope makes live reviews, stored-envelope replay,
/// and forge summaries share exactly the same blocking semantics.
pub fn finding_blocks_gate(
finding: &Finding,
fail_on: &str,
block_on_kinds: &[String],
provider_error_is_advisory: bool,
) -> bool {
if finding.kind == Kind::HumanEscalation
&& finding.confidence < HUMAN_ESCALATION_GATE_MIN_CONFIDENCE
{
return false;
}

let severity_blocks = if provider_error_is_advisory || fail_on.eq_ignore_ascii_case("never") {
false
} else {
Severity::parse(fail_on).is_some_and(|threshold| finding.severity >= threshold)
};
let kind_blocks = block_on_kinds
.iter()
.any(|kind| Kind::parse(kind).is_some_and(|kind| kind == finding.kind));
severity_blocks || kind_blocks
}

/// Path marker for synthetic unusable-output findings (the model answered but
/// the output could not be validated). A malicious diff can induce this class
/// via prompt injection, so it always fails the gate — even under
Expand Down Expand Up @@ -347,6 +383,29 @@ mod tests {
assert_eq!(Kind::parse("unknown"), None);
}

#[test]
fn weak_human_escalation_never_blocks_but_floor_does() {
let mut escalation = finding(Severity::Error, 0.05);
escalation.kind = Kind::HumanEscalation;
let block_on_kinds = vec!["humanEscalation".to_string()];

assert!(!finding_blocks_gate(
&escalation,
"error",
&block_on_kinds,
false
));

escalation.severity = Severity::Warn;
escalation.confidence = HUMAN_ESCALATION_GATE_MIN_CONFIDENCE;
assert!(finding_blocks_gate(
&escalation,
"error",
&block_on_kinds,
false
));
}

#[test]
fn envelope_serializes_camel_case() {
let env = Envelope {
Expand Down
2 changes: 1 addition & 1 deletion src/forge/azure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ impl Forge for Azure {
&head,
"postil/review",
map(advisory),
&check_summary(envelope, false),
&check_summary(envelope, false, Default::default()),
)
.await?;
let gate_desc = if envelope.gate.failing {
Expand Down
2 changes: 1 addition & 1 deletion src/forge/bitbucket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ impl Forge for Bitbucket {
&head,
"postil/review",
map(advisory),
&check_summary(envelope, false),
&check_summary(envelope, false, Default::default()),
)
.await?;
let gate_desc = if envelope.gate.failing {
Expand Down
138 changes: 126 additions & 12 deletions src/forge/github.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ use anyhow::{Context, Result, anyhow};
use serde::Deserialize;
use serde_json::json;

use super::{CheckState, Forge, PrMeta, ThreadKind, check_summary, check_title, wrap_plain_text};
use super::{
CheckState, Forge, PrMeta, SummaryContext, ThreadKind, check_summary, check_title,
wrap_plain_text,
};
use crate::envelope::{Envelope, Finding, Severity};
use crate::filter;

Expand All @@ -16,6 +19,7 @@ pub struct GitHub {
owner: String,
repo: String,
pr: u64,
web_base: String,
}

impl GitHub {
Expand All @@ -28,6 +32,8 @@ impl GitHub {
let api_base = std::env::var("GITHUB_API_URL")
.unwrap_or_else(|_| "https://api.github.com".to_string());
let details_url = valid_details_url(std::env::var("POSTIL_DETAILS_URL").ok());
let web_base = valid_details_url(std::env::var("GITHUB_SERVER_URL").ok())
.unwrap_or_else(|| "https://github.com".to_string());
Ok(GitHub {
http: reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
Expand All @@ -38,6 +44,7 @@ impl GitHub {
owner: owner.to_string(),
repo: repo.to_string(),
pr,
web_base: web_base.trim_end_matches('/').to_string(),
})
}

Expand Down Expand Up @@ -89,7 +96,7 @@ fn gate_title(envelope: &Envelope) -> &'static str {
fn gate_summary(envelope: &Envelope) -> String {
if !envelope.gate.failing {
return format!(
"Merge gate passed: no findings at or above the blocking threshold ({}).\n",
"Merge gate passed: no findings block under the configured policy (failOn: {}).\n",
envelope.gate.fail_on
);
}
Expand All @@ -98,20 +105,28 @@ fn gate_summary(envelope: &Envelope) -> String {
.findings
.iter()
.filter(|f| {
crate::config::GateLevel::parse(&envelope.gate.fail_on)
.map(|g| g.fails(f.severity))
.unwrap_or(false)
crate::envelope::finding_blocks_gate(
f,
&envelope.gate.fail_on,
&envelope.gate.block_on_kinds,
false,
)
})
.map(|f| format!("- `{}:{}` {}", f.path, f.line, f.title))
.collect();
let noun = if failing.len() == 1 {
"finding"
if failing.is_empty() {
return format!(
"Merge gate failed under the configured operational error policy (failOn: {}).\n",
envelope.gate.fail_on
);
}
let subject = if failing.len() == 1 {
"1 finding blocks".to_string()
} else {
"findings"
format!("{} findings block", failing.len())
};
format!(
"Merge gate failed: {} {noun} at or above the blocking threshold ({}).\n\n{}\n",
failing.len(),
"Merge gate failed: {subject} under the configured policy (failOn: {}).\n\n{}\n",
envelope.gate.fail_on,
failing.join("\n")
)
Expand Down Expand Up @@ -140,6 +155,23 @@ impl Forge for GitHub {
true
}

fn review_summary(&self, envelope: &Envelope) -> String {
let commit_url = envelope.head_sha.as_deref().map(|sha| {
format!(
"{}/{}/{}/commit/{sha}",
self.web_base, self.owner, self.repo
)
});
check_summary(
envelope,
true,
SummaryContext {
commit_url: commit_url.as_deref(),
details_url: self.details_url.as_deref(),
},
)
}

async fn fetch_pr_meta(&self) -> Result<PrMeta> {
let resp = self
.request(
Expand Down Expand Up @@ -305,7 +337,7 @@ impl Forge for GitHub {
let gate_note = if name == "postil/gate" {
gate_summary(envelope)
} else {
check_summary(envelope, true)
self.review_summary(envelope)
};
let title = if name == "postil/gate" {
gate_title(envelope).to_string()
Expand Down Expand Up @@ -373,7 +405,8 @@ impl Forge for GitHub {

#[cfg(test)]
mod tests {
use super::valid_details_url;
use super::{gate_summary, valid_details_url};
use crate::envelope::{Envelope, Finding, Gate, Kind, Severity, Usage};

#[test]
fn details_url_accepts_only_http_and_https_urls() {
Expand All @@ -390,4 +423,85 @@ mod tests {
assert_eq!(valid_details_url(Some("not a URL".into())), None);
assert_eq!(valid_details_url(None), None);
}

#[test]
fn gate_summary_counts_qualified_kind_blocks_from_stored_envelope() {
let finding = Finding {
path: "src/auth.rs".into(),
line: 41,
end_line: None,
severity: Severity::Warn,
kind: Kind::HumanEscalation,
confidence: 0.30,
generator_confidence: None,
scorer_confidence: None,
generator_kind: None,
scorer_kind: None,
scorer_reason: None,
title: "Human judgment required".into(),
body: "Concrete compatibility concern.".into(),
id: None,
};
let env = Envelope {
version: 1,
summary: String::new(),
silent: false,
findings: vec![finding],
resolved: vec![],
counts: Default::default(),
confidence_buckets: [0; 5],
gate: Gate {
fail_on: "error".into(),
failing: true,
block_on_kinds: vec!["humanEscalation".into()],
},
model_used: "m".into(),
scorer_model: None,
scorer_error: None,
scorer_disagreements: None,
usage: Usage::default(),
duration_ms: 0,
base_sha: None,
head_sha: None,
since_sha: None,
};

let summary = gate_summary(&env);
assert!(summary.contains("1 finding blocks under the configured policy"));
assert!(summary.contains("src/auth.rs:41"));
}

#[test]
fn gate_summary_reports_operational_failure_without_inventing_blockers() {
let mut env = Envelope {
version: 1,
summary: String::new(),
silent: false,
findings: vec![],
resolved: vec![],
counts: Default::default(),
confidence_buckets: [0; 5],
gate: Gate {
fail_on: "never".into(),
failing: true,
block_on_kinds: vec![],
},
model_used: "m".into(),
scorer_model: None,
scorer_error: None,
scorer_disagreements: None,
usage: Usage::default(),
duration_ms: 0,
base_sha: None,
head_sha: None,
since_sha: None,
};
env.findings
.push(crate::envelope::provider_error_finding("timeout"));

assert_eq!(
gate_summary(&env),
"Merge gate failed under the configured operational error policy (failOn: never).\n"
);
}
}
2 changes: 1 addition & 1 deletion src/forge/gitlab.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ impl Forge for GitLab {
&head,
"postil/review",
map(advisory),
&check_summary(envelope, true),
&check_summary(envelope, true, Default::default()),
)
.await?;
let gate_desc = if envelope.gate.failing {
Expand Down
Loading
Loading