diff --git a/crates/fractal-api-core/src/api/operations.rs b/crates/fractal-api-core/src/api/operations.rs index 8f1ec388db5..61cace17d3c 100644 --- a/crates/fractal-api-core/src/api/operations.rs +++ b/crates/fractal-api-core/src/api/operations.rs @@ -135,6 +135,7 @@ pub async fn doctor( client_now_ms(), )); report.checks.extend(crate::github_admission::doctor_lines(client_now_ms())); + report.checks.extend(crate::github_webhook::doctor_line(client_now_ms())); report.checks.push(reader_store.mailbox_wake_health()); report.checks.push(descriptor_check()); report.checks.push(reader_memory_check( diff --git a/crates/fractal-daemon/src/github_webhook.rs b/crates/fractal-daemon/src/github_webhook.rs new file mode 100644 index 00000000000..c21e1d2e32f --- /dev/null +++ b/crates/fractal-daemon/src/github_webhook.rs @@ -0,0 +1,321 @@ +//! The GitHub webhook receiver: one loopback route, `POST /fractal/github-webhook`, that verifies +//! each delivery's signature and asks the observers of the repository it names to poll now. +//! A public proxy forwards the route; the secret's custody is the operator's. +use std::{io::Read as _, net::SocketAddr, path::{Path, PathBuf}, sync::Arc}; + +use axum::{ + Json, Router, + body::{Body, Bytes}, + extract::State, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, +}; +use fractal_kernel::github_webhook::{self as webhook, Outcome}; +use hmac::{Hmac, Mac as _}; +use sha2::Sha256; +use tokio::{net::TcpListener, sync::{Notify, watch}}; + +use crate::store::Store; + +#[derive(Clone)] +struct Receiver { + secret_file: Arc, + store: Arc, + notify: Arc, + event_notify: watch::Sender, +} + +/// A bound receiver, ready to serve. +pub struct Bound { + listener: TcpListener, + router: Router, +} + +impl Bound { + pub fn local_addr(&self) -> std::io::Result { + self.listener.local_addr() + } + + pub async fn serve(self) -> anyhow::Result<()> { + axum::serve(self.listener, self.router).await?; + Ok(()) + } +} + +/// Bind the receiver `[github].webhook_listen` configures, or `None` when it is unset. The +/// secret file is not read here: it is read for every delivery, so it may appear later. +pub async fn bind( + config: &crate::config::GithubConfig, + store: Arc, + notify: Arc, + event_notify: watch::Sender, +) -> anyhow::Result> { + let (Some(listen), Some(secret_file)) = (config.webhook_listen, config.webhook_secret_file.clone()) else { + return Ok(None); + }; + let listener = TcpListener::bind(listen).await + .map_err(|error| anyhow::anyhow!("bind the GitHub webhook receiver at {listen}: {error}"))?; + let receiver = Receiver { secret_file: Arc::new(secret_file), store, notify, event_notify }; + let router = Router::new() + .route(webhook::ROUTE, post(deliver).fallback(not_found)) + .fallback(not_found) + .with_state(receiver); + webhook::listening(&listener.local_addr()?.to_string()); + Ok(Some(Bound { listener, router })) +} + +async fn not_found() -> StatusCode { + StatusCode::NOT_FOUND +} + +async fn deliver(State(receiver): State, headers: HeaderMap, body: Body) -> Response { + let event = header(&headers, "x-github-event"); + let (outcome, response) = receive(&receiver, &headers, event, body).await; + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_millis(); + webhook::record(outcome, now); + crate::otel::github_webhook_delivery(webhook::event_label(event.unwrap_or("")), outcome.as_str()); + response +} + +fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name).and_then(|value| value.to_str().ok()).filter(|value| !value.is_empty()) +} + +async fn receive(receiver: &Receiver, headers: &HeaderMap, event: Option<&str>, body: Body) -> (Outcome, Response) { + let reply = |outcome, status: StatusCode| (outcome, status.into_response()); + let Ok(body) = axum::body::to_bytes(body, webhook::BODY_LIMIT).await else { + return reply(Outcome::Invalid, StatusCode::PAYLOAD_TOO_LARGE); + }; + let secret = match read_secret(&receiver.secret_file) { + Ok(secret) => secret, + Err(error) => { + tracing::warn!(error = format!("{error:#}"), "GitHub webhook secret unavailable; refusing the delivery"); + return reply(Outcome::Unavailable, StatusCode::SERVICE_UNAVAILABLE); + } + }; + if !signed(&secret, header(headers, "x-hub-signature-256"), &body) { + return reply(Outcome::Unauthorized, StatusCode::UNAUTHORIZED); + } + let (Some(event), Some(delivery)) = (event, header(headers, "x-github-delivery") + .filter(|delivery| delivery.len() <= 128 && delivery.bytes().all(|byte| byte.is_ascii_graphic()))) + else { + return reply(Outcome::Invalid, StatusCode::BAD_REQUEST); + }; + if event == "ping" { + return reply(Outcome::Ignored, StatusCode::OK); + } + if matches!(webhook::event_label(event), "other" | "ping") { + return reply(Outcome::Ignored, StatusCode::NO_CONTENT); + } + let scope = match serde_json::from_slice(&body).map_err(|_| "the payload is not JSON") + .and_then(|payload| webhook::scope(event, &payload)) + { + Ok(Some(scope)) => scope, + Ok(None) => return reply(Outcome::Ignored, StatusCode::NO_CONTENT), + Err(_) => return reply(Outcome::Invalid, StatusCode::BAD_REQUEST), + }; + let store = receiver.store.clone(); + let delivery = delivery.to_owned(); + let refreshed = store.connection.write_workers() + .spawn_blocking({ + let store = store.clone(); + move || store.request_github_webhook_refresh(&delivery, &scope) + }) + .await; + match refreshed { + Ok(Ok(refreshed)) => { + if refreshed > 0 { + receiver.notify.notify_one(); + receiver.event_notify.send_modify(|generation| *generation = generation.saturating_add(1)); + } + (Outcome::Refreshed, (StatusCode::ACCEPTED, Json(serde_json::json!({"refreshed": refreshed}))).into_response()) + } + Ok(Err(error)) => { + tracing::warn!(code = error.code, message = error.message, "GitHub webhook refresh failed"); + reply(Outcome::Unavailable, StatusCode::SERVICE_UNAVAILABLE) + } + Err(error) => { + tracing::warn!(%error, "GitHub webhook refresh stopped"); + reply(Outcome::Unavailable, StatusCode::SERVICE_UNAVAILABLE) + } + } +} + +/// The secret, from an owner-only regular file read again for every delivery. +fn read_secret(path: &Path) -> anyhow::Result> { + let mut secret = String::new(); + crate::config::open_token_file(path)? + .read_to_string(&mut secret) + .map_err(|error| anyhow::anyhow!("read {}: {error}", path.display()))?; + let secret = secret.trim_end_matches(['\r', '\n']); + anyhow::ensure!(!secret.is_empty(), "{} is empty", path.display()); + Ok(secret.as_bytes().to_vec()) +} + +/// Whether `signature` is `sha256=` and the HMAC-SHA256 of `body` under `secret`, compared in +/// constant time. +fn signed(secret: &[u8], signature: Option<&str>, body: &Bytes) -> bool { + let Some(expected) = signature + .and_then(|signature| signature.strip_prefix("sha256=")) + .and_then(|digest| hex::decode(digest).ok()) + else { + return false; + }; + let Ok(mut mac) = Hmac::::new_from_slice(secret) else { + return false; + }; + mac.update(body); + mac.verify_slice(&expected).is_ok() +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt as _; + + use reqwest::StatusCode; + use serde_json::{Value, json}; + + use super::*; + + const SECRET: &str = "fixture-webhook-secret"; + const OBSERVERS: &str = r#"version 2 +resource "garden" { kind "vcs.repository" } +observer "garden" { resource "resource/garden"; provider "github.repository"; locator "Acme/Garden"; field "pull_requests" } +resource "garden-main" { kind "vcs.ref" } +observer "garden-main" { resource "resource/garden-main"; provider "github.ref"; locator "acme/garden@main"; field "head" } +"#; + + struct Fixture { + _root: tempfile::TempDir, + secret_file: PathBuf, + store: Arc, + base: String, + client: reqwest::Client, + } + + impl Fixture { + async fn start() -> Self { + let root = tempfile::tempdir().unwrap(); + let secret_file = root.path().join("secret"); + let store = Arc::new(Store::open_memory("node").unwrap()); + let intent = crate::graph::parse_internal_intent(OBSERVERS, "node").unwrap(); + let planned = store + .mission(&intent, crate::model::IntentInput { kdl: OBSERVERS.into(), source_name: None }) + .unwrap(); + store.apply(&intent, &planned.subject_tokens, "webhook-observers").unwrap(); + let config = crate::config::GithubConfig { + webhook_listen: Some("127.0.0.1:0".parse().unwrap()), + webhook_secret_file: Some(secret_file.clone()), + ..Default::default() + }; + let bound = bind(&config, store.clone(), Arc::new(Notify::new()), watch::channel(0).0) + .await.unwrap().unwrap(); + let base = format!("http://{}", bound.local_addr().unwrap()); + tokio::spawn(bound.serve()); + Self { _root: root, secret_file, store, base, client: reqwest::Client::new() } + } + + fn write_secret(&self, mode: u32) { + std::fs::write(&self.secret_file, format!("{SECRET}\n")).unwrap(); + std::fs::set_permissions(&self.secret_file, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + async fn send(&self, event: &str, delivery: &str, payload: &Value, secret: &str) -> (StatusCode, String) { + let body = payload.to_string(); + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); + mac.update(body.as_bytes()); + let response = self.client.post(format!("{}{}", self.base, webhook::ROUTE)) + .header("x-github-event", event) + .header("x-github-delivery", delivery) + .header("x-hub-signature-256", format!("sha256={}", hex::encode(mac.finalize().into_bytes()))) + .header("content-type", "application/json") + .body(body) + .send().await.unwrap(); + (response.status(), response.text().await.unwrap()) + } + + fn requests(&self, observer: &str) -> Vec { + self.store.graph.claims_for(observer, Some("observer.refresh-requested")).unwrap() + .into_iter().map(|claim| claim.body["fields"].clone()).collect() + } + } + + fn push(branch: &str) -> Value { + json!({"ref": format!("refs/heads/{branch}"), "repository": {"full_name": "ACME/garden"}}) + } + + #[tokio::test] + async fn only_a_signed_delivery_with_a_present_secret_refreshes() { + let fixture = Fixture::start().await; + let (status, _) = fixture.send("push", "d-1", &push("main"), SECRET).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "an absent secret file never accepts a delivery"); + fixture.write_secret(0o644); + let (status, _) = fixture.send("push", "d-1", &push("main"), SECRET).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "a secret others can read is refused"); + fixture.write_secret(0o600); + let (status, _) = fixture.send("push", "d-1", &push("main"), "another-secret").await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + let unsigned = fixture.client.post(format!("{}{}", fixture.base, webhook::ROUTE)) + .header("x-github-event", "push").header("x-github-delivery", "d-1") + .body(push("main").to_string()).send().await.unwrap(); + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + assert!(fixture.requests("observer/garden").is_empty()); + + let (status, body) = fixture.send("push", "d-1", &push("main"), SECRET).await; + assert_eq!(status, StatusCode::ACCEPTED); + assert_eq!(serde_json::from_str::(&body).unwrap(), json!({"refreshed": 2})); + for observer in ["observer/garden", "observer/garden-main"] { + let revision = fixture.store.selected_desired_revision(observer).unwrap().unwrap(); + assert_eq!(fixture.requests(observer), [json!({ + "attempt": webhook::attempt("d-1", observer), + "revision": revision, + })]); + } + // A redelivery asks nothing more; another delivery asks again, as a poll may be running. + let (status, body) = fixture.send("push", "d-1", &push("main"), SECRET).await; + assert_eq!((status, body.as_str()), (StatusCode::ACCEPTED, r#"{"refreshed":0}"#)); + let opened = json!({"action": "opened", "repository": {"full_name": "acme/garden"}}); + let (status, body) = fixture.send("pull_request", "d-2", &opened, SECRET).await; + assert_eq!((status, body.as_str()), (StatusCode::ACCEPTED, r#"{"refreshed":1}"#)); + assert_eq!(fixture.requests("observer/garden").len(), 2); + } + + #[tokio::test] + async fn the_receiver_filters_events_and_serves_only_its_route() { + let fixture = Fixture::start().await; + fixture.write_secret(0o600); + let repository = json!({"full_name": "acme/garden"}); + assert_eq!(fixture.send("ping", "d-ping", &json!({"zen": "Keep it logically awesome."}), SECRET).await.0, StatusCode::OK); + assert_eq!(fixture.send("star", "d-star", &json!({"repository": repository}), SECRET).await.0, StatusCode::NO_CONTENT); + assert_eq!(fixture.send("check_run", "d-run", &json!({"action": "created", "repository": repository}), SECRET).await.0, StatusCode::NO_CONTENT); + assert_eq!(fixture.send("issues", "d-bad", &json!({"action": "opened"}), SECRET).await.0, StatusCode::BAD_REQUEST); + assert_eq!(fixture.send("push", "d-other", &json!({"ref": "refs/heads/main", "repository": {"full_name": "acme/greenhouse"}}), SECRET).await.0, StatusCode::ACCEPTED); + assert!(fixture.requests("observer/garden").is_empty()); + let (status, body) = fixture.send("check_run", "d-done", &json!({"action": "completed", "repository": repository}), SECRET).await; + assert_eq!((status, body.as_str()), (StatusCode::ACCEPTED, r#"{"refreshed":1}"#)); + assert!(fixture.requests("observer/garden-main").is_empty(), "a check run moves no branch"); + + let oversized = fixture.client.post(format!("{}{}", fixture.base, webhook::ROUTE)) + .body(vec![b' '; webhook::BODY_LIMIT + 1]).send().await.unwrap(); + assert_eq!(oversized.status(), StatusCode::PAYLOAD_TOO_LARGE); + for (method, path) in [(reqwest::Method::GET, webhook::ROUTE), (reqwest::Method::POST, "/"), (reqwest::Method::POST, "/v1/apply")] { + let response = fixture.client.request(method.clone(), format!("{}{path}", fixture.base)).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{method} {path}"); + } + } + + #[test] + fn the_signature_is_hmac_sha256_of_the_exact_body() { + let body = Bytes::from_static(b"{\"zen\":\"x\"}"); + let mut mac = Hmac::::new_from_slice(b"key").unwrap(); + mac.update(&body); + let signature = format!("sha256={}", hex::encode(mac.finalize().into_bytes())); + assert!(signed(b"key", Some(&signature), &body)); + assert!(!signed(b"other", Some(&signature), &body)); + assert!(!signed(b"key", Some(&signature), &Bytes::from_static(b"{\"zen\":\"y\"}"))); + assert!(!signed(b"key", Some(signature.trim_start_matches("sha256=")), &body)); + assert!(!signed(b"key", Some("sha256=not-hex"), &body)); + assert!(!signed(b"key", None, &body)); + } +} diff --git a/crates/fractal-daemon/src/lib.rs b/crates/fractal-daemon/src/lib.rs index 6e357dc5368..388a3463658 100644 --- a/crates/fractal-daemon/src/lib.rs +++ b/crates/fractal-daemon/src/lib.rs @@ -13,6 +13,8 @@ mod ci_test_paths; pub use fractal_kernel::accounts; pub mod api; pub mod web; +/// Verified GitHub webhook deliveries ask the observers of their repository to poll now. +pub mod github_webhook; pub use fractal_kernel::archive; pub mod attach_image; pub use fractal_store::backup; diff --git a/crates/fractal-github-base/src/resource.rs b/crates/fractal-github-base/src/resource.rs index 50daa078d32..8dac696e867 100644 --- a/crates/fractal-github-base/src/resource.rs +++ b/crates/fractal-github-base/src/resource.rs @@ -34,6 +34,9 @@ pub struct ObservationRequest { /// A declared refresh asked for this observation. It revalidates with GitHub instead of /// reusing a cached response; a conditional request costs nothing when nothing changed. pub refresh: bool, + /// A verified webhook delivery refreshed this observer recently, so this poll is a backstop + /// and a repository observer waits the settled interval between GraphQL reads. + pub webhook_covered: bool, } #[derive(Clone, Debug, PartialEq)] @@ -1490,6 +1493,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }; let empty = GithubAuth::test(""); let blank = GithubAuth::test(" "); @@ -1535,6 +1539,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }) .await .unwrap(); @@ -1610,6 +1615,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }) .await .unwrap(); @@ -1632,6 +1638,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }) .await .unwrap(); @@ -1686,6 +1693,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }; let observed = tokio::time::timeout( Duration::from_secs(10), @@ -1721,7 +1729,7 @@ mod tests { requests }); let request = || ObservationRequest { provider:"github.ref".into(), locator:"acme/garden@main".into(), fields:BTreeSet::from(["head".into()]), - cursor:None, previous_facts:None, every_ms:Some(GITHUB_REF_WATCH_MS), refresh:true }; + cursor:None, previous_facts:None, every_ms:Some(GITHUB_REF_WATCH_MS), refresh:true, webhook_covered:false }; let first = observe_github_ref_at(request(), &base, &GithubAuth::test("fixture-token")).await.unwrap(); let unchanged = observe_github_ref_at(request(), &base, &GithubAuth::test("fixture-token")).await.unwrap(); let changed = observe_github_ref_at(request(), &base, &GithubAuth::test("fixture-token")).await.unwrap(); @@ -1814,6 +1822,7 @@ mod tests { previous_facts: None, every_ms: None, refresh: false, + webhook_covered: false, }, &base, Some(&GithubAuth::test("orchid-test-token")), diff --git a/crates/fractal-github-base/src/resource/github_repository.rs b/crates/fractal-github-base/src/resource/github_repository.rs index 77f398f1040..e3826aa11ab 100644 --- a/crates/fractal-github-base/src/resource/github_repository.rs +++ b/crates/fractal-github-base/src/resource/github_repository.rs @@ -42,7 +42,8 @@ pub const DATA_TYPES: [&str; 5] = [ const SETTLED_PULL_REQUEST_CHECK: Duration = Duration::from_secs(15 * 60); /// The least time between two GraphQL reads of one repository on this host, however often its -/// observers poll and whatever they saw change. Only a refresh asks sooner. +/// observers poll and whatever they saw change. Only a refresh asks sooner. A webhook-covered +/// repository's refreshes carry its changes, so its polls wait `SETTLED_PULL_REQUEST_CHECK`. const PULL_REQUEST_CHECK_INTERVAL: Duration = Duration::from_secs(5 * 60); /// The most comments and reviews each item keeps in `recent_comments`, newest last. @@ -1663,6 +1664,7 @@ async fn observe_poll( &request.locator, current_location, request.refresh, + request.webhook_covered, changed_open_pull || !cursor.seen.contains_key("pull-requests") || items @@ -1864,10 +1866,11 @@ fn merge_legacy_statuses(node: &mut Value, combined: &Value) -> Result<()> { /// The repository's open pull requests as GraphQL last answered on this host. GraphQL has no /// conditional request, so it is asked again only when `changed` says the REST reads saw an open /// pull request change, while a pull request is settling, or once the last answer is old, and -/// never sooner than a minute after the last read unless `refresh` asks or the REST reads name -/// an open pull request (`listed`) the last answer lacks: a new pull request's first record +/// never sooner than the active floor after the last read unless `refresh` asks or the REST reads +/// name an open pull request (`listed`) the last answer lacks: a new pull request's first record /// carries its head, since only a head that differs from a known one asks for a review. A change -/// seen within that minute is remembered, so the first read after it asks. +/// seen within the floor is remembered, so the first read after it asks. A `webhook_covered` +/// repository's floor is the settled interval: its webhook refreshes read changes as they happen. #[allow(clippy::too_many_arguments)] async fn open_pull_requests( client: &reqwest::Client, @@ -1877,6 +1880,7 @@ async fn open_pull_requests( locator: &str, (owner, repository): (&str, &str), refresh: bool, + webhook_covered: bool, changed: bool, listed: &BTreeSet, emit_pulls: bool, @@ -1914,7 +1918,7 @@ async fn open_pull_requests( .iter() .any(|facts| facts.get("number").and_then(Value::as_u64) == Some(*number)) }) - && (previous.at.elapsed() < PULL_REQUEST_CHECK_INTERVAL + && (previous.at.elapsed() < if webhook_covered { SETTLED_PULL_REQUEST_CHECK } else { PULL_REQUEST_CHECK_INTERVAL } || (!previous.changed && !previous.settling && previous.at.elapsed() < SETTLED_PULL_REQUEST_CHECK)) @@ -2219,6 +2223,7 @@ mod tests { // Each poll revalidates instead of reusing a cached response. every_ms: Some(0), refresh: false, + webhook_covered: false, } } @@ -2272,7 +2277,7 @@ mod tests { }).await; let details = open_pull_requests(&reqwest::Client::new(), &fixture.base, &GithubAuth::test("fixture-pat"), Duration::ZERO, "pat-fixture/repo", - ("pat-fixture", "repo"), true, true, &BTreeSet::new(), false, + ("pat-fixture", "repo"), true, false, true, &BTreeSet::new(), false, &[("opaque-issue".into(), false), ("opaque-pull".into(), true)]).await.unwrap(); assert_eq!(details.identities_read, 2); assert_eq!(details.resolved.len(), 2); @@ -2306,7 +2311,7 @@ mod tests { }).await; for _ in 0..2 { open_pull_requests(&reqwest::Client::new(), &fixture.base, &auth, - Duration::ZERO, "forbidden-pulls/repo", (owner, "repo"), false, true, + Duration::ZERO, "forbidden-pulls/repo", (owner, "repo"), false, false, true, &BTreeSet::new(), true, &[("opaque-issue".into(), false), ("opaque-pull".into(), true)]).await.unwrap(); } let calls = calls.lock(); @@ -2360,7 +2365,7 @@ mod tests { } }).await; let details = open_pull_requests(&reqwest::Client::new(), &fixture.base, &auth, - Duration::ZERO, "mixed-fixture/repo", (owner, "repo"), true, true, + Duration::ZERO, "mixed-fixture/repo", (owner, "repo"), true, false, true, &BTreeSet::new(), true, &[("opaque-issue".into(), false), ("opaque-pull".into(), true)]).await.unwrap(); assert_eq!(details.identities_read, 2); assert_eq!(details.resolved.len(), 2); @@ -2371,7 +2376,7 @@ mod tests { assert!(pull["checks"].as_array().unwrap().iter().any(|check| check["name"] == "lint")); } let reused = open_pull_requests(&reqwest::Client::new(), &fixture.base, &auth, - Duration::ZERO, "mixed-fixture/repo", (owner, "repo"), false, false, + Duration::ZERO, "mixed-fixture/repo", (owner, "repo"), false, false, false, &BTreeSet::new(), true, &[("opaque-issue".into(), false), ("opaque-pull".into(), true)]).await.unwrap(); assert_eq!(reused.identities_read, 0, "intentional PAT status reads must preserve the GraphQL detail cache"); assert!(Arc::ptr_eq(&details.open, &reused.open)); @@ -2411,7 +2416,7 @@ mod tests { }).await; for _ in 0..2 { let details = open_pull_requests(&reqwest::Client::new(), &fixture.base, &auth, - Duration::ZERO, "status-etag-fixture/repo", (owner, "repo"), true, true, + Duration::ZERO, "status-etag-fixture/repo", (owner, "repo"), true, false, true, &BTreeSet::new(), true, &[]).await.unwrap(); assert!(details.open[0]["checks"].as_array().unwrap().iter().any(|check| check["name"] == "legacy/build")); } @@ -2435,11 +2440,36 @@ mod tests { open_pull_requests( &reqwest::Client::new(), base, &GithubAuth::test("fixture-auth"), Duration::ZERO, "github/acme/garden", ("acme", "garden"), - true, true, &BTreeSet::new(), true, &identities, + true, false, true, &BTreeSet::new(), true, &identities, ) .await } + #[tokio::test] + async fn webhook_coverage_raises_the_graphql_floor_to_the_settled_interval() { + let (github, base) = FakeGithub::start().await; + github.open_pull_requests(json!([open_pull(1, "head", "SUCCESS")])); + let read = |webhook_covered: bool| { + let base = base.clone(); + async move { + open_pull_requests(&reqwest::Client::new(), &base, &GithubAuth::test("fixture-auth"), + Duration::ZERO, "acme/garden", ("acme", "garden"), false, webhook_covered, true, + &BTreeSet::new(), true, &[]).await.unwrap(); + } + }; + let reads = || std::mem::take(&mut *github.graphql_requests.lock()).len(); + read(false).await; + assert_eq!(reads(), 1); + age_pull_request_check(&base, PULL_REQUEST_CHECK_INTERVAL); + read(true).await; + assert_eq!(reads(), 0, "a covered repository's polls wait out the settled interval"); + read(false).await; + assert_eq!(reads(), 1, "an uncovered repository reads a change after the active floor"); + age_pull_request_check(&base, SETTLED_PULL_REQUEST_CHECK); + read(true).await; + assert_eq!(reads(), 1, "the settled interval still bounds a covered repository"); + } + #[tokio::test] async fn smaller_pages_fold_to_the_same_facts_as_a_single_page() { let pulls = json!([ @@ -3365,6 +3395,7 @@ mod tests { ("acme", "garden"), false, false, + false, &BTreeSet::new(), true, &[], @@ -3400,6 +3431,7 @@ mod tests { ("acme", "garden"), false, false, + false, &BTreeSet::new(), true, &[], diff --git a/crates/fractal-kernel/src/config.rs b/crates/fractal-kernel/src/config.rs index 7f878a9de72..c8bd05c8aa8 100644 --- a/crates/fractal-kernel/src/config.rs +++ b/crates/fractal-kernel/src/config.rs @@ -254,6 +254,12 @@ pub struct GithubConfig { /// current usage and freshness are qualified; Factory's proposed cap is1000. #[serde(skip_serializing_if = "Option::is_none")] pub observer_requests_per_hour: Option, + /// Loopback address of the GitHub webhook receiver; unset runs no receiver. + #[serde(skip_serializing_if = "Option::is_none")] + pub webhook_listen: Option, + /// Absolute owner-only file holding the webhook secret, read again for every delivery. + #[serde(skip_serializing_if = "Option::is_none")] + pub webhook_secret_file: Option, } impl GithubConfig { @@ -271,6 +277,15 @@ impl GithubConfig { anyhow::ensure!(!profile.trim().is_empty() && profile.trim() == profile, "github.sekrets_profile must name a non-empty profile without surrounding whitespace"); } + if let Some(listen) = self.webhook_listen { + anyhow::ensure!(listen.ip().is_loopback() && listen.port() != 0, + "github.webhook_listen must be a loopback address with a nonzero port"); + } + anyhow::ensure!(self.webhook_listen.is_some() == self.webhook_secret_file.is_some(), + "github.webhook_listen and github.webhook_secret_file must be set together"); + if let Some(path) = &self.webhook_secret_file { + anyhow::ensure!(path.is_absolute(), "github.webhook_secret_file must be absolute"); + } if let Some(path) = &self.token_file { open_token_file(path)?; } @@ -1595,6 +1610,29 @@ mod tests { assert_eq!(decoded, config); } + #[test] + fn github_webhook_listens_on_loopback_with_an_absolute_secret_file() { + let parse = |text: &str| toml::from_str::(text).map(|config| config.validate().map(|()| config)); + let config = parse("webhook_listen = \"127.0.0.1:41871\"\nwebhook_secret_file = \"/run/fractal/github-webhook-secret\"\n") + .unwrap().unwrap(); + assert_eq!(config.webhook_listen, Some("127.0.0.1:41871".parse().unwrap())); + assert_eq!(toml::from_str::(&toml::to_string(&config).unwrap()).unwrap(), config); + // The secret file need not exist at startup; the receiver reads it per delivery. + parse("webhook_listen = \"[::1]:41871\"\nwebhook_secret_file = \"/absent/secret\"\n").unwrap().unwrap(); + for (text, expected) in [ + ("webhook_listen = \"0.0.0.0:41871\"\nwebhook_secret_file = \"/s\"\n", "loopback"), + ("webhook_listen = \"127.0.0.1:0\"\nwebhook_secret_file = \"/s\"\n", "loopback"), + ("webhook_listen = \"127.0.0.1:41871\"\n", "together"), + ("webhook_secret_file = \"/s\"\n", "together"), + ("webhook_listen = \"127.0.0.1:41871\"\nwebhook_secret_file = \"relative\"\n", "absolute"), + ] { + let error = parse(text).unwrap().unwrap_err().to_string(); + assert!(error.contains(expected), "{text}: {error}"); + } + assert!(parse("webhook_listen = \"localhost:41871\"\nwebhook_secret_file = \"/s\"\n").is_err()); + assert!(parse("webhook_url = \"x\"\n").is_err()); + } + #[test] fn the_work_root_defaults_to_a_folder_inside_home_that_macos_does_not_guard() { let Ok(home) = super::home_dir() else { return }; diff --git a/crates/fractal-kernel/src/github_webhook.rs b/crates/fractal-kernel/src/github_webhook.rs new file mode 100644 index 00000000000..7e25a5d0c3f --- /dev/null +++ b/crates/fractal-kernel/src/github_webhook.rs @@ -0,0 +1,272 @@ +//! Verified GitHub webhook deliveries ask the observers of the repository they name to poll now. +//! See `docs/st3/github-observer-admission.md#webhook-delivery`. + +use std::collections::BTreeMap; +use std::sync::{Mutex, PoisonError}; + +use serde_json::Value; +use sha2::{Digest as _, Sha256}; + +use crate::model::{DoctorCheck, ObserverSpec}; + +/// The receiver's only route; the public proxy forwards this path verbatim. +pub const ROUTE: &str = "/fractal/github-webhook"; + +/// The prefix of `observer.refresh-requested.attempt` on a refresh a verified delivery requested. +/// Declared refreshes carry a bare hex digest, so the prefix alone marks webhook refreshes. +pub const ATTEMPT_PREFIX: &str = "github-webhook:"; + +/// How long after a webhook-requested refresh its observer counts as webhook-covered, so its +/// polls are only a backstop. +pub const COVERAGE_MS: u128 = 30 * 60 * 1_000; + +/// GitHub's own limit on a delivery's payload. +pub const BODY_LIMIT: usize = 5 * 1024 * 1024; + +/// Events that can change what a repository or ref observer reads. +const REFRESHING: &[&str] = &[ + "pull_request", + "pull_request_review", + "pull_request_review_comment", + "pull_request_review_thread", + "issue_comment", + "issues", + "push", + "check_suite", + "check_run", + "status", + "workflow_run", + "merge_group", + "create", + "delete", +]; + +/// Events whose deliveries refresh only once their run has finished. +const ON_COMPLETION: &[&str] = &["check_suite", "check_run", "workflow_run"]; + +/// The metric and doctor outcome of one delivery. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum Outcome { + Refreshed, + Ignored, + Unauthorized, + Unavailable, + Invalid, +} + +impl Outcome { + pub fn as_str(self) -> &'static str { + match self { + Self::Refreshed => "refreshed", + Self::Ignored => "ignored", + Self::Unauthorized => "unauthorized", + Self::Unavailable => "unavailable", + Self::Invalid => "invalid", + } + } +} + +/// A bounded metric label for an unauthenticated event header. +pub fn event_label(event: &str) -> &'static str { + REFRESHING + .iter() + .chain(&["ping"]) + .find(|known| **known == event) + .copied() + .unwrap_or("other") +} + +/// What one delivery refreshes: the repository's observers and, for a branch it moved, that +/// branch's ref observers. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Scope { + /// `owner/repo`, lowercased. + pub repository: String, + pub branch: Option, +} + +/// The scope of a delivery, `None` for an event that changes nothing an observer reads, or an +/// error for a payload without the repository it concerns. +pub fn scope(event: &str, payload: &Value) -> Result, &'static str> { + if !REFRESHING.contains(&event) + || (ON_COMPLETION.contains(&event) + && payload.get("action").and_then(Value::as_str) != Some("completed")) + { + return Ok(None); + } + let repository = payload + .pointer("/repository/full_name") + .and_then(Value::as_str) + .filter(|name| name.split_once('/').is_some_and(|(owner, repo)| !owner.is_empty() && !repo.is_empty() && !repo.contains('/'))) + .ok_or("the payload names no repository.full_name")? + .to_ascii_lowercase(); + let reference = payload.get("ref").and_then(Value::as_str); + let branch = match event { + "push" => reference.and_then(|name| name.strip_prefix("refs/heads/")), + "create" | "delete" => reference + .filter(|_| payload.get("ref_type").and_then(Value::as_str) == Some("branch")), + _ => None, + }; + Ok(Some(Scope { + repository, + branch: branch.filter(|branch| !branch.is_empty()).map(str::to_owned), + })) +} + +/// Whether a delivery with `scope` refreshes an observer of `spec`: a repository observer whose +/// locator names the repository in any case, or a ref observer of the branch it moved. +pub fn refreshes(spec: &ObserverSpec, scope: &Scope) -> bool { + match spec.provider.as_str() { + "github.repository" => spec.locator.eq_ignore_ascii_case(&scope.repository), + "github.ref" => scope.branch.as_deref().is_some_and(|branch| { + spec.locator.rsplit_once('@').is_some_and(|(repository, name)| { + repository.eq_ignore_ascii_case(&scope.repository) && name == branch + }) + }), + _ => false, + } +} + +/// The refresh attempt one delivery asks of one observer; a redelivery asks the same one. +pub fn attempt(delivery: &str, observer: &str) -> String { + format!( + "{ATTEMPT_PREFIX}{}", + hex::encode(Sha256::digest(format!("fractal.github-webhook.v1\0{delivery}\0{observer}"))) + ) +} + +#[derive(Default)] +struct Receiver { + listen: Option, + last_verified_unix_ms: Option, + last: Option, + counts: BTreeMap, +} + +fn receiver() -> &'static Mutex { + static RECEIVER: Mutex = Mutex::new(Receiver { + listen: None, + last_verified_unix_ms: None, + last: None, + counts: BTreeMap::new(), + }); + &RECEIVER +} + +/// Mark this daemon as running the receiver, so doctor reports it. +pub fn listening(listen: &str) { + receiver().lock().unwrap_or_else(PoisonError::into_inner).listen = Some(listen.to_owned()); +} + +/// Count one delivery for doctor. A delivery whose signature verified sets the last verified time. +pub fn record(outcome: Outcome, now_unix_ms: u128) { + let mut receiver = receiver().lock().unwrap_or_else(PoisonError::into_inner); + if matches!(outcome, Outcome::Refreshed | Outcome::Ignored) { + receiver.last_verified_unix_ms = Some(now_unix_ms); + } + receiver.last = Some(outcome); + *receiver.counts.entry(outcome).or_default() += 1; +} + +/// The `github-webhook` doctor line, only on a daemon that runs the receiver. +pub fn doctor_line(now_unix_ms: u128) -> Option { + let receiver = receiver().lock().unwrap_or_else(PoisonError::into_inner); + let listen = receiver.listen.as_deref()?; + let last = match receiver.last_verified_unix_ms { + Some(at) => format!( + "last verified delivery {} ({}s ago)", + chrono::DateTime::from_timestamp_millis(i64::try_from(at).unwrap_or(i64::MAX)) + .map_or_else(|| at.to_string(), |time| time.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), + now_unix_ms.saturating_sub(at) / 1_000, + ), + None => "no verified delivery since start".into(), + }; + let counts = [Outcome::Refreshed, Outcome::Ignored, Outcome::Unauthorized, Outcome::Unavailable, Outcome::Invalid] + .map(|outcome| format!("{} {}", outcome.as_str(), receiver.counts.get(&outcome).copied().unwrap_or(0))) + .join(", "); + Some(DoctorCheck { + name: "github-webhook".into(), + status: if matches!(receiver.last, Some(Outcome::Unauthorized | Outcome::Unavailable)) { "warn" } else { "pass" }.into(), + message: format!("listening at http://{listen}{ROUTE}; {last}; deliveries since start: {counts}"), + }) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn spec(provider: &str, locator: &str) -> ObserverSpec { + ObserverSpec { + resource: "resource/github/acme/garden".into(), + provider: provider.into(), + locator: locator.into(), + fields: Vec::new(), + every_ms: None, + stopped: false, + } + } + + #[test] + fn only_events_that_change_observed_facts_have_a_scope() { + let repository = json!({"full_name": "Acme/Garden"}); + let garden = |branch: Option<&str>| Some(Scope { repository: "acme/garden".into(), branch: branch.map(Into::into) }); + assert_eq!(scope("pull_request", &json!({"action": "opened", "repository": repository})), Ok(garden(None))); + assert_eq!(scope("push", &json!({"ref": "refs/heads/main", "repository": repository})), Ok(garden(Some("main")))); + assert_eq!(scope("push", &json!({"ref": "refs/tags/v1", "repository": repository})), Ok(garden(None))); + assert_eq!(scope("create", &json!({"ref": "feature/x", "ref_type": "branch", "repository": repository})), Ok(garden(Some("feature/x")))); + assert_eq!(scope("delete", &json!({"ref": "v1", "ref_type": "tag", "repository": repository})), Ok(garden(None))); + for event in ["check_run", "check_suite", "workflow_run"] { + assert_eq!(scope(event, &json!({"action": "created", "repository": repository})), Ok(None), "{event}"); + assert_eq!(scope(event, &json!({"action": "completed", "repository": repository})), Ok(garden(None)), "{event}"); + } + for event in ["ping", "star", "fork", "deployment", "package"] { + assert_eq!(scope(event, &json!({"repository": repository})), Ok(None), "{event}"); + } + assert!(scope("issues", &json!({"action": "opened"})).is_err()); + assert!(scope("issues", &json!({"repository": {"full_name": "garden"}})).is_err()); + assert_eq!(event_label("check_run"), "check_run"); + assert_eq!(event_label("ping"), "ping"); + assert_eq!(event_label("attacker-chosen"), "other"); + } + + #[test] + fn repository_observers_match_in_any_case_and_ref_observers_only_their_branch() { + let main = Scope { repository: "acme/garden".into(), branch: Some("main".into()) }; + assert!(refreshes(&spec("github.repository", "Acme/Garden"), &main)); + assert!(!refreshes(&spec("github.repository", "acme/greenhouse"), &main)); + assert!(refreshes(&spec("github.ref", "ACME/garden@main"), &main)); + assert!(!refreshes(&spec("github.ref", "acme/garden@Main"), &main)); + assert!(!refreshes(&spec("github.ref", "acme/garden@release"), &main)); + let unbranched = Scope { branch: None, ..main }; + assert!(!refreshes(&spec("github.ref", "acme/garden@main"), &unbranched)); + assert!(!refreshes(&spec("command.json", "acme/garden"), &unbranched)); + } + + #[test] + fn the_attempt_binds_the_delivery_and_the_observer() { + let attempt_one = attempt("delivery-1", "observer/github/acme/garden"); + assert_eq!(attempt_one, format!( + "github-webhook:{}", + hex::encode(Sha256::digest(b"fractal.github-webhook.v1\0delivery-1\0observer/github/acme/garden")), + )); + assert_ne!(attempt_one, attempt("delivery-2", "observer/github/acme/garden")); + assert_ne!(attempt_one, attempt("delivery-1", "observer/garden-ref")); + } + + #[test] + fn doctor_reports_the_receiver_once_it_listens() { + assert!(doctor_line(0).is_none(), "a daemon without a receiver shows no line"); + listening("127.0.0.1:41871"); + record(Outcome::Unauthorized, 1_000); + let line = doctor_line(2_000).unwrap(); + assert_eq!((line.name.as_str(), line.status.as_str()), ("github-webhook", "warn")); + assert!(line.message.contains("http://127.0.0.1:41871/fractal/github-webhook; no verified delivery since start"), "{}", line.message); + record(Outcome::Refreshed, 1_762_000_000_000); + let line = doctor_line(1_762_000_030_000).unwrap(); + assert_eq!(line.status, "pass"); + assert!(line.message.contains("last verified delivery 2025-11-01T12:26:40Z (30s ago)"), "{}", line.message); + assert!(line.message.ends_with("refreshed 1, ignored 0, unauthorized 1, unavailable 0, invalid 0"), "{}", line.message); + } +} diff --git a/crates/fractal-kernel/src/lib.rs b/crates/fractal-kernel/src/lib.rs index 0673fba7e1c..f6fc6c9028c 100644 --- a/crates/fractal-kernel/src/lib.rs +++ b/crates/fractal-kernel/src/lib.rs @@ -42,3 +42,4 @@ pub mod terminal_binding; /// What a mission run waits on, and the fleet's blocked runs by cause. pub mod waiting_on; pub mod github_admission; +pub mod github_webhook; diff --git a/crates/fractal-missions/src/store/missions.rs b/crates/fractal-missions/src/store/missions.rs index 2db3483b464..45d380b9020 100644 --- a/crates/fractal-missions/src/store/missions.rs +++ b/crates/fractal-missions/src/store/missions.rs @@ -7147,32 +7147,7 @@ impl MissionsStore for Store { } fn pending_observer_refresh_attempt(&self, observer: &str) -> Result> { - let connection = self.readers.get(); - // The observations are listed once, not scanned for each request; see - // `pending_subscription_mission_requests`. - connection - .query_row( - /* sqlite-large-value: allow-for-now lv-h9b4924d4a878 owner=fractal_missions::store::missions TODO=trace-statement-and-value-representation */ &canonical_sql( - "SELECT json_extract(request.body, '$.fields.attempt') - FROM claims request - WHERE request.subject=?1 - AND request.kind='observer.refresh-requested' - AND ( - json_extract(request.body, '$.fields.attempt') IS NULL - OR json_extract(request.body, '$.fields.attempt') NOT IN ( - SELECT json_extract(result.body, '$.fields.attempt') FROM claims result - WHERE result.subject=?1 - AND result.kind IN ('observer.observed', 'observer.state') - AND json_extract(result.body, '$.fields.attempt') IS NOT NULL - ) - ) - ORDER BY CANONICAL_ASC(request) - LIMIT 1", - ), - [observer], - |row| row.get(0), - ) - .optional() + fractal_store::store::observer_refresh::pending_attempt(&self.readers.get(), observer) .map_err(Into::into) } diff --git a/crates/fractal-reconcile/src/reconcile.rs b/crates/fractal-reconcile/src/reconcile.rs index 52bf4ce18eb..9c35e064de3 100644 --- a/crates/fractal-reconcile/src/reconcile.rs +++ b/crates/fractal-reconcile/src/reconcile.rs @@ -17546,6 +17546,12 @@ impl Reconciler { let _look = look_lock.clone().lock_owned().await; let look_deadline = tokio::time::Instant::now() + look_limit; if !armed.lock().unwrap_or_else(PoisonError::into_inner).contains(&operation) { return; } + // The arm keeps the oldest pending attempt as its name, so a delivery while it + // looks does not retire it. Right before the reads, a webhook refresh takes the + // newest pending webhook refresh from the same member, whose answer answers them all. + let refresh_attempt = refresh_attempt.as_ref().map(|attempt| { + store.coalesced_refresh_attempt(&observer_subject, attempt).unwrap_or_else(|_| attempt.clone()) + }); if let Some(latest) = cursors.lock().unwrap_or_else(PoisonError::into_inner).get(&deadline_key).cloned() { kept_cursor = Some(latest.clone()); cursor = latest; @@ -17576,6 +17582,10 @@ impl Reconciler { kept_cursor = Some(pending.clone()); cursor = pending; } + // Refresh claims replicate, so a webhook delivery any member received covers the + // repository here, where it is polled. A covered repository's polls are a backstop. + let webhook_covered = spec.provider == "github.repository" + && store.github_webhook_covered(&observer_subject, now_ms()).unwrap_or(false); let request = ObservationRequest { provider: spec.provider.clone(), locator: spec.locator.clone(), @@ -17584,6 +17594,7 @@ impl Reconciler { previous_facts: previous_facts.clone(), every_ms: spec.every_ms, refresh: refresh_attempt.is_some(), + webhook_covered, }; // Reload under the look lock: a retired arm may have completed while we waited. let refreshed = github_plugin.observer_freshness(&observer_subject, now_ms()); @@ -17683,6 +17694,7 @@ impl Reconciler { if standing { let interval = match standing_interval { _ if outcome.baseline || !outcome.changed_fields.is_empty() => 180_000, + _ if webhook_covered && !continuing => 600_000, None => 180_000, Some(interval) if continuing => interval, Some(180_000) => 360_000, @@ -19511,6 +19523,7 @@ async fn refill_mirror( previous_facts: None, every_ms: spec.every_ms, refresh: false, + webhook_covered: false, }; let filled = async { let observation = crate::resource::spend_as(observer.to_owned(), provider.observe(request)).await?; @@ -39846,6 +39859,7 @@ observer "repo" { calls: Arc, changed: Arc, continuation: Arc, + covered: Arc, } impl ResourceProvider for AdaptiveProvider { fn observe(&self, request: ObservationRequest) -> std::pin::Pin>(); + let desired = store.desired_subjects().unwrap(); + reconciler.reconcile_resource_observers(&desired, &[]).unwrap(); + settle_observer_tasks().await; + requests.try_recv().unwrap().1.send(observation("a")).unwrap(); + settle_observer_tasks().await; + + refresh("d-1"); + reconciler.reconcile_resource_observers(&desired, &[]).unwrap(); + settle_observer_tasks().await; + let (_, look) = requests.try_recv().unwrap(); + refresh("d-2"); + refresh("d-3"); + reconciler.reconcile_resource_observers(&desired, &[]).unwrap(); + settle_observer_tasks().await; + assert_eq!(reconciler.armed_observers.lock().unwrap_or_else(PoisonError::into_inner).len(), 1); + look.send(observation("b")).unwrap(); + settle_observer_tasks().await; + assert_eq!(answered(), [attempt("d-1")], "the running look kept its answer"); + assert!(requests.try_recv().is_err()); + + reconciler.reconcile_resource_observers(&desired, &[]).unwrap(); + settle_observer_tasks().await; + requests.try_recv().unwrap().1.send(observation("c")).unwrap(); + settle_observer_tasks().await; + assert_eq!(answered(), [attempt("d-1"), attempt("d-3")]); + assert!(store.pending_observer_refresh_attempt("observer/ref").unwrap().is_none()); + reconciler.reconcile_resource_observers(&desired, &[]).unwrap(); + settle_observer_tasks().await; + assert!(requests.try_recv().is_err(), "d-2 was answered with d-3"); +} + const BACKOFF_REF_OBSERVER: &str = "version 2\nresource \"ref\" { kind \"vcs.ref\" }\nobserver \"ref\" { resource \"resource/ref\"; provider \"github.ref\"; locator \"acme/garden@main\"; field \"head\" }"; /// The first error receipt wakes once; identical repeats write and wake nothing, so the arm diff --git a/crates/fractal-store/src/store.rs b/crates/fractal-store/src/store.rs index bf9aa84742e..2ad701fb5fc 100644 --- a/crates/fractal-store/src/store.rs +++ b/crates/fractal-store/src/store.rs @@ -26,6 +26,7 @@ pub mod seats; pub mod helper_launchers; pub mod trace_principal; pub mod workspace_observations; +pub mod observer_refresh; #[cfg(test)] pub mod accountable_tests; pub mod custom; @@ -8311,31 +8312,9 @@ fn request_declared_resource_refresh_tx( operation: &ResourceRefreshOperation, batch_id: &str, ) -> Result, FractalError> { - let desired = { - let mut statement = transaction - .prepare("SELECT subject, body, revision FROM desired WHERE kind='observer'") - .map_err(internal)?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }) - .map_err(internal)? - .collect::, _>>() - .map_err(internal)? - }; - let observers = desired + let observers = observer_refresh::active_observers_tx(transaction)? .into_iter() - .filter_map(|(subject, body, revision)| { - serde_json::from_str::(&body) - .ok() - .and_then(|desired| crate::graph::observer_spec(&desired)) - .filter(|spec| !spec.stopped && spec.resource == operation.resource) - .map(|_| (subject, revision)) - }) + .filter(|(_, _, spec)| spec.resource == operation.resource) .collect::>(); if observers.is_empty() { return Err(FractalError::new( @@ -8344,25 +8323,18 @@ fn request_declared_resource_refresh_tx( )); } let mut claims = Vec::new(); - for (observer, revision) in observers { + for (observer, revision, _) in observers { let attempt = hex::encode(Sha256::digest(format!( "st3.resource-refresh.v1\0{}\0{}\0{}", operation.resource, operation.id, observer ))); - let claim = append_claim_tx( + let claim = observer_refresh::append_observer_refresh_tx( transaction, origin, - &observer, - "observer.refresh-requested", - None, - &json!({"fields": { - "revision": revision, - "attempt": attempt, - }}), - &[], + (&observer, &revision), + &attempt, Some(batch_id), - ) - .map_err(internal)?; + )?; claims.push(claim.id); } Ok(claims) diff --git a/crates/fractal-store/src/store/observer_refresh.rs b/crates/fractal-store/src/store/observer_refresh.rs new file mode 100644 index 00000000000..74b240ad8fb --- /dev/null +++ b/crates/fractal-store/src/store/observer_refresh.rs @@ -0,0 +1,327 @@ +//! `observer.refresh-requested`: a declared `refresh` or a verified GitHub webhook delivery asks +//! an observer to poll now. The claim replicates, so the member that polls the observer sees it +//! wherever it was written. +use super::*; +use crate::github_webhook; +use crate::model::ObserverSpec; + +/// The active observers this store declares, with their selected revision and spec. +pub(super) fn active_observers_tx( + transaction: &Transaction<'_>, +) -> Result, FractalError> { + let mut statement = transaction + .prepare("SELECT subject, body, revision FROM desired WHERE kind='observer'") + .map_err(internal)?; + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(internal)? + .collect::, _>>() + .map_err(internal)?; + Ok(rows + .into_iter() + .filter_map(|(subject, body, revision)| { + serde_json::from_str::(&body) + .ok() + .and_then(|desired| crate::graph::observer_spec(&desired)) + .filter(|spec| !spec.stopped) + .map(|spec| (subject, revision, spec)) + }) + .collect()) +} + +/// Ask `observer` at `revision` to poll once for `attempt`. +pub(super) fn append_observer_refresh_tx( + transaction: &Transaction<'_>, + origin: &str, + (observer, revision): (&str, &str), + attempt: &str, + batch_id: Option<&str>, +) -> Result { + let fields = json!({"revision": revision, "attempt": attempt}); + append_claim_tx( + transaction, + origin, + observer, + "observer.refresh-requested", + None, + &json!({"fields": fields}), + &[], + batch_id, + ) + .map_err(internal) +} + +/// The oldest refresh attempt `observer` has not yet answered, or `None` when none is pending. A +/// request is pending until an observation or a state carries its attempt. It names the poll that +/// answers it, so it stays the same while that poll runs. +pub fn pending_attempt(connection: &Connection, observer: &str) -> rusqlite::Result> { + Ok(pending_requests(connection, observer)?.into_iter().next().map(|(attempt, _)| attempt)) +} + +/// The attempt a poll taken for pending `attempt` answers, read right before the poll's reads. +/// For a webhook refresh it is the newest pending webhook refresh from the member that wrote +/// `attempt`; answering it answers every earlier webhook refresh from that member, since each +/// was written after the change it announced and before those reads. Any other attempt answers +/// itself. +pub fn coalesced_attempt(connection: &Connection, observer: &str, attempt: &str) -> rusqlite::Result { + if !attempt.starts_with(github_webhook::ATTEMPT_PREFIX) { + return Ok(attempt.to_owned()); + } + let pending = pending_requests(connection, observer)?; + let Some((_, origin)) = pending.iter().find(|(pending, _)| pending == attempt) else { + return Ok(attempt.to_owned()); + }; + Ok(pending + .iter() + .rev() + .find(|(newest, from)| from == origin && newest.starts_with(github_webhook::ATTEMPT_PREFIX)) + .map_or_else(|| attempt.to_owned(), |(newest, _)| newest.clone())) +} + +/// The pending refresh attempts of `observer` with the member that wrote each, oldest first. A +/// webhook refresh is no longer pending once a later webhook refresh from the same member is +/// answered. +fn pending_requests(connection: &Connection, observer: &str) -> rusqlite::Result)>> { + // The observations are listed once, not scanned for each request; see + // `pending_subscription_mission_requests`. + let mut statement = connection.prepare( + /* sqlite-large-value: allow-for-now lv-h1e605cea76d4 owner=fractal_store::store::observer_refresh TODO=trace-statement-and-value-representation */ &canonical_sql(&format!( + "WITH answered(attempt) AS ( + SELECT json_extract(result.body, '$.fields.attempt') FROM claims result + WHERE result.subject=?1 + AND result.kind IN ('observer.observed', 'observer.state') + AND json_extract(result.body, '$.fields.attempt') IS NOT NULL + ) + SELECT json_extract(request.body, '$.fields.attempt'), + (SELECT origin FROM batches WHERE id=request.batch_id) + FROM claims request + WHERE request.subject=?1 + AND request.kind='observer.refresh-requested' + AND ( + json_extract(request.body, '$.fields.attempt') IS NULL + OR json_extract(request.body, '$.fields.attempt') NOT IN (SELECT attempt FROM answered) + ) + AND NOT ( + json_extract(request.body, '$.fields.attempt') LIKE ?2 + AND EXISTS ( + SELECT 1 FROM claims later + WHERE later.subject=?1 + AND later.kind='observer.refresh-requested' + AND json_extract(later.body, '$.fields.attempt') LIKE ?2 + AND json_extract(later.body, '$.fields.attempt') IN (SELECT attempt FROM answered) + AND (SELECT origin FROM batches WHERE id=later.batch_id) + IS (SELECT origin FROM batches WHERE id=request.batch_id) + AND {} + ) + ) + ORDER BY CANONICAL_ASC(request)", + canonical::after_sql("later", "request"), + )), + )?; + statement + .query_map(params![observer, format!("{}%", github_webhook::ATTEMPT_PREFIX)], |row| { + Ok((row.get(0)?, row.get(1)?)) + })? + .collect() +} + +/// Whether `observer` already has a refresh request for `attempt`. +fn requested(connection: &Connection, observer: &str, attempt: &str) -> rusqlite::Result { + connection.query_row( + "SELECT EXISTS(SELECT 1 FROM claims + WHERE subject=?1 AND kind='observer.refresh-requested' + AND json_extract(body, '$.fields.attempt')=?2)", + params![observer, attempt], + |row| row.get(0), + ) +} + +impl Store { + /// Ask every active observer that a verified delivery `delivery` concerns to poll now, and + /// return how many it asked. A redelivery asks nothing more. Deliveries that arrive while a + /// poll runs each add a request, which the next poll answers together (see + /// [`coalesced_attempt`]). + pub fn request_github_webhook_refresh( + &self, + delivery: &str, + scope: &github_webhook::Scope, + ) -> Result { + let mut connection = self.connection.write(); + let transaction = connection.transaction().map_err(internal)?; + let mut refreshed = 0; + for (observer, revision, spec) in active_observers_tx(&transaction)? { + let attempt = github_webhook::attempt(delivery, &observer); + if !github_webhook::refreshes(&spec, scope) + || requested(&transaction, &observer, &attempt).map_err(internal)? + { + continue; + } + append_observer_refresh_tx(&transaction, &self.origin, (&observer, &revision), &attempt, None)?; + refreshed += 1; + } + transaction.commit().map_err(internal)?; + Ok(refreshed) + } + + /// [`coalesced_attempt`] on this store's readers. + pub fn coalesced_refresh_attempt(&self, observer: &str, attempt: &str) -> Result { + coalesced_attempt(&self.readers.get(), observer, attempt).map_err(Into::into) + } + + /// Whether a webhook delivery asked `observer` to refresh within the coverage window, on + /// whichever member it arrived; its attempt carries [`github_webhook::ATTEMPT_PREFIX`]. While + /// it is covered, its polls are a slow backstop. + pub fn github_webhook_covered(&self, observer: &str, now_unix_ms: u128) -> Result { + let since = now_unix_ms.saturating_sub(github_webhook::COVERAGE_MS).to_string(); + self.readers + .get() + .query_row( + "SELECT EXISTS(SELECT 1 FROM claims + WHERE subject=?1 AND kind='observer.refresh-requested' + AND (length(accepted_at_unix_ms) > length(?2) + OR (length(accepted_at_unix_ms) = length(?2) AND accepted_at_unix_ms >= ?2)) + AND json_extract(body, '$.fields.attempt') LIKE ?3)", + params![observer, since, format!("{}%", github_webhook::ATTEMPT_PREFIX)], + |row| row.get(0), + ) + .map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SOURCE: &str = r#"version 2 +resource "garden" { kind "vcs.repository" } +observer "garden" { resource "resource/garden"; provider "github.repository"; locator "Acme/Garden"; field "pull_requests" } +observer "greenhouse" { resource "resource/garden"; provider "github.repository"; locator "acme/greenhouse"; field "pull_requests" } +resource "garden-main" { kind "vcs.ref" } +observer "garden-main" { resource "resource/garden-main"; provider "github.ref"; locator "acme/garden@main"; field "head" } +observer "garden-release" { resource "resource/garden-main"; provider "github.ref"; locator "acme/garden@release"; field "head" } +"#; + + fn store() -> Store { + let store = Store::open_memory("node").unwrap(); + let intent = crate::graph::parse_internal_intent(SOURCE, "node").unwrap(); + let planned = store + .mission(&intent, IntentInput { kdl: SOURCE.into(), source_name: None }) + .unwrap(); + store.apply(&intent, &planned.subject_tokens, "webhook-observers").unwrap(); + store + } + + fn requests(store: &Store, observer: &str) -> Vec { + store + .graph + .claims_for(observer, Some("observer.refresh-requested")) + .unwrap() + .into_iter() + .map(|claim| claim.body["fields"].clone()) + .collect() + } + + #[test] + fn a_delivery_refreshes_the_repository_and_moved_branch_observers_with_a_bound_attempt() { + let store = store(); + let scope = github_webhook::Scope { repository: "acme/garden".into(), branch: Some("main".into()) }; + assert_eq!(store.request_github_webhook_refresh("delivery-1", &scope).unwrap(), 2); + for observer in ["observer/garden", "observer/garden-main"] { + let revision = store.selected_desired_revision(observer).unwrap().unwrap(); + assert_eq!(requests(&store, observer), [json!({ + "attempt": github_webhook::attempt("delivery-1", observer), + "revision": revision, + })]); + assert_eq!( + pending_attempt(&store.readers.get(), observer).unwrap(), + Some(github_webhook::attempt("delivery-1", observer)), + ); + assert!(store.github_webhook_covered(observer, now_ms()).unwrap()); + } + for observer in ["observer/greenhouse", "observer/garden-release"] { + assert!(requests(&store, observer).is_empty(), "{observer}"); + assert!(!store.github_webhook_covered(observer, now_ms()).unwrap()); + } + } + + #[test] + fn deliveries_during_a_poll_are_answered_together_by_the_next_one() { + let store = store(); + let scope = github_webhook::Scope { repository: "acme/garden".into(), branch: None }; + let attempt = |delivery: &str| github_webhook::attempt(delivery, "observer/garden"); + let pending = || pending_attempt(&store.readers.get(), "observer/garden").unwrap(); + let revision = store.selected_desired_revision("observer/garden").unwrap().unwrap(); + let answer = |attempt: String| { + store + .append_claim(&ClaimInput { + subject: "observer/garden".into(), + kind: "observer.state".into(), + actor: None, + fields: BTreeMap::from([ + ("attempt".into(), json!(attempt)), + ("revision".into(), json!(revision)), + ("state".into(), json!("healthy")), + ]), + evidence: Vec::new(), + expected_subject: None, + idempotency_key: None, + }) + .unwrap(); + }; + let coalesced = |pending: String| store.coalesced_refresh_attempt("observer/garden", &pending).unwrap(); + assert_eq!(store.request_github_webhook_refresh("delivery-1", &scope).unwrap(), 1); + assert_eq!(pending(), Some(attempt("delivery-1"))); + assert_eq!(coalesced(attempt("delivery-1")), attempt("delivery-1")); + // The poller has taken delivery-1's refresh and may have read GitHub already, so each + // delivery while it polls adds a request; a redelivery adds none. The running poll keeps + // its name. + assert_eq!(store.request_github_webhook_refresh("delivery-2", &scope).unwrap(), 1); + assert_eq!(store.request_github_webhook_refresh("delivery-3", &scope).unwrap(), 1); + assert_eq!(store.request_github_webhook_refresh("delivery-1", &scope).unwrap(), 0); + assert_eq!(requests(&store, "observer/garden").len(), 3); + assert_eq!(pending(), Some(attempt("delivery-1"))); + answer(attempt("delivery-1")); + // The next poll is named for the oldest and answers the newest, which answers both. + assert_eq!(pending(), Some(attempt("delivery-2"))); + assert_eq!(coalesced(attempt("delivery-2")), attempt("delivery-3")); + answer(attempt("delivery-3")); + assert_eq!(pending(), None); + assert_eq!(store.request_github_webhook_refresh("delivery-3", &scope).unwrap(), 0); + assert_eq!(pending(), None); + + // A declared refresh keeps its place and answers only itself. + let mut connection = store.connection.write(); + let transaction = connection.transaction().unwrap(); + append_observer_refresh_tx(&transaction, "node", ("observer/garden", &revision), "declared", None).unwrap(); + transaction.commit().unwrap(); + drop(connection); + assert_eq!(store.request_github_webhook_refresh("delivery-4", &scope).unwrap(), 1); + assert_eq!(pending(), Some("declared".into())); + assert_eq!(coalesced("declared".into()), "declared"); + answer("declared".into()); + assert_eq!(pending(), Some(attempt("delivery-4"))); + } + + #[test] + fn coverage_ends_after_the_window_and_ignores_declared_refreshes() { + let store = store(); + let revision = store.selected_desired_revision("observer/greenhouse").unwrap().unwrap(); + let mut connection = store.connection.write(); + let transaction = connection.transaction().unwrap(); + append_observer_refresh_tx(&transaction, "node", ("observer/greenhouse", &revision), "declared", None).unwrap(); + transaction.commit().unwrap(); + drop(connection); + assert!(!store.github_webhook_covered("observer/greenhouse", now_ms()).unwrap()); + let scope = github_webhook::Scope { repository: "acme/garden".into(), branch: None }; + store.request_github_webhook_refresh("delivery-1", &scope).unwrap(); + let later = now_ms() + github_webhook::COVERAGE_MS + 1_000; + assert!(store.github_webhook_covered("observer/garden", now_ms()).unwrap()); + assert!(!store.github_webhook_covered("observer/garden", later).unwrap()); + } +} diff --git a/crates/fractal-telemetry/src/otel.rs b/crates/fractal-telemetry/src/otel.rs index 6f84e18caeb..86f0b5c2569 100644 --- a/crates/fractal-telemetry/src/otel.rs +++ b/crates/fractal-telemetry/src/otel.rs @@ -921,6 +921,27 @@ fn record_github_response_metric( ]); } +static GITHUB_WEBHOOK_DELIVERIES: std::sync::LazyLock> = + std::sync::LazyLock::new(|| { + opentelemetry::global::meter("st3.github") + .u64_counter("st.github.webhook.deliveries") + .with_description("GitHub webhook deliveries this daemon received, by event and outcome") + .with_unit("{delivery}") + .build() + }); + +/// One webhook delivery. `event` is a bounded label (`other` for events the receiver does not +/// know); `outcome` is `refreshed`, `ignored`, `unauthorized`, `unavailable` or `invalid`. +pub fn github_webhook_delivery(event: &'static str, outcome: &'static str) { + if !metrics_enabled() { + return; + } + GITHUB_WEBHOOK_DELIVERIES.add(1, &[ + KeyValue::new("github.event", event), + KeyValue::new("outcome", outcome), + ]); +} + #[cfg(test)] mod github_response_metrics { use super::*; diff --git a/crates/fractal/src/main.rs b/crates/fractal/src/main.rs index 96cf1e1b779..f4e0de93aed 100644 --- a/crates/fractal/src/main.rs +++ b/crates/fractal/src/main.rs @@ -7971,6 +7971,9 @@ async fn run_up(args: UpArgs) -> Result<()> { &args.web_listen, web_dist, &state.state_dir, config.person.clone(), ).await?; let web_status = axum::Extension(web.status.clone()); + let github_webhook = fractal_daemon::github_webhook::bind( + &config.github, store.clone(), notify.clone(), event_notify.clone(), + ).await?; let web_state = state.clone(); let receipts = state.assistant_receipts.clone(); // A stop signal first commits the assistant read receipts of reads that already returned, @@ -8002,6 +8005,12 @@ async fn run_up(args: UpArgs) -> Result<()> { } result }, + async { + match github_webhook { + Some(receiver) => receiver.serve().await, + None => std::future::pending().await, + } + }, ); Err(served) }; diff --git a/crates/fractal/tests/daemon_environment.rs b/crates/fractal/tests/daemon_environment.rs index 078e836baca..11170afbe3b 100644 --- a/crates/fractal/tests/daemon_environment.rs +++ b/crates/fractal/tests/daemon_environment.rs @@ -391,7 +391,7 @@ fn a_profile_configured_gate_uses_the_gateway_and_node_identity_without_local_cr let config = fractal_daemon::config::Config { node: "fixture-node".into(), person: Some("person/fixture".into()), state_dir: root.path().join("node-state"), - github: fractal_daemon::config::GithubConfig { sekrets_profile: Some("owner/daemon-gh".into()), token_file: None, observer_requests_per_hour: None, app_token_root: None }, + github: fractal_daemon::config::GithubConfig { sekrets_profile: Some("owner/daemon-gh".into()), token_file: None, observer_requests_per_hour: None, app_token_root: None, webhook_listen: None, webhook_secret_file: None }, ..Default::default() }; let key = smallclaims::fleet::MemberKey::load_or_create( diff --git a/docs/st3/github-observer-admission.md b/docs/st3/github-observer-admission.md index 926dc585c33..30c7417ef46 100644 --- a/docs/st3/github-observer-admission.md +++ b/docs/st3/github-observer-admission.md @@ -333,3 +333,75 @@ metrics. A missing observer refuses the request rather than silently dropping accounting. Resource wrappers do not recount the final response. Artifact URL invalidation clears every stable principal-qualified cache entry for that exact URL, retaining unrelated URLs. + +## Webhook delivery + +GitHub webhooks push freshness; polling remains the backstop. A daemon runs the +receiver only when both keys are set: + +```toml +[github] +webhook_listen = "127.0.0.1:41871" +webhook_secret_file = "/absolute/owner-only/secret" +``` + +`webhook_listen` must be a loopback address with a nonzero port, and +`webhook_secret_file` must be absolute; one without the other is a configuration +error. The receiver serves exactly `POST /fractal/github-webhook` and answers +everything else 404. A public proxy forwards that path verbatim; the daemon never +listens publicly. The webhook uses content type `application/json`. + +**Security.** Every delivery must carry `X-GitHub-Event`, `X-GitHub-Delivery` +and `X-Hub-Signature-256`, the HMAC-SHA256 of the raw body under the shared +secret, compared in constant time. A missing or wrong signature is 401. The +secret file is read again for every delivery, under the same custody rules as +token files (an owner-owned regular file with no group or other permissions, +opened without following links), so it may appear or rotate after the daemon +starts. While it is absent, unreadable, empty or too permissive, every delivery +is 503: an unsigned delivery is never accepted. The secret's custody, the GitHub +hook and the proxy are external to Fractal. Bodies over 5 MiB are 413. + +**Events.** `ping` is 200. `pull_request`, `pull_request_review`, +`pull_request_review_comment`, `pull_request_review_thread`, `issue_comment`, +`issues`, `push`, `status`, `merge_group`, `create`, `delete`, and +`check_suite`, `check_run` and `workflow_run` once `completed`, refresh +observers; every other event or action is 204. The receiver reads only +`repository.full_name`, plus `ref` for `push` and `ref`/`ref_type` for +`create`/`delete`; a refreshing event without a repository is 400. + +**Refresh.** For each active observer whose provider is `github.repository` with a +locator equal to the repository in any case, and, for a `push`, `create` or +`delete` of branch `B`, each `github.ref` observer of `OWNER/REPO@B`, the +receiver writes an `observer.refresh-requested` claim with attempt +`"github-webhook:" + hex(sha256("fractal.github-webhook.v1\0" + delivery + "\0" + observer))`, +the same claim a declared `refresh` writes; the attempt prefix alone marks it as +webhook-requested, since declared attempts are bare hex digests. A redelivery +writes nothing more, since its attempt is already requested. The answer is 202 +with `{"refreshed": n}`. Refresh claims replicate, so the member that polls the +observer acts on a delivery any member received; a refresh poll bypasses the +GraphQL floor, as a declared refresh does. + +**Coalescing.** The poller decides, on its own replica, so a delivery that +arrives while a poll runs is never lost to replication lag. Requests are taken +oldest first, and a poll is named for the request it took, so later requests +never retire a running poll. Under the observer's look lock, right before its +reads, a poll taken for a webhook refresh answers the newest pending webhook +refresh written by the same member instead; that answer also answers every +earlier webhook refresh from that member, since each was written after the +change it announced and before those reads. A burst of deliveries during one +poll thus costs one more poll, not one per delivery. + +**Backstop cadence.** A repository observer with a webhook refresh claim +accepted within the last 30 minutes is webhook-covered. The member that polls it +reads that from its own replicated claims, not from the receiving host's memory. +While covered, the repository's GraphQL floor is the 15-minute settled interval +instead of 5 minutes, and a quiet standing-observer poll goes straight to the +10-minute ceiling instead of stepping through 6 minutes; a poll that sees a +change still returns to the 3-minute active cadence. Without coverage nothing +changes. The provider learns coverage through `ObservationRequest.webhook_covered`. + +**Observability.** `st.github.webhook.deliveries` is a monotonic OpenTelemetry +counter labeled `github.event` (the event name, or `other` for unknown events) +and `outcome` (`refreshed`, `ignored`, `unauthorized`, `unavailable`, `invalid`). +Doctor shows a `github-webhook` line on a daemon that runs the receiver: its +address, the last verified delivery time, and counts by outcome since start. diff --git a/docs/st3/large-value-sites.json b/docs/st3/large-value-sites.json index 5ee34d656cc..1bdd81801ed 100644 --- a/docs/st3/large-value-sites.json +++ b/docs/st3/large-value-sites.json @@ -1305,6 +1305,7 @@ {"id": "lv-h1e27c5bc693c", "path": "crates/fractal-store/src/store/agent_view.rs", "kind": "sql-template", "fingerprint": "384dc72b149df9e50dae97703444c8c15d47137c61626dca9307ec02a0dbcf65", "owner": "fractal_store::store::agent_view", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 429, "excerpt": "CREATE TABLE IF NOT EXISTS local_agent_harness_rows(subject TEXT PRIMARY KEY, observed TEXT);"}, {"id": "lv-h1e30d65a948a", "path": "crates/fractal-store/src/store/run_eta_tests.rs", "kind": "sql-template", "fingerprint": "c4a744d197d165cc930eb95b6a1f721854c8cd02460bd0f1dd96db71497bc7ed", "owner": "fractal_store::store::run_eta_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 263, "excerpt": "UPDATE mission_runs SET phase='terminal',status='completed' WHERE id=?1"}, {"id": "lv-h1e3e5d5be065", "path": "crates/fractal-seats/src/store/person_work/reconcile_candidates_tests.rs", "kind": "sql-template", "fingerprint": "3c6aae4cccf6cfd4f4bcde133ebf6c8638b09a6f161aea9cfdbfc02ec95686f3", "owner": "fractal_seats::store::person_work::reconcile_candidates_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 46, "excerpt": "INSERT INTO step_runs(subject,run_id,generation_id,step_path,definition_hash,status, attempt,assignee,available_to,agentless,title,goals,created_at_unix_ms,updated_at_unix_ms,constraints) SELECT ?1,ru"}, + {"id": "lv-h1e605cea76d4", "path": "crates/fractal-store/src/store/observer_refresh.rs", "kind": "unresolved-sql-sink", "fingerprint": "79a55996658a52026e07f9609b757cd4bef7465830b0777bf351f8920a3d5536", "owner": "fractal_store::store::observer_refresh", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 88, "excerpt": "& canonical_sql ( & format ! ( \"WITH answered(attempt) AS ( SELECT json_extract(result.body, '$.fields.attempt') FROM claims result WHERE result.subject=?1 AND result.kind IN ('observer.observed', 'ob"}, {"id": "lv-h1e6ea42f1255", "path": "crates/fractal-store/src/store/message_send_tests.rs", "kind": "sql-template", "fingerprint": "97bd9b2d9b5d8ecfd294950ad4fde6ba2af020fc5cbce1d3d053f98aacef9736", "owner": "fractal_store::store::message_send_tests", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 41, "excerpt": "INSERT INTO replica_records(record_ref,writer,sequence,envelope_hash,position, raw,state,claim_id,updated_at_unix_ms) VALUES ('record/invented-repair','invented-peer', 1,'invented-envelope',0,X'','rep"}, {"id": "lv-h1e9a7dd429b3", "path": "crates/fractal-store/src/store/message_marks_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "4ec7c53222c8a758c722e2111541035ce700d5ae7bd0898c5f1b1a743e6450fd", "owner": "fractal_store::store::message_marks_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 83, "excerpt": "sql"}, {"id": "lv-h1eca2dcd8e6e", "path": "crates/fractal-store/src/store/attention_snapshot.rs", "kind": "unresolved-sql-sink", "fingerprint": "1a89efc152789a3f0ef56ff275fbdbf2285f5210f812c0990935267d14a5b9f9", "owner": "fractal_store::store::attention_snapshot", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1910, "excerpt": "params ! [ seat , i64 : : MAX ] , | row | row . get : : < _ , String > ( 0 )"}, @@ -3443,7 +3444,6 @@ {"id": "lv-h9b34dc483f39", "path": "crates/fractal-seats/src/store/resource_preparation/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "dc372daedd324b47f1261987900c1a08eff3ccdf253bfcf666d0ebecf17142ad", "owner": "fractal_seats::store::resource_preparation::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 156, "excerpt": "& store , | _ | ( )"}, {"id": "lv-h9b34dd42fcf7", "path": "crates/fractal-store/src/store/observer_actuals_tests.rs", "kind": "sql-template", "fingerprint": "8501af8ee20076569f886d3675507790807a8bcb152b1696b1d633effa58beae", "owner": "fractal_store::store::observer_actuals_tests", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 160, "excerpt": "DROP TABLE projection_digest_repaired_claims; CREATE TABLE projection_digest_repaired_claims(id TEXT PRIMARY KEY);"}, {"id": "lv-h9b3b2893e731", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "8edb630b01a1f78d4abc5ee08b714ce7a8578b35d2269aec45684bf19ac1e4fc", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 6783, "excerpt": "& query"}, - {"id": "lv-h9b4924d4a878", "path": "crates/fractal-missions/src/store/missions.rs", "kind": "unresolved-sql-sink", "fingerprint": "f72cc7aca17eb71574b8e970455c1013b67d78da66a206519950483606343b19", "owner": "fractal_missions::store::missions", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 6898, "excerpt": "& canonical_sql ( \"SELECT json_extract(request.body, '$.fields.attempt') FROM claims request WHERE request.subject=?1 AND request.kind='observer.refresh-requested' AND ( json_extract(request.body, '$."}, {"id": "lv-h9b594f38a048", "path": "crates/smallclaims/src/store/schema_install_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "9ff21cc7560331f62097a428335acc6ef7c7c1320dc75b2e3399b991e9c6fceb", "owner": "smallclaims::store::schema_install_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 65, "excerpt": "FRESH_SCHEMA"}, {"id": "lv-h9b93bbbdbb39", "path": "crates/fractal-missions/src/store/lanes.rs", "kind": "unresolved-sql-sink", "fingerprint": "ff9e4ca5b7a4370a3a2d73456bca304f1badd8e2dfe36651c8e8ecadce0e00eb", "owner": "fractal_missions::store::lanes", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 36, "excerpt": "if run . is_some ( ) { \"SELECT subject, kind, body, member, owner_run, owner_generation, owner_step FROM desired WHERE owner_run=?1 AND kind='lane' ORDER BY subject\" } else { \"SELECT subject, kind, bo"}, {"id": "lv-h9bc0d44f925c", "path": "crates/smallclaims/src/store/saved_inventory_tests.rs", "kind": "sql-template", "fingerprint": "b3146e6453d128849b29448d71f9ae9c7fdd07692029bee76e7de8983a3056ac", "owner": "smallclaims::store::saved_inventory_tests", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 488, "excerpt": "UPDATE retired_envelopes SET sequence=sequence+1000000"}, diff --git a/release-notes/github-webhook-refresh.json b/release-notes/github-webhook-refresh.json new file mode 100644 index 00000000000..1f64b99ef33 --- /dev/null +++ b/release-notes/github-webhook-refresh.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Verified GitHub webhook deliveries refresh the matching repository and ref observers, and webhook-covered repositories poll GraphQL only as a slow backstop", + "replay": {"status": "changed", "detail": "A daemon with [github].webhook_listen and webhook_secret_file serves POST /fractal/github-webhook on loopback, verifies X-Hub-Signature-256 against the per-request secret file, and writes observer.refresh-requested claims whose attempt starts with github-webhook: for matching active observers, once per delivery. Without those keys nothing runs. The polling member coalesces webhook refreshes: a poll still takes the oldest pending refresh, but one taken for a webhook refresh answers, right before its reads, the newest pending webhook refresh from the same member, which also answers the earlier ones from that member; declared refreshes answer only themselves. A github.repository observer with such a webhook refresh claim accepted within 30 minutes waits the 15-minute settled interval between GraphQL reads instead of 5 minutes, and a quiet standing poll uses the 10-minute ceiling at once; uncovered observers keep their cadence and refresh polls still bypass the floor."}, + "schema": {"status": "none", "detail": "observer.refresh-requested keeps its fields; a webhook refresh is marked only by the github-webhook: prefix of its required attempt string. No kind, table, projection, registry digest or numeric schema version changes."}, + "rules": {"status": "none", "detail": "No checkpoint rule changes; observer.refresh-requested keeps its retention class."}, + "compatibility": {"status": "none", "detail": "Webhook refresh claims are ordinary observer.refresh-requested claims that every reader accepts. An N-1 member that polls a covered observer applies them as refreshes, one poll per delivery oldest first since it does not coalesce, and keeps its own cadence between them, since it does not read the attempt prefix as coverage. Declared refreshes are unchanged in both directions."}, + "downtime": {"status": "none", "detail": "No migration, rebuild or restart sequence."}, + "manual": {"status": "changed", "detail": "To enable, configure [github].webhook_listen (loopback) and webhook_secret_file (absolute, owner-only) on the ingress member, the matching GitHub repository webhook (application/json, the same secret) and a proxy that forwards /fractal/github-webhook verbatim. Unconfigured daemons are unaffected."}, + "recovery": {"status": "none", "detail": "Remove the two keys to stop the receiver, and before rolling back a member that has them, since older configs reject unknown [github] keys; coverage lapses 30 minutes after the last webhook refresh and polling returns to its normal cadence. Rollback leaves webhook refresh claims in history; an older poller answers any it finds pending once each, like any other refresh."} +}