From f909d348a01aa342f1d9a8ca82e97d5e8d2deae9 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Wed, 14 Oct 2026 13:49:02 +0530 Subject: [PATCH 1/2] fix(daemon): reset the cloud-policy rollback floor when the cloud identity changes Root cause: PolicyStore's rollback guard keeps `server_high_water`, the highest deployment the server has offered, for the life of the process. The maintenance lane re-reads credentials on every tick so a key or org change applies without a restart, but the floor was never tied to the identity. The server numbers deployments per (org, machine). After an org switch, a re-registered machine or a fresh enrolment, the new identity legitimately answers deployment 0. The daemon refused it ("deployment rollback from N to 0 is not allowed") until it restarted, and the previous org's active.json kept enforcing. Fix: - PolicyStore::reset_high_water() clears the floor. - The maintenance lane hashes each resolved client's identity: SHA-256 over the length-prefixed normalised URL, machine id and token. Only the digest is kept. When the identity changes (None->Some, Some->None or Some(a)->Some(b)), the lane resets the floor. A credential resolution error is not a change. Until the new identity's first reconcile succeeds, the floor is reset before each poll. Otherwise a failed first poll lets repair re-apply the old desired-state.json and raise the floor again. Within one identity the replay guard is unchanged. - Enrolled -> not enrolled calls PolicyStore::deactivate(), which removes desired-state.json and then active.json. Removing only active.json, as the CLI's --disconnect does, was undone on the next tick: repair rebuilt it from the leftover desired state. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + crates/failproofaid/src/cloud_client.rs | 237 +++++++++++++++++++++- crates/failproofaid/src/cloud_policies.rs | 104 ++++++++++ 3 files changed, 339 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1576b070..e8c6c0169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.0.11-beta.0 — 2026-10-14 + +### Fixes + +- **The daemon no longer refuses a new cloud identity's policies as a "rollback".** Its replay guard kept the highest deployment number it had seen for the life of the process, but the server numbers deployments per organisation and machine. After switching organisation, re-registering the machine or enrolling afresh, the new identity's deployment (often 0) was refused with "deployment rollback from N to 0 is not allowed" until the daemon restarted, and the previous organisation's policies kept enforcing. The guard now starts again whenever the URL, machine id or token it polls with changes; within one identity it is unchanged. When the enrolment is removed, the daemon also deletes `desired-state.json` along with `active.json`. Before, the maintenance pass rebuilt `active.json` from the leftover desired state and the departed organisation's policies came back. + ## 1.0.10 — 2026-10-05 Action needed if you use `instruct` policies with Hermes: an `instruct` no longer stops the call it reminds about. The call runs and the reminder is placed first in its result, so a policy written to prevent an action (for example, a direct API write that should go through an approved script) must become a `deny` before you upgrade. Hermes 0.20.0 is supported again: on 1.0.9 the plugin loaded there with no hooks and every tool call ran unchecked. After upgrading, run `failproofai update` and restart each Hermes gateway profile by name. Collects 1.0.10-beta.0 to beta.1 below. diff --git a/crates/failproofaid/src/cloud_client.rs b/crates/failproofaid/src/cloud_client.rs index 84a428110..f04700447 100644 --- a/crates/failproofaid/src/cloud_client.rs +++ b/crates/failproofaid/src/cloud_client.rs @@ -3,6 +3,7 @@ use crate::cloud_policies::{ }; use reqwest::Url; use reqwest::blocking::Client; +use sha2::{Digest, Sha256}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::JoinHandle; @@ -11,6 +12,10 @@ use std::time::{Duration, Instant}; const DEFAULT_POLL_MS: u64 = 30_000; const MINIMUM_POLL_MS: u64 = 100; +/// Digest of a [`CloudClient`]'s identity. See [`CloudClient::identity`]. +#[derive(Clone, PartialEq, Eq)] +struct CloudIdentity(Vec); + #[derive(Clone)] pub struct CloudClient { base_url: Url, @@ -318,6 +323,28 @@ impl CloudClient { }) } + /// Who this client polls AS: a SHA-256 over the normalised base URL, the + /// machine id and the token. Only the digest is kept, so the token never + /// sits in a second place and cannot reach a log through this value. + /// + /// The server numbers deployments per (org, machine), so any of the three + /// changing can mean a counter that starts again at 0. The token is in it + /// because switching org with the same machine id (or rotating to a fresh + /// enrolment) changes nothing else the daemon can see. + fn identity(&self) -> CloudIdentity { + let mut hasher = Sha256::new(); + for part in [ + self.base_url.as_str(), + self.machine_id.as_str(), + self.token.as_str(), + ] { + // Length-prefixed, so ("ab","c") and ("a","bc") cannot collide. + hasher.update((part.len() as u64).to_be_bytes()); + hasher.update(part.as_bytes()); + } + CloudIdentity(hasher.finalize().to_vec()) + } + /// `applied` is the deployment this machine is ACTUALLY enforcing right now, /// read from `active.json`. /// @@ -446,15 +473,62 @@ pub fn spawn_maintenance( ) -> JoinHandle<()> { std::thread::spawn(move || { let mut last_state: Option = None; + // `None` until the first successful resolution; then the identity seen + // last (itself `None` when not enrolled). + let mut last_identity: Option> = None; + // Set when the identity changed and the new one has not yet completed a + // reconcile. See below. + let mut floor_reset_pending = false; while !shutdown.load(Ordering::Relaxed) { - let cloud = match CloudClient::from_env_or_file() { - Ok(client) => client, + let (cloud, identity) = match CloudClient::from_env_or_file() { + Ok(client) => { + let identity = client.as_ref().map(CloudClient::identity); + (client, Some(identity)) + } + // A resolution ERROR is not a change of identity: broken + // credentials keep the last known-good deployment (see above), + // so they must neither reset the floor nor deactivate anything. Err(err) => { eprintln!("[failproofaid] cloud enrolment error: {err}"); - None + (None, None) } }; + // The rollback floor belongs to ONE cloud identity. The server + // numbers deployments per (org, machine), so after an org switch, a + // re-registration or a fresh enrolment the new identity legitimately + // answers with a lower deployment — often 0. Carrying the old floor + // across refused it ("deployment rollback from N to 0") until the + // daemon restarted, and the previous org's policies kept enforcing. + if let Some(identity) = identity + && last_identity.as_ref() != Some(&identity) + { + let first = last_identity.is_none(); + if !first { + store.reset_high_water(); + floor_reset_pending = identity.is_some(); + eprintln!( + "[failproofaid] cloud identity changed; the cloud-policy deployment floor starts again" + ); + } + // Enrolled → not enrolled (`--disconnect`, credentials removed): + // stop enforcing the departed org's policies. Removing only + // `active.json`, as the CLI does, is undone by the repair below, + // which rebuilds it from `desired-state.json`. + if !first && identity.is_none() { + match store.deactivate() { + Ok(true) => eprintln!( + "[failproofaid] cloud enrolment removed; cloud-managed policies deactivated" + ), + Ok(false) => {} + Err(err) => eprintln!( + "[failproofaid] could not deactivate cloud-managed policies: {err}" + ), + } + } + last_identity = Some(identity); + } + // Log only on transition, so a disconnected machine does not print // a line every 30 seconds forever. let enrolled = cloud.is_some(); @@ -467,7 +541,17 @@ pub fn spawn_maintenance( } if let Some(cloud) = cloud.as_ref() { - poll_once(&store, cloud); + // Re-reset until the new identity's first reconcile lands. If + // that poll fails, the repair below re-applies the PREVIOUS + // identity's `desired-state.json` through `reconcile`, which + // raises the floor straight back to the old number. Replay + // protection resumes from the new identity's first success. + if floor_reset_pending { + store.reset_high_water(); + } + if poll_once(&store, cloud) { + floor_reset_pending = false; + } } // Runs whether or not cloud is reachable: poll failures never @@ -488,7 +572,8 @@ pub fn spawn_maintenance( }) } -fn poll_once(store: &PolicyStore, cloud: &CloudClient) { +/// Whether the server's desired state was reconciled. +fn poll_once(store: &PolicyStore, cloud: &CloudClient) -> bool { // Read BEFORE the request, so what we report is what was in force when we // asked. Reading after would race this poll's own reconcile and could claim // a deployment the server is about to be told about anyway — reporting the @@ -515,12 +600,19 @@ fn poll_once(store: &PolicyStore, cloud: &CloudClient) { "[failproofaid] cloud policy deployment {} active (downloaded {}, repaired {})", outcome.deployment, outcome.downloaded, outcome.repaired ); + true + } + Ok(_) => true, + Err(err) => { + eprintln!("[failproofaid] cloud policy reconcile error: {err}"); + false } - Ok(_) => {} - Err(err) => eprintln!("[failproofaid] cloud policy reconcile error: {err}"), } } - Err(err) => eprintln!("[failproofaid] cloud policy poll error: {err}"), + Err(err) => { + eprintln!("[failproofaid] cloud policy poll error: {err}"); + false + } } } @@ -1151,4 +1243,133 @@ mod tests { assert!(CloudClient::from_file().unwrap().is_none()); unsafe { std::env::remove_var("FAILPROOFAI_HOME") }; } + + // ── The rollback floor follows the cloud identity ──────────────────────── + + /// A fake control plane that answers each token with its own deployment, + /// as the real server does per (org, machine). Empty sets, so no artifact + /// fetch is involved. + fn serve_deployments_by_token(by_token: Vec<(&'static str, u64)>) -> std::net::SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut request = [0_u8; 4096]; + let read = stream.read(&mut request).unwrap_or(0); + let request = String::from_utf8_lossy(&request[..read]).to_lowercase(); + let deployment = by_token + .iter() + .find(|(token, _)| request.contains(&format!("bearer {token}"))) + .map(|(_, deployment)| *deployment); + let response = match deployment { + Some(d) => { + let body = + format!(r#"{{"schemaVersion":2,"deployment":{d},"policies":[]}}"#); + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + } + None => "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string(), + }; + let _ = stream.write_all(response.as_bytes()); + } + }); + address + } + + fn wait_for(what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if done() { + return; + } + std::thread::sleep(Duration::from_millis(20)); + } + panic!("timed out waiting for {what}"); + } + + /// The reported bug, end to end through the maintenance lane: switch the + /// credential to another org (fresh token and machine) while the daemon + /// runs, and that org's deployment 0 must activate on the next poll — + /// not be refused as a rollback from the old org's 3 until a restart. + /// Then remove the credential: the old org's policies stop enforcing and + /// stay stopped across the repair pass. + #[test] + fn an_identity_change_resets_the_floor_and_a_disconnect_deactivates() { + let _lock = lock_env(); + let address = serve_deployments_by_token(vec![("org-a-token", 3), ("org-b-token", 0)]); + let creds = |machine: &str, token: &str| { + format!( + r#"{{"schemaVersion":1,"url":"http://{address}","machineId":"{machine}","token":"{token}"}}"# + ) + }; + let guard = with_credentials_file(Some(&creds("machine-a", "org-a-token"))); + let credentials = guard.0.join("cloud.json"); + let root = guard.0.join("cloud-policies"); + let store = PolicyStore::new(root.clone()); + let shutdown = Arc::new(AtomicBool::new(false)); + let lane = spawn_maintenance( + store.clone(), + shutdown.clone(), + Duration::from_millis(50), + Duration::from_millis(50), + ); + + wait_for("org A's deployment 3", || { + store + .read_active() + .ok() + .flatten() + .is_some_and(|a| a.deployment == 3) + }); + + fs::write(&credentials, creds("machine-b", "org-b-token")).unwrap(); + wait_for("org B's deployment 0 after the switch", || { + store + .read_active() + .ok() + .flatten() + .is_some_and(|a| a.deployment == 0) + }); + assert_eq!(store.read_desired().unwrap().unwrap().deployment, 0); + + // Back to org A: its 3 is above the floor anyway, and must still win. + fs::write(&credentials, creds("machine-a", "org-a-token")).unwrap(); + wait_for("org A's deployment 3 again", || { + store + .read_active() + .ok() + .flatten() + .is_some_and(|a| a.deployment == 3) + }); + + fs::remove_file(&credentials).unwrap(); + wait_for("deactivation after the credential is removed", || { + store.read_active().ok().flatten().is_none() + && store.read_desired().ok().flatten().is_none() + }); + // Several more ticks: repair must not resurrect the departed org. + std::thread::sleep(Duration::from_millis(300)); + assert!(store.read_active().unwrap().is_none()); + + shutdown.store(true, Ordering::Relaxed); + lane.join().unwrap(); + } + + #[test] + fn the_identity_covers_url_machine_and_token_and_is_stable() { + let a = CloudClient::new("https://cloud.example", "t".into(), "m".into()).unwrap(); + let same = CloudClient::new("https://CLOUD.example/", "t".into(), "m".into()).unwrap(); + assert!(a.identity() == same.identity()); + for other in [ + CloudClient::new("https://other.example", "t".into(), "m".into()).unwrap(), + CloudClient::new("https://cloud.example", "t2".into(), "m".into()).unwrap(), + CloudClient::new("https://cloud.example", "t".into(), "m2".into()).unwrap(), + ] { + assert!(a.identity() != other.identity()); + } + } } diff --git a/crates/failproofaid/src/cloud_policies.rs b/crates/failproofaid/src/cloud_policies.rs index 8acd7993c..3bc3a0fb8 100644 --- a/crates/failproofaid/src/cloud_policies.rs +++ b/crates/failproofaid/src/cloud_policies.rs @@ -306,6 +306,41 @@ impl PolicyStore { &self.root } + /// Forgets the highest deployment the server has offered this process. + /// + /// The deployment counter is the server's per (org, machine) — a different + /// org, a re-registered machine or a fresh enrolment starts again at 0. A + /// floor carried over from the previous identity would refuse every + /// deployment the new one legitimately sends ("deployment rollback from 3 + /// to 0") until the daemon restarted, while the previous org's policies + /// went on enforcing. The maintenance lane calls this when, and only when, + /// the cloud identity it polls with changes; within one identity the + /// replay guard is untouched. + pub fn reset_high_water(&self) { + self.server_high_water.store(0, Ordering::Relaxed); + } + + /// Stops enforcing cloud-managed policy: removes `desired-state.json`, then + /// `active.json`. Returns whether anything was removed. + /// + /// Both files, in that order. Removing only `active.json` (what the CLI's + /// `--disconnect` does) is undone on the next maintenance tick, because + /// `repair_active_from_cache` rebuilds the pointer from the desired state + /// still on disk. Desired first, so a crash between the two leaves nothing + /// that repair could resurrect. The content-addressed artifacts stay: they + /// are inert without a pointer, and a reconnect may reuse them. + pub fn deactivate(&self) -> Result { + let mut removed = false; + for path in [self.desired_state_path(), self.active_manifest_path()] { + match fs::remove_file(&path) { + Ok(()) => removed = true, + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + } + Ok(removed) + } + pub fn active_manifest_path(&self) -> PathBuf { self.root.join("active.json") } @@ -1046,6 +1081,75 @@ mod tests { fs::remove_dir_all(store.root()).ok(); } + /// The floor belongs to one cloud identity. After a reset (org switch, + /// re-registration) the new identity's deployment 0 — and its empty set — + /// must activate; without one, the same 0 is still a rollback. + #[test] + fn a_reset_floor_accepts_the_next_identitys_lower_deployment() { + let store = temp_store("reset-floor"); + let bytes = b"policy"; + store + .reconcile(&desired(3, "guard", bytes), &|_: &DesiredPolicy| { + Ok(bytes.to_vec()) + }) + .unwrap(); + + let empty = DesiredState { + schema_version: DESIRED_STATE_SCHEMA_VERSION, + deployment: 0, + policies: vec![], + }; + let refuse = |_: &DesiredPolicy| -> Result, String> { Err("no fetch".into()) }; + assert!( + matches!( + store.reconcile(&empty, &refuse), + Err(ReconcileError::InvalidDesiredState(_)) + ), + "without a reset, a lower deployment is still a rollback" + ); + + store.reset_high_water(); + let outcome = store + .reconcile(&empty, &refuse) + .expect("a new identity's deployment 0 must be accepted after a reset"); + assert!(outcome.activated); + let active = store.read_active().unwrap().unwrap(); + assert_eq!(active.deployment, 0); + assert!(active.policies.is_empty()); + + // And the guard is live again for the new identity. + store + .reconcile(&desired(2, "guard", bytes), &|_: &DesiredPolicy| { + Ok(bytes.to_vec()) + }) + .unwrap(); + assert!(matches!( + store.reconcile(&empty, &refuse), + Err(ReconcileError::InvalidDesiredState(_)) + )); + fs::remove_dir_all(store.root()).ok(); + } + + /// Deactivation must survive the next maintenance tick: repair rebuilds + /// `active.json` from `desired-state.json`, so both have to go. + #[test] + fn deactivate_removes_what_repair_would_resurrect() { + let store = temp_store("deactivate"); + let bytes = b"policy"; + store + .reconcile(&desired(4, "guard", bytes), &|_: &DesiredPolicy| { + Ok(bytes.to_vec()) + }) + .unwrap(); + assert!(store.deactivate().unwrap()); + assert!(store.read_active().unwrap().is_none()); + assert!(store.read_desired().unwrap().is_none()); + assert_eq!(store.repair_active_from_cache().unwrap(), 0); + assert!(store.read_active().unwrap().is_none()); + assert!(!store.deactivate().unwrap(), "idempotent"); + fs::remove_dir_all(store.root()).ok(); + } + #[test] fn rejects_traversal_duplicate_ids_and_deployment_rollback() { let store = temp_store("validation"); From f83320bd63b0966974edc6af96cbbd7000d25db1 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Wed, 14 Oct 2026 13:54:55 +0530 Subject: [PATCH 2/2] docs(changelog): reference #899 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8c6c0169..0b510fade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixes -- **The daemon no longer refuses a new cloud identity's policies as a "rollback".** Its replay guard kept the highest deployment number it had seen for the life of the process, but the server numbers deployments per organisation and machine. After switching organisation, re-registering the machine or enrolling afresh, the new identity's deployment (often 0) was refused with "deployment rollback from N to 0 is not allowed" until the daemon restarted, and the previous organisation's policies kept enforcing. The guard now starts again whenever the URL, machine id or token it polls with changes; within one identity it is unchanged. When the enrolment is removed, the daemon also deletes `desired-state.json` along with `active.json`. Before, the maintenance pass rebuilt `active.json` from the leftover desired state and the departed organisation's policies came back. +- **The daemon no longer refuses a new cloud identity's policies as a "rollback".** Its replay guard kept the highest deployment number it had seen for the life of the process, but the server numbers deployments per organisation and machine. After switching organisation, re-registering the machine or enrolling afresh, the new identity's deployment (often 0) was refused with "deployment rollback from N to 0 is not allowed" until the daemon restarted, and the previous organisation's policies kept enforcing. The guard now starts again whenever the URL, machine id or token it polls with changes; within one identity it is unchanged. When the enrolment is removed, the daemon also deletes `desired-state.json` along with `active.json`. Before, the maintenance pass rebuilt `active.json` from the leftover desired state and the departed organisation's policies came back. (#899) ## 1.0.10 — 2026-10-05