From 3f9d938130d0fa5a24bea256198362df500542eb Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 12:39:35 -0600 Subject: [PATCH 01/15] fix(settings): keep saved sources, plugin state and sign-ins a newer RustGIS wrote Saved data sources dropped rows of unknown kinds, unknown fields and lists on the next save (recording a recent source is enough), and overwrote a preference they couldn't read. They now keep those and write them back unchanged, keep an unreadable preference byte for byte, and say what wasn't used. Data they can't read that holds a password, key or token is dropped and said. A change made before the first read of the preference is now saved too. The desktop plugin state.json is no longer overwritten when it can't be read (load reports it, writes are refused) and keeps keys it doesn't know. The web app's plugins_web keeps entries and fields it can't read and isn't written over when it isn't a list. The ChatGPT sign-in store refuses a version above 2 instead of rewriting it as 2. Part of #1405 (item 3). Co-Authored-By: Claude Opus 5.5 --- crates/assistant/src/chatgpt.rs | 42 ++++- crates/engine/src/sources.rs | 262 +++++++++++++++++++++++++++--- crates/plugins/src/store.rs | 28 +++- crates/plugins/src/tests.rs | 29 ++++ crates/ui-egui/src/pluginsui.rs | 84 +++++++++- crates/ui-egui/src/sourcespane.rs | 15 +- crates/ui-egui/tests/smoke.rs | 48 ++++++ 7 files changed, 475 insertions(+), 33 deletions(-) diff --git a/crates/assistant/src/chatgpt.rs b/crates/assistant/src/chatgpt.rs index 24936c6413..3939bc2789 100644 --- a/crates/assistant/src/chatgpt.rs +++ b/crates/assistant/src/chatgpt.rs @@ -838,6 +838,9 @@ mod native { } } + /// The sign-in store at `path` (none: no account). One that isn't JSON, or that a newer + /// RustGIS saved (a `version` above [`STORE_VERSION`]), is an error: every change reads it + /// first, so it is never written over. fn read_accounts(path: &Path) -> Result { let text = match std::fs::read_to_string(path) { Ok(text) => text, @@ -845,6 +848,16 @@ mod native { Err(e) => return Err(e.to_string().into()), }; let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|_| "the sign-in store isn't JSON")?; + // One a newer RustGIS wrote may hold what this version would lose by writing it again: it + // is neither used nor written over. + if let Some(version) = v.get("version").and_then(Value::as_u64).filter(|v| *v > STORE_VERSION) { + return Err(Error::new( + Recovery::Auth, + format!( + "the ChatGPT sign-ins were saved by a newer RustGIS (version {version}; this one reads {STORE_VERSION}): use that RustGIS, or update this one" + ), + )); + } if let Some(entries) = v.get("accounts").and_then(Value::as_array) { let entries = entries.iter().map(Credentials::from_json).collect::, _>>()?; let active = @@ -857,6 +870,11 @@ mod native { } } + /// The version of the sign-in store this RustGIS reads and writes. + const STORE_VERSION: u64 = 2; + + /// Writes the sign-in store as version [`STORE_VERSION`], without tokens except a settings + /// file's own from before the keychain. fn write_accounts(path: &Path, accounts: &Accounts) -> Result<(), Error> { let active = accounts.active.as_ref().map(|(client, subject)| serde_json::json!({"client_id": client, "subject": subject})); let entry = |c: &Credentials| { @@ -871,7 +889,7 @@ mod native { }; write_private( path, - &serde_json::json!({"version": 2, "active": active, "accounts": accounts.entries.iter().map(entry).collect::>()}) + &serde_json::json!({"version": STORE_VERSION, "active": active, "accounts": accounts.entries.iter().map(entry).collect::>()}) .to_string(), ) } @@ -2023,6 +2041,28 @@ mod native { assert!(remove_at(&path, &v, "oaiapp_other", "user-1").unwrap().is_none()); } + /// A store a newer RustGIS saved is neither used nor written over, whatever is asked of it. + #[test] + fn a_store_from_a_newer_version_is_kept_as_it_is() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("auth.json"); + let store = Arc::new(MemorySecrets::new()); + let v = sharing(&store); + write_credentials(&path, &v, &synthetic(now() + 100_000)).unwrap(); + let mut newer: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + newer["version"] = serde_json::json!(3); + newer["devices"] = serde_json::json!([{"id": "d1"}]); + let text = newer.to_string(); + std::fs::write(&path, &text).unwrap(); + let e = read_accounts(&path).err().unwrap(); + assert!(e.to_string().contains("newer RustGIS (version 3; this one reads 2)"), "{e}"); + assert!(read_credentials(&path, &v).is_err()); + assert!(write_credentials(&path, &v, &synthetic(now() + 100_000)).is_err()); + assert!(sign_out_at(&path, &v).is_err()); + assert!(remove_at(&path, &v, "oaiapp_synthetic", "user-1").is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), text); + } + #[test] fn an_unreadable_keychain_signs_out_here_and_keeps_its_item_noted() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index bc3aa7eeff..00754ded9a 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -4,8 +4,10 @@ //! without their passwords, tokens or signatures. use rustgis_data::source::credentials::{redact_query, secret_key}; +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; /// How many recent sources are kept. pub const MAX_RECENT: usize = 20; @@ -43,31 +45,159 @@ pub struct Sources { /// Bumped on every change, so the app knows when to keep them. #[serde(skip)] pub generation: u64, + /// What of the preference this version couldn't use, written back as it was. + #[serde(skip)] + kept: Kept, +} + +/// The three lists, as the preference names them. +const LISTS: [&str; 3] = ["favourites", "recent", "saved"]; + +/// What of the saved sources this version couldn't use. A newer RustGIS may add kinds, fields and +/// lists: they are kept and written back unchanged, so a source noted or kept here never drops +/// what the newer one wrote. +#[derive(Clone, Debug, Default, PartialEq)] +struct Kept { + /// The whole preference as read, when it couldn't be read at all, and why: it is never + /// written over, so what changes in this session isn't kept. + whole: Option<(String, String)>, + /// Rows of each list that couldn't be read (an unknown kind, an address that isn't one…), or + /// that are past the list's limit. + rows: BTreeMap<&'static str, Vec>, + /// How many of those rows are past the limit, rather than unreadable. + over: usize, + /// Fields of read sources this version doesn't know, by list, kind and address. + fields: BTreeMap<(&'static str, String, String), Map>, + /// Keys beside the three lists this version doesn't know. + top: Map, + /// Rows and fields that couldn't be read and held a password, key or token: not kept, since + /// this version can't strip it from what it doesn't understand. + secrets: usize, +} + +impl Kept { + /// The entries of `o` without a password, key or token, counting those left out. + fn without_secrets(&mut self, mut o: Map) -> Map { + let before = o.len(); + o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v }))); + self.secrets += before - o.len(); + o + } +} + +/// A saved row: the source, and the fields this version doesn't know. +#[derive(Deserialize)] +struct Row { + #[serde(flatten)] + source: Source, + #[serde(flatten)] + other: Map, } impl Sources { /// Sources read back from the app's preferences: each one checked and cleaned again (the - /// preferences are a file anyone could have edited). + /// preferences are a file anyone could have edited). What this version can't use is kept, to + /// be written back by [`Sources::to_json`] as it was: rows it can't read, fields and lists it + /// doesn't know, or the whole preference when it can't be read at all. [`Sources::unread`] + /// says what was kept. pub fn from_json(text: &str) -> Sources { - let v: Value = serde_json::from_str(text).unwrap_or(Value::Null); - // Entry by entry: one that doesn't read is dropped, not the whole list. - let read = |k: &str| -> Vec { - v.get(k) - .and_then(Value::as_array) - .map(|a| a.iter().filter_map(|x| serde_json::from_value::(x.clone()).ok()).filter_map(|x| clean(x).ok()).collect()) - .unwrap_or_default() + let mut s = Sources::default(); + if text.trim().is_empty() { + return s; + } + let whole = |why: &str| Kept { whole: Some((text.to_string(), why.to_string())), ..Kept::default() }; + let mut o = match serde_json::from_str::(text) { + Ok(Value::Object(o)) => o, + _ => { + s.kept = whole("they aren't a JSON object"); + return s; + } }; - let mut s = Sources { favourites: read("favourites"), recent: read("recent"), saved: read("saved"), generation: 0 }; - s.favourites.truncate(MAX_KEPT); - s.saved.truncate(MAX_KEPT); - s.recent.truncate(MAX_RECENT); - // Preferences carry provenance, never authority from an earlier process's login. - crate::s3_connections::clear_bindings( - s.favourites.iter_mut().chain(&mut s.recent).chain(&mut s.saved).map(|source| &mut source.address), - ); + let mut lists = Vec::with_capacity(LISTS.len()); + for list in LISTS { + match o.remove(list) { + None => lists.push(Vec::new()), + Some(Value::Array(rows)) => lists.push(rows), + Some(_) => { + s.kept = whole("a list of them isn't a JSON array"); + return s; + } + } + } + s.kept.top = s.kept.without_secrets(o); + for (list, rows) in LISTS.into_iter().zip(lists) { + let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; + let mut into = Vec::new(); + for row in rows { + // Entry by entry: one that doesn't read is kept as it was, not the whole list. + let read = serde_json::from_value::(row.clone()).ok().and_then(|r| Some((clean(r.source).ok()?, r.other))); + match read { + Some((mut source, other)) if into.len() < cap => { + // Preferences carry provenance, never authority from an earlier process's login. + crate::s3_connections::clear_bindings(std::iter::once(&mut source.address)); + let other = s.kept.without_secrets(other); + if !other.is_empty() { + s.kept.fields.insert((list, source.kind.clone(), source.address.clone()), other); + } + into.push(source); + } + _ if holds_secret(&row) => s.kept.secrets += 1, + read => { + s.kept.over += usize::from(read.is_some()); + s.kept.rows.entry(list).or_default().push(row); + } + } + } + if let Ok(l) = s.list_mut(list) { + *l = into; + } + } s } + /// What of the preference read by [`Sources::from_json`] isn't used here, one line per kind + /// (none when all of it is). + pub fn unread(&self) -> Vec { + let k = &self.kept; + if let Some((_, why)) = &k.whole { + return vec![format!( + "The saved data sources couldn't be read ({why}): none is listed, and the favourites, recent and saved sources of this session aren't kept over them; fix or remove \"data_sources\" in the preferences file to keep them" + )]; + } + let mut said = Vec::new(); + let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over); + if k.over > 0 { + let n = k.over; + said.push(format!( + "{n} saved data source{} past the lists' limits ({MAX_KEPT} favourites or saved, {MAX_RECENT} recent): not listed, and kept as {} when the sources are saved", + if n == 1 { " is" } else { "s are" }, + if n == 1 { "it was" } else { "they were" } + )); + } + if rows > 0 { + said.push(format!( + "{rows} saved data source{} couldn't be read (from a newer RustGIS, or damaged): not listed, and kept as {} when the sources are saved", + if rows == 1 { "" } else { "s" }, + if rows == 1 { "it was" } else { "they were" } + )); + } + if !k.fields.is_empty() { + let n = k.fields.len(); + said.push(format!( + "{n} saved data source{} settings this version doesn't know (from a newer RustGIS): ignored here, and kept when the sources are saved", + if n == 1 { " has" } else { "s have" } + )); + } + if k.secrets > 0 { + let n = k.secrets; + said.push(format!( + "{n} entr{} of the saved data sources couldn't be read and held a password, key or token: dropped, and not saved again", + if n == 1 { "y" } else { "ies" } + )); + } + said + } + /// Takes in the sources kept with the preferences, keeping what this session already has: /// in-memory entries stay (and first), stored ones are added after them unless the same source /// ([`Source::is`]) is there already. @@ -85,11 +215,35 @@ impl Sources { join(&mut self.favourites, stored.favourites, MAX_KEPT); join(&mut self.saved, stored.saved, MAX_KEPT); join(&mut self.recent, stored.recent, MAX_RECENT); + self.kept = stored.kept; self.generation += 1; } + /// The `data_sources` preference: these sources, with what [`Sources::from_json`] kept + /// written back as it was (a preference it couldn't read at all, unchanged). pub fn to_json(&self) -> String { - serde_json::to_string(self).unwrap_or_default() + if let Some((text, _)) = &self.kept.whole { + return text.clone(); + } + let mut o = self.kept.top.clone(); + for (list, sources) in LISTS.into_iter().zip([&self.favourites, &self.recent, &self.saved]) { + let mut rows: Vec = sources + .iter() + .map(|source| { + let mut row = serde_json::to_value(source).unwrap_or(Value::Null); + let key = (list, source.kind.clone(), source.address.clone()); + if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&key)) { + for (k, v) in other { + row.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + row + }) + .collect(); + rows.extend(self.kept.rows.get(list).into_iter().flatten().cloned()); + o.insert(list.into(), Value::Array(rows)); + } + Value::Object(o).to_string() } /// Notes a source just added (the most recent first, each once). @@ -98,6 +252,9 @@ impl Sources { self.recent.retain(|r| !same(r, &s)); self.recent.insert(0, s); self.recent.truncate(MAX_RECENT); + // A source no longer recent takes the fields kept for it along. + let recent = &self.recent; + self.kept.fields.retain(|(list, kind, address), _| *list != "recent" || recent.iter().any(|r| r.is(kind, address))); self.generation += 1; } @@ -126,6 +283,9 @@ impl Sources { let before = l.len(); l.retain(|r| !r.is(kind, address)); let gone = l.len() != before; + if let Some(list) = LISTS.into_iter().find(|l| *l == list) { + self.kept.fields.remove(&(list, kind.to_string(), address.to_string())); + } if gone { self.generation += 1; } @@ -712,6 +872,68 @@ MIIE assert!(s.generation > before); } + /// Rows, fields and lists a newer RustGIS wrote are kept through recording a recent source + /// and keeping a favourite here; a preference that can't be read at all is kept byte for + /// byte. Clean input says nothing. + #[test] + fn sources_keep_what_this_version_cannot_read() { + let future = json!({"kind": "gcs", "name": "Bucket", "address": "gs://b/a.tif"}); + let text = json!({ + "favourites": [{"kind": "file", "name": "A", "address": "/a.gpkg", "pinned": true}, future.clone()], + "recent": [{"kind": "file", "name": "R", "address": "/r.csv", "opened": 3}], + "saved": [], + "folders": ["/data"] + }) + .to_string(); + let mut s = Sources::default(); + s.merge(Sources::from_json(&text)); + assert_eq!(s.unread().len(), 2, "{:?}", s.unread()); + assert!(s.unread()[0].starts_with("1 saved data source couldn't be read"), "{:?}", s.unread()); + s.note(Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }); + s.keep("favourites", Source { kind: "file".into(), address: "/b.gpkg".into(), ..Source::default() }).unwrap(); + let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); + assert_eq!(saved["folders"], json!(["/data"])); + assert_eq!(saved["favourites"][0]["pinned"], true); + assert_eq!(saved["favourites"][1]["address"], "/b.gpkg"); + assert_eq!(saved["favourites"][2], future); + assert_eq!(saved["recent"][1]["opened"], 3); + // A source that is no longer recent, or that is forgotten, takes its fields along. + for i in 0..MAX_RECENT { + s.note(Source { kind: "file".into(), address: format!("/{i}.csv"), ..Source::default() }); + } + s.forget("favourites", "file", "/a.gpkg").unwrap(); + assert!(!s.to_json().contains("opened") && !s.to_json().contains("pinned")); + assert!(s.to_json().contains("gs://b/a.tif")); + + for whole in ["[1]", r#"{"favourites": {"a": 1}}"#] { + let mut s = Sources::default(); + s.merge(Sources::from_json(whole)); + assert!(matches!(s.unread().as_slice(), [one] if one.contains("couldn't be read")), "{whole}"); + s.note(Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }); + assert_eq!(s.recent.len(), 1, "this session's sources are still listed"); + assert_eq!(s.to_json(), whole); + } + // Rows past a list's limit are kept, and said as that. + let recent: Vec = (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + let s = Sources::from_json(&json!({ "recent": recent }).to_string()); + assert_eq!(s.recent.len(), MAX_RECENT); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source is past")), "{:?}", s.unread()); + assert!(s.to_json().contains(&format!("/{MAX_RECENT}.csv"))); + // What it can't read and that holds a secret isn't kept: this version can't strip it. + let secret = json!({"token": "hidden", "saved": [ + {"kind": "url", "name": "A", "address": "https://a/x.fgb", "apiKey": "hidden"}, + {"kind": "gcs", "name": "B", "address": "https://g.example/b?sig=hidden"} + ]}); + let s = Sources::from_json(&secret.to_string()); + assert!(s.unread().iter().any(|t| t.starts_with("3 entries of the saved data sources")), "{:?}", s.unread()); + assert!(!s.to_json().contains("hidden")); + for clean in + ["", r#"{"favourites": [], "recent": [], "saved": []}"#, r#"{"recent": [{"kind": "file", "name": "R", "address": "/r.csv"}]}"#] + { + assert!(Sources::from_json(clean).unread().is_empty(), "{clean}"); + } + } + #[test] fn identity_is_kind_and_address() { let mut s = Sources::default(); @@ -930,7 +1152,9 @@ MIIE assert_eq!(back.favourites.len(), 1); assert_eq!(back.favourites[0].address, "https://h/x.fgb"); assert!(back.recent.is_empty()); - assert_eq!(Sources::from_json("not json"), Sources::default()); + assert_eq!(back.unread().len(), 1, "{:?}", back.unread()); + let unreadable = Sources::from_json("not json"); + assert!(unreadable.favourites.is_empty() && unreadable.recent.is_empty() && unreadable.saved.is_empty()); } #[test] diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 8febc712da..7782bc4a1a 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -51,17 +51,38 @@ struct State { disabled: Vec, #[serde(default)] folders: Vec, + /// Keys a newer RustGIS added, written back as they were. + #[serde(flatten)] + other: serde_json::Map, + /// Why `state.json` couldn't be read, when it couldn't: it is then never written over. + #[serde(skip)] + unreadable: Option, } fn state_path() -> Option { dir().map(|d| d.join("state.json")) } +/// `state.json`: none is no plugin disabled or added from a folder. One that can't be read (or that +/// a newer RustGIS wrote in a shape this one doesn't know) is said by [`load_installed`] and kept as +/// it is: [`write_state`] refuses to write over it. fn read_state() -> State { - state_path().and_then(|p| std::fs::read_to_string(p).ok()).and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() + let Some(path) = state_path() else { return State::default() }; + let read = match std::fs::read_to_string(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return State::default(), + Err(e) => Err(e.to_string()), + Ok(text) => serde_json::from_str(&text).map_err(|e| e.to_string()), + }; + read.unwrap_or_else(|e| State { unreadable: Some(format!("{}: {e}", path.display())), ..State::default() }) } +/// Writes `state.json`, refused when [`read_state`] couldn't read the one there. fn write_state(st: &State) -> Result<(), String> { + if let Some(why) = &st.unreadable { + return Err(format!( + "{why}; it is kept as it is, so which plugins are disabled or added from folders can't be changed until it is fixed or removed" + )); + } let path = state_path().ok_or("there is no folder for plugins")?; if let Some(d) = path.parent() { std::fs::create_dir_all(d).map_err(|e| format!("{}: {e}", d.display()))?; @@ -94,6 +115,11 @@ pub fn load_installed() -> Vec { )), } let st = read_state(); + if let Some(why) = &st.unreadable { + errors.push(format!( + "{why}; it was ignored and is kept as it is, so plugins it disables are on and plugin folders it adds aren't loaded" + )); + } let mut paths: Vec = std::fs::read_dir(&dir) .map(|rd| rd.flatten().map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("rgplugin"))).collect()) .unwrap_or_default(); diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index 1ee24fe92c..ab16c49223 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1830,6 +1830,35 @@ fn cached_blocklist_survives_refused_refreshes() { let _ = std::fs::remove_dir_all(&plugins); } +/// `state.json` keeps what a newer RustGIS added through a change here, and one that can't be +/// read is said at start and never written over. +#[test] +fn plugin_state_keeps_what_this_version_cannot_read() { + let plugins = std::env::temp_dir().join(format!("rustgis-test-plugin-state-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&plugins); + let _store = store_lock(); + crate::store::set_dir(Some(plugins.clone())); + std::fs::create_dir_all(&plugins).unwrap(); + let state = plugins.join("state.json"); + std::fs::write(&state, r#"{"disabled": [], "folders": [], "pinned": ["kept"]}"#).unwrap(); + crate::store::install_bytes(&testing::package("state-test", &[])).unwrap(); + crate::store::set_enabled("state-test", false).unwrap(); + let saved: Value = serde_json::from_str(&std::fs::read_to_string(&state).unwrap()).unwrap(); + assert_eq!((&saved["disabled"], &saved["pinned"]), (&json!(["state-test"]), &json!(["kept"]))); + for unreadable in ["not json", r#"{"disabled": [{"id": "state-test", "until": 2}]}"#] { + std::fs::write(&state, unreadable).unwrap(); + assert!(crate::store::load_installed().iter().any(|e| e.contains("state.json")), "{unreadable}"); + let e = crate::store::set_enabled("state-test", true).unwrap_err(); + assert!(e.contains("kept as it is"), "{e}"); + assert!(crate::store::uninstall("state-test").is_err()); + assert_eq!(std::fs::read_to_string(&state).unwrap(), unreadable); + } + std::fs::remove_file(&state).unwrap(); + assert!(crate::store::load_installed().is_empty()); + crate::store::uninstall("state-test").unwrap(); + let _ = std::fs::remove_dir_all(&plugins); +} + #[test] fn process_packages_unpack_safely() { use std::io::Write; diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 76ab3f30ab..378aff65f5 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -40,6 +40,9 @@ struct Saved { data: Option, #[serde(default = "yes")] enabled: bool, + /// Fields a newer RustGIS added, written back as they were. + #[serde(flatten)] + other: serde_json::Map, } fn yes() -> bool { @@ -282,23 +285,46 @@ fn keeps_prefs() -> bool { cfg!(target_arch = "wasm32") } -fn saved(app: &RustGisApp) -> Vec { - app.services.load_pref.as_ref().and_then(|l| l("plugins_web")).and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() +/// The installed plugins the web app keeps (`plugins_web`): those it can read, and the entries it +/// can't (from a newer RustGIS, or damaged), kept to be written back as they were. `None` when +/// the preference isn't a JSON list: it is then never written over. None kept is an empty list. +fn read_saved(text: Option<&str>) -> Option<(Vec, Vec)> { + let Some(text) = text.filter(|t| !t.trim().is_empty()) else { return Some((Vec::new(), Vec::new())) }; + let Ok(Value::Array(rows)) = serde_json::from_str::(text) else { return None }; + let (mut list, mut kept) = (Vec::new(), Vec::new()); + for row in rows { + match serde_json::from_value::(row.clone()) { + Ok(s) => list.push(s), + Err(_) => kept.push(row), + } + } + Some((list, kept)) } -fn save(app: &RustGisApp, list: &[Saved]) { - if let (Some(s), Ok(text)) = (&app.services.save_pref, serde_json::to_string(list)) { - s("plugins_web", &text); - } +/// The `plugins_web` preference as [`read_saved`] reads it. +fn saved(app: &RustGisApp) -> Option<(Vec, Vec)> { + read_saved(app.services.load_pref.as_ref().and_then(|l| l("plugins_web")).as_deref()) } +/// The `plugins_web` preference for `list`, with the entries [`read_saved`] couldn't read after it. +fn saved_text(list: &[Saved], kept: &[Value]) -> String { + let mut rows: Vec = list.iter().filter_map(|s| serde_json::to_value(s).ok()).collect(); + rows.extend(kept.iter().cloned()); + Value::Array(rows).to_string() +} + +/// Changes the installed plugins the web app keeps with `f`, writing back the entries it couldn't +/// read as they were; nothing on the desktop, or when the preference couldn't be read. fn update_saved(app: &RustGisApp, f: impl FnOnce(&mut Vec)) { if !keeps_prefs() { return; } - let mut list = saved(app); + // One that can't be read was said at start, and isn't written over. + let Some((mut list, kept)) = saved(app) else { return }; f(&mut list); - save(app, &list); + if let Some(s) = &app.services.save_pref { + s("plugins_web", &saved_text(&list, &kept)); + } } /// Loads the plugins the web app installed in earlier visits. @@ -309,7 +335,22 @@ pub fn restore(app: &mut RustGisApp) { if let Some(u) = app.services.load_pref.as_ref().and_then(|l| l("plugin_registry")) { app.plugins.url = u; } - restore_list(app, saved(app)); + let (list, kept) = match saved(app) { + Some(read) => read, + None => { + app.session.warn(tl!( + "The plugins installed in this browser couldn't be read (they aren't a JSON list): none is loaded, and plugins installed or removed now aren't remembered." + )); + (Vec::new(), Vec::new()) + } + }; + if !kept.is_empty() { + app.session.warn(crate::i18n::fmt( + "{n} installed plugins couldn't be read (from a newer RustGIS, or damaged): they aren't loaded, and are kept as they were.", + &[("n", &kept.len().to_string())], + )); + } + restore_list(app, list); // The blocklist, so a pulled plugin isn't loaded again. let registry = app.plugins.registry_url(); let slot = fetch(app, &rp::registry::blocklist_url(®istry), 4 << 20, None); @@ -483,6 +524,7 @@ fn install_confirmed(app: &mut RustGisApp) { base64::engine::general_purpose::STANDARD.encode(&c.bytes) }), enabled: true, + other: serde_json::Map::new(), }; let kept = entry.url.is_some() || entry.data.is_some(); update_saved(app, |list| { @@ -1263,6 +1305,7 @@ mod tests { url: Some(format!("https://example.invalid/{id}.rgplugin")), data: None, enabled: true, + other: serde_json::Map::new(), } } @@ -1285,6 +1328,29 @@ mod tests { app.session.notices_since(mark).iter().map(|n| (n.text.clone(), n.error)).collect() } + /// What a newer RustGIS wrote to `plugins_web` survives a change here: entries this version + /// can't read and fields it doesn't know are written back as they were; a preference that + /// isn't a list isn't read at all. + #[test] + fn saved_web_plugins_keep_what_this_version_cannot_read() { + let text = json!([ + {"id": "a", "sha256": "0", "url": "https://example.invalid/a.rgplugin", "enabled": true, "channel": "beta"}, + {"id": "b", "sha256": {"v2": "0"}} + ]) + .to_string(); + let (mut list, kept) = read_saved(Some(&text)).unwrap(); + assert_eq!((list.len(), kept.len()), (1, 1)); + list[0].enabled = false; + list.push(record("c", "1")); + let again: Value = serde_json::from_str(&saved_text(&list, &kept)).unwrap(); + assert_eq!(again[0]["channel"], "beta"); + assert_eq!(again[0]["enabled"], false); + assert_eq!(again[1]["id"], "c"); + assert_eq!(again[2], json!({"id": "b", "sha256": {"v2": "0"}})); + assert!(read_saved(Some("{\"plugins\": []}")).is_none()); + assert_eq!(read_saved(None).map(|(l, k)| (l.len(), k.len())), Some((0, 0))); + } + /// Plugins the web app couldn't load at start are listed, kept, and told once (#1352). #[test] fn plugins_that_fail_at_start_are_listed_and_told_once() { diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 14779e6cdc..d3cce0b994 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -53,12 +53,21 @@ pub fn load(app: &mut RustGisApp) { return; } app.ui.sources.loaded = true; + // A change made before this first read (a source just added, which is what usually brings + // it on) is still to be saved, with the stored lists. + let changed = app.session.sources.generation != app.ui.sources_saved; if let Some(text) = app.services.load_pref.as_ref().and_then(|load| load(PREF)) { // What this session already has (added, saved or made favourite before the first read) - // stays; the stored lists are added after it. - app.session.sources.merge(rustgis_engine::sources::Sources::from_json(&text)); + // stays; the stored lists are added after it. What of them isn't used here is said. + let stored = rustgis_engine::sources::Sources::from_json(&text); + for said in stored.unread() { + app.session.warn(said); + } + app.session.sources.merge(stored); + } + if !changed { + app.ui.sources_saved = app.session.sources.generation; } - app.ui.sources_saved = app.session.sources.generation; } /// Keeps the sources with the preferences when they changed. diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index f8da5b5cb6..9dfb270796 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1457,6 +1457,54 @@ fn connections_persist_and_import_once() { assert!(empty.session.connections.connections.is_empty(), "an empty saved list must not import again"); } +/// Saved data sources from a newer RustGIS (#1405): rows, fields and lists this version can't read +/// are said at start and survive recording a recent source; a preference that can't be read at +/// all stays byte for byte. +#[test] +fn saved_sources_from_a_newer_version_survive_recording_a_recent_one() { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("points.geojson"); + std::fs::write( + &file, + r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1,2]},"properties":{}}]}"#, + ) + .unwrap(); + let future = json!({"kind": "gcs", "name": "Bucket", "address": "gs://b/a.tif"}); + let newer = json!({"favourites": [future.clone()], "recent": [{"kind": "file", "name": "R", "address": "/r.csv", "opened": 3}], "folders": ["/data"]}) + .to_string(); + for text in [newer, "[\"not\", \"an object\"]".to_string()] { + let store = Arc::new(Mutex::new(HashMap::from([ + ("data_sources".to_string(), text.clone()), + ("connections".to_string(), "{}".to_string()), + ]))); + let (load, save) = (store.clone(), store.clone()); + let services = rustgis_ui_egui::Services { + load_pref: Some(Box::new(move |k| load.lock().unwrap().get(k).cloned())), + save_pref: Some(Box::new(move |k, v| { + save.lock().unwrap().insert(k.to_string(), v.to_string()); + })), + ..Default::default() + }; + let mut app = RustGisApp::with_sample().with_services(services); + app.run("layer.addData", json!({"path": file.display().to_string()})).unwrap(); + assert!(app.session.sources.recent.iter().any(|r| r.address.ends_with("points.geojson")), "{text}"); + let said: Vec<_> = app.session.notices_since(0).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("saved data source")), "{said:?}"); + let stored = store.lock().unwrap()["data_sources"].clone(); + if text.starts_with('[') { + assert_eq!(stored, text, "a preference that can't be read stays as it was"); + } else { + let stored: serde_json::Value = serde_json::from_str(&stored).unwrap(); + assert_eq!(stored["favourites"], json!([future])); + assert_eq!(stored["folders"], json!(["/data"])); + assert!(stored["recent"][0]["address"].as_str().unwrap().ends_with("points.geojson")); + assert_eq!(stored["recent"][1]["opened"], 3); + } + } +} + /// Settings › Start-up › Start With a project (#377): the last one opened or saved, or one picked /// once. It opens with its own projection and view, a shell's `blank` still wins, one that no /// longer opens leaves the sample with a notice, and after a crash none is opened. From af475cf4634e1490093aca0c7a34fc5d94ec1a22 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 18:23:19 -0600 Subject: [PATCH 02/15] fix(settings): refuse before changing a plugin, keep a web plugin's unknown fields on update, say what cleaning cut from a source Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 87 ++++++++++++++++++++++++++++----- crates/plugins/src/store.rs | 25 +++++++--- crates/plugins/src/tests.rs | 7 +++ crates/ui-egui/src/pluginsui.rs | 25 +++++++--- 4 files changed, 119 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 00754ded9a..51c15edb77 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -73,18 +73,40 @@ struct Kept { /// Rows and fields that couldn't be read and held a password, key or token: not kept, since /// this version can't strip it from what it doesn't understand. secrets: usize, + /// Readable sources whose name or details [`clean`] cut down (a name past 200 characters, a + /// detail that isn't short text, more than eight, or one that looks secret): saved cut down. + cut: usize, } impl Kept { - /// The entries of `o` without a password, key or token, counting those left out. - fn without_secrets(&mut self, mut o: Map) -> Map { + /// The entries of `o` without a password, key or token, counting those left out. `address` is + /// the one the entries sit beside (a source's own, kept apart from them): a bare `key` is a + /// secret beside a web address, which one entry alone doesn't show. + fn without_secrets(&mut self, mut o: Map, address: &str) -> Map { let before = o.len(); o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v }))); + let mut beside = o.clone(); + beside.insert("address".into(), address.into()); + if holds_secret(&Value::Object(beside)) { + o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v, "address": "https://a" }))); + } self.secrets += before - o.len(); o } } +/// Whether [`clean`] took something of `before` away that isn't just tidied (whitespace, an +/// empty name or details). +fn cut(before: &Source, after: &Source) -> bool { + let name = before.name.trim(); + let extra = match &before.extra { + Value::Null => false, + Value::Object(o) => !o.is_empty() && before.extra != after.extra, + _ => true, + }; + (!name.is_empty() && after.name != name) || extra +} + /// A saved row: the source, and the fields this version doesn't know. #[derive(Deserialize)] struct Row { @@ -124,18 +146,22 @@ impl Sources { } } } - s.kept.top = s.kept.without_secrets(o); + s.kept.top = s.kept.without_secrets(o, ""); for (list, rows) in LISTS.into_iter().zip(lists) { let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; let mut into = Vec::new(); for row in rows { // Entry by entry: one that doesn't read is kept as it was, not the whole list. - let read = serde_json::from_value::(row.clone()).ok().and_then(|r| Some((clean(r.source).ok()?, r.other))); + let read = serde_json::from_value::(row.clone()).ok().and_then(|r| { + let cleaned = clean(r.source.clone()).ok()?; + Some((cut(&r.source, &cleaned), cleaned, r.other)) + }); match read { - Some((mut source, other)) if into.len() < cap => { + Some((was_cut, mut source, other)) if into.len() < cap => { + s.kept.cut += usize::from(was_cut); // Preferences carry provenance, never authority from an earlier process's login. crate::s3_connections::clear_bindings(std::iter::once(&mut source.address)); - let other = s.kept.without_secrets(other); + let other = s.kept.without_secrets(other, &source.address); if !other.is_empty() { s.kept.fields.insert((list, source.kind.clone(), source.address.clone()), other); } @@ -188,11 +214,20 @@ impl Sources { if n == 1 { " has" } else { "s have" } )); } + if k.cut > 0 { + let n = k.cut; + said.push(format!( + "{n} saved data source{} a name past 200 characters, or details this version can't keep (not short text, more than eight, or a password, key or token): cut down, and saved cut down", + if n == 1 { " has" } else { "s have" } + )); + } if k.secrets > 0 { let n = k.secrets; said.push(format!( - "{n} entr{} of the saved data sources couldn't be read and held a password, key or token: dropped, and not saved again", - if n == 1 { "y" } else { "ies" } + "{n} value{} in the saved data sources {} dropped because {} held a password, key or token", + if n == 1 { "" } else { "s" }, + if n == 1 { "was" } else { "were" }, + if n == 1 { "it" } else { "they" } )); } said @@ -925,11 +960,39 @@ MIIE {"kind": "gcs", "name": "B", "address": "https://g.example/b?sig=hidden"} ]}); let s = Sources::from_json(&secret.to_string()); - assert!(s.unread().iter().any(|t| t.starts_with("3 entries of the saved data sources")), "{:?}", s.unread()); + assert!( + s.unread() + .iter() + .any(|t| t.starts_with("3 values in the saved data sources were dropped because they held a password, key or token")), + "{:?}", + s.unread() + ); assert!(!s.to_json().contains("hidden")); - for clean in - ["", r#"{"favourites": [], "recent": [], "saved": []}"#, r#"{"recent": [{"kind": "file", "name": "R", "address": "/r.csv"}]}"#] - { + // A key beside a web address is one, in a readable source and among the preference's own keys. + let beside = json!({"endpoint": "https://api.example", "key": "abc", "saved": [ + {"kind": "url", "name": "A", "address": "https://a/x.fgb", "mirror": "https://m.example", "key": "abc"} + ]}); + let s = Sources::from_json(&beside.to_string()); + assert!( + !s.to_json().contains("abc") && s.to_json().contains("api.example") && s.to_json().contains("m.example"), + "{}", + s.to_json() + ); + assert!(s.unread().iter().any(|t| t.starts_with("2 values in the saved data sources were dropped")), "{:?}", s.unread()); + // Details and a name this version cuts down when it reads them are said. + let long = "n".repeat(300); + let cut = json!({"saved": [ + {"kind": "service", "name": "S", "address": "https://h/wms", "extra": {"type": "wms", "future": {"mode": 1}}}, + {"kind": "file", "name": long, "address": "/a.csv"} + ]}); + let s = Sources::from_json(&cut.to_string()); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("2 saved data sources have")), "{:?}", s.unread()); + for clean in [ + "", + r#"{"favourites": [], "recent": [], "saved": []}"#, + r#"{"recent": [{"kind": "file", "name": " R ", "address": "/r.csv", "extra": {}}]}"#, + r#"{"saved": [{"kind": "service", "name": "", "address": "https://h/wms", "extra": {"type": "wms"}}]}"#, + ] { assert!(Sources::from_json(clean).unread().is_empty(), "{clean}"); } } diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 7782bc4a1a..f1e4f16af5 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -76,13 +76,20 @@ fn read_state() -> State { read.unwrap_or_else(|e| State { unreadable: Some(format!("{}: {e}", path.display())), ..State::default() }) } -/// Writes `state.json`, refused when [`read_state`] couldn't read the one there. -fn write_state(st: &State) -> Result<(), String> { - if let Some(why) = &st.unreadable { - return Err(format!( +/// Refuses a change to `state.json` when [`read_state`] couldn't read the one there. Callers ask +/// before they change anything in memory, so a refused change isn't half made. +fn writable(st: &State) -> Result<(), String> { + match &st.unreadable { + Some(why) => Err(format!( "{why}; it is kept as it is, so which plugins are disabled or added from folders can't be changed until it is fixed or removed" - )); + )), + None => Ok(()), } +} + +/// Writes `state.json`, refused when [`read_state`] couldn't read the one there. +fn write_state(st: &State) -> Result<(), String> { + writable(st)?; let path = state_path().ok_or("there is no folder for plugins")?; if let Some(d) = path.parent() { std::fs::create_dir_all(d).map_err(|e| format!("{}: {e}", d.display()))?; @@ -324,9 +331,12 @@ pub fn install_path(path: &Path) -> Result, String> { return install_bytes(&bytes); } let pkg = Package::from_dir(path)?; - let plugin = crate::load(pkg, &path.display().to_string())?; let mut st = read_state(); let p = path.display().to_string(); + if !st.folders.contains(&p) { + writable(&st)?; + } + let plugin = crate::load(pkg, &p)?; if !st.folders.contains(&p) { st.folders.push(p); write_state(&st)?; @@ -360,8 +370,9 @@ pub fn uninstall(id: &str) -> Result<(), String> { /// Enables or disables a plugin, and remembers it. pub fn set_enabled(id: &str, enabled: bool) -> Result<(), String> { - crate::set_enabled(id, enabled)?; let mut st = read_state(); + writable(&st)?; + crate::set_enabled(id, enabled)?; st.disabled.retain(|d| d != id); if !enabled { st.disabled.push(id.to_string()); diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index ab16c49223..f5e0c1e52b 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1850,6 +1850,13 @@ fn plugin_state_keeps_what_this_version_cannot_read() { assert!(crate::store::load_installed().iter().any(|e| e.contains("state.json")), "{unreadable}"); let e = crate::store::set_enabled("state-test", true).unwrap_err(); assert!(e.contains("kept as it is"), "{e}"); + // A refused change leaves the plugin as it was, not changed for this session only. + assert!(crate::get("state-test").is_some_and(|p| !p.enabled)); + let folder = plugins.join("folder-plugin"); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::write(folder.join("plugin.json"), crate::testing::manifest("folder-test", &[])).unwrap(); + std::fs::write(folder.join("plugin.wasm"), testing::wasm()).unwrap(); + assert!(crate::store::install_path(&folder).is_err() && crate::get("folder-test").is_none()); assert!(crate::store::uninstall("state-test").is_err()); assert_eq!(std::fs::read_to_string(&state).unwrap(), unreadable); } diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 378aff65f5..03ca360e73 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -327,6 +327,18 @@ fn update_saved(app: &RustGisApp, f: impl FnOnce(&mut Vec)) { } } +/// Puts `entry` in `list` in place of the one with its id, which hands it the fields this version +/// doesn't know; `keep` false only removes the old one. +fn put_saved(list: &mut Vec, mut entry: Saved, keep: bool) { + if let Some(old) = list.iter().find(|s| s.id == entry.id) { + entry.other = old.other.clone(); + } + list.retain(|s| s.id != entry.id); + if keep { + list.push(entry); + } +} + /// Loads the plugins the web app installed in earlier visits. pub fn restore(app: &mut RustGisApp) { if !keeps_prefs() { @@ -527,12 +539,7 @@ fn install_confirmed(app: &mut RustGisApp) { other: serde_json::Map::new(), }; let kept = entry.url.is_some() || entry.data.is_some(); - update_saved(app, |list| { - list.retain(|s| s.id != entry.id); - if kept { - list.push(entry); - } - }); + update_saved(app, |list| put_saved(list, entry, kept)); let mut text = crate::i18n::fmt( "Installed {name} {version}. Find it on the Plugins tab; its tools are also in the Tools pane.", &[("name", &p.manifest.name), ("version", &p.manifest.version)], @@ -1347,6 +1354,12 @@ mod tests { assert_eq!(again[0]["enabled"], false); assert_eq!(again[1]["id"], "c"); assert_eq!(again[2], json!({"id": "b", "sha256": {"v2": "0"}})); + // Installing it again (an update) keeps the fields this version doesn't know. + put_saved(&mut list, record("a", "2"), true); + assert_eq!( + (list.len(), list.last().map(|s| s.sha256.as_str()), list.last().and_then(|s| s.other.get("channel"))), + (2, Some("2"), Some(&json!("beta"))) + ); assert!(read_saved(Some("{\"plugins\": []}")).is_none()); assert_eq!(read_saved(None).map(|(l, k)| (l.len(), k.len())), Some((0, 0))); } From 5c1205a01dde67a45274c4555f23e6acd480b9d9 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 19:43:46 -0600 Subject: [PATCH 03/15] fix(settings): say what cleaning changed in an address, keep stored sources a full list pushes out and each twin's fields, say an unsaved plugin change Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 118 +++++++++++++++++++++++--------- crates/ui-egui/src/pluginsui.rs | 13 +++- 2 files changed, 97 insertions(+), 34 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 51c15edb77..7c87840153 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -73,9 +73,13 @@ struct Kept { /// Rows and fields that couldn't be read and held a password, key or token: not kept, since /// this version can't strip it from what it doesn't understand. secrets: usize, - /// Readable sources whose name or details [`clean`] cut down (a name past 200 characters, a - /// detail that isn't short text, more than eight, or one that looks secret): saved cut down. + /// Readable sources whose name, address or details [`clean`] changed (a name too long or with + /// control characters, an address with a password, key or token, details that aren't short + /// text, more than eight, or secret-looking): saved that way. cut: usize, + /// Readable rows that repeat the kind and address of an earlier row of their list: kept as + /// they were, not listed, so each row keeps its own unknown fields. + twins: usize, } impl Kept { @@ -86,7 +90,9 @@ impl Kept { let before = o.len(); o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v }))); let mut beside = o.clone(); - beside.insert("address".into(), address.into()); + if !address.is_empty() { + beside.insert("address".into(), address.into()); + } if holds_secret(&Value::Object(beside)) { o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v, "address": "https://a" }))); } @@ -95,8 +101,8 @@ impl Kept { } } -/// Whether [`clean`] took something of `before` away that isn't just tidied (whitespace, an -/// empty name or details). +/// Whether [`clean`] changed something of `before` that isn't just tidied (whitespace, an empty +/// name or details). fn cut(before: &Source, after: &Source) -> bool { let name = before.name.trim(); let extra = match &before.extra { @@ -104,7 +110,7 @@ fn cut(before: &Source, after: &Source) -> bool { Value::Object(o) => !o.is_empty() && before.extra != after.extra, _ => true, }; - (!name.is_empty() && after.name != name) || extra + (!name.is_empty() && after.name != name) || after.address != before.address.trim() || extra } /// A saved row: the source, and the fields this version doesn't know. @@ -157,6 +163,10 @@ impl Sources { Some((cut(&r.source, &cleaned), cleaned, r.other)) }); match read { + Some((_, source, _)) if into.len() < cap && into.iter().any(|r| same(r, &source)) && !holds_secret(&row) => { + s.kept.twins += 1; + s.kept.rows.entry(list).or_default().push(row); + } Some((was_cut, mut source, other)) if into.len() < cap => { s.kept.cut += usize::from(was_cut); // Preferences carry provenance, never authority from an earlier process's login. @@ -191,7 +201,15 @@ impl Sources { )]; } let mut said = Vec::new(); - let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over); + let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over + k.twins); + if k.twins > 0 { + let n = k.twins; + said.push(format!( + "{n} saved data source{} another in the same list: not listed, and kept as {} when the sources are saved", + if n == 1 { " repeats" } else { "s repeat" }, + if n == 1 { "it was" } else { "they were" } + )); + } if k.over > 0 { let n = k.over; said.push(format!( @@ -217,7 +235,7 @@ impl Sources { if k.cut > 0 { let n = k.cut; said.push(format!( - "{n} saved data source{} a name past 200 characters, or details this version can't keep (not short text, more than eight, or a password, key or token): cut down, and saved cut down", + "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key or token, details that aren't short text, more than eight, or secret-looking): saved changed", if n == 1 { " has" } else { "s have" } )); } @@ -236,21 +254,31 @@ impl Sources { /// Takes in the sources kept with the preferences, keeping what this session already has: /// in-memory entries stay (and first), stored ones are added after them unless the same source /// ([`Source::is`]) is there already. - pub fn merge(&mut self, stored: Sources) { - fn join(now: &mut Vec, stored: Vec, cap: usize) { - for s in stored { - if now.len() >= cap { - break; + pub fn merge(&mut self, mut stored: Sources) { + // Stored sources that no longer fit (a change before the first read filled the list) are + // kept as rows, so they are said and written back. + let mut over = Vec::new(); + for list in LISTS { + let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; + let Ok(incoming) = stored.list_mut(list).map(std::mem::take) else { continue }; + let Ok(now) = self.list_mut(list) else { continue }; + for source in incoming { + if now.iter().any(|n| same(n, &source)) { + continue; } - if !now.iter().any(|n| n.is(&s.kind, &s.address)) { - now.push(s); + if now.len() < cap { + now.push(source); + } else { + over.push((list, stored.row(list, &source))); + stored.kept.fields.remove(&(list, source.kind.clone(), source.address.clone())); } } } - join(&mut self.favourites, stored.favourites, MAX_KEPT); - join(&mut self.saved, stored.saved, MAX_KEPT); - join(&mut self.recent, stored.recent, MAX_RECENT); self.kept = stored.kept; + self.kept.over += over.len(); + for (list, row) in over { + self.kept.rows.entry(list).or_default().push(row); + } self.generation += 1; } @@ -262,25 +290,25 @@ impl Sources { } let mut o = self.kept.top.clone(); for (list, sources) in LISTS.into_iter().zip([&self.favourites, &self.recent, &self.saved]) { - let mut rows: Vec = sources - .iter() - .map(|source| { - let mut row = serde_json::to_value(source).unwrap_or(Value::Null); - let key = (list, source.kind.clone(), source.address.clone()); - if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&key)) { - for (k, v) in other { - row.entry(k.clone()).or_insert_with(|| v.clone()); - } - } - row - }) - .collect(); + let mut rows: Vec = sources.iter().map(|source| self.row(list, source)).collect(); rows.extend(self.kept.rows.get(list).into_iter().flatten().cloned()); o.insert(list.into(), Value::Array(rows)); } Value::Object(o).to_string() } + /// A source as a preference row: with the fields kept for it that this version doesn't know. + fn row(&self, list: &'static str, source: &Source) -> Value { + let mut row = serde_json::to_value(source).unwrap_or(Value::Null); + let key = (list, source.kind.clone(), source.address.clone()); + if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&key)) { + for (k, v) in other { + row.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + row + } + /// Notes a source just added (the most recent first, each once). pub fn note(&mut self, s: Source) { let Ok(s) = clean(s) else { return }; @@ -979,6 +1007,31 @@ MIIE s.to_json() ); assert!(s.unread().iter().any(|t| t.starts_with("2 values in the saved data sources were dropped")), "{:?}", s.unread()); + // A key beside an address among the preference's own keys. + let top = json!({"address": "https://api.example", "key": "abc"}); + assert!(!Sources::from_json(&top.to_string()).to_json().contains("abc")); + // A change before the first read that fills a list doesn't drop what was stored. + let stored: Vec = + (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "pinned": i})).collect(); + let mut s = Sources::default(); + s.keep("favourites", Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }).unwrap(); + s.merge(Sources::from_json(&json!({ "favourites": stored }).to_string())); + assert_eq!(s.favourites.len(), MAX_KEPT); + assert!(s.unread()[0].starts_with("1 saved data source is past"), "{:?}", s.unread()); + assert!(s.to_json().contains(&format!("/{}.csv", MAX_KEPT - 1)) && s.to_json().contains(&format!("\"pinned\":{}", MAX_KEPT - 1))); + // Two rows of one identity each keep their own fields. + let twins = json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "future": 1}, + {"kind": "file", "name": "A", "address": "/a.csv", "future": 2} + ]}); + let s = Sources::from_json(&twins.to_string()); + let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); + assert_eq!((&saved["saved"][0]["future"], &saved["saved"][1]["future"]), (&json!(1), &json!(2))); + assert!(s.unread()[0].starts_with("1 saved data source repeats"), "{:?}", s.unread()); + // An address with a signed query is changed, and said. + let signed = json!({"saved": [{"kind": "url", "name": "U", "address": "https://a/x.fgb?sig=hidden"}]}); + let s = Sources::from_json(&signed.to_string()); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source has")), "{:?}", s.unread()); // Details and a name this version cuts down when it reads them are said. let long = "n".repeat(300); let cut = json!({"saved": [ @@ -1215,7 +1268,8 @@ MIIE assert_eq!(back.favourites.len(), 1); assert_eq!(back.favourites[0].address, "https://h/x.fgb"); assert!(back.recent.is_empty()); - assert_eq!(back.unread().len(), 1, "{:?}", back.unread()); + // The unreadable rows, and the address cut of its token. + assert_eq!(back.unread().len(), 2, "{:?}", back.unread()); let unreadable = Sources::from_json("not json"); assert!(unreadable.favourites.is_empty() && unreadable.recent.is_empty() && unreadable.saved.is_empty()); } diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 03ca360e73..068be4fc17 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -97,6 +97,8 @@ pub struct PluginsState { /// Ids of plugins that failed to load at start, not yet told (one notice for all, once every /// start download is done). start_failures: Vec, + /// Whether this session was told that changes to the installed plugins aren't remembered. + unsaved_told: bool, } /// A plugin's Python environment being made on a background thread. @@ -315,12 +317,19 @@ fn saved_text(list: &[Saved], kept: &[Value]) -> String { /// Changes the installed plugins the web app keeps with `f`, writing back the entries it couldn't /// read as they were; nothing on the desktop, or when the preference couldn't be read. -fn update_saved(app: &RustGisApp, f: impl FnOnce(&mut Vec)) { +fn update_saved(app: &mut RustGisApp, f: impl FnOnce(&mut Vec)) { if !keeps_prefs() { return; } // One that can't be read was said at start, and isn't written over. - let Some((mut list, kept)) = saved(app) else { return }; + let Some((mut list, kept)) = saved(app) else { + if !std::mem::replace(&mut app.plugins.unsaved_told, true) { + app.session.warn(tl!( + "The plugins installed in this browser can't be read, so this change isn't remembered: fix or remove \"plugins_web\" in the browser's saved preferences." + )); + } + return; + }; f(&mut list); if let Some(s) = &app.services.save_pref { s("plugins_web", &saved_text(&list, &kept)); From 7a72ab7ca20ec98e806d1b2ca1407aa655d339f6 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 20:37:38 -0600 Subject: [PATCH 04/15] fix(settings): classify a source's unknown fields on the whole object before removing any Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 7c87840153..ffad54b146 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -83,24 +83,41 @@ struct Kept { } impl Kept { - /// The entries of `o` without a password, key or token, counting those left out. `address` is - /// the one the entries sit beside (a source's own, kept apart from them): a bare `key` is a - /// secret beside a web address, which one entry alone doesn't show. + /// The entries of `o` without a password, key or token, counting those left out. Every entry is + /// classified on the original object first, beside the others and `address` (the one a row + /// keeps apart from them): a bare `key` is a secret beside a web address, and an address + /// holding a secret doesn't stop being one. Only then are the classified entries removed. fn without_secrets(&mut self, mut o: Map, address: &str) -> Map { - let before = o.len(); - o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v }))); - let mut beside = o.clone(); - if !address.is_empty() { - beside.insert("address".into(), address.into()); - } - if holds_secret(&Value::Object(beside)) { - o.retain(|k, v| !holds_secret(&json!({ k.as_str(): v, "address": "https://a" }))); - } - self.secrets += before - o.len(); + let secret: Vec = o + .iter() + .filter(|(k, v)| { + let mut probe = Map::new(); + probe.insert((*k).clone(), (*v).clone()); + if web_address(&Value::String(address.into()), 0) || o.iter().any(|(other, v)| other != *k && web_address(v, 0)) { + probe.insert("\u{1}beside".into(), "https://a".into()); + } + holds_secret(&Value::Object(probe)) + }) + .map(|(k, _)| k.clone()) + .collect(); + self.secrets += secret.len(); + o.retain(|k, _| !secret.contains(k)); o } } +/// Whether `v` is, or holds, a web address (not under a file's `data` or `text`). +fn web_address(v: &Value, depth: usize) -> bool { + match v { + Value::String(t) => t.trim_start().get(..4).is_some_and(|p| p.eq_ignore_ascii_case("http")), + Value::Object(o) if depth < 4 => { + o.iter().any(|(k, v)| !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, depth + 1)) + } + Value::Array(a) if depth < 4 => a.iter().any(|v| web_address(v, depth + 1)), + _ => false, + } +} + /// Whether [`clean`] changed something of `before` that isn't just tidied (whitespace, an empty /// name or details). fn cut(before: &Source, after: &Source) -> bool { @@ -1010,6 +1027,10 @@ MIIE // A key beside an address among the preference's own keys. let top = json!({"address": "https://api.example", "key": "abc"}); assert!(!Sources::from_json(&top.to_string()).to_json().contains("abc")); + // Even when that address holds a secret itself. + let top = json!({"address": "https://api.example?token=hidden", "key": "abc"}); + let saved = Sources::from_json(&top.to_string()).to_json(); + assert!(!saved.contains("abc") && !saved.contains("hidden"), "{saved}"); // A change before the first read that fills a list doesn't drop what was stored. let stored: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "pinned": i})).collect(); From 6525bd4da83684aec28e70131408e89a2d010dcf Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 21:33:33 -0600 Subject: [PATCH 05/15] fix(settings): a source listed twice is one source, merged at load; a file's contents aren't an address for a field Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 58 +++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index ffad54b146..7cd4456085 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -77,9 +77,9 @@ struct Kept { /// control characters, an address with a password, key or token, details that aren't short /// text, more than eight, or secret-looking): saved that way. cut: usize, - /// Readable rows that repeat the kind and address of an earlier row of their list: kept as - /// they were, not listed, so each row keeps its own unknown fields. - twins: usize, + /// Readable rows that repeat the kind and address of an earlier row of their list: merged + /// into it (a list is keyed by that identity), keeping unknown fields it lacks. + merged: usize, } impl Kept { @@ -93,7 +93,11 @@ impl Kept { .filter(|(k, v)| { let mut probe = Map::new(); probe.insert((*k).clone(), (*v).clone()); - if web_address(&Value::String(address.into()), 0) || o.iter().any(|(other, v)| other != *k && web_address(v, 0)) { + if web_address(&Value::String(address.into()), 0) + || o.iter().any(|(other, v)| { + other != *k && !other.eq_ignore_ascii_case("data") && !other.eq_ignore_ascii_case("text") && web_address(v, 0) + }) + { probe.insert("\u{1}beside".into(), "https://a".into()); } holds_secret(&Value::Object(probe)) @@ -180,19 +184,23 @@ impl Sources { Some((cut(&r.source, &cleaned), cleaned, r.other)) }); match read { - Some((_, source, _)) if into.len() < cap && into.iter().any(|r| same(r, &source)) && !holds_secret(&row) => { - s.kept.twins += 1; - s.kept.rows.entry(list).or_default().push(row); - } Some((was_cut, mut source, other)) if into.len() < cap => { s.kept.cut += usize::from(was_cut); // Preferences carry provenance, never authority from an earlier process's login. crate::s3_connections::clear_bindings(std::iter::once(&mut source.address)); let other = s.kept.without_secrets(other, &source.address); + let repeat = into.iter().any(|r| same(r, &source)); + s.kept.merged += usize::from(repeat); if !other.is_empty() { - s.kept.fields.insert((list, source.kind.clone(), source.address.clone()), other); + // A repeated row adds the unknown fields the first lacks. + let kept = s.kept.fields.entry((list, source.kind.clone(), source.address.clone())).or_default(); + for (k, v) in other { + kept.entry(k).or_insert(v); + } + } + if !repeat { + into.push(source); } - into.push(source); } _ if holds_secret(&row) => s.kept.secrets += 1, read => { @@ -218,13 +226,12 @@ impl Sources { )]; } let mut said = Vec::new(); - let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over + k.twins); - if k.twins > 0 { - let n = k.twins; + let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over); + if k.merged > 0 { + let n = k.merged; said.push(format!( - "{n} saved data source{} another in the same list: not listed, and kept as {} when the sources are saved", - if n == 1 { " repeats" } else { "s repeat" }, - if n == 1 { "it was" } else { "they were" } + "{n} saved data source{} another in the same list: merged into it, keeping its settings and adding the unknown ones it lacks", + if n == 1 { " repeats" } else { "s repeat" } )); } if k.over > 0 { @@ -1040,15 +1047,26 @@ MIIE assert_eq!(s.favourites.len(), MAX_KEPT); assert!(s.unread()[0].starts_with("1 saved data source is past"), "{:?}", s.unread()); assert!(s.to_json().contains(&format!("/{}.csv", MAX_KEPT - 1)) && s.to_json().contains(&format!("\"pinned\":{}", MAX_KEPT - 1))); - // Two rows of one identity each keep their own fields. + // Rows of one identity are one source: the first row's fields stay, a later row adds the + // ones it lacks (without a secret), and forgetting it leaves nothing behind. let twins = json!({"saved": [ {"kind": "file", "name": "A", "address": "/a.csv", "future": 1}, - {"kind": "file", "name": "A", "address": "/a.csv", "future": 2} + {"kind": "file", "name": "A", "address": "/a.csv", "future": 2, "more": 3, "token": "hidden"} ]}); - let s = Sources::from_json(&twins.to_string()); + let mut s = Sources::from_json(&twins.to_string()); + assert_eq!(s.saved.len(), 1); let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); - assert_eq!((&saved["saved"][0]["future"], &saved["saved"][1]["future"]), (&json!(1), &json!(2))); + assert_eq!( + (saved["saved"].as_array().map(Vec::len), &saved["saved"][0]["future"], &saved["saved"][0]["more"]), + (Some(1), &json!(1), &json!(3)) + ); + assert!(!s.to_json().contains("hidden")); assert!(s.unread()[0].starts_with("1 saved data source repeats"), "{:?}", s.unread()); + s.forget("saved", "file", "/a.csv").unwrap(); + assert!(!s.to_json().contains("/a.csv")); + // A file's contents beside a field isn't an address for it. + let file = json!({"data": "https://example.invalid/a.csv", "key": "feature_id"}); + assert!(Sources::from_json(&file.to_string()).to_json().contains("feature_id")); // An address with a signed query is changed, and said. let signed = json!({"saved": [{"kind": "url", "name": "U", "address": "https://a/x.fgb?sig=hidden"}]}); let s = Sources::from_json(&signed.to_string()); From ea1a7f2b38db4f0864cdf2e57078d923d865af9d Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 08:17:20 -0600 Subject: [PATCH 06/15] fix(engine,assistant): count cleared login bindings, refuse odd store versions, merge repeats before the limit A login binding cleared from a saved address is counted in the unread notice. A sign-in store whose version isn't a whole number is refused as unreadable. Rows repeating a source merge before the list limit applies, and the merged fields are checked for secrets together. Co-Authored-By: Claude Sonnet 5.5 --- crates/assistant/src/chatgpt.rs | 26 +++++++++++---- crates/engine/src/sources.rs | 59 +++++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/crates/assistant/src/chatgpt.rs b/crates/assistant/src/chatgpt.rs index 3939bc2789..35eed912f7 100644 --- a/crates/assistant/src/chatgpt.rs +++ b/crates/assistant/src/chatgpt.rs @@ -850,13 +850,18 @@ mod native { let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|_| "the sign-in store isn't JSON")?; // One a newer RustGIS wrote may hold what this version would lose by writing it again: it // is neither used nor written over. - if let Some(version) = v.get("version").and_then(Value::as_u64).filter(|v| *v > STORE_VERSION) { - return Err(Error::new( - Recovery::Auth, - format!( - "the ChatGPT sign-ins were saved by a newer RustGIS (version {version}; this one reads {STORE_VERSION}): use that RustGIS, or update this one" - ), - )); + match v.get("version").map(|version| version.as_u64().ok_or(version)) { + Some(Ok(version)) if version > STORE_VERSION => { + return Err(Error::new( + Recovery::Auth, + format!( + "the ChatGPT sign-ins were saved by a newer RustGIS (version {version}; this one reads {STORE_VERSION}): use that RustGIS, or update this one" + ), + )); + } + // A version that isn't a plain integer can't be told from a newer one's. + Some(Err(_)) => return Err("the sign-in store's version isn't a whole number".into()), + _ => {} } if let Some(entries) = v.get("accounts").and_then(Value::as_array) { let entries = entries.iter().map(Credentials::from_json).collect::, _>>()?; @@ -2061,6 +2066,13 @@ mod native { assert!(sign_out_at(&path, &v).is_err()); assert!(remove_at(&path, &v, "oaiapp_synthetic", "user-1").is_err()); assert_eq!(std::fs::read_to_string(&path).unwrap(), text); + // A version that isn't a plain integer is unreadable, not newer-or-not. + for odd in [serde_json::json!("3"), serde_json::json!(3.0), serde_json::json!(true)] { + let mut v: Value = serde_json::from_str(&text).unwrap(); + v["version"] = odd; + std::fs::write(&path, v.to_string()).unwrap(); + assert!(read_accounts(&path).is_err()); + } } #[test] diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 7cd4456085..c89df7024c 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -74,7 +74,7 @@ struct Kept { /// this version can't strip it from what it doesn't understand. secrets: usize, /// Readable sources whose name, address or details [`clean`] changed (a name too long or with - /// control characters, an address with a password, key or token, details that aren't short + /// control characters, an address with a password, key, token or login binding, details that aren't short /// text, more than eight, or secret-looking): saved that way. cut: usize, /// Readable rows that repeat the kind and address of an earlier row of their list: merged @@ -180,22 +180,31 @@ impl Sources { for row in rows { // Entry by entry: one that doesn't read is kept as it was, not the whole list. let read = serde_json::from_value::(row.clone()).ok().and_then(|r| { - let cleaned = clean(r.source.clone()).ok()?; + let mut cleaned = clean(r.source.clone()).ok()?; + // Preferences carry provenance, never authority from an earlier process's login; + // clearing it is a change `cut` counts. + crate::s3_connections::clear_bindings(std::iter::once(&mut cleaned.address)); Some((cut(&r.source, &cleaned), cleaned, r.other)) }); + // A repeat of a listed source merges even at the limit: it adds no row. + let repeat = read.as_ref().is_some_and(|(_, source, _)| into.iter().any(|r| same(r, source))); match read { - Some((was_cut, mut source, other)) if into.len() < cap => { + Some((was_cut, source, other)) if repeat || into.len() < cap => { s.kept.cut += usize::from(was_cut); - // Preferences carry provenance, never authority from an earlier process's login. - crate::s3_connections::clear_bindings(std::iter::once(&mut source.address)); let other = s.kept.without_secrets(other, &source.address); - let repeat = into.iter().any(|r| same(r, &source)); s.kept.merged += usize::from(repeat); if !other.is_empty() { - // A repeated row adds the unknown fields the first lacks. - let kept = s.kept.fields.entry((list, source.kind.clone(), source.address.clone())).or_default(); + // A repeated row adds the unknown fields the first lacks; the combined + // fields are classified again, since a key beside a web address + // that another row held is a secret only together. + let key = (list, source.kind.clone(), source.address.clone()); + let mut merged = s.kept.fields.remove(&key).unwrap_or_default(); for (k, v) in other { - kept.entry(k).or_insert(v); + merged.entry(k).or_insert(v); + } + let merged = if repeat { s.kept.without_secrets(merged, &source.address) } else { merged }; + if !merged.is_empty() { + s.kept.fields.insert(key, merged); } } if !repeat { @@ -259,7 +268,7 @@ impl Sources { if k.cut > 0 { let n = k.cut; said.push(format!( - "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key or token, details that aren't short text, more than eight, or secret-looking): saved changed", + "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key, token or login binding, details that aren't short text, more than eight, or secret-looking): saved changed", if n == 1 { " has" } else { "s have" } )); } @@ -1064,6 +1073,36 @@ MIIE assert!(s.unread()[0].starts_with("1 saved data source repeats"), "{:?}", s.unread()); s.forget("saved", "file", "/a.csv").unwrap(); assert!(!s.to_json().contains("/a.csv")); + // Fields that are fine in each row but a secret together are dropped, in either order. + for rows in [ + [json!({"endpoint": "https://api.example"}), json!({"key": "abc"})], + [json!({"key": "abc"}), json!({"endpoint": "https://api.example"})], + ] { + let rows: Vec = rows + .into_iter() + .map(|mut r| { + r.as_object_mut().into_iter().for_each(|o| { + o.extend([("kind".into(), json!("file")), ("name".into(), json!("A")), ("address".into(), json!("/a.csv"))]) + }); + r + }) + .collect(); + let saved = Sources::from_json(&json!({"saved": rows}).to_string()).to_json(); + assert!(!saved.contains("abc") && saved.contains("api.example"), "{saved}"); + } + // A repeat past the limit merges into its source, whether or not it holds a secret. + let mut full: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + full.push(json!({"kind": "file", "name": "", "address": "/0.csv", "token": "hidden", "future": 1})); + full.push(json!({"kind": "file", "name": "", "address": "/1.csv"})); + let mut s = Sources::from_json(&json!({"saved": full}).to_string()); + assert!(!s.to_json().contains("hidden") && s.to_json().contains("\"future\":1"), "{}", s.to_json()); + assert_eq!(s.kept.over, 0); + s.forget("saved", "file", "/1.csv").unwrap(); + assert!(!s.to_json().contains("/1.csv")); + // A login binding removed from an address is said. + let bound = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + let s = Sources::from_json(&json!({"saved": [{"kind": "url", "name": "B", "address": bound}]}).to_string()); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source has")), "{:?}", s.unread()); // A file's contents beside a field isn't an address for it. let file = json!({"data": "https://example.invalid/a.csv", "key": "feature_id"}); assert!(Sources::from_json(&file.to_string()).to_json().contains("feature_id")); From add3f837c51d851e8bbeb3fbf530d5b574990fe5 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 08:40:09 -0600 Subject: [PATCH 07/15] fix(plugins): load no plugins while state.json can't be read Which plugins are disabled is in that file, and one it can't read may have been a native program. The notice names the file and says repairing or removing it brings plugins back. Co-Authored-By: Claude Sonnet 5.5 --- crates/plugins/src/store.rs | 9 ++++++--- crates/plugins/src/tests.rs | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index f1e4f16af5..eaa5f54235 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -64,8 +64,8 @@ fn state_path() -> Option { } /// `state.json`: none is no plugin disabled or added from a folder. One that can't be read (or that -/// a newer RustGIS wrote in a shape this one doesn't know) is said by [`load_installed`] and kept as -/// it is: [`write_state`] refuses to write over it. +/// a newer RustGIS wrote in a shape this one doesn't know) is said by [`load_installed`], which then +/// loads no plugin, and kept as it is: [`write_state`] refuses to write over it. fn read_state() -> State { let Some(path) = state_path() else { return State::default() }; let read = match std::fs::read_to_string(&path) { @@ -122,10 +122,13 @@ pub fn load_installed() -> Vec { )), } let st = read_state(); + // Which plugins are disabled is in that file, and a disabled one may be a native program: with + // it unreadable none is loaded, rather than starting what the user turned off. if let Some(why) = &st.unreadable { errors.push(format!( - "{why}; it was ignored and is kept as it is, so plugins it disables are on and plugin folders it adds aren't loaded" + "{why}: plugins are off because this file couldn't be read; repairing or removing it brings them back (it is kept as it is until then)" )); + return errors; } let mut paths: Vec = std::fs::read_dir(&dir) .map(|rd| rd.flatten().map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("rgplugin"))).collect()) diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index 3334288af1..4b123a09c6 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1875,9 +1875,19 @@ fn plugin_state_keeps_what_this_version_cannot_read() { assert!(crate::store::install_path(&folder).is_err() && crate::get("folder-test").is_none()); assert!(crate::store::uninstall("state-test").is_err()); assert_eq!(std::fs::read_to_string(&state).unwrap(), unreadable); + // At start none is loaded (it may have been disabled, and be a native program), and it is said. + crate::unload("state-test"); + let errors = crate::store::load_installed(); + assert!(errors.len() == 1 && errors[0].contains("state.json") && errors[0].contains("plugins are off"), "{errors:?}"); + assert!(crate::get("state-test").is_none()); + std::fs::write(&state, r#"{"disabled": ["state-test"]}"#).unwrap(); + assert!(crate::store::load_installed().is_empty()); } std::fs::remove_file(&state).unwrap(); + // No file is nothing disabled, with no notice. + crate::unload("state-test"); assert!(crate::store::load_installed().is_empty()); + assert!(crate::get("state-test").is_some_and(|p| p.enabled)); crate::store::uninstall("state-test").unwrap(); let _ = std::fs::remove_dir_all(&plugins); } From d81b76d0c421b7ef9d11e3cabb2b17c728f31b51 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 10:02:14 -0600 Subject: [PATCH 08/15] fix(engine,ui): forgetting a source drops its kept over-limit row; context and notices for kept sources Forgetting a source also drops a readable row kept past the list's limit with the same identity. Unknown fields are classified beside the row's own details (extra). The notice for sources moved past a limit by the first-read merge is said after the merge. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 44 +++++++++++++++++++++++++++---- crates/ui-egui/src/sourcespane.rs | 5 ++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index c89df7024c..93acfe2a93 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -86,14 +86,16 @@ impl Kept { /// The entries of `o` without a password, key or token, counting those left out. Every entry is /// classified on the original object first, beside the others and `address` (the one a row /// keeps apart from them): a bare `key` is a secret beside a web address, and an address - /// holding a secret doesn't stop being one. Only then are the classified entries removed. - fn without_secrets(&mut self, mut o: Map, address: &str) -> Map { + /// holding a secret doesn't stop being one; `known` is the row's known details (its `extra`), a + /// sibling too. Only then are the classified entries removed. + fn without_secrets(&mut self, mut o: Map, address: &str, known: &Value) -> Map { let secret: Vec = o .iter() .filter(|(k, v)| { let mut probe = Map::new(); probe.insert((*k).clone(), (*v).clone()); if web_address(&Value::String(address.into()), 0) + || web_address(known, 0) || o.iter().any(|(other, v)| { other != *k && !other.eq_ignore_ascii_case("data") && !other.eq_ignore_ascii_case("text") && web_address(v, 0) }) @@ -173,7 +175,7 @@ impl Sources { } } } - s.kept.top = s.kept.without_secrets(o, ""); + s.kept.top = s.kept.without_secrets(o, "", &Value::Null); for (list, rows) in LISTS.into_iter().zip(lists) { let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; let mut into = Vec::new(); @@ -191,7 +193,7 @@ impl Sources { match read { Some((was_cut, source, other)) if repeat || into.len() < cap => { s.kept.cut += usize::from(was_cut); - let other = s.kept.without_secrets(other, &source.address); + let other = s.kept.without_secrets(other, &source.address, &source.extra); s.kept.merged += usize::from(repeat); if !other.is_empty() { // A repeated row adds the unknown fields the first lacks; the combined @@ -202,7 +204,7 @@ impl Sources { for (k, v) in other { merged.entry(k).or_insert(v); } - let merged = if repeat { s.kept.without_secrets(merged, &source.address) } else { merged }; + let merged = if repeat { s.kept.without_secrets(merged, &source.address, &source.extra) } else { merged }; if !merged.is_empty() { s.kept.fields.insert(key, merged); } @@ -381,6 +383,18 @@ impl Sources { let gone = l.len() != before; if let Some(list) = LISTS.into_iter().find(|l| *l == list) { self.kept.fields.remove(&(list, kind.to_string(), address.to_string())); + // A readable row kept past the list's limit of the same source would be listed again + // by the next start. + if let Some(rows) = self.kept.rows.get_mut(list) { + let text = |r: &Value, k: &str| r.get(k).and_then(Value::as_str).map(|s| s.trim().to_string()); + let mut dropped = 0; + rows.retain(|r| { + let same = text(r, "kind").as_deref() == Some(kind) && text(r, "address").as_deref() == Some(address.trim()); + dropped += usize::from(same && serde_json::from_value::(r.clone()).is_ok()); + !same + }); + self.kept.over = self.kept.over.saturating_sub(dropped); + } } if gone { self.generation += 1; @@ -1090,6 +1104,26 @@ MIIE let saved = Sources::from_json(&json!({"saved": rows}).to_string()).to_json(); assert!(!saved.contains("abc") && saved.contains("api.example"), "{saved}"); } + // A readable row past the limit is kept raw; once the source is listed and forgotten it + // doesn't come back with the next start. + let recent: Vec = (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + let mut s = Sources::from_json(&json!({ "recent": recent }).to_string()); + assert!(s.to_json().contains(&format!("/{MAX_RECENT}.csv"))); + s.forget("recent", "file", "/0.csv").unwrap(); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + s.forget("recent", "file", &format!("/{MAX_RECENT}.csv")).unwrap(); + assert!(!s.to_json().contains(&format!("/{MAX_RECENT}.csv")), "{}", s.to_json()); + assert!(s.unread().iter().all(|t| !t.contains("past the lists' limits")), "{:?}", s.unread()); + // The row's known details count beside an unknown key, as its address does. + let extra = json!({"saved": [{"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}, "key": "abc"}]}); + let saved = Sources::from_json(&extra.to_string()).to_json(); + assert!(!saved.contains("abc") && saved.contains("api.example"), "{saved}"); + // Sources merged past the limit are said by the merged sources. + let mut s = Sources::default(); + s.keep("favourites", Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }).unwrap(); + let stored: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + s.merge(Sources::from_json(&json!({ "favourites": stored }).to_string())); + assert!(s.unread().iter().any(|t| t.contains("past the lists' limits")), "{:?}", s.unread()); // A repeat past the limit merges into its source, whether or not it holds a secret. let mut full: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); full.push(json!({"kind": "file", "name": "", "address": "/0.csv", "token": "hidden", "future": 1})); diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 6610258a9b..8fad253834 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -60,10 +60,11 @@ pub fn load(app: &mut RustGisApp) { // What this session already has (added, saved or made favourite before the first read) // stays; the stored lists are added after it. What of them isn't used here is said. let stored = rustgis_engine::sources::Sources::from_json(&text); - for said in stored.unread() { + app.session.sources.merge(stored); + // After the merge: sources it moved past a list's limit are said too. + for said in app.session.sources.unread() { app.session.warn(said); } - app.session.sources.merge(stored); } if !changed { app.ui.sources_saved = app.session.sources.generation; From 54e3bc658f3721b41e5bfb2a455bb027363dad52 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 11:11:32 -0600 Subject: [PATCH 09/15] refactor(engine): one identity for saved sources; a repeated row is dropped, not merged Rows, lists, forget, note and keep compare a source by kind and its address with a login binding cleared. The first row of an identity wins and later ones are counted in one notice. Forgetting removes only readable kept rows with that identity and saves when only those went. A cleared login binding is the app's own output and is no longer counted. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 204 ++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 100 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 93acfe2a93..14af3f44d5 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -74,12 +74,12 @@ struct Kept { /// this version can't strip it from what it doesn't understand. secrets: usize, /// Readable sources whose name, address or details [`clean`] changed (a name too long or with - /// control characters, an address with a password, key, token or login binding, details that aren't short + /// control characters, an address with a password, key or token, details that aren't short /// text, more than eight, or secret-looking): saved that way. cut: usize, - /// Readable rows that repeat the kind and address of an earlier row of their list: merged - /// into it (a list is keyed by that identity), keeping unknown fields it lacks. - merged: usize, + /// Readable rows that repeat the identity of an earlier row of their list: dropped (the first + /// one wins, with its unknown fields). + repeated: usize, } impl Kept { @@ -178,40 +178,29 @@ impl Sources { s.kept.top = s.kept.without_secrets(o, "", &Value::Null); for (list, rows) in LISTS.into_iter().zip(lists) { let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; - let mut into = Vec::new(); + let mut into: Vec = Vec::new(); + let mut seen = std::collections::HashSet::new(); for row in rows { // Entry by entry: one that doesn't read is kept as it was, not the whole list. let read = serde_json::from_value::(row.clone()).ok().and_then(|r| { let mut cleaned = clean(r.source.clone()).ok()?; - // Preferences carry provenance, never authority from an earlier process's login; - // clearing it is a change `cut` counts. + let was_cut = cut(&r.source, &cleaned); + // A login binding is the app's own output, never authority from an earlier + // process's login: it isn't kept, and clearing it is no loss. crate::s3_connections::clear_bindings(std::iter::once(&mut cleaned.address)); - Some((cut(&r.source, &cleaned), cleaned, r.other)) + Some((was_cut, cleaned, r.other)) }); - // A repeat of a listed source merges even at the limit: it adds no row. - let repeat = read.as_ref().is_some_and(|(_, source, _)| into.iter().any(|r| same(r, source))); match read { - Some((was_cut, source, other)) if repeat || into.len() < cap => { + // The first row of an identity wins; a repeat is dropped, and said. + Some((_, source, _)) if seen.contains(&identity(&source.kind, &source.address)) => s.kept.repeated += 1, + Some((was_cut, source, other)) if into.len() < cap => { + seen.insert(identity(&source.kind, &source.address)); s.kept.cut += usize::from(was_cut); let other = s.kept.without_secrets(other, &source.address, &source.extra); - s.kept.merged += usize::from(repeat); if !other.is_empty() { - // A repeated row adds the unknown fields the first lacks; the combined - // fields are classified again, since a key beside a web address - // that another row held is a secret only together. - let key = (list, source.kind.clone(), source.address.clone()); - let mut merged = s.kept.fields.remove(&key).unwrap_or_default(); - for (k, v) in other { - merged.entry(k).or_insert(v); - } - let merged = if repeat { s.kept.without_secrets(merged, &source.address, &source.extra) } else { merged }; - if !merged.is_empty() { - s.kept.fields.insert(key, merged); - } - } - if !repeat { - into.push(source); + s.kept.fields.insert((list, source.kind.clone(), source.address.clone()), other); } + into.push(source); } _ if holds_secret(&row) => s.kept.secrets += 1, read => { @@ -238,11 +227,11 @@ impl Sources { } let mut said = Vec::new(); let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over); - if k.merged > 0 { - let n = k.merged; + if k.repeated > 0 { + let n = k.repeated; said.push(format!( - "{n} saved data source{} another in the same list: merged into it, keeping its settings and adding the unknown ones it lacks", - if n == 1 { " repeats" } else { "s repeat" } + "{n} repeated saved data source{} removed (the first of each is kept, with its settings)", + if n == 1 { " was" } else { "s were" } )); } if k.over > 0 { @@ -270,7 +259,7 @@ impl Sources { if k.cut > 0 { let n = k.cut; said.push(format!( - "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key, token or login binding, details that aren't short text, more than eight, or secret-looking): saved changed", + "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key or token, details that aren't short text, more than eight, or secret-looking): saved changed", if n == 1 { " has" } else { "s have" } )); } @@ -335,8 +324,8 @@ impl Sources { /// A source as a preference row: with the fields kept for it that this version doesn't know. fn row(&self, list: &'static str, source: &Source) -> Value { let mut row = serde_json::to_value(source).unwrap_or(Value::Null); - let key = (list, source.kind.clone(), source.address.clone()); - if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&key)) { + let (kind, address) = identity(&source.kind, &source.address); + if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&(list, kind, address))) { for (k, v) in other { row.entry(k.clone()).or_insert_with(|| v.clone()); } @@ -352,7 +341,9 @@ impl Sources { self.recent.truncate(MAX_RECENT); // A source no longer recent takes the fields kept for it along. let recent = &self.recent; - self.kept.fields.retain(|(list, kind, address), _| *list != "recent" || recent.iter().any(|r| r.is(kind, address))); + self.kept.fields.retain(|(list, kind, address), _| { + *list != "recent" || recent.iter().any(|r| identity(&r.kind, &r.address) == identity(kind, address)) + }); self.generation += 1; } @@ -377,26 +368,24 @@ impl Sources { /// Forgets the source of `kind` at `address` from `list`; whether there was one. pub fn forget(&mut self, list: &str, kind: &str, address: &str) -> Result { + let id = identity(kind, address); let l = self.list_mut(list)?; let before = l.len(); - l.retain(|r| !r.is(kind, address)); + l.retain(|r| identity(&r.kind, &r.address) != id); let gone = l.len() != before; + let mut kept_gone = false; if let Some(list) = LISTS.into_iter().find(|l| *l == list) { - self.kept.fields.remove(&(list, kind.to_string(), address.to_string())); - // A readable row kept past the list's limit of the same source would be listed again - // by the next start. + self.kept.fields.retain(|(l, k, a), _| *l != list || identity(k, a) != id); + // A readable row kept past the list's limit with the same identity would be listed + // again by the next start; a row this version can't read is not this source's. if let Some(rows) = self.kept.rows.get_mut(list) { - let text = |r: &Value, k: &str| r.get(k).and_then(Value::as_str).map(|s| s.trim().to_string()); - let mut dropped = 0; - rows.retain(|r| { - let same = text(r, "kind").as_deref() == Some(kind) && text(r, "address").as_deref() == Some(address.trim()); - dropped += usize::from(same && serde_json::from_value::(r.clone()).is_ok()); - !same - }); - self.kept.over = self.kept.over.saturating_sub(dropped); + let before = rows.len(); + rows.retain(|r| row_identity(r).as_ref() != Some(&id)); + self.kept.over = self.kept.over.saturating_sub(before - rows.len()); + kept_gone = rows.len() != before; } } - if gone { + if gone || kept_gone { self.generation += 1; } Ok(gone) @@ -412,8 +401,22 @@ impl Sources { } } +/// A source's identity, the one every list is matched by: its kind and its address with a login +/// binding cleared, as [`Sources::from_json`] reads it. +fn identity(kind: &str, address: &str) -> (String, String) { + let mut address = address.to_string(); + crate::s3_connections::clear_bindings(std::iter::once(&mut address)); + (kind.to_string(), address) +} + +/// The identity of a saved row [`Sources::from_json`] reads (none for one it can't). +fn row_identity(row: &Value) -> Option<(String, String)> { + let source = clean(serde_json::from_value::(row.clone()).ok()?.source).ok()?; + Some(identity(&source.kind, &source.address)) +} + fn same(a: &Source, b: &Source) -> bool { - a.is(&b.kind, &b.address) + identity(&a.kind, &a.address) == identity(&b.kind, &b.address) } impl Source { @@ -1070,50 +1073,30 @@ MIIE assert_eq!(s.favourites.len(), MAX_KEPT); assert!(s.unread()[0].starts_with("1 saved data source is past"), "{:?}", s.unread()); assert!(s.to_json().contains(&format!("/{}.csv", MAX_KEPT - 1)) && s.to_json().contains(&format!("\"pinned\":{}", MAX_KEPT - 1))); - // Rows of one identity are one source: the first row's fields stay, a later row adds the - // ones it lacks (without a secret), and forgetting it leaves nothing behind. - let twins = json!({"saved": [ - {"kind": "file", "name": "A", "address": "/a.csv", "future": 1}, - {"kind": "file", "name": "A", "address": "/a.csv", "future": 2, "more": 3, "token": "hidden"} - ]}); - let mut s = Sources::from_json(&twins.to_string()); - assert_eq!(s.saved.len(), 1); - let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); - assert_eq!( - (saved["saved"].as_array().map(Vec::len), &saved["saved"][0]["future"], &saved["saved"][0]["more"]), - (Some(1), &json!(1), &json!(3)) - ); - assert!(!s.to_json().contains("hidden")); - assert!(s.unread()[0].starts_with("1 saved data source repeats"), "{:?}", s.unread()); - s.forget("saved", "file", "/a.csv").unwrap(); - assert!(!s.to_json().contains("/a.csv")); - // Fields that are fine in each row but a secret together are dropped, in either order. - for rows in [ - [json!({"endpoint": "https://api.example"}), json!({"key": "abc"})], - [json!({"key": "abc"}), json!({"endpoint": "https://api.example"})], + // Rows of one identity are one source: the first row wins, with its fields, a later row is + // dropped and said (whatever it holds), and forgetting it leaves nothing behind. + for twins in [ + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "future": 1}, + {"kind": "file", "name": "A", "address": "/a.csv", "future": 2, "more": 3, "token": "hidden", "endpoint": "https://api.example"} + ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"}, + {"kind": "file", "name": "A", "address": "/a.csv", "endpoint": "https://api.example"} + ]}), ] { - let rows: Vec = rows - .into_iter() - .map(|mut r| { - r.as_object_mut().into_iter().for_each(|o| { - o.extend([("kind".into(), json!("file")), ("name".into(), json!("A")), ("address".into(), json!("/a.csv"))]) - }); - r - }) - .collect(); - let saved = Sources::from_json(&json!({"saved": rows}).to_string()).to_json(); - assert!(!saved.contains("abc") && saved.contains("api.example"), "{saved}"); + let mut s = Sources::from_json(&twins.to_string()); + assert_eq!(s.saved.len(), 1); + let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); + assert_eq!(saved["saved"].as_array().map(Vec::len), Some(1)); + assert!(saved["saved"][0].get("more").is_none() && !s.to_json().contains("hidden") && !s.to_json().contains("api.example")); + assert_eq!( + s.unread().first().map(String::as_str), + Some("1 repeated saved data source was removed (the first of each is kept, with its settings)") + ); + s.forget("saved", "file", "/a.csv").unwrap(); + assert!(!s.to_json().contains("/a.csv")); } - // A readable row past the limit is kept raw; once the source is listed and forgotten it - // doesn't come back with the next start. - let recent: Vec = (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); - let mut s = Sources::from_json(&json!({ "recent": recent }).to_string()); - assert!(s.to_json().contains(&format!("/{MAX_RECENT}.csv"))); - s.forget("recent", "file", "/0.csv").unwrap(); - s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); - s.forget("recent", "file", &format!("/{MAX_RECENT}.csv")).unwrap(); - assert!(!s.to_json().contains(&format!("/{MAX_RECENT}.csv")), "{}", s.to_json()); - assert!(s.unread().iter().all(|t| !t.contains("past the lists' limits")), "{:?}", s.unread()); // The row's known details count beside an unknown key, as its address does. let extra = json!({"saved": [{"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}, "key": "abc"}]}); let saved = Sources::from_json(&extra.to_string()).to_json(); @@ -1124,19 +1107,40 @@ MIIE let stored: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); s.merge(Sources::from_json(&json!({ "favourites": stored }).to_string())); assert!(s.unread().iter().any(|t| t.contains("past the lists' limits")), "{:?}", s.unread()); - // A repeat past the limit merges into its source, whether or not it holds a secret. + // A repeat past the limit is dropped, not kept as an overflow row. A bound address is the + // bare one's identity: forgetting by either form removes a kept over-limit row of it, never + // an unreadable row that merely shares the address. + let bound = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); let mut full: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); - full.push(json!({"kind": "file", "name": "", "address": "/0.csv", "token": "hidden", "future": 1})); - full.push(json!({"kind": "file", "name": "", "address": "/1.csv"})); + full.push(json!({"kind": "file", "name": "", "address": "/0.csv", "token": "hidden"})); + let s = Sources::from_json(&json!({"saved": full.clone()}).to_string()); + assert!(!s.to_json().contains("hidden") && s.kept.over == 0 && s.kept.repeated == 1); + full.truncate(MAX_KEPT); + full.push(json!({"kind": "url", "name": "", "address": bound})); + let twin = rustgis_data::s3_address::bind("fedcba9876543210", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + full.push(json!({"kind": "url", "name": "", "address": twin, "gcs": 1})); + full.push(json!({"kind": "gcs", "name": "", "address": "gs://b/a.tif"})); + full.push(json!({"kind": "future", "name": "", "address": twin})); + full.push(json!({"kind": " url ", "name": 5, "address": twin})); let mut s = Sources::from_json(&json!({"saved": full}).to_string()); - assert!(!s.to_json().contains("hidden") && s.to_json().contains("\"future\":1"), "{}", s.to_json()); + assert_eq!(s.kept.over, 2, "{:?}", s.unread()); + let generation = s.generation; + // Only kept rows go: the pane must still save. + assert!(!s.forget("saved", "url", &twin).unwrap() && s.generation != generation); + assert!( + s.to_json().contains("gs://b/a.tif") + && s.to_json().contains("\"future\"") + && !s.to_json().contains("\"gcs\":1") + && s.to_json().contains("\"name\":5") + ); assert_eq!(s.kept.over, 0); - s.forget("saved", "file", "/1.csv").unwrap(); - assert!(!s.to_json().contains("/1.csv")); - // A login binding removed from an address is said. - let bound = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + // A login binding is the app's own output: cleared on read without a notice. let s = Sources::from_json(&json!({"saved": [{"kind": "url", "name": "B", "address": bound}]}).to_string()); - assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source has")), "{:?}", s.unread()); + assert!( + s.unread().is_empty() && rustgis_data::s3_address::parse(&s.saved[0].address).unwrap().unwrap().binding.is_empty(), + "{:?}", + s.unread() + ); // A file's contents beside a field isn't an address for it. let file = json!({"data": "https://example.invalid/a.csv", "key": "feature_id"}); assert!(Sources::from_json(&file.to_string()).to_json().contains("feature_id")); From d71cb05ceb5c6251bfcb375bc523d1980d93487e Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 11:58:31 -0600 Subject: [PATCH 10/15] fix(plugins): an empty state.json is nothing disabled, as a missing one It is written whole each time, so an empty file was made empty on purpose rather than cut short; it no longer switches every plugin off. Co-Authored-By: Claude Opus 5.5 --- crates/plugins/src/store.rs | 9 ++++++--- crates/plugins/src/tests.rs | 15 ++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index eaa5f54235..8e73eb5bbb 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -63,13 +63,16 @@ fn state_path() -> Option { dir().map(|d| d.join("state.json")) } -/// `state.json`: none is no plugin disabled or added from a folder. One that can't be read (or that -/// a newer RustGIS wrote in a shape this one doesn't know) is said by [`load_installed`], which then -/// loads no plugin, and kept as it is: [`write_state`] refuses to write over it. +/// `state.json`: none, or an empty one, is no plugin disabled or added from a folder. One that can't +/// be read (or that a newer RustGIS wrote in a shape this one doesn't know) is said by +/// [`load_installed`], which then loads no plugin, and kept as it is: [`write_state`] refuses to +/// write over it. fn read_state() -> State { let Some(path) = state_path() else { return State::default() }; let read = match std::fs::read_to_string(&path) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => return State::default(), + // Written whole each time, so an empty file was made empty on purpose, not cut short. + Ok(text) if text.trim().is_empty() => return State::default(), Err(e) => Err(e.to_string()), Ok(text) => serde_json::from_str(&text).map_err(|e| e.to_string()), }; diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index 4b123a09c6..b7a8da7c15 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1883,11 +1883,16 @@ fn plugin_state_keeps_what_this_version_cannot_read() { std::fs::write(&state, r#"{"disabled": ["state-test"]}"#).unwrap(); assert!(crate::store::load_installed().is_empty()); } - std::fs::remove_file(&state).unwrap(); - // No file is nothing disabled, with no notice. - crate::unload("state-test"); - assert!(crate::store::load_installed().is_empty()); - assert!(crate::get("state-test").is_some_and(|p| p.enabled)); + // No file, or an empty one, is nothing disabled, with no notice. + for empty in [None, Some(" \n")] { + match empty { + Some(text) => std::fs::write(&state, text).unwrap(), + None => std::fs::remove_file(&state).unwrap(), + } + crate::unload("state-test"); + assert!(crate::store::load_installed().is_empty(), "{empty:?}"); + assert!(crate::get("state-test").is_some_and(|p| p.enabled), "{empty:?}"); + } crate::store::uninstall("state-test").unwrap(); let _ = std::fs::remove_dir_all(&plugins); } From 8bc93e0ece4a5dd0576d10375b78dc33a94f479f Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 12:06:59 -0600 Subject: [PATCH 11/15] fix(engine): forgetting a source kept only past the list's limit counts as a removal Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/sources.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 14af3f44d5..19fbe9f9ea 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -388,7 +388,7 @@ impl Sources { if gone || kept_gone { self.generation += 1; } - Ok(gone) + Ok(gone || kept_gone) } fn list_mut(&mut self, list: &str) -> Result<&mut Vec, String> { @@ -1125,8 +1125,8 @@ MIIE let mut s = Sources::from_json(&json!({"saved": full}).to_string()); assert_eq!(s.kept.over, 2, "{:?}", s.unread()); let generation = s.generation; - // Only kept rows go: the pane must still save. - assert!(!s.forget("saved", "url", &twin).unwrap() && s.generation != generation); + // Only kept rows go: that is still a removal, and the pane must still save. + assert!(s.forget("saved", "url", &twin).unwrap() && s.generation != generation); assert!( s.to_json().contains("gs://b/a.tif") && s.to_json().contains("\"future\"") From eda708f1ad4a7cee55bf9d1e79e2c4d713654dc7 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 12:28:15 -0600 Subject: [PATCH 12/15] fix(engine): check kept fields again when a source's details change; listing a source adopts its kept row Fields kept for a source are classified again against the source as it is after keep, note or merge, and a secret one is dropped and counted. Listing a source takes the fields of a readable row of it kept past the list's limit and removes that row. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/sources.rs | 132 ++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 19fbe9f9ea..13bffa8ce0 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -266,10 +266,9 @@ impl Sources { if k.secrets > 0 { let n = k.secrets; said.push(format!( - "{n} value{} in the saved data sources {} dropped because {} held a password, key or token", + "{n} value{} in the saved data sources {} a password, key or token: not kept, and left out the next time the sources are saved", if n == 1 { "" } else { "s" }, - if n == 1 { "was" } else { "were" }, - if n == 1 { "it" } else { "they" } + if n == 1 { "holds" } else { "hold" } )); } said @@ -303,9 +302,72 @@ impl Sources { for (list, row) in over { self.kept.rows.entry(list).or_default().push(row); } + // A readable row kept past a limit for a source that is listed gives it its fields. + for list in LISTS { + if self.kept.rows.contains_key(list) { + let ids: Vec<_> = match list { + "favourites" => &self.favourites, + "saved" => &self.saved, + _ => &self.recent, + } + .iter() + .map(|s| identity(&s.kind, &s.address)) + .collect(); + for id in ids { + self.adopt(list, &id); + } + } + } + self.recheck_fields(); self.generation += 1; } + /// Classifies the fields kept for each listed source again, against the source as it is now + /// (a changed `extra` or address can make a field that passed at load a secret), and drops + /// what is one, counted with the rest. Every change to a list that keeps fields ends here, so + /// [`Sources::to_json`] writes only what passed. + fn recheck_fields(&mut self) { + let keys: Vec<_> = self.kept.fields.keys().cloned().collect(); + for key in keys { + let (list, kind, address) = &key; + let listed = match *list { + "favourites" => &self.favourites, + "saved" => &self.saved, + _ => &self.recent, + }; + let id = (kind.clone(), address.clone()); + let Some(source) = listed.iter().find(|s| identity(&s.kind, &s.address) == id).cloned() else { continue }; + if let Some(fields) = self.kept.fields.remove(&key) { + let fields = self.kept.without_secrets(fields, &source.address, &source.extra); + if !fields.is_empty() { + self.kept.fields.insert(key, fields); + } + } + } + } + + /// Listing a source takes along the fields of a readable row of it kept past the list's limit + /// (the first fields of an identity win), and removes that row: it would list it twice. + fn adopt(&mut self, list: &'static str, id: &(String, String)) { + let Some(rows) = self.kept.rows.get_mut(list) else { return }; + let mut taken = Vec::new(); + rows.retain(|r| { + let ours = row_identity(r).as_ref() == Some(id); + if ours && let Ok(row) = serde_json::from_value::(r.clone()) { + taken.push(row.other); + } + !ours + }); + self.kept.over = self.kept.over.saturating_sub(taken.len()); + for other in taken { + let fields = self.kept.fields.entry((list, id.0.clone(), id.1.clone())).or_default(); + for (k, v) in other { + fields.entry(k).or_insert(v); + } + } + self.kept.fields.retain(|_, f| !f.is_empty()); + } + /// The `data_sources` preference: these sources, with what [`Sources::from_json`] kept /// written back as it was (a preference it couldn't read at all, unchanged). pub fn to_json(&self) -> String { @@ -339,11 +401,14 @@ impl Sources { self.recent.retain(|r| !same(r, &s)); self.recent.insert(0, s); self.recent.truncate(MAX_RECENT); + let id = identity(&self.recent[0].kind, &self.recent[0].address); + self.adopt("recent", &id); // A source no longer recent takes the fields kept for it along. let recent = &self.recent; self.kept.fields.retain(|(list, kind, address), _| { *list != "recent" || recent.iter().any(|r| identity(&r.kind, &r.address) == identity(kind, address)) }); + self.recheck_fields(); self.generation += 1; } @@ -351,6 +416,7 @@ impl Sources { /// identity ([`Source::is`]: kind and address), whose name and `extra` the new one's take over. pub fn keep(&mut self, list: &str, s: Source) -> Result { let s = clean(s)?; + let list = LISTS.into_iter().find(|l| *l == list).ok_or_else(|| format!("unknown list '{list}' (favourites, saved or recent)"))?; let l = self.list_mut(list)?; // The same source keeps its place. if let Some(slot) = l.iter_mut().find(|r| same(r, &s)) { @@ -361,6 +427,8 @@ impl Sources { } l.push(s.clone()); } + self.adopt(list, &identity(&s.kind, &s.address)); + self.recheck_fields(); self.generation += 1; // What was stored (cleaned): the caller reports it, wherever it sits in the list. Ok(s) @@ -1039,9 +1107,7 @@ MIIE ]}); let s = Sources::from_json(&secret.to_string()); assert!( - s.unread() - .iter() - .any(|t| t.starts_with("3 values in the saved data sources were dropped because they held a password, key or token")), + s.unread().iter().any(|t| t.starts_with("3 values in the saved data sources hold a password, key or token")), "{:?}", s.unread() ); @@ -1056,7 +1122,7 @@ MIIE "{}", s.to_json() ); - assert!(s.unread().iter().any(|t| t.starts_with("2 values in the saved data sources were dropped")), "{:?}", s.unread()); + assert!(s.unread().iter().any(|t| t.starts_with("2 values in the saved data sources hold")), "{:?}", s.unread()); // A key beside an address among the preference's own keys. let top = json!({"address": "https://api.example", "key": "abc"}); assert!(!Sources::from_json(&top.to_string()).to_json().contains("abc")); @@ -1084,12 +1150,22 @@ MIIE {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"}, {"kind": "file", "name": "A", "address": "/a.csv", "endpoint": "https://api.example"} ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}}, + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"} + ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"}, + {"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}} + ]}), ] { let mut s = Sources::from_json(&twins.to_string()); assert_eq!(s.saved.len(), 1); let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); assert_eq!(saved["saved"].as_array().map(Vec::len), Some(1)); - assert!(saved["saved"][0].get("more").is_none() && !s.to_json().contains("hidden") && !s.to_json().contains("api.example")); + assert!(saved["saved"][0].get("more").is_none() && !s.to_json().contains("hidden")); + // Nothing of a dropped row combines with the first: no key beside an endpoint. + assert!(!(s.to_json().contains("abc") && s.to_json().contains("api.example")), "{}", s.to_json()); assert_eq!( s.unread().first().map(String::as_str), Some("1 repeated saved data source was removed (the first of each is kept, with its settings)") @@ -1097,6 +1173,46 @@ MIIE s.forget("saved", "file", "/a.csv").unwrap(); assert!(!s.to_json().contains("/a.csv")); } + // A source's known details can be changed after its fields were kept, as a key beside an + // endpoint it now has: checked again when it is written, by keep and by merge. + let a = || Source { + kind: "file".into(), + address: "/a.csv".into(), + extra: json!({"endpoint": "https://api.example"}), + ..Source::default() + }; + let kept = json!({"favourites": [{"kind": "file", "name": "", "address": "/a.csv", "key": "abc", "future": 1}]}).to_string(); + let mut s = Sources::from_json(&kept); + assert!(s.to_json().contains("abc")); + s.keep("favourites", a()).unwrap(); + assert!( + !s.to_json().contains("abc") && s.to_json().contains("\"future\":1") && s.to_json().contains("api.example"), + "{}", + s.to_json() + ); + assert!(s.unread().iter().any(|t| t.starts_with("1 value in the saved data sources holds")), "{:?}", s.unread()); + let mut s = Sources::default(); + s.keep("favourites", a()).unwrap(); + s.merge(Sources::from_json(&kept)); + assert!(!s.to_json().contains("abc") && s.to_json().contains("api.example"), "{}", s.to_json()); + // A readable row kept past the limit gives its fields to the source when it is listed. + let recent: Vec = + (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "future": i})).collect(); + let mut s = Sources::from_json(&json!({ "recent": recent }).to_string()); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let mut s = Sources::from_json(&s.to_json()); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let saved: Value = serde_json::from_str(&Sources::from_json(&s.to_json()).to_json()).unwrap(); + assert_eq!(saved["recent"][0]["future"], json!(MAX_RECENT), "{saved}"); + assert_eq!(saved["recent"].as_array().map(Vec::len), Some(MAX_RECENT)); + // The same by merge: the session lists a source the stored preference kept past the limit. + let mut s = Sources::default(); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let recent: Vec = + (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "future": i})).collect(); + s.merge(Sources::from_json(&json!({ "recent": recent }).to_string())); + assert!(s.to_json().contains(&format!("\"future\":{MAX_RECENT}")), "{}", s.to_json()); + assert_eq!(s.to_json().matches(&format!("/{MAX_RECENT}.csv")).count(), 1, "{}", s.to_json()); // The row's known details count beside an unknown key, as its address does. let extra = json!({"saved": [{"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}, "key": "abc"}]}); let saved = Sources::from_json(&extra.to_string()).to_json(); From 4adf225a2c83bd49ee53285473a58f41d77c8be0 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 13:33:32 -0600 Subject: [PATCH 13/15] fix(engine, plugins): say a secret left out when a source changes; a byte-order mark in state.json is not unreadable A kept field that only holds a secret once a source's details change is left out at that change, and is now said then, not only counted for the start-up notice. An editor's byte-order mark at the start of state.json no longer makes the file unreadable and switch every plugin off. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/sources.rs | 5 +++++ crates/engine/src/policy/dispatch.rs | 5 +++++ crates/engine/src/sources.rs | 23 +++++++++++++++++------ crates/engine/src/tests.rs | 21 +++++++++++++++++++++ crates/plugins/src/store.rs | 7 ++++--- crates/plugins/src/tests.rs | 7 ++++++- 6 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/sources.rs b/crates/engine/src/cmd/sources.rs index 7498992862..e411d33e00 100644 --- a/crates/engine/src/cmd/sources.rs +++ b/crates/engine/src/cmd/sources.rs @@ -68,7 +68,12 @@ fn source(p: &Value) -> Result { fn keep(s: &mut Session, p: &Value, list: &str) -> CmdResult { let src = source(p)?; + let before = s.sources.secrets_left_out(); let kept = s.sources.keep(list, src)?; + let left_out = s.sources.secrets_left_out().saturating_sub(before); + if left_out > 0 { + s.notify(crate::sources::secrets_notice(left_out), false); + } Ok(shown(&kept)) } diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 288e17a9c2..57f7f63341 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -212,7 +212,12 @@ impl Session { if result.is_ok() && let Some(source) = crate::sources::added_by(command, ¶ms) { + let before = s.sources.secrets_left_out(); s.sources.note(source); + let left_out = s.sources.secrets_left_out().saturating_sub(before); + if left_out > 0 { + s.notify(crate::sources::secrets_notice(left_out), false); + } } result })? diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 13bffa8ce0..e65c086dcf 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -264,16 +264,18 @@ impl Sources { )); } if k.secrets > 0 { - let n = k.secrets; - said.push(format!( - "{n} value{} in the saved data sources {} a password, key or token: not kept, and left out the next time the sources are saved", - if n == 1 { "" } else { "s" }, - if n == 1 { "holds" } else { "hold" } - )); + said.push(secrets_notice(k.secrets)); } said } + /// How many values holding a password, key or token have been left out of what is kept so far. + /// A caller that changes a source compares it before and after, to say what that change left + /// out ([`secrets_notice`]); [`Sources::unread`] says what reading did. + pub fn secrets_left_out(&self) -> usize { + self.kept.secrets + } + /// Takes in the sources kept with the preferences, keeping what this session already has: /// in-memory entries stay (and first), stored ones are added after them unless the same source /// ([`Source::is`]) is there already. @@ -496,6 +498,15 @@ impl Source { } } +/// The notice for `n` values left out of the saved data sources because they hold a secret. +pub fn secrets_notice(n: usize) -> String { + format!( + "{n} value{} in the saved data sources {} a password, key or token: not kept, and left out the next time the sources are saved", + if n == 1 { "" } else { "s" }, + if n == 1 { "holds" } else { "hold" } + ) +} + /// A source made safe to keep: a known kind, bounded text, and an address without secrets. pub fn clean(mut s: Source) -> Result { if !KINDS.contains(&s.kind.as_str()) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 10fd16574e..e51a6f49fc 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -28342,6 +28342,27 @@ fn connection_browse_levels_and_adds() { assert!(!s.can_undo()); } +/// A kept field that only holds a secret once a source's details change is left out then, and +/// said then, not only at start (the start-up notice has already been shown). +#[test] +fn a_secret_left_out_when_a_source_changes_is_said() { + let mut s = Session::new(Project::default()); + s.sources = crate::sources::Sources::from_json(r#"{"saved":[{"kind":"file","name":"A","address":"/a.csv","key":"abc"}]}"#); + // Kept at start as a setting this version doesn't know: nothing beside it makes it a secret. + assert!(!s.sources.unread().iter().any(|t| t.contains("password")), "{:?}", s.sources.unread()); + let mark = s.notices_given(); + s.run("sources.save", &json!({"kind": "file", "name": "A", "address": "/a.csv"})).unwrap(); + assert!(s.notices_since(mark).is_empty(), "{:?}", s.notices_since(mark)); + s.run("sources.save", &json!({"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}})) + .unwrap(); + assert!( + s.notices_since(mark).iter().any(|n| n.text.starts_with("1 value in the saved data sources holds a password")), + "{:?}", + s.notices_since(mark) + ); + assert!(!s.sources.to_json().contains("abc")); +} + fn rustgis_engine_sources_roundtrip(text: &str) -> crate::sources::Sources { crate::sources::Sources::from_json(text) } diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 8e73eb5bbb..8a75ab5521 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -71,10 +71,11 @@ fn read_state() -> State { let Some(path) = state_path() else { return State::default() }; let read = match std::fs::read_to_string(&path) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => return State::default(), - // Written whole each time, so an empty file was made empty on purpose, not cut short. - Ok(text) if text.trim().is_empty() => return State::default(), + // Written whole each time, so an empty file was made empty on purpose, not cut short. A + // byte-order mark (an editor's) isn't part of the JSON. + Ok(text) if text.trim_start_matches('\u{feff}').trim().is_empty() => return State::default(), Err(e) => Err(e.to_string()), - Ok(text) => serde_json::from_str(&text).map_err(|e| e.to_string()), + Ok(text) => serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|e| e.to_string()), }; read.unwrap_or_else(|e| State { unreadable: Some(format!("{}: {e}", path.display())), ..State::default() }) } diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index b7a8da7c15..b145bac15b 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1884,7 +1884,7 @@ fn plugin_state_keeps_what_this_version_cannot_read() { assert!(crate::store::load_installed().is_empty()); } // No file, or an empty one, is nothing disabled, with no notice. - for empty in [None, Some(" \n")] { + for empty in [None, Some(" \n"), Some("\u{feff}")] { match empty { Some(text) => std::fs::write(&state, text).unwrap(), None => std::fs::remove_file(&state).unwrap(), @@ -1893,6 +1893,11 @@ fn plugin_state_keeps_what_this_version_cannot_read() { assert!(crate::store::load_installed().is_empty(), "{empty:?}"); assert!(crate::get("state-test").is_some_and(|p| p.enabled), "{empty:?}"); } + // An editor's byte-order mark isn't part of the JSON. + std::fs::write(&state, "\u{feff}{\"disabled\": [\"state-test\"]}").unwrap(); + crate::unload("state-test"); + assert!(crate::store::load_installed().is_empty()); + assert!(crate::get("state-test").is_some_and(|p| !p.enabled)); crate::store::uninstall("state-test").unwrap(); let _ = std::fs::remove_dir_all(&plugins); } From a0a78aee51625b3b028cc0322cd253c4792f662a Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 14:20:26 -0600 Subject: [PATCH 14/15] fix(engine): every way of recording a recent source says a secret it left out Recording a recent source goes through Session::note_source, which says any value left out as a secret, from a command, a read and the connection tree alike. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/lib.rs | 13 ++++++++++++- crates/engine/src/policy/dispatch.rs | 7 +------ crates/ui-egui/src/connectiontree.rs | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 6d0193d985..95b8b2539f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1689,7 +1689,7 @@ impl Session { if res.is_ok() && let Some(src) = sources::added_by(id, params) { - self.sources.note(src); + self.note_source(src); } res } @@ -2341,6 +2341,17 @@ impl Session { started } + /// Records `src` in the Data Sources pane's Recent list ([`sources::Sources::note`]), and says + /// any value it left out because, beside the source as now listed, it holds a secret. + pub fn note_source(&mut self, src: sources::Source) { + let before = self.sources.secrets_left_out(); + self.sources.note(src); + let left_out = self.sources.secrets_left_out().saturating_sub(before); + if left_out > 0 { + self.notify(sources::secrets_notice(left_out), false); + } + } + pub fn notify(&mut self, text: impl Into, error: bool) { use rustgis_data::diagnostics::Severity; self.notice(text.into(), if error { Severity::Error } else { Severity::Info }); diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 57f7f63341..e44a7cf194 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -212,12 +212,7 @@ impl Session { if result.is_ok() && let Some(source) = crate::sources::added_by(command, ¶ms) { - let before = s.sources.secrets_left_out(); - s.sources.note(source); - let left_out = s.sources.secrets_left_out().saturating_sub(before); - if left_out > 0 { - s.notify(crate::sources::secrets_notice(left_out), false); - } + s.note_source(source); } result })? diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index d7c381d2c8..51a2ed8c0d 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -99,7 +99,7 @@ pub(crate) fn name_recent(app: &mut RustGisApp, before: u64, origin: Option<&str } let Some(mut added) = app.session.sources.recent.first().cloned() else { return }; added.name = origin.to_string(); - app.session.sources.note(added); + app.session.note_source(added); } pub enum Entry { From 073592c1da421465dcbc54020fd30b273a056e7f Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 14:27:43 -0600 Subject: [PATCH 15/15] fix(engine, plugins): a byte-order mark in saved sources reads as JSON; no install while state.json can't be read An editor's byte-order mark no longer makes the saved data sources unreadable (a preference kept whole keeps it). Installing a plugin while state.json can't be read is refused like the other changes: it would start enabled, and may be one the user disabled. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/sources.rs | 9 +++++++-- crates/plugins/src/store.rs | 3 +++ crates/plugins/src/tests.rs | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index e65c086dcf..fa9c373ebd 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -153,11 +153,13 @@ impl Sources { /// says what was kept. pub fn from_json(text: &str) -> Sources { let mut s = Sources::default(); - if text.trim().is_empty() { + // An editor's byte-order mark isn't part of the JSON; a preference kept whole keeps it. + let body = text.trim_start_matches('\u{feff}'); + if body.trim().is_empty() { return s; } let whole = |why: &str| Kept { whole: Some((text.to_string(), why.to_string())), ..Kept::default() }; - let mut o = match serde_json::from_str::(text) { + let mut o = match serde_json::from_str::(body) { Ok(Value::Object(o)) => o, _ => { s.kept = whole("they aren't a JSON object"); @@ -1077,6 +1079,9 @@ MIIE "folders": ["/data"] }) .to_string(); + // With an editor's byte-order mark it reads the same. + let bom = Sources::from_json(&format!("\u{feff}{text}")); + assert_eq!((bom.favourites.len(), bom.unread().len()), (1, 2), "{:?}", bom.unread()); let mut s = Sources::default(); s.merge(Sources::from_json(&text)); assert_eq!(s.unread().len(), 2, "{:?}", s.unread()); diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 8a75ab5521..6d0182684d 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -288,6 +288,9 @@ fn extract(bytes: &[u8], to: &Path) -> Result<(), String> { /// Installs a package's bytes: checks it, copies it into the plugins folder (replacing an /// older copy) and loads it. pub fn install_bytes(bytes: &[u8]) -> Result, String> { + // Installed plugins start enabled: while `state.json` can't be read, the one being installed + // may be one the user disabled (and a native program), so it isn't installed either. + writable(&read_state())?; let mut pkg = Package::from_zip(bytes)?; // A process plugin is checked (blocklist, platform) before its files replace an installed // copy's: a package that won't load leaves the old one working. diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index b145bac15b..9e9b8a73eb 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1873,6 +1873,7 @@ fn plugin_state_keeps_what_this_version_cannot_read() { std::fs::write(folder.join("plugin.json"), crate::testing::manifest("folder-test", &[])).unwrap(); std::fs::write(folder.join("plugin.wasm"), testing::wasm()).unwrap(); assert!(crate::store::install_path(&folder).is_err() && crate::get("folder-test").is_none()); + assert!(crate::store::install_bytes(&testing::package("state-test", &[])).is_err_and(|e| e.contains("kept as it is"))); assert!(crate::store::uninstall("state-test").is_err()); assert_eq!(std::fs::read_to_string(&state).unwrap(), unreadable); // At start none is loaded (it may have been disabled, and be a native program), and it is said.