diff --git a/crates/assistant/src/chatgpt.rs b/crates/assistant/src/chatgpt.rs index 24936c6413..35eed912f7 100644 --- a/crates/assistant/src/chatgpt.rs +++ b/crates/assistant/src/chatgpt.rs @@ -838,6 +838,9 @@ mod native { } } + /// The sign-in store at `path` (none: no account). One that isn't JSON, or that a newer + /// RustGIS saved (a `version` above [`STORE_VERSION`]), is an error: every change reads it + /// first, so it is never written over. fn read_accounts(path: &Path) -> Result { let text = match std::fs::read_to_string(path) { Ok(text) => text, @@ -845,6 +848,21 @@ mod native { Err(e) => return Err(e.to_string().into()), }; let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|_| "the sign-in store isn't JSON")?; + // One a newer RustGIS wrote may hold what this version would lose by writing it again: it + // is neither used nor written over. + match v.get("version").map(|version| version.as_u64().ok_or(version)) { + Some(Ok(version)) if version > STORE_VERSION => { + return Err(Error::new( + Recovery::Auth, + format!( + "the ChatGPT sign-ins were saved by a newer RustGIS (version {version}; this one reads {STORE_VERSION}): use that RustGIS, or update this one" + ), + )); + } + // A version that isn't a plain integer can't be told from a newer one's. + Some(Err(_)) => return Err("the sign-in store's version isn't a whole number".into()), + _ => {} + } if let Some(entries) = v.get("accounts").and_then(Value::as_array) { let entries = entries.iter().map(Credentials::from_json).collect::, _>>()?; let active = @@ -857,6 +875,11 @@ mod native { } } + /// The version of the sign-in store this RustGIS reads and writes. + const STORE_VERSION: u64 = 2; + + /// Writes the sign-in store as version [`STORE_VERSION`], without tokens except a settings + /// file's own from before the keychain. fn write_accounts(path: &Path, accounts: &Accounts) -> Result<(), Error> { let active = accounts.active.as_ref().map(|(client, subject)| serde_json::json!({"client_id": client, "subject": subject})); let entry = |c: &Credentials| { @@ -871,7 +894,7 @@ mod native { }; write_private( path, - &serde_json::json!({"version": 2, "active": active, "accounts": accounts.entries.iter().map(entry).collect::>()}) + &serde_json::json!({"version": STORE_VERSION, "active": active, "accounts": accounts.entries.iter().map(entry).collect::>()}) .to_string(), ) } @@ -2023,6 +2046,35 @@ mod native { assert!(remove_at(&path, &v, "oaiapp_other", "user-1").unwrap().is_none()); } + /// A store a newer RustGIS saved is neither used nor written over, whatever is asked of it. + #[test] + fn a_store_from_a_newer_version_is_kept_as_it_is() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("auth.json"); + let store = Arc::new(MemorySecrets::new()); + let v = sharing(&store); + write_credentials(&path, &v, &synthetic(now() + 100_000)).unwrap(); + let mut newer: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + newer["version"] = serde_json::json!(3); + newer["devices"] = serde_json::json!([{"id": "d1"}]); + let text = newer.to_string(); + std::fs::write(&path, &text).unwrap(); + let e = read_accounts(&path).err().unwrap(); + assert!(e.to_string().contains("newer RustGIS (version 3; this one reads 2)"), "{e}"); + assert!(read_credentials(&path, &v).is_err()); + assert!(write_credentials(&path, &v, &synthetic(now() + 100_000)).is_err()); + assert!(sign_out_at(&path, &v).is_err()); + assert!(remove_at(&path, &v, "oaiapp_synthetic", "user-1").is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), text); + // A version that isn't a plain integer is unreadable, not newer-or-not. + for odd in [serde_json::json!("3"), serde_json::json!(3.0), serde_json::json!(true)] { + let mut v: Value = serde_json::from_str(&text).unwrap(); + v["version"] = odd; + std::fs::write(&path, v.to_string()).unwrap(); + assert!(read_accounts(&path).is_err()); + } + } + #[test] fn an_unreadable_keychain_signs_out_here_and_keeps_its_item_noted() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/engine/src/cmd/sources.rs b/crates/engine/src/cmd/sources.rs index 7498992862..e411d33e00 100644 --- a/crates/engine/src/cmd/sources.rs +++ b/crates/engine/src/cmd/sources.rs @@ -68,7 +68,12 @@ fn source(p: &Value) -> Result { fn keep(s: &mut Session, p: &Value, list: &str) -> CmdResult { let src = source(p)?; + let before = s.sources.secrets_left_out(); let kept = s.sources.keep(list, src)?; + let left_out = s.sources.secrets_left_out().saturating_sub(before); + if left_out > 0 { + s.notify(crate::sources::secrets_notice(left_out), false); + } Ok(shown(&kept)) } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 6d0193d985..95b8b2539f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1689,7 +1689,7 @@ impl Session { if res.is_ok() && let Some(src) = sources::added_by(id, params) { - self.sources.note(src); + self.note_source(src); } res } @@ -2341,6 +2341,17 @@ impl Session { started } + /// Records `src` in the Data Sources pane's Recent list ([`sources::Sources::note`]), and says + /// any value it left out because, beside the source as now listed, it holds a secret. + pub fn note_source(&mut self, src: sources::Source) { + let before = self.sources.secrets_left_out(); + self.sources.note(src); + let left_out = self.sources.secrets_left_out().saturating_sub(before); + if left_out > 0 { + self.notify(sources::secrets_notice(left_out), false); + } + } + pub fn notify(&mut self, text: impl Into, error: bool) { use rustgis_data::diagnostics::Severity; self.notice(text.into(), if error { Severity::Error } else { Severity::Info }); diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 288e17a9c2..e44a7cf194 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -212,7 +212,7 @@ impl Session { if result.is_ok() && let Some(source) = crate::sources::added_by(command, ¶ms) { - s.sources.note(source); + s.note_source(source); } result })? diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index bc3aa7eeff..fa9c373ebd 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -4,8 +4,10 @@ //! without their passwords, tokens or signatures. use rustgis_data::source::credentials::{redact_query, secret_key}; +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; /// How many recent sources are kept. pub const MAX_RECENT: usize = 20; @@ -43,53 +45,358 @@ pub struct Sources { /// Bumped on every change, so the app knows when to keep them. #[serde(skip)] pub generation: u64, + /// What of the preference this version couldn't use, written back as it was. + #[serde(skip)] + kept: Kept, +} + +/// The three lists, as the preference names them. +const LISTS: [&str; 3] = ["favourites", "recent", "saved"]; + +/// What of the saved sources this version couldn't use. A newer RustGIS may add kinds, fields and +/// lists: they are kept and written back unchanged, so a source noted or kept here never drops +/// what the newer one wrote. +#[derive(Clone, Debug, Default, PartialEq)] +struct Kept { + /// The whole preference as read, when it couldn't be read at all, and why: it is never + /// written over, so what changes in this session isn't kept. + whole: Option<(String, String)>, + /// Rows of each list that couldn't be read (an unknown kind, an address that isn't one…), or + /// that are past the list's limit. + rows: BTreeMap<&'static str, Vec>, + /// How many of those rows are past the limit, rather than unreadable. + over: usize, + /// Fields of read sources this version doesn't know, by list, kind and address. + fields: BTreeMap<(&'static str, String, String), Map>, + /// Keys beside the three lists this version doesn't know. + top: Map, + /// Rows and fields that couldn't be read and held a password, key or token: not kept, since + /// this version can't strip it from what it doesn't understand. + secrets: usize, + /// Readable sources whose name, address or details [`clean`] changed (a name too long or with + /// control characters, an address with a password, key or token, details that aren't short + /// text, more than eight, or secret-looking): saved that way. + cut: usize, + /// Readable rows that repeat the identity of an earlier row of their list: dropped (the first + /// one wins, with its unknown fields). + repeated: usize, +} + +impl Kept { + /// The entries of `o` without a password, key or token, counting those left out. Every entry is + /// classified on the original object first, beside the others and `address` (the one a row + /// keeps apart from them): a bare `key` is a secret beside a web address, and an address + /// holding a secret doesn't stop being one; `known` is the row's known details (its `extra`), a + /// sibling too. Only then are the classified entries removed. + fn without_secrets(&mut self, mut o: Map, address: &str, known: &Value) -> Map { + let secret: Vec = o + .iter() + .filter(|(k, v)| { + let mut probe = Map::new(); + probe.insert((*k).clone(), (*v).clone()); + if web_address(&Value::String(address.into()), 0) + || web_address(known, 0) + || o.iter().any(|(other, v)| { + other != *k && !other.eq_ignore_ascii_case("data") && !other.eq_ignore_ascii_case("text") && web_address(v, 0) + }) + { + probe.insert("\u{1}beside".into(), "https://a".into()); + } + holds_secret(&Value::Object(probe)) + }) + .map(|(k, _)| k.clone()) + .collect(); + self.secrets += secret.len(); + o.retain(|k, _| !secret.contains(k)); + o + } +} + +/// Whether `v` is, or holds, a web address (not under a file's `data` or `text`). +fn web_address(v: &Value, depth: usize) -> bool { + match v { + Value::String(t) => t.trim_start().get(..4).is_some_and(|p| p.eq_ignore_ascii_case("http")), + Value::Object(o) if depth < 4 => { + o.iter().any(|(k, v)| !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, depth + 1)) + } + Value::Array(a) if depth < 4 => a.iter().any(|v| web_address(v, depth + 1)), + _ => false, + } +} + +/// Whether [`clean`] changed something of `before` that isn't just tidied (whitespace, an empty +/// name or details). +fn cut(before: &Source, after: &Source) -> bool { + let name = before.name.trim(); + let extra = match &before.extra { + Value::Null => false, + Value::Object(o) => !o.is_empty() && before.extra != after.extra, + _ => true, + }; + (!name.is_empty() && after.name != name) || after.address != before.address.trim() || extra +} + +/// A saved row: the source, and the fields this version doesn't know. +#[derive(Deserialize)] +struct Row { + #[serde(flatten)] + source: Source, + #[serde(flatten)] + other: Map, } impl Sources { /// Sources read back from the app's preferences: each one checked and cleaned again (the - /// preferences are a file anyone could have edited). + /// preferences are a file anyone could have edited). What this version can't use is kept, to + /// be written back by [`Sources::to_json`] as it was: rows it can't read, fields and lists it + /// doesn't know, or the whole preference when it can't be read at all. [`Sources::unread`] + /// says what was kept. pub fn from_json(text: &str) -> Sources { - let v: Value = serde_json::from_str(text).unwrap_or(Value::Null); - // Entry by entry: one that doesn't read is dropped, not the whole list. - let read = |k: &str| -> Vec { - v.get(k) - .and_then(Value::as_array) - .map(|a| a.iter().filter_map(|x| serde_json::from_value::(x.clone()).ok()).filter_map(|x| clean(x).ok()).collect()) - .unwrap_or_default() + let mut s = Sources::default(); + // An editor's byte-order mark isn't part of the JSON; a preference kept whole keeps it. + let body = text.trim_start_matches('\u{feff}'); + if body.trim().is_empty() { + return s; + } + let whole = |why: &str| Kept { whole: Some((text.to_string(), why.to_string())), ..Kept::default() }; + let mut o = match serde_json::from_str::(body) { + Ok(Value::Object(o)) => o, + _ => { + s.kept = whole("they aren't a JSON object"); + return s; + } }; - let mut s = Sources { favourites: read("favourites"), recent: read("recent"), saved: read("saved"), generation: 0 }; - s.favourites.truncate(MAX_KEPT); - s.saved.truncate(MAX_KEPT); - s.recent.truncate(MAX_RECENT); - // Preferences carry provenance, never authority from an earlier process's login. - crate::s3_connections::clear_bindings( - s.favourites.iter_mut().chain(&mut s.recent).chain(&mut s.saved).map(|source| &mut source.address), - ); + let mut lists = Vec::with_capacity(LISTS.len()); + for list in LISTS { + match o.remove(list) { + None => lists.push(Vec::new()), + Some(Value::Array(rows)) => lists.push(rows), + Some(_) => { + s.kept = whole("a list of them isn't a JSON array"); + return s; + } + } + } + s.kept.top = s.kept.without_secrets(o, "", &Value::Null); + for (list, rows) in LISTS.into_iter().zip(lists) { + let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; + let mut into: Vec = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for row in rows { + // Entry by entry: one that doesn't read is kept as it was, not the whole list. + let read = serde_json::from_value::(row.clone()).ok().and_then(|r| { + let mut cleaned = clean(r.source.clone()).ok()?; + let was_cut = cut(&r.source, &cleaned); + // A login binding is the app's own output, never authority from an earlier + // process's login: it isn't kept, and clearing it is no loss. + crate::s3_connections::clear_bindings(std::iter::once(&mut cleaned.address)); + Some((was_cut, cleaned, r.other)) + }); + match read { + // The first row of an identity wins; a repeat is dropped, and said. + Some((_, source, _)) if seen.contains(&identity(&source.kind, &source.address)) => s.kept.repeated += 1, + Some((was_cut, source, other)) if into.len() < cap => { + seen.insert(identity(&source.kind, &source.address)); + s.kept.cut += usize::from(was_cut); + let other = s.kept.without_secrets(other, &source.address, &source.extra); + if !other.is_empty() { + s.kept.fields.insert((list, source.kind.clone(), source.address.clone()), other); + } + into.push(source); + } + _ if holds_secret(&row) => s.kept.secrets += 1, + read => { + s.kept.over += usize::from(read.is_some()); + s.kept.rows.entry(list).or_default().push(row); + } + } + } + if let Ok(l) = s.list_mut(list) { + *l = into; + } + } s } + /// What of the preference read by [`Sources::from_json`] isn't used here, one line per kind + /// (none when all of it is). + pub fn unread(&self) -> Vec { + let k = &self.kept; + if let Some((_, why)) = &k.whole { + return vec![format!( + "The saved data sources couldn't be read ({why}): none is listed, and the favourites, recent and saved sources of this session aren't kept over them; fix or remove \"data_sources\" in the preferences file to keep them" + )]; + } + let mut said = Vec::new(); + let rows = k.rows.values().map(Vec::len).sum::().saturating_sub(k.over); + if k.repeated > 0 { + let n = k.repeated; + said.push(format!( + "{n} repeated saved data source{} removed (the first of each is kept, with its settings)", + if n == 1 { " was" } else { "s were" } + )); + } + if k.over > 0 { + let n = k.over; + said.push(format!( + "{n} saved data source{} past the lists' limits ({MAX_KEPT} favourites or saved, {MAX_RECENT} recent): not listed, and kept as {} when the sources are saved", + if n == 1 { " is" } else { "s are" }, + if n == 1 { "it was" } else { "they were" } + )); + } + if rows > 0 { + said.push(format!( + "{rows} saved data source{} couldn't be read (from a newer RustGIS, or damaged): not listed, and kept as {} when the sources are saved", + if rows == 1 { "" } else { "s" }, + if rows == 1 { "it was" } else { "they were" } + )); + } + if !k.fields.is_empty() { + let n = k.fields.len(); + said.push(format!( + "{n} saved data source{} settings this version doesn't know (from a newer RustGIS): ignored here, and kept when the sources are saved", + if n == 1 { " has" } else { "s have" } + )); + } + if k.cut > 0 { + let n = k.cut; + said.push(format!( + "{n} saved data source{} a name, address or details this version changes (a name too long or with control characters, an address with a password, key or token, details that aren't short text, more than eight, or secret-looking): saved changed", + if n == 1 { " has" } else { "s have" } + )); + } + if k.secrets > 0 { + said.push(secrets_notice(k.secrets)); + } + said + } + + /// How many values holding a password, key or token have been left out of what is kept so far. + /// A caller that changes a source compares it before and after, to say what that change left + /// out ([`secrets_notice`]); [`Sources::unread`] says what reading did. + pub fn secrets_left_out(&self) -> usize { + self.kept.secrets + } + /// Takes in the sources kept with the preferences, keeping what this session already has: /// in-memory entries stay (and first), stored ones are added after them unless the same source /// ([`Source::is`]) is there already. - pub fn merge(&mut self, stored: Sources) { - fn join(now: &mut Vec, stored: Vec, cap: usize) { - for s in stored { - if now.len() >= cap { - break; + pub fn merge(&mut self, mut stored: Sources) { + // Stored sources that no longer fit (a change before the first read filled the list) are + // kept as rows, so they are said and written back. + let mut over = Vec::new(); + for list in LISTS { + let cap = if list == "recent" { MAX_RECENT } else { MAX_KEPT }; + let Ok(incoming) = stored.list_mut(list).map(std::mem::take) else { continue }; + let Ok(now) = self.list_mut(list) else { continue }; + for source in incoming { + if now.iter().any(|n| same(n, &source)) { + continue; } - if !now.iter().any(|n| n.is(&s.kind, &s.address)) { - now.push(s); + if now.len() < cap { + now.push(source); + } else { + over.push((list, stored.row(list, &source))); + stored.kept.fields.remove(&(list, source.kind.clone(), source.address.clone())); } } } - join(&mut self.favourites, stored.favourites, MAX_KEPT); - join(&mut self.saved, stored.saved, MAX_KEPT); - join(&mut self.recent, stored.recent, MAX_RECENT); + self.kept = stored.kept; + self.kept.over += over.len(); + for (list, row) in over { + self.kept.rows.entry(list).or_default().push(row); + } + // A readable row kept past a limit for a source that is listed gives it its fields. + for list in LISTS { + if self.kept.rows.contains_key(list) { + let ids: Vec<_> = match list { + "favourites" => &self.favourites, + "saved" => &self.saved, + _ => &self.recent, + } + .iter() + .map(|s| identity(&s.kind, &s.address)) + .collect(); + for id in ids { + self.adopt(list, &id); + } + } + } + self.recheck_fields(); self.generation += 1; } + /// Classifies the fields kept for each listed source again, against the source as it is now + /// (a changed `extra` or address can make a field that passed at load a secret), and drops + /// what is one, counted with the rest. Every change to a list that keeps fields ends here, so + /// [`Sources::to_json`] writes only what passed. + fn recheck_fields(&mut self) { + let keys: Vec<_> = self.kept.fields.keys().cloned().collect(); + for key in keys { + let (list, kind, address) = &key; + let listed = match *list { + "favourites" => &self.favourites, + "saved" => &self.saved, + _ => &self.recent, + }; + let id = (kind.clone(), address.clone()); + let Some(source) = listed.iter().find(|s| identity(&s.kind, &s.address) == id).cloned() else { continue }; + if let Some(fields) = self.kept.fields.remove(&key) { + let fields = self.kept.without_secrets(fields, &source.address, &source.extra); + if !fields.is_empty() { + self.kept.fields.insert(key, fields); + } + } + } + } + + /// Listing a source takes along the fields of a readable row of it kept past the list's limit + /// (the first fields of an identity win), and removes that row: it would list it twice. + fn adopt(&mut self, list: &'static str, id: &(String, String)) { + let Some(rows) = self.kept.rows.get_mut(list) else { return }; + let mut taken = Vec::new(); + rows.retain(|r| { + let ours = row_identity(r).as_ref() == Some(id); + if ours && let Ok(row) = serde_json::from_value::(r.clone()) { + taken.push(row.other); + } + !ours + }); + self.kept.over = self.kept.over.saturating_sub(taken.len()); + for other in taken { + let fields = self.kept.fields.entry((list, id.0.clone(), id.1.clone())).or_default(); + for (k, v) in other { + fields.entry(k).or_insert(v); + } + } + self.kept.fields.retain(|_, f| !f.is_empty()); + } + + /// The `data_sources` preference: these sources, with what [`Sources::from_json`] kept + /// written back as it was (a preference it couldn't read at all, unchanged). pub fn to_json(&self) -> String { - serde_json::to_string(self).unwrap_or_default() + if let Some((text, _)) = &self.kept.whole { + return text.clone(); + } + let mut o = self.kept.top.clone(); + for (list, sources) in LISTS.into_iter().zip([&self.favourites, &self.recent, &self.saved]) { + let mut rows: Vec = sources.iter().map(|source| self.row(list, source)).collect(); + rows.extend(self.kept.rows.get(list).into_iter().flatten().cloned()); + o.insert(list.into(), Value::Array(rows)); + } + Value::Object(o).to_string() + } + + /// A source as a preference row: with the fields kept for it that this version doesn't know. + fn row(&self, list: &'static str, source: &Source) -> Value { + let mut row = serde_json::to_value(source).unwrap_or(Value::Null); + let (kind, address) = identity(&source.kind, &source.address); + if let (Value::Object(row), Some(other)) = (&mut row, self.kept.fields.get(&(list, kind, address))) { + for (k, v) in other { + row.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + row } /// Notes a source just added (the most recent first, each once). @@ -98,6 +405,14 @@ impl Sources { self.recent.retain(|r| !same(r, &s)); self.recent.insert(0, s); self.recent.truncate(MAX_RECENT); + let id = identity(&self.recent[0].kind, &self.recent[0].address); + self.adopt("recent", &id); + // A source no longer recent takes the fields kept for it along. + let recent = &self.recent; + self.kept.fields.retain(|(list, kind, address), _| { + *list != "recent" || recent.iter().any(|r| identity(&r.kind, &r.address) == identity(kind, address)) + }); + self.recheck_fields(); self.generation += 1; } @@ -105,6 +420,7 @@ impl Sources { /// identity ([`Source::is`]: kind and address), whose name and `extra` the new one's take over. pub fn keep(&mut self, list: &str, s: Source) -> Result { let s = clean(s)?; + let list = LISTS.into_iter().find(|l| *l == list).ok_or_else(|| format!("unknown list '{list}' (favourites, saved or recent)"))?; let l = self.list_mut(list)?; // The same source keeps its place. if let Some(slot) = l.iter_mut().find(|r| same(r, &s)) { @@ -115,6 +431,8 @@ impl Sources { } l.push(s.clone()); } + self.adopt(list, &identity(&s.kind, &s.address)); + self.recheck_fields(); self.generation += 1; // What was stored (cleaned): the caller reports it, wherever it sits in the list. Ok(s) @@ -122,14 +440,27 @@ impl Sources { /// Forgets the source of `kind` at `address` from `list`; whether there was one. pub fn forget(&mut self, list: &str, kind: &str, address: &str) -> Result { + let id = identity(kind, address); let l = self.list_mut(list)?; let before = l.len(); - l.retain(|r| !r.is(kind, address)); + l.retain(|r| identity(&r.kind, &r.address) != id); let gone = l.len() != before; - if gone { + let mut kept_gone = false; + if let Some(list) = LISTS.into_iter().find(|l| *l == list) { + self.kept.fields.retain(|(l, k, a), _| *l != list || identity(k, a) != id); + // A readable row kept past the list's limit with the same identity would be listed + // again by the next start; a row this version can't read is not this source's. + if let Some(rows) = self.kept.rows.get_mut(list) { + let before = rows.len(); + rows.retain(|r| row_identity(r).as_ref() != Some(&id)); + self.kept.over = self.kept.over.saturating_sub(before - rows.len()); + kept_gone = rows.len() != before; + } + } + if gone || kept_gone { self.generation += 1; } - Ok(gone) + Ok(gone || kept_gone) } fn list_mut(&mut self, list: &str) -> Result<&mut Vec, String> { @@ -142,8 +473,22 @@ impl Sources { } } +/// A source's identity, the one every list is matched by: its kind and its address with a login +/// binding cleared, as [`Sources::from_json`] reads it. +fn identity(kind: &str, address: &str) -> (String, String) { + let mut address = address.to_string(); + crate::s3_connections::clear_bindings(std::iter::once(&mut address)); + (kind.to_string(), address) +} + +/// The identity of a saved row [`Sources::from_json`] reads (none for one it can't). +fn row_identity(row: &Value) -> Option<(String, String)> { + let source = clean(serde_json::from_value::(row.clone()).ok()?.source).ok()?; + Some(identity(&source.kind, &source.address)) +} + fn same(a: &Source, b: &Source) -> bool { - a.is(&b.kind, &b.address) + identity(&a.kind, &a.address) == identity(&b.kind, &b.address) } impl Source { @@ -155,6 +500,15 @@ impl Source { } } +/// The notice for `n` values left out of the saved data sources because they hold a secret. +pub fn secrets_notice(n: usize) -> String { + format!( + "{n} value{} in the saved data sources {} a password, key or token: not kept, and left out the next time the sources are saved", + if n == 1 { "" } else { "s" }, + if n == 1 { "holds" } else { "hold" } + ) +} + /// A source made safe to keep: a known kind, bounded text, and an address without secrets. pub fn clean(mut s: Source) -> Result { if !KINDS.contains(&s.kind.as_str()) { @@ -712,6 +1066,238 @@ MIIE assert!(s.generation > before); } + /// Rows, fields and lists a newer RustGIS wrote are kept through recording a recent source + /// and keeping a favourite here; a preference that can't be read at all is kept byte for + /// byte. Clean input says nothing. + #[test] + fn sources_keep_what_this_version_cannot_read() { + let future = json!({"kind": "gcs", "name": "Bucket", "address": "gs://b/a.tif"}); + let text = json!({ + "favourites": [{"kind": "file", "name": "A", "address": "/a.gpkg", "pinned": true}, future.clone()], + "recent": [{"kind": "file", "name": "R", "address": "/r.csv", "opened": 3}], + "saved": [], + "folders": ["/data"] + }) + .to_string(); + // With an editor's byte-order mark it reads the same. + let bom = Sources::from_json(&format!("\u{feff}{text}")); + assert_eq!((bom.favourites.len(), bom.unread().len()), (1, 2), "{:?}", bom.unread()); + let mut s = Sources::default(); + s.merge(Sources::from_json(&text)); + assert_eq!(s.unread().len(), 2, "{:?}", s.unread()); + assert!(s.unread()[0].starts_with("1 saved data source couldn't be read"), "{:?}", s.unread()); + s.note(Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }); + s.keep("favourites", Source { kind: "file".into(), address: "/b.gpkg".into(), ..Source::default() }).unwrap(); + let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); + assert_eq!(saved["folders"], json!(["/data"])); + assert_eq!(saved["favourites"][0]["pinned"], true); + assert_eq!(saved["favourites"][1]["address"], "/b.gpkg"); + assert_eq!(saved["favourites"][2], future); + assert_eq!(saved["recent"][1]["opened"], 3); + // A source that is no longer recent, or that is forgotten, takes its fields along. + for i in 0..MAX_RECENT { + s.note(Source { kind: "file".into(), address: format!("/{i}.csv"), ..Source::default() }); + } + s.forget("favourites", "file", "/a.gpkg").unwrap(); + assert!(!s.to_json().contains("opened") && !s.to_json().contains("pinned")); + assert!(s.to_json().contains("gs://b/a.tif")); + + for whole in ["[1]", r#"{"favourites": {"a": 1}}"#] { + let mut s = Sources::default(); + s.merge(Sources::from_json(whole)); + assert!(matches!(s.unread().as_slice(), [one] if one.contains("couldn't be read")), "{whole}"); + s.note(Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }); + assert_eq!(s.recent.len(), 1, "this session's sources are still listed"); + assert_eq!(s.to_json(), whole); + } + // Rows past a list's limit are kept, and said as that. + let recent: Vec = (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + let s = Sources::from_json(&json!({ "recent": recent }).to_string()); + assert_eq!(s.recent.len(), MAX_RECENT); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source is past")), "{:?}", s.unread()); + assert!(s.to_json().contains(&format!("/{MAX_RECENT}.csv"))); + // What it can't read and that holds a secret isn't kept: this version can't strip it. + let secret = json!({"token": "hidden", "saved": [ + {"kind": "url", "name": "A", "address": "https://a/x.fgb", "apiKey": "hidden"}, + {"kind": "gcs", "name": "B", "address": "https://g.example/b?sig=hidden"} + ]}); + let s = Sources::from_json(&secret.to_string()); + assert!( + s.unread().iter().any(|t| t.starts_with("3 values in the saved data sources hold a password, key or token")), + "{:?}", + s.unread() + ); + assert!(!s.to_json().contains("hidden")); + // A key beside a web address is one, in a readable source and among the preference's own keys. + let beside = json!({"endpoint": "https://api.example", "key": "abc", "saved": [ + {"kind": "url", "name": "A", "address": "https://a/x.fgb", "mirror": "https://m.example", "key": "abc"} + ]}); + let s = Sources::from_json(&beside.to_string()); + assert!( + !s.to_json().contains("abc") && s.to_json().contains("api.example") && s.to_json().contains("m.example"), + "{}", + s.to_json() + ); + assert!(s.unread().iter().any(|t| t.starts_with("2 values in the saved data sources hold")), "{:?}", s.unread()); + // A key beside an address among the preference's own keys. + let top = json!({"address": "https://api.example", "key": "abc"}); + assert!(!Sources::from_json(&top.to_string()).to_json().contains("abc")); + // Even when that address holds a secret itself. + let top = json!({"address": "https://api.example?token=hidden", "key": "abc"}); + let saved = Sources::from_json(&top.to_string()).to_json(); + assert!(!saved.contains("abc") && !saved.contains("hidden"), "{saved}"); + // A change before the first read that fills a list doesn't drop what was stored. + let stored: Vec = + (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "pinned": i})).collect(); + let mut s = Sources::default(); + s.keep("favourites", Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }).unwrap(); + s.merge(Sources::from_json(&json!({ "favourites": stored }).to_string())); + assert_eq!(s.favourites.len(), MAX_KEPT); + assert!(s.unread()[0].starts_with("1 saved data source is past"), "{:?}", s.unread()); + assert!(s.to_json().contains(&format!("/{}.csv", MAX_KEPT - 1)) && s.to_json().contains(&format!("\"pinned\":{}", MAX_KEPT - 1))); + // Rows of one identity are one source: the first row wins, with its fields, a later row is + // dropped and said (whatever it holds), and forgetting it leaves nothing behind. + for twins in [ + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "future": 1}, + {"kind": "file", "name": "A", "address": "/a.csv", "future": 2, "more": 3, "token": "hidden", "endpoint": "https://api.example"} + ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"}, + {"kind": "file", "name": "A", "address": "/a.csv", "endpoint": "https://api.example"} + ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}}, + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"} + ]}), + json!({"saved": [ + {"kind": "file", "name": "A", "address": "/a.csv", "key": "abc"}, + {"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}} + ]}), + ] { + let mut s = Sources::from_json(&twins.to_string()); + assert_eq!(s.saved.len(), 1); + let saved: Value = serde_json::from_str(&s.to_json()).unwrap(); + assert_eq!(saved["saved"].as_array().map(Vec::len), Some(1)); + assert!(saved["saved"][0].get("more").is_none() && !s.to_json().contains("hidden")); + // Nothing of a dropped row combines with the first: no key beside an endpoint. + assert!(!(s.to_json().contains("abc") && s.to_json().contains("api.example")), "{}", s.to_json()); + assert_eq!( + s.unread().first().map(String::as_str), + Some("1 repeated saved data source was removed (the first of each is kept, with its settings)") + ); + s.forget("saved", "file", "/a.csv").unwrap(); + assert!(!s.to_json().contains("/a.csv")); + } + // A source's known details can be changed after its fields were kept, as a key beside an + // endpoint it now has: checked again when it is written, by keep and by merge. + let a = || Source { + kind: "file".into(), + address: "/a.csv".into(), + extra: json!({"endpoint": "https://api.example"}), + ..Source::default() + }; + let kept = json!({"favourites": [{"kind": "file", "name": "", "address": "/a.csv", "key": "abc", "future": 1}]}).to_string(); + let mut s = Sources::from_json(&kept); + assert!(s.to_json().contains("abc")); + s.keep("favourites", a()).unwrap(); + assert!( + !s.to_json().contains("abc") && s.to_json().contains("\"future\":1") && s.to_json().contains("api.example"), + "{}", + s.to_json() + ); + assert!(s.unread().iter().any(|t| t.starts_with("1 value in the saved data sources holds")), "{:?}", s.unread()); + let mut s = Sources::default(); + s.keep("favourites", a()).unwrap(); + s.merge(Sources::from_json(&kept)); + assert!(!s.to_json().contains("abc") && s.to_json().contains("api.example"), "{}", s.to_json()); + // A readable row kept past the limit gives its fields to the source when it is listed. + let recent: Vec = + (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "future": i})).collect(); + let mut s = Sources::from_json(&json!({ "recent": recent }).to_string()); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let mut s = Sources::from_json(&s.to_json()); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let saved: Value = serde_json::from_str(&Sources::from_json(&s.to_json()).to_json()).unwrap(); + assert_eq!(saved["recent"][0]["future"], json!(MAX_RECENT), "{saved}"); + assert_eq!(saved["recent"].as_array().map(Vec::len), Some(MAX_RECENT)); + // The same by merge: the session lists a source the stored preference kept past the limit. + let mut s = Sources::default(); + s.note(Source { kind: "file".into(), address: format!("/{MAX_RECENT}.csv"), ..Source::default() }); + let recent: Vec = + (0..=MAX_RECENT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv"), "future": i})).collect(); + s.merge(Sources::from_json(&json!({ "recent": recent }).to_string())); + assert!(s.to_json().contains(&format!("\"future\":{MAX_RECENT}")), "{}", s.to_json()); + assert_eq!(s.to_json().matches(&format!("/{MAX_RECENT}.csv")).count(), 1, "{}", s.to_json()); + // The row's known details count beside an unknown key, as its address does. + let extra = json!({"saved": [{"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}, "key": "abc"}]}); + let saved = Sources::from_json(&extra.to_string()).to_json(); + assert!(!saved.contains("abc") && saved.contains("api.example"), "{saved}"); + // Sources merged past the limit are said by the merged sources. + let mut s = Sources::default(); + s.keep("favourites", Source { kind: "file".into(), address: "/new.csv".into(), ..Source::default() }).unwrap(); + let stored: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + s.merge(Sources::from_json(&json!({ "favourites": stored }).to_string())); + assert!(s.unread().iter().any(|t| t.contains("past the lists' limits")), "{:?}", s.unread()); + // A repeat past the limit is dropped, not kept as an overflow row. A bound address is the + // bare one's identity: forgetting by either form removes a kept over-limit row of it, never + // an unreadable row that merely shares the address. + let bound = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + let mut full: Vec = (0..MAX_KEPT).map(|i| json!({"kind": "file", "name": "", "address": format!("/{i}.csv")})).collect(); + full.push(json!({"kind": "file", "name": "", "address": "/0.csv", "token": "hidden"})); + let s = Sources::from_json(&json!({"saved": full.clone()}).to_string()); + assert!(!s.to_json().contains("hidden") && s.kept.over == 0 && s.kept.repeated == 1); + full.truncate(MAX_KEPT); + full.push(json!({"kind": "url", "name": "", "address": bound})); + let twin = rustgis_data::s3_address::bind("fedcba9876543210", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + full.push(json!({"kind": "url", "name": "", "address": twin, "gcs": 1})); + full.push(json!({"kind": "gcs", "name": "", "address": "gs://b/a.tif"})); + full.push(json!({"kind": "future", "name": "", "address": twin})); + full.push(json!({"kind": " url ", "name": 5, "address": twin})); + let mut s = Sources::from_json(&json!({"saved": full}).to_string()); + assert_eq!(s.kept.over, 2, "{:?}", s.unread()); + let generation = s.generation; + // Only kept rows go: that is still a removal, and the pane must still save. + assert!(s.forget("saved", "url", &twin).unwrap() && s.generation != generation); + assert!( + s.to_json().contains("gs://b/a.tif") + && s.to_json().contains("\"future\"") + && !s.to_json().contains("\"gcs\":1") + && s.to_json().contains("\"name\":5") + ); + assert_eq!(s.kept.over, 0); + // A login binding is the app's own output: cleared on read without a notice. + let s = Sources::from_json(&json!({"saved": [{"kind": "url", "name": "B", "address": bound}]}).to_string()); + assert!( + s.unread().is_empty() && rustgis_data::s3_address::parse(&s.saved[0].address).unwrap().unwrap().binding.is_empty(), + "{:?}", + s.unread() + ); + // A file's contents beside a field isn't an address for it. + let file = json!({"data": "https://example.invalid/a.csv", "key": "feature_id"}); + assert!(Sources::from_json(&file.to_string()).to_json().contains("feature_id")); + // An address with a signed query is changed, and said. + let signed = json!({"saved": [{"kind": "url", "name": "U", "address": "https://a/x.fgb?sig=hidden"}]}); + let s = Sources::from_json(&signed.to_string()); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("1 saved data source has")), "{:?}", s.unread()); + // Details and a name this version cuts down when it reads them are said. + let long = "n".repeat(300); + let cut = json!({"saved": [ + {"kind": "service", "name": "S", "address": "https://h/wms", "extra": {"type": "wms", "future": {"mode": 1}}}, + {"kind": "file", "name": long, "address": "/a.csv"} + ]}); + let s = Sources::from_json(&cut.to_string()); + assert!(matches!(s.unread().as_slice(), [one] if one.starts_with("2 saved data sources have")), "{:?}", s.unread()); + for clean in [ + "", + r#"{"favourites": [], "recent": [], "saved": []}"#, + r#"{"recent": [{"kind": "file", "name": " R ", "address": "/r.csv", "extra": {}}]}"#, + r#"{"saved": [{"kind": "service", "name": "", "address": "https://h/wms", "extra": {"type": "wms"}}]}"#, + ] { + assert!(Sources::from_json(clean).unread().is_empty(), "{clean}"); + } + } + #[test] fn identity_is_kind_and_address() { let mut s = Sources::default(); @@ -930,7 +1516,10 @@ MIIE assert_eq!(back.favourites.len(), 1); assert_eq!(back.favourites[0].address, "https://h/x.fgb"); assert!(back.recent.is_empty()); - assert_eq!(Sources::from_json("not json"), Sources::default()); + // The unreadable rows, and the address cut of its token. + assert_eq!(back.unread().len(), 2, "{:?}", back.unread()); + let unreadable = Sources::from_json("not json"); + assert!(unreadable.favourites.is_empty() && unreadable.recent.is_empty() && unreadable.saved.is_empty()); } #[test] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 10fd16574e..e51a6f49fc 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -28342,6 +28342,27 @@ fn connection_browse_levels_and_adds() { assert!(!s.can_undo()); } +/// A kept field that only holds a secret once a source's details change is left out then, and +/// said then, not only at start (the start-up notice has already been shown). +#[test] +fn a_secret_left_out_when_a_source_changes_is_said() { + let mut s = Session::new(Project::default()); + s.sources = crate::sources::Sources::from_json(r#"{"saved":[{"kind":"file","name":"A","address":"/a.csv","key":"abc"}]}"#); + // Kept at start as a setting this version doesn't know: nothing beside it makes it a secret. + assert!(!s.sources.unread().iter().any(|t| t.contains("password")), "{:?}", s.sources.unread()); + let mark = s.notices_given(); + s.run("sources.save", &json!({"kind": "file", "name": "A", "address": "/a.csv"})).unwrap(); + assert!(s.notices_since(mark).is_empty(), "{:?}", s.notices_since(mark)); + s.run("sources.save", &json!({"kind": "file", "name": "A", "address": "/a.csv", "extra": {"endpoint": "https://api.example"}})) + .unwrap(); + assert!( + s.notices_since(mark).iter().any(|n| n.text.starts_with("1 value in the saved data sources holds a password")), + "{:?}", + s.notices_since(mark) + ); + assert!(!s.sources.to_json().contains("abc")); +} + fn rustgis_engine_sources_roundtrip(text: &str) -> crate::sources::Sources { crate::sources::Sources::from_json(text) } diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 8febc712da..6d0182684d 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -51,17 +51,49 @@ struct State { disabled: Vec, #[serde(default)] folders: Vec, + /// Keys a newer RustGIS added, written back as they were. + #[serde(flatten)] + other: serde_json::Map, + /// Why `state.json` couldn't be read, when it couldn't: it is then never written over. + #[serde(skip)] + unreadable: Option, } fn state_path() -> Option { dir().map(|d| d.join("state.json")) } +/// `state.json`: none, or an empty one, is no plugin disabled or added from a folder. One that can't +/// be read (or that a newer RustGIS wrote in a shape this one doesn't know) is said by +/// [`load_installed`], which then loads no plugin, and kept as it is: [`write_state`] refuses to +/// write over it. fn read_state() -> State { - state_path().and_then(|p| std::fs::read_to_string(p).ok()).and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() + let Some(path) = state_path() else { return State::default() }; + let read = match std::fs::read_to_string(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return State::default(), + // Written whole each time, so an empty file was made empty on purpose, not cut short. A + // byte-order mark (an editor's) isn't part of the JSON. + Ok(text) if text.trim_start_matches('\u{feff}').trim().is_empty() => return State::default(), + Err(e) => Err(e.to_string()), + Ok(text) => serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|e| e.to_string()), + }; + read.unwrap_or_else(|e| State { unreadable: Some(format!("{}: {e}", path.display())), ..State::default() }) +} + +/// Refuses a change to `state.json` when [`read_state`] couldn't read the one there. Callers ask +/// before they change anything in memory, so a refused change isn't half made. +fn writable(st: &State) -> Result<(), String> { + match &st.unreadable { + Some(why) => Err(format!( + "{why}; it is kept as it is, so which plugins are disabled or added from folders can't be changed until it is fixed or removed" + )), + None => Ok(()), + } } +/// Writes `state.json`, refused when [`read_state`] couldn't read the one there. fn write_state(st: &State) -> Result<(), String> { + writable(st)?; let path = state_path().ok_or("there is no folder for plugins")?; if let Some(d) = path.parent() { std::fs::create_dir_all(d).map_err(|e| format!("{}: {e}", d.display()))?; @@ -94,6 +126,14 @@ pub fn load_installed() -> Vec { )), } let st = read_state(); + // Which plugins are disabled is in that file, and a disabled one may be a native program: with + // it unreadable none is loaded, rather than starting what the user turned off. + if let Some(why) = &st.unreadable { + errors.push(format!( + "{why}: plugins are off because this file couldn't be read; repairing or removing it brings them back (it is kept as it is until then)" + )); + return errors; + } let mut paths: Vec = std::fs::read_dir(&dir) .map(|rd| rd.flatten().map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("rgplugin"))).collect()) .unwrap_or_default(); @@ -248,6 +288,9 @@ fn extract(bytes: &[u8], to: &Path) -> Result<(), String> { /// Installs a package's bytes: checks it, copies it into the plugins folder (replacing an /// older copy) and loads it. pub fn install_bytes(bytes: &[u8]) -> Result, String> { + // Installed plugins start enabled: while `state.json` can't be read, the one being installed + // may be one the user disabled (and a native program), so it isn't installed either. + writable(&read_state())?; let mut pkg = Package::from_zip(bytes)?; // A process plugin is checked (blocklist, platform) before its files replace an installed // copy's: a package that won't load leaves the old one working. @@ -298,9 +341,12 @@ pub fn install_path(path: &Path) -> Result, String> { return install_bytes(&bytes); } let pkg = Package::from_dir(path)?; - let plugin = crate::load(pkg, &path.display().to_string())?; let mut st = read_state(); let p = path.display().to_string(); + if !st.folders.contains(&p) { + writable(&st)?; + } + let plugin = crate::load(pkg, &p)?; if !st.folders.contains(&p) { st.folders.push(p); write_state(&st)?; @@ -334,8 +380,9 @@ pub fn uninstall(id: &str) -> Result<(), String> { /// Enables or disables a plugin, and remembers it. pub fn set_enabled(id: &str, enabled: bool) -> Result<(), String> { - crate::set_enabled(id, enabled)?; let mut st = read_state(); + writable(&st)?; + crate::set_enabled(id, enabled)?; st.disabled.retain(|d| d != id); if !enabled { st.disabled.push(id.to_string()); diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index a5e21d53be..9e9b8a73eb 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -1846,6 +1846,63 @@ fn cached_blocklist_survives_refused_refreshes() { let _ = std::fs::remove_dir_all(&plugins); } +/// `state.json` keeps what a newer RustGIS added through a change here, and one that can't be +/// read is said at start and never written over. +#[test] +fn plugin_state_keeps_what_this_version_cannot_read() { + let plugins = std::env::temp_dir().join(format!("rustgis-test-plugin-state-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&plugins); + let _store = store_lock(); + crate::store::set_dir(Some(plugins.clone())); + std::fs::create_dir_all(&plugins).unwrap(); + let state = plugins.join("state.json"); + std::fs::write(&state, r#"{"disabled": [], "folders": [], "pinned": ["kept"]}"#).unwrap(); + crate::store::install_bytes(&testing::package("state-test", &[])).unwrap(); + crate::store::set_enabled("state-test", false).unwrap(); + let saved: Value = serde_json::from_str(&std::fs::read_to_string(&state).unwrap()).unwrap(); + assert_eq!((&saved["disabled"], &saved["pinned"]), (&json!(["state-test"]), &json!(["kept"]))); + for unreadable in ["not json", r#"{"disabled": [{"id": "state-test", "until": 2}]}"#] { + std::fs::write(&state, unreadable).unwrap(); + assert!(crate::store::load_installed().iter().any(|e| e.contains("state.json")), "{unreadable}"); + let e = crate::store::set_enabled("state-test", true).unwrap_err(); + assert!(e.contains("kept as it is"), "{e}"); + // A refused change leaves the plugin as it was, not changed for this session only. + assert!(crate::get("state-test").is_some_and(|p| !p.enabled)); + let folder = plugins.join("folder-plugin"); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::write(folder.join("plugin.json"), crate::testing::manifest("folder-test", &[])).unwrap(); + std::fs::write(folder.join("plugin.wasm"), testing::wasm()).unwrap(); + assert!(crate::store::install_path(&folder).is_err() && crate::get("folder-test").is_none()); + assert!(crate::store::install_bytes(&testing::package("state-test", &[])).is_err_and(|e| e.contains("kept as it is"))); + assert!(crate::store::uninstall("state-test").is_err()); + assert_eq!(std::fs::read_to_string(&state).unwrap(), unreadable); + // At start none is loaded (it may have been disabled, and be a native program), and it is said. + crate::unload("state-test"); + let errors = crate::store::load_installed(); + assert!(errors.len() == 1 && errors[0].contains("state.json") && errors[0].contains("plugins are off"), "{errors:?}"); + assert!(crate::get("state-test").is_none()); + std::fs::write(&state, r#"{"disabled": ["state-test"]}"#).unwrap(); + assert!(crate::store::load_installed().is_empty()); + } + // No file, or an empty one, is nothing disabled, with no notice. + for empty in [None, Some(" \n"), Some("\u{feff}")] { + match empty { + Some(text) => std::fs::write(&state, text).unwrap(), + None => std::fs::remove_file(&state).unwrap(), + } + crate::unload("state-test"); + assert!(crate::store::load_installed().is_empty(), "{empty:?}"); + assert!(crate::get("state-test").is_some_and(|p| p.enabled), "{empty:?}"); + } + // An editor's byte-order mark isn't part of the JSON. + std::fs::write(&state, "\u{feff}{\"disabled\": [\"state-test\"]}").unwrap(); + crate::unload("state-test"); + assert!(crate::store::load_installed().is_empty()); + assert!(crate::get("state-test").is_some_and(|p| !p.enabled)); + crate::store::uninstall("state-test").unwrap(); + let _ = std::fs::remove_dir_all(&plugins); +} + #[test] fn process_packages_unpack_safely() { use std::io::Write; diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index d7c381d2c8..51a2ed8c0d 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -99,7 +99,7 @@ pub(crate) fn name_recent(app: &mut RustGisApp, before: u64, origin: Option<&str } let Some(mut added) = app.session.sources.recent.first().cloned() else { return }; added.name = origin.to_string(); - app.session.sources.note(added); + app.session.note_source(added); } pub enum Entry { diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 43192ad936..ade993aaa8 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -40,6 +40,9 @@ struct Saved { data: Option, #[serde(default = "yes")] enabled: bool, + /// Fields a newer RustGIS added, written back as they were. + #[serde(flatten)] + other: serde_json::Map, } fn yes() -> bool { @@ -94,6 +97,8 @@ pub struct PluginsState { /// Ids of plugins that failed to load at start, not yet told (one notice for all, once every /// start download is done). start_failures: Vec, + /// Whether this session was told that changes to the installed plugins aren't remembered. + unsaved_told: bool, } /// A plugin's Python environment being made on a background thread. @@ -282,23 +287,65 @@ fn keeps_prefs() -> bool { cfg!(target_arch = "wasm32") } -fn saved(app: &RustGisApp) -> Vec { - app.services.load_pref.as_ref().and_then(|l| l("plugins_web")).and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() +/// The installed plugins the web app keeps (`plugins_web`): those it can read, and the entries it +/// can't (from a newer RustGIS, or damaged), kept to be written back as they were. `None` when +/// the preference isn't a JSON list: it is then never written over. None kept is an empty list. +fn read_saved(text: Option<&str>) -> Option<(Vec, Vec)> { + let Some(text) = text.filter(|t| !t.trim().is_empty()) else { return Some((Vec::new(), Vec::new())) }; + let Ok(Value::Array(rows)) = serde_json::from_str::(text) else { return None }; + let (mut list, mut kept) = (Vec::new(), Vec::new()); + for row in rows { + match serde_json::from_value::(row.clone()) { + Ok(s) => list.push(s), + Err(_) => kept.push(row), + } + } + Some((list, kept)) } -fn save(app: &RustGisApp, list: &[Saved]) { - if let (Some(s), Ok(text)) = (&app.services.save_pref, serde_json::to_string(list)) { - s("plugins_web", &text); - } +/// The `plugins_web` preference as [`read_saved`] reads it. +fn saved(app: &RustGisApp) -> Option<(Vec, Vec)> { + read_saved(app.services.load_pref.as_ref().and_then(|l| l("plugins_web")).as_deref()) } -fn update_saved(app: &RustGisApp, f: impl FnOnce(&mut Vec)) { +/// The `plugins_web` preference for `list`, with the entries [`read_saved`] couldn't read after it. +fn saved_text(list: &[Saved], kept: &[Value]) -> String { + let mut rows: Vec = list.iter().filter_map(|s| serde_json::to_value(s).ok()).collect(); + rows.extend(kept.iter().cloned()); + Value::Array(rows).to_string() +} + +/// Changes the installed plugins the web app keeps with `f`, writing back the entries it couldn't +/// read as they were; nothing on the desktop, or when the preference couldn't be read. +fn update_saved(app: &mut RustGisApp, f: impl FnOnce(&mut Vec)) { if !keeps_prefs() { return; } - let mut list = saved(app); + // One that can't be read was said at start, and isn't written over. + let Some((mut list, kept)) = saved(app) else { + if !std::mem::replace(&mut app.plugins.unsaved_told, true) { + app.session.warn(tl!( + "The plugins installed in this browser can't be read, so this change isn't remembered: fix or remove \"plugins_web\" in the browser's saved preferences." + )); + } + return; + }; f(&mut list); - save(app, &list); + if let Some(s) = &app.services.save_pref { + s("plugins_web", &saved_text(&list, &kept)); + } +} + +/// Puts `entry` in `list` in place of the one with its id, which hands it the fields this version +/// doesn't know; `keep` false only removes the old one. +fn put_saved(list: &mut Vec, mut entry: Saved, keep: bool) { + if let Some(old) = list.iter().find(|s| s.id == entry.id) { + entry.other = old.other.clone(); + } + list.retain(|s| s.id != entry.id); + if keep { + list.push(entry); + } } /// Loads the plugins the web app installed in earlier visits. @@ -309,7 +356,22 @@ pub fn restore(app: &mut RustGisApp) { if let Some(u) = app.services.load_pref.as_ref().and_then(|l| l("plugin_registry")) { app.plugins.url = u; } - restore_list(app, saved(app)); + let (list, kept) = match saved(app) { + Some(read) => read, + None => { + app.session.warn(tl!( + "The plugins installed in this browser couldn't be read (they aren't a JSON list): none is loaded, and plugins installed or removed now aren't remembered." + )); + (Vec::new(), Vec::new()) + } + }; + if !kept.is_empty() { + app.session.warn(crate::i18n::fmt( + "{n} installed plugins couldn't be read (from a newer RustGIS, or damaged): they aren't loaded, and are kept as they were.", + &[("n", &kept.len().to_string())], + )); + } + restore_list(app, list); // The blocklist, so a pulled plugin isn't loaded again. let registry = app.plugins.registry_url(); let slot = fetch(app, &rp::registry::blocklist_url(®istry), 4 << 20, None); @@ -483,14 +545,10 @@ fn install_confirmed(app: &mut RustGisApp) { base64::engine::general_purpose::STANDARD.encode(&c.bytes) }), enabled: true, + other: serde_json::Map::new(), }; let kept = entry.url.is_some() || entry.data.is_some(); - update_saved(app, |list| { - list.retain(|s| s.id != entry.id); - if kept { - list.push(entry); - } - }); + update_saved(app, |list| put_saved(list, entry, kept)); let mut text = crate::i18n::fmt( "Installed {name} {version}. Find it on the Plugins tab; its tools are also in the Tools pane.", &[("name", &p.manifest.name), ("version", &p.manifest.version)], @@ -1263,6 +1321,7 @@ mod tests { url: Some(format!("https://example.invalid/{id}.rgplugin")), data: None, enabled: true, + other: serde_json::Map::new(), } } @@ -1285,6 +1344,35 @@ mod tests { app.session.notices_since(mark).iter().map(|n| (n.text.clone(), n.error)).collect() } + /// What a newer RustGIS wrote to `plugins_web` survives a change here: entries this version + /// can't read and fields it doesn't know are written back as they were; a preference that + /// isn't a list isn't read at all. + #[test] + fn saved_web_plugins_keep_what_this_version_cannot_read() { + let text = json!([ + {"id": "a", "sha256": "0", "url": "https://example.invalid/a.rgplugin", "enabled": true, "channel": "beta"}, + {"id": "b", "sha256": {"v2": "0"}} + ]) + .to_string(); + let (mut list, kept) = read_saved(Some(&text)).unwrap(); + assert_eq!((list.len(), kept.len()), (1, 1)); + list[0].enabled = false; + list.push(record("c", "1")); + let again: Value = serde_json::from_str(&saved_text(&list, &kept)).unwrap(); + assert_eq!(again[0]["channel"], "beta"); + assert_eq!(again[0]["enabled"], false); + assert_eq!(again[1]["id"], "c"); + assert_eq!(again[2], json!({"id": "b", "sha256": {"v2": "0"}})); + // Installing it again (an update) keeps the fields this version doesn't know. + put_saved(&mut list, record("a", "2"), true); + assert_eq!( + (list.len(), list.last().map(|s| s.sha256.as_str()), list.last().and_then(|s| s.other.get("channel"))), + (2, Some("2"), Some(&json!("beta"))) + ); + assert!(read_saved(Some("{\"plugins\": []}")).is_none()); + assert_eq!(read_saved(None).map(|(l, k)| (l.len(), k.len())), Some((0, 0))); + } + /// Plugins the web app couldn't load at start are listed, kept, and told once (#1352). #[test] fn plugins_that_fail_at_start_are_listed_and_told_once() { diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 9713dbffbd..8fad253834 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -53,12 +53,22 @@ pub fn load(app: &mut RustGisApp) { return; } app.ui.sources.loaded = true; + // A change made before this first read (a source just added, which is what usually brings + // it on) is still to be saved, with the stored lists. + let changed = app.session.sources.generation != app.ui.sources_saved; if let Some(text) = app.services.load_pref.as_ref().and_then(|load| load(PREF)) { // What this session already has (added, saved or made favourite before the first read) - // stays; the stored lists are added after it. - app.session.sources.merge(rustgis_engine::sources::Sources::from_json(&text)); + // stays; the stored lists are added after it. What of them isn't used here is said. + let stored = rustgis_engine::sources::Sources::from_json(&text); + app.session.sources.merge(stored); + // After the merge: sources it moved past a list's limit are said too. + for said in app.session.sources.unread() { + app.session.warn(said); + } + } + if !changed { + app.ui.sources_saved = app.session.sources.generation; } - app.ui.sources_saved = app.session.sources.generation; } /// Keeps the sources with the preferences when they changed. diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index b0baa0f599..4117193cc6 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1457,6 +1457,54 @@ fn connections_persist_and_import_once() { assert!(empty.session.connections.connections.is_empty(), "an empty saved list must not import again"); } +/// Saved data sources from a newer RustGIS (#1405): rows, fields and lists this version can't read +/// are said at start and survive recording a recent source; a preference that can't be read at +/// all stays byte for byte. +#[test] +fn saved_sources_from_a_newer_version_survive_recording_a_recent_one() { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("points.geojson"); + std::fs::write( + &file, + r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1,2]},"properties":{}}]}"#, + ) + .unwrap(); + let future = json!({"kind": "gcs", "name": "Bucket", "address": "gs://b/a.tif"}); + let newer = json!({"favourites": [future.clone()], "recent": [{"kind": "file", "name": "R", "address": "/r.csv", "opened": 3}], "folders": ["/data"]}) + .to_string(); + for text in [newer, "[\"not\", \"an object\"]".to_string()] { + let store = Arc::new(Mutex::new(HashMap::from([ + ("data_sources".to_string(), text.clone()), + ("connections".to_string(), "{}".to_string()), + ]))); + let (load, save) = (store.clone(), store.clone()); + let services = rustgis_ui_egui::Services { + load_pref: Some(Box::new(move |k| load.lock().unwrap().get(k).cloned())), + save_pref: Some(Box::new(move |k, v| { + save.lock().unwrap().insert(k.to_string(), v.to_string()); + })), + ..Default::default() + }; + let mut app = RustGisApp::with_sample().with_services(services); + app.run("layer.addData", json!({"path": file.display().to_string()})).unwrap(); + assert!(app.session.sources.recent.iter().any(|r| r.address.ends_with("points.geojson")), "{text}"); + let said: Vec<_> = app.session.notices_since(0).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("saved data source")), "{said:?}"); + let stored = store.lock().unwrap()["data_sources"].clone(); + if text.starts_with('[') { + assert_eq!(stored, text, "a preference that can't be read stays as it was"); + } else { + let stored: serde_json::Value = serde_json::from_str(&stored).unwrap(); + assert_eq!(stored["favourites"], json!([future])); + assert_eq!(stored["folders"], json!(["/data"])); + assert!(stored["recent"][0]["address"].as_str().unwrap().ends_with("points.geojson")); + assert_eq!(stored["recent"][1]["opened"], 3); + } + } +} + /// Saved connections from a newer RustGIS (#1405): a preference of a newer version is said, isn't /// used and stays byte for byte through other changes to the preferences, recording a recent /// source among them; rows and fields this version can't read survive a connection made here.