Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 29 additions & 11 deletions crates/engine/src/cmd/plugins.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,14 +150,28 @@ fn enable(_: &mut Session, p: &Value) -> CmdResult {

/// The registry at `url` with each plugin's installed version. Applies its blocklist.
#[cfg(not(target_arch = "wasm32"))]
fn fetch_registry(url: &str) -> Result<Vec<rustgis_plugins::registry::Entry>, String> {
fn fetch_registry(s: &mut Session, url: &str) -> Result<Vec<rustgis_plugins::registry::Entry>, String> {
use rustgis_plugins::registry;
let text = registry::fetch(url, registry::MAX_REGISTRY_BYTES)?;
let entries = registry::parse(&String::from_utf8_lossy(&text), url)?;
// A registry without a blocklist blocks nothing; one that can't be fetched leaves the
// cached list in place.
if let Ok(b) = registry::fetch(&registry::blocklist_url(url), 4 * 1024 * 1024) {
let _ = rustgis_plugins::store::save_blocklist(url, &String::from_utf8_lossy(&b));
// A blocklist that can't be fetched or read (one that isn't text included: a replaced byte in
// an id would match nothing) leaves the list in force, with a warning. Another registry may
// have no blocklist (it blocks nothing); the hosted one always has one.
let blocklist = registry::blocklist_url(url);
let applied = match registry::fetch(&blocklist, 4 * 1024 * 1024) {
Ok(b) => {
std::str::from_utf8(&b).map_err(|e| format!("it isn't text: {e}")).and_then(|t| rustgis_plugins::store::save_blocklist(url, t))
}
Err(e) if registry::is_default(url) => Err(e),
Err(_) => Ok(Vec::new()),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
};
match applied {
Ok(pulled) if !pulled.is_empty() => s.notify(
format!("{} plugin(s) were withdrawn from the plugin registry and turned off: {}", pulled.len(), pulled.join(", ")),
true,
),
Ok(_) => {}
Err(e) => s.warn(format!("{blocklist}: {e}; the last blocklist read, if any, stays in force")),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
Ok(entries)
}
Expand All @@ -178,15 +192,17 @@ pub fn entry_status(e: &rustgis_plugins::registry::Entry) -> Value {
v
}

fn browse(_: &mut Session, p: &Value) -> CmdResult {
/// `plugin.registry`: the registry's plugins, each with its installed version (see
/// [`fetch_registry`]).
fn browse(s: &mut Session, p: &Value) -> CmdResult {
let url = ps(p, "url").map(str::trim).filter(|s| !s.is_empty()).unwrap_or(rustgis_plugins::registry::DEFAULT_URL);
#[cfg(not(target_arch = "wasm32"))]
{
Ok(Value::Array(fetch_registry(url)?.iter().map(entry_status).collect()))
Ok(Value::Array(fetch_registry(s, url)?.iter().map(entry_status).collect()))
}
#[cfg(target_arch = "wasm32")]
{
let _ = url;
let _ = (s, url);
Err("in the browser, browse plugins in the Plugin Manager".into())
}
}
Expand Down Expand Up @@ -220,19 +236,21 @@ pub fn install_checked(e: &rustgis_plugins::registry::Entry, bytes: &[u8]) -> Re
install_bytes(bytes)
}

fn install_from_registry(_: &mut Session, p: &Value) -> CmdResult {
/// Installs the plugin `id` from the registry, checked against its entry (see [`install_checked`]),
/// after applying the registry's blocklist.
fn install_from_registry(s: &mut Session, p: &Value) -> CmdResult {
let id = ps(p, "id").ok_or("id is required")?;
let url = ps(p, "url").map(str::trim).filter(|s| !s.is_empty()).unwrap_or(rustgis_plugins::registry::DEFAULT_URL);
#[cfg(not(target_arch = "wasm32"))]
{
let entries = fetch_registry(url)?;
let entries = fetch_registry(s, url)?;
let e = entries.iter().find(|e| e.id == id).ok_or_else(|| format!("the registry has no plugin '{id}'"))?;
let bytes = rustgis_plugins::registry::fetch(&e.package, rustgis_plugins::package::MAX_PACKAGE)?;
Ok(install_checked(e, &bytes)?.info())
}
#[cfg(target_arch = "wasm32")]
{
let _ = (id, url);
let _ = (s, id, url);
Err("in the browser, install plugins from the Plugin Manager".into())
}
}
Expand Down
39 changes: 39 additions & 0 deletions crates/engine/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12680,6 +12680,45 @@ fn serve_files(files: Vec<(String, Vec<u8>)>) -> (String, std::sync::Arc<std::sy
(base, asked)
}

/// A registry whose blocklist can't be read warns that the last blocklist read stays in force;
/// one with a readable blocklist says nothing, apart from naming the plugins it turned off.
#[test]
fn unreadable_plugin_blocklist_warns() {
let warned = |blocklist: &[u8]| {
let (base, _) =
serve_files(vec![("/registry.json".into(), br#"{"plugins": []}"#.to_vec()), ("/blocklist.json".into(), blocklist.to_vec())]);
let mut s = Session::new(Project::default());
let mark = s.notices_given();
s.run("plugin.registry", &json!({"url": format!("{base}/registry.json")})).unwrap();
let said: Vec<_> = s.notices_since(mark).iter().filter(|n| n.text.contains("stays in force")).collect();
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
assert!(said.iter().all(|n| n.severity == rustgis_data::diagnostics::Severity::Warning), "{said:?}");
// It names the blocklist and why it was refused.
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
assert!(
said.iter().all(|n| n.text.contains(&format!("{base}/blocklist.json")) && n.text.contains("no 'blocked' list")),
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
"{said:?}"
);
said.len()
};
assert_eq!(warned(br#"{"blocked": {}}"#), 1);
assert_eq!(warned(br#"{"blocked": []}"#), 0);
// In the same test, as both apply blocklists, which unload plugins process-wide: one that
// isn't text is refused with a warning, and one that withdraws a loaded plugin says so.
let run = |blocklist: &[u8]| {
let (base, _) =
serve_files(vec![("/registry.json".into(), br#"{"plugins": []}"#.to_vec()), ("/blocklist.json".into(), blocklist.to_vec())]);
let mut s = Session::new(Project::default());
let mark = s.notices_given();
s.run("plugin.registry", &json!({"url": format!("{base}/registry.json")})).unwrap();
s.notices_since(mark).iter().map(|n| n.text.clone()).collect::<Vec<_>>()
};
let said = run(b"[{\"id\": \"eng-\xff\"}]");
assert!(said.iter().any(|t| t.contains("isn't text") && t.contains("stays in force")), "{said:?}");
plugin("eng-withdrawn", &[]);
let said = run(br#"[{"id": "eng-withdrawn"}]"#);
assert!(said.iter().any(|t| t.contains("1 plugin(s) were withdrawn") && t.contains("Test eng-withdrawn")), "{said:?}");
assert!(rustgis_plugins::get("eng-withdrawn").is_none());
}

/// A Shapefile published as loose files is added by its `.shp` address (#1238): its `.dbf`,
/// `.shx`, `.prj` and `.cpg` are fetched beside it with the address's query, and its `.prj` places
/// it; the parts that aren't there are done without, said, but not its `.dbf`, nor a part a
Expand Down
39 changes: 32 additions & 7 deletions crates/plugins/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use crate::manifest::{Permission, Runtime, valid_id, version_parts};
pub const DEFAULT_URL: &str = "https://plugins.rustgis.app/registry.json";

/// The most entries read from a registry (or a blocklist): room for tens of thousands of plugins.
const MAX_ENTRIES: usize = 100_000;
pub(crate) const MAX_ENTRIES: usize = 100_000;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit, outside this change's logic: this constant is also what the registry reader uses at registry.rs:154 (.take(MAX_ENTRIES)), which still drops registry entries past the cap with no notice. Same rule as the blocklist fix, so it can be handled in the release quality pass or in this PR. Confidence: medium that it matters, since the cap is unlikely to be reached in practice.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The registry reader's .take(MAX_ENTRIES) predates this PR and isn't changed by it, so it stays out of this blocklist fix.


/// The largest registry downloaded (a registry entry is about 1 KB, so this is ample).
pub const MAX_REGISTRY_BYTES: usize = 64 * 1024 * 1024;
Expand Down Expand Up @@ -192,15 +192,40 @@ pub fn blocklist_url(registry: &str) -> String {
resolve(registry, "blocklist.json")
}

/// Reads a blocklist (an object with a `blocked` array, or a bare array).
/// Reads a blocklist (an object with a `blocked` array, or a bare array). Any other shape, an
/// entry whose `id` isn't a plugin id or whose `sha256`, when given, isn't a SHA-256 hash (either
/// would match nothing; no `sha256`, or `null`, withdraws every version), or more than 100,000
/// entries, is an error rather than a shorter list, so a malformed refresh leaves the last good
/// list in place: only `[]` or `{"blocked": []}` clears it.
pub fn parse_blocklist(text: &str) -> Result<Vec<Blocked>, String> {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Silent discard of the cached blocklist at startup (Never silent). The new strict parser makes parse_blocklist fail in cases where it used to return a partial list, and the startup reader drops that failure without a word. crates/plugins/src/store.rs:78 (unchanged by this PR) does read_to_string(...).ok().and_then(|t| registry::parse_blocklist(&t).ok()), so a cache that now fails to parse is ignored entirely. Example: a cache written by an older version containing [{"id": "a"}, {"reason": "no id"}] kept a withdrawn before this PR, and now loads with no withdrawals at all, with no notice. The same happens to a cache over 100,000 entries, which the comment thread already accepts, but the silence is not covered by that answer.

Suggested direction: match on the parse result in load_installed, and push the reason into the errors it returns (e.g. format!("{}: {e}", path.display())), so the caller reports it, and add a test that writes a bad cache and asserts the error is reported. Confidence: high on the behaviour, medium on how often it happens (it needs an older cache that the new validation refuses, read on an offline start).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 7a442ad: load_installed puts a cached blocklist it can't read into the errors it returns, with the reason and that withdrawn plugins aren't blocked until the blocklist is fetched again. The desktop app shows those errors as notices and the command line prints them. New test cached_blocklist_survives_refused_refreshes checks the report for [{"id": "a"}, {"reason": "no id"}].

let root: Value = serde_json::from_str(text).map_err(|e| format!("the blocklist isn't valid JSON: {e}"))?;
let list = match &root {
Value::Array(a) => a.clone(),
Value::Object(o) => o.get("blocked").and_then(Value::as_array).cloned().unwrap_or_default(),
_ => Vec::new(),
let list = match root {
Value::Array(a) => a,
Value::Object(mut o) => match o.remove("blocked") {
Some(Value::Array(a)) => a,
_ => return Err("the blocklist has no 'blocked' list".into()),
},
_ => return Err("the blocklist isn't a list or an object with a 'blocked' list".into()),
};
Ok(list.into_iter().take(MAX_ENTRIES).filter_map(|v| serde_json::from_value::<Blocked>(v).ok()).collect())
if list.len() > MAX_ENTRIES {
return Err(format!("the blocklist has {} entries; at most {MAX_ENTRIES} can be read", list.len()));
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
}
list.into_iter()
.enumerate()
.map(|(i, v)| {
// The reason is cut short: it can quote the entry, which comes from the network.
let b = serde_json::from_value::<Blocked>(v)
.map_err(|e| format!("blocklist entry {}: {}", i + 1, e.to_string().chars().take(200).collect::<String>()))?;
if !valid_id(&b.id, false) {
return Err(format!("blocklist entry {}: id isn't a plugin id", i + 1));
}
// Either case: `blocked` compares hashes ignoring it.
match &b.sha256 {
Some(h) if !is_sha256(&h.to_ascii_lowercase()) => Err(format!("blocklist entry {}: sha256 isn't a SHA-256 hash", i + 1)),
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
_ => Ok(b),
}
})
.collect()
}

/// Why a package is blocked, if it is.
Expand Down
19 changes: 17 additions & 2 deletions crates/plugins/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,23 @@ fn write_state(st: &State) -> Result<(), String> {
pub fn load_installed() -> Vec<String> {
let mut errors = Vec::new();
let Some(dir) = dir() else { return errors };
if let Some(list) = std::fs::read_to_string(dir.join("blocklist.json")).ok().and_then(|t| registry::parse_blocklist(&t).ok()) {
crate::set_blocklist(list);
// A cache that can't be read (or one an older version saved that this one refuses) is
// reported, not trusted; there being none is fine.
let cached = dir.join("blocklist.json");
let list = match std::fs::read_to_string(&cached) {
Ok(text) => registry::parse_blocklist(&text).map(Some),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.to_string()),
};
match list {
Ok(Some(list)) => {
crate::set_blocklist(list);
}
Ok(None) => {}
Err(e) => errors.push(format!(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Silent loss on the Python default path (new with this PR, medium-high confidence, about 75%). Before this PR, a cache with one bad entry still applied its good withdrawals, because filter_map kept them. Now the whole cache is refused, so load_installed returns this error and nothing is applied. apps/rustgis-python/python/rustgis/__init__.py:762 calls load_plugins() inside Session(plugins=True) and discards its result. A fresh Python process therefore loads a plugin the cache withdrew, with no notice. The earlier answer, that Session(plugins=True) always discarded errors, is true of other errors. What is new is that this error now means no withdrawals at all, so it is worth another look. Either the Python session should report the errors it gets back from load_plugins() (for example through take_notices), or the PR should say in its description that Python users must call rustgis.load_plugins() themselves.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed that the stakes are higher now. A refused cache means no withdrawals until the next fetch, instead of the good part of the list. I've said so in the PR description, along with where Python callers get the report: rustgis.load_plugins() returns it, the cache error included. Making Session(plugins=True) raise or warn on what load_plugins() returns would change the Python API for every plugin load error, so it's still a separate change. The case also needs a cache saved by an older version holding an entry this version refuses. Every cache written from now on has passed the same checks.

"{}: {e}; it was ignored, so withdrawn plugins may not be blocked until the blocklist is fetched again",
cached.display()
)),
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
}
let st = read_state();
let mut paths: Vec<PathBuf> = std::fs::read_dir(&dir)
Expand Down
57 changes: 57 additions & 0 deletions crates/plugins/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,18 @@ fn registries_and_blocklists_are_read_defensively() {
assert!(registry::blocked(&list, "x", "anything").is_some());
assert!(registry::blocked(&list, "y", &sha).is_some());
assert!(registry::blocked(&list, "y", &"b".repeat(64)).is_none());
// A list too long to read whole is refused, not cut short.
let entry = json!({"id": "x"});
let full = Value::Array(vec![entry.clone(); registry::MAX_ENTRIES]);
assert_eq!(registry::parse_blocklist(&full.to_string()).unwrap().len(), registry::MAX_ENTRIES);
let over = Value::Array(vec![entry; registry::MAX_ENTRIES + 1]);
assert!(registry::parse_blocklist(&over.to_string()).unwrap_err().contains("at most"));
// A refused entry isn't quoted at length.
let long = json!([{"id": "x".repeat(100_000)}, "y".repeat(100_000)]);
for list in [json!([long[0].clone()]), json!([long[1].clone()])] {
let e = registry::parse_blocklist(&list.to_string()).unwrap_err();
assert!(e.starts_with("blocklist entry 1") && e.len() < 400, "{}", e.len());
}
}

/// Blocklisting unloads affected plugins and prevents them from being loaded again.
Expand All @@ -403,6 +415,25 @@ fn blocked_plugins_are_unloaded_and_refused() {
assert!(crate::get("unit-blocked").is_none());
let pkg = Package::from_zip(&testing::package("unit-blocked", &[])).unwrap();
assert!(crate::load(pkg, "t").unwrap_err().contains("malware"));
// A refresh of the wrong shape is refused and leaves the withdrawal in place.
for text in [
"{}",
r#"{"blocked": {}}"#,
r#"{"blocked": null}"#,
"1",
r#"[{"reason": "no id"}]"#,
"[1]",
r#"[{"id": "ok"}, {"reason": "no id"}]"#,
r#"[{"id": "unit-blocked", "sha256": "sha256:abc"}]"#,
r#"[{"id": "Unit Blocked"}]"#,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
r#"[{"id": "unit blocked"}]"#,
] {
assert!(crate::store::save_blocklist(registry::DEFAULT_URL, text).is_err(), "{text}");
assert!(crate::blocked("unit-blocked", "any").is_some(), "{text} cleared the blocklist");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
// Only an explicitly empty list clears it.
assert_eq!(registry::parse_blocklist(r#"{"blocked": []}"#).unwrap(), Vec::new());
assert_eq!(registry::parse_blocklist("[]").unwrap(), Vec::new());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
crate::set_blocklist(Vec::new());
// Another registry's blocklist only adds: it can't bring back what the hosted one withdrew.
load("unit-blocked", &[]);
Expand Down Expand Up @@ -1190,6 +1221,32 @@ fn process_manifests_are_checked() {
assert!(crate::prepare(pkg, "test").unwrap_err().contains("doesn't run on this platform"));
}

/// A refused refresh leaves the cached hosted blocklist byte for byte, and a cache that can't be
/// read at start is reported rather than ignored.
#[test]
fn cached_blocklist_survives_refused_refreshes() {
let plugins = std::env::temp_dir().join(format!("rustgis-test-blocklist-cache-{}", std::process::id()));
let _store = store_lock();
crate::store::set_dir(Some(plugins.clone()));
std::fs::create_dir_all(&plugins).unwrap();
let cache = plugins.join("blocklist.json");
let good = r#"{"blocked": [{"id": "cached-withdrawal"}]}"#;
std::fs::write(&cache, good).unwrap();
for text in ["{}", r#"{"blocked": {}}"#, r#"[{"id": "ok"}, {"reason": "no id"}]"#] {
assert!(crate::store::save_blocklist(registry::DEFAULT_URL, text).is_err(), "{text}");
assert_eq!(std::fs::read_to_string(&cache).unwrap(), good, "{text} overwrote the cache");
}
std::fs::write(&cache, r#"[{"id": "a"}, {"reason": "no id"}]"#).unwrap();
let errors = crate::store::load_installed();
assert!(errors.iter().any(|e| e.contains("blocklist.json") && e.contains("entry 2")), "{errors:?}");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
// So is one that isn't text; none at all is fine.
std::fs::write(&cache, [0xff, 0xfe]).unwrap();
assert!(crate::store::load_installed().iter().any(|e| e.contains("blocklist.json")));
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
std::fs::remove_file(&cache).unwrap();
assert!(crate::store::load_installed().is_empty());
let _ = std::fs::remove_dir_all(&plugins);
}

#[test]
fn process_packages_unpack_safely() {
use std::io::Write;
Expand Down
27 changes: 22 additions & 5 deletions crates/ui-egui/src/pluginsui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -613,20 +613,37 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) {
Job::Blocklist { registry, slot } => match take(&slot) {
None => waiting.push(Job::Blocklist { registry, slot }),
Some(Ok(b)) => {
let text = String::from_utf8_lossy(&b).to_string();
// Not text is refused too: a replaced byte in an id would match nothing.
let text = std::str::from_utf8(&b).map_err(|e| format!("it isn't text: {e}"));
#[cfg(not(target_arch = "wasm32"))]
let pulled = rp::store::save_blocklist(&registry, &text).unwrap_or_default();
let pulled = text.and_then(|t| rp::store::save_blocklist(&registry, t));
#[cfg(target_arch = "wasm32")]
let pulled = rp::registry::parse_blocklist(&text).map(|l| rp::apply_blocklist(&registry, l)).unwrap_or_default();
let pulled = text.and_then(rp::registry::parse_blocklist).map(|l| rp::apply_blocklist(&registry, l));
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
// One that can't be read leaves the list in force as it was.
let pulled = pulled.unwrap_or_else(|e| {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same gap in the Plugin Manager. The Some(Err(_)) => {} arm (line 637, unchanged) drops a blocklist that is too large to fetch: registry::fetch fails above the 4 MiB limit, see the note on cmd/plugins.rs. Nothing is shown and the list in force stays. The refusal should be reported with its reason, as the parse-error path at line 622 already is. Confidence: high.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the engine thread: Some(Err(_)) is also the no-blocklist case (a 404), so the arm stays as it was. This PR only reports blocklists that arrive and can't be read.

app.session.warn(crate::i18n::fmt(
"The plugin blocklist couldn't be read ({error}); the last one read, if any, stays in force.",
&[("error", &e)],
));
Vec::new()
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
for name in pulled {
app.session.notify(
crate::i18n::fmt("{name} was withdrawn from the plugin registry and has been turned off.", &[("name", &name)]),
true,
);
}
}
// No blocklist: nothing is blocked.
Some(Err(_)) => {}
// Another registry may have no blocklist (it blocks nothing); the hosted one always
// has one, so the list in force may be out of date.
Some(Err(e)) => {
if rp::registry::is_default(&registry) {
app.session.warn(crate::i18n::fmt(
"The plugin blocklist couldn't be downloaded ({error}); the last one read, if any, stays in force.",
&[("error", &e)],
));
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
},
Job::Download { entry, slot } => match take(&slot) {
None => waiting.push(Job::Download { entry, slot }),
Expand Down
Loading