From e3356405662e6962e124e5b33484f0547c90cca4 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 14:35:14 -0600 Subject: [PATCH 1/6] feat(plugins): session process plugins stay running and hear the map A process plugin with "session": true keeps one program running between its commands. It talks to RustGIS over its own standard input and output (no port, no token), sends host requests at any time within its permissions, and hears view, layers, selection and project events. - Requests between commands run in the plugin's scope (its own permissions), each change an undo step and a session-log entry attributed to the plugin; reads leave no step; host.fetch is refused; downloads are left to the app where there is one. - RustGIS stops the program, with what it started, when the plugin is turned off, removed or updated and when the app closes; before an update replaces its files; one that exits is started again by the next command, its last requests answered first. - Output, events and answers are bounded (log budget, held events, queued-answer limit, drain bound for an exited program), and why a program was stopped is said. - The session log's CSV has a plugin column. Part of #1518. --- apps/rustgis-cli/src/main.rs | 9 + apps/rustgis/src/main.rs | 2 + crates/engine/src/cmd/plugins.rs | 357 +++++++++++++++- crates/engine/src/cmd/project.rs | 2 +- crates/engine/src/lib.rs | 4 + crates/engine/src/policy/dispatch.rs | 14 + crates/engine/src/project_log.rs | 15 +- crates/engine/src/tests.rs | 272 ++++++++++++- crates/plugins/src/lib.rs | 23 +- crates/plugins/src/manifest.rs | 12 +- crates/plugins/src/process.rs | 215 ++++++++-- crates/plugins/src/session.rs | 589 +++++++++++++++++++++++++++ crates/plugins/src/store.rs | 15 +- crates/plugins/src/testing.rs | 202 +++++++++ crates/plugins/src/tests.rs | 203 +++++++++ crates/ui-egui/src/lib.rs | 20 + crates/ui-egui/src/sessionlog.rs | 2 +- docs/plugins.md | 52 +++ docs/user-guide/workspace.md | 4 +- 19 files changed, 1938 insertions(+), 74 deletions(-) create mode 100644 crates/plugins/src/session.rs diff --git a/apps/rustgis-cli/src/main.rs b/apps/rustgis-cli/src/main.rs index be53219218..990bf70284 100644 --- a/apps/rustgis-cli/src/main.rs +++ b/apps/rustgis-cli/src/main.rs @@ -685,7 +685,16 @@ fn mcp(s: &mut Session, args: &[String]) -> Result<(), String> { Ok(()) } +/// Runs the command line, then stops the session plugins' programs its commands started. fn main() -> ExitCode { + let code = cli(); + // Session plugins' programs a command started end with the command line. + rustgis_engine::plugins::session::stop_all(); + code +} + +/// The command line: loads the plugins, then runs the subcommand the arguments name. +fn cli() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); let Some(cmd) = args.first().map(|s| s.as_str()) else { return usage() }; if !args.iter().any(|a| a == "--no-plugins") { diff --git a/apps/rustgis/src/main.rs b/apps/rustgis/src/main.rs index 833cb48a25..c42ef52ad6 100644 --- a/apps/rustgis/src/main.rs +++ b/apps/rustgis/src/main.rs @@ -342,6 +342,8 @@ impl eframe::App for App { fn on_exit(&mut self) { self.0.end_exports(); self.0.end_autosave(); + // Session plugins' programs end with the app, with what they started. + rustgis_engine::plugins::session::stop_all(); // The installer starts before the marker goes, so a RustGIS started meanwhile finds this // one still running. #[cfg(windows)] diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index da786e3190..2ffb2a286c 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -282,6 +282,25 @@ pub fn reaches(spec: &C, params: &Value) -> Option { } } +/// The project's layers as `host.layers` answers them. +fn layers_of(s: &Session) -> Value { + let active = s.active_layer; + Value::Array( + s.map() + .layers + .iter() + .map(|l| { + json!({ + "id": l.id, "name": l.name, "raster": l.is_raster(), "visible": l.visible, + "geometry": format!("{:?}", l.dataset.geometry_type), "features": l.dataset.features.len(), + "selected": l.selection.len(), "active": Some(l.id) == active, + "fields": l.dataset.fields.iter().map(|f| json!({"name": f.name, "type": format!("{:?}", f.ty)})).collect::>(), + }) + }) + .collect(), + ) +} + /// Answers a running plugin's requests. struct Bridge<'a> { s: &'a mut Session, @@ -310,22 +329,7 @@ impl Bridge<'_> { } fn layers(&self) -> Value { - let active = self.s.active_layer; - Value::Array( - self.s - .map() - .layers - .iter() - .map(|l| { - json!({ - "id": l.id, "name": l.name, "raster": l.is_raster(), "visible": l.visible, - "geometry": format!("{:?}", l.dataset.geometry_type), "features": l.dataset.features.len(), - "selected": l.selection.len(), "active": Some(l.id) == active, - "fields": l.dataset.fields.iter().map(|f| json!({"name": f.name, "type": format!("{:?}", f.ty)})).collect::>(), - }) - }) - .collect(), - ) + layers_of(self.s) } /// A layer's features as GeoJSON: all of them, or its selection (`"selected": true`), at @@ -437,6 +441,14 @@ impl Bridge<'_> { } } +impl Bridge<'_> { + /// Whether host request `op` only reads (or says something): no undo step for it. + #[cfg(not(target_arch = "wasm32"))] + fn reads_only(op: &str) -> bool { + matches!(op, "host.info" | "host.layers" | "host.features" | "host.view" | "host.notify") + } +} + impl Host for Bridge<'_> { fn call(&mut self, op: &str, params: &Value) -> Result { match op { @@ -444,11 +456,7 @@ impl Host for Bridge<'_> { "plugin": self.plugin.manifest.id, "permissions": self.plugin.manifest.permissions})), "host.layers" => Ok(self.layers()), "host.features" => self.features(params), - "host.view" => { - let c = &self.s.map().camera; - let center = c.center_lonlat(); - Ok(json!({"center": [center.x, center.y], "scale": c.scale(), "projection": self.s.map().projection.code()})) - } + "host.view" => Ok(view_of(self.s)), "host.notify" => { let text: String = ps(params, "text").unwrap_or_default().chars().take(500).collect(); let error = b(params, "error").unwrap_or(false); @@ -488,6 +496,37 @@ impl Session { let label = leak(&cmd.decl.label); let mutates = cmd.mutates(); let plugin = cmd.plugin.clone(); + #[cfg(not(target_arch = "wasm32"))] + let (plugin_is_session, plugin_name) = (plugin.manifest.session, plugin.manifest.name.clone()); + // A session's program may start with this command: what it changes from its first + // command on is told to it, so the map as it was is noted first. + // What its program sent since the last poll (after answering its last command, or as it + // exited) is answered as between its commands, each request its own undo step, not part + // of this command's. + // The command's own merge key (a held key repeating it) is the command's, not theirs. + #[cfg(not(target_arch = "wasm32"))] + let merge = self.merge.take(); + #[cfg(not(target_arch = "wasm32"))] + for _ in 0..rustgis_plugins::session::DRAIN_BATCHES { + let id = &plugin.manifest.id; + if !(plugin.manifest.session && rustgis_plugins::session::is_running(id)) { + break; + } + let exited = rustgis_plugins::session::has_exited(id); + if self.poll_plugin_session(id, self.leave_downloads) == 0 && !exited { + break; + } + } + #[cfg(not(target_arch = "wasm32"))] + { + self.merge = merge; + } + #[cfg(not(target_arch = "wasm32"))] + if plugin.manifest.session && !self.plugin_watch.seen.iter().any(|(p, _)| *p == plugin.manifest.id) { + let view = view_of(self); + let now = seen(self, &view); + self.plugin_watch.seen.push((plugin.manifest.id.clone(), now)); + } // A plugin with a panel gets its values with every command (`panel` in the request), // after the ones `params` sets (`{"panel": {"id": value}}`, which is taken out of them), // and may answer the panel's new state. A failed command leaves the values as they were. @@ -561,6 +600,12 @@ impl Session { } } self.call_requests = outer; + // A session's program this command found failed, and started again: the command line + // has no poll, so this is where it hears of it. + #[cfg(not(target_arch = "wasm32"))] + if plugin_is_session && let Some(ended) = rustgis_plugins::session::take_ended(&pid) { + self.say_session_ended(&plugin_name, ended); + } if pending.is_empty() && requested.is_empty() { return res; } @@ -615,10 +660,280 @@ impl Session { } } +/// Answers a session plugin's requests made between its commands: each one that can change the +/// project is an undo step of its own; one that only reads runs as it is, leaving no step. +#[cfg(not(target_arch = "wasm32"))] +struct Between<'a> { + s: &'a mut Session, + plugin: Arc, +} + +#[cfg(not(target_arch = "wasm32"))] +impl Host for Between<'_> { + fn call(&mut self, op: &str, params: &Value) -> Result { + let plugin = self.plugin.clone(); + let bridge = |s: &mut Session| { + let fetches = fetch::Calls::for_plugin(&plugin.manifest.id); + let mut bridge = Bridge { s, plugin: plugin.clone(), fetches, changed: false, requested: false }; + let res = bridge.call(op, params); + (res, bridge.changed, bridge.requested) + }; + // A fetch waits on the app's thread: between commands the program fetches for itself. + if op == "host.fetch" { + return Err("'host.fetch' is for commands; between them, a session's program fetches what it needs itself".into()); + } + // An undo step only for what can change the project, as for an agent's command: a read + // leaves the user's undo and redo history alone. + let reads = Bridge::reads_only(op); + let mutates = + plugin.manifest.has(Permission::Edit) && (op == "host.addLayer" || self.s.registry.get(op).is_some_and(|c| c.mutates)); + // With the plugin's authority, exactly as during one of its commands. + self.s + .as_session_plugin(plugin.clone(), |s| { + if reads { + return std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| bridge(s).0)) + .unwrap_or_else(|_| Err(format!("'{op}' failed unexpectedly"))); + } + let (at, start) = (crate::project_log::now_ms(), web_time::Instant::now()); + let res = s.guarded(leak(&plugin.manifest.name), mutates, |s| { + let (res, changed, requested) = bridge(s); + // Only left downloads for the app (they add their layers when they land): no step. + s.request_only = res.is_ok() && requested && !changed; + // The answer goes to the program, which says what it makes of a refusal. + s.quiet_failure = res.is_err(); + res + }); + // No command of its own carries it into the session log: it goes in by itself, as + // a command run would. + if s.log.on() && (op == "host.addLayer" || crate::project_log::is_logged(s, op, params)) { + let (kept, partial) = crate::project_log::kept_and_partial(params); + let project = (crate::project_log::kept_name(&s.title()), s.opened); + let mut e = crate::project_log::entry(project, op, kept, partial, &res, at, start.elapsed()); + e.plugin = Some(plugin.manifest.id.clone()); + s.log.push(e); + } + res + }) + .and_then(|r| r) + } + + fn log(&mut self, text: &str) { + log::info!("{}: {text}", self.plugin.manifest.id); + } +} + +/// What session plugins were last told of the map, to tell them what changes. +#[derive(Debug, Default)] +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +pub(crate) struct Watch { + /// Each running program's, by plugin id. + seen: Vec<(String, Seen)>, + /// The layers' symbology and fields and the web tile layers, and the selection, each + /// fingerprinted, with the [`Session::revision`] and the time they were taken at. They're + /// taken again when a command bumps the revision, and at least every [`RESTYLE_EVERY`] for + /// what changes them without one: not every frame. + styled: Option<(u64, web_time::Instant, u64, u64)>, +} + +/// How often a session plugin's layer symbology is looked at again without a command. +#[cfg(not(target_arch = "wasm32"))] +const RESTYLE_EVERY: web_time::Duration = web_time::Duration::from_secs(1); + +/// Fingerprints of what a session plugin hears about. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +pub(crate) struct Seen { + view: u64, + /// Their list and order, which is active, names, visibility, opacity, symbology, fields and + /// features, and the web tile layers. + layers: u64, + selection: u64, + /// Bumped when another project replaces the open one ([`Session::opened`]). + project: u64, +} + +/// What a session plugin would be told of the map now, `view` being [`view_of`] it. +#[cfg(not(target_arch = "wasm32"))] +fn seen(s: &mut Session, view: &Value) -> Seen { + use std::hash::{Hash, Hasher}; + let hash = |f: &dyn Fn(&mut std::collections::hash_map::DefaultHasher)| { + let mut h = std::collections::hash_map::DefaultHasher::new(); + f(&mut h); + h.finish() + }; + let (styled, selection) = match s.plugin_watch.styled { + Some((revision, at, styled, selection)) if revision == s.revision && at.elapsed() < RESTYLE_EVERY => (styled, selection), + _ => { + let map = s.map(); + let styled = hash(&|h| { + for l in &map.layers { + serde_json::to_string(&(&l.renderer, &l.labels, &l.line_strokes, &l.fill_layers, l.raster.as_ref().map(|r| &r.style))) + .unwrap_or_default() + .hash(h); + for f in &l.dataset.fields { + (&f.name, format!("{:?}", f.ty)).hash(h); + } + } + serde_json::to_string(&map.tiles).unwrap_or_default().hash(h); + }); + // A large selection is many rows to walk: with the symbology, not every frame. Layers + // with nothing selected don't count: adding one changes no selection. + let selection = hash(&|h| { + for l in map.layers.iter().filter(|l| !l.selection.is_empty()) { + (l.id, &l.selection).hash(h); + } + }); + s.plugin_watch.styled = Some((s.revision, web_time::Instant::now(), styled, selection)); + (styled, selection) + } + }; + let map = s.map(); + let view = hash(&|h| view.to_string().hash(h)); + // The rest is a few numbers a layer: looked at every frame. + let layers = hash(&|h| { + (styled, s.active_layer).hash(h); + for l in &map.layers { + (l.id, &l.name, l.visible, l.transparency.to_bits(), l.geometry_rev, l.dataset.features.len()).hash(h); + // Which rows these are: an edit writes new ones and undo puts the old ones back, so it + // changes even where undo brings `geometry_rev` back to the edit's number. + (l.dataset.features.allocation_key().as_ptr() as usize).hash(h); + } + }); + Seen { view, layers, selection, project: s.opened } +} + +/// The view as `host.view` answers it. +fn view_of(s: &Session) -> Value { + let c = &s.map().camera; + let center = c.center_lonlat(); + json!({"center": [center.x, center.y], "scale": c.scale(), "projection": s.map().projection.code()}) +} + +impl Session { + /// Says that a session plugin's program failed; one that ended by itself (its window closed) + /// goes quietly. + #[cfg(not(target_arch = "wasm32"))] + fn say_session_ended(&mut self, name: &str, ended: rustgis_plugins::session::Ended) { + if ended.unread { + self.notify( + format!("{name}: its program kept sending after it exited (something it left running), and the rest wasn't answered; it was stopped"), + true, + ); + } + if let Some(reason) = ended.reason { + self.notify(format!("{name} was stopped: {reason}"), true); + } else if ended.failed { + let why = ended.last_error.map(|l| format!(": {l}")).unwrap_or_default(); + self.notify(format!("{name} stopped unexpectedly{why}"), true); + } + } + + /// Answers what plugin `id`'s program sent since the last call, between its commands (leaving + /// downloads to the app when `defer`), and says + /// if it failed, or if a command found it failed. Returns how many of its lines it handled. + #[cfg(not(target_arch = "wasm32"))] + fn poll_plugin_session(&mut self, id: &str, defer: bool) -> usize { + use rustgis_plugins::session; + let Some(plugin) = rustgis_plugins::get(id).filter(|p| p.enabled) else { + session::stop(id); + return 0; + }; + // A download it asks for is left to the app when there is one to take it (`defer`: the + // desktop's frame, or a command the app runs leaving its downloads); otherwise (the + // command line, Python) it's fetched now, as a command run there fetches it. + let polled = if defer { + self.with_deferred_downloads(|s| session::poll(id, &mut Between { s, plugin: plugin.clone() })) + } else { + session::poll(id, &mut Between { s: self, plugin: plugin.clone() }) + }; + for ended in polled.ended.into_iter().chain(session::take_ended(id)) { + self.say_session_ended(&plugin.manifest.name, ended); + } + polled.handled + } + + /// Answers what session plugins' programs asked since the last call, and tells each what + /// changed on the map since it was last told. The desktop app calls it every frame. Returns + /// how many of their lines it handled. + #[cfg(not(target_arch = "wasm32"))] + pub fn poll_plugin_sessions(&mut self) -> usize { + use rustgis_plugins::session; + session::polled(); + let mut handled = 0; + for id in session::running() { + handled += self.poll_plugin_session(&id, true); + } + let running = session::running(); + self.plugin_watch.seen.retain(|(id, _)| running.contains(id)); + if running.is_empty() { + self.plugin_watch.styled = None; + return handled; + } + let view = view_of(self); + let now = seen(self, &view); + for id in running { + let Some((_, was)) = self.plugin_watch.seen.iter_mut().find(|(p, _)| *p == id) else { + // Started without a command (none yet): it asks for what it needs first. + self.plugin_watch.seen.push((id, now)); + continue; + }; + let was = std::mem::replace(was, now); + if was.view != now.view { + session::event(&id, &json!({"event": "view", "view": view})); + } + if was.layers != now.layers { + session::event(&id, &json!({"event": "layers"})); + } + if was.project != now.project { + session::event(&id, &json!({"event": "project"})); + } + if was.selection != now.selection { + session::event(&id, &json!({"event": "selection"})); + } + } + handled + } +} + #[cfg(test)] mod tests { use super::*; + /// Why a session plugin's program was stopped is said; one that ended by itself says nothing. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_stopped_session_program_says_why() { + use rustgis_plugins::session::Ended; + let mut s = Session::new(crate::Project::default()); + let mark = s.notice_mark(); + s.say_session_ended("Globe", Ended { failed: false, last_error: None, unread: false, reason: None }); + assert!(s.notices_since(mark).is_empty(), "a clean end says nothing"); + let why = "it sent a line longer than 256 MB"; + s.say_session_ended("Globe", Ended { failed: true, last_error: Some("boom".into()), unread: false, reason: Some(why.into()) }); + let said: Vec<&str> = s.notices_since(mark).iter().map(|n| n.text.as_str()).collect(); + assert_eq!(said, [format!("Globe was stopped: {why}")], "the reason, not the last error line"); + } + + /// A geometry edit and its undo each change what a session plugin is told of the layers, + /// though undo, putting the old rows back, brings `geometry_rev` back to the edit's number. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn an_undone_geometry_edit_is_a_layer_change() { + let mut s = Session::new(crate::Project::default()); + let fc = r#"{"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [1, 2]}}]}"#; + let (ds, _) = rustgis_data::io::read_geojson(fc, "Pts").unwrap(); + super::super::layer::add_dataset(&mut s, ds); + s.run("edit.enable", &json!({"on": true})).unwrap(); + let view = view_of(&s); + let before = seen(&mut s, &view); + s.run("edit.setGeometry", &json!({"layer": "Pts", "row": 0, "geometry": {"type": "Point", "coordinates": [3, 4]}})).unwrap(); + let edited = seen(&mut s, &view); + assert_ne!(before.layers, edited.layers); + assert!(s.undo()); + let undone = seen(&mut s, &view); + assert_ne!(edited.layers, undone.layers); + } + /// A command that only left a download for the front end changed nothing, and a later one /// doesn't hide an earlier edit: the plugin's command keeps its undo step. #[test] diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index e962726acf..b3ce8b0e3c 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -34,7 +34,7 @@ pub fn commands() -> Vec { // The session log (#797): read, written out, emptied, turned on or off. It sits outside // the project, so none of these changes the project or takes an undo step. C::new("project.log", "Session Log", "Project", log).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( - r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis", "project": the project open when it ran, "partial"?: true when params were cut or had secrets taken out}], "count": matching, "total": in the log, "on": bool}"#, + r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis", "project": the project open when it ran, "partial"?: true when params were cut or had secrets taken out, "plugin"?: the session plugin that asked for it between its commands}], "count": matching, "total": in the log, "on": bool}"#, ), C::new("project.exportLog", "Export Session Log", "Project", export_log).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) .view() diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 447882a4e7..e0535f368b 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -809,6 +809,9 @@ pub struct Session { /// them (it keeps nothing between calls). Not saved with the project: a plugin's state /// command reads what it needs from the map when the panel opens. pub plugin_panels: std::collections::BTreeMap, + /// What running session plugins were last told of the map, to send them events. + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + pub(crate) plugin_watch: cmd::plugins::Watch, /// Data downloads of this project under way (Home › Add › Add Data › From URL), by id. downloading: Vec, /// Downloads that landed, waiting for `layer.addDownload`. @@ -1150,6 +1153,7 @@ impl Session { started: Vec::new(), plugin_runs: Vec::new(), plugin_panels: std::collections::BTreeMap::new(), + plugin_watch: Default::default(), downloading: Vec::new(), downloaded: Vec::new(), landed_reads: Vec::new(), diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 20ef59a352..a17a9db50d 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -277,6 +277,20 @@ impl Session { Err(p) => std::panic::resume_unwind(p), } } + /// Runs what a session plugin's program asks between its commands with the plugin's own + /// authority, as its commands run: the exact load, still current, narrowing the scope the app + /// is in (none at the frame's root, the trusted native convention). + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn as_session_plugin( + &mut self, + plugin: Arc, + f: impl FnOnce(&mut Session) -> T, + ) -> Result { + if !rustgis_plugins::is_current(&plugin) { + return Err("the plugin was disabled, updated or removed".into()); + } + self.narrowed(Narrow::Plugin(super::plan::PluginPin(plugin)), f) + } /// Intersects nested plugin/item authority with its initiating scope. pub(crate) fn narrowed(&mut self, to: Narrow, f: impl FnOnce(&mut Session) -> T) -> Result { let mut scope = self.scope(); diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index fc91d8f0bf..3d6f75f7fc 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -519,6 +519,7 @@ pub(crate) fn entry( project: project.0, opened: project.1, partial, + plugin: None, } } @@ -573,6 +574,10 @@ pub struct LogEntry { /// wouldn't run as they ran. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub partial: bool, + /// The plugin whose session program asked for it between its commands: it counts with the + /// plugins' commands toward [`LOG_PLUGIN_LIMIT`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub plugin: Option, } impl LogEntry { @@ -623,7 +628,7 @@ impl SessionLog { if self.off && !matches!(e.command.as_str(), "project.logCommands" | "project.clearLog") { return; } - let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX); + let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX) || e.plugin.is_some(); if plugin(&e) && self.plugin_entries >= LOG_PLUGIN_LIMIT && let Some(oldest) = self.entries.iter().position(plugin) @@ -649,7 +654,8 @@ impl SessionLog { /// into the last undo step ([`crate::Session::merge_undo`]). The entry keeps when the first /// started, takes the last one's parameters, and adds up how long they took. pub fn join_last(&mut self, mut e: LogEntry) { - if let Some(last) = self.entries.back_mut().filter(|l| l.ok && l.command == e.command) { + // A session plugin's entry between the steps isn't the user's step to join. + if let Some(last) = self.entries.back_mut().filter(|l| l.ok && l.command == e.command && l.plugin == e.plugin) { e.at = last.at; e.millis = last.millis.saturating_add(e.millis); *last = e; @@ -713,7 +719,7 @@ impl SessionLog { /// line). Entry by entry, so a long log isn't held as text first. pub fn write(&self, w: &mut dyn std::io::Write, csv: bool) -> std::io::Result<()> { if csv { - w.write_all(b"time_utc,at,command,ok,millis,error,params,project\n")?; + w.write_all(b"time_utc,at,command,ok,millis,error,params,plugin,project\n")?; } self.entries.iter().try_for_each(|e| write_entry(w, e, csv)) } @@ -740,7 +746,7 @@ fn write_entry(w: &mut dyn std::io::Write, e: &LogEntry, csv: bool) -> std::io:: if csv { writeln!( w, - "{},{},{},{},{},{},{},{}", + "{},{},{},{},{},{},{},{},{}", e.time_utc(), e.at, csv_cell(&e.command), @@ -748,6 +754,7 @@ fn write_entry(w: &mut dyn std::io::Write, e: &LogEntry, csv: bool) -> std::io:: e.millis, csv_cell(e.error.as_deref().unwrap_or("")), csv_cell(&e.params.to_string()), + csv_cell(e.plugin.as_deref().unwrap_or("")), csv_cell(&e.project) ) } else { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f0c80f39ae..29e8a522b1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -10499,6 +10499,237 @@ fn process_plugins_reach_the_project_like_modules() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn session_plugins_ask_between_commands_and_hear_the_map() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess", &["edit"], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let poll_until = |s: &mut Session, done: &dyn Fn(&Session) -> bool| { + let t0 = std::time::Instant::now(); + while !done(s) && t0.elapsed() < std::time::Duration::from_secs(20) { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + // Long enough for a request it sent to be answered. + let settle = |s: &mut Session| { + for _ in 0..25 { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + // Its first command starts it, and what that command changes is told to it too. + let point = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [1, 2]}}]}); + s.run("plugin:eng-sess/f", &json!({"op": "host.addLayer", "params": {"name": "First", "geojson": point}})).unwrap(); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + + // Reads between commands, a host request or a command that changes nothing, leave the project + // and the undo history as they were. + s.run("plugin:eng-sess/k", &json!({"op": "host.layers", "params": {}})).unwrap(); + let (revision, steps) = (s.revision, s.undo_depth()); + settle(&mut s); + assert_eq!((s.revision, s.undo_depth()), (revision, steps)); + s.run("plugin:eng-sess/k", &json!({"op": "map.extent", "params": {}})).unwrap(); + let (steps, top) = (s.undo_depth(), s.undo_label()); + settle(&mut s); + assert_eq!((s.undo_depth(), s.undo_label()), (steps, top)); + // An edit between commands goes through the same checks, as an undo step of its own. + let fc = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [3, 4]}}]}); + s.log.set_on(true); + s.run("plugin:eng-sess/k", &json!({"op": "host.addLayer", "params": {"name": "Later", "geojson": fc}})).unwrap(); + poll_until(&mut s, &|s| s.map().layer_by_name("Later").is_some()); + assert_eq!(s.undo_label(), Some("Test eng-sess"), "its own step"); + assert!(s.log.entries().any(|e| e.command == "host.addLayer"), "in the session log"); + assert!(s.undo()); + assert!(s.map().layer_by_name("Later").is_none()); + assert!(s.redo()); + assert!(s.map().layer_by_name("Later").is_some()); + s.run("plugin:eng-sess/k", &json!({"op": "layer.addData", "params": {"path": "/etc/passwd"}})).unwrap(); + settle(&mut s); + // A fetch would wait on the app's thread: only during commands. + s.run("plugin:eng-sess/k", &json!({"op": "host.fetch", "params": {"url": "https://example.org/a.json"}})).unwrap(); + settle(&mut s); + // Told what changed: the layers (the one it added), the view, the selection. + s.map_mut().camera.pan_px(40.0, 0.0); + s.run("select.rows", &json!({"layer": "Cities", "rows": [0]})).unwrap(); + s.poll_plugin_sessions(); + let heard = s.run("plugin:eng-sess/v", &Value::Null).unwrap(); + // (The undo and redo above count as layer changes too.) + let events: Vec<&str> = heard["events"].as_array().unwrap().iter().filter_map(Value::as_str).collect(); + assert!(events.first() == Some(&"layers") && events.ends_with(&["view", "selection"]), "{heard}"); + let refused = heard["answers"].as_array().unwrap().iter().filter_map(|a| a["error"].as_str()).collect::>(); + assert!(refused.iter().any(|e| e.contains("'files' permission")), "{heard}"); + assert!(refused.iter().any(|e| e.contains("'host.fetch' is for commands")), "{heard}"); + // Restyled or made see-through counts as a layer change; another project is told too. + if let Some(l) = s.map_mut().layers.first_mut() { + l.transparency = 0.5; + } + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + // Another layer made active is a layer change too: `host.layers` says which one is. + let other = s.map().layers.iter().map(|l| l.id).find(|id| Some(*id) != s.active_layer).unwrap(); + s.active_layer = Some(other); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + s.run("project.new", &json!({})).unwrap(); + s.poll_plugin_sessions(); + let events = s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"].clone(); + assert!(events.as_array().unwrap().contains(&json!("project")), "{events}"); + // Turned off, it stops. + s.run("plugin.enable", &json!({"id": "eng-sess", "enabled": false})).unwrap(); + assert!(!rustgis_plugins::session::is_running("eng-sess")); + rustgis_plugins::unload("eng-sess"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Session plugins' programs are the process's, and a session's poll answers every running one: +/// the tests that run them take turns, so none answers another's in its session. +fn session_plugins_turn() -> std::sync::MutexGuard<'static, ()> { + static TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + TURN.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Waits (a few seconds at most) for a session plugin's program to exit, without answering it. +fn wait_for_exit(plugin: &str) { + let t0 = std::time::Instant::now(); + while !rustgis_plugins::session::has_exited(plugin) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(10)); + } +} + +/// What a program sent as it exited, after many lines, is answered before the next command, as +/// between commands: its edit is an undo step of its own, not part of that command's. +#[test] +fn a_session_programs_last_request_is_its_own_step_before_the_next_command() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-late-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-late", &["edit"], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let fc = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [3, 4]}}]}); + s.run("plugin:eng-sess-late/y", &json!({"op": "host.addLayer", "params": {"name": "Late", "geojson": fc}})).unwrap(); + wait_for_exit("eng-sess-late"); + // No poll in between: the next command finds it exited. + assert_eq!(s.run("plugin:eng-sess-late/h", &Value::Null).unwrap(), json!({"hello": "world"})); + assert!(s.map().layer_by_name("Late").is_some()); + assert_eq!(s.undo_label(), Some("Hello")); + assert!(s.undo()); + assert!(s.map().layer_by_name("Late").is_some(), "not part of the next command's step"); + assert_eq!(s.undo_label(), Some("Test eng-sess-late")); + assert!(s.notices_since(0).iter().all(|n| !n.text.contains("Test eng-sess-late")), "a clean exit says nothing"); + rustgis_plugins::unload("eng-sess-late"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// On the command line (no app to leave downloads to), a download a session program asked for +/// between commands is fetched when the next command answers it: never left `pending`. +#[test] +fn session_plugin_downloads_between_commands_are_fetched_without_an_app() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-cli-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-cli", &["edit", "network"], &run)) + .unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-cli/k", &json!({"op": "layer.addUrl", "params": {"url": "https://example.invalid/points.geojson"}})).unwrap(); + // Each command first answers what the program sent since: `v` reports once it has. + let t0 = std::time::Instant::now(); + let answer = loop { + let heard = s.run("plugin:eng-sess-cli/v", &Value::Null).unwrap(); + if let Some(a) = heard["answers"].as_array().and_then(|a| a.first()).cloned() { + break a; + } + assert!(t0.elapsed() < std::time::Duration::from_secs(20), "no answer"); + std::thread::sleep(std::time::Duration::from_millis(50)); + }; + assert!(answer["error"].is_string() && answer["ok"]["pending"].is_null(), "{answer}"); + assert!(s.take_url_requests().is_empty(), "nothing left for an app that isn't there"); + rustgis_plugins::unload("eng-sess-cli"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that exits while something it left running keeps writing into its output is ended +/// by the next command, and the notice says the rest wasn't answered; one that ends cleanly +/// gets no notice (`a_session_programs_last_request_is_its_own_step_before_the_next_command`). +#[cfg(unix)] +#[test] +fn a_session_program_whose_output_kept_coming_is_said() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-flow-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-flow", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-flow/r", &Value::Null).unwrap(); + wait_for_exit("eng-sess-flow"); + let mark = s.notice_mark(); + assert_eq!(s.run("plugin:eng-sess-flow/h", &Value::Null).unwrap(), json!({"hello": "world"})); + let said: Vec = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("kept sending after it exited")), "{said:?}"); + rustgis_plugins::session::stop("eng-sess-flow"); + rustgis_plugins::unload("eng-sess-flow"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that failed after answering is said to have when the next command finds it, as +/// the frame's poll would: the command line has no poll. +#[test] +fn a_session_program_found_failed_by_a_command_is_said() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-bad", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + assert_eq!(s.run("plugin:eng-sess-bad/b", &Value::Null).unwrap(), "bye"); + wait_for_exit("eng-sess-bad"); + let mark = s.notice_mark(); + assert_eq!(s.run("plugin:eng-sess-bad/h", &Value::Null).unwrap(), json!({"hello": "world"})); + let said: Vec = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("Test eng-sess-bad stopped unexpectedly: boom")), "{said:?}"); + rustgis_plugins::unload("eng-sess-bad"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A download a session plugin asks for between its commands is left to the app, as during its +/// commands: the frame never waits for it. +#[test] +fn session_plugin_downloads_between_commands_are_left_to_the_app() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-url-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-url", &["edit", "network"], &run)) + .unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let url = "https://example.invalid/points.geojson"; + s.run("plugin:eng-sess-url/k", &json!({"op": "layer.addUrl", "params": {"url": url}})).unwrap(); + let t0 = std::time::Instant::now(); + while s.url_requests.is_empty() && t0.elapsed() < std::time::Duration::from_secs(20) { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let asked = s.take_url_requests(); + let urls: Vec<&Value> = asked.iter().map(|q| &q.params()["url"]).collect(); + assert!(urls.contains(&&json!(url)), "{urls:?}"); + let heard = s.run("plugin:eng-sess-url/v", &Value::Null).unwrap(); + assert_eq!(heard["answers"][0]["ok"]["pending"], true, "{heard}"); + rustgis_plugins::unload("eng-sess-url"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn plugins_install_and_uninstall_from_files() { isolate_plugins(); @@ -29440,7 +29671,7 @@ fn the_session_log_records_each_command_run_in_the_session() { if isolate_process_test("tests::the_session_log_records_each_command_run_in_the_session") { return; } - use crate::project_log::{LOG_LIMIT, LOG_PARAMS_BYTES, LogEntry}; + use crate::project_log::{LOG_LIMIT, LOG_PARAMS_BYTES, LOG_PLUGIN_LIMIT, LogEntry}; let mut s = world(); // The sample project's making is its first entry: counted from after it. assert_eq!(s.log.clear(), 1); @@ -29552,6 +29783,17 @@ fn the_session_log_records_each_command_run_in_the_session() { ..Default::default() }; s.log.push(formula); + // A session plugin's request between its commands keeps its attribution in both. + s.log.push(LogEntry { + at: 3, + command: "host.addLayer".into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + plugin: Some("globe".into()), + ..Default::default() + }); // Written out as CSV and JSON Lines. let dir = std::env::temp_dir().join(format!("rustgis-session-log-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -29559,14 +29801,22 @@ fn the_session_log_records_each_command_run_in_the_session() { s.run("project.exportLog", &json!({"path": csv.to_string_lossy()})).unwrap(); let csv_text = std::fs::read_to_string(&csv).unwrap(); assert!( - csv_text.starts_with("time_utc,at,command,ok,millis,error,params,project\n") && csv_text.contains("\"edit.undo\""), + csv_text.starts_with("time_utc,at,command,ok,millis,error,params,plugin,project\n") && csv_text.contains("\"edit.undo\""), "{csv_text}" ); assert!(csv_text.contains("\"'=HYPERLINK(1)\"") && !csv_text.contains(",\"=HYPERLINK"), "{csv_text}"); + // The plugin is the cell before the project's name, the last. + let plugin_cell = |l: &str| l.rsplit(',').nth(1).map(str::to_string); + assert!(csv_text.lines().any(|l| l.contains("\"host.addLayer\"") && plugin_cell(l).as_deref() == Some("\"globe\"")), "{csv_text}"); + assert!( + csv_text.lines().any(|l| l.contains("\"edit.undo\"") && plugin_cell(l).as_deref() == Some("\"\"")), + "the user's own carry none: {csv_text}" + ); let jsonl = dir.join("log.jsonl"); s.run("project.exportLog", &json!({"path": jsonl.to_string_lossy()})).unwrap(); let lines: Vec = std::fs::read_to_string(&jsonl).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); assert_eq!(lines.len() + 1, s.log.len(), "every entry but the export's own"); + assert!(lines.iter().any(|e| e.plugin.as_deref() == Some("globe"))); let _ = std::fs::remove_dir_all(&dir); // Cleared, it says so; off, nothing goes in; on again, it does. Neither changes the project, // which holds no log, nor takes an undo step. @@ -29616,6 +29866,24 @@ fn the_session_log_records_each_command_run_in_the_session() { } assert_eq!(s.log.len(), LOG_LIMIT); assert_eq!(s.log.entries().next().unwrap().at, 5); + // A session plugin's requests between its commands count with the plugins': past their limit + // they make room among themselves, not by pushing out the user's entries. + let user = s.log.entries().next().unwrap().at; + for i in 0..LOG_PLUGIN_LIMIT + 5 { + let e = LogEntry { + at: 1_000_000 + i as u64, + command: "host.addLayer".into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + plugin: Some("globe".into()), + ..Default::default() + }; + s.log.push(e); + } + assert_eq!(s.log.entries().filter(|e| e.plugin.is_some()).count(), LOG_PLUGIN_LIMIT); + assert_eq!(s.log.entries().next().unwrap().at, user + LOG_PLUGIN_LIMIT as u64); // What was dropped is counted, and an export says so; a clear starts the count again. assert!(s.log.dropped() >= 5, "{}", s.log.dropped()); let out = std::env::temp_dir().join(format!("rustgis-log-dropped-{}.csv", std::process::id())); diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs index 24970a5389..e5421304be 100644 --- a/crates/plugins/src/lib.rs +++ b/crates/plugins/src/lib.rs @@ -9,7 +9,8 @@ //! //! A process plugin (`"runtime": "process"`, see [`process`]) is instead a program on the //! user's computer, not sandboxed, for the desktop and the command line: it speaks the same -//! requests over its standard input and output, and the user is told it isn't sandboxed. +//! requests over its standard input and output, and the user is told it isn't sandboxed. With +//! `"session": true` its program is kept running between commands (see [`session`]). //! //! Loaded plugins live for the whole process (like the geoprocessing tools they add), in a //! list shared by every session. See `docs/plugins.md` for the format and the ABI. @@ -25,6 +26,8 @@ pub mod pyenv; pub mod registry; pub mod runtime; #[cfg(not(target_arch = "wasm32"))] +pub mod session; +#[cfg(not(target_arch = "wasm32"))] pub mod store; #[cfg(any(test, feature = "testing"))] pub mod testing; @@ -111,6 +114,7 @@ impl Plugin { "panel": m.panel.as_ref().map(|p| p.title.clone()), "skills": m.skills.iter().map(|s| format!("{}:{s}", m.id)).collect::>(), "runtime": match m.runtime { Runtime::Wasm => "wasm", Runtime::Process => "process", Runtime::Skills => "skills" }, + "session": m.session, "enabled": self.enabled, "source": self.source, "sha256": self.sha256, }) } @@ -232,6 +236,7 @@ fn process_code(pkg: &Package) -> Result { let argv = pkg.manifest.run.as_ref().and_then(|r| r.for_this_platform()).ok_or_else(|| format!("{name} doesn't run on this platform"))?; let mut spec = process::Spec::new(argv, dir)?; + spec.session = pkg.manifest.session; // With a Python environment, `python` is that environment's interpreter. if let Some(check) = pyenv::EnvCheck::of(&pkg.manifest, &spec.dir).map_err(|e| format!("{name}: {e}"))? { if matches!(spec.program.as_str(), "python" | "python3" | "python.exe") { @@ -251,6 +256,8 @@ fn process_code(pkg: &Package) -> Result { /// was enabled). pub fn register(mut plugin: Plugin) -> Arc { plugin.enabled = get(&plugin.manifest.id).is_none_or(|p| p.enabled); + // The program running is the version being replaced. + stop_session(&plugin.manifest.id); let plugin = Arc::new(plugin); if let Ok(mut v) = PLUGINS.write() { v.retain(|p| p.manifest.id != plugin.manifest.id); @@ -263,6 +270,7 @@ pub fn register(mut plugin: Plugin) -> Arc { /// Removes a loaded plugin; returns whether there was one. pub fn unload(id: &str) -> bool { + stop_session(id); let removed = PLUGINS.write().map(|mut v| { let n = v.len(); v.retain(|p| p.manifest.id != id); @@ -293,10 +301,19 @@ pub fn set_enabled(id: &str, enabled: bool) -> Result<(), String> { changed(&v); } drop(v); + if !enabled { + stop_session(id); + } tools::reload(); Ok(()) } +/// Stops a session plugin's program, if it's running. +fn stop_session(_id: &str) { + #[cfg(not(target_arch = "wasm32"))] + session::stop(_id); +} + /// Replaces the blocklist; loaded plugins it names are unloaded. Returns their names. pub fn set_blocklist(list: Vec) -> Vec { if let Ok(mut b) = BLOCKED.write() { @@ -398,6 +415,10 @@ pub(crate) fn call(plugin: &Plugin, host: &mut dyn Host, request: &Value, tool: match &plugin.code { Code::Wasm(module) => runtime::call(module, host, request, if tool { runtime::TOOL_FUEL } else { runtime::COMMAND_FUEL }), #[cfg(not(target_arch = "wasm32"))] + Code::Process(spec) if spec.session && !tool => { + session::command(&plugin.manifest.id, spec, host, request, process::COMMAND_TIMEOUT) + } + #[cfg(not(target_arch = "wasm32"))] Code::Process(spec) => process::call(spec, host, request, (!tool).then_some(process::COMMAND_TIMEOUT)), Code::None => Err(format!("{} has only skills, nothing to run", plugin.manifest.name)), } diff --git a/crates/plugins/src/manifest.rs b/crates/plugins/src/manifest.rs index 7b32bf656d..207283e247 100644 --- a/crates/plugins/src/manifest.rs +++ b/crates/plugins/src/manifest.rs @@ -318,6 +318,10 @@ pub struct Manifest { /// For a process plugin in Python: the environment RustGIS makes for it. #[serde(default)] pub python: Option, + /// For a process plugin: keep the program running between its commands (a session), so it + /// can ask RustGIS for things at any time and hear when the map changes. See the `session` module. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub session: bool, /// The plugin ABI the module was built for. #[serde(default = "default_abi")] pub abi: u32, @@ -447,8 +451,8 @@ impl Manifest { if !valid_path(&self.entry) || !self.entry.ends_with(".wasm") { return Err("plugin.json: entry must be a relative path to a .wasm file inside the package".into()); } - if self.run.is_some() || self.python.is_some() { - return Err("plugin.json: 'run' and 'python' are for process plugins (\"runtime\": \"process\")".into()); + if self.run.is_some() || self.python.is_some() || self.session { + return Err("plugin.json: 'run', 'python' and 'session' are for process plugins (\"runtime\": \"process\")".into()); } } Runtime::Process => { @@ -487,8 +491,8 @@ impl Manifest { } } Runtime::Skills => { - if self.run.is_some() || self.python.is_some() { - return Err("plugin.json: 'run' and 'python' are for process plugins (\"runtime\": \"process\")".into()); + if self.run.is_some() || self.python.is_some() || self.session { + return Err("plugin.json: 'run', 'python' and 'session' are for process plugins (\"runtime\": \"process\")".into()); } if !self.commands.is_empty() || !self.tools.is_empty() diff --git a/crates/plugins/src/process.rs b/crates/plugins/src/process.rs index 4d6e8f0677..662d95297a 100644 --- a/crates/plugins/src/process.rs +++ b/crates/plugins/src/process.rs @@ -18,6 +18,8 @@ use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel, sync_channel}; use std::time::{Duration, Instant}; @@ -28,11 +30,11 @@ use crate::runtime::Host; /// How long a command may run; tools run until they finish (a large dataset can take a while). pub const COMMAND_TIMEOUT: Duration = Duration::from_secs(120); /// How long a program has to exit after answering, before it's stopped. -const EXIT_GRACE: Duration = Duration::from_secs(3); +pub(crate) const EXIT_GRACE: Duration = Duration::from_secs(3); /// The longest line read from a program (a reply with large layers). -const MAX_LINE: usize = 256 * 1024 * 1024; +pub(crate) const MAX_LINE: usize = 256 * 1024 * 1024; /// The most host requests one call may make. -const MAX_HOST_CALLS: u32 = 100_000; +pub(crate) const MAX_HOST_CALLS: u32 = 100_000; /// The most lines of its own output a program may log in one call. const MAX_LOGGED: usize = 200; /// The longest line of a program's error output kept (a longer one is split). @@ -54,6 +56,8 @@ pub struct Spec { pub(crate) env: Vec<(String, String)>, /// For a plugin with a Python environment (`"python"`): the check that it's ready. pub env_check: Option, + /// Whether its commands go to one program kept running (`"session"`, see [`crate::session`]). + pub session: bool, } impl Spec { @@ -63,7 +67,7 @@ impl Spec { // Absolute: the program starts in the folder, so a relative folder would be followed // twice (`./plugin/plugin/bin/tool`). let dir = std::path::absolute(dir).map_err(|e| format!("{}: {e}", dir.display()))?; - Ok(Spec { program: program.clone(), args: args.to_vec(), dir, env: Vec::new(), env_check: None }) + Ok(Spec { program: program.clone(), args: args.to_vec(), dir, env: Vec::new(), env_check: None, session: false }) } /// The program to start: a path relative to the plugin's folder, or a name for `PATH`. @@ -93,7 +97,7 @@ impl Spec { } /// What the program sent: a line of its output, or the end of it. -enum Line { +pub(crate) enum Line { Text(Vec), End, TooLong, @@ -102,7 +106,7 @@ enum Line { /// Reads the program's output lines on a thread, so waiting for one can time out. A few lines /// at most wait to be handled: a program printing faster than RustGIS reads is made to wait /// rather than filling memory. -fn lines(out: impl Read + Send + 'static) -> Receiver { +fn lines(out: impl Read + Send + 'static, wake: Option) -> Receiver { let (tx, rx) = sync_channel(2); std::thread::spawn(move || { let mut r = BufReader::new(out); @@ -111,16 +115,20 @@ fn lines(out: impl Read + Send + 'static) -> Receiver { match (&mut r).take(MAX_LINE as u64 + 1).read_until(b'\n', &mut buf) { Ok(0) | Err(_) => { let _ = tx.send(Line::End); + wake.as_ref().inspect(|w| w()); return; } Ok(_) if buf.len() > MAX_LINE => { let _ = tx.send(Line::TooLong); + wake.as_ref().inspect(|w| w()); return; } Ok(_) => { if tx.send(Line::Text(buf)).is_err() { return; } + // A session's line can arrive while the app is idle: it is asked to look. + wake.as_ref().inspect(|w| w()); } } } @@ -129,11 +137,24 @@ fn lines(out: impl Read + Send + 'static) -> Receiver { } /// The last lines of the program's error output, for the log and for an error message. -type Kept = std::sync::Arc>>; +pub(crate) type Kept = std::sync::Arc>; -/// Keeps the program's error output as it comes. Read, never joined: a process the program +/// The last lines of a program's error output, and how many earlier ones weren't kept. +#[derive(Debug, Default)] +pub(crate) struct ErrorLines { + pub lines: std::collections::VecDeque, + pub dropped: usize, + /// Whether the error output has ended (everything holding it open has closed it). + pub done: bool, +} + +/// The most lines of a program's error output kept until they're read. +const KEPT_ERROR_LINES: usize = 20; + +/// Keeps the program's error output as it comes; `wake` is called as each line arrives (a +/// session's, so it is logged while the app is idle). Read, never joined: a process the program /// started can hold the pipe open after the program itself has exited. -fn errors(err: impl Read + Send + 'static) -> Kept { +pub(crate) fn errors(err: impl Read + Send + 'static, wake: Option) -> Kept { let kept = Kept::default(); let k = kept.clone(); std::thread::spawn(move || { @@ -142,22 +163,30 @@ fn errors(err: impl Read + Send + 'static) -> Kept { // Bounded: endless output without a line end is read in pieces. let mut buf = Vec::new(); match (&mut r).take(MAX_ERROR_LINE as u64).read_until(b'\n', &mut buf) { - Ok(0) | Err(_) => break, + Ok(0) | Err(_) => { + if let Ok(mut k) = k.lock() { + k.done = true; + } + break; + } Ok(_) => {} } let line = String::from_utf8_lossy(&buf).trim_end().to_string(); let Ok(mut k) = k.lock() else { break }; - k.push_back(line.chars().take(1_000).collect::()); - if k.len() > 20 { - k.pop_front(); + k.lines.push_back(line.chars().take(1_000).collect::()); + if k.lines.len() > KEPT_ERROR_LINES { + k.lines.pop_front(); + k.dropped += 1; } + drop(k); + wake.as_ref().inspect(|w| w()); } }); kept } /// Stops the program, and what it started, if it hasn't exited on its own within `grace`. -fn finish(child: &mut Child, grace: Duration) { +pub(crate) fn finish(child: &mut Child, grace: Duration) { let start = Instant::now(); loop { match child.try_wait() { @@ -174,6 +203,70 @@ fn finish(child: &mut Child, grace: Duration) { } } +/// Whether the program has exited. On Unix it isn't reaped yet: until it is, its id can't be +/// reused, so its process group can still be stopped safely ([`end`]). +pub(crate) fn exited(child: &mut Child) -> bool { + #[cfg(unix)] + { + use rustix::io::Errno; + use rustix::process::{Pid, WaitId, WaitIdOptions, waitid}; + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(Pid::from_child(child)), options) { + Ok(found) => return found.is_some(), + Err(Errno::INTR) => continue, + // Reaped already (by its own status check, or by something else when the app + // reaps every child or ignores SIGCHLD): gone either way. + // (Not waited on again: its id may be another child's by now.) + Err(Errno::CHILD) => return true, + // Not known to have exited: asking `try_wait` would reap it, so it counts as running. + Err(_) => return false, + } + } + } + #[cfg(not(unix))] + matches!(child.try_wait(), Ok(Some(_))) +} + +/// Ends a program kept running: gives it `grace` to exit, stops what it left running, then stops +/// it if it's still running and reaps it. Returns its exit status when it exited by itself (and +/// RustGIS reaped it), and whether it had to be stopped. +/// +/// On Unix what it left running is its process group, stopped before the program is reaped so +/// the group's id can't belong to anything else yet. On Windows a process tree can't be found +/// from a program that has exited, so only a running one's is stopped. +pub(crate) fn end(child: &mut Child, grace: Duration) -> (Option, bool) { + let start = Instant::now(); + while !exited(child) && start.elapsed() < grace { + std::thread::sleep(Duration::from_millis(10)); + } + let by_itself = exited(child); + // Reaped already (by something else: an embedding host reaping every child, SIGCHLD + // ignored): its id may be another process's by now, so nothing is signalled by it. + #[cfg(unix)] + let reaped = { + use rustix::process::{Pid, WaitId, WaitIdOptions, waitid}; + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + matches!(waitid(WaitId::Pid(Pid::from_child(child)), options), Err(rustix::io::Errno::CHILD)) + }; + #[cfg(not(unix))] + let reaped = false; + #[cfg(unix)] + if !reaped { + let _ = rustix::process::kill_process_group(rustix::process::Pid::from_child(child), rustix::process::Signal::KILL); + } + // Reaped elsewhere, it's gone: waiting on its id could wait on another child that has it now. + if reaped { + return (None, false); + } + if !by_itself { + kill_tree(child); + let _ = child.kill(); + } + let status = child.wait().ok(); + (status.filter(|_| by_itself), !by_itself) +} + /// Stops the processes a program started: its process group (Unix), its process tree (Windows). fn kill_tree(child: &Child) { #[cfg(unix)] @@ -194,28 +287,76 @@ fn kill_tree(child: &Child) { } } -/// Writes lines to the program's input on a thread, so a program that doesn't read can't block -/// RustGIS (the timeout still applies). Dropping the sender closes the input. -fn writer(mut stdin: impl Write + Send + 'static) -> Sender> { +/// The program's input: lines written on a thread, so a program that doesn't read can't block +/// RustGIS (the timeout still applies). Dropping it closes the input. +pub(crate) struct Writer { + tx: Sender>, + /// Lines sent and not yet written: how far behind the program is in reading. + backlog: Arc, + /// Their bytes. + bytes: Arc, +} + +/// How many lines may wait for the program to read them before a session's events are held +/// (`session::event`); when it catches up below this, the app is asked to send them. +pub(crate) const HOLD_EVENTS_AT: usize = 16; + +impl Writer { + /// Queues `v` as one line. + pub(crate) fn send(&self, v: &Value) -> Result<(), String> { + self.send_line(v.to_string().into_bytes()) + } + + /// Queues `line`, JSON already written out, as one line. + pub(crate) fn send_line(&self, line: Vec) -> Result<(), String> { + self.backlog.fetch_add(1, Ordering::Relaxed); + self.bytes.fetch_add(line.len(), Ordering::Relaxed); + self.tx.send(line).map_err(|_| "it stopped reading".to_string()) + } + + /// Lines waiting for the program to read them. + pub(crate) fn backlog(&self) -> usize { + self.backlog.load(Ordering::Relaxed) + } + + /// Bytes waiting for the program to read them. + pub(crate) fn backlog_bytes(&self) -> usize { + self.bytes.load(Ordering::Relaxed) + } +} + +/// `wake` is called when the program, behind in reading, catches up below [`HOLD_EVENTS_AT`]. +fn writer(mut stdin: impl Write + Send + 'static, wake: Option) -> Writer { let (tx, rx) = channel::>(); + let backlog = Arc::new(AtomicUsize::new(0)); + let bytes = Arc::new(AtomicUsize::new(0)); + let (left, left_bytes) = (backlog.clone(), bytes.clone()); std::thread::spawn(move || { for mut line in rx { + let n = line.len(); line.push(b'\n'); if stdin.write_all(&line).and_then(|_| stdin.flush()).is_err() { return; } + left_bytes.fetch_sub(n, Ordering::Relaxed); + if left.fetch_sub(1, Ordering::Relaxed) == HOLD_EVENTS_AT { + wake.as_ref().inspect(|w| w()); + } } }); - tx + Writer { tx, backlog, bytes } } -fn send(stdin: &Sender>, v: &Value) -> Result<(), String> { - stdin.send(v.to_string().into_bytes()).map_err(|_| "it stopped reading".to_string()) +/// A program started, with its input, its output lines and its error output. +pub(crate) struct Started { + pub child: Child, + pub stdin: Writer, + pub rx: Receiver, + pub errs: Kept, } -/// Runs one call: starts the program, sends `request`, answers its host requests through -/// `host`, and returns its `ok` value or its error. `timeout` (none for tools) stops it. -pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { +/// Starts the program, its pipes connected; `wake` is called as each line of its output arrives. +pub(crate) fn spawn(spec: &Spec, wake: Option) -> Result { if let Some(check) = spec.env_check.as_ref().filter(|c| !c.ready()) { return Err(format!( "its Python environment isn't ready: prepare it in Plugins › Plugin Manager › Installed, or with rustgis-cli plugins prepare {}", @@ -233,9 +374,13 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { + let Started { mut child, stdin, rx, errs } = spawn(spec, None)?; let res = converse(&stdin, &rx, host, request, timeout); // Its input closed, a program that answered exits; one that didn't is stopped. Its output // isn't read any more: one still printing gets a broken pipe rather than waiting on a full @@ -247,7 +392,7 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option = errs.lock().map(|k| k.iter().cloned().collect()).unwrap_or_default(); + let stderr: Vec = errs.lock().map(|k| k.lines.iter().cloned().collect()).unwrap_or_default(); for line in stderr.iter().filter(|l| !l.trim().is_empty()) { log::info!("plugin {}: {line}", spec.program); } @@ -257,14 +402,8 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option>, - rx: &Receiver, - host: &mut dyn Host, - request: &Value, - timeout: Option, -) -> Result { - send(stdin, request)?; +fn converse(stdin: &Writer, rx: &Receiver, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { + stdin.send(request)?; let deadline = timeout.map(|t| Instant::now() + t); let (mut calls, mut logged) = (0u32, 0usize); loop { @@ -274,6 +413,10 @@ fn converse( if host.cancelled() { return Err(CANCELLED.into()); } + // Checked whatever it sends: a program that keeps talking still runs out of time. + if deadline.is_some_and(|d| Instant::now() >= d) { + return Err("it ran too long and was stopped".into()); + } let wait = deadline.map_or(CANCEL_POLL, |d| d.saturating_duration_since(Instant::now()).min(CANCEL_POLL)); match rx.recv_timeout(wait) { Ok(l) => break l, @@ -316,7 +459,7 @@ fn converse( Ok(v) => json!({"ok": v}), Err(e) => json!({"error": e}), }; - send(stdin, &reply)?; + stdin.send(&reply)?; } else if let Some(text) = msg.get("log") { if logged < MAX_LOGGED { logged += 1; diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs new file mode 100644 index 0000000000..cac9b42d29 --- /dev/null +++ b/crates/plugins/src/session.rs @@ -0,0 +1,589 @@ +//! Session process plugins (`"session": true`, desktop and command line): one program kept +//! running between the plugin's commands, so it can ask RustGIS for things at any time and hear +//! when the map changes. A linked view in a window of its own (a 3D globe) is one. +//! +//! RustGIS starts the program the first time one of the plugin's commands runs, in the plugin's +//! folder, and writes a first line `{"session": {"abi", "plugin", "rustgis", "lang"}}`. From +//! then on the lines go both ways, one JSON object each: +//! +//! - RustGIS to the program: `{"command": request, "id": n}`, a command to run (the request a +//! per-call plugin gets), answered with `{"ok": value, "id": n}` or `{"error": "text", "id": n}`; +//! the reply to one of its host requests, `{"ok": value}` or `{"error": "text"}` with the +//! request's `id` when it gave one; and `{"event": "view" | "layers" | "selection" | "project", +//! ...}` when the map changes or another project opens. +//! - The program to RustGIS: `{"call": {"op", "params"}, "id": n}` (the `id` optional), `log` +//! and `progress` lines, and its answers to commands. +//! +//! The channel is the program's own input and output: no port, no token, nothing another program +//! can join. Its requests go through the same [`Host`] checks as a command's, within the plugin's +//! permissions. The program is stopped, with what it started, when the plugin is turned off, +//! removed or updated and when RustGIS closes ([`stop`], [`stop_all`]); its input closing means +//! it is being stopped (RustGIS closing or gone, however it ended; the plugin turned off, removed +//! or updated; the program failed) and is its sign to exit. A program that exits on its own (its window +//! closed) is started again by the plugin's next command; on Unix what it left running is stopped. +//! +//! The plugin's tools still start the program once per run, as a per-call process plugin's do. + +use std::process::{Child, ExitStatus}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use crate::process::{self, Kept, Line, Spec, Started, Writer}; +use crate::runtime::Host; + +/// Asks the app to look at its sessions (a repaint): a line arrived while it was idle. +pub type Wake = Arc; + +/// The most lines of one program handled in one [`poll`]; the rest wait for the next. +pub const POLL_BATCH: usize = 64; +/// Why a program was stopped when the answers it hadn't read passed [`MAX_QUEUED_BYTES`]. +const STALLED: &str = "the answers waiting for it to read passed 256 MB"; +/// How long a program has to exit when its plugin is turned off, removed or updated: the app +/// waits, so a program that ignores its closed input holds it this long at most. +const STOP_GRACE: Duration = Duration::from_secs(1); +/// How long a program whose output ended has to finish exiting before it's stopped. +const END_GRACE: Duration = Duration::from_millis(200); +/// The most lines of a program's other output and `log` lines logged per [`LOG_WINDOW`]. +const MAX_LOGGED: usize = 200; +const LOG_WINDOW: Duration = Duration::from_secs(60); +/// The most bytes of replies that may wait for the program to read them: one that keeps asking +/// without reading the answers is stopped rather than fill memory. +const MAX_QUEUED_BYTES: usize = 256 * 1024 * 1024; +/// How often a command waiting on the program checks whether it was cancelled. +const CANCEL_POLL: Duration = Duration::from_millis(100); + +/// A program kept running for its plugin. +struct Running { + plugin: String, + program: String, + child: Child, + stdin: Writer, + rx: Receiver, + errs: Kept, + /// The next command's `id`. + next_id: u64, + /// Lines logged since the window began: a program printing endlessly can't flood the log. + logged: (Instant, usize), + /// Events held while the program is behind in reading: the latest of each kind. + held: Vec, + /// How many polls have found it exited with its output still coming: something it left + /// running keeps writing. At [`DRAIN_BATCHES`] it's stopped. + drained: usize, + /// The last line of its error output logged so far. + last_stderr: Option, +} + +impl Running { + /// Whether one more line may be logged now; says once when lines stop being logged. + fn may_log(&mut self) -> bool { + if self.logged.0.elapsed() > LOG_WINDOW { + self.logged = (Instant::now(), 0); + } + self.logged.1 += 1; + if self.logged.1 == MAX_LOGGED + 1 { + log::info!("{}: more output than {MAX_LOGGED} lines a minute; the rest isn't logged", self.plugin); + } + self.logged.1 <= MAX_LOGGED + } +} + +static RUNNING: Mutex> = Mutex::new(Vec::new()); +/// How programs found ended by a command (not by [`poll`]) ended: for the app to say ([`take_ended`]). +static ENDED: Mutex> = Mutex::new(Vec::new()); +/// The most [`poll`] batches read of a program found exited before it is stopped: what it sent +/// before exiting is finite, but what it left running may keep its output open and writing. +pub const DRAIN_BATCHES: usize = 256; + +/// Keeps how `plugin`'s program ended when it failed, for [`take_ended`]. +fn note_ended(plugin: &str, ended: Ended) { + if (ended.failed || ended.unread) + && let Ok(mut v) = ENDED.lock() + { + // One kept a plugin: the latest. + v.retain(|(p, _)| p != plugin); + v.push((plugin.to_string(), ended)); + } +} + +/// How `plugin`'s program ended, if a command found it failed (and started it again) since +/// the last call: the app says so as it does for one [`poll`] finds. +pub fn take_ended(plugin: &str) -> Option { + let mut v = ENDED.lock().ok()?; + let i = v.iter().position(|(p, _)| p == plugin)?; + Some(v.swap_remove(i).1) +} + +/// Whether `plugin`'s program is in the list but has exited: what it sent last is still to be +/// read ([`poll`]). +pub fn has_exited(plugin: &str) -> bool { + RUNNING.lock().is_ok_and(|mut v| v.iter_mut().any(|r| r.plugin == plugin && process::exited(&mut r.child))) +} +static WAKE: Mutex> = Mutex::new(None); + +/// Sets what a session's line arriving calls, for programs started from now on. +pub fn set_wake(wake: Wake) { + if let Ok(mut w) = WAKE.lock() { + *w = Some(wake); + } +} + +/// What a new program's lines call as they arrive ([`set_wake`]). +fn wake() -> Option { + // What the app set when it's called, not when the program started: one started before the + // app's first frame wakes it too. At most one asked for until the app polls ([`polled`]): + // a program printing line after line asks for one frame, not one a line. + Some(Arc::new(|| { + if !WOKEN.swap(true, std::sync::atomic::Ordering::Relaxed) + && let Some(w) = WAKE.lock().ok().and_then(|w| w.clone()) + { + w(); + } + })) +} + +/// Whether a wake was asked for since the app last polled. +static WOKEN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// The app is about to poll: the next line asks for a frame again. +pub fn polled() { + WOKEN.store(false, std::sync::atomic::Ordering::Relaxed); +} + +/// The plugins whose programs are running. +pub fn running() -> Vec { + RUNNING.lock().map(|v| v.iter().map(|r| r.plugin.clone()).collect()).unwrap_or_default() +} + +/// Whether `plugin`'s program is running. +pub fn is_running(plugin: &str) -> bool { + RUNNING.lock().is_ok_and(|v| v.iter().any(|r| r.plugin == plugin)) +} + +/// Takes a running program out of the list while it's talked to: what its requests run can't +/// reach the list's lock. +fn take(plugin: &str) -> Option { + let mut v = RUNNING.lock().ok()?; + let i = v.iter().position(|r| r.plugin == plugin)?; + Some(v.swap_remove(i)) +} + +/// Puts a program back in the list after it was talked to; one that can't be kept is stopped. +fn put(r: Running) { + match RUNNING.lock() { + Ok(mut v) => v.push(r), + // Not kept, so not left running either. + Err(_) => { + shut(r, Duration::ZERO); + } + } +} + +/// Starts `plugin`'s program and greets it with the session's first line. +fn start(plugin: &str, spec: &Spec) -> Result { + let Started { child, stdin, rx, errs } = process::spawn(spec, wake())?; + let r = Running { + plugin: plugin.to_string(), + program: spec.program.clone(), + child, + stdin, + rx, + errs, + next_id: 1, + logged: (Instant::now(), 0), + held: Vec::new(), + drained: 0, + last_stderr: None, + }; + let hello = json!({"session": {"abi": crate::manifest::ABI, "plugin": plugin, "rustgis": crate::manifest::RUSTGIS_VERSION, "lang": crate::language()}}); + if let Err(e) = r.stdin.send(&hello) { + shut(r, Duration::ZERO); + return Err(e); + } + Ok(r) +} + +/// Closes the program's input, gives it `grace` to exit, then stops it with what it started. +/// Returns its exit status when it exited by itself, whether it had to be stopped, and the last +/// line of its error output. +fn shut(mut r: Running, grace: Duration) -> (Option, bool, Option) { + log_errors(&mut r); + let Running { mut child, stdin, rx, errs, program, last_stderr, .. } = r; + drop(stdin); + drop(rx); + let (status, stopped) = process::end(&mut child, grace); + // What it wrote as it ended: until its error output has ended, or a moment at most, as + // something holding the pipe open (which `end` couldn't stop) mustn't hold this. + let until = Instant::now() + END_GRACE; + while !errs.lock().is_ok_and(|k| k.done) && Instant::now() < until { + std::thread::sleep(Duration::from_millis(5)); + } + let last = drain_errors(&errs, &program).or(last_stderr); + (status, stopped, last) +} + +/// Logs the error output the program wrote since the last call, as it runs. +fn log_errors(r: &mut Running) { + let (lines, dropped) = r.errs.lock().map(|mut k| (k.lines.drain(..).collect(), std::mem::take(&mut k.dropped))).unwrap_or_default(); + let lines: Vec = lines; + if dropped > 0 && r.may_log() { + log::info!("plugin {}: {dropped} earlier lines of its error output weren't kept", r.program); + } + for line in lines.into_iter().filter(|l| !l.trim().is_empty()) { + if r.may_log() { + log::info!("plugin {}: {line}", r.program); + } + r.last_stderr = Some(line); + } +} + +/// Logs what is left of a program's error output; returns its last line. +fn drain_errors(errs: &Kept, program: &str) -> Option { + let (lines, dropped) = errs.lock().map(|mut k| (k.lines.drain(..).collect(), std::mem::take(&mut k.dropped))).unwrap_or_default(); + let lines: Vec = lines; + if dropped > 0 { + log::info!("plugin {program}: {dropped} earlier lines of its error output weren't kept"); + } + for line in lines.iter().filter(|l| !l.trim().is_empty()) { + log::info!("plugin {program}: {line}"); + } + lines.into_iter().rev().find(|l| !l.trim().is_empty()) +} + +/// Stops `plugin`'s program, if it's running. +pub fn stop(plugin: &str) { + if let Some(r) = take(plugin) { + shut(r, STOP_GRACE); + } +} + +/// Stops every program (RustGIS is closing): their inputs close together, and they share the +/// time to exit. +pub fn stop_all() { + let all: Vec = RUNNING.lock().map(|mut v| std::mem::take(&mut *v)).unwrap_or_default(); + let mut waiting = Vec::new(); + for r in all { + let Running { child, stdin, rx, errs, program, .. } = r; + drop(stdin); + drop(rx); + waiting.push((child, errs, program)); + } + let end = Instant::now() + process::EXIT_GRACE; + for (mut child, errs, program) in waiting { + process::end(&mut child, end.saturating_duration_since(Instant::now())); + drain_errors(&errs, &program); + } +} + +/// A protocol line from the program, or `None` for other output (logged, within limits). +fn message(r: &mut Running, bytes: &[u8]) -> Option> { + match serde_json::from_slice::(bytes) { + Ok(Value::Object(m)) if ["call", "log", "progress", "error", "ok"].iter().any(|k| m.contains_key(*k)) => Some(m), + _ => { + let text = String::from_utf8_lossy(bytes); + let text = text.trim_end(); + if !text.is_empty() && r.may_log() { + log::info!("plugin output: {}", text.chars().take(1_000).collect::()); + } + None + } + } +} + +/// A message's text: a string as it is, anything else as JSON. +fn text(v: &Value) -> String { + v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string()) +} + +/// Handles a line that isn't an answer to a command: a host request (answered), a log or +/// progress line. Returns the answer's `id` and value when it is one. +fn handle(m: &Map, host: &mut dyn Host, r: &mut Running) -> Result)>, String> { + let id = m.get("id").cloned().unwrap_or(Value::Null); + if let Some(c) = m.get("call") { + let op = c.get("op").and_then(Value::as_str).unwrap_or_default(); + let params = c.get("params").cloned().unwrap_or(Value::Null); + // A request that fails unexpectedly is an error answer, as in a WebAssembly plugin: the + // program, out of the list while it's talked to, is never left running untracked. + let answer = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| host.call(op, ¶ms))) + .unwrap_or_else(|_| Err(format!("'{op}' failed unexpectedly"))); + let mut reply = match answer { + Ok(v) => json!({"ok": v}), + Err(e) => json!({"error": e}), + }; + if !id.is_null() + && let Some(o) = reply.as_object_mut() + { + o.insert("id".into(), id); + } + // An answer too large to send at all is refused to it, whether it reads or not; one + // that would only overflow what it hasn't read yet means it stopped reading. + // Written out once: measured, then sent as it is (an answer can be hundreds of MB). + let mut line = reply.to_string().into_bytes(); + let size = line.len(); + if size > MAX_QUEUED_BYTES { + let id = reply.get("id").cloned(); + reply = json!({"error": format!("the answer is larger than {} MB", MAX_QUEUED_BYTES / (1024 * 1024))}); + if let (Some(id), Some(o)) = (id, reply.as_object_mut()) { + o.insert("id".into(), id); + } + line = reply.to_string().into_bytes(); + } + // What is sent, refusal or not, must fit with what it hasn't read yet. + if r.stdin.backlog_bytes().saturating_add(line.len()) > MAX_QUEUED_BYTES { + return Err(STALLED.into()); + } + r.stdin.send_line(line)?; + } else if let Some(t) = m.get("log") { + if r.may_log() { + host.log(&text(t)); + } + } else if let Some(t) = m.get("progress") { + host.progress(&text(t)); + } else if let Some(e) = m.get("error") { + return Ok(Some((id, Err(text(e))))); + } else if let Some(v) = m.get("ok") { + return Ok(Some((id, Ok(v.clone())))); + } + Ok(None) +} + +/// Runs a command of `plugin` in its program, starting it if it isn't running: sends `request`, +/// answers its host requests through `host`, and returns its answer. A program that doesn't +/// answer within `timeout` is stopped. +pub fn command(plugin: &str, spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Duration) -> Result { + let mut r = match take(plugin) { + Some(mut r) => { + if !process::exited(&mut r.child) { + r + } else { + // It exited on its own (its window closed): what it sent last is handled first, + // then it is started again. + put(r); + // All of it, a batch at a time; then how it ended is kept for the app to say. + for _ in 0..DRAIN_BATCHES { + let polled = poll(plugin, host); + if let Some(ended) = polled.ended { + note_ended(plugin, ended); + break; + } + } + // Still output past that: what it left running keeps writing. Stopped, and its + // exit status says whether it failed. + if let Some(r) = take(plugin) { + log::warn!("{plugin}: its output was still coming after it exited; the rest wasn't read"); + let (status, _, last_error) = shut(r, Duration::ZERO); + // No status (stopped here, or reaped elsewhere) isn't a failure: only its own exit says so. + note_ended(plugin, Ended { failed: status.is_some_and(|s| !s.success()), last_error, unread: true, reason: None }); + } + start(plugin, spec)? + } + } + None => start(plugin, spec)?, + }; + let id = r.next_id; + r.next_id += 1; + if let Err(e) = r.stdin.send(&json!({"command": request, "id": id})) { + shut(r, Duration::ZERO); + return Err(e); + } + let deadline = Instant::now() + timeout; + let mut calls = 0u32; + let res: Result<(), String> = loop { + if host.cancelled() { + break Err(process::CANCELLED.to_string()); + } + // Checked whatever it sends: a program that keeps talking still runs out of time. + if Instant::now() >= deadline { + break Err("it ran too long and was stopped".to_string()); + } + let line = match r.rx.recv_timeout(deadline.saturating_duration_since(Instant::now()).min(CANCEL_POLL)) { + Ok(l) => l, + Err(RecvTimeoutError::Timeout) if Instant::now() >= deadline => break Err("it ran too long and was stopped".to_string()), + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => Line::End, + }; + let bytes = match line { + Line::Text(b) => b, + Line::End => break Err("it exited without answering".to_string()), + Line::TooLong => break Err(format!("it sent a line longer than {} MB", process::MAX_LINE / (1024 * 1024))), + }; + let Some(m) = message(&mut r, &bytes) else { continue }; + if m.contains_key("call") { + calls += 1; + if calls > process::MAX_HOST_CALLS { + break Err("it made too many requests".to_string()); + } + } + match handle(&m, host, &mut r) { + Err(e) => break Err(e), + Ok(Some((answered, res))) if answered == json!(id) => { + put(r); + return res; + } + // Not this command's: an answer it had given up on. + Ok(Some(_)) => { + if r.may_log() { + log::info!("{plugin}: an answer to no command it was running"); + } + } + Ok(None) => {} + } + }; + // It stopped answering: stopped, so the next command starts it afresh. + let err = res.err().unwrap_or_default(); + match shut(r, Duration::ZERO) { + (_, _, Some(last)) if err.contains("without answering") => Err(format!("{err}: {last}")), + _ => Err(err), + } +} + +/// What [`poll`] found. +#[derive(Debug, Default, PartialEq)] +pub struct Polled { + /// Lines handled. + pub handled: usize, + /// The program exited, or was stopped for what it sent. + pub ended: Option, +} + +/// How a session's program ended. +#[derive(Debug, PartialEq)] +pub struct Ended { + /// Whether it failed: exited with an error, or was stopped. Exiting by itself (its window + /// closed) isn't failing. + pub failed: bool, + /// The last line of its error output. + pub last_error: Option, + /// Whether what it sent after exiting wasn't all read: something it left running kept + /// writing past [`DRAIN_BATCHES`], and its requests from then on weren't answered. + pub unread: bool, + /// Why RustGIS stopped it, when it did for what it sent. + pub reason: Option, +} + +/// Answers what `plugin`'s program sent since the last call, at most [`POLL_BATCH`] lines, +/// without waiting for more. Each host request goes to `host`. +pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { + let Some(mut r) = take(plugin) else { return Polled::default() }; + log_errors(&mut r); + let mut out = Polled::default(); + // Whether it ended, and whether it was stopped for what it sent. + let mut ended = None; + let mut reason = None; + // It exited: what it wrote last may still be on its way from the pipe, and is read first. + // Checked first too: what it left running may keep its output flowing, so the channel never + // runs empty to say so. + let mut gone = process::exited(&mut r.child).then(|| Instant::now() + END_GRACE); + for _ in 0..POLL_BATCH { + let line = if let Some(until) = gone { + // One moment for all of it, not one a line. + r.rx.recv_timeout(until.saturating_duration_since(Instant::now())).ok() + } else { + match r.rx.try_recv() { + Ok(l) => Some(l), + // Nothing new: it may have exited, its output held open by what it started. + Err(TryRecvError::Empty) => { + if process::exited(&mut r.child) { + gone = Some(Instant::now() + END_GRACE); + continue; + } + break; + } + Err(TryRecvError::Disconnected) => None, + } + }; + let bytes = match line { + Some(Line::Text(b)) => b, + None | Some(Line::End) => { + ended = Some(false); + break; + } + Some(Line::TooLong) => { + let why = format!("it sent a line longer than {} MB", process::MAX_LINE / (1024 * 1024)); + log::warn!("{plugin}: {why}, and was stopped"); + reason = Some(why); + ended = Some(true); + break; + } + }; + out.handled += 1; + let Some(m) = message(&mut r, &bytes) else { continue }; + match handle(&m, host, &mut r) { + Ok(None) => {} + Ok(Some(_)) => { + if r.may_log() { + log::info!("{plugin}: an answer to no command it was running"); + } + } + // The answers it hasn't read passed the limit: stopped. + Err(e) if e == STALLED => { + log::warn!("{plugin}: {e}, and was stopped"); + reason = Some(e); + ended = Some(true); + break; + } + // Exited, it can't be answered; what it asked was done. + Err(_) if gone.is_some() => {} + // It can't be written to: it's exiting, and what it sent last is read first. + Err(_) => gone = Some(Instant::now() + END_GRACE), + } + } + // Exited, and its output still coming after this batch: something it left running keeps + // writing. Poll after poll, that's bounded too: past it, stopped with what it left running. + if ended.is_none() && gone.is_some() { + r.drained += 1; + if r.drained >= DRAIN_BATCHES { + log::warn!("{plugin}: its output was still coming after it exited; the rest wasn't read"); + let (status, _, last_error) = shut(r, Duration::ZERO); + out.ended = Some(Ended { failed: status.is_some_and(|s| !s.success()), last_error, unread: true, reason: None }); + return out; + } + } + let Some(stopped) = ended else { + flush(&mut r); + put(r); + // A full batch may have left more: another frame for them, not whenever one comes. + if out.handled >= POLL_BATCH + && let Some(w) = wake() + { + w(); + } + return out; + }; + // Its own exit status says whether it failed; one stopped for what it sent did. Its output + // ends as it exits: a moment for the exit to land, on the app's thread. One still running + // after it is stopped, but closing its output was its own doing: no failure. + let (status, had_to_stop, last_error) = shut(r, if stopped { Duration::ZERO } else { END_GRACE }); + // Its output closed but it didn't exit: it was stopped, and that's said. (No status of its + // own isn't enough: something else may have reaped it after a clean exit.) + if !stopped && had_to_stop && reason.is_none() { + reason = Some("it closed its output but kept running".to_string()); + } + out.ended = Some(Ended { failed: stopped || status.is_some_and(|s| !s.success()), last_error, unread: false, reason }); + out +} + +/// Tells `plugin`'s program what changed: `event` is `{"event": name, ...}`. While the program +/// is behind in reading, events wait, the latest of each kind replacing an earlier one, so a +/// program that stalls while the map moves doesn't fill memory. One that exited is found out by +/// the next [`poll`]. +pub fn event(plugin: &str, event: &Value) { + if let Ok(mut v) = RUNNING.lock() + && let Some(r) = v.iter_mut().find(|r| r.plugin == plugin) + { + r.held.retain(|e| e.get("event") != event.get("event")); + r.held.push(event.clone()); + flush(r); + } +} + +/// Sends the events held for the program, unless it's still behind in reading. +fn flush(r: &mut Running) { + if r.stdin.backlog() < process::HOLD_EVENTS_AT { + for e in std::mem::take(&mut r.held) { + let _ = r.stdin.send(&e); + } + } +} diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 08bf38d3da..345537043b 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -125,16 +125,22 @@ pub fn unpack(bytes: &[u8], pkg: &mut Package) -> Result<(), String> { return Ok(()); } let target = unpacked_dir(&pkg.manifest.id).ok_or("there is no folder for plugins")?; - unpack_to(bytes, &pkg.sha256, &target)?; + unpack_to(bytes, &pkg.sha256, &target, &pkg.manifest.id)?; pkg.dir = Some(target); Ok(()) } /// Unpacks a package to `target`, the folder a process plugin runs from, unless that copy is /// already there. -fn unpack_to(bytes: &[u8], sha256: &str, target: &Path) -> Result<(), String> { +fn unpack_to(bytes: &[u8], sha256: &str, target: &Path, id: &str) -> Result<(), String> { match stage(bytes, sha256, target)? { - Some(staged) => swap_in(&staged, target), + Some(staged) => { + // A session plugin's program runs from these files (on Windows they can't be + // replaced while it does): it stops first. + let was_running = crate::session::is_running(id); + crate::session::stop(id); + swap_in(&staged, target).map_err(|e| if was_running { format!("{e} (its running program was stopped first)") } else { e }) + } None => Ok(()), } } @@ -255,6 +261,9 @@ pub fn install_bytes(bytes: &[u8]) -> Result, String> { return Err(format!("{}: {e}", path.display())); } if let Some((s, target)) = &staged { + // A session plugin's program runs from these files (on Windows they can't be replaced + // while it does): it stops first. + crate::session::stop(&plugin.manifest.id); swap_in(s, target)?; } Ok(crate::register(plugin)) diff --git a/crates/plugins/src/testing.rs b/crates/plugins/src/testing.rs index ed5e7602c7..c7ab6649b4 100644 --- a/crates/plugins/src/testing.rs +++ b/crates/plugins/src/testing.rs @@ -256,6 +256,26 @@ pub fn process_manifest(id: &str, permissions: &[&str], run: &[String]) -> Strin m.to_string() } +/// A manifest for the session test plugin (`"session": true`): the process test plugin's +/// commands, and `k`, `v`, `i` and `z` (see [`serve_if_child`]). +pub fn session_manifest(id: &str, permissions: &[&str], run: &[String]) -> String { + let mut m: serde_json::Value = serde_json::from_str(&process_manifest(id, permissions, run)).unwrap_or_default(); + m["session"] = serde_json::json!(true); + if let Some(c) = m["commands"].as_array_mut() { + c.push(serde_json::json!({"id": "k", "label": "Ask Later"})); + c.push(serde_json::json!({"id": "v", "label": "Heard"})); + c.push(serde_json::json!({"id": "i", "label": "Process"})); + c.push(serde_json::json!({"id": "z", "label": "Quit"})); + c.push(serde_json::json!({"id": "l", "label": "Launch"})); + c.push(serde_json::json!({"id": "p", "label": "Pause"})); + c.push(serde_json::json!({"id": "w", "label": "Last Words"})); + c.push(serde_json::json!({"id": "b", "label": "Bad Exit"})); + c.push(serde_json::json!({"id": "y", "label": "Long Goodbye"})); + c.push(serde_json::json!({"id": "r", "label": "Leave a Writer"})); + } + m.to_string() +} + /// In a test binary started by a process test plugin ([`child_run`]): answers one request as the /// test module does, then exits. Elsewhere it returns at once. /// @@ -267,6 +287,18 @@ pub fn process_manifest(id: &str, permissions: &[&str], run: &[String]) -> Strin /// - Tools: `m` returns the layer "Made"; `x` selects rows 0 and 2 of "Cities"; `r` returns the /// names of the files it was given as its messages; `w` logs "working", reports /// progress "step 1 of 3" and never answers; `m` reports progress "making" first. +/// +/// Started as a session (its first line `{"session": …}`), it keeps answering commands until +/// its input closes: `h`, `e`, `c` and `s` as above; `f` sends its params as a host request and +/// returns the reply, skipping events; `k` sends that request and answers at once, leaving the +/// reply for later; `v` returns what it has heard since (`events`, `answers`, and the latest +/// `view` event's view as `last_view`); `i` returns its +/// process id; `z` answers and exits; `l` starts a copy of itself that never answers, answers +/// with its process id and exits, leaving it running; `p` answers, then reads nothing for two +/// seconds; `w` answers, then sends its params as a host request, logs "last words" and exits at +/// once; `b` answers, writes "boom" to its error output and exits with status 3; `y` answers, +/// logs 70 lines, sends its params as a host request and exits at once; `r` starts a copy of +/// itself that writes into its output forever, answers with that copy's process id and exits. #[cfg(not(target_arch = "wasm32"))] pub fn serve_if_child() { use serde_json::{Value, json}; @@ -281,6 +313,9 @@ pub fn serve_if_child() { let mut line = String::new(); let _ = input.read_line(&mut line); let req: Value = serde_json::from_str(&line).unwrap_or_default(); + if req.get("session").is_some() { + serve_session(input, out); + } let params = req.get("params").cloned().unwrap_or_default(); let reply = match req.get("id").and_then(Value::as_str).and_then(|s| s.chars().next()) { Some('h') => { @@ -334,6 +369,17 @@ pub fn serve_if_child() { Some('s') => loop { std::thread::sleep(std::time::Duration::from_secs(1)); }, + // Logs forever, as fast as it can, saying on its error output once it has written more + // than a pipe holds: from then on its output is waiting to be read, whenever it runs. + Some('u') => { + for _ in 0..8_192 { + let _ = writeln!(out, "{}", json!({"log": "more"})); + } + eprintln!("writing"); + loop { + let _ = writeln!(out, "{}", json!({"log": "more"})); + } + } Some('g') => { let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); let spawned = argv.split_first().and_then(|(exe, args)| { @@ -381,3 +427,159 @@ pub fn serve_if_child() { let _ = out.flush(); std::process::exit(0); } + +/// The session test plugin: see [`serve_if_child`]. +#[cfg(not(target_arch = "wasm32"))] +fn serve_session(mut input: impl std::io::BufRead, mut out: impl std::io::Write) -> ! { + use serde_json::{Value, json}; + let (mut events, mut answers) = (Vec::::new(), Vec::::new()); + let mut last_view = Value::Null; + let mut asked = 0u64; + loop { + let mut line = String::new(); + if input.read_line(&mut line).map_or(true, |n| n == 0) { + std::process::exit(0); + } + let msg: Value = serde_json::from_str(&line).unwrap_or_default(); + if let Some(e) = msg.get("event") { + if e == "view" { + last_view = msg["view"].clone(); + } + events.push(e.clone()); + continue; + } + let Some(req) = msg.get("command") else { + answers.push(msg); + continue; + }; + let id = msg["id"].clone(); + let params = req.get("params").cloned().unwrap_or_default(); + let mut ask = |out: &mut dyn std::io::Write| { + asked += 1; + let _ = writeln!(out, "{}", json!({"call": {"op": params["op"], "params": params["params"]}, "id": format!("a{asked}")})); + let _ = out.flush(); + format!("a{asked}") + }; + let reply = match req.get("id").and_then(Value::as_str).and_then(|s| s.chars().next()) { + Some('h') => json!({"ok": {"hello": "world"}}), + Some('f') => { + let want = ask(&mut out); + loop { + let mut answer = String::new(); + if input.read_line(&mut answer).map_or(true, |n| n == 0) { + std::process::exit(0); + } + let a: Value = serde_json::from_str(&answer).unwrap_or_default(); + if let Some(e) = a.get("event") { + if e == "view" { + last_view = a["view"].clone(); + } + events.push(e.clone()); + } else if a["id"] == json!(want) { + break a; + } + } + } + Some('r') => { + let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); + // Its output is this program's: it keeps writing there after this one exits. + let spawned = argv.split_first().and_then(|(exe, args)| { + std::process::Command::new(exe) + .args(args) + .stdin(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .ok() + }); + let pid = spawned.map(|mut g| { + if let Some(mut stdin) = g.stdin.take() { + let _ = writeln!(stdin, "{}", json!({"id": "u", "params": {}})); + } + // Exits only once the copy is writing: its output is flowing when it's gone. + if let Some(err) = g.stderr.take() { + let mut started = String::new(); + let _ = std::io::BufRead::read_line(&mut std::io::BufReader::new(err), &mut started); + } + g.id() + }); + let _ = writeln!(out, "{}", json!({"ok": pid, "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + Some('y') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + for i in 0..70 { + let _ = writeln!(out, "{}", json!({"log": format!("line {i}")})); + } + ask(&mut out); + std::process::exit(0); + } + Some('b') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + let _ = out.flush(); + eprintln!("boom"); + std::process::exit(3); + } + Some('w') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + ask(&mut out); + let _ = writeln!(out, "{}", json!({"log": "last words"})); + let _ = out.flush(); + std::process::exit(0); + } + Some('p') => { + let _ = writeln!(out, "{}", json!({"ok": "paused", "id": id})); + let _ = out.flush(); + std::thread::sleep(std::time::Duration::from_secs(2)); + continue; + } + // Answered first, so the request comes after the command. + Some('k') => { + let _ = writeln!(out, "{}", json!({"ok": "asked", "id": id})); + ask(&mut out); + continue; + } + Some('v') => { + json!({"ok": {"events": std::mem::take(&mut events), "answers": std::mem::take(&mut answers), "last_view": last_view}}) + } + Some('i') => json!({"ok": std::process::id()}), + Some('c') => { + eprintln!("boom"); + std::process::exit(3); + } + Some('s') => loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + }, + Some('z') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + Some('l') => { + let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); + let spawned = argv.split_first().and_then(|(exe, args)| { + std::process::Command::new(exe) + .args(args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .spawn() + .ok() + }); + let pid = spawned.map(|mut g| { + if let Some(mut stdin) = g.stdin.take() { + let _ = writeln!(stdin, "{}", json!({"id": "s", "params": {}})); + } + g.id() + }); + let _ = writeln!(out, "{}", json!({"ok": pid, "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + _ => json!({"error": "nope"}), + }; + let mut reply = reply; + reply["id"] = id; + let _ = writeln!(out, "{reply}"); + let _ = out.flush(); + } +} diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index be87984cde..762a68e1f6 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -15,6 +15,9 @@ impl Host for Recorder { self.ops.push(op.to_string()); match op { "host.echo" => Ok(params.clone()), + // A host request that fails unexpectedly. + #[allow(clippy::panic)] + "host.panic" => panic!("a host request failed unexpectedly"), _ => Err(format!("no '{op}'")), } } @@ -1148,6 +1151,203 @@ fn process_tools_show_progress_and_can_be_cancelled() { let _ = std::fs::remove_dir_all(&dir); } +/// Whether a process is gone: not there, or (on Linux) a zombie nothing has reaped yet, as in a +/// container with no init to reap orphans. +#[cfg(unix)] +fn gone(pid: rustix::process::Pid) -> bool { + rustix::process::test_kill_process(pid).is_err() + || std::fs::read_to_string(format!("/proc/{}/stat", pid.as_raw_nonzero())) + .ok() + .and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('Z'))) + .unwrap_or(false) +} + +/// A session test plugin played by this binary, prepared (not loaded). +fn session_plugin(id: &str) -> (crate::Plugin, std::path::PathBuf) { + let dir = std::env::temp_dir().join(format!("rustgis-test-session-{id}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = testing::child_run("tests::process_child"); + std::fs::write(dir.join("plugin.json"), testing::session_manifest(id, &[], &run)).unwrap(); + (crate::prepare(Package::from_dir(&dir).unwrap(), "test").unwrap(), dir) +} + +/// The request for the test plugin's command `id`, without parameters. +fn command(id: &str) -> Value { + json!({"abi": 1, "kind": "command", "id": id, "params": {}, "lang": "en"}) +} + +/// Polls `plugin`'s program until `done` says so, for a few seconds at most. +fn poll_until(plugin: &str, host: &mut Recorder, done: impl Fn(&crate::session::Polled, &Recorder) -> bool) -> crate::session::Polled { + let t0 = std::time::Instant::now(); + loop { + let polled = crate::session::poll(plugin, host); + if done(&polled, host) || t0.elapsed() > std::time::Duration::from_secs(20) { + return polled; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +/// A program's error output past what is kept is counted, so the log can say so. +#[test] +fn error_output_past_what_is_kept_is_counted() { + let text: String = (0..25).map(|i| format!("line {i}\n")).collect(); + let kept = crate::process::errors(std::io::Cursor::new(text.into_bytes()), None); + let t0 = std::time::Instant::now(); + while kept.lock().unwrap().dropped < 5 && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let k = kept.lock().unwrap(); + assert_eq!((k.dropped, k.lines.len(), k.lines.front().map(String::as_str)), (5, 20, Some("line 5"))); +} + +#[test] +fn session_plugins_keep_one_program_running() { + let (p, dir) = session_plugin("sess-keep"); + assert_eq!(p.info()["session"], true); + let mut host = Recorder::default(); + let pid = crate::call(&p, &mut host, &command("i"), false).unwrap(); + assert!(crate::session::is_running("sess-keep")); + assert_eq!(crate::call(&p, &mut host, &command("h"), false).unwrap(), json!({"hello": "world"})); + assert_eq!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid, "the same program answers"); + // A host request inside a command is answered as a per-call plugin's is. + let forward = json!({"abi": 1, "kind": "command", "id": "f", "params": {"op": "host.echo", "params": {"a": 1}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &forward, false).unwrap(), json!({"a": 1})); + assert_eq!(crate::call(&p, &mut host, &command("e"), false).unwrap_err(), "nope"); + + // Between commands it asks when it likes; the answer carries its id. It hears events. + let later = json!({"abi": 1, "kind": "command", "id": "k", "params": {"op": "host.echo", "params": {"a": 1}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &later, false).unwrap(), "asked"); + let polled = poll_until("sess-keep", &mut host, |_, h| h.ops.len() >= 2); + assert_eq!(host.ops, ["host.echo", "host.echo"]); + assert!(polled.ended.is_none()); + crate::session::event("sess-keep", &json!({"event": "selection"})); + let heard = crate::call(&p, &mut host, &command("v"), false).unwrap(); + assert_eq!(heard["answers"], json!([{"ok": {"a": 1}, "id": "a2"}])); + assert_eq!(heard["events"], json!(["selection"])); + + // A program that stops reading while the map moves gets the latest view when it reads again, + // not every one that came meanwhile. + assert_eq!(crate::call(&p, &mut host, &command("p"), false).unwrap(), "paused"); + for i in 0..5_000 { + crate::session::event("sess-keep", &json!({"event": "view", "view": {"scale": i, "pad": "x".repeat(100)}})); + } + std::thread::sleep(std::time::Duration::from_secs(3)); + poll_until("sess-keep", &mut host, |_, _| true); + let heard = crate::call(&p, &mut host, &command("v"), false).unwrap(); + let views = heard["events"].as_array().unwrap().len(); + assert!(views < 5_000, "{views} views queued"); + assert_eq!(heard["last_view"]["scale"], 4_999, "the latest view is the one kept"); + + // A host request that fails unexpectedly is an error answer; the program stays running. + let panics = json!({"abi": 1, "kind": "command", "id": "f", "params": {"op": "host.panic", "params": {}}, "lang": "en"}); + let e = crate::call(&p, &mut host, &panics, false).unwrap_err(); + assert!(e.contains("'host.panic' failed unexpectedly"), "{e}"); + assert_eq!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid, "the same program still answers"); + + // Stopped, it's gone; the next command starts it again. + crate::session::stop("sess-keep"); + assert!(!crate::session::is_running("sess-keep")); + #[cfg(unix)] + { + let old = rustix::process::Pid::from_raw(pid.as_i64().unwrap() as i32).unwrap(); + assert!(rustix::process::test_kill_process(old).is_err(), "{old:?} still runs"); + } + assert_ne!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid); + // A tool still runs a program of its own, once. + let tool = json!({"abi": 1, "kind": "tool", "id": "m", "params": {}, "layers": {}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &tool, true).unwrap()["messages"], json!(["made"])); + crate::session::stop("sess-keep"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn session_plugins_that_end_or_fail_are_started_again() { + let (p, dir) = session_plugin("sess-end"); + let mut host = Recorder::default(); + // Exiting by itself (its window closed) is no failure, and it is no longer running. + assert_eq!(crate::call(&p, &mut host, &command("z"), false).unwrap(), "bye"); + let polled = poll_until("sess-end", &mut host, |p, _| p.ended.is_some()); + assert_eq!(polled.ended, Some(crate::session::Ended { failed: false, last_error: None, unread: false, reason: None })); + assert!(!crate::session::is_running("sess-end")); + // What a program left running when it exited is stopped with it (on Unix). + #[cfg(unix)] + { + let left = crate::call(&p, &mut host, &command("l"), false).unwrap(); + assert!(poll_until("sess-end", &mut host, |p, _| p.ended.is_some()).ended.is_some(), "the session didn't end"); + let pid = rustix::process::Pid::from_raw(left.as_i64().unwrap() as i32).unwrap(); + let t0 = std::time::Instant::now(); + while !gone(pid) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone(pid), "{pid:?} still runs"); + } + // What it sent as it exited is still handled: a last request, a last log line. + let last = json!({"abi": 1, "kind": "command", "id": "w", "params": {"op": "host.echo", "params": {"last": true}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &last, false).unwrap(), "bye"); + let polled = poll_until("sess-end", &mut host, |p, _| p.ended.is_some()); + assert_eq!(polled.ended.map(|e| e.failed), Some(false)); + assert!(host.ops.contains(&"host.echo".to_string()) && host.logs.contains(&"last words".to_string()), "{:?} {:?}", host.ops, host.logs); + // A crash is an error naming what it wrote last; the next command starts it again. + let e = crate::call(&p, &mut host, &command("c"), false).unwrap_err(); + assert!(e.contains("exited without answering") && e.contains("boom"), "{e}"); + assert!(!crate::session::is_running("sess-end")); + assert_eq!(crate::call(&p, &mut host, &command("h"), false).unwrap(), json!({"hello": "world"})); + // One that stops answering is stopped. + let spec = + crate::process::Spec { session: true, ..crate::process::Spec::new(&testing::child_run("tests::process_child"), &dir).unwrap() }; + let t0 = std::time::Instant::now(); + let e = crate::session::command("sess-spin", &spec, &mut host, &command("s"), std::time::Duration::from_secs(2)).unwrap_err(); + assert!(e.contains("too long") && t0.elapsed() < std::time::Duration::from_secs(20), "{e}"); + assert!(!crate::session::is_running("sess-spin")); + // Updated, the running program is the old version's: stopped. Turned off, it stops too. + let p = crate::register(p); + assert!(!crate::session::is_running("sess-end")); + crate::call(&p, &mut host, &command("h"), false).unwrap(); + assert!(crate::session::is_running("sess-end")); + crate::set_enabled("sess-end", false).unwrap(); + assert!(!crate::session::is_running("sess-end")); + // Removed, it stops as well. + crate::set_enabled("sess-end", true).unwrap(); + let p = crate::get("sess-end").unwrap(); + crate::call(&p, &mut host, &command("h"), false).unwrap(); + assert!(crate::session::is_running("sess-end")); + assert!(crate::unload("sess-end")); + assert!(!crate::session::is_running("sess-end") && crate::get("sess-end").is_none()); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that exits while something it left running keeps writing into its output, faster +/// than it's read, is still ended poll after poll, with what it left running. +#[cfg(unix)] +#[test] +fn an_exited_session_whose_output_keeps_flowing_is_ended() { + let (p, dir) = session_plugin("sess-flow"); + let mut host = Recorder::default(); + let left = crate::call(&p, &mut host, &command("r"), false).unwrap(); + let pid = rustix::process::Pid::from_raw(left.as_i64().unwrap() as i32).unwrap(); + // Polled as frames would be, for a while at most: until it has exited there's nothing to + // count, and a poll with nothing new returns at once. + let mut ended = None; + let t0 = std::time::Instant::now(); + while t0.elapsed() < std::time::Duration::from_secs(30) { + if let Some(e) = crate::session::poll("sess-flow", &mut host).ended { + ended = Some(e); + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let ended = ended.expect("the session didn't end"); + assert!(ended.unread, "{ended:?}"); + assert!(!crate::session::is_running("sess-flow")); + let t0 = std::time::Instant::now(); + while !gone(pid) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone(pid), "{pid:?} still runs"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn process_manifests_are_checked() { let base = |extra: Value| { @@ -1164,6 +1364,9 @@ fn process_manifests_are_checked() { assert!(base(json!({"runtime": "process", "run": {"windos": ["x"]}})).unwrap_err().contains("names no program")); assert!(base(json!({"runtime": "process", "run": vec!["a"; 65]})).is_err()); assert!(base(json!({"run": ["python3"]})).unwrap_err().contains("process plugins")); + // Only a process plugin keeps a program running. + assert!(base(json!({"runtime": "process", "run": ["x"], "session": true})).unwrap().session); + assert!(base(json!({"session": true})).unwrap_err().contains("process plugins")); assert!(base(json!({"runtime": "native"})).is_err()); // A program by a relative path stays in the plugin's folder. assert!(base(json!({"runtime": "process", "run": ["bin/tool"]})).is_ok()); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 0fdcf54a97..c235222e9c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1100,6 +1100,9 @@ pub struct RustGisApp { /// to give its wake callback yet: `logic` installs the wake as soon as a context exists. #[cfg(not(target_arch = "wasm32"))] relay_wake_pending: bool, + /// Whether session plugins' programs ask for a repaint when they send a line. + #[cfg(not(target_arch = "wasm32"))] + plugin_wake_set: bool, /// Extra font faces installed with the interface fonts: (name, face, script). The current /// language's face comes right after the interface font (before it for Arabic script), so its /// glyph shapes win; the others are last-resort fallbacks. Faces are shared, so reinstalling @@ -1304,6 +1307,8 @@ impl RustGisApp { frame_ctx: None, #[cfg(not(target_arch = "wasm32"))] relay_wake_pending: false, + #[cfg(not(target_arch = "wasm32"))] + plugin_wake_set: false, extra_fonts: Vec::new(), installed_region: None, fonts_dirty: false, @@ -4996,6 +5001,21 @@ impl RustGisApp { } } relayshot::drive(self, ctx); + // Session plugins' programs: what they asked, and what changed on the map for them. + if !self.plugin_wake_set { + let ctx = ctx.clone(); + rustgis_engine::plugins::session::set_wake(std::sync::Arc::new(move || ctx.request_repaint())); + self.plugin_wake_set = true; + } + let state = self.session.state(); + if self.session.poll_plugin_sessions() > 0 && self.session.state() != state { + self.map.cancel_label_drag(); + } + // A program that exits without a last line (something it left running holds its output + // open) wakes nothing: while one runs, the app looks at least once a second. + if !rustgis_engine::plugins::session::running().is_empty() { + ctx.request_repaint_after(std::time::Duration::from_secs(1)); + } // A script an agent wrote to the file bridge. if let Some(bridge) = self.file_bridge.clone() { if !bridge.has_wake() { diff --git a/crates/ui-egui/src/sessionlog.rs b/crates/ui-egui/src/sessionlog.rs index c9923f2850..17602121d7 100644 --- a/crates/ui-egui/src/sessionlog.rs +++ b/crates/ui-egui/src/sessionlog.rs @@ -527,7 +527,7 @@ mod tests { save(&mut app); let (name, text) = got.borrow_mut().take().expect("downloaded"); assert_eq!(name, "session-log.csv"); - assert!(text.starts_with("time_utc,at,command,ok,millis,error,params,project\n"), "{text}"); + assert!(text.starts_with("time_utc,at,command,ok,millis,error,params,plugin,project\n"), "{text}"); assert!(text.contains("\"map.new\",true"), "{text}"); let dropped = |app: &RustGisApp| { app.session.notices.iter().any(|n| n.text.contains("session-log.csv: not in it: 4 oldest entries were dropped past")) diff --git a/docs/plugins.md b/docs/plugins.md index 53b5446f15..ffc6bc3d78 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -693,6 +693,58 @@ except Exception as e: print(json.dumps(result), flush=True) ``` +### Session plugins + +A process plugin with `"session": true` keeps one program running between its commands, so it +can ask RustGIS for things whenever it likes and hear when the map changes: a view of the map in +a window of its own (a 3D globe linked with it) is one. The program talks to RustGIS only over +its standard input and output: no port, no token, nothing another program can join. + +```json +"runtime": "process", +"session": true, +"run": ["bin/globe"] +``` + +- RustGIS starts the program the first time one of the plugin's commands runs and writes a first + line `{"session": {"abi": 1, "plugin": "", "rustgis": "", "lang": ""}}`. +- Each command comes as `{"command": request, "id": n}`, where `request` is what a per-call + program gets. The program answers `{"ok": value, "id": n}` or `{"error": "text", "id": n}`, + within 2 minutes. +- At any time, during a command or between commands, the program can send a host request + `{"call": {"op": "...", "params": {...}}, "id": "any"}`. RustGIS answers + `{"ok": value, "id": "any"}` or `{"error": "text", "id": "any"}`; the `id` is optional and comes + back as given. Requests go through the same checks as a command's, within the plugin's + permissions. Between commands, each request that can change the project is an undo step of its + own, named after the plugin, and an entry in the session log; `host.fetch` is refused: it would make the app wait, and the + program can fetch for itself. The desktop app answers requests as they come; `rustgis-cli` and + the Python package only while one of the plugin's commands runs (or as the next one starts); + `rustgis-cli` ends the program after its last command, and what it asks after that isn't + answered. +- When the map changes, the desktop app sends `{"event": "view", "view": {center, scale, + projection}}` (the view, as `host.view` answers it), `{"event": "layers"}` (layers added, + removed, reordered, made active, renamed, shown or hidden, restyled, made more or less transparent, their + fields or features changed, and web tile layers changed), `{"event": "selection"}` or `{"event": "project"}` (another project + opened). Ask for what changed with host requests. Changes the program made itself are told too: a view it set comes back as + a `view` event, so compare before following it. An event can arrive while the program waits + for an answer: match answers by their `id`. While the program is behind in reading its input, + events wait, and only the latest of each kind is sent. +- `log` and `progress` lines work as for a per-call program. +- The program RustGIS started is the session: keep it running, rather than starting another + program and exiting. RustGIS stops it, with what it started, when the plugin is turned off, + removed or updated and when the desktop app closes. Its input closing means it is being + stopped (RustGIS closing or gone, however it ended; the plugin turned off, removed or updated; + the program failed or stopped answering): the program should then exit, and stop what it + started. A program may also + exit by itself (its window closed); the plugin's next command starts it again. One that fails + or stops answering a command is stopped, and the error says so. +- On Linux and macOS, while RustGIS runs, what a program left running when it exited is stopped + with it. Windows can't find what an exited program started, so there the program must stop it + itself, as every program must once RustGIS has gone. +- The plugin's tools still start the program once per run, with the request as its first line. +- RustGIS versions before sessions run the program once per command instead, so set + `minRustgisVersion` to the first version with them. + ### Python environments A Python plugin can have RustGIS make its environment, so its users don't install Python or diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index fcff74bc06..572e4f0507 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -132,7 +132,9 @@ isn't itself an entry in the log, as a saved file is). **Clear Log** empties it that waited for its downloads, when its last run started), `command`, `params`, `ok`, `error` when it failed, `millis`, how long it took, `project`, the name of the project open when it ran (the one it opened, for a command that opens one), and `partial`, true when its parameters were - cut short or had secrets taken out. + cut short or had secrets taken out, and, for what a session plugin's program asked + for between the plugin's commands, `plugin`, that plugin's id (left out of `project.log` and + JSON Lines for other entries; an empty cell in CSV). - **For the session only:** the log is kept in memory, across the projects opened or made in the session, until RustGIS closes or the log is cleared. A saved project, its recovery copy, a package or a template carries none of it, so sharing a project shares no log. Reading, writing From d69c49a8accd075e7a42fa9746992e9de198fbab Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 17:52:14 -0600 Subject: [PATCH 2/6] fix(plugins): requests sent before a command are answered between commands; view gives bearing and pitch - What a session program sent before RustGIS writes a command is set aside for the next poll and answered as between commands; docs say a request read while a command runs is part of it. - host.view, and so the view event, give the camera's bearing and pitch. - A failed install says its running program was stopped first, on both install paths. - Test: the between-command log entry carries the plugin. --- crates/engine/src/cmd/plugins.rs | 2 +- crates/engine/src/tests.rs | 5 ++++- crates/plugins/src/session.rs | 14 +++++++++++++- crates/plugins/src/store.rs | 3 ++- docs/plugins.md | 9 ++++++--- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 2ffb2a286c..0fa069c32a 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -806,7 +806,7 @@ fn seen(s: &mut Session, view: &Value) -> Seen { fn view_of(s: &Session) -> Value { let c = &s.map().camera; let center = c.center_lonlat(); - json!({"center": [center.x, center.y], "scale": c.scale(), "projection": s.map().projection.code()}) + json!({"center": [center.x, center.y], "scale": c.scale(), "projection": s.map().projection.code(), "bearing": c.bearing, "pitch": c.pitch}) } impl Session { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 29e8a522b1..58b565b1da 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -10544,7 +10544,10 @@ fn session_plugins_ask_between_commands_and_hear_the_map() { s.run("plugin:eng-sess/k", &json!({"op": "host.addLayer", "params": {"name": "Later", "geojson": fc}})).unwrap(); poll_until(&mut s, &|s| s.map().layer_by_name("Later").is_some()); assert_eq!(s.undo_label(), Some("Test eng-sess"), "its own step"); - assert!(s.log.entries().any(|e| e.command == "host.addLayer"), "in the session log"); + assert!( + s.log.entries().any(|e| e.command == "host.addLayer" && e.plugin.as_deref() == Some("eng-sess")), + "in the session log, as the plugin's" + ); assert!(s.undo()); assert!(s.map().layer_by_name("Later").is_none()); assert!(s.redo()); diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs index cac9b42d29..18c2f86660 100644 --- a/crates/plugins/src/session.rs +++ b/crates/plugins/src/session.rs @@ -72,6 +72,9 @@ struct Running { /// How many polls have found it exited with its output still coming: something it left /// running keeps writing. At [`DRAIN_BATCHES`] it's stopped. drained: usize, + /// What it sent before a command was written to it, set aside for the next [`poll`] to + /// answer as between its commands, not as part of that command. + before_command: std::collections::VecDeque, /// The last line of its error output logged so far. last_stderr: Option, } @@ -195,6 +198,7 @@ fn start(plugin: &str, spec: &Spec) -> Result { logged: (Instant::now(), 0), held: Vec::new(), drained: 0, + before_command: std::collections::VecDeque::new(), last_stderr: None, }; let hello = json!({"session": {"abi": crate::manifest::ABI, "plugin": plugin, "rustgis": crate::manifest::RUSTGIS_VERSION, "lang": crate::language()}}); @@ -384,6 +388,12 @@ pub fn command(plugin: &str, spec: &Spec, host: &mut dyn Host, request: &Value, }; let id = r.next_id; r.next_id += 1; + // What it sent before this command is written isn't part of it: set aside for the next + // poll (as between its commands, each request its own step). What it sends from here on, + // until it answers, is the command's. + while let Ok(line) = r.rx.try_recv() { + r.before_command.push_back(line); + } if let Err(e) = r.stdin.send(&json!({"command": request, "id": id})) { shut(r, Duration::ZERO); return Err(e); @@ -477,7 +487,9 @@ pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { // runs empty to say so. let mut gone = process::exited(&mut r.child).then(|| Instant::now() + END_GRACE); for _ in 0..POLL_BATCH { - let line = if let Some(until) = gone { + let line = if let Some(l) = r.before_command.pop_front() { + Some(l) + } else if let Some(until) = gone { // One moment for all of it, not one a line. r.rx.recv_timeout(until.saturating_duration_since(Instant::now())).ok() } else { diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 345537043b..15c3a52bed 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -263,8 +263,9 @@ pub fn install_bytes(bytes: &[u8]) -> Result, String> { if let Some((s, target)) = &staged { // A session plugin's program runs from these files (on Windows they can't be replaced // while it does): it stops first. + let was_running = crate::session::is_running(&plugin.manifest.id); crate::session::stop(&plugin.manifest.id); - swap_in(s, target)?; + swap_in(s, target).map_err(|e| if was_running { format!("{e} (its running program was stopped first)") } else { e })?; } Ok(crate::register(plugin)) } diff --git a/docs/plugins.md b/docs/plugins.md index ffc6bc3d78..23ef6f1a0a 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -436,7 +436,7 @@ Inside a request, `host::call(op, params)` sends one of these to RustGIS: | `host.info` | `{rustgis, abi, plugin, permissions}`: version, ABI, plugin id and permissions | | `host.layers` | `[{id, name, raster, visible, geometry, features, selected, active, fields: [{name, type}]}]` | | `host.features` `{layer, selected?, limit?}` | the layer's features as GeoJSON; each feature has its `row` | -| `host.view` | `{center: [lon, lat], scale, projection}` | +| `host.view` | `{center: [lon, lat], scale, projection, bearing, pitch}` (bearing and pitch in degrees, 0 for a flat north-up map) | | `host.notify` `{text, error?}` | shows a message in the status bar | | `host.addLayer` `{name, geojson}` | adds a layer (`edit`) | | `host.fetch` `{url}` | `{url, text, json?}`: the document at a web address (`network`; see [Fetching a document](#fetching-a-document)) | @@ -717,12 +717,15 @@ its standard input and output: no port, no token, nothing another program can jo back as given. Requests go through the same checks as a command's, within the plugin's permissions. Between commands, each request that can change the project is an undo step of its own, named after the plugin, and an entry in the session log; `host.fetch` is refused: it would make the app wait, and the - program can fetch for itself. The desktop app answers requests as they come; `rustgis-cli` and + program can fetch for itself. What RustGIS has read before it writes a command is answered as + between commands; a request read after that, until the program answers, is part of that + command (its undo step), so send one meant to stand alone after answering, not while a command + may be on its way. The desktop app answers requests as they come; `rustgis-cli` and the Python package only while one of the plugin's commands runs (or as the next one starts); `rustgis-cli` ends the program after its last command, and what it asks after that isn't answered. - When the map changes, the desktop app sends `{"event": "view", "view": {center, scale, - projection}}` (the view, as `host.view` answers it), `{"event": "layers"}` (layers added, + projection, bearing, pitch}}` (the view, as `host.view` answers it), `{"event": "layers"}` (layers added, removed, reordered, made active, renamed, shown or hidden, restyled, made more or less transparent, their fields or features changed, and web tile layers changed), `{"event": "selection"}` or `{"event": "project"}` (another project opened). Ask for what changed with host requests. Changes the program made itself are told too: a view it set comes back as From faab2fc66f7287bbb31886e08cdd4d2f0ce19db6 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 19:18:15 -0600 Subject: [PATCH 3/6] fix(plugins): what's set aside before a command is bounded A program writing without pause can't keep a command from being sent: setting aside what arrived before it is bounded like the drain of an exited program; past the bound, what the program sends is read with the command. A program a command starts (none running, or the last one gone) takes the map as it is then, not what an earlier program was last told: no events for what changed before it started. --- crates/engine/src/cmd/plugins.rs | 6 ++++++ crates/engine/src/tests.rs | 26 ++++++++++++++++++++++++++ crates/plugins/src/session.rs | 5 ++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 0fa069c32a..2a5e2ce41b 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -521,6 +521,12 @@ impl Session { { self.merge = merge; } + // A program this command starts (none running, or the last one gone) takes the map as it + // is now, not what an earlier program was last told. + #[cfg(not(target_arch = "wasm32"))] + if plugin.manifest.session && !rustgis_plugins::session::is_running(&plugin.manifest.id) { + self.plugin_watch.seen.retain(|(p, _)| *p != plugin.manifest.id); + } #[cfg(not(target_arch = "wasm32"))] if plugin.manifest.session && !self.plugin_watch.seen.iter().any(|(p, _)| *p == plugin.manifest.id) { let view = view_of(self); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 58b565b1da..f9f6de6985 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -10684,6 +10684,32 @@ fn a_session_program_whose_output_kept_coming_is_said() { let _ = std::fs::remove_dir_all(&dir); } +/// A program started again (the last one exited, no poll in between) takes the map as it is +/// then: it isn't told of what changed before it started. +#[test] +fn a_restarted_session_program_starts_from_the_map_as_it_is() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-restart-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-restart", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-restart/h", &Value::Null).unwrap(); + s.poll_plugin_sessions(); + // The view moves, and the program exits, with no poll between. + s.map_mut().camera.pan_px(40.0, 0.0); + assert_eq!(s.run("plugin:eng-sess-restart/z", &Value::Null).unwrap(), "bye"); + wait_for_exit("eng-sess-restart"); + // Its next command starts another, which hears nothing of the earlier move. + s.run("plugin:eng-sess-restart/h", &Value::Null).unwrap(); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess-restart/v", &Value::Null).unwrap()["events"], json!([])); + rustgis_plugins::session::stop("eng-sess-restart"); + rustgis_plugins::unload("eng-sess-restart"); + let _ = std::fs::remove_dir_all(&dir); +} + /// A program that failed after answering is said to have when the next command finds it, as /// the frame's poll would: the command line has no poll. #[test] diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs index 18c2f86660..109dcef7cc 100644 --- a/crates/plugins/src/session.rs +++ b/crates/plugins/src/session.rs @@ -391,7 +391,10 @@ pub fn command(plugin: &str, spec: &Spec, host: &mut dyn Host, request: &Value, // What it sent before this command is written isn't part of it: set aside for the next // poll (as between its commands, each request its own step). What it sends from here on, // until it answers, is the command's. - while let Ok(line) = r.rx.try_recv() { + // Bounded: a program writing without pause can't keep the command from being sent; past + // the bound, what it sends is read with the command. + for _ in 0..POLL_BATCH * DRAIN_BATCHES { + let Ok(line) = r.rx.try_recv() else { break }; r.before_command.push_back(line); } if let Err(e) = r.stdin.send(&json!({"command": request, "id": id})) { From 221a67fca719701c7e86be286af6c36762bb5fdd Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 20:00:25 -0600 Subject: [PATCH 4/6] fix(plugins): no set-aside before a command (it could deadlock); symbology cache keyed on edits - The set-aside of what arrived just before a command is taken back out: a program that waits for its reply before reading the command would never read it. The engine's drain before each command still answers what was sent until then as between commands. - The symbology and selection fingerprints are cached on the project's state, which edits change and reads or view moves (map.goToXY) don't, with the one-second refresh kept. --- crates/engine/src/cmd/plugins.rs | 11 ++++++----- crates/plugins/src/session.rs | 17 +---------------- docs/plugins.md | 8 ++++---- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 2a5e2ce41b..6b8e430af4 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -735,9 +735,10 @@ pub(crate) struct Watch { /// Each running program's, by plugin id. seen: Vec<(String, Seen)>, /// The layers' symbology and fields and the web tile layers, and the selection, each - /// fingerprinted, with the [`Session::revision`] and the time they were taken at. They're - /// taken again when a command bumps the revision, and at least every [`RESTYLE_EVERY`] for - /// what changes them without one: not every frame. + /// fingerprinted, with the project's [`Session::state`] and the time they were taken at. + /// They're taken again when an edit changes the state (a read or a move of the view + /// doesn't), and at least every [`RESTYLE_EVERY`] for what changes them without one: not + /// every frame. styled: Option<(u64, web_time::Instant, u64, u64)>, } @@ -768,7 +769,7 @@ fn seen(s: &mut Session, view: &Value) -> Seen { h.finish() }; let (styled, selection) = match s.plugin_watch.styled { - Some((revision, at, styled, selection)) if revision == s.revision && at.elapsed() < RESTYLE_EVERY => (styled, selection), + Some((state, at, styled, selection)) if state == s.state() && at.elapsed() < RESTYLE_EVERY => (styled, selection), _ => { let map = s.map(); let styled = hash(&|h| { @@ -789,7 +790,7 @@ fn seen(s: &mut Session, view: &Value) -> Seen { (l.id, &l.selection).hash(h); } }); - s.plugin_watch.styled = Some((s.revision, web_time::Instant::now(), styled, selection)); + s.plugin_watch.styled = Some((s.state(), web_time::Instant::now(), styled, selection)); (styled, selection) } }; diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs index 109dcef7cc..cac9b42d29 100644 --- a/crates/plugins/src/session.rs +++ b/crates/plugins/src/session.rs @@ -72,9 +72,6 @@ struct Running { /// How many polls have found it exited with its output still coming: something it left /// running keeps writing. At [`DRAIN_BATCHES`] it's stopped. drained: usize, - /// What it sent before a command was written to it, set aside for the next [`poll`] to - /// answer as between its commands, not as part of that command. - before_command: std::collections::VecDeque, /// The last line of its error output logged so far. last_stderr: Option, } @@ -198,7 +195,6 @@ fn start(plugin: &str, spec: &Spec) -> Result { logged: (Instant::now(), 0), held: Vec::new(), drained: 0, - before_command: std::collections::VecDeque::new(), last_stderr: None, }; let hello = json!({"session": {"abi": crate::manifest::ABI, "plugin": plugin, "rustgis": crate::manifest::RUSTGIS_VERSION, "lang": crate::language()}}); @@ -388,15 +384,6 @@ pub fn command(plugin: &str, spec: &Spec, host: &mut dyn Host, request: &Value, }; let id = r.next_id; r.next_id += 1; - // What it sent before this command is written isn't part of it: set aside for the next - // poll (as between its commands, each request its own step). What it sends from here on, - // until it answers, is the command's. - // Bounded: a program writing without pause can't keep the command from being sent; past - // the bound, what it sends is read with the command. - for _ in 0..POLL_BATCH * DRAIN_BATCHES { - let Ok(line) = r.rx.try_recv() else { break }; - r.before_command.push_back(line); - } if let Err(e) = r.stdin.send(&json!({"command": request, "id": id})) { shut(r, Duration::ZERO); return Err(e); @@ -490,9 +477,7 @@ pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { // runs empty to say so. let mut gone = process::exited(&mut r.child).then(|| Instant::now() + END_GRACE); for _ in 0..POLL_BATCH { - let line = if let Some(l) = r.before_command.pop_front() { - Some(l) - } else if let Some(until) = gone { + let line = if let Some(until) = gone { // One moment for all of it, not one a line. r.rx.recv_timeout(until.saturating_duration_since(Instant::now())).ok() } else { diff --git a/docs/plugins.md b/docs/plugins.md index 23ef6f1a0a..6aa7c90bdd 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -717,10 +717,10 @@ its standard input and output: no port, no token, nothing another program can jo back as given. Requests go through the same checks as a command's, within the plugin's permissions. Between commands, each request that can change the project is an undo step of its own, named after the plugin, and an entry in the session log; `host.fetch` is refused: it would make the app wait, and the - program can fetch for itself. What RustGIS has read before it writes a command is answered as - between commands; a request read after that, until the program answers, is part of that - command (its undo step), so send one meant to stand alone after answering, not while a command - may be on its way. The desktop app answers requests as they come; `rustgis-cli` and + program can fetch for itself. Before one of the plugin's commands, what the program sent until then + is answered as between commands; a request read while the command runs, until the program + answers, is part of that command (its undo step), so send one meant to stand alone after + answering, not while a command may be on its way. The desktop app answers requests as they come; `rustgis-cli` and the Python package only while one of the plugin's commands runs (or as the next one starts); `rustgis-cli` ends the program after its last command, and what it asks after that isn't answered. From b0b289c5ddbf9fd9ab7008eb5960dbaa6a3cd226 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 20:54:06 -0600 Subject: [PATCH 5/6] fix(plugins): selection changes are told even though they change no project state - The selection fingerprint has its own cache, keyed on the revision a selection command bumps; the symbology stays keyed on the project's state. - Docs: whether a request is part of a command depends on when RustGIS handles it. --- crates/engine/src/cmd/plugins.rs | 35 +++++++++++++++++++++----------- docs/plugins.md | 10 +++++---- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 6b8e430af4..c42775da80 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -734,12 +734,14 @@ impl Host for Between<'_> { pub(crate) struct Watch { /// Each running program's, by plugin id. seen: Vec<(String, Seen)>, - /// The layers' symbology and fields and the web tile layers, and the selection, each - /// fingerprinted, with the project's [`Session::state`] and the time they were taken at. - /// They're taken again when an edit changes the state (a read or a move of the view - /// doesn't), and at least every [`RESTYLE_EVERY`] for what changes them without one: not - /// every frame. - styled: Option<(u64, web_time::Instant, u64, u64)>, + /// The layers' symbology and fields and the web tile layers, fingerprinted, with the + /// project's [`Session::state`] and the time it was taken at. It's taken again when an edit + /// changes the state (a read or a move of the view doesn't), and at least every + /// [`RESTYLE_EVERY`] for what changes it without one: not every frame. + styled: Option<(u64, web_time::Instant, u64)>, + /// The selection, fingerprinted, with the [`Session::revision`] it was taken at: selecting + /// changes no project state but runs a command, which bumps the revision. + selected: Option<(u64, u64)>, } /// How often a session plugin's layer symbology is looked at again without a command. @@ -768,8 +770,8 @@ fn seen(s: &mut Session, view: &Value) -> Seen { f(&mut h); h.finish() }; - let (styled, selection) = match s.plugin_watch.styled { - Some((state, at, styled, selection)) if state == s.state() && at.elapsed() < RESTYLE_EVERY => (styled, selection), + let styled = match s.plugin_watch.styled { + Some((state, at, styled)) if state == s.state() && at.elapsed() < RESTYLE_EVERY => styled, _ => { let map = s.map(); let styled = hash(&|h| { @@ -783,15 +785,23 @@ fn seen(s: &mut Session, view: &Value) -> Seen { } serde_json::to_string(&map.tiles).unwrap_or_default().hash(h); }); - // A large selection is many rows to walk: with the symbology, not every frame. Layers - // with nothing selected don't count: adding one changes no selection. + s.plugin_watch.styled = Some((s.state(), web_time::Instant::now(), styled)); + styled + } + }; + // A large selection is many rows to walk: again only after a command. Layers with nothing + // selected don't count: adding one changes no selection. + let selection = match s.plugin_watch.selected { + Some((revision, selection)) if revision == s.revision => selection, + _ => { + let map = s.map(); let selection = hash(&|h| { for l in map.layers.iter().filter(|l| !l.selection.is_empty()) { (l.id, &l.selection).hash(h); } }); - s.plugin_watch.styled = Some((s.state(), web_time::Instant::now(), styled, selection)); - (styled, selection) + s.plugin_watch.selected = Some((s.revision, selection)); + selection } }; let map = s.map(); @@ -874,6 +884,7 @@ impl Session { self.plugin_watch.seen.retain(|(id, _)| running.contains(id)); if running.is_empty() { self.plugin_watch.styled = None; + self.plugin_watch.selected = None; return handled; } let view = view_of(self); diff --git a/docs/plugins.md b/docs/plugins.md index 6aa7c90bdd..ae10e4813b 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -717,10 +717,12 @@ its standard input and output: no port, no token, nothing another program can jo back as given. Requests go through the same checks as a command's, within the plugin's permissions. Between commands, each request that can change the project is an undo step of its own, named after the plugin, and an entry in the session log; `host.fetch` is refused: it would make the app wait, and the - program can fetch for itself. Before one of the plugin's commands, what the program sent until then - is answered as between commands; a request read while the command runs, until the program - answers, is part of that command (its undo step), so send one meant to stand alone after - answering, not while a command may be on its way. The desktop app answers requests as they come; `rustgis-cli` and + program can fetch for itself. Whether a request is part of a command depends on when RustGIS handles + it, not when the program sent it: one handled before the command starts (by a poll, or as the + command is about to run) is answered as between commands; one handled while the command runs, + until the program answers, is part of that command (its undo step), even if it was sent just + before. Send a request meant to stand alone after answering the command, and wait for its + reply. The desktop app answers requests as they come; `rustgis-cli` and the Python package only while one of the plugin's commands runs (or as the next one starts); `rustgis-cli` ends the program after its last command, and what it asks after that isn't answered. From 66369fc18aedf88ac032912aa4143d8c83527165 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 21:30:34 -0600 Subject: [PATCH 6/6] fix(plugins): a session program that stops reading its input is stopped, and that's said A failed write to the program (an event, an answer) marks its input closed; the next poll stops one still running with "it stopped reading its input while running", instead of keeping it as running while nothing reaches it, and that reason replaces the one about its output. The selection fingerprint is also taken again at least every second (a streamed layer's window can change it without a command); docs: a request sent after answering is a separate step likely, not certain. --- crates/engine/src/cmd/plugins.rs | 12 +++++++----- crates/plugins/src/session.rs | 22 +++++++++++++++++++--- docs/plugins.md | 5 +++-- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index c42775da80..7188a925d8 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -739,9 +739,11 @@ pub(crate) struct Watch { /// changes the state (a read or a move of the view doesn't), and at least every /// [`RESTYLE_EVERY`] for what changes it without one: not every frame. styled: Option<(u64, web_time::Instant, u64)>, - /// The selection, fingerprinted, with the [`Session::revision`] it was taken at: selecting - /// changes no project state but runs a command, which bumps the revision. - selected: Option<(u64, u64)>, + /// The selection, fingerprinted, with the [`Session::revision`] and the time it was taken + /// at: selecting changes no project state but runs a command, which bumps the revision; what + /// changes it without one (a streamed layer reading another window) is caught within + /// [`RESTYLE_EVERY`]. + selected: Option<(u64, web_time::Instant, u64)>, } /// How often a session plugin's layer symbology is looked at again without a command. @@ -792,7 +794,7 @@ fn seen(s: &mut Session, view: &Value) -> Seen { // A large selection is many rows to walk: again only after a command. Layers with nothing // selected don't count: adding one changes no selection. let selection = match s.plugin_watch.selected { - Some((revision, selection)) if revision == s.revision => selection, + Some((revision, at, selection)) if revision == s.revision && at.elapsed() < RESTYLE_EVERY => selection, _ => { let map = s.map(); let selection = hash(&|h| { @@ -800,7 +802,7 @@ fn seen(s: &mut Session, view: &Value) -> Seen { (l.id, &l.selection).hash(h); } }); - s.plugin_watch.selected = Some((s.revision, selection)); + s.plugin_watch.selected = Some((s.revision, web_time::Instant::now(), selection)); selection } }; diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs index cac9b42d29..645358b1e6 100644 --- a/crates/plugins/src/session.rs +++ b/crates/plugins/src/session.rs @@ -72,6 +72,8 @@ struct Running { /// How many polls have found it exited with its output still coming: something it left /// running keeps writing. At [`DRAIN_BATCHES`] it's stopped. drained: usize, + /// Writing to it failed: it closed its input (stopped reading) while it may still run. + input_closed: bool, /// The last line of its error output logged so far. last_stderr: Option, } @@ -195,6 +197,7 @@ fn start(plugin: &str, spec: &Spec) -> Result { logged: (Instant::now(), 0), held: Vec::new(), drained: 0, + input_closed: false, last_stderr: None, }; let hello = json!({"session": {"abi": crate::manifest::ABI, "plugin": plugin, "rustgis": crate::manifest::RUSTGIS_VERSION, "lang": crate::language()}}); @@ -527,7 +530,10 @@ pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { // Exited, it can't be answered; what it asked was done. Err(_) if gone.is_some() => {} // It can't be written to: it's exiting, and what it sent last is read first. - Err(_) => gone = Some(Instant::now() + END_GRACE), + Err(_) => { + r.input_closed = true; + gone = Some(Instant::now() + END_GRACE); + } } } // Exited, and its output still coming after this batch: something it left running keeps @@ -541,6 +547,11 @@ pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { return out; } } + // It stopped reading while it still runs: nothing RustGIS says reaches it. Stopped, and said. + if ended.is_none() && r.input_closed && !process::exited(&mut r.child) { + reason = Some("it stopped reading its input while running".to_string()); + ended = Some(true); + } let Some(stopped) = ended else { flush(&mut r); put(r); @@ -555,11 +566,14 @@ pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { // Its own exit status says whether it failed; one stopped for what it sent did. Its output // ends as it exits: a moment for the exit to land, on the app's thread. One still running // after it is stopped, but closing its output was its own doing: no failure. + let input_closed = r.input_closed; let (status, had_to_stop, last_error) = shut(r, if stopped { Duration::ZERO } else { END_GRACE }); // Its output closed but it didn't exit: it was stopped, and that's said. (No status of its // own isn't enough: something else may have reaped it after a clean exit.) if !stopped && had_to_stop && reason.is_none() { - reason = Some("it closed its output but kept running".to_string()); + reason = Some( + if input_closed { "it stopped reading its input while running" } else { "it closed its output but kept running" }.to_string(), + ); } out.ended = Some(Ended { failed: stopped || status.is_some_and(|s| !s.success()), last_error, unread: false, reason }); out @@ -583,7 +597,9 @@ pub fn event(plugin: &str, event: &Value) { fn flush(r: &mut Running) { if r.stdin.backlog() < process::HOLD_EVENTS_AT { for e in std::mem::take(&mut r.held) { - let _ = r.stdin.send(&e); + if r.stdin.send(&e).is_err() { + r.input_closed = true; + } } } } diff --git a/docs/plugins.md b/docs/plugins.md index ae10e4813b..b07ecb8c99 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -721,8 +721,9 @@ its standard input and output: no port, no token, nothing another program can jo it, not when the program sent it: one handled before the command starts (by a poll, or as the command is about to run) is answered as between commands; one handled while the command runs, until the program answers, is part of that command (its undo step), even if it was sent just - before. Send a request meant to stand alone after answering the command, and wait for its - reply. The desktop app answers requests as they come; `rustgis-cli` and + before. Sending a request meant to stand alone after answering the command, and + waiting for its reply, makes that likely but not certain: if another of its commands starts + before RustGIS handles the request, the request is part of that command. The desktop app answers requests as they come; `rustgis-cli` and the Python package only while one of the plugin's commands runs (or as the next one starts); `rustgis-cli` ends the program after its last command, and what it asks after that isn't answered.