diff --git a/apps/rustgis-cli/src/main.rs b/apps/rustgis-cli/src/main.rs index be53219218..990bf70284 100644 --- a/apps/rustgis-cli/src/main.rs +++ b/apps/rustgis-cli/src/main.rs @@ -685,7 +685,16 @@ fn mcp(s: &mut Session, args: &[String]) -> Result<(), String> { Ok(()) } +/// Runs the command line, then stops the session plugins' programs its commands started. fn main() -> ExitCode { + let code = cli(); + // Session plugins' programs a command started end with the command line. + rustgis_engine::plugins::session::stop_all(); + code +} + +/// The command line: loads the plugins, then runs the subcommand the arguments name. +fn cli() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); let Some(cmd) = args.first().map(|s| s.as_str()) else { return usage() }; if !args.iter().any(|a| a == "--no-plugins") { diff --git a/apps/rustgis/src/main.rs b/apps/rustgis/src/main.rs index 833cb48a25..c42ef52ad6 100644 --- a/apps/rustgis/src/main.rs +++ b/apps/rustgis/src/main.rs @@ -342,6 +342,8 @@ impl eframe::App for App { fn on_exit(&mut self) { self.0.end_exports(); self.0.end_autosave(); + // Session plugins' programs end with the app, with what they started. + rustgis_engine::plugins::session::stop_all(); // The installer starts before the marker goes, so a RustGIS started meanwhile finds this // one still running. #[cfg(windows)] diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index da786e3190..7188a925d8 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -282,6 +282,25 @@ pub fn reaches(spec: &C, params: &Value) -> Option { } } +/// The project's layers as `host.layers` answers them. +fn layers_of(s: &Session) -> Value { + let active = s.active_layer; + Value::Array( + s.map() + .layers + .iter() + .map(|l| { + json!({ + "id": l.id, "name": l.name, "raster": l.is_raster(), "visible": l.visible, + "geometry": format!("{:?}", l.dataset.geometry_type), "features": l.dataset.features.len(), + "selected": l.selection.len(), "active": Some(l.id) == active, + "fields": l.dataset.fields.iter().map(|f| json!({"name": f.name, "type": format!("{:?}", f.ty)})).collect::>(), + }) + }) + .collect(), + ) +} + /// Answers a running plugin's requests. struct Bridge<'a> { s: &'a mut Session, @@ -310,22 +329,7 @@ impl Bridge<'_> { } fn layers(&self) -> Value { - let active = self.s.active_layer; - Value::Array( - self.s - .map() - .layers - .iter() - .map(|l| { - json!({ - "id": l.id, "name": l.name, "raster": l.is_raster(), "visible": l.visible, - "geometry": format!("{:?}", l.dataset.geometry_type), "features": l.dataset.features.len(), - "selected": l.selection.len(), "active": Some(l.id) == active, - "fields": l.dataset.fields.iter().map(|f| json!({"name": f.name, "type": format!("{:?}", f.ty)})).collect::>(), - }) - }) - .collect(), - ) + layers_of(self.s) } /// A layer's features as GeoJSON: all of them, or its selection (`"selected": true`), at @@ -437,6 +441,14 @@ impl Bridge<'_> { } } +impl Bridge<'_> { + /// Whether host request `op` only reads (or says something): no undo step for it. + #[cfg(not(target_arch = "wasm32"))] + fn reads_only(op: &str) -> bool { + matches!(op, "host.info" | "host.layers" | "host.features" | "host.view" | "host.notify") + } +} + impl Host for Bridge<'_> { fn call(&mut self, op: &str, params: &Value) -> Result { match op { @@ -444,11 +456,7 @@ impl Host for Bridge<'_> { "plugin": self.plugin.manifest.id, "permissions": self.plugin.manifest.permissions})), "host.layers" => Ok(self.layers()), "host.features" => self.features(params), - "host.view" => { - let c = &self.s.map().camera; - let center = c.center_lonlat(); - Ok(json!({"center": [center.x, center.y], "scale": c.scale(), "projection": self.s.map().projection.code()})) - } + "host.view" => Ok(view_of(self.s)), "host.notify" => { let text: String = ps(params, "text").unwrap_or_default().chars().take(500).collect(); let error = b(params, "error").unwrap_or(false); @@ -488,6 +496,43 @@ impl Session { let label = leak(&cmd.decl.label); let mutates = cmd.mutates(); let plugin = cmd.plugin.clone(); + #[cfg(not(target_arch = "wasm32"))] + let (plugin_is_session, plugin_name) = (plugin.manifest.session, plugin.manifest.name.clone()); + // A session's program may start with this command: what it changes from its first + // command on is told to it, so the map as it was is noted first. + // What its program sent since the last poll (after answering its last command, or as it + // exited) is answered as between its commands, each request its own undo step, not part + // of this command's. + // The command's own merge key (a held key repeating it) is the command's, not theirs. + #[cfg(not(target_arch = "wasm32"))] + let merge = self.merge.take(); + #[cfg(not(target_arch = "wasm32"))] + for _ in 0..rustgis_plugins::session::DRAIN_BATCHES { + let id = &plugin.manifest.id; + if !(plugin.manifest.session && rustgis_plugins::session::is_running(id)) { + break; + } + let exited = rustgis_plugins::session::has_exited(id); + if self.poll_plugin_session(id, self.leave_downloads) == 0 && !exited { + break; + } + } + #[cfg(not(target_arch = "wasm32"))] + { + self.merge = merge; + } + // A program this command starts (none running, or the last one gone) takes the map as it + // is now, not what an earlier program was last told. + #[cfg(not(target_arch = "wasm32"))] + if plugin.manifest.session && !rustgis_plugins::session::is_running(&plugin.manifest.id) { + self.plugin_watch.seen.retain(|(p, _)| *p != plugin.manifest.id); + } + #[cfg(not(target_arch = "wasm32"))] + if plugin.manifest.session && !self.plugin_watch.seen.iter().any(|(p, _)| *p == plugin.manifest.id) { + let view = view_of(self); + let now = seen(self, &view); + self.plugin_watch.seen.push((plugin.manifest.id.clone(), now)); + } // A plugin with a panel gets its values with every command (`panel` in the request), // after the ones `params` sets (`{"panel": {"id": value}}`, which is taken out of them), // and may answer the panel's new state. A failed command leaves the values as they were. @@ -561,6 +606,12 @@ impl Session { } } self.call_requests = outer; + // A session's program this command found failed, and started again: the command line + // has no poll, so this is where it hears of it. + #[cfg(not(target_arch = "wasm32"))] + if plugin_is_session && let Some(ended) = rustgis_plugins::session::take_ended(&pid) { + self.say_session_ended(&plugin_name, ended); + } if pending.is_empty() && requested.is_empty() { return res; } @@ -615,10 +666,294 @@ impl Session { } } +/// Answers a session plugin's requests made between its commands: each one that can change the +/// project is an undo step of its own; one that only reads runs as it is, leaving no step. +#[cfg(not(target_arch = "wasm32"))] +struct Between<'a> { + s: &'a mut Session, + plugin: Arc, +} + +#[cfg(not(target_arch = "wasm32"))] +impl Host for Between<'_> { + fn call(&mut self, op: &str, params: &Value) -> Result { + let plugin = self.plugin.clone(); + let bridge = |s: &mut Session| { + let fetches = fetch::Calls::for_plugin(&plugin.manifest.id); + let mut bridge = Bridge { s, plugin: plugin.clone(), fetches, changed: false, requested: false }; + let res = bridge.call(op, params); + (res, bridge.changed, bridge.requested) + }; + // A fetch waits on the app's thread: between commands the program fetches for itself. + if op == "host.fetch" { + return Err("'host.fetch' is for commands; between them, a session's program fetches what it needs itself".into()); + } + // An undo step only for what can change the project, as for an agent's command: a read + // leaves the user's undo and redo history alone. + let reads = Bridge::reads_only(op); + let mutates = + plugin.manifest.has(Permission::Edit) && (op == "host.addLayer" || self.s.registry.get(op).is_some_and(|c| c.mutates)); + // With the plugin's authority, exactly as during one of its commands. + self.s + .as_session_plugin(plugin.clone(), |s| { + if reads { + return std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| bridge(s).0)) + .unwrap_or_else(|_| Err(format!("'{op}' failed unexpectedly"))); + } + let (at, start) = (crate::project_log::now_ms(), web_time::Instant::now()); + let res = s.guarded(leak(&plugin.manifest.name), mutates, |s| { + let (res, changed, requested) = bridge(s); + // Only left downloads for the app (they add their layers when they land): no step. + s.request_only = res.is_ok() && requested && !changed; + // The answer goes to the program, which says what it makes of a refusal. + s.quiet_failure = res.is_err(); + res + }); + // No command of its own carries it into the session log: it goes in by itself, as + // a command run would. + if s.log.on() && (op == "host.addLayer" || crate::project_log::is_logged(s, op, params)) { + let (kept, partial) = crate::project_log::kept_and_partial(params); + let project = (crate::project_log::kept_name(&s.title()), s.opened); + let mut e = crate::project_log::entry(project, op, kept, partial, &res, at, start.elapsed()); + e.plugin = Some(plugin.manifest.id.clone()); + s.log.push(e); + } + res + }) + .and_then(|r| r) + } + + fn log(&mut self, text: &str) { + log::info!("{}: {text}", self.plugin.manifest.id); + } +} + +/// What session plugins were last told of the map, to tell them what changes. +#[derive(Debug, Default)] +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +pub(crate) struct Watch { + /// Each running program's, by plugin id. + seen: Vec<(String, Seen)>, + /// The layers' symbology and fields and the web tile layers, fingerprinted, with the + /// project's [`Session::state`] and the time it was taken at. It's taken again when an edit + /// changes the state (a read or a move of the view doesn't), and at least every + /// [`RESTYLE_EVERY`] for what changes it without one: not every frame. + styled: Option<(u64, web_time::Instant, u64)>, + /// The selection, fingerprinted, with the [`Session::revision`] and the time it was taken + /// at: selecting changes no project state but runs a command, which bumps the revision; what + /// changes it without one (a streamed layer reading another window) is caught within + /// [`RESTYLE_EVERY`]. + selected: Option<(u64, web_time::Instant, u64)>, +} + +/// How often a session plugin's layer symbology is looked at again without a command. +#[cfg(not(target_arch = "wasm32"))] +const RESTYLE_EVERY: web_time::Duration = web_time::Duration::from_secs(1); + +/// Fingerprints of what a session plugin hears about. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +pub(crate) struct Seen { + view: u64, + /// Their list and order, which is active, names, visibility, opacity, symbology, fields and + /// features, and the web tile layers. + layers: u64, + selection: u64, + /// Bumped when another project replaces the open one ([`Session::opened`]). + project: u64, +} + +/// What a session plugin would be told of the map now, `view` being [`view_of`] it. +#[cfg(not(target_arch = "wasm32"))] +fn seen(s: &mut Session, view: &Value) -> Seen { + use std::hash::{Hash, Hasher}; + let hash = |f: &dyn Fn(&mut std::collections::hash_map::DefaultHasher)| { + let mut h = std::collections::hash_map::DefaultHasher::new(); + f(&mut h); + h.finish() + }; + let styled = match s.plugin_watch.styled { + Some((state, at, styled)) if state == s.state() && at.elapsed() < RESTYLE_EVERY => styled, + _ => { + let map = s.map(); + let styled = hash(&|h| { + for l in &map.layers { + serde_json::to_string(&(&l.renderer, &l.labels, &l.line_strokes, &l.fill_layers, l.raster.as_ref().map(|r| &r.style))) + .unwrap_or_default() + .hash(h); + for f in &l.dataset.fields { + (&f.name, format!("{:?}", f.ty)).hash(h); + } + } + serde_json::to_string(&map.tiles).unwrap_or_default().hash(h); + }); + s.plugin_watch.styled = Some((s.state(), web_time::Instant::now(), styled)); + styled + } + }; + // A large selection is many rows to walk: again only after a command. Layers with nothing + // selected don't count: adding one changes no selection. + let selection = match s.plugin_watch.selected { + Some((revision, at, selection)) if revision == s.revision && at.elapsed() < RESTYLE_EVERY => selection, + _ => { + let map = s.map(); + let selection = hash(&|h| { + for l in map.layers.iter().filter(|l| !l.selection.is_empty()) { + (l.id, &l.selection).hash(h); + } + }); + s.plugin_watch.selected = Some((s.revision, web_time::Instant::now(), selection)); + selection + } + }; + let map = s.map(); + let view = hash(&|h| view.to_string().hash(h)); + // The rest is a few numbers a layer: looked at every frame. + let layers = hash(&|h| { + (styled, s.active_layer).hash(h); + for l in &map.layers { + (l.id, &l.name, l.visible, l.transparency.to_bits(), l.geometry_rev, l.dataset.features.len()).hash(h); + // Which rows these are: an edit writes new ones and undo puts the old ones back, so it + // changes even where undo brings `geometry_rev` back to the edit's number. + (l.dataset.features.allocation_key().as_ptr() as usize).hash(h); + } + }); + Seen { view, layers, selection, project: s.opened } +} + +/// The view as `host.view` answers it. +fn view_of(s: &Session) -> Value { + let c = &s.map().camera; + let center = c.center_lonlat(); + json!({"center": [center.x, center.y], "scale": c.scale(), "projection": s.map().projection.code(), "bearing": c.bearing, "pitch": c.pitch}) +} + +impl Session { + /// Says that a session plugin's program failed; one that ended by itself (its window closed) + /// goes quietly. + #[cfg(not(target_arch = "wasm32"))] + fn say_session_ended(&mut self, name: &str, ended: rustgis_plugins::session::Ended) { + if ended.unread { + self.notify( + format!("{name}: its program kept sending after it exited (something it left running), and the rest wasn't answered; it was stopped"), + true, + ); + } + if let Some(reason) = ended.reason { + self.notify(format!("{name} was stopped: {reason}"), true); + } else if ended.failed { + let why = ended.last_error.map(|l| format!(": {l}")).unwrap_or_default(); + self.notify(format!("{name} stopped unexpectedly{why}"), true); + } + } + + /// Answers what plugin `id`'s program sent since the last call, between its commands (leaving + /// downloads to the app when `defer`), and says + /// if it failed, or if a command found it failed. Returns how many of its lines it handled. + #[cfg(not(target_arch = "wasm32"))] + fn poll_plugin_session(&mut self, id: &str, defer: bool) -> usize { + use rustgis_plugins::session; + let Some(plugin) = rustgis_plugins::get(id).filter(|p| p.enabled) else { + session::stop(id); + return 0; + }; + // A download it asks for is left to the app when there is one to take it (`defer`: the + // desktop's frame, or a command the app runs leaving its downloads); otherwise (the + // command line, Python) it's fetched now, as a command run there fetches it. + let polled = if defer { + self.with_deferred_downloads(|s| session::poll(id, &mut Between { s, plugin: plugin.clone() })) + } else { + session::poll(id, &mut Between { s: self, plugin: plugin.clone() }) + }; + for ended in polled.ended.into_iter().chain(session::take_ended(id)) { + self.say_session_ended(&plugin.manifest.name, ended); + } + polled.handled + } + + /// Answers what session plugins' programs asked since the last call, and tells each what + /// changed on the map since it was last told. The desktop app calls it every frame. Returns + /// how many of their lines it handled. + #[cfg(not(target_arch = "wasm32"))] + pub fn poll_plugin_sessions(&mut self) -> usize { + use rustgis_plugins::session; + session::polled(); + let mut handled = 0; + for id in session::running() { + handled += self.poll_plugin_session(&id, true); + } + let running = session::running(); + self.plugin_watch.seen.retain(|(id, _)| running.contains(id)); + if running.is_empty() { + self.plugin_watch.styled = None; + self.plugin_watch.selected = None; + return handled; + } + let view = view_of(self); + let now = seen(self, &view); + for id in running { + let Some((_, was)) = self.plugin_watch.seen.iter_mut().find(|(p, _)| *p == id) else { + // Started without a command (none yet): it asks for what it needs first. + self.plugin_watch.seen.push((id, now)); + continue; + }; + let was = std::mem::replace(was, now); + if was.view != now.view { + session::event(&id, &json!({"event": "view", "view": view})); + } + if was.layers != now.layers { + session::event(&id, &json!({"event": "layers"})); + } + if was.project != now.project { + session::event(&id, &json!({"event": "project"})); + } + if was.selection != now.selection { + session::event(&id, &json!({"event": "selection"})); + } + } + handled + } +} + #[cfg(test)] mod tests { use super::*; + /// Why a session plugin's program was stopped is said; one that ended by itself says nothing. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_stopped_session_program_says_why() { + use rustgis_plugins::session::Ended; + let mut s = Session::new(crate::Project::default()); + let mark = s.notice_mark(); + s.say_session_ended("Globe", Ended { failed: false, last_error: None, unread: false, reason: None }); + assert!(s.notices_since(mark).is_empty(), "a clean end says nothing"); + let why = "it sent a line longer than 256 MB"; + s.say_session_ended("Globe", Ended { failed: true, last_error: Some("boom".into()), unread: false, reason: Some(why.into()) }); + let said: Vec<&str> = s.notices_since(mark).iter().map(|n| n.text.as_str()).collect(); + assert_eq!(said, [format!("Globe was stopped: {why}")], "the reason, not the last error line"); + } + + /// A geometry edit and its undo each change what a session plugin is told of the layers, + /// though undo, putting the old rows back, brings `geometry_rev` back to the edit's number. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn an_undone_geometry_edit_is_a_layer_change() { + let mut s = Session::new(crate::Project::default()); + let fc = r#"{"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [1, 2]}}]}"#; + let (ds, _) = rustgis_data::io::read_geojson(fc, "Pts").unwrap(); + super::super::layer::add_dataset(&mut s, ds); + s.run("edit.enable", &json!({"on": true})).unwrap(); + let view = view_of(&s); + let before = seen(&mut s, &view); + s.run("edit.setGeometry", &json!({"layer": "Pts", "row": 0, "geometry": {"type": "Point", "coordinates": [3, 4]}})).unwrap(); + let edited = seen(&mut s, &view); + assert_ne!(before.layers, edited.layers); + assert!(s.undo()); + let undone = seen(&mut s, &view); + assert_ne!(edited.layers, undone.layers); + } + /// A command that only left a download for the front end changed nothing, and a later one /// doesn't hide an earlier edit: the plugin's command keeps its undo step. #[test] diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index e962726acf..b3ce8b0e3c 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -34,7 +34,7 @@ pub fn commands() -> Vec { // The session log (#797): read, written out, emptied, turned on or off. It sits outside // the project, so none of these changes the project or takes an undo step. C::new("project.log", "Session Log", "Project", log).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( - r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis", "project": the project open when it ran, "partial"?: true when params were cut or had secrets taken out}], "count": matching, "total": in the log, "on": bool}"#, + r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis", "project": the project open when it ran, "partial"?: true when params were cut or had secrets taken out, "plugin"?: the session plugin that asked for it between its commands}], "count": matching, "total": in the log, "on": bool}"#, ), C::new("project.exportLog", "Export Session Log", "Project", export_log).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) .view() diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 447882a4e7..e0535f368b 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -809,6 +809,9 @@ pub struct Session { /// them (it keeps nothing between calls). Not saved with the project: a plugin's state /// command reads what it needs from the map when the panel opens. pub plugin_panels: std::collections::BTreeMap, + /// What running session plugins were last told of the map, to send them events. + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + pub(crate) plugin_watch: cmd::plugins::Watch, /// Data downloads of this project under way (Home › Add › Add Data › From URL), by id. downloading: Vec, /// Downloads that landed, waiting for `layer.addDownload`. @@ -1150,6 +1153,7 @@ impl Session { started: Vec::new(), plugin_runs: Vec::new(), plugin_panels: std::collections::BTreeMap::new(), + plugin_watch: Default::default(), downloading: Vec::new(), downloaded: Vec::new(), landed_reads: Vec::new(), diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 20ef59a352..a17a9db50d 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -277,6 +277,20 @@ impl Session { Err(p) => std::panic::resume_unwind(p), } } + /// Runs what a session plugin's program asks between its commands with the plugin's own + /// authority, as its commands run: the exact load, still current, narrowing the scope the app + /// is in (none at the frame's root, the trusted native convention). + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn as_session_plugin( + &mut self, + plugin: Arc, + f: impl FnOnce(&mut Session) -> T, + ) -> Result { + if !rustgis_plugins::is_current(&plugin) { + return Err("the plugin was disabled, updated or removed".into()); + } + self.narrowed(Narrow::Plugin(super::plan::PluginPin(plugin)), f) + } /// Intersects nested plugin/item authority with its initiating scope. pub(crate) fn narrowed(&mut self, to: Narrow, f: impl FnOnce(&mut Session) -> T) -> Result { let mut scope = self.scope(); diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index fc91d8f0bf..3d6f75f7fc 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -519,6 +519,7 @@ pub(crate) fn entry( project: project.0, opened: project.1, partial, + plugin: None, } } @@ -573,6 +574,10 @@ pub struct LogEntry { /// wouldn't run as they ran. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub partial: bool, + /// The plugin whose session program asked for it between its commands: it counts with the + /// plugins' commands toward [`LOG_PLUGIN_LIMIT`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub plugin: Option, } impl LogEntry { @@ -623,7 +628,7 @@ impl SessionLog { if self.off && !matches!(e.command.as_str(), "project.logCommands" | "project.clearLog") { return; } - let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX); + let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX) || e.plugin.is_some(); if plugin(&e) && self.plugin_entries >= LOG_PLUGIN_LIMIT && let Some(oldest) = self.entries.iter().position(plugin) @@ -649,7 +654,8 @@ impl SessionLog { /// into the last undo step ([`crate::Session::merge_undo`]). The entry keeps when the first /// started, takes the last one's parameters, and adds up how long they took. pub fn join_last(&mut self, mut e: LogEntry) { - if let Some(last) = self.entries.back_mut().filter(|l| l.ok && l.command == e.command) { + // A session plugin's entry between the steps isn't the user's step to join. + if let Some(last) = self.entries.back_mut().filter(|l| l.ok && l.command == e.command && l.plugin == e.plugin) { e.at = last.at; e.millis = last.millis.saturating_add(e.millis); *last = e; @@ -713,7 +719,7 @@ impl SessionLog { /// line). Entry by entry, so a long log isn't held as text first. pub fn write(&self, w: &mut dyn std::io::Write, csv: bool) -> std::io::Result<()> { if csv { - w.write_all(b"time_utc,at,command,ok,millis,error,params,project\n")?; + w.write_all(b"time_utc,at,command,ok,millis,error,params,plugin,project\n")?; } self.entries.iter().try_for_each(|e| write_entry(w, e, csv)) } @@ -740,7 +746,7 @@ fn write_entry(w: &mut dyn std::io::Write, e: &LogEntry, csv: bool) -> std::io:: if csv { writeln!( w, - "{},{},{},{},{},{},{},{}", + "{},{},{},{},{},{},{},{},{}", e.time_utc(), e.at, csv_cell(&e.command), @@ -748,6 +754,7 @@ fn write_entry(w: &mut dyn std::io::Write, e: &LogEntry, csv: bool) -> std::io:: e.millis, csv_cell(e.error.as_deref().unwrap_or("")), csv_cell(&e.params.to_string()), + csv_cell(e.plugin.as_deref().unwrap_or("")), csv_cell(&e.project) ) } else { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f0c80f39ae..f9f6de6985 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -10499,6 +10499,266 @@ fn process_plugins_reach_the_project_like_modules() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn session_plugins_ask_between_commands_and_hear_the_map() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess", &["edit"], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let poll_until = |s: &mut Session, done: &dyn Fn(&Session) -> bool| { + let t0 = std::time::Instant::now(); + while !done(s) && t0.elapsed() < std::time::Duration::from_secs(20) { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + // Long enough for a request it sent to be answered. + let settle = |s: &mut Session| { + for _ in 0..25 { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + // Its first command starts it, and what that command changes is told to it too. + let point = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [1, 2]}}]}); + s.run("plugin:eng-sess/f", &json!({"op": "host.addLayer", "params": {"name": "First", "geojson": point}})).unwrap(); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + + // Reads between commands, a host request or a command that changes nothing, leave the project + // and the undo history as they were. + s.run("plugin:eng-sess/k", &json!({"op": "host.layers", "params": {}})).unwrap(); + let (revision, steps) = (s.revision, s.undo_depth()); + settle(&mut s); + assert_eq!((s.revision, s.undo_depth()), (revision, steps)); + s.run("plugin:eng-sess/k", &json!({"op": "map.extent", "params": {}})).unwrap(); + let (steps, top) = (s.undo_depth(), s.undo_label()); + settle(&mut s); + assert_eq!((s.undo_depth(), s.undo_label()), (steps, top)); + // An edit between commands goes through the same checks, as an undo step of its own. + let fc = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [3, 4]}}]}); + s.log.set_on(true); + s.run("plugin:eng-sess/k", &json!({"op": "host.addLayer", "params": {"name": "Later", "geojson": fc}})).unwrap(); + poll_until(&mut s, &|s| s.map().layer_by_name("Later").is_some()); + assert_eq!(s.undo_label(), Some("Test eng-sess"), "its own step"); + assert!( + s.log.entries().any(|e| e.command == "host.addLayer" && e.plugin.as_deref() == Some("eng-sess")), + "in the session log, as the plugin's" + ); + assert!(s.undo()); + assert!(s.map().layer_by_name("Later").is_none()); + assert!(s.redo()); + assert!(s.map().layer_by_name("Later").is_some()); + s.run("plugin:eng-sess/k", &json!({"op": "layer.addData", "params": {"path": "/etc/passwd"}})).unwrap(); + settle(&mut s); + // A fetch would wait on the app's thread: only during commands. + s.run("plugin:eng-sess/k", &json!({"op": "host.fetch", "params": {"url": "https://example.org/a.json"}})).unwrap(); + settle(&mut s); + // Told what changed: the layers (the one it added), the view, the selection. + s.map_mut().camera.pan_px(40.0, 0.0); + s.run("select.rows", &json!({"layer": "Cities", "rows": [0]})).unwrap(); + s.poll_plugin_sessions(); + let heard = s.run("plugin:eng-sess/v", &Value::Null).unwrap(); + // (The undo and redo above count as layer changes too.) + let events: Vec<&str> = heard["events"].as_array().unwrap().iter().filter_map(Value::as_str).collect(); + assert!(events.first() == Some(&"layers") && events.ends_with(&["view", "selection"]), "{heard}"); + let refused = heard["answers"].as_array().unwrap().iter().filter_map(|a| a["error"].as_str()).collect::>(); + assert!(refused.iter().any(|e| e.contains("'files' permission")), "{heard}"); + assert!(refused.iter().any(|e| e.contains("'host.fetch' is for commands")), "{heard}"); + // Restyled or made see-through counts as a layer change; another project is told too. + if let Some(l) = s.map_mut().layers.first_mut() { + l.transparency = 0.5; + } + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + // Another layer made active is a layer change too: `host.layers` says which one is. + let other = s.map().layers.iter().map(|l| l.id).find(|id| Some(*id) != s.active_layer).unwrap(); + s.active_layer = Some(other); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"], json!(["layers"])); + s.run("project.new", &json!({})).unwrap(); + s.poll_plugin_sessions(); + let events = s.run("plugin:eng-sess/v", &Value::Null).unwrap()["events"].clone(); + assert!(events.as_array().unwrap().contains(&json!("project")), "{events}"); + // Turned off, it stops. + s.run("plugin.enable", &json!({"id": "eng-sess", "enabled": false})).unwrap(); + assert!(!rustgis_plugins::session::is_running("eng-sess")); + rustgis_plugins::unload("eng-sess"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Session plugins' programs are the process's, and a session's poll answers every running one: +/// the tests that run them take turns, so none answers another's in its session. +fn session_plugins_turn() -> std::sync::MutexGuard<'static, ()> { + static TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + TURN.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Waits (a few seconds at most) for a session plugin's program to exit, without answering it. +fn wait_for_exit(plugin: &str) { + let t0 = std::time::Instant::now(); + while !rustgis_plugins::session::has_exited(plugin) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(10)); + } +} + +/// What a program sent as it exited, after many lines, is answered before the next command, as +/// between commands: its edit is an undo step of its own, not part of that command's. +#[test] +fn a_session_programs_last_request_is_its_own_step_before_the_next_command() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-late-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-late", &["edit"], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let fc = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": [3, 4]}}]}); + s.run("plugin:eng-sess-late/y", &json!({"op": "host.addLayer", "params": {"name": "Late", "geojson": fc}})).unwrap(); + wait_for_exit("eng-sess-late"); + // No poll in between: the next command finds it exited. + assert_eq!(s.run("plugin:eng-sess-late/h", &Value::Null).unwrap(), json!({"hello": "world"})); + assert!(s.map().layer_by_name("Late").is_some()); + assert_eq!(s.undo_label(), Some("Hello")); + assert!(s.undo()); + assert!(s.map().layer_by_name("Late").is_some(), "not part of the next command's step"); + assert_eq!(s.undo_label(), Some("Test eng-sess-late")); + assert!(s.notices_since(0).iter().all(|n| !n.text.contains("Test eng-sess-late")), "a clean exit says nothing"); + rustgis_plugins::unload("eng-sess-late"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// On the command line (no app to leave downloads to), a download a session program asked for +/// between commands is fetched when the next command answers it: never left `pending`. +#[test] +fn session_plugin_downloads_between_commands_are_fetched_without_an_app() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-cli-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-cli", &["edit", "network"], &run)) + .unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-cli/k", &json!({"op": "layer.addUrl", "params": {"url": "https://example.invalid/points.geojson"}})).unwrap(); + // Each command first answers what the program sent since: `v` reports once it has. + let t0 = std::time::Instant::now(); + let answer = loop { + let heard = s.run("plugin:eng-sess-cli/v", &Value::Null).unwrap(); + if let Some(a) = heard["answers"].as_array().and_then(|a| a.first()).cloned() { + break a; + } + assert!(t0.elapsed() < std::time::Duration::from_secs(20), "no answer"); + std::thread::sleep(std::time::Duration::from_millis(50)); + }; + assert!(answer["error"].is_string() && answer["ok"]["pending"].is_null(), "{answer}"); + assert!(s.take_url_requests().is_empty(), "nothing left for an app that isn't there"); + rustgis_plugins::unload("eng-sess-cli"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that exits while something it left running keeps writing into its output is ended +/// by the next command, and the notice says the rest wasn't answered; one that ends cleanly +/// gets no notice (`a_session_programs_last_request_is_its_own_step_before_the_next_command`). +#[cfg(unix)] +#[test] +fn a_session_program_whose_output_kept_coming_is_said() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-flow-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-flow", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-flow/r", &Value::Null).unwrap(); + wait_for_exit("eng-sess-flow"); + let mark = s.notice_mark(); + assert_eq!(s.run("plugin:eng-sess-flow/h", &Value::Null).unwrap(), json!({"hello": "world"})); + let said: Vec = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("kept sending after it exited")), "{said:?}"); + rustgis_plugins::session::stop("eng-sess-flow"); + rustgis_plugins::unload("eng-sess-flow"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program started again (the last one exited, no poll in between) takes the map as it is +/// then: it isn't told of what changed before it started. +#[test] +fn a_restarted_session_program_starts_from_the_map_as_it_is() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-restart-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-restart", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + s.run("plugin:eng-sess-restart/h", &Value::Null).unwrap(); + s.poll_plugin_sessions(); + // The view moves, and the program exits, with no poll between. + s.map_mut().camera.pan_px(40.0, 0.0); + assert_eq!(s.run("plugin:eng-sess-restart/z", &Value::Null).unwrap(), "bye"); + wait_for_exit("eng-sess-restart"); + // Its next command starts another, which hears nothing of the earlier move. + s.run("plugin:eng-sess-restart/h", &Value::Null).unwrap(); + s.poll_plugin_sessions(); + assert_eq!(s.run("plugin:eng-sess-restart/v", &Value::Null).unwrap()["events"], json!([])); + rustgis_plugins::session::stop("eng-sess-restart"); + rustgis_plugins::unload("eng-sess-restart"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that failed after answering is said to have when the next command finds it, as +/// the frame's poll would: the command line has no poll. +#[test] +fn a_session_program_found_failed_by_a_command_is_said() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-bad", &[], &run)).unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + assert_eq!(s.run("plugin:eng-sess-bad/b", &Value::Null).unwrap(), "bye"); + wait_for_exit("eng-sess-bad"); + let mark = s.notice_mark(); + assert_eq!(s.run("plugin:eng-sess-bad/h", &Value::Null).unwrap(), json!({"hello": "world"})); + let said: Vec = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t.contains("Test eng-sess-bad stopped unexpectedly: boom")), "{said:?}"); + rustgis_plugins::unload("eng-sess-bad"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A download a session plugin asks for between its commands is left to the app, as during its +/// commands: the frame never waits for it. +#[test] +fn session_plugin_downloads_between_commands_are_left_to_the_app() { + let _turn = session_plugins_turn(); + let mut s = world(); + let dir = std::env::temp_dir().join(format!("rustgis-test-eng-session-url-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = rustgis_plugins::testing::child_run("tests::plugin_process_child"); + std::fs::write(dir.join("plugin.json"), rustgis_plugins::testing::session_manifest("eng-sess-url", &["edit", "network"], &run)) + .unwrap(); + rustgis_plugins::load(rustgis_plugins::Package::from_dir(&dir).unwrap(), "test").unwrap(); + let url = "https://example.invalid/points.geojson"; + s.run("plugin:eng-sess-url/k", &json!({"op": "layer.addUrl", "params": {"url": url}})).unwrap(); + let t0 = std::time::Instant::now(); + while s.url_requests.is_empty() && t0.elapsed() < std::time::Duration::from_secs(20) { + s.poll_plugin_sessions(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let asked = s.take_url_requests(); + let urls: Vec<&Value> = asked.iter().map(|q| &q.params()["url"]).collect(); + assert!(urls.contains(&&json!(url)), "{urls:?}"); + let heard = s.run("plugin:eng-sess-url/v", &Value::Null).unwrap(); + assert_eq!(heard["answers"][0]["ok"]["pending"], true, "{heard}"); + rustgis_plugins::unload("eng-sess-url"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn plugins_install_and_uninstall_from_files() { isolate_plugins(); @@ -29440,7 +29700,7 @@ fn the_session_log_records_each_command_run_in_the_session() { if isolate_process_test("tests::the_session_log_records_each_command_run_in_the_session") { return; } - use crate::project_log::{LOG_LIMIT, LOG_PARAMS_BYTES, LogEntry}; + use crate::project_log::{LOG_LIMIT, LOG_PARAMS_BYTES, LOG_PLUGIN_LIMIT, LogEntry}; let mut s = world(); // The sample project's making is its first entry: counted from after it. assert_eq!(s.log.clear(), 1); @@ -29552,6 +29812,17 @@ fn the_session_log_records_each_command_run_in_the_session() { ..Default::default() }; s.log.push(formula); + // A session plugin's request between its commands keeps its attribution in both. + s.log.push(LogEntry { + at: 3, + command: "host.addLayer".into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + plugin: Some("globe".into()), + ..Default::default() + }); // Written out as CSV and JSON Lines. let dir = std::env::temp_dir().join(format!("rustgis-session-log-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -29559,14 +29830,22 @@ fn the_session_log_records_each_command_run_in_the_session() { s.run("project.exportLog", &json!({"path": csv.to_string_lossy()})).unwrap(); let csv_text = std::fs::read_to_string(&csv).unwrap(); assert!( - csv_text.starts_with("time_utc,at,command,ok,millis,error,params,project\n") && csv_text.contains("\"edit.undo\""), + csv_text.starts_with("time_utc,at,command,ok,millis,error,params,plugin,project\n") && csv_text.contains("\"edit.undo\""), "{csv_text}" ); assert!(csv_text.contains("\"'=HYPERLINK(1)\"") && !csv_text.contains(",\"=HYPERLINK"), "{csv_text}"); + // The plugin is the cell before the project's name, the last. + let plugin_cell = |l: &str| l.rsplit(',').nth(1).map(str::to_string); + assert!(csv_text.lines().any(|l| l.contains("\"host.addLayer\"") && plugin_cell(l).as_deref() == Some("\"globe\"")), "{csv_text}"); + assert!( + csv_text.lines().any(|l| l.contains("\"edit.undo\"") && plugin_cell(l).as_deref() == Some("\"\"")), + "the user's own carry none: {csv_text}" + ); let jsonl = dir.join("log.jsonl"); s.run("project.exportLog", &json!({"path": jsonl.to_string_lossy()})).unwrap(); let lines: Vec = std::fs::read_to_string(&jsonl).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); assert_eq!(lines.len() + 1, s.log.len(), "every entry but the export's own"); + assert!(lines.iter().any(|e| e.plugin.as_deref() == Some("globe"))); let _ = std::fs::remove_dir_all(&dir); // Cleared, it says so; off, nothing goes in; on again, it does. Neither changes the project, // which holds no log, nor takes an undo step. @@ -29616,6 +29895,24 @@ fn the_session_log_records_each_command_run_in_the_session() { } assert_eq!(s.log.len(), LOG_LIMIT); assert_eq!(s.log.entries().next().unwrap().at, 5); + // A session plugin's requests between its commands count with the plugins': past their limit + // they make room among themselves, not by pushing out the user's entries. + let user = s.log.entries().next().unwrap().at; + for i in 0..LOG_PLUGIN_LIMIT + 5 { + let e = LogEntry { + at: 1_000_000 + i as u64, + command: "host.addLayer".into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + plugin: Some("globe".into()), + ..Default::default() + }; + s.log.push(e); + } + assert_eq!(s.log.entries().filter(|e| e.plugin.is_some()).count(), LOG_PLUGIN_LIMIT); + assert_eq!(s.log.entries().next().unwrap().at, user + LOG_PLUGIN_LIMIT as u64); // What was dropped is counted, and an export says so; a clear starts the count again. assert!(s.log.dropped() >= 5, "{}", s.log.dropped()); let out = std::env::temp_dir().join(format!("rustgis-log-dropped-{}.csv", std::process::id())); diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs index 24970a5389..e5421304be 100644 --- a/crates/plugins/src/lib.rs +++ b/crates/plugins/src/lib.rs @@ -9,7 +9,8 @@ //! //! A process plugin (`"runtime": "process"`, see [`process`]) is instead a program on the //! user's computer, not sandboxed, for the desktop and the command line: it speaks the same -//! requests over its standard input and output, and the user is told it isn't sandboxed. +//! requests over its standard input and output, and the user is told it isn't sandboxed. With +//! `"session": true` its program is kept running between commands (see [`session`]). //! //! Loaded plugins live for the whole process (like the geoprocessing tools they add), in a //! list shared by every session. See `docs/plugins.md` for the format and the ABI. @@ -25,6 +26,8 @@ pub mod pyenv; pub mod registry; pub mod runtime; #[cfg(not(target_arch = "wasm32"))] +pub mod session; +#[cfg(not(target_arch = "wasm32"))] pub mod store; #[cfg(any(test, feature = "testing"))] pub mod testing; @@ -111,6 +114,7 @@ impl Plugin { "panel": m.panel.as_ref().map(|p| p.title.clone()), "skills": m.skills.iter().map(|s| format!("{}:{s}", m.id)).collect::>(), "runtime": match m.runtime { Runtime::Wasm => "wasm", Runtime::Process => "process", Runtime::Skills => "skills" }, + "session": m.session, "enabled": self.enabled, "source": self.source, "sha256": self.sha256, }) } @@ -232,6 +236,7 @@ fn process_code(pkg: &Package) -> Result { let argv = pkg.manifest.run.as_ref().and_then(|r| r.for_this_platform()).ok_or_else(|| format!("{name} doesn't run on this platform"))?; let mut spec = process::Spec::new(argv, dir)?; + spec.session = pkg.manifest.session; // With a Python environment, `python` is that environment's interpreter. if let Some(check) = pyenv::EnvCheck::of(&pkg.manifest, &spec.dir).map_err(|e| format!("{name}: {e}"))? { if matches!(spec.program.as_str(), "python" | "python3" | "python.exe") { @@ -251,6 +256,8 @@ fn process_code(pkg: &Package) -> Result { /// was enabled). pub fn register(mut plugin: Plugin) -> Arc { plugin.enabled = get(&plugin.manifest.id).is_none_or(|p| p.enabled); + // The program running is the version being replaced. + stop_session(&plugin.manifest.id); let plugin = Arc::new(plugin); if let Ok(mut v) = PLUGINS.write() { v.retain(|p| p.manifest.id != plugin.manifest.id); @@ -263,6 +270,7 @@ pub fn register(mut plugin: Plugin) -> Arc { /// Removes a loaded plugin; returns whether there was one. pub fn unload(id: &str) -> bool { + stop_session(id); let removed = PLUGINS.write().map(|mut v| { let n = v.len(); v.retain(|p| p.manifest.id != id); @@ -293,10 +301,19 @@ pub fn set_enabled(id: &str, enabled: bool) -> Result<(), String> { changed(&v); } drop(v); + if !enabled { + stop_session(id); + } tools::reload(); Ok(()) } +/// Stops a session plugin's program, if it's running. +fn stop_session(_id: &str) { + #[cfg(not(target_arch = "wasm32"))] + session::stop(_id); +} + /// Replaces the blocklist; loaded plugins it names are unloaded. Returns their names. pub fn set_blocklist(list: Vec) -> Vec { if let Ok(mut b) = BLOCKED.write() { @@ -398,6 +415,10 @@ pub(crate) fn call(plugin: &Plugin, host: &mut dyn Host, request: &Value, tool: match &plugin.code { Code::Wasm(module) => runtime::call(module, host, request, if tool { runtime::TOOL_FUEL } else { runtime::COMMAND_FUEL }), #[cfg(not(target_arch = "wasm32"))] + Code::Process(spec) if spec.session && !tool => { + session::command(&plugin.manifest.id, spec, host, request, process::COMMAND_TIMEOUT) + } + #[cfg(not(target_arch = "wasm32"))] Code::Process(spec) => process::call(spec, host, request, (!tool).then_some(process::COMMAND_TIMEOUT)), Code::None => Err(format!("{} has only skills, nothing to run", plugin.manifest.name)), } diff --git a/crates/plugins/src/manifest.rs b/crates/plugins/src/manifest.rs index 7b32bf656d..207283e247 100644 --- a/crates/plugins/src/manifest.rs +++ b/crates/plugins/src/manifest.rs @@ -318,6 +318,10 @@ pub struct Manifest { /// For a process plugin in Python: the environment RustGIS makes for it. #[serde(default)] pub python: Option, + /// For a process plugin: keep the program running between its commands (a session), so it + /// can ask RustGIS for things at any time and hear when the map changes. See the `session` module. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub session: bool, /// The plugin ABI the module was built for. #[serde(default = "default_abi")] pub abi: u32, @@ -447,8 +451,8 @@ impl Manifest { if !valid_path(&self.entry) || !self.entry.ends_with(".wasm") { return Err("plugin.json: entry must be a relative path to a .wasm file inside the package".into()); } - if self.run.is_some() || self.python.is_some() { - return Err("plugin.json: 'run' and 'python' are for process plugins (\"runtime\": \"process\")".into()); + if self.run.is_some() || self.python.is_some() || self.session { + return Err("plugin.json: 'run', 'python' and 'session' are for process plugins (\"runtime\": \"process\")".into()); } } Runtime::Process => { @@ -487,8 +491,8 @@ impl Manifest { } } Runtime::Skills => { - if self.run.is_some() || self.python.is_some() { - return Err("plugin.json: 'run' and 'python' are for process plugins (\"runtime\": \"process\")".into()); + if self.run.is_some() || self.python.is_some() || self.session { + return Err("plugin.json: 'run', 'python' and 'session' are for process plugins (\"runtime\": \"process\")".into()); } if !self.commands.is_empty() || !self.tools.is_empty() diff --git a/crates/plugins/src/process.rs b/crates/plugins/src/process.rs index 4d6e8f0677..662d95297a 100644 --- a/crates/plugins/src/process.rs +++ b/crates/plugins/src/process.rs @@ -18,6 +18,8 @@ use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel, sync_channel}; use std::time::{Duration, Instant}; @@ -28,11 +30,11 @@ use crate::runtime::Host; /// How long a command may run; tools run until they finish (a large dataset can take a while). pub const COMMAND_TIMEOUT: Duration = Duration::from_secs(120); /// How long a program has to exit after answering, before it's stopped. -const EXIT_GRACE: Duration = Duration::from_secs(3); +pub(crate) const EXIT_GRACE: Duration = Duration::from_secs(3); /// The longest line read from a program (a reply with large layers). -const MAX_LINE: usize = 256 * 1024 * 1024; +pub(crate) const MAX_LINE: usize = 256 * 1024 * 1024; /// The most host requests one call may make. -const MAX_HOST_CALLS: u32 = 100_000; +pub(crate) const MAX_HOST_CALLS: u32 = 100_000; /// The most lines of its own output a program may log in one call. const MAX_LOGGED: usize = 200; /// The longest line of a program's error output kept (a longer one is split). @@ -54,6 +56,8 @@ pub struct Spec { pub(crate) env: Vec<(String, String)>, /// For a plugin with a Python environment (`"python"`): the check that it's ready. pub env_check: Option, + /// Whether its commands go to one program kept running (`"session"`, see [`crate::session`]). + pub session: bool, } impl Spec { @@ -63,7 +67,7 @@ impl Spec { // Absolute: the program starts in the folder, so a relative folder would be followed // twice (`./plugin/plugin/bin/tool`). let dir = std::path::absolute(dir).map_err(|e| format!("{}: {e}", dir.display()))?; - Ok(Spec { program: program.clone(), args: args.to_vec(), dir, env: Vec::new(), env_check: None }) + Ok(Spec { program: program.clone(), args: args.to_vec(), dir, env: Vec::new(), env_check: None, session: false }) } /// The program to start: a path relative to the plugin's folder, or a name for `PATH`. @@ -93,7 +97,7 @@ impl Spec { } /// What the program sent: a line of its output, or the end of it. -enum Line { +pub(crate) enum Line { Text(Vec), End, TooLong, @@ -102,7 +106,7 @@ enum Line { /// Reads the program's output lines on a thread, so waiting for one can time out. A few lines /// at most wait to be handled: a program printing faster than RustGIS reads is made to wait /// rather than filling memory. -fn lines(out: impl Read + Send + 'static) -> Receiver { +fn lines(out: impl Read + Send + 'static, wake: Option) -> Receiver { let (tx, rx) = sync_channel(2); std::thread::spawn(move || { let mut r = BufReader::new(out); @@ -111,16 +115,20 @@ fn lines(out: impl Read + Send + 'static) -> Receiver { match (&mut r).take(MAX_LINE as u64 + 1).read_until(b'\n', &mut buf) { Ok(0) | Err(_) => { let _ = tx.send(Line::End); + wake.as_ref().inspect(|w| w()); return; } Ok(_) if buf.len() > MAX_LINE => { let _ = tx.send(Line::TooLong); + wake.as_ref().inspect(|w| w()); return; } Ok(_) => { if tx.send(Line::Text(buf)).is_err() { return; } + // A session's line can arrive while the app is idle: it is asked to look. + wake.as_ref().inspect(|w| w()); } } } @@ -129,11 +137,24 @@ fn lines(out: impl Read + Send + 'static) -> Receiver { } /// The last lines of the program's error output, for the log and for an error message. -type Kept = std::sync::Arc>>; +pub(crate) type Kept = std::sync::Arc>; -/// Keeps the program's error output as it comes. Read, never joined: a process the program +/// The last lines of a program's error output, and how many earlier ones weren't kept. +#[derive(Debug, Default)] +pub(crate) struct ErrorLines { + pub lines: std::collections::VecDeque, + pub dropped: usize, + /// Whether the error output has ended (everything holding it open has closed it). + pub done: bool, +} + +/// The most lines of a program's error output kept until they're read. +const KEPT_ERROR_LINES: usize = 20; + +/// Keeps the program's error output as it comes; `wake` is called as each line arrives (a +/// session's, so it is logged while the app is idle). Read, never joined: a process the program /// started can hold the pipe open after the program itself has exited. -fn errors(err: impl Read + Send + 'static) -> Kept { +pub(crate) fn errors(err: impl Read + Send + 'static, wake: Option) -> Kept { let kept = Kept::default(); let k = kept.clone(); std::thread::spawn(move || { @@ -142,22 +163,30 @@ fn errors(err: impl Read + Send + 'static) -> Kept { // Bounded: endless output without a line end is read in pieces. let mut buf = Vec::new(); match (&mut r).take(MAX_ERROR_LINE as u64).read_until(b'\n', &mut buf) { - Ok(0) | Err(_) => break, + Ok(0) | Err(_) => { + if let Ok(mut k) = k.lock() { + k.done = true; + } + break; + } Ok(_) => {} } let line = String::from_utf8_lossy(&buf).trim_end().to_string(); let Ok(mut k) = k.lock() else { break }; - k.push_back(line.chars().take(1_000).collect::()); - if k.len() > 20 { - k.pop_front(); + k.lines.push_back(line.chars().take(1_000).collect::()); + if k.lines.len() > KEPT_ERROR_LINES { + k.lines.pop_front(); + k.dropped += 1; } + drop(k); + wake.as_ref().inspect(|w| w()); } }); kept } /// Stops the program, and what it started, if it hasn't exited on its own within `grace`. -fn finish(child: &mut Child, grace: Duration) { +pub(crate) fn finish(child: &mut Child, grace: Duration) { let start = Instant::now(); loop { match child.try_wait() { @@ -174,6 +203,70 @@ fn finish(child: &mut Child, grace: Duration) { } } +/// Whether the program has exited. On Unix it isn't reaped yet: until it is, its id can't be +/// reused, so its process group can still be stopped safely ([`end`]). +pub(crate) fn exited(child: &mut Child) -> bool { + #[cfg(unix)] + { + use rustix::io::Errno; + use rustix::process::{Pid, WaitId, WaitIdOptions, waitid}; + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(Pid::from_child(child)), options) { + Ok(found) => return found.is_some(), + Err(Errno::INTR) => continue, + // Reaped already (by its own status check, or by something else when the app + // reaps every child or ignores SIGCHLD): gone either way. + // (Not waited on again: its id may be another child's by now.) + Err(Errno::CHILD) => return true, + // Not known to have exited: asking `try_wait` would reap it, so it counts as running. + Err(_) => return false, + } + } + } + #[cfg(not(unix))] + matches!(child.try_wait(), Ok(Some(_))) +} + +/// Ends a program kept running: gives it `grace` to exit, stops what it left running, then stops +/// it if it's still running and reaps it. Returns its exit status when it exited by itself (and +/// RustGIS reaped it), and whether it had to be stopped. +/// +/// On Unix what it left running is its process group, stopped before the program is reaped so +/// the group's id can't belong to anything else yet. On Windows a process tree can't be found +/// from a program that has exited, so only a running one's is stopped. +pub(crate) fn end(child: &mut Child, grace: Duration) -> (Option, bool) { + let start = Instant::now(); + while !exited(child) && start.elapsed() < grace { + std::thread::sleep(Duration::from_millis(10)); + } + let by_itself = exited(child); + // Reaped already (by something else: an embedding host reaping every child, SIGCHLD + // ignored): its id may be another process's by now, so nothing is signalled by it. + #[cfg(unix)] + let reaped = { + use rustix::process::{Pid, WaitId, WaitIdOptions, waitid}; + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + matches!(waitid(WaitId::Pid(Pid::from_child(child)), options), Err(rustix::io::Errno::CHILD)) + }; + #[cfg(not(unix))] + let reaped = false; + #[cfg(unix)] + if !reaped { + let _ = rustix::process::kill_process_group(rustix::process::Pid::from_child(child), rustix::process::Signal::KILL); + } + // Reaped elsewhere, it's gone: waiting on its id could wait on another child that has it now. + if reaped { + return (None, false); + } + if !by_itself { + kill_tree(child); + let _ = child.kill(); + } + let status = child.wait().ok(); + (status.filter(|_| by_itself), !by_itself) +} + /// Stops the processes a program started: its process group (Unix), its process tree (Windows). fn kill_tree(child: &Child) { #[cfg(unix)] @@ -194,28 +287,76 @@ fn kill_tree(child: &Child) { } } -/// Writes lines to the program's input on a thread, so a program that doesn't read can't block -/// RustGIS (the timeout still applies). Dropping the sender closes the input. -fn writer(mut stdin: impl Write + Send + 'static) -> Sender> { +/// The program's input: lines written on a thread, so a program that doesn't read can't block +/// RustGIS (the timeout still applies). Dropping it closes the input. +pub(crate) struct Writer { + tx: Sender>, + /// Lines sent and not yet written: how far behind the program is in reading. + backlog: Arc, + /// Their bytes. + bytes: Arc, +} + +/// How many lines may wait for the program to read them before a session's events are held +/// (`session::event`); when it catches up below this, the app is asked to send them. +pub(crate) const HOLD_EVENTS_AT: usize = 16; + +impl Writer { + /// Queues `v` as one line. + pub(crate) fn send(&self, v: &Value) -> Result<(), String> { + self.send_line(v.to_string().into_bytes()) + } + + /// Queues `line`, JSON already written out, as one line. + pub(crate) fn send_line(&self, line: Vec) -> Result<(), String> { + self.backlog.fetch_add(1, Ordering::Relaxed); + self.bytes.fetch_add(line.len(), Ordering::Relaxed); + self.tx.send(line).map_err(|_| "it stopped reading".to_string()) + } + + /// Lines waiting for the program to read them. + pub(crate) fn backlog(&self) -> usize { + self.backlog.load(Ordering::Relaxed) + } + + /// Bytes waiting for the program to read them. + pub(crate) fn backlog_bytes(&self) -> usize { + self.bytes.load(Ordering::Relaxed) + } +} + +/// `wake` is called when the program, behind in reading, catches up below [`HOLD_EVENTS_AT`]. +fn writer(mut stdin: impl Write + Send + 'static, wake: Option) -> Writer { let (tx, rx) = channel::>(); + let backlog = Arc::new(AtomicUsize::new(0)); + let bytes = Arc::new(AtomicUsize::new(0)); + let (left, left_bytes) = (backlog.clone(), bytes.clone()); std::thread::spawn(move || { for mut line in rx { + let n = line.len(); line.push(b'\n'); if stdin.write_all(&line).and_then(|_| stdin.flush()).is_err() { return; } + left_bytes.fetch_sub(n, Ordering::Relaxed); + if left.fetch_sub(1, Ordering::Relaxed) == HOLD_EVENTS_AT { + wake.as_ref().inspect(|w| w()); + } } }); - tx + Writer { tx, backlog, bytes } } -fn send(stdin: &Sender>, v: &Value) -> Result<(), String> { - stdin.send(v.to_string().into_bytes()).map_err(|_| "it stopped reading".to_string()) +/// A program started, with its input, its output lines and its error output. +pub(crate) struct Started { + pub child: Child, + pub stdin: Writer, + pub rx: Receiver, + pub errs: Kept, } -/// Runs one call: starts the program, sends `request`, answers its host requests through -/// `host`, and returns its `ok` value or its error. `timeout` (none for tools) stops it. -pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { +/// Starts the program, its pipes connected; `wake` is called as each line of its output arrives. +pub(crate) fn spawn(spec: &Spec, wake: Option) -> Result { if let Some(check) = spec.env_check.as_ref().filter(|c| !c.ready()) { return Err(format!( "its Python environment isn't ready: prepare it in Plugins › Plugin Manager › Installed, or with rustgis-cli plugins prepare {}", @@ -233,9 +374,13 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { + let Started { mut child, stdin, rx, errs } = spawn(spec, None)?; let res = converse(&stdin, &rx, host, request, timeout); // Its input closed, a program that answered exits; one that didn't is stopped. Its output // isn't read any more: one still printing gets a broken pipe rather than waiting on a full @@ -247,7 +392,7 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option = errs.lock().map(|k| k.iter().cloned().collect()).unwrap_or_default(); + let stderr: Vec = errs.lock().map(|k| k.lines.iter().cloned().collect()).unwrap_or_default(); for line in stderr.iter().filter(|l| !l.trim().is_empty()) { log::info!("plugin {}: {line}", spec.program); } @@ -257,14 +402,8 @@ pub fn call(spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Option>, - rx: &Receiver, - host: &mut dyn Host, - request: &Value, - timeout: Option, -) -> Result { - send(stdin, request)?; +fn converse(stdin: &Writer, rx: &Receiver, host: &mut dyn Host, request: &Value, timeout: Option) -> Result { + stdin.send(request)?; let deadline = timeout.map(|t| Instant::now() + t); let (mut calls, mut logged) = (0u32, 0usize); loop { @@ -274,6 +413,10 @@ fn converse( if host.cancelled() { return Err(CANCELLED.into()); } + // Checked whatever it sends: a program that keeps talking still runs out of time. + if deadline.is_some_and(|d| Instant::now() >= d) { + return Err("it ran too long and was stopped".into()); + } let wait = deadline.map_or(CANCEL_POLL, |d| d.saturating_duration_since(Instant::now()).min(CANCEL_POLL)); match rx.recv_timeout(wait) { Ok(l) => break l, @@ -316,7 +459,7 @@ fn converse( Ok(v) => json!({"ok": v}), Err(e) => json!({"error": e}), }; - send(stdin, &reply)?; + stdin.send(&reply)?; } else if let Some(text) = msg.get("log") { if logged < MAX_LOGGED { logged += 1; diff --git a/crates/plugins/src/session.rs b/crates/plugins/src/session.rs new file mode 100644 index 0000000000..645358b1e6 --- /dev/null +++ b/crates/plugins/src/session.rs @@ -0,0 +1,605 @@ +//! Session process plugins (`"session": true`, desktop and command line): one program kept +//! running between the plugin's commands, so it can ask RustGIS for things at any time and hear +//! when the map changes. A linked view in a window of its own (a 3D globe) is one. +//! +//! RustGIS starts the program the first time one of the plugin's commands runs, in the plugin's +//! folder, and writes a first line `{"session": {"abi", "plugin", "rustgis", "lang"}}`. From +//! then on the lines go both ways, one JSON object each: +//! +//! - RustGIS to the program: `{"command": request, "id": n}`, a command to run (the request a +//! per-call plugin gets), answered with `{"ok": value, "id": n}` or `{"error": "text", "id": n}`; +//! the reply to one of its host requests, `{"ok": value}` or `{"error": "text"}` with the +//! request's `id` when it gave one; and `{"event": "view" | "layers" | "selection" | "project", +//! ...}` when the map changes or another project opens. +//! - The program to RustGIS: `{"call": {"op", "params"}, "id": n}` (the `id` optional), `log` +//! and `progress` lines, and its answers to commands. +//! +//! The channel is the program's own input and output: no port, no token, nothing another program +//! can join. Its requests go through the same [`Host`] checks as a command's, within the plugin's +//! permissions. The program is stopped, with what it started, when the plugin is turned off, +//! removed or updated and when RustGIS closes ([`stop`], [`stop_all`]); its input closing means +//! it is being stopped (RustGIS closing or gone, however it ended; the plugin turned off, removed +//! or updated; the program failed) and is its sign to exit. A program that exits on its own (its window +//! closed) is started again by the plugin's next command; on Unix what it left running is stopped. +//! +//! The plugin's tools still start the program once per run, as a per-call process plugin's do. + +use std::process::{Child, ExitStatus}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use crate::process::{self, Kept, Line, Spec, Started, Writer}; +use crate::runtime::Host; + +/// Asks the app to look at its sessions (a repaint): a line arrived while it was idle. +pub type Wake = Arc; + +/// The most lines of one program handled in one [`poll`]; the rest wait for the next. +pub const POLL_BATCH: usize = 64; +/// Why a program was stopped when the answers it hadn't read passed [`MAX_QUEUED_BYTES`]. +const STALLED: &str = "the answers waiting for it to read passed 256 MB"; +/// How long a program has to exit when its plugin is turned off, removed or updated: the app +/// waits, so a program that ignores its closed input holds it this long at most. +const STOP_GRACE: Duration = Duration::from_secs(1); +/// How long a program whose output ended has to finish exiting before it's stopped. +const END_GRACE: Duration = Duration::from_millis(200); +/// The most lines of a program's other output and `log` lines logged per [`LOG_WINDOW`]. +const MAX_LOGGED: usize = 200; +const LOG_WINDOW: Duration = Duration::from_secs(60); +/// The most bytes of replies that may wait for the program to read them: one that keeps asking +/// without reading the answers is stopped rather than fill memory. +const MAX_QUEUED_BYTES: usize = 256 * 1024 * 1024; +/// How often a command waiting on the program checks whether it was cancelled. +const CANCEL_POLL: Duration = Duration::from_millis(100); + +/// A program kept running for its plugin. +struct Running { + plugin: String, + program: String, + child: Child, + stdin: Writer, + rx: Receiver, + errs: Kept, + /// The next command's `id`. + next_id: u64, + /// Lines logged since the window began: a program printing endlessly can't flood the log. + logged: (Instant, usize), + /// Events held while the program is behind in reading: the latest of each kind. + held: Vec, + /// How many polls have found it exited with its output still coming: something it left + /// running keeps writing. At [`DRAIN_BATCHES`] it's stopped. + drained: usize, + /// Writing to it failed: it closed its input (stopped reading) while it may still run. + input_closed: bool, + /// The last line of its error output logged so far. + last_stderr: Option, +} + +impl Running { + /// Whether one more line may be logged now; says once when lines stop being logged. + fn may_log(&mut self) -> bool { + if self.logged.0.elapsed() > LOG_WINDOW { + self.logged = (Instant::now(), 0); + } + self.logged.1 += 1; + if self.logged.1 == MAX_LOGGED + 1 { + log::info!("{}: more output than {MAX_LOGGED} lines a minute; the rest isn't logged", self.plugin); + } + self.logged.1 <= MAX_LOGGED + } +} + +static RUNNING: Mutex> = Mutex::new(Vec::new()); +/// How programs found ended by a command (not by [`poll`]) ended: for the app to say ([`take_ended`]). +static ENDED: Mutex> = Mutex::new(Vec::new()); +/// The most [`poll`] batches read of a program found exited before it is stopped: what it sent +/// before exiting is finite, but what it left running may keep its output open and writing. +pub const DRAIN_BATCHES: usize = 256; + +/// Keeps how `plugin`'s program ended when it failed, for [`take_ended`]. +fn note_ended(plugin: &str, ended: Ended) { + if (ended.failed || ended.unread) + && let Ok(mut v) = ENDED.lock() + { + // One kept a plugin: the latest. + v.retain(|(p, _)| p != plugin); + v.push((plugin.to_string(), ended)); + } +} + +/// How `plugin`'s program ended, if a command found it failed (and started it again) since +/// the last call: the app says so as it does for one [`poll`] finds. +pub fn take_ended(plugin: &str) -> Option { + let mut v = ENDED.lock().ok()?; + let i = v.iter().position(|(p, _)| p == plugin)?; + Some(v.swap_remove(i).1) +} + +/// Whether `plugin`'s program is in the list but has exited: what it sent last is still to be +/// read ([`poll`]). +pub fn has_exited(plugin: &str) -> bool { + RUNNING.lock().is_ok_and(|mut v| v.iter_mut().any(|r| r.plugin == plugin && process::exited(&mut r.child))) +} +static WAKE: Mutex> = Mutex::new(None); + +/// Sets what a session's line arriving calls, for programs started from now on. +pub fn set_wake(wake: Wake) { + if let Ok(mut w) = WAKE.lock() { + *w = Some(wake); + } +} + +/// What a new program's lines call as they arrive ([`set_wake`]). +fn wake() -> Option { + // What the app set when it's called, not when the program started: one started before the + // app's first frame wakes it too. At most one asked for until the app polls ([`polled`]): + // a program printing line after line asks for one frame, not one a line. + Some(Arc::new(|| { + if !WOKEN.swap(true, std::sync::atomic::Ordering::Relaxed) + && let Some(w) = WAKE.lock().ok().and_then(|w| w.clone()) + { + w(); + } + })) +} + +/// Whether a wake was asked for since the app last polled. +static WOKEN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// The app is about to poll: the next line asks for a frame again. +pub fn polled() { + WOKEN.store(false, std::sync::atomic::Ordering::Relaxed); +} + +/// The plugins whose programs are running. +pub fn running() -> Vec { + RUNNING.lock().map(|v| v.iter().map(|r| r.plugin.clone()).collect()).unwrap_or_default() +} + +/// Whether `plugin`'s program is running. +pub fn is_running(plugin: &str) -> bool { + RUNNING.lock().is_ok_and(|v| v.iter().any(|r| r.plugin == plugin)) +} + +/// Takes a running program out of the list while it's talked to: what its requests run can't +/// reach the list's lock. +fn take(plugin: &str) -> Option { + let mut v = RUNNING.lock().ok()?; + let i = v.iter().position(|r| r.plugin == plugin)?; + Some(v.swap_remove(i)) +} + +/// Puts a program back in the list after it was talked to; one that can't be kept is stopped. +fn put(r: Running) { + match RUNNING.lock() { + Ok(mut v) => v.push(r), + // Not kept, so not left running either. + Err(_) => { + shut(r, Duration::ZERO); + } + } +} + +/// Starts `plugin`'s program and greets it with the session's first line. +fn start(plugin: &str, spec: &Spec) -> Result { + let Started { child, stdin, rx, errs } = process::spawn(spec, wake())?; + let r = Running { + plugin: plugin.to_string(), + program: spec.program.clone(), + child, + stdin, + rx, + errs, + next_id: 1, + logged: (Instant::now(), 0), + held: Vec::new(), + drained: 0, + input_closed: false, + last_stderr: None, + }; + let hello = json!({"session": {"abi": crate::manifest::ABI, "plugin": plugin, "rustgis": crate::manifest::RUSTGIS_VERSION, "lang": crate::language()}}); + if let Err(e) = r.stdin.send(&hello) { + shut(r, Duration::ZERO); + return Err(e); + } + Ok(r) +} + +/// Closes the program's input, gives it `grace` to exit, then stops it with what it started. +/// Returns its exit status when it exited by itself, whether it had to be stopped, and the last +/// line of its error output. +fn shut(mut r: Running, grace: Duration) -> (Option, bool, Option) { + log_errors(&mut r); + let Running { mut child, stdin, rx, errs, program, last_stderr, .. } = r; + drop(stdin); + drop(rx); + let (status, stopped) = process::end(&mut child, grace); + // What it wrote as it ended: until its error output has ended, or a moment at most, as + // something holding the pipe open (which `end` couldn't stop) mustn't hold this. + let until = Instant::now() + END_GRACE; + while !errs.lock().is_ok_and(|k| k.done) && Instant::now() < until { + std::thread::sleep(Duration::from_millis(5)); + } + let last = drain_errors(&errs, &program).or(last_stderr); + (status, stopped, last) +} + +/// Logs the error output the program wrote since the last call, as it runs. +fn log_errors(r: &mut Running) { + let (lines, dropped) = r.errs.lock().map(|mut k| (k.lines.drain(..).collect(), std::mem::take(&mut k.dropped))).unwrap_or_default(); + let lines: Vec = lines; + if dropped > 0 && r.may_log() { + log::info!("plugin {}: {dropped} earlier lines of its error output weren't kept", r.program); + } + for line in lines.into_iter().filter(|l| !l.trim().is_empty()) { + if r.may_log() { + log::info!("plugin {}: {line}", r.program); + } + r.last_stderr = Some(line); + } +} + +/// Logs what is left of a program's error output; returns its last line. +fn drain_errors(errs: &Kept, program: &str) -> Option { + let (lines, dropped) = errs.lock().map(|mut k| (k.lines.drain(..).collect(), std::mem::take(&mut k.dropped))).unwrap_or_default(); + let lines: Vec = lines; + if dropped > 0 { + log::info!("plugin {program}: {dropped} earlier lines of its error output weren't kept"); + } + for line in lines.iter().filter(|l| !l.trim().is_empty()) { + log::info!("plugin {program}: {line}"); + } + lines.into_iter().rev().find(|l| !l.trim().is_empty()) +} + +/// Stops `plugin`'s program, if it's running. +pub fn stop(plugin: &str) { + if let Some(r) = take(plugin) { + shut(r, STOP_GRACE); + } +} + +/// Stops every program (RustGIS is closing): their inputs close together, and they share the +/// time to exit. +pub fn stop_all() { + let all: Vec = RUNNING.lock().map(|mut v| std::mem::take(&mut *v)).unwrap_or_default(); + let mut waiting = Vec::new(); + for r in all { + let Running { child, stdin, rx, errs, program, .. } = r; + drop(stdin); + drop(rx); + waiting.push((child, errs, program)); + } + let end = Instant::now() + process::EXIT_GRACE; + for (mut child, errs, program) in waiting { + process::end(&mut child, end.saturating_duration_since(Instant::now())); + drain_errors(&errs, &program); + } +} + +/// A protocol line from the program, or `None` for other output (logged, within limits). +fn message(r: &mut Running, bytes: &[u8]) -> Option> { + match serde_json::from_slice::(bytes) { + Ok(Value::Object(m)) if ["call", "log", "progress", "error", "ok"].iter().any(|k| m.contains_key(*k)) => Some(m), + _ => { + let text = String::from_utf8_lossy(bytes); + let text = text.trim_end(); + if !text.is_empty() && r.may_log() { + log::info!("plugin output: {}", text.chars().take(1_000).collect::()); + } + None + } + } +} + +/// A message's text: a string as it is, anything else as JSON. +fn text(v: &Value) -> String { + v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string()) +} + +/// Handles a line that isn't an answer to a command: a host request (answered), a log or +/// progress line. Returns the answer's `id` and value when it is one. +fn handle(m: &Map, host: &mut dyn Host, r: &mut Running) -> Result)>, String> { + let id = m.get("id").cloned().unwrap_or(Value::Null); + if let Some(c) = m.get("call") { + let op = c.get("op").and_then(Value::as_str).unwrap_or_default(); + let params = c.get("params").cloned().unwrap_or(Value::Null); + // A request that fails unexpectedly is an error answer, as in a WebAssembly plugin: the + // program, out of the list while it's talked to, is never left running untracked. + let answer = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| host.call(op, ¶ms))) + .unwrap_or_else(|_| Err(format!("'{op}' failed unexpectedly"))); + let mut reply = match answer { + Ok(v) => json!({"ok": v}), + Err(e) => json!({"error": e}), + }; + if !id.is_null() + && let Some(o) = reply.as_object_mut() + { + o.insert("id".into(), id); + } + // An answer too large to send at all is refused to it, whether it reads or not; one + // that would only overflow what it hasn't read yet means it stopped reading. + // Written out once: measured, then sent as it is (an answer can be hundreds of MB). + let mut line = reply.to_string().into_bytes(); + let size = line.len(); + if size > MAX_QUEUED_BYTES { + let id = reply.get("id").cloned(); + reply = json!({"error": format!("the answer is larger than {} MB", MAX_QUEUED_BYTES / (1024 * 1024))}); + if let (Some(id), Some(o)) = (id, reply.as_object_mut()) { + o.insert("id".into(), id); + } + line = reply.to_string().into_bytes(); + } + // What is sent, refusal or not, must fit with what it hasn't read yet. + if r.stdin.backlog_bytes().saturating_add(line.len()) > MAX_QUEUED_BYTES { + return Err(STALLED.into()); + } + r.stdin.send_line(line)?; + } else if let Some(t) = m.get("log") { + if r.may_log() { + host.log(&text(t)); + } + } else if let Some(t) = m.get("progress") { + host.progress(&text(t)); + } else if let Some(e) = m.get("error") { + return Ok(Some((id, Err(text(e))))); + } else if let Some(v) = m.get("ok") { + return Ok(Some((id, Ok(v.clone())))); + } + Ok(None) +} + +/// Runs a command of `plugin` in its program, starting it if it isn't running: sends `request`, +/// answers its host requests through `host`, and returns its answer. A program that doesn't +/// answer within `timeout` is stopped. +pub fn command(plugin: &str, spec: &Spec, host: &mut dyn Host, request: &Value, timeout: Duration) -> Result { + let mut r = match take(plugin) { + Some(mut r) => { + if !process::exited(&mut r.child) { + r + } else { + // It exited on its own (its window closed): what it sent last is handled first, + // then it is started again. + put(r); + // All of it, a batch at a time; then how it ended is kept for the app to say. + for _ in 0..DRAIN_BATCHES { + let polled = poll(plugin, host); + if let Some(ended) = polled.ended { + note_ended(plugin, ended); + break; + } + } + // Still output past that: what it left running keeps writing. Stopped, and its + // exit status says whether it failed. + if let Some(r) = take(plugin) { + log::warn!("{plugin}: its output was still coming after it exited; the rest wasn't read"); + let (status, _, last_error) = shut(r, Duration::ZERO); + // No status (stopped here, or reaped elsewhere) isn't a failure: only its own exit says so. + note_ended(plugin, Ended { failed: status.is_some_and(|s| !s.success()), last_error, unread: true, reason: None }); + } + start(plugin, spec)? + } + } + None => start(plugin, spec)?, + }; + let id = r.next_id; + r.next_id += 1; + if let Err(e) = r.stdin.send(&json!({"command": request, "id": id})) { + shut(r, Duration::ZERO); + return Err(e); + } + let deadline = Instant::now() + timeout; + let mut calls = 0u32; + let res: Result<(), String> = loop { + if host.cancelled() { + break Err(process::CANCELLED.to_string()); + } + // Checked whatever it sends: a program that keeps talking still runs out of time. + if Instant::now() >= deadline { + break Err("it ran too long and was stopped".to_string()); + } + let line = match r.rx.recv_timeout(deadline.saturating_duration_since(Instant::now()).min(CANCEL_POLL)) { + Ok(l) => l, + Err(RecvTimeoutError::Timeout) if Instant::now() >= deadline => break Err("it ran too long and was stopped".to_string()), + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => Line::End, + }; + let bytes = match line { + Line::Text(b) => b, + Line::End => break Err("it exited without answering".to_string()), + Line::TooLong => break Err(format!("it sent a line longer than {} MB", process::MAX_LINE / (1024 * 1024))), + }; + let Some(m) = message(&mut r, &bytes) else { continue }; + if m.contains_key("call") { + calls += 1; + if calls > process::MAX_HOST_CALLS { + break Err("it made too many requests".to_string()); + } + } + match handle(&m, host, &mut r) { + Err(e) => break Err(e), + Ok(Some((answered, res))) if answered == json!(id) => { + put(r); + return res; + } + // Not this command's: an answer it had given up on. + Ok(Some(_)) => { + if r.may_log() { + log::info!("{plugin}: an answer to no command it was running"); + } + } + Ok(None) => {} + } + }; + // It stopped answering: stopped, so the next command starts it afresh. + let err = res.err().unwrap_or_default(); + match shut(r, Duration::ZERO) { + (_, _, Some(last)) if err.contains("without answering") => Err(format!("{err}: {last}")), + _ => Err(err), + } +} + +/// What [`poll`] found. +#[derive(Debug, Default, PartialEq)] +pub struct Polled { + /// Lines handled. + pub handled: usize, + /// The program exited, or was stopped for what it sent. + pub ended: Option, +} + +/// How a session's program ended. +#[derive(Debug, PartialEq)] +pub struct Ended { + /// Whether it failed: exited with an error, or was stopped. Exiting by itself (its window + /// closed) isn't failing. + pub failed: bool, + /// The last line of its error output. + pub last_error: Option, + /// Whether what it sent after exiting wasn't all read: something it left running kept + /// writing past [`DRAIN_BATCHES`], and its requests from then on weren't answered. + pub unread: bool, + /// Why RustGIS stopped it, when it did for what it sent. + pub reason: Option, +} + +/// Answers what `plugin`'s program sent since the last call, at most [`POLL_BATCH`] lines, +/// without waiting for more. Each host request goes to `host`. +pub fn poll(plugin: &str, host: &mut dyn Host) -> Polled { + let Some(mut r) = take(plugin) else { return Polled::default() }; + log_errors(&mut r); + let mut out = Polled::default(); + // Whether it ended, and whether it was stopped for what it sent. + let mut ended = None; + let mut reason = None; + // It exited: what it wrote last may still be on its way from the pipe, and is read first. + // Checked first too: what it left running may keep its output flowing, so the channel never + // runs empty to say so. + let mut gone = process::exited(&mut r.child).then(|| Instant::now() + END_GRACE); + for _ in 0..POLL_BATCH { + let line = if let Some(until) = gone { + // One moment for all of it, not one a line. + r.rx.recv_timeout(until.saturating_duration_since(Instant::now())).ok() + } else { + match r.rx.try_recv() { + Ok(l) => Some(l), + // Nothing new: it may have exited, its output held open by what it started. + Err(TryRecvError::Empty) => { + if process::exited(&mut r.child) { + gone = Some(Instant::now() + END_GRACE); + continue; + } + break; + } + Err(TryRecvError::Disconnected) => None, + } + }; + let bytes = match line { + Some(Line::Text(b)) => b, + None | Some(Line::End) => { + ended = Some(false); + break; + } + Some(Line::TooLong) => { + let why = format!("it sent a line longer than {} MB", process::MAX_LINE / (1024 * 1024)); + log::warn!("{plugin}: {why}, and was stopped"); + reason = Some(why); + ended = Some(true); + break; + } + }; + out.handled += 1; + let Some(m) = message(&mut r, &bytes) else { continue }; + match handle(&m, host, &mut r) { + Ok(None) => {} + Ok(Some(_)) => { + if r.may_log() { + log::info!("{plugin}: an answer to no command it was running"); + } + } + // The answers it hasn't read passed the limit: stopped. + Err(e) if e == STALLED => { + log::warn!("{plugin}: {e}, and was stopped"); + reason = Some(e); + ended = Some(true); + break; + } + // Exited, it can't be answered; what it asked was done. + Err(_) if gone.is_some() => {} + // It can't be written to: it's exiting, and what it sent last is read first. + Err(_) => { + r.input_closed = true; + gone = Some(Instant::now() + END_GRACE); + } + } + } + // Exited, and its output still coming after this batch: something it left running keeps + // writing. Poll after poll, that's bounded too: past it, stopped with what it left running. + if ended.is_none() && gone.is_some() { + r.drained += 1; + if r.drained >= DRAIN_BATCHES { + log::warn!("{plugin}: its output was still coming after it exited; the rest wasn't read"); + let (status, _, last_error) = shut(r, Duration::ZERO); + out.ended = Some(Ended { failed: status.is_some_and(|s| !s.success()), last_error, unread: true, reason: None }); + return out; + } + } + // It stopped reading while it still runs: nothing RustGIS says reaches it. Stopped, and said. + if ended.is_none() && r.input_closed && !process::exited(&mut r.child) { + reason = Some("it stopped reading its input while running".to_string()); + ended = Some(true); + } + let Some(stopped) = ended else { + flush(&mut r); + put(r); + // A full batch may have left more: another frame for them, not whenever one comes. + if out.handled >= POLL_BATCH + && let Some(w) = wake() + { + w(); + } + return out; + }; + // Its own exit status says whether it failed; one stopped for what it sent did. Its output + // ends as it exits: a moment for the exit to land, on the app's thread. One still running + // after it is stopped, but closing its output was its own doing: no failure. + let input_closed = r.input_closed; + let (status, had_to_stop, last_error) = shut(r, if stopped { Duration::ZERO } else { END_GRACE }); + // Its output closed but it didn't exit: it was stopped, and that's said. (No status of its + // own isn't enough: something else may have reaped it after a clean exit.) + if !stopped && had_to_stop && reason.is_none() { + reason = Some( + if input_closed { "it stopped reading its input while running" } else { "it closed its output but kept running" }.to_string(), + ); + } + out.ended = Some(Ended { failed: stopped || status.is_some_and(|s| !s.success()), last_error, unread: false, reason }); + out +} + +/// Tells `plugin`'s program what changed: `event` is `{"event": name, ...}`. While the program +/// is behind in reading, events wait, the latest of each kind replacing an earlier one, so a +/// program that stalls while the map moves doesn't fill memory. One that exited is found out by +/// the next [`poll`]. +pub fn event(plugin: &str, event: &Value) { + if let Ok(mut v) = RUNNING.lock() + && let Some(r) = v.iter_mut().find(|r| r.plugin == plugin) + { + r.held.retain(|e| e.get("event") != event.get("event")); + r.held.push(event.clone()); + flush(r); + } +} + +/// Sends the events held for the program, unless it's still behind in reading. +fn flush(r: &mut Running) { + if r.stdin.backlog() < process::HOLD_EVENTS_AT { + for e in std::mem::take(&mut r.held) { + if r.stdin.send(&e).is_err() { + r.input_closed = true; + } + } + } +} diff --git a/crates/plugins/src/store.rs b/crates/plugins/src/store.rs index 08bf38d3da..15c3a52bed 100644 --- a/crates/plugins/src/store.rs +++ b/crates/plugins/src/store.rs @@ -125,16 +125,22 @@ pub fn unpack(bytes: &[u8], pkg: &mut Package) -> Result<(), String> { return Ok(()); } let target = unpacked_dir(&pkg.manifest.id).ok_or("there is no folder for plugins")?; - unpack_to(bytes, &pkg.sha256, &target)?; + unpack_to(bytes, &pkg.sha256, &target, &pkg.manifest.id)?; pkg.dir = Some(target); Ok(()) } /// Unpacks a package to `target`, the folder a process plugin runs from, unless that copy is /// already there. -fn unpack_to(bytes: &[u8], sha256: &str, target: &Path) -> Result<(), String> { +fn unpack_to(bytes: &[u8], sha256: &str, target: &Path, id: &str) -> Result<(), String> { match stage(bytes, sha256, target)? { - Some(staged) => swap_in(&staged, target), + Some(staged) => { + // A session plugin's program runs from these files (on Windows they can't be + // replaced while it does): it stops first. + let was_running = crate::session::is_running(id); + crate::session::stop(id); + swap_in(&staged, target).map_err(|e| if was_running { format!("{e} (its running program was stopped first)") } else { e }) + } None => Ok(()), } } @@ -255,7 +261,11 @@ pub fn install_bytes(bytes: &[u8]) -> Result, String> { return Err(format!("{}: {e}", path.display())); } if let Some((s, target)) = &staged { - swap_in(s, target)?; + // A session plugin's program runs from these files (on Windows they can't be replaced + // while it does): it stops first. + let was_running = crate::session::is_running(&plugin.manifest.id); + crate::session::stop(&plugin.manifest.id); + swap_in(s, target).map_err(|e| if was_running { format!("{e} (its running program was stopped first)") } else { e })?; } Ok(crate::register(plugin)) } diff --git a/crates/plugins/src/testing.rs b/crates/plugins/src/testing.rs index ed5e7602c7..c7ab6649b4 100644 --- a/crates/plugins/src/testing.rs +++ b/crates/plugins/src/testing.rs @@ -256,6 +256,26 @@ pub fn process_manifest(id: &str, permissions: &[&str], run: &[String]) -> Strin m.to_string() } +/// A manifest for the session test plugin (`"session": true`): the process test plugin's +/// commands, and `k`, `v`, `i` and `z` (see [`serve_if_child`]). +pub fn session_manifest(id: &str, permissions: &[&str], run: &[String]) -> String { + let mut m: serde_json::Value = serde_json::from_str(&process_manifest(id, permissions, run)).unwrap_or_default(); + m["session"] = serde_json::json!(true); + if let Some(c) = m["commands"].as_array_mut() { + c.push(serde_json::json!({"id": "k", "label": "Ask Later"})); + c.push(serde_json::json!({"id": "v", "label": "Heard"})); + c.push(serde_json::json!({"id": "i", "label": "Process"})); + c.push(serde_json::json!({"id": "z", "label": "Quit"})); + c.push(serde_json::json!({"id": "l", "label": "Launch"})); + c.push(serde_json::json!({"id": "p", "label": "Pause"})); + c.push(serde_json::json!({"id": "w", "label": "Last Words"})); + c.push(serde_json::json!({"id": "b", "label": "Bad Exit"})); + c.push(serde_json::json!({"id": "y", "label": "Long Goodbye"})); + c.push(serde_json::json!({"id": "r", "label": "Leave a Writer"})); + } + m.to_string() +} + /// In a test binary started by a process test plugin ([`child_run`]): answers one request as the /// test module does, then exits. Elsewhere it returns at once. /// @@ -267,6 +287,18 @@ pub fn process_manifest(id: &str, permissions: &[&str], run: &[String]) -> Strin /// - Tools: `m` returns the layer "Made"; `x` selects rows 0 and 2 of "Cities"; `r` returns the /// names of the files it was given as its messages; `w` logs "working", reports /// progress "step 1 of 3" and never answers; `m` reports progress "making" first. +/// +/// Started as a session (its first line `{"session": …}`), it keeps answering commands until +/// its input closes: `h`, `e`, `c` and `s` as above; `f` sends its params as a host request and +/// returns the reply, skipping events; `k` sends that request and answers at once, leaving the +/// reply for later; `v` returns what it has heard since (`events`, `answers`, and the latest +/// `view` event's view as `last_view`); `i` returns its +/// process id; `z` answers and exits; `l` starts a copy of itself that never answers, answers +/// with its process id and exits, leaving it running; `p` answers, then reads nothing for two +/// seconds; `w` answers, then sends its params as a host request, logs "last words" and exits at +/// once; `b` answers, writes "boom" to its error output and exits with status 3; `y` answers, +/// logs 70 lines, sends its params as a host request and exits at once; `r` starts a copy of +/// itself that writes into its output forever, answers with that copy's process id and exits. #[cfg(not(target_arch = "wasm32"))] pub fn serve_if_child() { use serde_json::{Value, json}; @@ -281,6 +313,9 @@ pub fn serve_if_child() { let mut line = String::new(); let _ = input.read_line(&mut line); let req: Value = serde_json::from_str(&line).unwrap_or_default(); + if req.get("session").is_some() { + serve_session(input, out); + } let params = req.get("params").cloned().unwrap_or_default(); let reply = match req.get("id").and_then(Value::as_str).and_then(|s| s.chars().next()) { Some('h') => { @@ -334,6 +369,17 @@ pub fn serve_if_child() { Some('s') => loop { std::thread::sleep(std::time::Duration::from_secs(1)); }, + // Logs forever, as fast as it can, saying on its error output once it has written more + // than a pipe holds: from then on its output is waiting to be read, whenever it runs. + Some('u') => { + for _ in 0..8_192 { + let _ = writeln!(out, "{}", json!({"log": "more"})); + } + eprintln!("writing"); + loop { + let _ = writeln!(out, "{}", json!({"log": "more"})); + } + } Some('g') => { let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); let spawned = argv.split_first().and_then(|(exe, args)| { @@ -381,3 +427,159 @@ pub fn serve_if_child() { let _ = out.flush(); std::process::exit(0); } + +/// The session test plugin: see [`serve_if_child`]. +#[cfg(not(target_arch = "wasm32"))] +fn serve_session(mut input: impl std::io::BufRead, mut out: impl std::io::Write) -> ! { + use serde_json::{Value, json}; + let (mut events, mut answers) = (Vec::::new(), Vec::::new()); + let mut last_view = Value::Null; + let mut asked = 0u64; + loop { + let mut line = String::new(); + if input.read_line(&mut line).map_or(true, |n| n == 0) { + std::process::exit(0); + } + let msg: Value = serde_json::from_str(&line).unwrap_or_default(); + if let Some(e) = msg.get("event") { + if e == "view" { + last_view = msg["view"].clone(); + } + events.push(e.clone()); + continue; + } + let Some(req) = msg.get("command") else { + answers.push(msg); + continue; + }; + let id = msg["id"].clone(); + let params = req.get("params").cloned().unwrap_or_default(); + let mut ask = |out: &mut dyn std::io::Write| { + asked += 1; + let _ = writeln!(out, "{}", json!({"call": {"op": params["op"], "params": params["params"]}, "id": format!("a{asked}")})); + let _ = out.flush(); + format!("a{asked}") + }; + let reply = match req.get("id").and_then(Value::as_str).and_then(|s| s.chars().next()) { + Some('h') => json!({"ok": {"hello": "world"}}), + Some('f') => { + let want = ask(&mut out); + loop { + let mut answer = String::new(); + if input.read_line(&mut answer).map_or(true, |n| n == 0) { + std::process::exit(0); + } + let a: Value = serde_json::from_str(&answer).unwrap_or_default(); + if let Some(e) = a.get("event") { + if e == "view" { + last_view = a["view"].clone(); + } + events.push(e.clone()); + } else if a["id"] == json!(want) { + break a; + } + } + } + Some('r') => { + let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); + // Its output is this program's: it keeps writing there after this one exits. + let spawned = argv.split_first().and_then(|(exe, args)| { + std::process::Command::new(exe) + .args(args) + .stdin(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .ok() + }); + let pid = spawned.map(|mut g| { + if let Some(mut stdin) = g.stdin.take() { + let _ = writeln!(stdin, "{}", json!({"id": "u", "params": {}})); + } + // Exits only once the copy is writing: its output is flowing when it's gone. + if let Some(err) = g.stderr.take() { + let mut started = String::new(); + let _ = std::io::BufRead::read_line(&mut std::io::BufReader::new(err), &mut started); + } + g.id() + }); + let _ = writeln!(out, "{}", json!({"ok": pid, "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + Some('y') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + for i in 0..70 { + let _ = writeln!(out, "{}", json!({"log": format!("line {i}")})); + } + ask(&mut out); + std::process::exit(0); + } + Some('b') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + let _ = out.flush(); + eprintln!("boom"); + std::process::exit(3); + } + Some('w') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + ask(&mut out); + let _ = writeln!(out, "{}", json!({"log": "last words"})); + let _ = out.flush(); + std::process::exit(0); + } + Some('p') => { + let _ = writeln!(out, "{}", json!({"ok": "paused", "id": id})); + let _ = out.flush(); + std::thread::sleep(std::time::Duration::from_secs(2)); + continue; + } + // Answered first, so the request comes after the command. + Some('k') => { + let _ = writeln!(out, "{}", json!({"ok": "asked", "id": id})); + ask(&mut out); + continue; + } + Some('v') => { + json!({"ok": {"events": std::mem::take(&mut events), "answers": std::mem::take(&mut answers), "last_view": last_view}}) + } + Some('i') => json!({"ok": std::process::id()}), + Some('c') => { + eprintln!("boom"); + std::process::exit(3); + } + Some('s') => loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + }, + Some('z') => { + let _ = writeln!(out, "{}", json!({"ok": "bye", "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + Some('l') => { + let argv = child_run(&std::env::args().nth(1).unwrap_or_default()); + let spawned = argv.split_first().and_then(|(exe, args)| { + std::process::Command::new(exe) + .args(args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .spawn() + .ok() + }); + let pid = spawned.map(|mut g| { + if let Some(mut stdin) = g.stdin.take() { + let _ = writeln!(stdin, "{}", json!({"id": "s", "params": {}})); + } + g.id() + }); + let _ = writeln!(out, "{}", json!({"ok": pid, "id": id})); + let _ = out.flush(); + std::process::exit(0); + } + _ => json!({"error": "nope"}), + }; + let mut reply = reply; + reply["id"] = id; + let _ = writeln!(out, "{reply}"); + let _ = out.flush(); + } +} diff --git a/crates/plugins/src/tests.rs b/crates/plugins/src/tests.rs index be87984cde..762a68e1f6 100644 --- a/crates/plugins/src/tests.rs +++ b/crates/plugins/src/tests.rs @@ -15,6 +15,9 @@ impl Host for Recorder { self.ops.push(op.to_string()); match op { "host.echo" => Ok(params.clone()), + // A host request that fails unexpectedly. + #[allow(clippy::panic)] + "host.panic" => panic!("a host request failed unexpectedly"), _ => Err(format!("no '{op}'")), } } @@ -1148,6 +1151,203 @@ fn process_tools_show_progress_and_can_be_cancelled() { let _ = std::fs::remove_dir_all(&dir); } +/// Whether a process is gone: not there, or (on Linux) a zombie nothing has reaped yet, as in a +/// container with no init to reap orphans. +#[cfg(unix)] +fn gone(pid: rustix::process::Pid) -> bool { + rustix::process::test_kill_process(pid).is_err() + || std::fs::read_to_string(format!("/proc/{}/stat", pid.as_raw_nonzero())) + .ok() + .and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('Z'))) + .unwrap_or(false) +} + +/// A session test plugin played by this binary, prepared (not loaded). +fn session_plugin(id: &str) -> (crate::Plugin, std::path::PathBuf) { + let dir = std::env::temp_dir().join(format!("rustgis-test-session-{id}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let run = testing::child_run("tests::process_child"); + std::fs::write(dir.join("plugin.json"), testing::session_manifest(id, &[], &run)).unwrap(); + (crate::prepare(Package::from_dir(&dir).unwrap(), "test").unwrap(), dir) +} + +/// The request for the test plugin's command `id`, without parameters. +fn command(id: &str) -> Value { + json!({"abi": 1, "kind": "command", "id": id, "params": {}, "lang": "en"}) +} + +/// Polls `plugin`'s program until `done` says so, for a few seconds at most. +fn poll_until(plugin: &str, host: &mut Recorder, done: impl Fn(&crate::session::Polled, &Recorder) -> bool) -> crate::session::Polled { + let t0 = std::time::Instant::now(); + loop { + let polled = crate::session::poll(plugin, host); + if done(&polled, host) || t0.elapsed() > std::time::Duration::from_secs(20) { + return polled; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +/// A program's error output past what is kept is counted, so the log can say so. +#[test] +fn error_output_past_what_is_kept_is_counted() { + let text: String = (0..25).map(|i| format!("line {i}\n")).collect(); + let kept = crate::process::errors(std::io::Cursor::new(text.into_bytes()), None); + let t0 = std::time::Instant::now(); + while kept.lock().unwrap().dropped < 5 && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let k = kept.lock().unwrap(); + assert_eq!((k.dropped, k.lines.len(), k.lines.front().map(String::as_str)), (5, 20, Some("line 5"))); +} + +#[test] +fn session_plugins_keep_one_program_running() { + let (p, dir) = session_plugin("sess-keep"); + assert_eq!(p.info()["session"], true); + let mut host = Recorder::default(); + let pid = crate::call(&p, &mut host, &command("i"), false).unwrap(); + assert!(crate::session::is_running("sess-keep")); + assert_eq!(crate::call(&p, &mut host, &command("h"), false).unwrap(), json!({"hello": "world"})); + assert_eq!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid, "the same program answers"); + // A host request inside a command is answered as a per-call plugin's is. + let forward = json!({"abi": 1, "kind": "command", "id": "f", "params": {"op": "host.echo", "params": {"a": 1}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &forward, false).unwrap(), json!({"a": 1})); + assert_eq!(crate::call(&p, &mut host, &command("e"), false).unwrap_err(), "nope"); + + // Between commands it asks when it likes; the answer carries its id. It hears events. + let later = json!({"abi": 1, "kind": "command", "id": "k", "params": {"op": "host.echo", "params": {"a": 1}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &later, false).unwrap(), "asked"); + let polled = poll_until("sess-keep", &mut host, |_, h| h.ops.len() >= 2); + assert_eq!(host.ops, ["host.echo", "host.echo"]); + assert!(polled.ended.is_none()); + crate::session::event("sess-keep", &json!({"event": "selection"})); + let heard = crate::call(&p, &mut host, &command("v"), false).unwrap(); + assert_eq!(heard["answers"], json!([{"ok": {"a": 1}, "id": "a2"}])); + assert_eq!(heard["events"], json!(["selection"])); + + // A program that stops reading while the map moves gets the latest view when it reads again, + // not every one that came meanwhile. + assert_eq!(crate::call(&p, &mut host, &command("p"), false).unwrap(), "paused"); + for i in 0..5_000 { + crate::session::event("sess-keep", &json!({"event": "view", "view": {"scale": i, "pad": "x".repeat(100)}})); + } + std::thread::sleep(std::time::Duration::from_secs(3)); + poll_until("sess-keep", &mut host, |_, _| true); + let heard = crate::call(&p, &mut host, &command("v"), false).unwrap(); + let views = heard["events"].as_array().unwrap().len(); + assert!(views < 5_000, "{views} views queued"); + assert_eq!(heard["last_view"]["scale"], 4_999, "the latest view is the one kept"); + + // A host request that fails unexpectedly is an error answer; the program stays running. + let panics = json!({"abi": 1, "kind": "command", "id": "f", "params": {"op": "host.panic", "params": {}}, "lang": "en"}); + let e = crate::call(&p, &mut host, &panics, false).unwrap_err(); + assert!(e.contains("'host.panic' failed unexpectedly"), "{e}"); + assert_eq!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid, "the same program still answers"); + + // Stopped, it's gone; the next command starts it again. + crate::session::stop("sess-keep"); + assert!(!crate::session::is_running("sess-keep")); + #[cfg(unix)] + { + let old = rustix::process::Pid::from_raw(pid.as_i64().unwrap() as i32).unwrap(); + assert!(rustix::process::test_kill_process(old).is_err(), "{old:?} still runs"); + } + assert_ne!(crate::call(&p, &mut host, &command("i"), false).unwrap(), pid); + // A tool still runs a program of its own, once. + let tool = json!({"abi": 1, "kind": "tool", "id": "m", "params": {}, "layers": {}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &tool, true).unwrap()["messages"], json!(["made"])); + crate::session::stop("sess-keep"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn session_plugins_that_end_or_fail_are_started_again() { + let (p, dir) = session_plugin("sess-end"); + let mut host = Recorder::default(); + // Exiting by itself (its window closed) is no failure, and it is no longer running. + assert_eq!(crate::call(&p, &mut host, &command("z"), false).unwrap(), "bye"); + let polled = poll_until("sess-end", &mut host, |p, _| p.ended.is_some()); + assert_eq!(polled.ended, Some(crate::session::Ended { failed: false, last_error: None, unread: false, reason: None })); + assert!(!crate::session::is_running("sess-end")); + // What a program left running when it exited is stopped with it (on Unix). + #[cfg(unix)] + { + let left = crate::call(&p, &mut host, &command("l"), false).unwrap(); + assert!(poll_until("sess-end", &mut host, |p, _| p.ended.is_some()).ended.is_some(), "the session didn't end"); + let pid = rustix::process::Pid::from_raw(left.as_i64().unwrap() as i32).unwrap(); + let t0 = std::time::Instant::now(); + while !gone(pid) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone(pid), "{pid:?} still runs"); + } + // What it sent as it exited is still handled: a last request, a last log line. + let last = json!({"abi": 1, "kind": "command", "id": "w", "params": {"op": "host.echo", "params": {"last": true}}, "lang": "en"}); + assert_eq!(crate::call(&p, &mut host, &last, false).unwrap(), "bye"); + let polled = poll_until("sess-end", &mut host, |p, _| p.ended.is_some()); + assert_eq!(polled.ended.map(|e| e.failed), Some(false)); + assert!(host.ops.contains(&"host.echo".to_string()) && host.logs.contains(&"last words".to_string()), "{:?} {:?}", host.ops, host.logs); + // A crash is an error naming what it wrote last; the next command starts it again. + let e = crate::call(&p, &mut host, &command("c"), false).unwrap_err(); + assert!(e.contains("exited without answering") && e.contains("boom"), "{e}"); + assert!(!crate::session::is_running("sess-end")); + assert_eq!(crate::call(&p, &mut host, &command("h"), false).unwrap(), json!({"hello": "world"})); + // One that stops answering is stopped. + let spec = + crate::process::Spec { session: true, ..crate::process::Spec::new(&testing::child_run("tests::process_child"), &dir).unwrap() }; + let t0 = std::time::Instant::now(); + let e = crate::session::command("sess-spin", &spec, &mut host, &command("s"), std::time::Duration::from_secs(2)).unwrap_err(); + assert!(e.contains("too long") && t0.elapsed() < std::time::Duration::from_secs(20), "{e}"); + assert!(!crate::session::is_running("sess-spin")); + // Updated, the running program is the old version's: stopped. Turned off, it stops too. + let p = crate::register(p); + assert!(!crate::session::is_running("sess-end")); + crate::call(&p, &mut host, &command("h"), false).unwrap(); + assert!(crate::session::is_running("sess-end")); + crate::set_enabled("sess-end", false).unwrap(); + assert!(!crate::session::is_running("sess-end")); + // Removed, it stops as well. + crate::set_enabled("sess-end", true).unwrap(); + let p = crate::get("sess-end").unwrap(); + crate::call(&p, &mut host, &command("h"), false).unwrap(); + assert!(crate::session::is_running("sess-end")); + assert!(crate::unload("sess-end")); + assert!(!crate::session::is_running("sess-end") && crate::get("sess-end").is_none()); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A program that exits while something it left running keeps writing into its output, faster +/// than it's read, is still ended poll after poll, with what it left running. +#[cfg(unix)] +#[test] +fn an_exited_session_whose_output_keeps_flowing_is_ended() { + let (p, dir) = session_plugin("sess-flow"); + let mut host = Recorder::default(); + let left = crate::call(&p, &mut host, &command("r"), false).unwrap(); + let pid = rustix::process::Pid::from_raw(left.as_i64().unwrap() as i32).unwrap(); + // Polled as frames would be, for a while at most: until it has exited there's nothing to + // count, and a poll with nothing new returns at once. + let mut ended = None; + let t0 = std::time::Instant::now(); + while t0.elapsed() < std::time::Duration::from_secs(30) { + if let Some(e) = crate::session::poll("sess-flow", &mut host).ended { + ended = Some(e); + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let ended = ended.expect("the session didn't end"); + assert!(ended.unread, "{ended:?}"); + assert!(!crate::session::is_running("sess-flow")); + let t0 = std::time::Instant::now(); + while !gone(pid) && t0.elapsed() < std::time::Duration::from_secs(10) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone(pid), "{pid:?} still runs"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn process_manifests_are_checked() { let base = |extra: Value| { @@ -1164,6 +1364,9 @@ fn process_manifests_are_checked() { assert!(base(json!({"runtime": "process", "run": {"windos": ["x"]}})).unwrap_err().contains("names no program")); assert!(base(json!({"runtime": "process", "run": vec!["a"; 65]})).is_err()); assert!(base(json!({"run": ["python3"]})).unwrap_err().contains("process plugins")); + // Only a process plugin keeps a program running. + assert!(base(json!({"runtime": "process", "run": ["x"], "session": true})).unwrap().session); + assert!(base(json!({"session": true})).unwrap_err().contains("process plugins")); assert!(base(json!({"runtime": "native"})).is_err()); // A program by a relative path stays in the plugin's folder. assert!(base(json!({"runtime": "process", "run": ["bin/tool"]})).is_ok()); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 0fdcf54a97..c235222e9c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1100,6 +1100,9 @@ pub struct RustGisApp { /// to give its wake callback yet: `logic` installs the wake as soon as a context exists. #[cfg(not(target_arch = "wasm32"))] relay_wake_pending: bool, + /// Whether session plugins' programs ask for a repaint when they send a line. + #[cfg(not(target_arch = "wasm32"))] + plugin_wake_set: bool, /// Extra font faces installed with the interface fonts: (name, face, script). The current /// language's face comes right after the interface font (before it for Arabic script), so its /// glyph shapes win; the others are last-resort fallbacks. Faces are shared, so reinstalling @@ -1304,6 +1307,8 @@ impl RustGisApp { frame_ctx: None, #[cfg(not(target_arch = "wasm32"))] relay_wake_pending: false, + #[cfg(not(target_arch = "wasm32"))] + plugin_wake_set: false, extra_fonts: Vec::new(), installed_region: None, fonts_dirty: false, @@ -4996,6 +5001,21 @@ impl RustGisApp { } } relayshot::drive(self, ctx); + // Session plugins' programs: what they asked, and what changed on the map for them. + if !self.plugin_wake_set { + let ctx = ctx.clone(); + rustgis_engine::plugins::session::set_wake(std::sync::Arc::new(move || ctx.request_repaint())); + self.plugin_wake_set = true; + } + let state = self.session.state(); + if self.session.poll_plugin_sessions() > 0 && self.session.state() != state { + self.map.cancel_label_drag(); + } + // A program that exits without a last line (something it left running holds its output + // open) wakes nothing: while one runs, the app looks at least once a second. + if !rustgis_engine::plugins::session::running().is_empty() { + ctx.request_repaint_after(std::time::Duration::from_secs(1)); + } // A script an agent wrote to the file bridge. if let Some(bridge) = self.file_bridge.clone() { if !bridge.has_wake() { diff --git a/crates/ui-egui/src/sessionlog.rs b/crates/ui-egui/src/sessionlog.rs index c9923f2850..17602121d7 100644 --- a/crates/ui-egui/src/sessionlog.rs +++ b/crates/ui-egui/src/sessionlog.rs @@ -527,7 +527,7 @@ mod tests { save(&mut app); let (name, text) = got.borrow_mut().take().expect("downloaded"); assert_eq!(name, "session-log.csv"); - assert!(text.starts_with("time_utc,at,command,ok,millis,error,params,project\n"), "{text}"); + assert!(text.starts_with("time_utc,at,command,ok,millis,error,params,plugin,project\n"), "{text}"); assert!(text.contains("\"map.new\",true"), "{text}"); let dropped = |app: &RustGisApp| { app.session.notices.iter().any(|n| n.text.contains("session-log.csv: not in it: 4 oldest entries were dropped past")) diff --git a/docs/plugins.md b/docs/plugins.md index 53b5446f15..b07ecb8c99 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -436,7 +436,7 @@ Inside a request, `host::call(op, params)` sends one of these to RustGIS: | `host.info` | `{rustgis, abi, plugin, permissions}`: version, ABI, plugin id and permissions | | `host.layers` | `[{id, name, raster, visible, geometry, features, selected, active, fields: [{name, type}]}]` | | `host.features` `{layer, selected?, limit?}` | the layer's features as GeoJSON; each feature has its `row` | -| `host.view` | `{center: [lon, lat], scale, projection}` | +| `host.view` | `{center: [lon, lat], scale, projection, bearing, pitch}` (bearing and pitch in degrees, 0 for a flat north-up map) | | `host.notify` `{text, error?}` | shows a message in the status bar | | `host.addLayer` `{name, geojson}` | adds a layer (`edit`) | | `host.fetch` `{url}` | `{url, text, json?}`: the document at a web address (`network`; see [Fetching a document](#fetching-a-document)) | @@ -693,6 +693,64 @@ except Exception as e: print(json.dumps(result), flush=True) ``` +### Session plugins + +A process plugin with `"session": true` keeps one program running between its commands, so it +can ask RustGIS for things whenever it likes and hear when the map changes: a view of the map in +a window of its own (a 3D globe linked with it) is one. The program talks to RustGIS only over +its standard input and output: no port, no token, nothing another program can join. + +```json +"runtime": "process", +"session": true, +"run": ["bin/globe"] +``` + +- RustGIS starts the program the first time one of the plugin's commands runs and writes a first + line `{"session": {"abi": 1, "plugin": "", "rustgis": "", "lang": ""}}`. +- Each command comes as `{"command": request, "id": n}`, where `request` is what a per-call + program gets. The program answers `{"ok": value, "id": n}` or `{"error": "text", "id": n}`, + within 2 minutes. +- At any time, during a command or between commands, the program can send a host request + `{"call": {"op": "...", "params": {...}}, "id": "any"}`. RustGIS answers + `{"ok": value, "id": "any"}` or `{"error": "text", "id": "any"}`; the `id` is optional and comes + back as given. Requests go through the same checks as a command's, within the plugin's + permissions. Between commands, each request that can change the project is an undo step of its + own, named after the plugin, and an entry in the session log; `host.fetch` is refused: it would make the app wait, and the + program can fetch for itself. Whether a request is part of a command depends on when RustGIS handles + it, not when the program sent it: one handled before the command starts (by a poll, or as the + command is about to run) is answered as between commands; one handled while the command runs, + until the program answers, is part of that command (its undo step), even if it was sent just + before. Sending a request meant to stand alone after answering the command, and + waiting for its reply, makes that likely but not certain: if another of its commands starts + before RustGIS handles the request, the request is part of that command. The desktop app answers requests as they come; `rustgis-cli` and + the Python package only while one of the plugin's commands runs (or as the next one starts); + `rustgis-cli` ends the program after its last command, and what it asks after that isn't + answered. +- When the map changes, the desktop app sends `{"event": "view", "view": {center, scale, + projection, bearing, pitch}}` (the view, as `host.view` answers it), `{"event": "layers"}` (layers added, + removed, reordered, made active, renamed, shown or hidden, restyled, made more or less transparent, their + fields or features changed, and web tile layers changed), `{"event": "selection"}` or `{"event": "project"}` (another project + opened). Ask for what changed with host requests. Changes the program made itself are told too: a view it set comes back as + a `view` event, so compare before following it. An event can arrive while the program waits + for an answer: match answers by their `id`. While the program is behind in reading its input, + events wait, and only the latest of each kind is sent. +- `log` and `progress` lines work as for a per-call program. +- The program RustGIS started is the session: keep it running, rather than starting another + program and exiting. RustGIS stops it, with what it started, when the plugin is turned off, + removed or updated and when the desktop app closes. Its input closing means it is being + stopped (RustGIS closing or gone, however it ended; the plugin turned off, removed or updated; + the program failed or stopped answering): the program should then exit, and stop what it + started. A program may also + exit by itself (its window closed); the plugin's next command starts it again. One that fails + or stops answering a command is stopped, and the error says so. +- On Linux and macOS, while RustGIS runs, what a program left running when it exited is stopped + with it. Windows can't find what an exited program started, so there the program must stop it + itself, as every program must once RustGIS has gone. +- The plugin's tools still start the program once per run, with the request as its first line. +- RustGIS versions before sessions run the program once per command instead, so set + `minRustgisVersion` to the first version with them. + ### Python environments A Python plugin can have RustGIS make its environment, so its users don't install Python or diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index fcff74bc06..572e4f0507 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -132,7 +132,9 @@ isn't itself an entry in the log, as a saved file is). **Clear Log** empties it that waited for its downloads, when its last run started), `command`, `params`, `ok`, `error` when it failed, `millis`, how long it took, `project`, the name of the project open when it ran (the one it opened, for a command that opens one), and `partial`, true when its parameters were - cut short or had secrets taken out. + cut short or had secrets taken out, and, for what a session plugin's program asked + for between the plugin's commands, `plugin`, that plugin's id (left out of `project.log` and + JSON Lines for other entries; an empty cell in CSV). - **For the session only:** the log is kept in memory, across the projects opened or made in the session, until RustGIS closes or the log is cleared. A saved project, its recovery copy, a package or a template carries none of it, so sharing a project shares no log. Reading, writing