diff --git a/AGENTS.md b/AGENTS.md index 800c932610..06da9da7dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,30 @@ Before picking work, read [ROADMAP.md](ROADMAP.md) (check Underway) and what this version couldn't read; a reader checks the version it is given ([saved files](docs/maintaining.md#saved-files-and-settings)). +## Never hold the window +On the desktop, a command whose work grows with its input (features, cells, tiles, files, +requests) runs off the UI thread, with progress and Cancel (#1562). The browser has one thread and +is out of scope. +- Use an existing background path: tools go through `gp::tool_start`; exports through + `exports::runs_in_background`; commands that make a raster through + `workflows::runs_in_background`; models and scripts through the copy (`workflows::ahead_on_copy`), + with the command on `ON_A_COPY` when running it twice changes nothing outside the session. +- The work looks at Cancel between items of its long loops (`control::check`, `Pace`, + `progress::check`) and counts its progress. Where progress can't be measured, a busy indicator + without a count is fine; Cancel is not optional. +- Work that splits into independent parts runs on several threads where practical: the same + output as one thread, bit for bit; one thread below a size threshold and in the browser; the + shared helpers (`parallel::for_each_row`, `try_map`, `try_map_on`); the thread count from + `rustgis_data::threads` (it follows the user's setting), never `available_parallelism` or a + fixed number; scratch memory within `memory::thread_scratch`; a thread the OS refuses costs + speed, not the run (`Builder::spawn_scoped`). +- A command that must run on the UI thread says why in its doc comment. +- `crates/engine/src/tests.rs::long_commands_stop_when_cancelled_while_they_run` cancels each + background command as its work starts and fails one that never looks; its lists must match the + background registries, so a new background command is added there. Passing shows the work looks + at Cancel, not that it looks often: a long step without a check (one union, one read) is said in + its doc comment. + ## Architecture | Layer | Crate | Job | |---|---|---| diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48b100f2b8..21d37a8b4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,8 @@ something on the roadmap (or not on it) interests you, propose it and open a pul - what you plan to build: the commands it adds, where it sits in the ribbon (following the [ribbon guidelines](docs/contributing.md#ribbon-guidelines)), which formats or tools are involved; - which crates it touches and whether it adds a dependency; + - for work that grows with the data, how it runs off the UI thread with progress and Cancel + (see [Never hold the window](AGENTS.md#never-hold-the-window)); - how you will test it. Put it in an issue with the diff --git a/crates/analysis/src/control.rs b/crates/analysis/src/control.rs index 3f0969b737..eedd7dd5c4 100644 --- a/crates/analysis/src/control.rs +++ b/crates/analysis/src/control.rs @@ -21,6 +21,10 @@ pub struct Control { total: AtomicU64, /// Someone can stop the run while it goes ([`Control::stoppable`]). stoppable: bool, + /// Cancelled as soon as its run starts ([`Control::cancelled_once_started`]). + cancel_at_start: bool, + /// How often the work looked at the flag ([`Control::looked`]). + looked: AtomicU64, } impl Control { @@ -34,6 +38,37 @@ impl Control { Arc::new(Control { stoppable: !cfg!(target_arch = "wasm32"), ..Control::default() }) } + /// A control that [`crate::run_under`] cancels as soon as the work has started, past the + /// check before it: the work's own looks at the flag ([`Control::looked`]) decide whether it + /// stops. For tests that a long command checks for Cancel (#1562), with no timing. + pub fn cancelled_once_started() -> Arc { + Arc::new(Control { cancel_at_start: true, ..Control::default() }) + } + + /// How often the work under a [`Control::cancelled_once_started`] control looked at its flag: + /// the tool's checks, and the data writers' (not those of whoever runs it, before and after). + /// Always 0 for other controls. + pub fn looked(&self) -> u64 { + self.looked.load(Ordering::Relaxed) + } + + /// Cancels a [`Control::cancelled_once_started`] control: called by [`crate::run_under`] once + /// the work is about to start. + pub(crate) fn cancel_if_started(&self) { + if self.cancel_at_start { + self.cancel(); + } + } + + /// The flag, looked at by the work: counted for a test's control only, so an ordinary run's + /// threads don't share a counter. + fn looked_at(&self) -> bool { + if self.cancel_at_start { + self.looked.fetch_add(1, Ordering::Relaxed); + } + self.is_cancelled() + } + /// Asks the tool to stop at its next check. pub fn cancel(&self) { self.cancel.store(true, Ordering::Relaxed); @@ -55,7 +90,7 @@ impl Control { /// their steps count here, and they stop when the run is cancelled. impl rustgis_data::progress::Probe for Control { fn cancelled(&self) -> bool { - self.is_cancelled() + self.looked_at() } fn begin(&self, total: u64) { self.done.store(0, Ordering::Relaxed); @@ -75,7 +110,7 @@ struct Progress { impl rustgis_data::progress::Probe for Progress { fn cancelled(&self) -> bool { - self.control.as_ref().is_some_and(|c| c.is_cancelled()) || rustgis_data::source::remote::check_still(&self.still).is_err() + self.control.as_ref().is_some_and(|c| c.looked_at()) || rustgis_data::source::remote::check_still(&self.still).is_err() } fn begin(&self, total: u64) { if let Some(control) = &self.control { @@ -145,7 +180,7 @@ impl Drop for Installed { /// Whether the run on this thread was asked to stop. pub(crate) fn cancelled() -> bool { - CURRENT.with(|c| c.borrow().as_ref().is_some_and(|c| c.is_cancelled())) + CURRENT.with(|c| c.borrow().as_ref().is_some_and(|c| c.looked_at())) || rustgis_data::source::remote::check_still(&rustgis_data::source::remote::current_still()).is_err() } @@ -224,6 +259,26 @@ pub(crate) fn advance(n: usize) { mod tests { use super::*; + /// A control cancelled once its run starts passes the check before the work, is cancelled for + /// the work's own looks, and counts them: work that looks stops, work that doesn't is seen. + #[test] + fn a_control_cancelled_once_started_counts_the_works_looks() { + let control = Control::cancelled_once_started(); + let looks = crate::run_under("Looks", &control, || { + check()?; + Ok::<_, String>(()) + }); + assert!(looks.unwrap_err().contains("Looks was cancelled")); + assert_eq!(control.looked(), 1); + let blind = Control::cancelled_once_started(); + let res = crate::run_under("Blind", &blind, || Ok::<_, String>(())); + assert!(res.is_err(), "cancelled all the same, by whoever ran it"); + assert_eq!(blind.looked(), 0, "but the work never looked"); + // An ordinary control isn't cancelled by starting. + let plain = Control::new(); + assert!(crate::run_under("Plain", &plain, || Ok::<_, String>(())).is_ok()); + } + /// Revocation during real file construction must reach the data writer's final rename, /// even though the person did not press Cancel. Its prior file and the next run survive. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index ba092a1a4a..92a0b7a825 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -696,6 +696,7 @@ pub fn run_under(name: &str, control: &std::sync::Arc, work let _data = control::install_progress(Some(control.clone())); // No count until the work starts one: a control used before doesn't show that run's. control::begin(0); + control.cancel_if_started(); let res = work(); rustgis_data::source::remote::check_still(&still)?; if !control.is_cancelled() { diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 253ee5f9e1..bda480871d 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1576,25 +1576,31 @@ fn part(g: &Geometry, mask: &MultiPolygon, inside: bool) -> Option Result { let ds = input(inp, p, "input")?; let clip_ds = input(inp, p, "clip")?; if clip_ds.geometry_type != GeometryType::Polygon { return Err(format!("'{}' is not a polygon layer", clip_ds.name)); } - let masks: Vec<_> = rows(inp, p, "clip", clip_ds) - .into_iter() - .filter_map(|r| clip_ds.features.get(r)?.geometry.as_ref().and_then(to_multipolygon)) - .map(|m| { - let ext = merc_extent(&m); - // Union even one polygon: collapsed rings have no filled area. Keep the original - // bounds too: rounding an edge must not hide a line the boolean overlay clips to it. - let mask = union_all(m.0.iter()); - let ext = ext.union(&merc_extent(&mask)); - (mask, ext) - }) - .filter(|(m, _)| !m.0.is_empty()) - .collect(); + let mut masks = Vec::new(); + let mask_rows = rows(inp, p, "clip", clip_ds); + crate::control::begin(mask_rows.len()); + for (done, r) in mask_rows.into_iter().enumerate() { + // Each mask is a union: Cancel is looked at, and progress counted, before each. + crate::control::check()?; + crate::control::reached(done); + let Some(m) = clip_ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon) else { continue }; + let ext = merc_extent(&m); + // Union even one polygon: collapsed rings have no filled area. Keep the original + // bounds too: rounding an edge must not hide a line the boolean overlay clips to it. + let mask = union_all(m.0.iter()); + let ext = ext.union(&merc_extent(&mask)); + if !mask.0.is_empty() { + masks.push((mask, ext)); + } + } let index = ExtentIndex::new(masks.iter().map(|(_, ext)| *ext)); // The masks reaching each pole: a point at one is in them at any longitude (#1033). let at_poles = AtPoles::new(masks.iter().map(|(m, _)| plane_poles(m))); @@ -1603,7 +1609,11 @@ fn clip(inp: &dyn Inputs, p: &Json) -> Result { const MAX_CACHED_MASKS: usize = 8; let mut cached_masks: std::collections::VecDeque<(Vec, MultiPolygon)> = std::collections::VecDeque::new(); let mut out = copy_schema(ds, out_name(p, format!("{}_Clip", ds.name)), ds.geometry_type); - for r in rows(inp, p, "input", ds) { + let input_rows = rows(inp, p, "input", ds); + crate::control::begin(input_rows.len()); + for (done, r) in input_rows.into_iter().enumerate() { + crate::control::check()?; + crate::control::reached(done); let Some(f) = ds.features.get(r) else { continue }; let Some(g) = f.geometry.as_ref() else { continue }; let ext = projected_extent(g); @@ -2145,6 +2155,9 @@ fn split(inp: &dyn Inputs, p: &Json) -> Result { /// of the (selected) polygons holding it, with the value, `count` (the polygons merged) and the /// statistics asked for, as `_`, over exactly the records merged: a record without a /// polygon is left out of both. +/// Dissolve: Cancel is looked at feature by feature while the groups are made, and group by group +/// while each is merged. A group's merge is one union of all its polygons, which can't be stopped +/// once it has started: with no field, the whole layer is one group (#1562). fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { let ds = input(inp, p, "input")?; if ds.geometry_type != GeometryType::Polygon { @@ -2156,7 +2169,11 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { // Each group's key, polygons and records, in the order first met; found by its key's hash. let mut groups: Vec<(Value, Vec>, Vec)> = Vec::new(); let mut index: HashMap, usize> = HashMap::new(); - for r in each_once(rows(inp, p, "input", ds)) { + let input_rows = each_once(rows(inp, p, "input", ds)); + crate::control::begin(input_rows.len()); + for (done, r) in input_rows.into_iter().enumerate() { + crate::control::check()?; + crate::control::reached(done); let Some(mp) = ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon) else { continue }; let value = fi.map_or(&Value::Null, |fi| ds.value(r, fi)); // A NaN equals no value, so each one stays a group of its own; −0 is 0, as it compares. @@ -2183,7 +2200,11 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { fields.extend(added); let mut out = Dataset::new(out_name(p, format!("{}_Dissolve", ds.name)), GeometryType::Polygon, fields); let mut rounded = Rounded::new(requests.len()); - for (key, parts, mut records) in groups { + // The unions are the work: counted, and Cancel looked at, group by group. + crate::control::begin(groups.len()); + for (done, (key, parts, mut records)) in groups.into_iter().enumerate() { + crate::control::check()?; + crate::control::reached(done); let n = parts.len() as i64; let u = union_all(parts.iter().flat_map(|m| m.0.iter())); let mut attrs = if fi.is_some() { vec![key, Value::Int(n)] } else { vec![Value::Int(n)] }; @@ -5665,7 +5686,11 @@ fn spatial_join(inp: &dyn Inputs, p: &Json) -> Result { let join_rows = rows(inp, p, "join", join); let mut join_geoms: Vec> = Vec::with_capacity(join_rows.len()); - for &jr in &join_rows { + // The join features converted: Cancel looked at, and progress counted, feature by feature. + crate::control::begin(join_rows.len()); + for (done, &jr) in join_rows.iter().enumerate() { + crate::control::check()?; + crate::control::reached(done); if let Some(f) = join.features.get(jr) && let Some(g) = f.geometry.as_ref() { @@ -5710,7 +5735,10 @@ fn spatial_join(inp: &dyn Inputs, p: &Json) -> Result { let mut written_features = 0usize; let mut cand_buf = Vec::new(); - for tr in target_rows { + crate::control::begin(target_rows.len()); + for (done, tr) in target_rows.into_iter().enumerate() { + crate::control::check()?; + crate::control::reached(done); let Some(tf) = target.features.get(tr) else { continue }; // The matches, in the order they're joined, with their distances when measured. let mut matches: Vec<(usize, Option)> = Vec::new(); diff --git a/crates/engine/src/cmd/cube.rs b/crates/engine/src/cmd/cube.rs index b4524f75f5..c765c41fc1 100644 --- a/crates/engine/src/cmd/cube.rs +++ b/crates/engine/src/cmd/cube.rs @@ -766,6 +766,8 @@ impl TrendRun { let slopes = n.saturating_mul(n).saturating_mul(4).max(1); let threads = rustgis_data::threads::max().min(1 + rustgis_analysis::scratch_budget() / slopes); let part = cells.div_ceil(threads).max(1); + // No more threads than there are parts (10 cells on 8 threads make 5). + let threads = cells.div_ceil(part); // The parts are taken from a list, by the threads started and by this one, so a // thread the OS refuses leaves no part undone (and costs speed, not the run). let parts = std::sync::Mutex::new(block.chunks(part * n).zip(out.chunks_mut(part * OUTPUTS)).enumerate().collect::>()); diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index 89d2555314..ed45a24e3d 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -37,7 +37,7 @@ type Start = fn(&Session, &Value) -> Result; /// The exports the app writes on another thread ([`export_start`]), each with what it is called /// (in its progress, and a cancelled run's error) and what it reads of the session. -const IN_BACKGROUND: &[(&str, &str, Start)] = &[ +pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ ("share.exportLayer", "Export Features", super::share::layer_work), ("share.exportRaster", "Export Raster", super::raster::export_work), ("share.exportMap", "Export Map", super::share::map_work), diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 0a1c894571..a1b248512b 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -181,7 +181,7 @@ type Start = fn(&Session, &Value) -> Result; /// what it is called (in its progress, and a cancelled run's error) and what it reads of the /// session: their work grows with the raster, and most of their passes spread it over the cores /// from that thread. Run again with `{"ran": id}`, each adds the raster made (#1291 step 3). -const IN_BACKGROUND: &[(&str, &str, Start)] = &[ +pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ ("analysis.viewshed", "Viewshed", viewshed_work), ("analysis.interpolate", "Interpolate (IDW)", interpolate_work), ("analysis.suitability", "Suitability Modeler", suitability_work), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 378e719938..4ec48ae7de 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -35326,3 +35326,129 @@ fn the_thread_setting_is_reserved() { assert!(spec.effects.contains(crate::policy::Effect::Reserved)); assert!(!spec.mutates, "no undo step"); } + +/// A long command stops when it is cancelled while it runs (#1562): every command the app runs +/// on another thread (its exports and the commands that make a raster) and the long-running tools +/// look at the Cancel flag in their work, not only before it starts. Their control is cancelled +/// as soon as the work starts ([`rustgis_analysis::control::Control::cancelled_once_started`]), +/// so no timing is involved: a run that never looks finishes and is caught by its count of looks. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn long_commands_stop_when_cancelled_while_they_run() { + use rustgis_analysis::control::Control; + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let dir = Dir(std::env::temp_dir().join(format!("rustgis-cancel-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(&dir.0); + std::fs::create_dir_all(&dir.0).unwrap(); + let out = |f: &str| dir.0.join(f).display().to_string(); + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "dem.tif", "data": dem_b64()})).unwrap(); + let dem = s.map().layers.iter().find(|l| l.is_raster()).unwrap().name.clone(); + // What some of them read: a later DEM, a slope, training squares, an imagery workspace, a + // two-class scene with its training polygons, and a layout. + s.run("raster.function", &json!({"layer": dem, "function": "bandMath", "expression": "b1 + 10", "name": "dem_later"})).unwrap(); + s.run("raster.function", &json!({"layer": dem, "function": "slope", "name": "dem_slope"})).unwrap(); + s.run("layer.template", &json!({"template": "polygons", "name": "Squares"})).unwrap(); + for (name, x) in [("low", 10.002), ("high", 10.018)] { + let square = [[x, 45.018], [x + 0.004, 45.018], [x + 0.004, 45.022], [x, 45.022], [x, 45.018]]; + let geometry = json!({"type": "Polygon", "coordinates": [square]}); + s.run("edit.addFeature", &json!({"layer": "Squares", "geometry": geometry, "attributes": {"name": name}})).unwrap(); + } + s.run("imagery.workspace", &json!({"kind": "ortho", "name": "Valley", "images": [dem, "dem_later"]})).unwrap(); + s.run("imagery.workspaceOptions", &json!({"name": "Valley", "cellSize": 0.002})).unwrap(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + let training = json!({"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"class": "water"}, "geometry": {"type": "Polygon", "coordinates": [[[10.002, 45.002], [10.015, 45.002], [10.015, 45.035], [10.002, 45.035], [10.002, 45.002]]]}}, + {"type": "Feature", "properties": {"class": "field"}, "geometry": {"type": "Polygon", "coordinates": [[[10.025, 45.002], [10.038, 45.002], [10.038, 45.035], [10.025, 45.035], [10.025, 45.002]]]}}]}); + s.run("layer.addDataText", &json!({"fileName": "training.geojson", "text": training.to_string()})).unwrap(); + s.run("layout.new", &json!({"page": "a4", "orientation": "landscape", "template": "blank"})).unwrap(); + s.run("layout.addFrame", &json!({"x": 20, "y": 20, "w": 150, "h": 150})).unwrap(); + + // What didn't stop, and how often it looked. + let mut kept_going = Vec::new(); + let mut note = |what: &str, result: Result<(), String>, control: &Control| { + let stopped = result.as_ref().is_err_and(|e| e.contains("cancelled")); + if !stopped || control.looked() == 0 { + kept_going.push(format!("{what}: {result:?}, looked {} times", control.looked())); + } + }; + + let rasters = [ + ("analysis.viewshed", json!({"layer": dem, "lon": 10.0055, "lat": 45.0395, "height": 2})), + ("analysis.interpolate", json!({"layer": "Cities", "field": "pop_max"})), + ( + "analysis.suitability", + json!({"criteria": [ + {"layer": dem, "ranges": [[0, 250, 1], [250, 1000, 10]], "weight": 2}, + {"layer": "dem_slope", "ranges": [[0, 5, 10], [5, 90, 1]], "weight": 1}]}), + ), + ("raster.function", json!({"layer": dem, "function": "hillshade"})), + ("raster.functionChain", json!({"layer": dem, "steps": [{"function": "slope"}, {"function": "threshold", "value": 10}]})), + ("raster.classify", json!({"layer": dem, "classes": 3})), + ("raster.classifySupervised", json!({"layer": dem, "training": "Squares", "classField": "name"})), + ("raster.change", json!({"before": dem, "after": "dem_later", "method": "difference"})), + ("raster.index", json!({"layer": dem, "index": "ndvi", "nir": 1, "red": 1})), + ("raster.process", json!({"layer": dem, "operation": "resample", "factor": 4})), + ("imagery.mosaic", json!({"name": "Valley"})), + ( + "dl.classify", + json!({"layer": "scene", "training": "training", "classField": "class", "bands": [1, 2], "epochs": 30, "model": "landcover"}), + ), + ]; + // Every command that runs in the background is here, and only those. + fn listed<'a>(l: &[(&'a str, Value)]) -> std::collections::BTreeSet<&'a str> { + l.iter().map(|(c, _)| *c).collect() + } + let registered = crate::cmd::workflows::IN_BACKGROUND.iter().map(|(c, ..)| *c).collect::>(); + assert_eq!(listed(&rasters), registered, "add a new background raster command here"); + for (command, p) in &rasters { + let run = crate::cmd::workflows::raster_start(&mut s, command, p).unwrap_or_else(|e| panic!("{command}: {e}")); + let id = run.id; + let control = Control::cancelled_once_started(); + note(command, run.run(&control).map(|_| ()), &control); + crate::cmd::workflows::raster_abandon(&mut s, id); + } + + let exports = [ + ("share.exportLayer", json!({"layer": "Cities", "path": out("cities.geojson")})), + ("share.exportRaster", json!({"layer": dem, "path": out("dem.tif")})), + ("share.exportMap", json!({"path": out("map.png"), "width": 320, "height": 200})), + ("layout.export", json!({"path": out("layout.pdf"), "dpi": 72})), + ]; + let registered = crate::cmd::exports::IN_BACKGROUND.iter().map(|(c, ..)| *c).collect::>(); + assert_eq!(listed(&exports), registered, "add a new background export here"); + for (command, p) in &exports { + let run = crate::cmd::exports::export_start(&mut s, command, p).unwrap_or_else(|e| panic!("{command}: {e}")); + let control = Control::cancelled_once_started(); + note(command, run.run(&control).map(|_| ()), &control); + } + + // The tools whose work grows past a second or two on real data (and whose work caps follow + // the threads): run in the background as the Analysis pane runs them. + let tools = [ + ("buffer", json!({"input": "Cities", "distance": 100})), + ("dissolve", json!({"input": "Countries"})), + ("spatialJoin", json!({"target": "Cities", "join": "Countries"})), + ("clip", json!({"input": "Rivers", "clip": "Countries"})), + ("kernelDensity", json!({"input": "Cities", "crs": "EPSG:3857"})), + ("kriging", json!({"input": "Cities", "field": "pop_max"})), + ]; + for (tool, params) in &tools { + // The parameters are good: a run that isn't cancelled succeeds. + s.run("gp.run", &json!({"tool": tool, "params": params})).unwrap_or_else(|e| panic!("{tool}: {e}")); + let run = crate::cmd::gp::tool_start(&mut s, tool, params).unwrap_or_else(|e| panic!("{tool}: {e}")); + let id = run.id; + let control = Control::cancelled_once_started(); + let done = run.run(&control); + // The result is kept, then `gp.finish` applies it (or says why it can't), as in the app. + let finished = crate::cmd::gp::tool_complete(&mut s, id, done).and_then(|()| s.run("gp.finish", &json!({"id": id})).map(|_| ())); + note(tool, finished, &control); + } + assert!(kept_going.is_empty(), "these ran on after Cancel:\n{}", kept_going.join("\n")); +}