From 0b5b2c8fa99a04adfc72ce334f8fe1dc8b548f69 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 09:46:30 +0700 Subject: [PATCH 01/56] feat(engine): batch geoprocessing runner (gp.batch) - Implements gp.batch in crates/engine/src/cmd/gp.rs to execute any tool over multiple project layers, file paths, wildcard *, or a directory folder - Supports dynamic parameter binding, output pattern templating ({input}_{tool}), and cancellation via Control - Provides isolated per-item fault tolerance adhering to Never Crash, Never Silent - Adds integration unit tests covering multi-layer, wildcard, fault tolerance, folder scanning, and validation - Adds documentation in docs/user-guide/analysis.md --- crates/engine/src/cmd/gp.rs | 321 +++++++++++++++++++++++++++++++++++- docs/user-guide/analysis.md | 158 ++++++------------ 2 files changed, 370 insertions(+), 109 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 67e7059654..0217681356 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -11,6 +11,9 @@ use crate::{CmdResult, CommandSpec as C, GpRecord, Session}; pub fn commands() -> Vec { vec![ C::new("gp.run", "Run Tool", "Analyze › Geoprocessing", run).when(need_layers).params(r#"{"tool": tool id, "params": {...} (a layer by name; a standalone table by name or id)}"#), + C::new("gp.batch", "Batch Run", "Analyze › Geoprocessing", batch).params( + r#"{"tool": tool id, "inputs"?: [layer names or file paths] | "*", "folder"?: folder path, "param"?: input parameter name (default "input"), "outputPattern"?: text (default "{input}_{tool}"), "params"?: {...}}"#, + ), C::new("gp.spatialJoin", "Spatial Join", "Analyze › Overlay", spatial_join) .when(need_layers) .params( @@ -746,12 +749,29 @@ fn inputs_of<'a>(snap: Snapshot<'a>, spec: Option<&rustgis_analysis::ToolSpec>, let mut full = full; #[cfg(not(target_arch = "wasm32"))] if let Some(spec) = spec.filter(|t| rustgis_analysis::external_source(t).is_none()) { - for parameter in spec.params.iter().filter(|p| p.kind == rustgis_analysis::ParamKind::VectorLayers) { - for name in rustgis_analysis::layer_list(params.get(parameter.name).unwrap_or(&Value::Null))? { - if snap.map.layer_by_name(name).is_none() && !full.contains_key(name) { - let (dataset, report) = rustgis_data::io::read_path(std::path::Path::new(name)).map_err(|e| format!("{name}: {e}"))?; + for parameter in spec.params.iter() { + let names: Vec = match parameter.kind { + rustgis_analysis::ParamKind::VectorLayers => { + rustgis_analysis::layer_list(params.get(parameter.name).unwrap_or(&Value::Null)) + .unwrap_or_default() + .into_iter() + .map(String::from) + .collect() + } + rustgis_analysis::ParamKind::Layer + | rustgis_analysis::ParamKind::PolygonLayer + | rustgis_analysis::ParamKind::PointLayer + | rustgis_analysis::ParamKind::VectorLayer => { + params.get(parameter.name).and_then(Value::as_str).into_iter().map(String::from).collect() + } + _ => Vec::new(), + }; + for name in names { + let path = std::path::Path::new(&name); + if snap.map.layer_by_name(&name).is_none() && !full.contains_key(&name) && path.is_file() { + let (dataset, report) = rustgis_data::io::read_path(path).map_err(|e| format!("{name}: {e}"))?; messages.extend(report.messages.into_iter().map(|message| format!("{name}: {message}"))); - full.insert(name.to_string(), dataset); + full.insert(name.clone(), dataset); } } } @@ -949,6 +969,159 @@ fn run(s: &mut Session, p: &Value) -> CmdResult { run_tool(s, &tool, ¶ms) } +/// Runs a geoprocessing tool in batch across multiple layers or files. +fn batch(s: &mut Session, p: &Value) -> CmdResult { + let tool_name = ps(p, "tool").ok_or("tool is required")?; + let tool_id = tool_id(tool_name).ok_or_else(|| format!("no tool '{tool_name}'"))?; + let spec = rustgis_analysis::tool(tool_id).ok_or_else(|| format!("no tool '{tool_name}'"))?; + + // Identify which parameter to bind the input to. + let batch_param = ps(p, "param").or_else(|| ps(p, "batch_param")).unwrap_or_else(|| { + spec.params + .iter() + .find(|param| { + matches!( + param.kind, + rustgis_analysis::ParamKind::Layer + | rustgis_analysis::ParamKind::PolygonLayer + | rustgis_analysis::ParamKind::PointLayer + | rustgis_analysis::ParamKind::VectorLayer + | rustgis_analysis::ParamKind::RasterLayer + ) + }) + .map(|param| param.name) + .unwrap_or("input") + }); + + let pattern = ps(p, "outputPattern").or_else(|| ps(p, "output_pattern")).unwrap_or("{input}_{tool}"); + let base_params = p.get("params").filter(|v| v.is_object()).cloned().unwrap_or(json!({})); + + // Collect inputs: from "inputs" array, wildcard "*", or "folder" directory path. + let mut targets: Vec = Vec::new(); + + let is_wildcard = p.get("inputs").and_then(Value::as_str).is_some_and(|s| s == "*") + || p.get("inputs").and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")); + + if is_wildcard { + for layer in &s.map().layers { + if !layer.is_raster() { + targets.push(layer.name.clone()); + } + } + } else if let Some(arr) = p.get("inputs").and_then(Value::as_array) { + for item in arr { + if let Some(str_val) = item.as_str() { + let trimmed = str_val.trim(); + if !trimmed.is_empty() { + targets.push(trimmed.to_string()); + } + } + } + } + + #[cfg(not(target_arch = "wasm32"))] + if targets.is_empty() + && let Some(folder) = ps(p, "folder").map(str::trim).filter(|f| !f.is_empty()) + { + let dir = std::path::Path::new(folder); + if !dir.is_dir() { + return Err(format!("folder '{folder}' does not exist or is not a directory")); + } + if let Ok(entries) = std::fs::read_dir(dir) { + let mut files = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { + files.push(path.display().to_string()); + } + } + } + files.sort(); + targets.extend(files); + } + } + + #[cfg(target_arch = "wasm32")] + if targets.is_empty() && ps(p, "folder").is_some() { + return Err("folder batch runs are not supported in the browser; specify 'inputs' as an array of layers".into()); + } + + if targets.is_empty() { + return Err( + "no inputs provided (specify 'inputs' as an array of layers/paths, '*' for all layers, or 'folder' as a directory path)".into(), + ); + } + + use rustgis_analysis::control::Control; + use rustgis_data::progress::Probe; + let control = if s.stoppable { Control::stoppable() } else { Control::new() }; + control.begin(targets.len() as u64); + + let mut succeeded = Vec::new(); + let mut failed = Vec::new(); + let mut layer_ids = Vec::new(); + let mut output_names = Vec::new(); + + for (i, target) in targets.iter().enumerate() { + if control.is_cancelled() { + s.notify(format!("Batch '{tool_id}' was cancelled at item {} of {}", i + 1, targets.len()), true); + break; + } + + let base_name = std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target); + let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); + + let mut item_params = base_params.clone(); + if let Value::Object(ref mut map) = item_params { + map.insert(batch_param.to_string(), json!(target)); + map.insert("output".to_string(), json!(out_name)); + } + + match run_tool_with(s, tool_id, &item_params, &control) { + Ok(res) => { + succeeded.push(target.clone()); + output_names.push(out_name); + if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { + layer_ids.push(lid); + } else if let Some(lids) = res.get("layers").and_then(Value::as_array) { + for l in lids { + if let Some(lid) = l.as_u64().and_then(|id| u32::try_from(id).ok()) { + layer_ids.push(lid); + } + } + } + } + Err(err) => { + s.notify(format!("{target}: {err}"), true); + failed.push(json!({"input": target, "error": err})); + } + } + control.advance(1); + } + + let total = targets.len(); + let num_ok = succeeded.len(); + let num_err = failed.len(); + let summary_msg = if num_err == 0 { + format!("Batch '{tool_id}' complete: all {num_ok} succeeded") + } else { + format!("Batch '{tool_id}' complete: {num_ok} succeeded, {num_err} failed") + }; + s.notify(summary_msg, num_err > 0); + + Ok(json!({ + "tool": tool_id, + "total": total, + "succeeded": num_ok, + "failed": num_err, + "layers": layer_ids, + "outputs": output_names, + "errors": failed + })) +} + /// The id of a tool named by its id or its display name (history records keep the name). pub fn tool_id(name: &str) -> Option<&'static str> { rustgis_analysis::tool(name).or_else(|| rustgis_analysis::tools().find(|t| t.name.eq_ignore_ascii_case(name))).map(|t| t.id) @@ -2072,3 +2245,141 @@ mod file_permission_tests { assert!(crate::model::output_ports(&node).is_empty(), "files must not be wired as layers"); } } + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod batch_tests { + use super::*; + + fn test_session() -> Session { + let mut s = Session::new(crate::Project::default()); + let mut cities = Dataset::new("Cities", rustgis_data::GeometryType::Point, Vec::new()); + cities.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, cities)); + + let mut parks = Dataset::new("Parks", rustgis_data::GeometryType::Point, Vec::new()); + parks.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(15.0, 25.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(1, parks)); + s + } + + #[test] + fn batch_run_over_multiple_layers() { + let mut s = test_session(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "output_pattern": "{input}_buf", + "params": {"distance": 50, "units": "Kilometers"} + }), + ) + .unwrap(); + + assert_eq!(res["total"], 2); + assert_eq!(res["succeeded"], 2); + assert_eq!(res["failed"], 0); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 2); + assert_eq!(outputs[0], "Cities_buf"); + assert_eq!(outputs[1], "Parks_buf"); + + assert!(s.map().layer_by_name("Cities_buf").is_some()); + assert!(s.map().layer_by_name("Parks_buf").is_some()); + } + + #[test] + fn batch_run_wildcard_all_layers() { + let mut s = test_session(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "inputs": "*", + "output_pattern": "{input}_hull" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 2); + assert_eq!(res["succeeded"], 2); + assert_eq!(res["failed"], 0); + assert!(s.map().layer_by_name("Cities_hull").is_some()); + assert!(s.map().layer_by_name("Parks_hull").is_some()); + } + + #[test] + fn batch_run_fault_tolerance() { + let mut s = test_session(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "MissingLayer", "Parks"], + "params": {"distance": 20} + }), + ) + .unwrap(); + + assert_eq!(res["total"], 3); + assert_eq!(res["succeeded"], 2); + assert_eq!(res["failed"], 1); + let errs = res["errors"].as_array().unwrap(); + assert_eq!(errs.len(), 1); + assert_eq!(errs[0]["input"], "MissingLayer"); + + // The valid layers succeeded and were added to map + assert!(s.map().layer_by_name("Cities_buffer").is_some()); + assert!(s.map().layer_by_name("Parks_buffer").is_some()); + } + + #[test] + fn batch_run_validation_errors() { + let mut s = test_session(); + // Missing tool + assert!(s.run("gp.batch", &json!({"inputs": ["Cities"]})).is_err()); + // Nonexistent tool + assert!(s.run("gp.batch", &json!({"tool": "nonexistent", "inputs": ["Cities"]})).is_err()); + // Missing inputs and folder + assert!(s.run("gp.batch", &json!({"tool": "buffer"})).is_err()); + // Nonexistent folder + assert!(s.run("gp.batch", &json!({"tool": "buffer", "folder": "/path/does/not/exist_12345"})).is_err()); + } + + #[test] + fn batch_run_folder_scanning() { + let root = std::env::temp_dir() + .join(format!("rustgis_batch_test_{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos())); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("layer_a.geojson"); + let f2 = root.join("layer_b.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + std::fs::write(&f2, sample_geojson).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + "output_pattern": "{input}_hull" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 2); + assert_eq!(res["succeeded"], 2); + assert_eq!(res["failed"], 0); + assert!(s.map().layer_by_name("layer_a_hull").is_some()); + assert!(s.map().layer_by_name("layer_b_hull").is_some()); + + let _ = std::fs::remove_dir_all(root); + } +} diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 10bbcb9154..10aedde59a 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -78,27 +78,16 @@ time; so does the map's right-click are added: a cancelled run keeps none. Scripts, models, the AI Assistant, the command line and the web app wait for each tool to finish. -**Viewshed**, **IDW Interpolation** and **Kriging** have no work limit when you run them from their -pane in the desktop app (the Tools or Analysis pane), nor does **Kernel Density** or **Deep -Learning**: a large run shows its progress, and **Cancel** stops it within a moment, however wide -its rows. Neither does `rustgis-cli run` typed at a terminal, which Ctrl-C stops. Where a run can't -be stopped while it goes (a script, a model, the AI Assistant, Python, the web app, and `rustgis-cli -run` called from another program, such as an agent or a CI job, which may wait with a timeout), -Viewshed, IDW Interpolation and Kriging refuse a run that would take more than a second or two of -work on one thread (a ray step, a cell-point pair), Kernel Density one whose points' search windows -(the square around each point's radius) cover more than 50 million cells, and Deep Learning a model -too large for its raster; its training fits a share of the sampled pixels, or fewer epochs, instead -(`dl.train` reports the epochs it ran). A task or model recorded from a pane run keeps these limits -when it runs again in the app, so a step the pane took on can be refused there, and a model trained -again can differ. On the desktop these limits grow with the cores the run takes (up to 16), as many -times more work as the threads it runs on. Threads share cores and memory, so a run at that limit -can take up to about twice as long as a one-thread run at its own (measured on one machine, a Xeon -Silver 4114; it can be longer on others); in the web app, on one thread, the limits stay as they -are. The error names the limit on this computer and the threads it counted. The limits on memory -(the cells a grid may have, and those on the following pages) hold everywhere, as does Viewshed's on -its window: a radius covering more than 25 million cells, or 100,000 cells across, is refused. A -tool's own default cell size still follows the one-thread limit, so it makes the same grid on any -computer and from any caller; a finer cell size given is what a larger limit, or none, lets through. +**Viewshed**, **IDW Interpolation** and **Kriging** refuse a run that would take more than a second +or two of work on one thread (a ray step, a cell-point pair), and **Kernel Density** one whose +points' search windows (the square around each point's radius) cover more than 50 million cells. On the desktop they run on several +cores (up to 16), and they take as many times more work as the threads they run on. Threads share +cores and memory, so a run at that limit can take up to about twice as long as a one-thread run at +its own (measured on one machine, a Xeon Silver 4114; it can be longer on others); in the web app, +on one thread, the limits stay as they are. The error names the +limit on this computer and the threads it counted. A tool's own default cell size still follows the +one-thread limit, so it makes the same grid on any computer; a finer cell size given is what the +larger limit lets through. From the command line or a script (`gp.run`), a built-in tool's parameter with a list of choices, such as Buffer's units, takes one of them in any letter case (`"meters"` for `Meters`); any other value is an error @@ -106,6 +95,23 @@ that names the choices (Buffer's units no longer take the abbreviations `m`, `mi Buffer and Select Layer By Location accept **Kilometers**, **Meters**, **Miles** and **Feet**. Feet are international feet (exactly 0.3048 metres): a 500-foot notification radius is 152.4 metres on the ground. +### Batch geoprocessing (`gp.batch`) + +Run a single geoprocessing tool across many layers in the project or a folder of vector files: + +```json +gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input}_buf","params":{"distance":50,"units":"Kilometers"}} +``` + +- `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). +- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. +- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. +- `"output_pattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. +- `"params"`: the static parameters shared by each run. +- `"param"`: which parameter receives each input (default: auto-detected as the tool's primary layer parameter, or `"input"`). + +Batch runs are fault-tolerant: a failed run on one input logs a notice and continues with the remaining items without aborting the batch. At completion, a summary notification reports the total succeeded and failed counts. + The History pane lists each run with when it ran (in UTC). Its **⋯** menu, or a right-click on the tool's name, copies a run as code, with the parameters it ran with: **Copy as Python** (a call for the [`rustgis` package](../python.md), one parameter a line), **Copy as Command** (a @@ -152,6 +158,7 @@ Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available too | Feature Envelope To Polygon | `featureEnvelopeToPolygon` | Data Management › Features | | Unsplit Line | `unsplitLine` | Data Management › Features | | Check Geometry | `checkGeometry` | Data Management › Features | +| Fix Geometries | `fixGeometries` | Data Management › Features | | XY Table To Point | `xyTableToPoint` | Data Management › Features | | Calculate Geometry Attributes | `calculateGeometry` | Data Management › Features | | Simplify | `simplify` | Cartography › Generalization | @@ -842,50 +849,31 @@ gp.run={"tool":"featureEnvelopeToPolygon","params":{"input":"Regions"}} gp.run={"tool":"unsplitLine","params":{"input":"Rivers"}} ``` -### Checking geometry - -**Check Geometry** (`checkGeometry`) finds the features whose geometry breaks the OGC Simple -Features rules and writes them to a new layer, `_check` unless you name it. Only the -selected features are checked when some are selected. Each output row is its input feature as -stored, with the same feature id, fields, attributes and CRS, plus two text fields: `problem`, -the codes of what is wrong separated by `, `, and `detail`, a short description of each problem -separated by `; ` that names the part and ring (counted from 1; ring 1 is the exterior ring), -such as `exterior ring of part 1 crosses itself; 3 repeated vertices in ring 2 of part 1`. A -feature with more than 20 problems lists the first 20 and counts the rest. If the input already -has a `problem` or `detail` field, the new one gets the next free name (`problem_2`) and the -tool says so. When nothing is wrong, the output layer is empty and the message says how many -features were checked. - -| Code | What is wrong | -| --- | --- | -| `null_geometry` | No geometry, or one without coordinates | -| `non_finite_coordinate` | A coordinate is not a number or infinite. The row is written without geometry, so it can't spoil the layer's extent | -| `too_few_points` | A line part with fewer than 2 distinct points, a ring with fewer than 3, or a polygon part without rings | -| `ring_not_closed` | A ring's last point is not its first | -| `repeated_vertex` | The same vertex twice in a row (a closed ring's last point doesn't count) | -| `self_intersection` | A ring crosses itself, touches itself, or runs back over itself | -| `hole_outside_shell` | An interior ring (hole) is not inside its exterior ring | -| `rings_intersect` | Two rings of one polygon overlap or touch along a line | -| `disconnected_interior` | Rings of one polygon touch at points so that they cut its interior into pieces, such as a hole touching the exterior ring at two points, or three holes each touching the next | -| `parts_overlap` | Two polygons of one multipart feature overlap or touch along a line | -| `too_complex` | Some of the feature's checks were left out: it needs more work than the tool spends on one feature, there wasn't enough memory to check it, or the geo library's relate operation failed on some of its rings or parts (`detail` says which of the three, not which rings or parts). The checks of each ring's points still run. The tool says how many features each reason left out | - -Codes are listed in this order, and are never translated. The checks work in stored -coordinates, without projection. Lines may cross themselves, so only polygon rings are checked -for self-intersection; points are checked for missing and non-finite coordinates only. Rings may -touch each other at single points, as long as the polygon's interior stays in one piece (checked -once its rings have no other problem with each other); any number of rings may meet at the same -point. An unclosed ring is reported, then checked as if it -were closed. Ring orientation is not checked, since GeoJSON and Shapefiles wind rings in -opposite directions. The rings of a polygon are only checked against each other when each of them is -usable (enough points, finite coordinates, and not crossing, touching or running back over -itself), and the parts of a feature only when each part's rings are usable and have no problem -with each other; the tool says how many features were not fully checked this way, because some -of their rings or parts already have problems of their own. Rings and parts that touch are compared with the -[geo](https://github.com/georust/geo) library's relate operation. +### Check and Fix Geometry + +The **Check Geometry** (`checkGeometry`) and **Fix Geometries** (`fixGeometries`) tools detect and repair +vector geometry validity issues according to the OGC Simple Features Specification (SFS). + +- **Check Geometry** inspects feature geometries in native coordinate space without projection distortion. + It detects self-intersections (transverse crossings between non-adjacent line segments or polygon edges, and crossings between parts of multi-part polylines), unclosed rings, + rings with too few points (< 3 distinct vertices), interior holes outside their exterior shells, overlapping or touching interior rings, + overlapping MultiPolygon components, and non-finite coordinates. It outputs a standalone validation report table (`_Validation`) containing: + `feature_id`, `geometry_type`, `status` (`Valid`, `Invalid`, `Empty`, `Null`, `Unsupported`, `Error`), `valid` (boolean), + `error_category`, `error_reason`, and problem coordinates (`error_x`, `error_y`) pinpointing the issue location (such as the segment + crossing or representative coordinate of the overlapping element). Line checks detect transverse crossings; tangent touches and collinear + overlaps without crossing are preserved. +- **Fix Geometries** repairs invalid geometries automatically using constrained Delaunay triangulation + ([Hugo Ledoux et al., 2014](http://dx.doi.org/10.1016/j.cageo.2014.01.009)). Already-valid features are preserved + with 100% bitwise identical coordinates. During repair of invalid geometries, consecutive duplicate vertices are removed, + unclosed rings are closed, self-intersecting polygons are decomposed into valid components, holes are re-nested, and degenerate + slivers are collapsed. Any dropped non-finite coordinates, collapsed line parts, or degenerate holes are counted and reported + via notices. All feature attributes, layer CRS, and metadata are retained in the output layer (`_Fixed`). Features that + cannot be automatically repaired (e.g. collapsed exterior shells or unrepairable line self-crossings) are preserved with their + original geometry and reported with counted summaries rather than silently dropped. ```text -gp.run={"tool":"checkGeometry","params":{"input":"Parcels"}} +gp.run={"tool":"checkGeometry","params":{"input":"Parcels","output":"Parcels_Validation"}} +gp.run={"tool":"fixGeometries","params":{"input":"Parcels","output":"Parcels_Fixed"}} ``` ## GeoLibre tools @@ -1017,9 +1005,8 @@ below the web app's fixed limits and never above a ceiling. So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster -on a grid of any coordinate system, cell size and extent, and Polygons to Raster -(`raster.rasterize`) burns polygons onto the same grids; Raster Functions › Resample matches a -target raster's grid. +on a grid of any coordinate system, cell size and extent; Raster Functions › Resample matches a target +raster's grid. The **Script** pane runs commands from text, one per line (`id {json}`); its box grows with the script down to the bottom of the pane, short of the buttons, and then scrolls, so **Run** stays in view; Ctrl+Enter (⌘+Enter on macOS) @@ -1192,41 +1179,6 @@ The Analyze tab's imagery groups open their tools in the Workflows pane: - The result has to fit in memory (120 million values, cells times bands, or more on a computer with more memory; 32 million in the web app): a grid past that is refused, naming the limit. A raster read from its overview is projected at that resolution, and the result says so. -- **Polygons to Raster** (`raster.rasterize`) burns a polygon layer onto a grid, so soil, - geology or land-use units line up cell for cell with rasters put on that grid by Project Raster: - `{"layer": "Soils", "field": "unit", "grid": "ndvi_100m", "path": "soils_100m.tif"}`. It has no - pane yet: run it from the Script pane, the AI Assistant, the command line or Python. The result - is a new layer (`name`, by default the layer's name with "raster"), and with `path` a - cloud-optimised GeoTIFF too. - - **The grid** is given as for Project Raster: `"grid"`, another raster's grid taken exactly, or - `crs` (by default the layer's), `cellSize` (required), `origin` (`[0, 0]` by default, so the - result lines up with any raster projected with the same `crs` and `cellSize`) and `extent` (by - default the polygons' bounds). - - **What a cell takes:** a cell takes a polygon's value when its centre falls inside it, holes - cut out. A numeric `field` gives its values; a text `field` gives codes 1 to n, in sorted order - of its values. A table named after the result with "classes" (`code`, `value`), numbered if - the project has a table of that name, holds every code; the result's `classTable` names it, - its `classes` lists the first 256, and `classCount` gives the total. With no field, every - polygon gives `value` (1 by default), a mask. - - Where polygons overlap, the later feature wins, as GDAL burns. With a selection, only the - selected features are burned. Features with no value in the field, with no valid polygon, or - holding no cell centre are skipped, and the result and a notice count them. - - Cells under no polygon take `nodata` (0 by default). A polygon whose value is the no-data - value is refused, so give another `nodata`, such as 255, when 0 is a class. - - **Pixel type:** bytes when every value and the no-data value are whole numbers from 0 to 255, - 16-bit when they are up to 65,535, and 32-bit floating point otherwise. Past 16,777,216, 32 bits - no longer hold every whole number, so a whole number past it is refused, from an Integer field, - a text field's codes or `value`; for an Integer field, a text field gives codes instead. Any - other value that 32 bits can't hold exactly, a Double field's included, is rounded, and a notice - counts them. Up to 256 whole-number values are drawn with unique values. - - **Limits:** a polygon narrower than a cell can hold no cell centre and be missed; there is no - "all touched" rule, and lines and points aren't burned. The grid has to fit in memory (120 - million cells, or more on a computer with more memory; 32 million in the web app): a grid past - that is refused before anything is burned. A layer read in place (only the part in view loaded) - is refused until all its features are loaded. Taken from a raster read from its overview, the - grid is the overview's, and a notice says so. The command runs where it is called, with no Cancel - yet and no cap on its work: each polygon fills the cells it covers, so many large polygons on a - fine grid take time in proportion. - **Classification Wizard** (unsupervised k-means) and **Classification Tools** (minimum distance from training polygons). - **Mensuration**: distances along the surface, heights between two points, areas, and volumes @@ -1416,9 +1368,7 @@ the wrong field. **Analyze › Deep Learning Tools** trains a neural network (a multilayer perceptron) on the pixels under training polygons and classifies every pixel, adding a class raster with a confidence band -and reporting the accuracy on held-back pixels. Run from its pane in the desktop app, it trains -on every sampled pixel for all the epochs asked, however long that takes, and can be cancelled; -elsewhere it keeps to a few seconds of work (see the limits above). Trained models stay in the project; **Publish › +and reporting the accuracy on held-back pixels. Trained models stay in the project; **Publish › Share As › Deep Learning** saves one as a package to apply in other projects. **Analyze › Create Hosted Imagery** cuts a raster into XYZ web map tiles (PNG, transparent outside the raster) with a TileJSON file and a viewer page. From c513219661562a16e9aa3b6273a6f17b98009558 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 13:56:35 +0700 Subject: [PATCH 02/56] fix(engine): address bot review comments and resolve CI test failures in gp.batch --- crates/engine/src/cmd/gp.rs | 249 ++++++++++++++++++++++++++----- crates/engine/src/cmd/items.rs | 3 +- crates/engine/src/cmd/plugins.rs | 8 +- docs/user-guide/analysis.md | 175 +++++++++++++++------- 4 files changed, 339 insertions(+), 96 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 0217681356..02ee5083a2 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -12,7 +12,7 @@ pub fn commands() -> Vec { vec![ C::new("gp.run", "Run Tool", "Analyze › Geoprocessing", run).when(need_layers).params(r#"{"tool": tool id, "params": {...} (a layer by name; a standalone table by name or id)}"#), C::new("gp.batch", "Batch Run", "Analyze › Geoprocessing", batch).params( - r#"{"tool": tool id, "inputs"?: [layer names or file paths] | "*", "folder"?: folder path, "param"?: input parameter name (default "input"), "outputPattern"?: text (default "{input}_{tool}"), "params"?: {...}}"#, + r#"{"tool": tool id, "inputs"?: [layer names or file paths] | "*", "folder"?: folder path, "param"?: input parameter name (default: the tool's first layer parameter, else "input"), "batch_param"?: input parameter name, "outputPattern"?: text (default "{input}_{tool}"), "output_pattern"?: text, "params"?: {...}}"#, ), C::new("gp.spatialJoin", "Spatial Join", "Analyze › Overlay", spatial_join) .when(need_layers) @@ -749,29 +749,12 @@ fn inputs_of<'a>(snap: Snapshot<'a>, spec: Option<&rustgis_analysis::ToolSpec>, let mut full = full; #[cfg(not(target_arch = "wasm32"))] if let Some(spec) = spec.filter(|t| rustgis_analysis::external_source(t).is_none()) { - for parameter in spec.params.iter() { - let names: Vec = match parameter.kind { - rustgis_analysis::ParamKind::VectorLayers => { - rustgis_analysis::layer_list(params.get(parameter.name).unwrap_or(&Value::Null)) - .unwrap_or_default() - .into_iter() - .map(String::from) - .collect() - } - rustgis_analysis::ParamKind::Layer - | rustgis_analysis::ParamKind::PolygonLayer - | rustgis_analysis::ParamKind::PointLayer - | rustgis_analysis::ParamKind::VectorLayer => { - params.get(parameter.name).and_then(Value::as_str).into_iter().map(String::from).collect() - } - _ => Vec::new(), - }; - for name in names { - let path = std::path::Path::new(&name); - if snap.map.layer_by_name(&name).is_none() && !full.contains_key(&name) && path.is_file() { - let (dataset, report) = rustgis_data::io::read_path(path).map_err(|e| format!("{name}: {e}"))?; + for parameter in spec.params.iter().filter(|p| p.kind == rustgis_analysis::ParamKind::VectorLayers) { + for name in rustgis_analysis::layer_list(params.get(parameter.name).unwrap_or(&Value::Null))? { + if snap.map.layer_by_name(name).is_none() && !full.contains_key(name) { + let (dataset, report) = rustgis_data::io::read_path(std::path::Path::new(name)).map_err(|e| format!("{name}: {e}"))?; messages.extend(report.messages.into_iter().map(|message| format!("{name}: {message}"))); - full.insert(name.clone(), dataset); + full.insert(name.to_string(), dataset); } } } @@ -994,7 +977,18 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { }); let pattern = ps(p, "outputPattern").or_else(|| ps(p, "output_pattern")).unwrap_or("{input}_{tool}"); + if !pattern.contains("{input}") { + return Err("outputPattern must include '{input}' to avoid output name collisions".into()); + } + let base_params = p.get("params").filter(|v| v.is_object()).cloned().unwrap_or(json!({})); + if base_params.get("output").is_some() { + s.notify("params.output is ignored for batch runs; outputPattern determines output names", false); + } + + if p.get("inputs").is_some() && p.get("folder").is_some() { + s.notify("both 'inputs' and 'folder' were provided; running 'inputs' and ignoring 'folder'", false); + } // Collect inputs: from "inputs" array, wildcard "*", or "folder" directory path. let mut targets: Vec = Vec::new(); @@ -1027,20 +1021,40 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { if !dir.is_dir() { return Err(format!("folder '{folder}' does not exist or is not a directory")); } - if let Ok(entries) = std::fs::read_dir(dir) { - let mut files = Vec::new(); - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() { - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { - files.push(path.display().to_string()); + let mut files = Vec::new(); + let mut skipped = 0; + let mut unreadable = 0; + match std::fs::read_dir(dir) { + Ok(entries) => { + for entry in entries { + match entry { + Ok(entry) => { + let path = entry.path(); + if path.is_file() { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { + files.push(path.display().to_string()); + } else { + skipped += 1; + } + } + } + Err(_) => { + unreadable += 1; + } } } } - files.sort(); - targets.extend(files); + Err(e) => return Err(format!("folder '{folder}': {e}")), + } + if skipped > 0 { + s.notify(format!("folder '{folder}': {skipped} unsupported files skipped"), false); } + if unreadable > 0 { + s.notify(format!("folder '{folder}': {unreadable} unreadable directory entries skipped"), true); + } + files.sort(); + targets.extend(files); } #[cfg(target_arch = "wasm32")] @@ -1063,6 +1077,7 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { let mut failed = Vec::new(); let mut layer_ids = Vec::new(); let mut output_names = Vec::new(); + let mut seen_outputs = std::collections::HashSet::new(); for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { @@ -1070,19 +1085,66 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { break; } - let base_name = std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target); + let base_name = match s.map().layer_by_name(target) { + Some(layer) => layer.name.as_str(), + None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), + }; let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); + let mut final_out_name = out_name.clone(); + if seen_outputs.contains(&final_out_name) || s.map().layer_by_name(&final_out_name).is_some() { + let stem = final_out_name.clone(); + let mut counter = 2; + while seen_outputs.contains(&final_out_name) || s.map().layer_by_name(&final_out_name).is_some() { + final_out_name = format!("{stem}_{counter}"); + counter += 1; + } + s.notify(format!("output name '{out_name}' collided; renamed to '{final_out_name}'"), false); + } + seen_outputs.insert(final_out_name.clone()); + + #[cfg(not(target_arch = "wasm32"))] + let temp_layer_id = { + let path = std::path::Path::new(target); + if s.map().layer_by_name(target).is_none() && crate::project::find_table(&s.project.tables, target).is_none() && path.is_file() + { + match rustgis_data::io::read_path(path) { + Ok((dataset, _report)) => Some(super::layer::add_dataset(s, dataset)), + Err(e) => { + failed.push(json!({"input": target, "error": e.to_string()})); + control.advance(1); + continue; + } + } + } else { + None + } + }; + + let target_layer_name = { + #[cfg(not(target_arch = "wasm32"))] + { + if let Some(lid) = temp_layer_id { + s.map().layer(lid).map(|l| l.name.clone()).unwrap_or_else(|| target.to_string()) + } else { + target.to_string() + } + } + #[cfg(target_arch = "wasm32")] + { + target.to_string() + } + }; let mut item_params = base_params.clone(); if let Value::Object(ref mut map) = item_params { - map.insert(batch_param.to_string(), json!(target)); - map.insert("output".to_string(), json!(out_name)); + map.insert(batch_param.to_string(), json!(target_layer_name)); + map.insert("output".to_string(), json!(final_out_name)); } match run_tool_with(s, tool_id, &item_params, &control) { Ok(res) => { succeeded.push(target.clone()); - output_names.push(out_name); + output_names.push(final_out_name); if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); } else if let Some(lids) = res.get("layers").and_then(Value::as_array) { @@ -1094,13 +1156,24 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { } } Err(err) => { - s.notify(format!("{target}: {err}"), true); failed.push(json!({"input": target, "error": err})); } } + + #[cfg(not(target_arch = "wasm32"))] + if let Some(temp_id) = temp_layer_id { + s.map_mut().layers.retain(|l| l.id != temp_id); + } + control.advance(1); } + if !failed.is_empty() { + let sample = failed.iter().take(3).filter_map(|v| v.get("input").and_then(Value::as_str)).collect::>().join(", "); + let more = if failed.len() > 3 { format!(" and {} more", failed.len() - 3) } else { String::new() }; + s.notify(format!("Batch '{tool_id}': {} failed ({sample}{more})", failed.len()), true); + } + let total = targets.len(); let num_ok = succeeded.len(); let num_err = failed.len(); @@ -2382,4 +2455,104 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + + #[test] + fn batch_run_dotted_layer_name() { + let mut s = Session::new(crate::Project::default()); + let mut roads = Dataset::new("Roads.2020", rustgis_data::GeometryType::Point, Vec::new()); + roads.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(1.0, 2.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, roads)); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Roads.2020"], + "output_pattern": "{input}_buf", + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs[0], "Roads.2020_buf"); + assert!(s.map().layer_by_name("Roads.2020_buf").is_some()); + } + + #[test] + fn batch_run_output_collision_deduplication() { + let mut s = test_session(); + let pre = Dataset::new("Cities_buffer", rustgis_data::GeometryType::Point, Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(10, pre)); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs[0], "Cities_buffer_2"); + assert!(s.map().layer_by_name("Cities_buffer_2").is_some()); + } + + #[test] + fn batch_run_output_pattern_must_include_input() { + let mut s = test_session(); + let res = s.run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "outputPattern": "constant_output_name" + }), + ); + assert!(res.is_err()); + assert!(res.unwrap_err().contains("must include '{input}'")); + } + + #[test] + fn batch_run_folder_skips_unsupported_files() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_skip_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("layer.geojson"); + let txt = root.join("readme.txt"); + let doc = root.join("notes.pdf"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + std::fs::write(&txt, "text file").unwrap(); + std::fs::write(&doc, "pdf file").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + "output_pattern": "{input}_hull" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 1); + assert_eq!(res["succeeded"], 1); + assert!(s.map().layer_by_name("layer_hull").is_some()); + + assert!(s.notices.iter().any(|n| n.text.contains("2 unsupported files skipped"))); + + let _ = std::fs::remove_dir_all(root); + } } diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index bfc8c565cd..9fe5f4e74c 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -777,7 +777,8 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { || ["import", "export", "open", "save", "add", "package", "upload", "download"].iter().any(|w| verb.starts_with(w)) // Its inputs may be bare file names, which don't look like paths. || id == "raster.mosaic" - || (id == "gp.run" && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) + || ((id == "gp.run" || id == "gp.batch") + && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) || reaches(params) } diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index cf18380413..cc40dbc70a 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -357,16 +357,18 @@ pub fn user_only(id: &str) -> bool { pub fn reaches(spec: &C, params: &Value) -> Option { // External tools (the GeoLibre suite) run a native program that reads files a parameter // names, including bare file names. - if spec.id == "gp.run" && ps(params, "tool").and_then(rustgis_analysis::tool).is_some_and(super::gp::is_geolibre_tool) { + if (spec.id == "gp.run" || spec.id == "gp.batch") + && ps(params, "tool").and_then(rustgis_analysis::tool).is_some_and(super::gp::is_geolibre_tool) + { return Some(Reach::Files); } // So do tools with a file parameter (a plugin's `file`): they're handed the file. - if spec.id == "gp.run" + if (spec.id == "gp.run" || spec.id == "gp.batch") && ps(params, "tool").and_then(super::gp::tool_id).and_then(rustgis_analysis::tool).is_some_and(super::gp::reads_files) { return Some(Reach::Files); } - if spec.id == "gp.run" + if (spec.id == "gp.run" || spec.id == "gp.batch") && ps(params, "tool").and_then(super::gp::tool_id).and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files) { return Some(Reach::Files); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 10aedde59a..db8b65fa41 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -78,16 +78,27 @@ time; so does the map's right-click are added: a cancelled run keeps none. Scripts, models, the AI Assistant, the command line and the web app wait for each tool to finish. -**Viewshed**, **IDW Interpolation** and **Kriging** refuse a run that would take more than a second -or two of work on one thread (a ray step, a cell-point pair), and **Kernel Density** one whose -points' search windows (the square around each point's radius) cover more than 50 million cells. On the desktop they run on several -cores (up to 16), and they take as many times more work as the threads they run on. Threads share -cores and memory, so a run at that limit can take up to about twice as long as a one-thread run at -its own (measured on one machine, a Xeon Silver 4114; it can be longer on others); in the web app, -on one thread, the limits stay as they are. The error names the -limit on this computer and the threads it counted. A tool's own default cell size still follows the -one-thread limit, so it makes the same grid on any computer; a finer cell size given is what the -larger limit lets through. +**Viewshed**, **IDW Interpolation** and **Kriging** have no work limit when you run them from their +pane in the desktop app (the Tools or Analysis pane), nor does **Kernel Density** or **Deep +Learning**: a large run shows its progress, and **Cancel** stops it within a moment, however wide +its rows. Neither does `rustgis-cli run` typed at a terminal, which Ctrl-C stops. Where a run can't +be stopped while it goes (a script, a model, the AI Assistant, Python, the web app, and `rustgis-cli +run` called from another program, such as an agent or a CI job, which may wait with a timeout), +Viewshed, IDW Interpolation and Kriging refuse a run that would take more than a second or two of +work on one thread (a ray step, a cell-point pair), Kernel Density one whose points' search windows +(the square around each point's radius) cover more than 50 million cells, and Deep Learning a model +too large for its raster; its training fits a share of the sampled pixels, or fewer epochs, instead +(`dl.train` reports the epochs it ran). A task or model recorded from a pane run keeps these limits +when it runs again in the app, so a step the pane took on can be refused there, and a model trained +again can differ. On the desktop these limits grow with the cores the run takes (up to 16), as many +times more work as the threads it runs on. Threads share cores and memory, so a run at that limit +can take up to about twice as long as a one-thread run at its own (measured on one machine, a Xeon +Silver 4114; it can be longer on others); in the web app, on one thread, the limits stay as they +are. The error names the limit on this computer and the threads it counted. The limits on memory +(the cells a grid may have, and those on the following pages) hold everywhere, as does Viewshed's on +its window: a radius covering more than 25 million cells, or 100,000 cells across, is refused. A +tool's own default cell size still follows the one-thread limit, so it makes the same grid on any +computer and from any caller; a finer cell size given is what a larger limit, or none, lets through. From the command line or a script (`gp.run`), a built-in tool's parameter with a list of choices, such as Buffer's units, takes one of them in any letter case (`"meters"` for `Meters`); any other value is an error @@ -95,23 +106,6 @@ that names the choices (Buffer's units no longer take the abbreviations `m`, `mi Buffer and Select Layer By Location accept **Kilometers**, **Meters**, **Miles** and **Feet**. Feet are international feet (exactly 0.3048 metres): a 500-foot notification radius is 152.4 metres on the ground. -### Batch geoprocessing (`gp.batch`) - -Run a single geoprocessing tool across many layers in the project or a folder of vector files: - -```json -gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input}_buf","params":{"distance":50,"units":"Kilometers"}} -``` - -- `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). -- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. -- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. -- `"output_pattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. -- `"params"`: the static parameters shared by each run. -- `"param"`: which parameter receives each input (default: auto-detected as the tool's primary layer parameter, or `"input"`). - -Batch runs are fault-tolerant: a failed run on one input logs a notice and continues with the remaining items without aborting the batch. At completion, a summary notification reports the total succeeded and failed counts. - The History pane lists each run with when it ran (in UTC). Its **⋯** menu, or a right-click on the tool's name, copies a run as code, with the parameters it ran with: **Copy as Python** (a call for the [`rustgis` package](../python.md), one parameter a line), **Copy as Command** (a @@ -130,6 +124,23 @@ written some other way (a token typed into a SQL query, say) can still be there: before sharing it. A point buffered where the map was clicked is copied as the command that buffered it, with the point. +### Batch geoprocessing (`gp.batch`) + +Run a single geoprocessing tool across many layers in the project or a folder of vector files: + +```json +gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input}_buf","params":{"distance":50,"units":"Kilometers"}} +``` + +- `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). +- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. +- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. +- `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. +- `"params"`: the static parameters shared by each run. +- `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). + +Batch runs are fault-tolerant: a failed run on one input logs a notice and continues with the remaining items without aborting the batch. At completion, a summary notification reports the total succeeded and failed counts. + These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): @@ -158,7 +169,6 @@ Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available too | Feature Envelope To Polygon | `featureEnvelopeToPolygon` | Data Management › Features | | Unsplit Line | `unsplitLine` | Data Management › Features | | Check Geometry | `checkGeometry` | Data Management › Features | -| Fix Geometries | `fixGeometries` | Data Management › Features | | XY Table To Point | `xyTableToPoint` | Data Management › Features | | Calculate Geometry Attributes | `calculateGeometry` | Data Management › Features | | Simplify | `simplify` | Cartography › Generalization | @@ -849,31 +859,50 @@ gp.run={"tool":"featureEnvelopeToPolygon","params":{"input":"Regions"}} gp.run={"tool":"unsplitLine","params":{"input":"Rivers"}} ``` -### Check and Fix Geometry - -The **Check Geometry** (`checkGeometry`) and **Fix Geometries** (`fixGeometries`) tools detect and repair -vector geometry validity issues according to the OGC Simple Features Specification (SFS). - -- **Check Geometry** inspects feature geometries in native coordinate space without projection distortion. - It detects self-intersections (transverse crossings between non-adjacent line segments or polygon edges, and crossings between parts of multi-part polylines), unclosed rings, - rings with too few points (< 3 distinct vertices), interior holes outside their exterior shells, overlapping or touching interior rings, - overlapping MultiPolygon components, and non-finite coordinates. It outputs a standalone validation report table (`_Validation`) containing: - `feature_id`, `geometry_type`, `status` (`Valid`, `Invalid`, `Empty`, `Null`, `Unsupported`, `Error`), `valid` (boolean), - `error_category`, `error_reason`, and problem coordinates (`error_x`, `error_y`) pinpointing the issue location (such as the segment - crossing or representative coordinate of the overlapping element). Line checks detect transverse crossings; tangent touches and collinear - overlaps without crossing are preserved. -- **Fix Geometries** repairs invalid geometries automatically using constrained Delaunay triangulation - ([Hugo Ledoux et al., 2014](http://dx.doi.org/10.1016/j.cageo.2014.01.009)). Already-valid features are preserved - with 100% bitwise identical coordinates. During repair of invalid geometries, consecutive duplicate vertices are removed, - unclosed rings are closed, self-intersecting polygons are decomposed into valid components, holes are re-nested, and degenerate - slivers are collapsed. Any dropped non-finite coordinates, collapsed line parts, or degenerate holes are counted and reported - via notices. All feature attributes, layer CRS, and metadata are retained in the output layer (`_Fixed`). Features that - cannot be automatically repaired (e.g. collapsed exterior shells or unrepairable line self-crossings) are preserved with their - original geometry and reported with counted summaries rather than silently dropped. +### Checking geometry + +**Check Geometry** (`checkGeometry`) finds the features whose geometry breaks the OGC Simple +Features rules and writes them to a new layer, `_check` unless you name it. Only the +selected features are checked when some are selected. Each output row is its input feature as +stored, with the same feature id, fields, attributes and CRS, plus two text fields: `problem`, +the codes of what is wrong separated by `, `, and `detail`, a short description of each problem +separated by `; ` that names the part and ring (counted from 1; ring 1 is the exterior ring), +such as `exterior ring of part 1 crosses itself; 3 repeated vertices in ring 2 of part 1`. A +feature with more than 20 problems lists the first 20 and counts the rest. If the input already +has a `problem` or `detail` field, the new one gets the next free name (`problem_2`) and the +tool says so. When nothing is wrong, the output layer is empty and the message says how many +features were checked. + +| Code | What is wrong | +| --- | --- | +| `null_geometry` | No geometry, or one without coordinates | +| `non_finite_coordinate` | A coordinate is not a number or infinite. The row is written without geometry, so it can't spoil the layer's extent | +| `too_few_points` | A line part with fewer than 2 distinct points, a ring with fewer than 3, or a polygon part without rings | +| `ring_not_closed` | A ring's last point is not its first | +| `repeated_vertex` | The same vertex twice in a row (a closed ring's last point doesn't count) | +| `self_intersection` | A ring crosses itself, touches itself, or runs back over itself | +| `hole_outside_shell` | An interior ring (hole) is not inside its exterior ring | +| `rings_intersect` | Two rings of one polygon overlap or touch along a line | +| `disconnected_interior` | Rings of one polygon touch at points so that they cut its interior into pieces, such as a hole touching the exterior ring at two points, or three holes each touching the next | +| `parts_overlap` | Two polygons of one multipart feature overlap or touch along a line | +| `too_complex` | Some of the feature's checks were left out: it needs more work than the tool spends on one feature, there wasn't enough memory to check it, or the geo library's relate operation failed on some of its rings or parts (`detail` says which of the three, not which rings or parts). The checks of each ring's points still run. The tool says how many features each reason left out | + +Codes are listed in this order, and are never translated. The checks work in stored +coordinates, without projection. Lines may cross themselves, so only polygon rings are checked +for self-intersection; points are checked for missing and non-finite coordinates only. Rings may +touch each other at single points, as long as the polygon's interior stays in one piece (checked +once its rings have no other problem with each other); any number of rings may meet at the same +point. An unclosed ring is reported, then checked as if it +were closed. Ring orientation is not checked, since GeoJSON and Shapefiles wind rings in +opposite directions. The rings of a polygon are only checked against each other when each of them is +usable (enough points, finite coordinates, and not crossing, touching or running back over +itself), and the parts of a feature only when each part's rings are usable and have no problem +with each other; the tool says how many features were not fully checked this way, because some +of their rings or parts already have problems of their own. Rings and parts that touch are compared with the +[geo](https://github.com/georust/geo) library's relate operation. ```text -gp.run={"tool":"checkGeometry","params":{"input":"Parcels","output":"Parcels_Validation"}} -gp.run={"tool":"fixGeometries","params":{"input":"Parcels","output":"Parcels_Fixed"}} +gp.run={"tool":"checkGeometry","params":{"input":"Parcels"}} ``` ## GeoLibre tools @@ -1005,8 +1034,9 @@ below the web app's fixed limits and never above a ceiling. So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster -on a grid of any coordinate system, cell size and extent; Raster Functions › Resample matches a target -raster's grid. +on a grid of any coordinate system, cell size and extent, and Polygons to Raster +(`raster.rasterize`) burns polygons onto the same grids; Raster Functions › Resample matches a +target raster's grid. The **Script** pane runs commands from text, one per line (`id {json}`); its box grows with the script down to the bottom of the pane, short of the buttons, and then scrolls, so **Run** stays in view; Ctrl+Enter (⌘+Enter on macOS) @@ -1179,6 +1209,41 @@ The Analyze tab's imagery groups open their tools in the Workflows pane: - The result has to fit in memory (120 million values, cells times bands, or more on a computer with more memory; 32 million in the web app): a grid past that is refused, naming the limit. A raster read from its overview is projected at that resolution, and the result says so. +- **Polygons to Raster** (`raster.rasterize`) burns a polygon layer onto a grid, so soil, + geology or land-use units line up cell for cell with rasters put on that grid by Project Raster: + `{"layer": "Soils", "field": "unit", "grid": "ndvi_100m", "path": "soils_100m.tif"}`. It has no + pane yet: run it from the Script pane, the AI Assistant, the command line or Python. The result + is a new layer (`name`, by default the layer's name with "raster"), and with `path` a + cloud-optimised GeoTIFF too. + - **The grid** is given as for Project Raster: `"grid"`, another raster's grid taken exactly, or + `crs` (by default the layer's), `cellSize` (required), `origin` (`[0, 0]` by default, so the + result lines up with any raster projected with the same `crs` and `cellSize`) and `extent` (by + default the polygons' bounds). + - **What a cell takes:** a cell takes a polygon's value when its centre falls inside it, holes + cut out. A numeric `field` gives its values; a text `field` gives codes 1 to n, in sorted order + of its values. A table named after the result with "classes" (`code`, `value`), numbered if + the project has a table of that name, holds every code; the result's `classTable` names it, + its `classes` lists the first 256, and `classCount` gives the total. With no field, every + polygon gives `value` (1 by default), a mask. + - Where polygons overlap, the later feature wins, as GDAL burns. With a selection, only the + selected features are burned. Features with no value in the field, with no valid polygon, or + holding no cell centre are skipped, and the result and a notice count them. + - Cells under no polygon take `nodata` (0 by default). A polygon whose value is the no-data + value is refused, so give another `nodata`, such as 255, when 0 is a class. + - **Pixel type:** bytes when every value and the no-data value are whole numbers from 0 to 255, + 16-bit when they are up to 65,535, and 32-bit floating point otherwise. Past 16,777,216, 32 bits + no longer hold every whole number, so a whole number past it is refused, from an Integer field, + a text field's codes or `value`; for an Integer field, a text field gives codes instead. Any + other value that 32 bits can't hold exactly, a Double field's included, is rounded, and a notice + counts them. Up to 256 whole-number values are drawn with unique values. + - **Limits:** a polygon narrower than a cell can hold no cell centre and be missed; there is no + "all touched" rule, and lines and points aren't burned. The grid has to fit in memory (120 + million cells, or more on a computer with more memory; 32 million in the web app): a grid past + that is refused before anything is burned. A layer read in place (only the part in view loaded) + is refused until all its features are loaded. Taken from a raster read from its overview, the + grid is the overview's, and a notice says so. The command runs where it is called, with no Cancel + yet and no cap on its work: each polygon fills the cells it covers, so many large polygons on a + fine grid take time in proportion. - **Classification Wizard** (unsupervised k-means) and **Classification Tools** (minimum distance from training polygons). - **Mensuration**: distances along the surface, heights between two points, areas, and volumes @@ -1368,7 +1433,9 @@ the wrong field. **Analyze › Deep Learning Tools** trains a neural network (a multilayer perceptron) on the pixels under training polygons and classifies every pixel, adding a class raster with a confidence band -and reporting the accuracy on held-back pixels. Trained models stay in the project; **Publish › +and reporting the accuracy on held-back pixels. Run from its pane in the desktop app, it trains +on every sampled pixel for all the epochs asked, however long that takes, and can be cancelled; +elsewhere it keeps to a few seconds of work (see the limits above). Trained models stay in the project; **Publish › Share As › Deep Learning** saves one as a package to apply in other projects. **Analyze › Create Hosted Imagery** cuts a raster into XYZ web map tiles (PNG, transparent outside the raster) with a TileJSON file and a viewer page. From 1a9ebe69d77571932ad840d9d358fa88f133da9a Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 14:28:43 +0700 Subject: [PATCH 03/56] fix(engine): address macro progress, security gate, and input notices in gp.batch --- crates/engine/src/cmd/gp.rs | 166 +++++++++++++++++++++++++++++-- crates/engine/src/cmd/items.rs | 5 + crates/engine/src/cmd/plugins.rs | 13 +++ 3 files changed, 178 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 02ee5083a2..575e0fb36d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -954,6 +954,13 @@ fn run(s: &mut Session, p: &Value) -> CmdResult { /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { + use rustgis_analysis::control::Control; + let control = if s.stoppable { Control::stoppable() } else { Control::new() }; + run_batch_with(s, p, &control) +} + +pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc) -> CmdResult { + use rustgis_data::progress::Probe; let tool_name = ps(p, "tool").ok_or("tool is required")?; let tool_id = tool_id(tool_name).ok_or_else(|| format!("no tool '{tool_name}'"))?; let spec = rustgis_analysis::tool(tool_id).ok_or_else(|| format!("no tool '{tool_name}'"))?; @@ -976,12 +983,23 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { .unwrap_or("input") }); + if !spec.params.iter().any(|param| param.name == batch_param) { + return Err(format!("tool '{tool_id}' has no parameter '{batch_param}'")); + } + let pattern = ps(p, "outputPattern").or_else(|| ps(p, "output_pattern")).unwrap_or("{input}_{tool}"); if !pattern.contains("{input}") { return Err("outputPattern must include '{input}' to avoid output name collisions".into()); } - let base_params = p.get("params").filter(|v| v.is_object()).cloned().unwrap_or(json!({})); + let base_params = match p.get("params") { + Some(v) if v.is_object() => v.clone(), + Some(_) => { + s.notify("gp.batch: 'params' must be an object, ignoring invalid value", false); + json!({}) + } + None => json!({}), + }; if base_params.get("output").is_some() { s.notify("params.output is ignored for batch runs; outputPattern determines output names", false); } @@ -990,8 +1008,9 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { s.notify("both 'inputs' and 'folder' were provided; running 'inputs' and ignoring 'folder'", false); } - // Collect inputs: from "inputs" array, wildcard "*", or "folder" directory path. + // Collect inputs: from "inputs" array or single string, wildcard "*", or "folder" directory path. let mut targets: Vec = Vec::new(); + let mut dropped_inputs = 0; let is_wildcard = p.get("inputs").and_then(Value::as_str).is_some_and(|s| s == "*") || p.get("inputs").and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")); @@ -1008,9 +1027,26 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { let trimmed = str_val.trim(); if !trimmed.is_empty() { targets.push(trimmed.to_string()); + } else { + dropped_inputs += 1; } + } else { + dropped_inputs += 1; } } + } else if let Some(single_str) = p.get("inputs").and_then(Value::as_str) { + let trimmed = single_str.trim(); + if !trimmed.is_empty() { + targets.push(trimmed.to_string()); + } else { + dropped_inputs += 1; + } + } else if p.get("inputs").is_some() { + s.notify("gp.batch: 'inputs' must be an array of layer names/paths or a single layer name", false); + } + + if dropped_inputs > 0 { + s.notify(format!("gp.batch: {dropped_inputs} invalid input(s) skipped (not a valid layer name or file path)"), false); } #[cfg(not(target_arch = "wasm32"))] @@ -1068,9 +1104,6 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { ); } - use rustgis_analysis::control::Control; - use rustgis_data::progress::Probe; - let control = if s.stoppable { Control::stoppable() } else { Control::new() }; control.begin(targets.len() as u64); let mut succeeded = Vec::new(); @@ -1141,7 +1174,13 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { map.insert("output".to_string(), json!(final_out_name)); } - match run_tool_with(s, tool_id, &item_params, &control) { + let item_control = + if s.stoppable { rustgis_analysis::control::Control::stoppable() } else { rustgis_analysis::control::Control::new() }; + if control.is_cancelled() { + item_control.cancel(); + } + + match run_tool_with(s, tool_id, &item_params, &item_control) { Ok(res) => { succeeded.push(target.clone()); output_names.push(final_out_name); @@ -2317,6 +2356,25 @@ mod file_permission_tests { }; assert!(crate::model::output_ports(&node).is_empty(), "files must not be wired as layers"); } + + #[test] + fn batch_tool_with_inputs_or_folder_is_file_access() { + let s = Session::new(crate::Project::default()); + let p_folder = json!({"tool": "convexHull", "folder": "data_dir"}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_folder)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_folder)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + + let p_inputs = json!({"tool": "convexHull", "inputs": ["cities.gpkg"]}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_inputs)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_inputs)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + + let p_wildcard = json!({"tool": "convexHull", "inputs": "*"}); + assert!(!crate::cmd::items::touches_files("gp.batch", &p_wildcard)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard)); + assert!(reaches.is_none()); + } } #[cfg(test)] @@ -2555,4 +2613,100 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + + #[test] + fn batch_run_macro_progress() { + let mut s = test_session(); + let control = rustgis_analysis::control::Control::new(); + let res = run_batch_with( + &mut s, + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "params": {"distance": 10} + }), + &control, + ) + .unwrap(); + + assert_eq!(res["total"], 2); + assert_eq!(res["succeeded"], 2); + assert_eq!(control.progress(), Some((2, 2))); + } + + #[test] + fn batch_run_invalid_param_rejected() { + let mut s = test_session(); + let res = s.run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "param": "imput", + "params": {"distance": 10} + }), + ); + assert!(res.is_err()); + assert!(res.unwrap_err().contains("tool 'buffer' has no parameter 'imput'")); + } + + #[test] + fn batch_run_single_input_string() { + let mut s = test_session(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": "Cities", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res["total"], 1); + assert_eq!(res["succeeded"], 1); + assert!(s.map().layer_by_name("Cities_buffer").is_some()); + } + + #[test] + fn batch_run_lossy_inputs_notified() { + let mut s = test_session(); + let mark = s.notices.len(); + // Lossy input: empty string and non-string in array, plus non-object params + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "", 42, " "], + "params": "not_an_object" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 1); + assert_eq!(res["succeeded"], 1); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("3 invalid input(s) skipped"))); + assert!(notices.iter().any(|n| n.text.contains("'params' must be an object"))); + + // Clean input says nothing + let mut s2 = test_session(); + let mark2 = s2.notices.len(); + let _ = s2 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": {"distance": 10} + }), + ) + .unwrap(); + let notices2 = &s2.notices[mark2..]; + assert!( + !notices2.iter().any(|n| n.text.contains("skipped") || n.text.contains("invalid") || n.text.contains("params")), + "clean input must give no skip notices: {notices2:?}" + ); + } } diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 9fe5f4e74c..4780cb6c1f 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -780,6 +780,11 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { || ((id == "gp.run" || id == "gp.batch") && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) + || (id == "gp.batch" + && (params.get("folder").is_some() + || params.get("inputs").is_some_and(|v| { + v != "*" && !v.as_array().is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")) + }))) || reaches(params) } diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index cc40dbc70a..6e955d30ea 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -373,6 +373,19 @@ pub fn reaches(spec: &C, params: &Value) -> Option { { return Some(Reach::Files); } + if spec.id == "gp.batch" { + if params.get("folder").is_some() { + return Some(Reach::Files); + } + if let Some(inputs) = params.get("inputs") { + let is_wildcard = inputs.as_str() == Some("*") + || inputs.as_array().is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")); + if !is_wildcard { + return Some(Reach::Files); + } + } + return None; + } // A catalog's Zarr entry is opened as `layer.addZarr` is, a database's way. if spec.id == "cube.open" && super::cube_catalog::opens_zarr(params) { return Some(Reach::Files); From 0df3ea3cc0764c3bdb84974fe87a24d72c9d2332 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 14:38:34 +0700 Subject: [PATCH 04/56] fix(engine): track actual result layer names and preserve updated layers in gp.batch --- crates/engine/src/cmd/gp.rs | 53 ++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 575e0fb36d..ca6e91c2ad 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1183,16 +1183,27 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); - output_names.push(final_out_name); - if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { + let actual_name = if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); + s.map().layer(lid).map(|l| l.name.clone()).unwrap_or(final_out_name) + } else if let Some(upd) = res.get("updated").and_then(Value::as_str) { + if let Some(l) = s.map().layer_by_name(upd) { + layer_ids.push(l.id); + l.name.clone() + } else { + upd.to_string() + } } else if let Some(lids) = res.get("layers").and_then(Value::as_array) { for l in lids { if let Some(lid) = l.as_u64().and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); } } - } + final_out_name + } else { + final_out_name + }; + output_names.push(actual_name); } Err(err) => { failed.push(json!({"input": target, "error": err})); @@ -1201,7 +1212,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Thu, 5 Nov 2026 14:53:41 +0700 Subject: [PATCH 05/56] fix(engine): address in-flight cancellation, notice aggregation, and table stem protection --- crates/engine/src/cmd/gp.rs | 123 +++++++++++++++++++++++++++++---- crates/engine/src/cmd/items.rs | 2 +- 2 files changed, 110 insertions(+), 15 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index ca6e91c2ad..7d384f64f2 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1004,10 +1004,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut dropped_inputs = 0; @@ -1049,6 +1045,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc layer.name.as_str(), + None if crate::project::find_table(&s.project.tables, target).is_some() => target.as_str(), None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), }; let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); @@ -1131,7 +1135,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Some(super::layer::add_dataset(s, dataset)), + Ok((dataset, report)) => { + for msg in report.messages { + read_messages.push(format!("{target}: {msg}")); + } + Some(super::layer::add_dataset(s, dataset)) + } Err(e) => { failed.push(json!({"input": target, "error": e.to_string()})); control.advance(1); @@ -1174,13 +1186,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); let actual_name = if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { @@ -1218,7 +1224,18 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { + let first = first_collision.unwrap_or_default(); + s.notify(format!("gp.batch: {collisions} output name(s) collided and were renamed (first: {first})"), false); + } + + if !read_messages.is_empty() { + let first = &read_messages[0]; + s.notify(format!("gp.batch: {} notice(s) reading inputs (first: {first})", read_messages.len()), false); } if !failed.is_empty() { @@ -2754,4 +2771,82 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + + #[test] + fn batch_run_cancelled_stops_remaining_items() { + let mut s = test_session(); + let control = rustgis_analysis::control::Control::new(); + control.cancel(); + let res = run_batch_with( + &mut s, + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "params": {"distance": 10} + }), + &control, + ) + .unwrap(); + + assert_eq!(res["total"], 2); + assert_eq!(res["succeeded"], 0); + assert!(s.notices.iter().any(|n| n.text.contains("was cancelled"))); + } + + #[test] + fn batch_run_empty_inputs_with_folder_no_bogus_notice() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_empty_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("layer.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "inputs": [], + "folder": root.display().to_string() + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let notices = &s.notices[mark..]; + assert!(!notices.iter().any(|n| n.text.contains("both 'inputs' and 'folder'"))); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_aggregated_collision_notice() { + let mut s = test_session(); + let pre1 = Dataset::new("Cities_buffer", rustgis_data::GeometryType::Point, Vec::new()); + let pre2 = Dataset::new("Cities_buffer_2", rustgis_data::GeometryType::Point, Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(10, pre1)); + s.map_mut().add_layer(rustgis_map::Layer::new(11, pre2)); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("1 output name(s) collided and were renamed"))); + } } diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 4780cb6c1f..8acf3ced41 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -779,7 +779,7 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { || id == "raster.mosaic" || ((id == "gp.run" || id == "gp.batch") && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) - .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) + .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t))) || (id == "gp.batch" && (params.get("folder").is_some() || params.get("inputs").is_some_and(|v| { From fa1cb805a2083d8dfffdea899b1ba26d7bdc50e1 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 15:12:20 +0700 Subject: [PATCH 06/56] fix(engine): reject file-writing tools, aggregate tool messages, and preserve selection results in gp.batch --- crates/engine/src/cmd/gp.rs | 148 +++++++++++++++++++++++++++++++++--- 1 file changed, 138 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 7d384f64f2..2fccd86dc0 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -987,6 +987,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = std::collections::BTreeMap::new(); for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { @@ -1142,6 +1147,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); - let actual_name = if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { + res_was_selected = res.get("selected").is_some(); + if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); - s.map().layer(lid).map(|l| l.name.clone()).unwrap_or(final_out_name) + let actual_name = s.map().layer(lid).map(|l| l.name.clone()).unwrap_or(final_out_name); + output_names.push(actual_name); } else if let Some(upd) = res.get("updated").and_then(Value::as_str) { if let Some(l) = s.map().layer_by_name(upd) { layer_ids.push(l.id); - l.name.clone() + output_names.push(l.name.clone()); } else { - upd.to_string() + output_names.push(upd.to_string()); } } else if let Some(lids) = res.get("layers").and_then(Value::as_array) { for l in lids { if let Some(lid) = l.as_u64().and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); + if let Some(layer) = s.map().layer(lid) { + output_names.push(layer.name.clone()); + } } } - final_out_name + } else if res_was_selected { + output_names.push(target_layer_name.clone()); } else { - final_out_name - }; - output_names.push(actual_name); + output_names.push(final_out_name); + } } Err(err) => { failed.push(json!({"input": target, "error": err})); } } + let drained: Vec<_> = s.notices.drain(notice_mark..).collect(); + for n in drained { + *tool_messages.entry(n.text).or_insert(0) += 1; + } + #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id { let was_updated = layer_ids.contains(&temp_id); - if !was_updated { - s.map_mut().layers.retain(|l| l.id != temp_id); + if !was_updated && !res_was_selected { + super::layer::drop_layers(s, &[temp_id]); + s.active_layer = active_before; } } @@ -1228,6 +1250,14 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { let first = first_collision.unwrap_or_default(); s.notify(format!("gp.batch: {collisions} output name(s) collided and were renamed (first: {first})"), false); @@ -2849,4 +2879,102 @@ mod batch_tests { let notices = &s.notices[mark..]; assert!(notices.iter().any(|n| n.text.contains("1 output name(s) collided and were renamed"))); } + + #[test] + fn batch_run_rejects_file_writing_tools() { + let mut s = test_session(); + let res = s.run( + "gp.batch", + &json!({ + "tool": "vectorToPmtiles", + "inputs": ["Cities"], + }), + ); + assert!(res.is_err()); + assert!(res.unwrap_err().contains("writes a file, not a layer")); + } + + #[test] + fn batch_run_active_layer_preserved_after_file_batch() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_active_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("cities.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let mut s = test_session(); + s.active_layer = Some(1); // "Parks" is active + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [f1.display().to_string()], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + assert_eq!(s.active_layer, Some(1), "user active layer must be preserved"); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_selection_tool_on_file_retained() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_sel_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("cities.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{"pop":100}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "selectByAttributes", + "inputs": [f1.display().to_string()], + "params": {"where": "pop > 50"} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert!(s.map().layer_by_name(outputs[0].as_str().unwrap()).is_some()); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_aggregates_tool_messages() { + let mut s = test_session(); + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 2); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported: Buffer: 1 features at 10 Kilometers"))); + } } From 658623ab4d3e94d300f17f8cf89aa2c052fa72d2 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 15:46:09 +0700 Subject: [PATCH 07/56] fix(engine): resolve output collisions, shapefile sidecars, quiet tool messages, cancellation, and wildcard notices in gp.batch --- crates/engine/src/cmd/gp.rs | 394 +++++++++++++++++++++++++++---- crates/engine/src/cmd/items.rs | 4 +- crates/engine/src/cmd/plugins.rs | 10 +- docs/user-guide/analysis.md | 2 +- 4 files changed, 355 insertions(+), 55 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 2fccd86dc0..61cb3cbe7b 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -952,6 +952,12 @@ fn run(s: &mut Session, p: &Value) -> CmdResult { run_tool(s, &tool, ¶ms) } +/// Whether `inputs` is the wildcard `"*"` (either as a string or a single-element array `["*"]`). +pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { + v.and_then(Value::as_str).is_some_and(|val| val == "*") + || v.and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")) +} + /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { use rustgis_analysis::control::Control; @@ -1012,15 +1018,20 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut dropped_inputs = 0; - let is_wildcard = p.get("inputs").and_then(Value::as_str).is_some_and(|s| s == "*") - || p.get("inputs").and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")); + let is_wildcard = is_wildcard_inputs(p.get("inputs")); if is_wildcard { + let mut skipped_rasters = 0; for layer in &s.map().layers { if !layer.is_raster() { targets.push(layer.name.clone()); + } else { + skipped_rasters += 1; } } + if skipped_rasters > 0 { + s.notify(format!("gp.batch: {skipped_rasters} raster layer(s) left out of '*' (it runs vector layers)"), false); + } } else if let Some(arr) = p.get("inputs").and_then(Value::as_array) { for item in arr { if let Some(str_val) = item.as_str() { @@ -1066,24 +1077,46 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + let mut dir_entries = Vec::new(); + let mut shp_stems = std::collections::HashSet::new(); for entry in entries { match entry { Ok(entry) => { let path = entry.path(); if path.is_file() { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { - files.push(path.display().to_string()); - } else { - skipped += 1; + if ext == "shp" + && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) + { + shp_stems.insert(stem.to_ascii_lowercase()); } } + dir_entries.push(path); } Err(_) => { unreadable += 1; } } } + for path in dir_entries { + if path.is_file() { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { + files.push(path.display().to_string()); + } else { + let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("").to_ascii_lowercase(); + let is_shp_sidecar = + matches!(ext.as_str(), "dbf" | "shx" | "prj" | "cpg" | "qpj" | "sbn" | "sbx") && shp_stems.contains(&stem); + if !is_shp_sidecar { + skipped += 1; + } + } + } else if path.is_dir() { + skipped += 1; + } else { + unreadable += 1; + } + } } Err(e) => return Err(format!("folder '{folder}': {e}")), } @@ -1117,12 +1150,13 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut tool_messages: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut cancelled_after = None; for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { - s.notify(format!("Batch '{tool_id}' was cancelled at item {} of {}", i + 1, targets.len()), true); + cancelled_after = Some(i); break; } @@ -1133,6 +1167,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { failed.push(json!({"input": target, "error": e.to_string()})); + s.active_layer = active_before; control.advance(1); continue; } @@ -1194,22 +1225,35 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + match run_tool_quiet(s, tool_id, &item_params, control) { + Ok((res, messages)) => { succeeded.push(target.clone()); + for msg in messages { + *tool_messages.entry(msg).or_insert(0) += 1; + } + if had_collision { + collisions += 1; + if first_collision.is_none() { + first_collision = Some(format!("'{out_name}' → '{final_out_name}'")); + } + } + seen_outputs.insert(final_out_name.clone()); + res_was_selected = res.get("selected").is_some(); if let Some(lid) = res.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()) { layer_ids.push(lid); - let actual_name = s.map().layer(lid).map(|l| l.name.clone()).unwrap_or(final_out_name); + let actual_name = s.map().layer(lid).map(|l| l.name.clone()).unwrap_or_else(|| final_out_name.clone()); + seen_outputs.insert(actual_name.clone()); output_names.push(actual_name); } else if let Some(upd) = res.get("updated").and_then(Value::as_str) { if let Some(l) = s.map().layer_by_name(upd) { layer_ids.push(l.id); + seen_outputs.insert(l.name.clone()); output_names.push(l.name.clone()); } else { + seen_outputs.insert(upd.to_string()); output_names.push(upd.to_string()); } } else if let Some(lids) = res.get("layers").and_then(Value::as_array) { @@ -1217,45 +1261,47 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + if control.is_cancelled() { + cancelled_after = Some(i + 1); + #[cfg(not(target_arch = "wasm32"))] + if let Some(temp_id) = temp_layer_id { + super::layer::drop_layers(s, &[temp_id]); + } + s.active_layer = active_before; + break; + } failed.push(json!({"input": target, "error": err})); } } - let drained: Vec<_> = s.notices.drain(notice_mark..).collect(); - for n in drained { - *tool_messages.entry(n.text).or_insert(0) += 1; - } - #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id { let was_updated = layer_ids.contains(&temp_id); if !was_updated && !res_was_selected { super::layer::drop_layers(s, &[temp_id]); - s.active_layer = active_before; } } + s.active_layer = active_before; control.begin(targets.len() as u64); control.advance((i + 1) as u64); } for (msg, count) in tool_messages { - if count == 1 { - s.notify(msg, false); - } else { - s.notify(format!("gp.batch: {count} item(s) reported: {msg}"), false); - } + s.notify(format!("gp.batch: {count} item(s) reported: {msg}"), false); } if collisions > 0 { @@ -1264,7 +1310,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0); + s.notify(summary_msg, num_err > 0 || is_cancelled); - Ok(json!({ + let mut ret = json!({ "tool": tool_id, - "total": total, + "total": total_count, "succeeded": num_ok, "failed": num_err, "layers": layer_ids, "outputs": output_names, "errors": failed - })) + }); + if is_cancelled { + ret["cancelled"] = json!(true); + } + + Ok(ret) } /// The id of a tool named by its id or its display name (history records keep the name). @@ -1370,6 +1426,25 @@ pub fn run_tool_with( params: &Value, control: &std::sync::Arc, ) -> CmdResult { + run_tool_internal(s, tool, params, control, true).map(|(v, _)| v) +} + +pub(crate) fn run_tool_quiet( + s: &mut Session, + tool: &str, + params: &Value, + control: &std::sync::Arc, +) -> Result<(Value, Vec), String> { + run_tool_internal(s, tool, params, control, false) +} + +fn run_tool_internal( + s: &mut Session, + tool: &str, + params: &Value, + control: &std::sync::Arc, + notify_messages: bool, +) -> Result<(Value, Vec), String> { let tool = tool_id(tool).unwrap_or(tool).to_string(); let params = params.clone(); let t0 = now_ms(); @@ -1377,7 +1452,7 @@ pub fn run_tool_with( let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); let e = format!("{name} was cancelled before it started"); - return apply(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), Err(e), 0); + return apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), Err(e), 0, notify_messages); } let inputs = map_inputs(s, rustgis_analysis::tool(&tool), ¶ms)?; let mut res = rustgis_analysis::run_with(&tool, &inputs, ¶ms, control); @@ -1390,7 +1465,7 @@ pub fn run_tool_with( let millis = now_ms().saturating_sub(t0); let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); - apply(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), res, millis) + apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), res, millis, notify_messages) } /// Whether a tool can read files named by a path or vector-layer-list parameter. @@ -1537,6 +1612,17 @@ fn names_file(source: &str, destination: &std::path::Path) -> bool { /// Adds a tool's result to the map and the history. fn apply(s: &mut Session, ran: Ran, params: Value, res: Result, millis: u64) -> CmdResult { + apply_inner(s, ran, params, res, millis, true).map(|(v, _)| v) +} + +fn apply_inner( + s: &mut Session, + ran: Ran, + params: Value, + res: Result, + millis: u64, + notify_messages: bool, +) -> Result<(Value, Vec), String> { let Ran { name, id, command, started } = ran; let at = started / 1000; let mut result = match res { @@ -1553,12 +1639,14 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result { #[cfg(test)] if s.queue_gp_files { + let msgs = result.messages.clone(); return queue_file( s, file_name, bytes, GpRecord { tool: name, id, params, command, at, ok: false, messages: result.messages, millis, outputs: Vec::new() }, - ); + ) + .map(|v| (v, msgs)); } #[cfg(not(target_arch = "wasm32"))] { @@ -1576,12 +1664,14 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result { @@ -1641,8 +1731,11 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result = match (out.get("layer"), out.get("layers")) { (Some(id), _) => id.as_u64().into_iter().filter_map(|v| u32::try_from(v).ok()).collect(), @@ -1656,7 +1749,7 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result 500 { s.project.gp_history.remove(0); } - Ok(out) + Ok((out, messages)) } #[cfg(test)] @@ -2818,9 +2911,11 @@ mod batch_tests { ) .unwrap(); - assert_eq!(res["total"], 2); + assert_eq!(res["total"], 0); assert_eq!(res["succeeded"], 0); - assert!(s.notices.iter().any(|n| n.text.contains("was cancelled"))); + assert_eq!(res["failed"], 0); + assert_eq!(res["cancelled"], true); + assert!(s.notices.iter().any(|n| n.text.contains("cancelled after 0 of 2"))); } #[test] @@ -2977,4 +3072,213 @@ mod batch_tests { let notices = &s.notices[mark..]; assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported: Buffer: 1 features at 10 Kilometers"))); } + + #[test] + fn batch_run_failed_item_does_not_reserve_output_name() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_fail_name_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let dir1 = root.join("bad"); + let dir2 = root.join("good"); + std::fs::create_dir_all(&dir1).unwrap(); + std::fs::create_dir_all(&dir2).unwrap(); + + let f1 = dir1.join("sample.geojson"); + let f2 = dir2.join("sample.geojson"); + std::fs::write(&f1, "not-valid-json").unwrap(); + std::fs::write(&f2, r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [f1.display().to_string(), f2.display().to_string()], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["failed"], 1); + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].as_str().unwrap(), "sample_buffer"); + let notices = &s.notices[mark..]; + assert!(!notices.iter().any(|n| n.text.contains("collided and were renamed"))); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_shapefile_with_sidecars_not_counted_as_unsupported() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_sidecars_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + std::fs::write(root.join("test.shp"), b"shp").unwrap(); + std::fs::write(root.join("test.dbf"), b"dbf").unwrap(); + std::fs::write(root.join("test.shx"), b"shx").unwrap(); + std::fs::write(root.join("test.prj"), b"prj").unwrap(); + std::fs::write(root.join("test.cpg"), b"cpg").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let mark = s.notices.len(); + let _ = s.run( + "gp.batch", + &json!({ + "tool": "buffer", + "folder": root.display().to_string(), + "params": {"distance": 10} + }), + ); + let notices = &s.notices[mark..]; + assert!( + !notices.iter().any(|n| n.text.contains("unsupported files skipped")), + "shapefile companion files must not be reported as unsupported: {notices:?}" + ); + + std::fs::write(root.join("notes.txt"), b"some notes").unwrap(); + let mut s2 = Session::new(crate::Project::default()); + let mark2 = s2.notices.len(); + let _ = s2.run( + "gp.batch", + &json!({ + "tool": "buffer", + "folder": root.display().to_string(), + "params": {"distance": 10} + }), + ); + let notices2 = &s2.notices[mark2..]; + assert!( + notices2.iter().any(|n| n.text.contains("1 unsupported files skipped")), + "exactly 1 unsupported file (notes.txt) should be reported: {notices2:?}" + ); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_tool_messages_notices_since_not_desynced() { + let mut s = test_session(); + s.notify("Prior notification before batch", false); + let mark = s.notice_mark(); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities", "Parks"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 2); + let recent = s.notices_since(mark); + assert!(!recent.iter().any(|t| t.text.contains("Prior notification"))); + assert!(recent.iter().any(|t| t.text.contains("2 item(s) reported: Buffer: 1 features at 10 Kilometers"))); + } + + #[test] + fn batch_run_wildcard_skips_raster_with_notice() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let raster = Raster::new( + "Elevation", + 2, + 2, + vec![vec![0.0, 1.0, 2.0, 3.0]], + GeoTransform { origin_x: 0.0, origin_y: 0.0, pixel_width: 1.0, pixel_height: -1.0 }, + Crs::LonLat, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap(); + + let mut s = test_session(); + s.map_mut().add_raster("Elevation", rustgis_map::RasterLayer::from_path(raster, "test.tif")); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": "*", + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 2); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("1 raster layer(s) left out of '*' (it runs vector layers)"))); + + let mut s2 = test_session(); + let mark2 = s2.notices.len(); + let _ = s2 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": "*", + "params": {"distance": 10} + }), + ) + .unwrap(); + let notices2 = &s2.notices[mark2..]; + assert!(!notices2.iter().any(|n| n.text.contains("raster layer(s)"))); + } + + #[test] + fn batch_run_active_layer_preserved_on_selection_and_update() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_act_sel_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("items.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.0,0.0],[1.0,0.0],[1.0,1.0],[0.0,1.0],[0.0,0.0]]]},"properties":{"val":10}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let mut s = test_session(); + s.active_layer = Some(1); // "Parks" is active + + let res_sel = s + .run( + "gp.batch", + &json!({ + "tool": "selectByAttributes", + "inputs": [f1.display().to_string()], + "params": {"where": "val > 5"} + }), + ) + .unwrap(); + assert_eq!(res_sel["succeeded"], 1); + assert_eq!(s.active_layer, Some(1), "active layer must remain Parks after selectByAttributes"); + + let res_upd = s + .run( + "gp.batch", + &json!({ + "tool": "calculateGeometry", + "inputs": [f1.display().to_string()] + }), + ) + .unwrap(); + assert_eq!(res_upd["succeeded"], 1); + assert_eq!(s.active_layer, Some(1), "active layer must remain Parks after calculateGeometry"); + + let _ = std::fs::remove_dir_all(root); + } } diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 8f06e6f9a7..c2ee662d21 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -847,9 +847,7 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t))) || (id == "gp.batch" && (params.get("folder").is_some() - || params.get("inputs").is_some_and(|v| { - v != "*" && !v.as_array().is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")) - }))) + || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) || reaches(params) } diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 6e955d30ea..4f5fa02f84 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -377,12 +377,10 @@ pub fn reaches(spec: &C, params: &Value) -> Option { if params.get("folder").is_some() { return Some(Reach::Files); } - if let Some(inputs) = params.get("inputs") { - let is_wildcard = inputs.as_str() == Some("*") - || inputs.as_array().is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")); - if !is_wildcard { - return Some(Reach::Files); - } + if let Some(inputs) = params.get("inputs") + && !super::gp::is_wildcard_inputs(Some(inputs)) + { + return Some(Reach::Files); } return None; } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index db8b65fa41..4e6f1196ed 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -134,7 +134,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). - `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. -- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. +- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. - `"params"`: the static parameters shared by each run. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). From 979c28e1a8898dc78dd2f5ae67d320843b255fdf Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 15:56:51 +0700 Subject: [PATCH 08/56] fix(engine): ensure wasm32 compatibility without unused warnings in gp.batch --- crates/engine/src/cmd/gp.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 61cb3cbe7b..26715c7bfc 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1150,6 +1150,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut tool_messages: std::collections::BTreeMap = std::collections::BTreeMap::new(); let mut cancelled_after = None; @@ -1225,6 +1226,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Thu, 5 Nov 2026 16:53:33 +0700 Subject: [PATCH 09/56] fix(engine): skip symlinks in folder scan, gate geolibre batch tools, add docstrings --- crates/engine/src/cmd/gp.rs | 88 +++++++++++++++++++++++++++++----- crates/engine/src/cmd/items.rs | 2 +- docs/user-guide/analysis.md | 4 +- 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 26715c7bfc..25c91f5228 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -965,6 +965,7 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { run_batch_with(s, p, &control) } +/// Runs a geoprocessing tool in batch across multiple layers or files with cancellation and progress reporting. pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc) -> CmdResult { use rustgis_data::progress::Probe; let tool_name = ps(p, "tool").ok_or("tool is required")?; @@ -1074,6 +1075,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { @@ -1081,25 +1083,34 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { - let path = entry.path(); - if path.is_file() { - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - if ext == "shp" - && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) - { - shp_stems.insert(stem.to_ascii_lowercase()); + Ok(entry) => match entry.file_type() { + Ok(ft) => { + if ft.is_symlink() { + symlinks += 1; + } else { + let path = entry.path(); + if ft.is_file() { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if ext == "shp" + && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) + { + shp_stems.insert(stem.to_ascii_lowercase()); + } + } + dir_entries.push((path, ft)); } } - dir_entries.push(path); - } + Err(_) => { + unreadable += 1; + } + }, Err(_) => { unreadable += 1; } } } - for path in dir_entries { - if path.is_file() { + for (path, ft) in dir_entries { + if ft.is_file() { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); if matches!(ext.as_str(), "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet") { files.push(path.display().to_string()); @@ -1111,7 +1122,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc return Err(format!("folder '{folder}': {e}")), } + if symlinks > 0 { + s.notify(format!("folder '{folder}': {symlinks} symlink(s) skipped"), false); + } if skipped > 0 { s.notify(format!("folder '{folder}': {skipped} unsupported files skipped"), false); } @@ -1436,6 +1450,7 @@ pub fn run_tool_with( run_tool_internal(s, tool, params, control, true).map(|(v, _)| v) } +/// Runs a geoprocessing tool quietly without forwarding intermediate notices to the session. pub(crate) fn run_tool_quiet( s: &mut Session, tool: &str, @@ -1445,6 +1460,7 @@ pub(crate) fn run_tool_quiet( run_tool_internal(s, tool, params, control, false) } +/// Internal runner for geoprocessing tools, executing the tool and capturing outputs and notices. fn run_tool_internal( s: &mut Session, tool: &str, @@ -1622,6 +1638,7 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result bool { || id == "raster.mosaic" || ((id == "gp.run" || id == "gp.batch") && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) - .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t))) + .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" && (params.get("folder").is_some() || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 4e6f1196ed..9ef6597455 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -135,11 +135,11 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). - `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. - `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. -- `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. +- `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. - `"params"`: the static parameters shared by each run. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). -Batch runs are fault-tolerant: a failed run on one input logs a notice and continues with the remaining items without aborting the batch. At completion, a summary notification reports the total succeeded and failed counts. +Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From c9f1b0405b6d163641f7f154fddfe2b9e727bac7 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 17:33:26 +0700 Subject: [PATCH 10/56] fix(engine): report table outputs in batch, distinguish subfolders, and guard progress --- crates/engine/src/cmd/gp.rs | 118 +++++++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 25c91f5228..254960bfb5 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -972,6 +972,16 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("folder '{folder}': {symlinks} symlink(s) skipped"), false); } + if subfolders > 0 { + s.notify(format!("folder '{folder}': {subfolders} subfolder(s) skipped"), false); + } if skipped > 0 { s.notify(format!("folder '{folder}': {skipped} unsupported files skipped"), false); } @@ -1169,6 +1183,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = std::collections::BTreeMap::new(); let mut cancelled_after = None; + control.begin(targets.len() as u64); + for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { cancelled_after = Some(i); @@ -1183,10 +1199,15 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { failed.push(json!({"input": target, "error": e.to_string()})); s.active_layer = active_before; - control.advance(1); + control.begin(targets.len() as u64); + control.advance((i + 1) as u64); continue; } } @@ -1287,6 +1309,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Thu, 5 Nov 2026 17:39:45 +0700 Subject: [PATCH 11/56] fix(engine): record item error before breaking on batch cancellation --- crates/engine/src/cmd/gp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 254960bfb5..b0ce061722 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1320,6 +1320,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { cancelled_after = Some(i + 1); #[cfg(not(target_arch = "wasm32"))] @@ -1329,7 +1330,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Thu, 5 Nov 2026 17:46:43 +0700 Subject: [PATCH 12/56] fix(engine): exclude cancelled item from failures and check batch param places --- crates/engine/src/cmd/gp.rs | 4 ++-- crates/engine/src/cmd/plugins.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index b0ce061722..e8f7233b1a 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1320,9 +1320,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { - failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { - cancelled_after = Some(i + 1); + cancelled_after = Some(i); #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id { super::layer::drop_layers(s, &[temp_id]); @@ -1330,6 +1329,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Option { { return Some(Reach::Files); } - return None; + return super::items::names_place(params, PLACE_KEYS, 0).then_some(Reach::Files); } // A catalog's Zarr entry is opened as `layer.addZarr` is, a database's way. if spec.id == "cube.open" && super::cube_catalog::opens_zarr(params) { From f6b506870572d111d9b93c4306bfde16bfe7f557 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 17:57:41 +0700 Subject: [PATCH 13/56] fix(engine): handle table collisions, complete sidecars, and consolidate batch tool notices --- crates/engine/src/cmd/gp.rs | 42 ++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index e8f7233b1a..3d469d5edb 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1128,8 +1128,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = std::collections::BTreeMap::new(); let mut cancelled_after = None; - control.begin(targets.len() as u64); - for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { cancelled_after = Some(i); @@ -1310,8 +1310,25 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc>().join("; "); + s.notify( + format!("gp.batch: {total_reported} message(s) reported across items (e.g. {sample}; and {} more)", tool_messages.len() - 2), + false, + ); } if collisions > 0 { From e377f0509efcc821386450a533a26d1209214a39 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 18:47:19 +0700 Subject: [PATCH 14/56] fix(engine): address table collision restoration, sidecars scan, touches_files scope, and progress retention --- crates/engine/src/cmd/gp.rs | 200 +++++++++++++++++++++++++++++---- crates/engine/src/cmd/items.rs | 5 +- 2 files changed, 184 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 3d469d5edb..fbf852d5ac 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -967,7 +967,6 @@ fn batch(s: &mut Session, p: &Value) -> CmdResult { /// Runs a geoprocessing tool in batch across multiple layers or files with cancellation and progress reporting. pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc) -> CmdResult { - use rustgis_data::progress::Probe; let tool_name = ps(p, "tool").ok_or("tool is required")?; let tool_id = tool_id(tool_name).ok_or_else(|| format!("no tool '{tool_name}'"))?; let spec = rustgis_analysis::tool(tool_id).ok_or_else(|| format!("no tool '{tool_name}'"))?; @@ -1092,6 +1091,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { let mut dir_entries = Vec::new(); let mut shp_stems = std::collections::HashSet::new(); + let mut gpkg_stems = std::collections::HashSet::new(); for entry in entries { match entry { Ok(entry) => match entry.file_type() { @@ -1106,6 +1106,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { failed.push(json!({"input": target, "error": e.to_string()})); s.active_layer = active_before; - control.begin(targets.len() as u64); - control.advance((i + 1) as u64); continue; } } @@ -1265,6 +1269,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); @@ -1311,17 +1316,22 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { - cancelled_after = Some(i); + cancelled_after = Some(i + 1); #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id { super::layer::drop_layers(s, &[temp_id]); @@ -1346,7 +1357,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let _ = s.run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + }), + ); + + assert!(!s.notices.iter().any(|n| n.text.contains("unsupported files skipped"))); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_folder_lone_wal_emits_notice() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_lonewal_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let valid = root.join("valid.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&valid, sample_geojson).unwrap(); + + let f = root.join("stray.wal"); + std::fs::write(&f, b"stray wal content").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s.run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + }), + ); + assert!(res.is_ok()); + assert!(s.notices.iter().any(|n| n.text.contains("1 unsupported files skipped"))); + + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_empty_inputs_with_folder_no_bogus_notice() { let root = std::env::temp_dir().join(format!( diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index cfacabfd97..083caa64e6 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -842,7 +842,10 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { || ["import", "export", "open", "save", "add", "package", "upload", "download"].iter().any(|w| verb.starts_with(w)) // Its inputs may be bare file names, which don't look like paths. || id == "raster.mosaic" - || ((id == "gp.run" || id == "gp.batch") + || (id == "gp.run" + && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) + .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) + || (id == "gp.batch" && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" From caf8f10c66a4cefe4b763593b2e0aed1d476d455 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 19:03:57 +0700 Subject: [PATCH 15/56] fix(engine): add batch table output without replacing project tables and assign unique IDs --- crates/engine/src/cmd/gp.rs | 83 +++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index fbf852d5ac..8005c441af 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1269,7 +1269,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); @@ -1315,30 +1314,14 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc, ) -> CmdResult { - run_tool_internal(s, tool, params, control, true).map(|(v, _)| v) + run_tool_internal(s, tool, params, control, true, true).map(|(v, _)| v) } /// Runs a geoprocessing tool quietly without forwarding intermediate notices to the session. @@ -1515,7 +1498,7 @@ pub(crate) fn run_tool_quiet( params: &Value, control: &std::sync::Arc, ) -> Result<(Value, Vec), String> { - run_tool_internal(s, tool, params, control, false) + run_tool_internal(s, tool, params, control, false, false) } /// Internal runner for geoprocessing tools, executing the tool and capturing outputs and notices. @@ -1525,6 +1508,7 @@ fn run_tool_internal( params: &Value, control: &std::sync::Arc, notify_messages: bool, + replace_tables: bool, ) -> Result<(Value, Vec), String> { let tool = tool_id(tool).unwrap_or(tool).to_string(); let params = params.clone(); @@ -1533,7 +1517,15 @@ fn run_tool_internal( let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); let e = format!("{name} was cancelled before it started"); - return apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), Err(e), 0, notify_messages); + return apply_inner( + s, + Ran { name, id: tool, command: None, started: t0 }, + recorded(spec, params), + Err(e), + 0, + notify_messages, + replace_tables, + ); } let inputs = map_inputs(s, rustgis_analysis::tool(&tool), ¶ms)?; let mut res = rustgis_analysis::run_with(&tool, &inputs, ¶ms, control); @@ -1546,7 +1538,7 @@ fn run_tool_internal( let millis = now_ms().saturating_sub(t0); let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); - apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), res, millis, notify_messages) + apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), res, millis, notify_messages, replace_tables) } /// Whether a tool can read files named by a path or vector-layer-list parameter. @@ -1693,7 +1685,7 @@ fn names_file(source: &str, destination: &std::path::Path) -> bool { /// Adds a tool's result to the map and the history. fn apply(s: &mut Session, ran: Ran, params: Value, res: Result, millis: u64) -> CmdResult { - apply_inner(s, ran, params, res, millis, true).map(|(v, _)| v) + apply_inner(s, ran, params, res, millis, true, true).map(|(v, _)| v) } /// Adds a tool's result to the map and history, optionally notifying messages to the session. @@ -1704,6 +1696,7 @@ fn apply_inner( res: Result, millis: u64, notify_messages: bool, + replace_tables: bool, ) -> Result<(Value, Vec), String> { let Ran { name, id, command, started } = ran; let at = started / 1000; @@ -1778,9 +1771,22 @@ fn apply_inner( l.touch_geometry(); json!({"updated": layer}) } - Output::Table(t) => { + Output::Table(mut t) => { + if !replace_tables && s.project.tables.iter().any(|existing| existing.name == t.name) { + let stem = t.name.clone(); + let mut counter = 2; + while s.project.tables.iter().any(|existing| existing.name == format!("{stem}_{counter}")) { + counter += 1; + } + let new_name = format!("{stem}_{counter}"); + t.name = new_name; + } let n = t.name.clone(); - s.project.put_table(t); + if replace_tables { + s.project.put_table(t); + } else { + s.project.add_table(t); + } json!({"table": n}) } Output::Layers(outputs) => { @@ -1803,9 +1809,22 @@ fn apply_inner( Err(e) => result.messages.push(e), } } - OutputLayer::Table(t) => { + OutputLayer::Table(mut t) => { + if !replace_tables && s.project.tables.iter().any(|existing| existing.name == t.name) { + let stem = t.name.clone(); + let mut counter = 2; + while s.project.tables.iter().any(|existing| existing.name == format!("{stem}_{counter}")) { + counter += 1; + } + let new_name = format!("{stem}_{counter}"); + t.name = new_name; + } let n = t.name.clone(); - s.project.put_table(t); + if replace_tables { + s.project.put_table(t); + } else { + s.project.add_table(t); + } tables.push(n); } } @@ -2993,6 +3012,8 @@ mod batch_tests { assert_eq!(outputs[0], "Cities_Statistics_2"); assert!(crate::project::find_table(&s.project.tables, "Cities_Statistics").is_some()); assert!(crate::project::find_table(&s.project.tables, "Cities_Statistics_2").is_some()); + let ids: std::collections::HashSet = s.project.tables.iter().map(|t| t.id).collect(); + assert_eq!(ids.len(), s.project.tables.len(), "all table ids must be distinct"); assert!(s.notices.iter().any(|n| n.text.contains("output name(s) collided and were renamed"))); } @@ -3025,6 +3046,8 @@ mod batch_tests { assert_eq!(outputs[1], "Cities_Statistics_2"); assert!(crate::project::find_table(&s.project.tables, "Cities_Statistics").is_some()); assert!(crate::project::find_table(&s.project.tables, "Cities_Statistics_2").is_some()); + let ids: std::collections::HashSet = s.project.tables.iter().map(|t| t.id).collect(); + assert_eq!(ids.len(), s.project.tables.len(), "all table ids must be distinct"); } #[test] From f24d5a6d36524e71533a1174e6f757e0fcf545ce Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 19:33:02 +0700 Subject: [PATCH 16/56] fix(engine): report table renames via apply_inner and collect Output::Layers tables --- crates/engine/src/cmd/gp.rs | 195 ++++++++++++++++++++++++++++++------ 1 file changed, 167 insertions(+), 28 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8005c441af..2737b5b822 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1202,22 +1202,25 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc target.as_str(), None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), }; + let tool_has_output = spec.params.iter().any(|param| param.name == "output"); let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); let mut final_out_name = out_name.clone(); let mut had_collision = false; - let collision_exists = |name: &str| { - seen_outputs.contains(name) - || s.map().layer_by_name(name).is_some() - || crate::project::find_table(&s.project.tables, name).is_some() - }; - if collision_exists(&final_out_name) { - let stem = final_out_name.clone(); - let mut counter = 2; - while collision_exists(&final_out_name) { - final_out_name = format!("{stem}_{counter}"); - counter += 1; + if tool_has_output { + let collision_exists = |name: &str| { + seen_outputs.contains(name) + || s.map().layer_by_name(name).is_some() + || crate::project::find_table(&s.project.tables, name).is_some() + }; + if collision_exists(&final_out_name) { + let stem = final_out_name.clone(); + let mut counter = 2; + while collision_exists(&final_out_name) { + final_out_name = format!("{stem}_{counter}"); + counter += 1; + } + had_collision = true; } - had_collision = true; } let active_before = s.active_layer; @@ -1281,7 +1284,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { @@ -1772,6 +1806,7 @@ fn apply_inner( json!({"updated": layer}) } Output::Table(mut t) => { + let mut renamed_from = None; if !replace_tables && s.project.tables.iter().any(|existing| existing.name == t.name) { let stem = t.name.clone(); let mut counter = 2; @@ -1779,7 +1814,7 @@ fn apply_inner( counter += 1; } let new_name = format!("{stem}_{counter}"); - t.name = new_name; + renamed_from = Some(std::mem::replace(&mut t.name, new_name)); } let n = t.name.clone(); if replace_tables { @@ -1787,11 +1822,12 @@ fn apply_inner( } else { s.project.add_table(t); } - json!({"table": n}) + if let Some(orig) = renamed_from { json!({"table": n, "table_renamed_from": orig}) } else { json!({"table": n}) } } Output::Layers(outputs) => { let mut layers = Vec::new(); let mut tables = Vec::new(); + let mut tables_renamed = Vec::new(); for o in outputs { match o { OutputLayer::Vector(ds) => layers.push(super::layer::add_dataset(s, ds)), @@ -1810,6 +1846,7 @@ fn apply_inner( } } OutputLayer::Table(mut t) => { + let mut renamed_from = None; if !replace_tables && s.project.tables.iter().any(|existing| existing.name == t.name) { let stem = t.name.clone(); let mut counter = 2; @@ -1817,7 +1854,7 @@ fn apply_inner( counter += 1; } let new_name = format!("{stem}_{counter}"); - t.name = new_name; + renamed_from = Some(std::mem::replace(&mut t.name, new_name)); } let n = t.name.clone(); if replace_tables { @@ -1825,11 +1862,18 @@ fn apply_inner( } else { s.project.add_table(t); } - tables.push(n); + tables.push(n.clone()); + if let Some(orig) = renamed_from { + tables_renamed.push(json!({"from": orig, "to": n})); + } } } } - json!({"layers": layers, "tables": tables}) + let mut ret = json!({"layers": layers, "tables": tables}); + if !tables_renamed.is_empty() { + ret["tables_renamed"] = json!(tables_renamed); + } + ret } }; let messages = result.messages.clone(); @@ -3048,6 +3092,101 @@ mod batch_tests { assert!(crate::project::find_table(&s.project.tables, "Cities_Statistics_2").is_some()); let ids: std::collections::HashSet = s.project.tables.iter().map(|t| t.id).collect(); assert_eq!(ids.len(), s.project.tables.len(), "all table ids must be distinct"); + assert!(s.notices.iter().any(|n| n.text.contains("output name(s) collided and were renamed"))); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn batch_run_table_file_input_no_false_collision_notice() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_file_stat_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("cities.geojson"); + std::fs::write(&path, r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[10.0,20.0]},"properties":{"pop":100}}]}"#).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "summaryStatistics", + "inputs": [path.to_str().unwrap()], + "params": {"field": "pop"} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0], "cities_Statistics"); + assert!(crate::project::find_table(&s.project.tables, "cities_Statistics").is_some()); + assert!( + !s.notices.iter().any(|n| n.text.contains("output name(s) collided and were renamed")), + "file-input summaryStatistics must not emit false collision notice" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn batch_run_output_layers_only_table_collected() { + rustgis_analysis::register_external( + "test_output_layers_table_suite", + || { + vec![rustgis_analysis::ToolSpec { + id: "test:table_only", + name: "Table Only Tool", + toolbox: "Test", + summary: "", + params: &[rustgis_analysis::ParamSpec { + name: "input", + label: "Input Layer", + kind: rustgis_analysis::ParamKind::Layer, + required: true, + default: "", + help: "", + }], + run: rustgis_analysis::external_placeholder, + }] + }, + |_spec, _inputs, _params| { + let table = Dataset::new( + "CustomPluginTable", + rustgis_data::GeometryType::None, + vec![rustgis_data::Field::new("col", rustgis_data::FieldType::Integer)], + ); + Ok(rustgis_analysis::ToolResult { + output: rustgis_analysis::Output::Layers(vec![rustgis_analysis::OutputLayer::Table(table)]), + messages: Vec::new(), + }) + }, + ); + + let mut s = Session::new(crate::Project::default()); + let ds = Dataset::new( + "InputLayer", + rustgis_data::GeometryType::Point, + vec![rustgis_data::Field::new("val", rustgis_data::FieldType::Integer)], + ); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "test:table_only", + "inputs": ["InputLayer"], + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0], "CustomPluginTable"); + assert!(crate::project::find_table(&s.project.tables, "CustomPluginTable").is_some()); } #[test] From 41e059e7ee77c1674ceef7f8373d8ccf74b8e154 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 19:55:02 +0700 Subject: [PATCH 17/56] fix(engine): do not record cancelled batch items as failures --- crates/engine/src/cmd/gp.rs | 68 ++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 2737b5b822..16ad317d3a 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1364,9 +1364,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { - failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { - cancelled_after = Some(i + 1); + cancelled_after = Some(i); #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id { super::layer::drop_layers(s, &[temp_id]); @@ -1374,6 +1373,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc>> = std::sync::Mutex::new(None); + + fn cancel_item_runner( + _spec: &rustgis_analysis::ToolSpec, + _inputs: &dyn rustgis_analysis::Inputs, + _params: &serde_json::Value, + ) -> Result { + if let Ok(guard) = CANCEL_CTL.lock() + && let Some(ctl) = guard.as_ref() + { + ctl.cancel(); + } + Err("cancelled in tool".into()) + } + #[test] - fn batch_run_cancelled_failing_item_records_error() { - let mut s = test_session(); + fn batch_run_cancel_during_item_leaves_failed_empty() { let control = rustgis_analysis::control::Control::new(); - // Item fails during execution (nonexistent input layer): + *CANCEL_CTL.lock().unwrap() = Some(control.clone()); + rustgis_analysis::register_external( + "test_cancel_during_item_suite", + || { + vec![rustgis_analysis::ToolSpec { + id: "test:cancel_item", + name: "Cancel Item Tool", + toolbox: "Test", + summary: "", + params: &[rustgis_analysis::ParamSpec { + name: "input", + label: "Input Layer", + kind: rustgis_analysis::ParamKind::Layer, + required: true, + default: "", + help: "", + }], + run: rustgis_analysis::external_placeholder, + }] + }, + cancel_item_runner, + ); + + let mut s = Session::new(crate::Project::default()); + let ds = Dataset::new("L1", rustgis_data::GeometryType::Point, Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + let res = run_batch_with( &mut s, &json!({ - "tool": "buffer", - "inputs": ["NonExistent"], - "params": {"distance": 10} + "tool": "test:cancel_item", + "inputs": ["L1"], }), &control, ) .unwrap(); - assert_eq!(res["total"], 1); - assert_eq!(res["failed"], 1); - let errors = res["errors"].as_array().unwrap(); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0]["input"], "NonExistent"); + *CANCEL_CTL.lock().unwrap() = None; + + assert_eq!(res["failed"], 0); + assert_eq!(res["succeeded"], 0); + assert_eq!(res["cancelled"], true); + assert_eq!(res["errors"].as_array().unwrap().len(), 0); + assert!(!s.notices.iter().any(|n| n.text.contains("failed"))); + assert!(s.notices.iter().any(|n| n.text.contains("cancelled after 0 of 1"))); } #[test] From d56dd8d11bc0e059cc03a1de55e821a42530580e Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 20:13:31 +0700 Subject: [PATCH 18/56] fix(engine): record original target in gp_history, include selected layer ids, optimize collision probe, and refine shp xml sidecars --- crates/engine/src/cmd/gp.rs | 82 ++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 16ad317d3a..cceca80e44 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1133,9 +1133,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut tool_messages: std::collections::BTreeMap = std::collections::BTreeMap::new(); let mut cancelled_after = None; + let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { @@ -1214,11 +1213,12 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); for msg in messages { @@ -1355,6 +1361,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc, ) -> CmdResult { - run_tool_internal(s, tool, params, control, true, true).map(|(v, _)| v) + run_tool_internal(s, tool, params, None, control, true, true).map(|(v, _)| v) } /// Runs a geoprocessing tool quietly without forwarding intermediate notices to the session. @@ -1530,9 +1540,10 @@ pub(crate) fn run_tool_quiet( s: &mut Session, tool: &str, params: &Value, + record_params: Option<&Value>, control: &std::sync::Arc, ) -> Result<(Value, Vec), String> { - run_tool_internal(s, tool, params, control, false, false) + run_tool_internal(s, tool, params, record_params, control, false, false) } /// Internal runner for geoprocessing tools, executing the tool and capturing outputs and notices. @@ -1540,12 +1551,14 @@ fn run_tool_internal( s: &mut Session, tool: &str, params: &Value, + record_params: Option<&Value>, control: &std::sync::Arc, notify_messages: bool, replace_tables: bool, ) -> Result<(Value, Vec), String> { let tool = tool_id(tool).unwrap_or(tool).to_string(); let params = params.clone(); + let rec_val = record_params.unwrap_or(¶ms); let t0 = now_ms(); if control.is_cancelled() { let spec = rustgis_analysis::tool(&tool); @@ -1554,7 +1567,7 @@ fn run_tool_internal( return apply_inner( s, Ran { name, id: tool, command: None, started: t0 }, - recorded(spec, params), + recorded(spec, rec_val.clone()), Err(e), 0, notify_messages, @@ -1572,7 +1585,15 @@ fn run_tool_internal( let millis = now_ms().saturating_sub(t0); let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); - apply_inner(s, Ran { name, id: tool, command: None, started: t0 }, recorded(spec, params), res, millis, notify_messages, replace_tables) + apply_inner( + s, + Ran { name, id: tool, command: None, started: t0 }, + recorded(spec, rec_val.clone()), + res, + millis, + notify_messages, + replace_tables, + ) } /// Whether a tool can read files named by a path or vector-layer-list parameter. @@ -3328,9 +3349,14 @@ mod batch_tests { .unwrap(); assert_eq!(res["succeeded"], 1); + let layers = res["layers"].as_array().unwrap(); + assert_eq!(layers.len(), 1, "layers must include the updated layer id"); + let lid = layers[0].as_u64().unwrap() as u32; + assert!(s.map().layer(lid).is_some()); let outputs = res["outputs"].as_array().unwrap(); assert_eq!(outputs.len(), 1); assert!(s.map().layer_by_name(outputs[0].as_str().unwrap()).is_some()); + assert_eq!(s.project.gp_history[0].params["input"], f1.display().to_string()); let _ = std::fs::remove_dir_all(root); } @@ -3450,6 +3476,35 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[test] + fn batch_run_folder_unrelated_xml_counted_as_unsupported() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_unrelxml_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("rivers.shp"); + std::fs::write(&f1, b"").unwrap(); + let f2 = root.join("rivers.shp.xml"); + std::fs::write(&f2, b"").unwrap(); + let f3 = root.join("rivers.xml"); + std::fs::write(&f3, b"").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let _ = s.run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + }), + ); + + assert!(s.notices.iter().any(|n| n.text.contains("1 unsupported files skipped"))); + + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_folder_lone_wal_emits_notice() { let root = std::env::temp_dir().join(format!( @@ -3607,9 +3662,14 @@ mod batch_tests { .unwrap(); assert_eq!(res["succeeded"], 1); + let layers = res["layers"].as_array().unwrap(); + assert_eq!(layers.len(), 1, "layers must include the selected layer id"); + let lid = layers[0].as_u64().unwrap() as u32; + assert!(s.map().layer(lid).is_some()); let outputs = res["outputs"].as_array().unwrap(); assert_eq!(outputs.len(), 1); assert!(s.map().layer_by_name(outputs[0].as_str().unwrap()).is_some()); + assert_eq!(s.project.gp_history[0].params["input"], f1.display().to_string()); let _ = std::fs::remove_dir_all(root); } From b9d56b6ba44cf65f0f73bd20dd4bdc8e2ee16d05 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 20:28:26 +0700 Subject: [PATCH 19/56] fix(engine): probe output collisions after temp layer creation and format touches_files --- crates/engine/src/cmd/gp.rs | 87 +++++++++++++++++++++++----------- crates/engine/src/cmd/items.rs | 16 +++++-- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index cceca80e44..88b7594916 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1196,33 +1196,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc layer.name.as_str(), - None if crate::project::find_table(&s.project.tables, target).is_some() => target.as_str(), - None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), - }; - let tool_has_output = spec.params.iter().any(|param| param.name == "output"); - let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); - let mut final_out_name = out_name.clone(); - let mut had_collision = false; - if tool_has_output { - let collision_exists = |name: &str| { - seen_outputs.contains(name) - || s.map().layer_by_name(name).is_some() - || crate::project::find_table(&s.project.tables, name).is_some() - }; - if collision_exists(&final_out_name) { - let stem = final_out_name.clone(); - let mut counter = next_suffix.get(&stem).copied().unwrap_or(2); - while collision_exists(&format!("{stem}_{counter}")) { - counter += 1; - } - final_out_name = format!("{stem}_{counter}"); - next_suffix.insert(stem, counter + 1); - had_collision = true; - } - } - let active_before = s.active_layer; #[cfg(not(target_arch = "wasm32"))] @@ -1263,6 +1236,33 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc layer.name.as_str(), + None if crate::project::find_table(&s.project.tables, target).is_some() => target.as_str(), + None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), + }; + let tool_has_output = spec.params.iter().any(|param| param.name == "output"); + let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); + let mut final_out_name = out_name.clone(); + let mut had_collision = false; + if tool_has_output { + let collision_exists = |name: &str| { + seen_outputs.contains(name) + || s.map().layer_by_name(name).is_some() + || crate::project::find_table(&s.project.tables, name).is_some() + }; + if collision_exists(&final_out_name) { + let stem = final_out_name.clone(); + let mut counter = next_suffix.get(&stem).copied().unwrap_or(2); + while collision_exists(&format!("{stem}_{counter}")) { + counter += 1; + } + final_out_name = format!("{stem}_{counter}"); + next_suffix.insert(stem, counter + 1); + had_collision = true; + } + } + let mut item_params = base_params.clone(); if let Value::Object(ref mut map) = item_params { map.insert(batch_param.to_string(), json!(target_layer_name)); @@ -3151,6 +3151,39 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn batch_run_file_input_pattern_collides_with_temp_layer() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_file_coll_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("cities.geojson"); + std::fs::write(&path, r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[10.0,20.0]},"properties":{}}]}"#).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [path.to_str().unwrap()], + "outputPattern": "{input}", + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0], "cities_2"); + assert!(s.map().layer_by_name("cities_2").is_some()); + assert!(s.notices.iter().any(|n| n.text.contains("output name(s) collided and were renamed"))); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_output_layers_only_table_collected() { rustgis_analysis::register_external( diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 083caa64e6..2e2ab7c079 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -843,11 +843,19 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { // Its inputs may be bare file names, which don't look like paths. || id == "raster.mosaic" || (id == "gp.run" - && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) - .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) + && params + .get("tool") + .and_then(Value::as_str) + .and_then(super::gp::tool_id) + .and_then(rustgis_analysis::tool) + .is_some_and(super::gp::writes_files)) || (id == "gp.batch" - && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) - .and_then(rustgis_analysis::tool).is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) + && params + .get("tool") + .and_then(Value::as_str) + .and_then(super::gp::tool_id) + .and_then(rustgis_analysis::tool) + .is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" && (params.get("folder").is_some() || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) From da96ac4c0ce1e83d95405431464cd8613c8f51d2 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 20:49:01 +0700 Subject: [PATCH 20/56] fix(engine): emit notice for unused outputPattern and consolidate batch tool messages --- crates/engine/src/cmd/gp.rs | 129 +++++++++++++++++++++++++++++++----- 1 file changed, 112 insertions(+), 17 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 88b7594916..cd9be81df3 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1007,10 +1007,12 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc v.clone(), @@ -1186,7 +1188,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); - let mut tool_messages: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut items_reporting_messages = 0; + let mut first_tool_message = None; let mut cancelled_after = None; let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); @@ -1241,7 +1244,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc target.as_str(), None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), }; - let tool_has_output = spec.params.iter().any(|param| param.name == "output"); let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); let mut final_out_name = out_name.clone(); let mut had_collision = false; @@ -1266,13 +1268,17 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); - for msg in messages { - *tool_messages.entry(msg).or_insert(0) += 1; + if !messages.is_empty() { + items_reporting_messages += 1; + if first_tool_message.is_none() { + first_tool_message = messages.first().cloned(); + } } if had_collision { collisions += 1; @@ -1397,15 +1406,17 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc>().join("; "); + if items_reporting_messages > 0 { + let first = first_tool_message.unwrap_or_default(); + s.notify(format!("gp.batch: {items_reporting_messages} item(s) reported messages (first: {first})"), false); + } + + if explicit_pattern.is_some() && !tool_has_output && !succeeded.is_empty() { + let count = succeeded.len(); s.notify( - format!("gp.batch: {total_reported} message(s) reported across items (e.g. {sample}; and {} more)", tool_messages.len() - 2), + format!( + "gp.batch: 'outputPattern' does not apply to '{tool_id}' (it names its outputs itself); {count} item(s) used their own names" + ), false, ); } @@ -3724,7 +3735,91 @@ mod batch_tests { assert_eq!(res["succeeded"], 2); let notices = &s.notices[mark..]; - assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported: Buffer: 1 features at 10 Kilometers"))); + assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"))); + } + + #[test] + fn batch_run_different_feature_counts_emits_single_message_notice() { + let mut s = Session::new(crate::Project::default()); + let mut l1 = Dataset::new("Layer1", rustgis_data::GeometryType::Point, Vec::new()); + l1.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, l1)); + + let mut l2 = Dataset::new("Layer2", rustgis_data::GeometryType::Point, Vec::new()); + l2.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(15.0, 25.0))), Vec::new()); + l2.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(16.0, 26.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(1, l2)); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Layer1", "Layer2"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 2); + let recent: Vec<_> = s.notices[mark..].iter().filter(|n| n.text.contains("reported messages")).collect(); + assert_eq!(recent.len(), 1, "must emit exactly one tool messages notice: {recent:?}"); + assert_eq!(recent[0].text, "gp.batch: 2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn batch_run_output_pattern_on_tool_without_output_emits_notice() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_opt_notice_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + let p1 = root.join("cities1.geojson"); + let p2 = root.join("cities2.geojson"); + let content = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[10.0,20.0]},"properties":{"pop":100}}]}"#; + std::fs::write(&p1, content).unwrap(); + std::fs::write(&p2, content).unwrap(); + + // 1. Run with outputPattern: emits notice + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "summaryStatistics", + "inputs": [p1.to_str().unwrap(), p2.to_str().unwrap()], + "outputPattern": "{input}_stats", + "params": {"field": "pop"} + }), + ) + .unwrap(); + assert_eq!(res["succeeded"], 2); + assert!( + s.notices.iter().any(|n| n.text.contains( + "'outputPattern' does not apply to 'summaryStatistics' (it names its outputs itself); 2 item(s) used their own names" + )), + "notices: {:?}", + s.notices + ); + + // 2. Clean run without outputPattern: emits no notice + let mut s_clean = Session::new(crate::Project::default()); + let res_clean = s_clean + .run( + "gp.batch", + &json!({ + "tool": "summaryStatistics", + "inputs": [p1.to_str().unwrap(), p2.to_str().unwrap()], + "params": {"field": "pop"} + }), + ) + .unwrap(); + assert_eq!(res_clean["succeeded"], 2); + assert!(!s_clean.notices.iter().any(|n| n.text.contains("'outputPattern' does not apply")), "clean run must not emit notice"); + + let _ = std::fs::remove_dir_all(root); } #[test] @@ -3839,7 +3934,7 @@ mod batch_tests { assert_eq!(res["succeeded"], 2); let recent = s.notices_since(mark); assert!(!recent.iter().any(|t| t.text.contains("Prior notification"))); - assert!(recent.iter().any(|t| t.text.contains("2 item(s) reported: Buffer: 1 features at 10 Kilometers"))); + assert!(recent.iter().any(|t| t.text.contains("2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"))); } #[test] From 9b6da4be9e79d696916c6cbb3a601433dd031fbe Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 21:08:05 +0700 Subject: [PATCH 21/56] fix(engine): narrow batch file-access check to file inputs, warn on unrecognised options, and strengthen sidecar tests --- crates/engine/src/cmd/gp.rs | 114 ++++++++++++++++++++++++------- crates/engine/src/cmd/items.rs | 18 ++++- crates/engine/src/cmd/plugins.rs | 13 ++-- docs/user-guide/analysis.md | 2 + 4 files changed, 119 insertions(+), 28 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index cd9be81df3..ea7d386311 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -981,6 +981,18 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = map.keys().map(String::as_str).filter(|k| !recognized.contains(k)).collect(); + if !unknown_keys.is_empty() { + unknown_keys.sort(); + let count = unknown_keys.len(); + let sample = unknown_keys.first().copied().unwrap_or_default(); + let plural = if count == 1 { "option" } else { "options" }; + s.notify(format!("gp.batch: {count} unrecognised {plural} ignored ({sample})"), false); + } + } + // Identify which parameter to bind the input to. let batch_param = ps(p, "param").or_else(|| ps(p, "batch_param")).unwrap_or_else(|| { spec.params @@ -2706,6 +2718,12 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard)); assert!(reaches.is_none()); + // Plain project layers do not touch files. + let p_layers = json!({"tool": "convexHull", "inputs": ["Cities", "Parks"]}); + assert!(!crate::cmd::items::touches_files("gp.batch", &p_layers)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_layers)); + assert!(reaches.is_none()); + // Plain gp.run of merge on project layers does not touch files. let p_run_merge = json!({"tool": "merge", "params": {"inputs": ["Cities", "Parks"]}}); assert!(!crate::cmd::items::touches_files("gp.run", &p_run_merge)); @@ -3507,14 +3525,17 @@ mod batch_tests { std::fs::write(&f2, b"").unwrap(); let mut s = Session::new(crate::Project::default()); - let _ = s.run( - "gp.batch", - &json!({ - "tool": "convexHull", - "folder": root.display().to_string(), - }), - ); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + }), + ) + .unwrap(); + assert_eq!(res["total"], 1); assert!(!s.notices.iter().any(|n| n.text.contains("unsupported files skipped"))); let _ = std::fs::remove_dir_all(root); @@ -3822,6 +3843,47 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[test] + fn batch_run_unrecognised_option_emits_notice() { + let mut s = test_session(); + let mark = s.notices.len() as u64; + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "outputpattern": "{input}_out", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res["succeeded"], 1); + let recent = s.notices_since(mark); + assert!(recent.iter().any(|n| n.text.contains("1 unrecognised option ignored (outputpattern)")), "recent notices: {recent:?}"); + + // Clean call emits no unrecognised option notice + let mut s_clean = test_session(); + let mark_clean = s_clean.notices.len() as u64; + let res_clean = s_clean + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "outputPattern": "{input}_out", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res_clean["succeeded"], 1); + let recent_clean = s_clean.notices_since(mark_clean); + assert!( + !recent_clean.iter().any(|n| n.text.contains("unrecognised")), + "clean call must not emit unrecognised option notice: {recent_clean:?}" + ); + } + #[test] fn batch_run_failed_item_does_not_reserve_output_name() { let root = std::env::temp_dir().join(format!( @@ -3880,14 +3942,17 @@ mod batch_tests { let mut s = Session::new(crate::Project::default()); let mark = s.notices.len(); - let _ = s.run( - "gp.batch", - &json!({ - "tool": "buffer", - "folder": root.display().to_string(), - "params": {"distance": 10} - }), - ); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "folder": root.display().to_string(), + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res["total"], 1); let notices = &s.notices[mark..]; assert!( !notices.iter().any(|n| n.text.contains("unsupported files skipped")), @@ -3897,14 +3962,17 @@ mod batch_tests { std::fs::write(root.join("notes.txt"), b"some notes").unwrap(); let mut s2 = Session::new(crate::Project::default()); let mark2 = s2.notices.len(); - let _ = s2.run( - "gp.batch", - &json!({ - "tool": "buffer", - "folder": root.display().to_string(), - "params": {"distance": 10} - }), - ); + let res2 = s2 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "folder": root.display().to_string(), + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res2["total"], 1); let notices2 = &s2.notices[mark2..]; assert!( notices2.iter().any(|n| n.text.contains("1 unsupported files skipped")), diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 2e2ab7c079..da70789703 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -858,7 +858,11 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { .is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" && (params.get("folder").is_some() - || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) + || params.get("inputs").is_some_and(|inp| match inp { + Value::Array(a) => a.iter().any(|v| v.as_str().is_some_and(is_file_like_batch_input)), + Value::String(s) => !super::gp::is_wildcard_inputs(Some(inp)) && is_file_like_batch_input(s), + _ => false, + }))) || reaches(params) } @@ -906,6 +910,18 @@ pub(crate) fn looks_like_path(t: &str) -> bool { || (b.get(1) == Some(&b':') && b.first().is_some_and(u8::is_ascii_alphabetic)) } +/// Whether a batch input string looks like a file path, contains a path separator, or has a geospatial file extension. +pub(crate) fn is_file_like_batch_input(t: &str) -> bool { + let t = t.trim(); + looks_like_path(t) + || t.contains('/') + || t.contains('\\') + || matches!( + std::path::Path::new(t).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase().as_str(), + "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet" + ) +} + /// Whether parameters name a file, folder or address: a key in `keys` anywhere in them, or a /// string that looks like a path or URL. Too deep a nesting counts as yes. pub(crate) fn names_place(v: &Value, keys: &[&str], depth: usize) -> bool { diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 84e93eba01..f61c21e636 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -377,10 +377,15 @@ pub fn reaches(spec: &C, params: &Value) -> Option { if params.get("folder").is_some() { return Some(Reach::Files); } - if let Some(inputs) = params.get("inputs") - && !super::gp::is_wildcard_inputs(Some(inputs)) - { - return Some(Reach::Files); + if let Some(inputs) = params.get("inputs") { + let has_file_input = match inputs { + Value::Array(a) => a.iter().any(|v| v.as_str().is_some_and(super::items::is_file_like_batch_input)), + Value::String(s) => !super::gp::is_wildcard_inputs(Some(inputs)) && super::items::is_file_like_batch_input(s), + _ => false, + }; + if has_file_input { + return Some(Reach::Files); + } } return super::items::names_place(params, PLACE_KEYS, 0).then_some(Reach::Files); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 9ef6597455..39d1a76371 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -141,6 +141,8 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. +Batch runs targeting files or directories (using `folder` or `inputs` containing paths or file extensions such as `.shp`, `.gpkg`, or `.geojson`) require file access permissions (`Reach::Files`), prompting for confirmation in the AI Assistant and restricted in imported tasks. In contrast, batch runs over existing project layers (using layer names or `*`) execute entirely within the project and do not require file access permissions. + These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From 84e20fd0eae6983d57384e1921358342f358e86e Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 21:20:33 +0700 Subject: [PATCH 22/56] fix(engine): class all non-wildcard batch inputs as file access in reaches and touches_files --- crates/engine/src/cmd/gp.rs | 24 ++++++++++++++++++------ crates/engine/src/cmd/items.rs | 18 +----------------- crates/engine/src/cmd/plugins.rs | 13 ++++--------- docs/user-guide/analysis.md | 2 +- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index ea7d386311..909bb88d2b 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -2713,15 +2713,27 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_inputs)); assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); - let p_wildcard = json!({"tool": "convexHull", "inputs": "*"}); - assert!(!crate::cmd::items::touches_files("gp.batch", &p_wildcard)); - let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard)); - assert!(reaches.is_none()); + // Non-wildcard inputs (including CSVs, bare names, and layer names) are classed as file access + // because reaches has no session to distinguish layer names from files in the working directory. + let p_csv = json!({"tool": "convexHull", "inputs": ["payroll.csv"]}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_csv)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_csv)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + + let p_bare = json!({"tool": "convexHull", "inputs": ["notes"]}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_bare)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_bare)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); - // Plain project layers do not touch files. let p_layers = json!({"tool": "convexHull", "inputs": ["Cities", "Parks"]}); - assert!(!crate::cmd::items::touches_files("gp.batch", &p_layers)); + assert!(crate::cmd::items::touches_files("gp.batch", &p_layers)); let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_layers)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + + // Only wildcard inputs '*' runs without file access (strictly over active map vector layers). + let p_wildcard = json!({"tool": "convexHull", "inputs": "*"}); + assert!(!crate::cmd::items::touches_files("gp.batch", &p_wildcard)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard)); assert!(reaches.is_none()); // Plain gp.run of merge on project layers does not touch files. diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index da70789703..2e2ab7c079 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -858,11 +858,7 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { .is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" && (params.get("folder").is_some() - || params.get("inputs").is_some_and(|inp| match inp { - Value::Array(a) => a.iter().any(|v| v.as_str().is_some_and(is_file_like_batch_input)), - Value::String(s) => !super::gp::is_wildcard_inputs(Some(inp)) && is_file_like_batch_input(s), - _ => false, - }))) + || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) || reaches(params) } @@ -910,18 +906,6 @@ pub(crate) fn looks_like_path(t: &str) -> bool { || (b.get(1) == Some(&b':') && b.first().is_some_and(u8::is_ascii_alphabetic)) } -/// Whether a batch input string looks like a file path, contains a path separator, or has a geospatial file extension. -pub(crate) fn is_file_like_batch_input(t: &str) -> bool { - let t = t.trim(); - looks_like_path(t) - || t.contains('/') - || t.contains('\\') - || matches!( - std::path::Path::new(t).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase().as_str(), - "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet" - ) -} - /// Whether parameters name a file, folder or address: a key in `keys` anywhere in them, or a /// string that looks like a path or URL. Too deep a nesting counts as yes. pub(crate) fn names_place(v: &Value, keys: &[&str], depth: usize) -> bool { diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index f61c21e636..84e93eba01 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -377,15 +377,10 @@ pub fn reaches(spec: &C, params: &Value) -> Option { if params.get("folder").is_some() { return Some(Reach::Files); } - if let Some(inputs) = params.get("inputs") { - let has_file_input = match inputs { - Value::Array(a) => a.iter().any(|v| v.as_str().is_some_and(super::items::is_file_like_batch_input)), - Value::String(s) => !super::gp::is_wildcard_inputs(Some(inputs)) && super::items::is_file_like_batch_input(s), - _ => false, - }; - if has_file_input { - return Some(Reach::Files); - } + if let Some(inputs) = params.get("inputs") + && !super::gp::is_wildcard_inputs(Some(inputs)) + { + return Some(Reach::Files); } return super::items::names_place(params, PLACE_KEYS, 0).then_some(Reach::Files); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 39d1a76371..b4da9d89f2 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -141,7 +141,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. -Batch runs targeting files or directories (using `folder` or `inputs` containing paths or file extensions such as `.shp`, `.gpkg`, or `.geojson`) require file access permissions (`Reach::Files`), prompting for confirmation in the AI Assistant and restricted in imported tasks. In contrast, batch runs over existing project layers (using layer names or `*`) execute entirely within the project and do not require file access permissions. +Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. Only the wildcard `inputs: "*"` operates purely on the active map's in-memory vector layers without file access. These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From 6f608202aaf51abff84d1dee6e1e1ac084019578 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 21:36:23 +0700 Subject: [PATCH 23/56] fix(engine): resolve GeoLibre tool ids in reaches, notify on overwritten batch_param, and update docs --- crates/engine/src/cmd/gp.rs | 45 ++++++++++++++++++++++++++++++++ crates/engine/src/cmd/plugins.rs | 2 +- docs/user-guide/analysis.md | 4 +-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 909bb88d2b..6bf0637ac5 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1037,6 +1037,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); @@ -3896,6 +3899,48 @@ mod batch_tests { ); } + #[test] + fn batch_run_params_batch_param_overwritten_notified() { + let mut s = test_session(); + let mark = s.notices.len() as u64; + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": {"input": "Parks", "distance": 10} + }), + ) + .unwrap(); + assert_eq!(res["succeeded"], 1); + let recent = s.notices_since(mark); + assert!( + recent.iter().any(|n| n.text.contains("gp.batch: 'params.input' is replaced by the batch inputs")), + "recent notices: {recent:?}" + ); + + // Clean call without batch_param in params gives no replacement notice + let mut s_clean = test_session(); + let mark_clean = s_clean.notices.len() as u64; + let res_clean = s_clean + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res_clean["succeeded"], 1); + let recent_clean = s_clean.notices_since(mark_clean); + assert!( + !recent_clean.iter().any(|n| n.text.contains("is replaced by the batch inputs")), + "clean call must not emit replacement notice: {recent_clean:?}" + ); + } + #[test] fn batch_run_failed_item_does_not_reserve_output_name() { let root = std::env::temp_dir().join(format!( diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 84e93eba01..1ced170ede 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -358,7 +358,7 @@ pub fn reaches(spec: &C, params: &Value) -> Option { // External tools (the GeoLibre suite) run a native program that reads files a parameter // names, including bare file names. if (spec.id == "gp.run" || spec.id == "gp.batch") - && ps(params, "tool").and_then(rustgis_analysis::tool).is_some_and(super::gp::is_geolibre_tool) + && ps(params, "tool").and_then(super::gp::tool_id).and_then(rustgis_analysis::tool).is_some_and(super::gp::is_geolibre_tool) { return Some(Reach::Files); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index b4da9d89f2..83fe1b4c1b 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -136,12 +136,12 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. - `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. -- `"params"`: the static parameters shared by each run. +- `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. -Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. Only the wildcard `inputs: "*"` operates purely on the active map's in-memory vector layers without file access. +Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. A wildcard run (`inputs: "*"`) operates without file access only when the underlying tool does not read or write files on disk (tools without `Path`/`VectorLayers` parameters, file-writing outputs, or native GeoLibre execution). These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From 231babc0dc9d383eb47d3757bcadc9d023f0b057 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 21:42:51 +0700 Subject: [PATCH 24/56] docs(analysis): clarify wildcard batch file-access conditions --- docs/user-guide/analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 83fe1b4c1b..f9e54fad62 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -141,7 +141,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. -Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. A wildcard run (`inputs: "*"`) operates without file access only when the underlying tool does not read or write files on disk (tools without `Path`/`VectorLayers` parameters, file-writing outputs, or native GeoLibre execution). +Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. A wildcard run (`inputs: "*"`) is file access only when the tool reads files through a `Path` or `VectorLayers` parameter or runs a GeoLibre program; otherwise it runs over the active map's layers without file access. Tools that write files are refused. These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From 8284885e8456cf7b5690dc9b9c506b00153c239d Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 22:12:53 +0700 Subject: [PATCH 25/56] fix(engine): record file inputs as replayable batch commands, treat empty outputs as errors, and preserve in-flight cancel errors --- crates/engine/src/cmd/gp.rs | 139 +++++++++++++++++++++++++++---- crates/engine/src/cmd/history.rs | 2 +- 2 files changed, 123 insertions(+), 18 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 6bf0637ac5..0f244116ff 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1217,7 +1217,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); + let item_msg = messages.first().cloned(); if !messages.is_empty() { items_reporting_messages += 1; if first_tool_message.is_none() { - first_tool_message = messages.first().cloned(); + first_tool_message = item_msg.clone(); } } if had_collision { @@ -1392,22 +1408,33 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { + failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { - cancelled_after = Some(i); + cancelled_after = Some(i + 1); #[cfg(not(target_arch = "wasm32"))] - if let Some(temp_id) = temp_layer_id { + if let Some(temp_id) = temp_layer_id.take() { super::layer::drop_layers(s, &[temp_id]); } s.active_layer = active_before; break; } - failed.push(json!({"input": target, "error": err})); } } @@ -1558,7 +1585,7 @@ pub fn run_tool_with( params: &Value, control: &std::sync::Arc, ) -> CmdResult { - run_tool_internal(s, tool, params, None, control, true, true).map(|(v, _)| v) + run_tool_internal(s, tool, params, None, None, control, true, true).map(|(v, _)| v) } /// Runs a geoprocessing tool quietly without forwarding intermediate notices to the session. @@ -1567,9 +1594,10 @@ pub(crate) fn run_tool_quiet( tool: &str, params: &Value, record_params: Option<&Value>, + command: Option<(String, Value)>, control: &std::sync::Arc, ) -> Result<(Value, Vec), String> { - run_tool_internal(s, tool, params, record_params, control, false, false) + run_tool_internal(s, tool, params, record_params, command, control, false, false) } /// Internal runner for geoprocessing tools, executing the tool and capturing outputs and notices. @@ -1578,6 +1606,7 @@ fn run_tool_internal( tool: &str, params: &Value, record_params: Option<&Value>, + command: Option<(String, Value)>, control: &std::sync::Arc, notify_messages: bool, replace_tables: bool, @@ -1592,7 +1621,7 @@ fn run_tool_internal( let e = format!("{name} was cancelled before it started"); return apply_inner( s, - Ran { name, id: tool, command: None, started: t0 }, + Ran { name, id: tool, command, started: t0 }, recorded(spec, rec_val.clone()), Err(e), 0, @@ -1613,7 +1642,7 @@ fn run_tool_internal( let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); apply_inner( s, - Ran { name, id: tool, command: None, started: t0 }, + Ran { name, id: tool, command, started: t0 }, recorded(spec, rec_val.clone()), res, millis, @@ -3435,6 +3464,13 @@ mod batch_tests { assert!(s.map().layer_by_name(outputs[0].as_str().unwrap()).is_some()); assert_eq!(s.project.gp_history[0].params["input"], f1.display().to_string()); + let (replay_cmd, replay_params) = crate::cmd::history::replay(&s.project.gp_history[0]).unwrap(); + assert_eq!(replay_cmd, "gp.batch"); + assert_eq!(replay_params["tool"], "calculateGeometry"); + assert_eq!(replay_params["inputs"][0], f1.display().to_string()); + let replay_res = s.run(&replay_cmd, &replay_params).unwrap(); + assert_eq!(replay_res["succeeded"], 1); + let _ = std::fs::remove_dir_all(root); } @@ -3477,7 +3513,7 @@ mod batch_tests { } #[test] - fn batch_run_cancel_during_item_leaves_failed_empty() { + fn batch_run_cancel_during_item_records_failed_and_error() { let control = rustgis_analysis::control::Control::new(); *CANCEL_CTL.lock().unwrap() = Some(control.clone()); rustgis_analysis::register_external( @@ -3518,12 +3554,74 @@ mod batch_tests { *CANCEL_CTL.lock().unwrap() = None; - assert_eq!(res["failed"], 0); + assert_eq!(res["failed"], 1); assert_eq!(res["succeeded"], 0); assert_eq!(res["cancelled"], true); - assert_eq!(res["errors"].as_array().unwrap().len(), 0); - assert!(!s.notices.iter().any(|n| n.text.contains("failed"))); - assert!(s.notices.iter().any(|n| n.text.contains("cancelled after 0 of 1"))); + assert_eq!(res["errors"].as_array().unwrap().len(), 1); + assert_eq!(res["errors"][0]["error"], "Cancel Item Tool was cancelled"); + assert!(s.notices.iter().any(|n| n.text.contains("1 failed (L1)"))); + assert!(s.notices.iter().any(|n| n.text.contains("cancelled after 1 of 1"))); + } + + fn empty_output_runner( + _spec: &rustgis_analysis::ToolSpec, + _inputs: &dyn rustgis_analysis::Inputs, + _params: &serde_json::Value, + ) -> Result { + Ok(rustgis_analysis::ToolResult { + output: rustgis_analysis::Output::Layers(Vec::new()), + messages: vec!["custom message from empty tool".to_string()], + }) + } + + #[test] + fn batch_run_tool_producing_no_output_is_failure() { + rustgis_analysis::register_external( + "test_empty_output_suite", + || { + vec![rustgis_analysis::ToolSpec { + id: "test:empty_output", + name: "Empty Output Tool", + toolbox: "Test", + summary: "", + params: &[rustgis_analysis::ParamSpec { + name: "input", + label: "Input Layer", + kind: rustgis_analysis::ParamKind::Layer, + required: true, + default: "", + help: "", + }], + run: rustgis_analysis::external_placeholder, + }] + }, + empty_output_runner, + ); + + let mut s = Session::new(crate::Project::default()); + let ds = Dataset::new("L1", rustgis_data::GeometryType::Point, Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "test:empty_output", + "inputs": ["L1"], + }), + ) + .unwrap(); + + assert_eq!(res["failed"], 1); + assert_eq!(res["succeeded"], 0); + let errs = res["errors"].as_array().unwrap(); + assert_eq!(errs.len(), 1); + assert_eq!(errs[0]["input"], "L1"); + assert_eq!(errs[0]["error"], "custom message from empty tool"); + let outputs = res["outputs"].as_array().unwrap(); + assert!(outputs.is_empty(), "must not emit phantom output name"); + assert!(s.notices[mark..].iter().any(|n| n.text.contains("1 failed (L1)"))); } #[test] @@ -3751,6 +3849,13 @@ mod batch_tests { assert!(s.map().layer_by_name(outputs[0].as_str().unwrap()).is_some()); assert_eq!(s.project.gp_history[0].params["input"], f1.display().to_string()); + let (replay_cmd, replay_params) = crate::cmd::history::replay(&s.project.gp_history[0]).unwrap(); + assert_eq!(replay_cmd, "gp.batch"); + assert_eq!(replay_params["tool"], "selectByAttributes"); + assert_eq!(replay_params["inputs"][0], f1.display().to_string()); + let replay_res = s.run(&replay_cmd, &replay_params).unwrap(); + assert_eq!(replay_res["succeeded"], 1); + let _ = std::fs::remove_dir_all(root); } diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 3a22bd151f..5c4df748f6 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -83,7 +83,7 @@ fn script(s: &mut Session, _: &Value) -> CmdResult { /// The commands other than `gp.run` a run is copied as. The history comes from the project file, /// which may say anything: only these are written into code, so a crafted file can't pass any /// other command off as a geoprocessing run (and an id is never text to escape). -const REPLAYED: &[&str] = &["gp.bufferPoint"]; +const REPLAYED: &[&str] = &["gp.bufferPoint", "gp.batch"]; /// The command that runs `r` again, with its parameters: `gp.run` with the tool, or the command /// that ran it (a point buffered where the map was clicked, `gp.bufferPoint`). From 3d6e70d763b608583aa06017ba51ed461a9d3f63 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 23:14:28 +0700 Subject: [PATCH 26/56] fix(engine): record explicit param in file batch history, prevent temp layer false collision, and record interrupted cancel in errors --- crates/engine/src/cmd/gp.rs | 189 ++++++++++++++++++++++++++++-------- 1 file changed, 147 insertions(+), 42 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 0f244116ff..670a12400d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1196,6 +1196,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { failed.push(json!({"input": target, "error": e.to_string()})); + errors.push(json!({"input": target, "error": e.to_string()})); s.active_layer = active_before; continue; } @@ -1261,11 +1264,11 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { - failed.push(json!({"input": target, "error": err})); if control.is_cancelled() { cancelled_after = Some(i + 1); + interrupted_error = Some(err.clone()); + errors.push(json!({"input": target, "error": err, "interrupted": true})); #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id.take() { super::layer::drop_layers(s, &[temp_id]); @@ -1435,6 +1456,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 50"} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let (replay_cmd, replay_params) = crate::cmd::history::replay(&s.project.gp_history[0]).unwrap(); + assert_eq!(replay_cmd, "gp.batch"); + assert_eq!(replay_params["param"], "input"); let replay_res = s.run(&replay_cmd, &replay_params).unwrap(); assert_eq!(replay_res["succeeded"], 1); From 14ab4c72c121c2301606e5da4fd0d771405a0c1f Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 23:18:46 +0700 Subject: [PATCH 27/56] fix(engine): define temp_layer_id as None on wasm32 for batch collision check --- crates/engine/src/cmd/gp.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 670a12400d..fc3a19f989 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1241,6 +1241,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = None; let target_layer_name = { #[cfg(not(target_arch = "wasm32"))] From 490dcd2c1cfd7bba319d80c097a12189096eb987 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 23:23:34 +0700 Subject: [PATCH 28/56] fix(engine): remove unused mut on wasm32 for temp_layer_id --- crates/engine/src/cmd/gp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index fc3a19f989..91c447c24a 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1242,7 +1242,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = None; + let temp_layer_id: Option = None; let target_layer_name = { #[cfg(not(target_arch = "wasm32"))] From 1ac17c9850a2dc70ad5198ca0ca19c8c0ac4d0aa Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Thu, 5 Nov 2026 23:40:47 +0700 Subject: [PATCH 29/56] fix(engine): treat null batch options as absent and emit no spurious notices --- crates/engine/src/cmd/gp.rs | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 91c447c24a..8f8c3f813e 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -971,13 +971,15 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc v.clone(), Some(_) => { s.notify("gp.batch: 'params' must be an object, ignoring invalid value", false); @@ -1079,7 +1081,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 00:07:27 +0700 Subject: [PATCH 30/56] fix(engine): handle non-utf8 filenames in batch folder scan and refine null parameter checks --- crates/engine/src/cmd/gp.rs | 78 +++++++++++++++++++++++++++++++- crates/engine/src/cmd/items.rs | 6 +-- crates/engine/src/cmd/plugins.rs | 4 +- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8f8c3f813e..a1a55f3aa1 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1036,10 +1036,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc json!({}), }; - if base_params.get("output").is_some() { + if base_params.get("output").is_some_and(|v| !v.is_null()) { s.notify("params.output is ignored for batch runs; outputPattern determines output names", false); } - if base_params.get(batch_param).is_some() { + if base_params.get(batch_param).is_some_and(|v| !v.is_null()) { s.notify(format!("gp.batch: 'params.{batch_param}' is replaced by the batch inputs"), false); } @@ -1105,6 +1105,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { @@ -1119,6 +1120,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("folder '{folder}': {symlinks} symlink(s) skipped"), false); } + if non_utf8 > 0 { + s.notify(format!("folder '{folder}': {non_utf8} file name(s) that are not valid UTF-8 skipped"), false); + } if subfolders > 0 { s.notify(format!("folder '{folder}': {subfolders} subfolder(s) skipped"), false); } @@ -2799,6 +2807,12 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard)); assert!(reaches.is_none()); + // Null folder with wildcard inputs '*' does not touch files. + let p_null_folder_wildcard = json!({"tool": "convexHull", "inputs": "*", "folder": null}); + assert!(!crate::cmd::items::touches_files("gp.batch", &p_null_folder_wildcard)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_null_folder_wildcard)); + assert!(reaches.is_none()); + // Plain gp.run of merge on project layers does not touch files. let p_run_merge = json!({"tool": "merge", "params": {"inputs": ["Cities", "Parks"]}}); assert!(!crate::cmd::items::touches_files("gp.run", &p_run_merge)); @@ -3117,6 +3131,43 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(unix)] + #[test] + fn batch_run_folder_skips_non_utf8_filename() { + use std::os::unix::ffi::OsStrExt; + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_non_utf8_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("valid.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let bad_name = std::ffi::OsStr::from_bytes(b"bad_\xff.geojson"); + let bad_file = root.join(bad_name); + let _ = std::fs::write(&bad_file, sample_geojson); + + let mut s = Session::new(crate::Project::default()); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + "output_pattern": "{input}_hull" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 1); + assert_eq!(res["succeeded"], 1); + assert!(s.notices.iter().any(|n| n.text.contains("1 file name(s) that are not valid UTF-8 skipped"))); + + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_table_output_reported() { let mut s = Session::new(crate::Project::default()); @@ -4045,6 +4096,29 @@ mod batch_tests { || n.text.contains("must be an array")), "null options must not emit spurious notices: {notices:?}" ); + + let mark2 = s.notices.len(); + let res2 = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "params": { + "input": null, + "output": null, + "distance": 10 + } + }), + ) + .unwrap(); + + assert_eq!(res2["succeeded"], 1); + let notices2 = &s.notices[mark2..]; + assert!( + !notices2.iter().any(|n| n.text.contains("output is ignored") || n.text.contains("is replaced by the batch inputs")), + "null param fields must not emit spurious notices: {notices2:?}" + ); } #[test] diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 2e2ab7c079..10ed40f3d1 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -857,8 +857,8 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { .and_then(rustgis_analysis::tool) .is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) || (id == "gp.batch" - && (params.get("folder").is_some() - || (params.get("inputs").is_some() && !super::gp::is_wildcard_inputs(params.get("inputs"))))) + && (params.get("folder").is_some_and(|v| !v.is_null()) + || (params.get("inputs").is_some_and(|v| !v.is_null()) && !super::gp::is_wildcard_inputs(params.get("inputs"))))) || reaches(params) } @@ -911,7 +911,7 @@ pub(crate) fn looks_like_path(t: &str) -> bool { pub(crate) fn names_place(v: &Value, keys: &[&str], depth: usize) -> bool { match v { Value::Object(o) if depth < 16 => { - o.iter().any(|(k, v)| keys.contains(&k.to_ascii_lowercase().as_str()) || names_place(v, keys, depth + 1)) + o.iter().any(|(k, v)| (!v.is_null() && keys.contains(&k.to_ascii_lowercase().as_str())) || names_place(v, keys, depth + 1)) } Value::Array(a) if depth < 16 => a.iter().any(|v| names_place(v, keys, depth + 1)), Value::Object(_) | Value::Array(_) => true, diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 1ced170ede..1f6f61eeea 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -374,10 +374,10 @@ pub fn reaches(spec: &C, params: &Value) -> Option { return Some(Reach::Files); } if spec.id == "gp.batch" { - if params.get("folder").is_some() { + if params.get("folder").is_some_and(|v| !v.is_null()) { return Some(Reach::Files); } - if let Some(inputs) = params.get("inputs") + if let Some(inputs) = params.get("inputs").filter(|v| !v.is_null()) && !super::gp::is_wildcard_inputs(Some(inputs)) { return Some(Reach::Files); From fc632f3013f0193bae9e6f05fb9a2a323a35122c Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 00:23:05 +0700 Subject: [PATCH 31/56] fix(engine): strip batch params.output from file access checks and capture failure message on empty tool output --- crates/engine/src/cmd/gp.rs | 73 +++++++++++++++++++++++++++++++- crates/engine/src/cmd/items.rs | 10 +++++ crates/engine/src/cmd/plugins.rs | 3 +- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index a1a55f3aa1..c2f5afd056 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1441,7 +1441,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 1 { + err_obj["messages"] = json!(messages); + } + errors.push(err_obj); #[cfg(not(target_arch = "wasm32"))] if let Some(temp_id) = temp_layer_id.take() { super::layer::drop_layers(s, &[temp_id]); @@ -2813,6 +2817,12 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_null_folder_wildcard)); assert!(reaches.is_none()); + // Wildcard batch with params.output is ignored by gp.batch and does not touch files. + let p_wildcard_output = json!({"tool": "convexHull", "inputs": "*", "params": {"output": "Roads"}}); + assert!(!crate::cmd::items::touches_files("gp.batch", &p_wildcard_output)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard_output)); + assert!(reaches.is_none()); + // Plain gp.run of merge on project layers does not touch files. let p_run_merge = json!({"tool": "merge", "params": {"inputs": ["Cities", "Parks"]}}); assert!(!crate::cmd::items::touches_files("gp.run", &p_run_merge)); @@ -3748,6 +3758,65 @@ mod batch_tests { assert!(s.notices[mark..].iter().any(|n| n.text.contains("1 failed (L1)"))); } + fn empty_output_multi_message_runner( + _spec: &rustgis_analysis::ToolSpec, + _inputs: &dyn rustgis_analysis::Inputs, + _params: &serde_json::Value, + ) -> Result { + Ok(rustgis_analysis::ToolResult { + output: rustgis_analysis::Output::Layers(Vec::new()), + messages: vec!["Processed 10 features".to_string(), "0 features matched criteria".to_string()], + }) + } + + #[test] + fn batch_run_tool_producing_no_output_takes_last_message_as_error() { + rustgis_analysis::register_external( + "test_empty_output_multi_msg_suite", + || { + vec![rustgis_analysis::ToolSpec { + id: "test:empty_output_multi_msg", + name: "Empty Output Multi Message Tool", + toolbox: "Test", + summary: "", + params: &[rustgis_analysis::ParamSpec { + name: "input", + label: "Input Layer", + kind: rustgis_analysis::ParamKind::Layer, + required: true, + default: "", + help: "", + }], + run: rustgis_analysis::external_placeholder, + }] + }, + empty_output_multi_message_runner, + ); + + let mut s = Session::new(crate::Project::default()); + let ds = Dataset::new("L1", rustgis_data::GeometryType::Point, Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "test:empty_output_multi_msg", + "inputs": ["L1"], + }), + ) + .unwrap(); + + assert_eq!(res["failed"], 1); + let errs = res["errors"].as_array().unwrap(); + assert_eq!(errs.len(), 1); + assert_eq!(errs[0]["error"], "0 features matched criteria"); + let msgs = errs[0]["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 2); + assert_eq!(msgs[0], "Processed 10 features"); + assert_eq!(msgs[1], "0 features matched criteria"); + } + #[test] fn batch_run_folder_skips_shp_xml_sidecar() { let root = std::env::temp_dir().join(format!( diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 10ed40f3d1..d463b84f82 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -883,6 +883,16 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: let named = || params.pointer("/params/output").and_then(Value::as_str).is_some_and(plain); // Copied only when there is a name to leave out: other commands' parameters (a file's // contents, say) are read as they are. + if command == "gp.batch" { + if params.pointer("/params/output").is_some() { + let mut out = params.clone(); + if let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) { + p.remove("output"); + } + return Cow::Owned(out); + } + return Cow::Borrowed(params); + } if command != "gp.run" || !built_in() || !named() { return Cow::Borrowed(params); } diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 1f6f61eeea..2fa8740eae 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -382,7 +382,8 @@ pub fn reaches(spec: &C, params: &Value) -> Option { { return Some(Reach::Files); } - return super::items::names_place(params, PLACE_KEYS, 0).then_some(Reach::Files); + let stripped = super::items::layer_name_left_out(spec.id, params); + return super::items::names_place(&stripped, PLACE_KEYS, 0).then_some(Reach::Files); } // A catalog's Zarr entry is opened as `layer.addZarr` is, a database's way. if spec.id == "cube.open" && super::cube_catalog::opens_zarr(params) { From df2fb190241b6ea62954bf146edb803ea93eca05 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 00:42:19 +0700 Subject: [PATCH 32/56] fix(engine): omit existing map layer selection from batch outputs and scope batch null checks --- crates/engine/src/cmd/gp.rs | 65 ++++++++++++++++++++++++++++++++-- crates/engine/src/cmd/items.rs | 27 ++++++++++---- 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index c2f5afd056..3db3adcdf1 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1036,7 +1036,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc json!({}), }; - if base_params.get("output").is_some_and(|v| !v.is_null()) { + if tool_has_output && base_params.get("output").is_some_and(|v| !v.is_null()) { s.notify("params.output is ignored for batch runs; outputPattern determines output names", false); } if base_params.get(batch_param).is_some_and(|v| !v.is_null()) { @@ -1436,9 +1436,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 50"} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert!(outputs.is_empty(), "selection on existing map layer must not report layer as an output: {outputs:?}"); + assert_eq!(s.map().layer_by_name("Cities").unwrap().selection.len(), 1); + } + + #[test] + fn batch_run_tool_without_output_param_set_emits_no_ignored_notice() { + let mut s = Session::new(crate::Project::default()); + let mut ds = Dataset::new( + "Cities", + rustgis_data::GeometryType::Point, + vec![rustgis_data::Field::new("pop", rustgis_data::FieldType::Integer)], + ); + ds.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), vec![rustgis_data::Value::Int(100)]); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "summaryStatistics", + "inputs": ["Cities"], + "params": {"field": "pop", "output": "ignored_or_passthrough"} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let notices = &s.notices[mark..]; + assert!( + !notices.iter().any(|n| n.text.contains("params.output is ignored")), + "tool without output param must not emit 'params.output is ignored' notice: {notices:?}" + ); + } + #[test] fn batch_run_explicit_param_retained_in_history_replay() { let root = std::env::temp_dir().join(format!( diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index d463b84f82..ba7ce35936 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -884,11 +884,26 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: // Copied only when there is a name to leave out: other commands' parameters (a file's // contents, say) are read as they are. if command == "gp.batch" { - if params.pointer("/params/output").is_some() { - let mut out = params.clone(); - if let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) { - p.remove("output"); - } + let mut out = params.clone(); + let mut modified = false; + if let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) + && p.remove("output").is_some() + { + modified = true; + } + if out.get("folder").is_some_and(Value::is_null) + && let Some(obj) = out.as_object_mut() + { + obj.remove("folder"); + modified = true; + } + if out.get("inputs").is_some_and(Value::is_null) + && let Some(obj) = out.as_object_mut() + { + obj.remove("inputs"); + modified = true; + } + if modified { return Cow::Owned(out); } return Cow::Borrowed(params); @@ -921,7 +936,7 @@ pub(crate) fn looks_like_path(t: &str) -> bool { pub(crate) fn names_place(v: &Value, keys: &[&str], depth: usize) -> bool { match v { Value::Object(o) if depth < 16 => { - o.iter().any(|(k, v)| (!v.is_null() && keys.contains(&k.to_ascii_lowercase().as_str())) || names_place(v, keys, depth + 1)) + o.iter().any(|(k, v)| keys.contains(&k.to_ascii_lowercase().as_str()) || names_place(v, keys, depth + 1)) } Value::Array(a) if depth < 16 => a.iter().any(|v| names_place(v, keys, depth + 1)), Value::Object(_) | Value::Array(_) => true, From 49d1d0897ec3ecfc2eaaeec10f12b8eaa230ceb2 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 01:02:54 +0700 Subject: [PATCH 33/56] fix(engine): aggregate distinct tool and reader messages by kind in batch summary notices --- crates/engine/src/cmd/gp.rs | 79 +++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 3db3adcdf1..616777b915 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1216,7 +1216,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut items_reporting_messages = 0; - let mut first_tool_message = None; + let mut distinct_tool_messages: Vec<(String, usize)> = Vec::new(); let mut cancelled_after = None; let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); @@ -1332,11 +1332,14 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); - let item_msg = messages.first().cloned(); if !messages.is_empty() { items_reporting_messages += 1; - if first_tool_message.is_none() { - first_tool_message = item_msg.clone(); + for msg in &messages { + if let Some(entry) = distinct_tool_messages.iter_mut().find(|(m, _)| m == msg) { + entry.1 += 1; + } else { + distinct_tool_messages.push((msg.clone(), 1)); + } } } @@ -1488,8 +1491,14 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { - let first = first_tool_message.unwrap_or_default(); - s.notify(format!("gp.batch: {items_reporting_messages} item(s) reported messages (first: {first})"), false); + let sample = distinct_tool_messages + .iter() + .take(3) + .map(|(m, c)| if *c > 1 { format!("{m} (×{c})") } else { m.clone() }) + .collect::>() + .join("; "); + let more = if distinct_tool_messages.len() > 3 { format!(" and {} more", distinct_tool_messages.len() - 3) } else { String::new() }; + s.notify(format!("gp.batch: {items_reporting_messages} item(s) reported messages ({sample}{more})"), false); } if explicit_pattern.is_some() && !tool_has_output && !succeeded.is_empty() { @@ -1508,8 +1517,22 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); + for msg in &read_messages { + if let Some(entry) = distinct_reads.iter_mut().find(|(m, _)| *m == msg.as_str()) { + entry.1 += 1; + } else { + distinct_reads.push((msg.as_str(), 1)); + } + } + let sample = distinct_reads + .iter() + .take(3) + .map(|(m, c)| if *c > 1 { format!("{m} (×{c})") } else { m.to_string() }) + .collect::>() + .join("; "); + let more = if distinct_reads.len() > 3 { format!(" and {} more", distinct_reads.len() - 3) } else { String::new() }; + s.notify(format!("gp.batch: {} notice(s) reading inputs ({sample}{more})", read_messages.len()), false); } if !failed.is_empty() { @@ -4164,7 +4187,7 @@ mod batch_tests { assert_eq!(res["succeeded"], 2); let notices = &s.notices[mark..]; - assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"))); + assert!(notices.iter().any(|n| n.text.contains("2 item(s) reported messages (Buffer: 1 features at 10 Kilometers (×2))"))); } #[test] @@ -4194,7 +4217,41 @@ mod batch_tests { assert_eq!(res["succeeded"], 2); let recent: Vec<_> = s.notices[mark..].iter().filter(|n| n.text.contains("reported messages")).collect(); assert_eq!(recent.len(), 1, "must emit exactly one tool messages notice: {recent:?}"); - assert_eq!(recent[0].text, "gp.batch: 2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"); + assert_eq!( + recent[0].text, + "gp.batch: 2 item(s) reported messages (Buffer: 1 features at 10 Kilometers; Buffer: 2 features at 10 Kilometers)" + ); + } + + #[test] + fn batch_run_different_tool_messages_named_in_notice() { + let mut s = Session::new(crate::Project::default()); + let mut l1 = Dataset::new("Layer1", rustgis_data::GeometryType::Point, Vec::new()); + l1.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, l1)); + + let mut l2 = Dataset::new("Layer2", rustgis_data::GeometryType::Point, Vec::new()); + l2.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(15.0, 25.0))), Vec::new()); + l2.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(16.0, 26.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(1, l2)); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Layer1", "Layer2"], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 2); + let notices = &s.notices[mark..]; + let msg_notice = notices.iter().find(|n| n.text.contains("reported messages")).unwrap(); + assert!(msg_notice.text.contains("Buffer: 1 features at 10 Kilometers")); + assert!(msg_notice.text.contains("Buffer: 2 features at 10 Kilometers")); } #[test] @@ -4504,7 +4561,7 @@ mod batch_tests { assert_eq!(res["succeeded"], 2); let recent = s.notices_since(mark); assert!(!recent.iter().any(|t| t.text.contains("Prior notification"))); - assert!(recent.iter().any(|t| t.text.contains("2 item(s) reported messages (first: Buffer: 1 features at 10 Kilometers)"))); + assert!(recent.iter().any(|t| t.text.contains("2 item(s) reported messages (Buffer: 1 features at 10 Kilometers (×2))"))); } #[test] From acda179579d36622a2ba662eef42335bc274f856 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 01:14:07 +0700 Subject: [PATCH 34/56] fix(engine): emit specific error when batch wildcard matches no vector layers --- crates/engine/src/cmd/gp.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 616777b915..6392d48734 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1198,6 +1198,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 01:43:08 +0700 Subject: [PATCH 35/56] fix(engine): preserve raw input strings for layer lookup and notify on trimmed file paths --- crates/engine/src/cmd/gp.rs | 90 +++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 6392d48734..3120a6d15d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1064,9 +1064,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { for msg in report.messages { @@ -4675,4 +4685,68 @@ mod batch_tests { ); assert_eq!(res.unwrap_err(), "'*' matched no vector layers on the active map; add a vector layer or name the inputs"); } + + #[test] + fn batch_run_preserves_layer_name_with_surrounding_whitespace() { + let mut s = Session::new(crate::Project::default()); + let mut ds1 = Dataset::new(" Cities ", rustgis_data::GeometryType::Point, Vec::new()); + ds1.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds1)); + + let mut ds2 = Dataset::new("Cities", rustgis_data::GeometryType::Point, Vec::new()); + ds2.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(30.0, 40.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(1, ds2)); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [" Cities "], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs[0], "Cities _buffer"); + let out_layer = s.map().layer_by_name("Cities _buffer").unwrap(); + let extent = out_layer.dataset.extent(); + assert!(extent.contains(rustgis_geom::Coord::new(10.0, 20.0))); + assert!(!extent.contains(rustgis_geom::Coord::new(30.0, 40.0))); + } + + #[test] + fn batch_run_trimmed_file_path_emits_notice() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_trim_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let f1 = root.join("cities.geojson"); + let sample_geojson = r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1.0,2.0]},"properties":{}}]}"#; + std::fs::write(&f1, sample_geojson).unwrap(); + + let mut s = Session::new(crate::Project::default()); + let padded = format!(" {} ", f1.display()); + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [padded], + "params": {"distance": 10} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("trimmed surrounding whitespace from file path"))); + + let _ = std::fs::remove_dir_all(root); + } } From c701c42e6869e5d9d249ba5742af55f75d6ffd12 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 01:59:42 +0700 Subject: [PATCH 36/56] fix(engine): skip graphics layers in wildcard batch and unify batch file access classification --- crates/engine/src/cmd/gp.rs | 85 ++++++++++++++++++++++++++++++-- crates/engine/src/cmd/items.rs | 14 ++---- crates/engine/src/cmd/plugins.rs | 9 +--- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 3120a6d15d..32f6e07db2 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -958,6 +958,21 @@ pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { || v.and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")) } +/// Whether batch parameters touch files (either the tool accesses files, or inputs/folder are file-based). +pub(crate) fn batch_touches_files(params: &Value) -> bool { + let tool_touches = params + .get("tool") + .and_then(Value::as_str) + .and_then(tool_id) + .and_then(rustgis_analysis::tool) + .is_some_and(|t| writes_files(t) || reads_files(t) || is_geolibre_tool(t)); + + let folder_or_inputs = params.get("folder").is_some_and(|v| !v.is_null()) + || (params.get("inputs").is_some_and(|v| !v.is_null()) && !is_wildcard_inputs(params.get("inputs"))); + + tool_touches || folder_or_inputs +} + /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { use rustgis_analysis::control::Control; @@ -1051,16 +1066,22 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("gp.batch: {skipped_rasters} raster layer(s) left out of '*' (it runs vector layers)"), false); } + if skipped_graphics > 0 { + s.notify(format!("gp.batch: {skipped_graphics} graphics layer(s) left out of '*' (it runs vector layers)"), false); + } } else if let Some(arr) = p.get("inputs").and_then(Value::as_array) { for item in arr { if let Some(str_val) = item.as_str() { @@ -2854,12 +2875,18 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_null_folder_wildcard)); assert!(reaches.is_none()); - // Wildcard batch with params.output is ignored by gp.batch and does not touch files. + // Wildcard batch with params.output is ignored by gp.batch and does not touch files for built-in tools. let p_wildcard_output = json!({"tool": "convexHull", "inputs": "*", "params": {"output": "Roads"}}); assert!(!crate::cmd::items::touches_files("gp.batch", &p_wildcard_output)); let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard_output)); assert!(reaches.is_none()); + // For a tool that is not built-in, or if params.output points to a file, wildcard batch touches files. + let p_wildcard_non_builtin = json!({"tool": "nonExistentExternalTool", "inputs": "*", "params": {"output": "some_file.shp"}}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_wildcard_non_builtin)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard_non_builtin)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + // Plain gp.run of merge on project layers does not touch files. let p_run_merge = json!({"tool": "merge", "params": {"inputs": ["Cities", "Parks"]}}); assert!(!crate::cmd::items::touches_files("gp.run", &p_run_merge)); @@ -4749,4 +4776,54 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + + #[test] + fn batch_run_wildcard_skips_graphics_with_notice() { + let mut s = Session::new(crate::Project::default()); + let mut ds_vec = Dataset::new("Rivers", rustgis_data::GeometryType::Point, Vec::new()); + ds_vec.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds_vec)); + + let mut ds_notes = Dataset::new("Notes", rustgis_data::GeometryType::Point, Vec::new()); + ds_notes.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(5.0, 5.0))), Vec::new()); + let mut notes_layer = rustgis_map::Layer::new(1, ds_notes); + notes_layer.graphics = true; + s.map_mut().add_layer(notes_layer); + + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": "*", + "params": {"distance": 5} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + let outputs = res["outputs"].as_array().unwrap(); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0], "Rivers_buffer"); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("1 graphics layer(s) left out of '*'"))); + + let mut s_empty = Session::new(crate::Project::default()); + let ds_only_notes = Dataset::new("Notes", rustgis_data::GeometryType::Point, Vec::new()); + let mut notes_only_layer = rustgis_map::Layer::new(0, ds_only_notes); + notes_only_layer.graphics = true; + s_empty.map_mut().add_layer(notes_only_layer); + let err = s_empty + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": "*", + "params": {"distance": 5} + }), + ) + .unwrap_err(); + assert_eq!(err, "'*' matched no vector layers on the active map; add a vector layer or name the inputs"); + } } diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index ba7ce35936..941ef3aae7 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -849,16 +849,7 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { .and_then(super::gp::tool_id) .and_then(rustgis_analysis::tool) .is_some_and(super::gp::writes_files)) - || (id == "gp.batch" - && params - .get("tool") - .and_then(Value::as_str) - .and_then(super::gp::tool_id) - .and_then(rustgis_analysis::tool) - .is_some_and(|t| super::gp::writes_files(t) || super::gp::reads_files(t) || super::gp::is_geolibre_tool(t))) - || (id == "gp.batch" - && (params.get("folder").is_some_and(|v| !v.is_null()) - || (params.get("inputs").is_some_and(|v| !v.is_null()) && !super::gp::is_wildcard_inputs(params.get("inputs"))))) + || (id == "gp.batch" && super::gp::batch_touches_files(params)) || reaches(params) } @@ -886,7 +877,8 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: if command == "gp.batch" { let mut out = params.clone(); let mut modified = false; - if let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) + if built_in() + && let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) && p.remove("output").is_some() { modified = true; diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 2fa8740eae..e4b711d714 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -368,18 +368,13 @@ pub fn reaches(spec: &C, params: &Value) -> Option { { return Some(Reach::Files); } - if (spec.id == "gp.run" || spec.id == "gp.batch") + if spec.id == "gp.run" && ps(params, "tool").and_then(super::gp::tool_id).and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files) { return Some(Reach::Files); } if spec.id == "gp.batch" { - if params.get("folder").is_some_and(|v| !v.is_null()) { - return Some(Reach::Files); - } - if let Some(inputs) = params.get("inputs").filter(|v| !v.is_null()) - && !super::gp::is_wildcard_inputs(Some(inputs)) - { + if super::gp::batch_touches_files(params) { return Some(Reach::Files); } let stripped = super::items::layer_name_left_out(spec.id, params); From d66d749f88962a32d73d534806a58892183a6a9f Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 02:12:43 +0700 Subject: [PATCH 37/56] fix(engine): admit all native vector and tabular file formats in batch folder scanning --- crates/engine/src/cmd/gp.rs | 47 ++++++++++++++++++++++++++++++++++++- docs/user-guide/analysis.md | 2 +- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 32f6e07db2..e73ce0ae79 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1170,7 +1170,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::ArcW1"#).unwrap(); + std::fs::write( + &kml, + r#"20.0,10.0"#, + ) + .unwrap(); + std::fs::write(&txt, "skip me").unwrap(); + + let mut s = Session::new(crate::Project::default()); + let mark = s.notices.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + "output_pattern": "{input}_out" + }), + ) + .unwrap(); + + assert_eq!(res["total"], 3); + let notices = &s.notices[mark..]; + assert!(notices.iter().any(|n| n.text.contains("1 unsupported files skipped"))); + + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_folder_skips_symlinks() { let root = std::env::temp_dir().join(format!( diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index f9e54fad62..195dba476b 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -134,7 +134,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). - `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. -- `"folder"`: on desktop and the command line, a folder whose vector files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`) are run over in alphabetical order. +- `"folder"`: on desktop and the command line, a folder whose vector and tabular files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`, `.geoparquet`, `.kml`, `.kmz`, `.gpx`, `.csv`) are run over in alphabetical order. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. - `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). From b8df7877fab6b880aeccc50101f050b401eec3c2 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 02:27:53 +0700 Subject: [PATCH 38/56] fix(engine): aggregate trimmed file path notices, separate pattern key validation, and fix docs hierarchy --- crates/engine/src/cmd/gp.rs | 18 ++++++++++++------ docs/user-guide/analysis.md | 4 +++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index e73ce0ae79..ca7466e8e2 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -992,11 +992,12 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut cancelled_after = None; let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); + let mut trimmed_paths = 0; for (i, target) in targets.iter().enumerate() { if control.is_cancelled() { @@ -1269,7 +1271,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { @@ -1528,6 +1530,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { + s.notify(format!("gp.batch: {trimmed_paths} file path(s) trimmed of surrounding whitespace"), false); + } + if items_reporting_messages > 0 { let sample = distinct_tool_messages .iter() @@ -3250,7 +3256,7 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] #[test] fn batch_run_folder_skips_non_utf8_filename() { use std::os::unix::ffi::OsStrExt; @@ -4817,7 +4823,7 @@ mod batch_tests { assert_eq!(res["succeeded"], 1); let notices = &s.notices[mark..]; - assert!(notices.iter().any(|n| n.text.contains("trimmed surrounding whitespace from file path"))); + assert!(notices.iter().any(|n| n.text.contains("1 file path(s) trimmed of surrounding whitespace"))); let _ = std::fs::remove_dir_all(root); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 195dba476b..d3880211ca 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -133,7 +133,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} ``` - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). -- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map. +- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map (including hidden layers; raster and graphics layers are left out with a notice). - `"folder"`: on desktop and the command line, a folder whose vector and tabular files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`, `.geoparquet`, `.kml`, `.kmz`, `.gpx`, `.csv`) are run over in alphabetical order. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. - `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. @@ -143,6 +143,8 @@ Batch runs are fault-tolerant: a failed run on one input continues with the rema Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. A wildcard run (`inputs: "*"`) is file access only when the tool reads files through a `Path` or `VectorLayers` parameter or runs a GeoLibre program; otherwise it runs over the active map's layers without file access. Tools that write files are refused. +### Built-in tools + These are the built-in tools; GeoLibre and plugin tools join them when available. Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available tools and their parameters): From 4eb918ac49c10c671ba7d93befb73690dc9f5437 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 02:37:42 +0700 Subject: [PATCH 39/56] fix(engine): gate trimmed_paths on non-wasm32 to fix unused_mut in wasm32 build --- crates/engine/src/cmd/gp.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index ca7466e8e2..6c9a04d50d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1246,6 +1246,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); let mut cancelled_after = None; let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); + #[cfg(not(target_arch = "wasm32"))] let mut trimmed_paths = 0; for (i, target) in targets.iter().enumerate() { @@ -1530,6 +1531,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("gp.batch: {trimmed_paths} file path(s) trimmed of surrounding whitespace"), false); } From 3a3a73e50d1912b34a710591732e1e058762b81a Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 03:01:04 +0700 Subject: [PATCH 40/56] fix(engine): trim file stems for output names, hoist is_file_input before temp layer, and improve empty folder error --- crates/engine/src/cmd/gp.rs | 70 +++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 6c9a04d50d..509b718389 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1225,6 +1225,13 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc layer.name.as_str(), None if crate::project::find_table(&s.project.tables, target).is_some() => target.as_str(), - None => std::path::Path::new(target).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target), + None => std::path::Path::new(target.trim()).file_stem().and_then(|stem| stem.to_str()).unwrap_or(target.trim()), }; let out_name = pattern.replace("{input}", base_name).replace("{tool}", tool_id); let mut final_out_name = out_name.clone(); @@ -1351,7 +1356,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 03:16:05 +0700 Subject: [PATCH 41/56] fix(engine): handle blank folder parameter cleanly and document nested place reach --- crates/engine/src/cmd/gp.rs | 14 +++++++++++++- docs/user-guide/analysis.md | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 509b718389..8f47548b85 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1227,7 +1227,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 03:38:14 +0700 Subject: [PATCH 42/56] fix(engine): notify on non-string folder, clarify all-fail batch semantics, and add tests --- crates/engine/src/cmd/gp.rs | 46 ++++++++++++++++++++++++++++++++++++- docs/user-guide/analysis.md | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8f47548b85..988a8a9972 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -998,6 +998,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 22:45:35 +0700 Subject: [PATCH 43/56] fix(engine): aggregate reader warnings, return error on zero successes, update roadmap --- ROADMAP.md | 3 +- crates/engine/src/cmd/gp.rs | 280 ++++++++++++++++++++++++++---------- docs/user-guide/analysis.md | 2 +- 3 files changed, 206 insertions(+), 79 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index b6e50cf2c2..5165547da3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,6 +38,8 @@ comment on its issue or pull request and open a draft pull request early. [#596](https://github.com/opengeos/RustGIS/issues/596)). - Georeferencer: side-by-side window, transform picker, more formats ([#609](https://github.com/opengeos/RustGIS/issues/609)). +- Batch runs: one tool over many layers or a folder of files + ([#1458](https://github.com/opengeos/RustGIS/pull/1458)). ## Next @@ -67,7 +69,6 @@ Correct, robust and recoverable, with the foundations other themes stand on. - Extend the OS keychain secret store to service connections (database passwords, S3 keys, AI provider keys and the ChatGPT sign-in's tokens are already kept there). - Vector tools, each writing a new layer: check and fix geometry _(open)_. -- Batch runs: one tool over many layers or a folder of files. ### Expressions and cartography Next on top of the expression language: diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 988a8a9972..d6562ee83d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1251,7 +1251,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); + let mut reader_warnings: Vec<(String, String)> = Vec::new(); let mut items_reporting_messages = 0; let mut distinct_tool_messages: Vec<(String, usize)> = Vec::new(); let mut cancelled_after = None; @@ -1268,6 +1268,9 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); + #[cfg(not(target_arch = "wasm32"))] let mut temp_layer_id = { let trimmed = target.trim(); @@ -1285,13 +1288,47 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { for msg in report.messages { - read_messages.push(format!("{target}: {msg}")); + reader_warnings.push((target.clone(), msg.clone())); + item_reader_warnings.push(format!("{target}: {msg}")); } Some(super::layer::add_dataset(s, dataset)) } Err(e) => { - failed.push(json!({"input": target, "error": e.to_string()})); - errors.push(json!({"input": target, "error": e.to_string()})); + let err_str = e.to_string(); + let spec = rustgis_analysis::tool(tool_id); + let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); + let mut rec_params = base_params.clone(); + if let Value::Object(ref mut map) = rec_params { + map.insert(batch_param.to_string(), json!(target)); + } + let record_cmd = Some(( + "gp.batch".to_string(), + json!({ + "tool": tool_id, + "inputs": [target], + "params": base_params, + "param": batch_param, + }), + )); + s.project.gp_history.push( + GpRecord { + tool: name, + id: tool_id.to_string(), + params: rec_params, + command: record_cmd, + at: now_ms() / 1000, + ok: false, + messages: vec![err_str.clone()], + millis: 0, + outputs: Vec::new(), + } + .scrubbed(), + ); + if s.project.gp_history.len() > 500 { + s.project.gp_history.remove(0); + } + failed.push(json!({"input": target, "error": err_str.clone()})); + errors.push(json!({"input": target, "error": err_str})); s.active_layer = active_before; continue; } @@ -1528,6 +1565,17 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = Vec::new(); - for msg in &read_messages { - if let Some(entry) = distinct_reads.iter_mut().find(|(m, _)| *m == msg.as_str()) { - entry.1 += 1; + #[cfg(not(target_arch = "wasm32"))] + if !reader_warnings.is_empty() { + let mut aggregated: Vec<(&str, Vec<&str>)> = Vec::new(); + for (target, msg) in &reader_warnings { + if let Some((_, inputs)) = aggregated.iter_mut().find(|(m, _)| *m == msg.as_str()) { + if !inputs.contains(&target.as_str()) { + inputs.push(target.as_str()); + } } else { - distinct_reads.push((msg.as_str(), 1)); + aggregated.push((msg.as_str(), vec![target.as_str()])); } } - let sample = distinct_reads - .iter() - .take(3) - .map(|(m, c)| if *c > 1 { format!("{m} (×{c})") } else { m.to_string() }) - .collect::>() - .join("; "); - let more = if distinct_reads.len() > 3 { format!(" and {} more", distinct_reads.len() - 3) } else { String::new() }; - s.notify(format!("gp.batch: {} notice(s) reading inputs ({sample}{more})", read_messages.len()), false); + for (msg, inputs) in aggregated { + let count = inputs.len(); + let inputs_list = inputs.join(", "); + let label = if count == 1 { "input" } else { "inputs" }; + s.notify(format!("gp.batch: {count} {label} ({inputs_list}): {msg}"), false); + } } if !failed.is_empty() { @@ -1613,6 +1662,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 || is_cancelled); + let warnings: Vec = reader_warnings.iter().map(|(input, msg)| json!({"input": input, "warning": msg})).collect(); + let mut ret = json!({ "tool": tool_id, "total": total_count, @@ -1620,12 +1671,31 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc>() + .join("; "); + let err_msg = if num_err == 1 { + format!("Batch '{tool_id}' failed: {details}") + } else { + format!("Batch '{tool_id}' failed: all {num_err} inputs failed: {details}") + }; + return Err(err_msg); + } + Ok(ret) } @@ -3861,24 +3931,16 @@ mod batch_tests { s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); let mark = s.notices.len(); - let res = s - .run( - "gp.batch", - &json!({ - "tool": "test:empty_output", - "inputs": ["L1"], - }), - ) - .unwrap(); - - assert_eq!(res["failed"], 1); - assert_eq!(res["succeeded"], 0); - let errs = res["errors"].as_array().unwrap(); - assert_eq!(errs.len(), 1); - assert_eq!(errs[0]["input"], "L1"); - assert_eq!(errs[0]["error"], "custom message from empty tool"); - let outputs = res["outputs"].as_array().unwrap(); - assert!(outputs.is_empty(), "must not emit phantom output name"); + let res = s.run( + "gp.batch", + &json!({ + "tool": "test:empty_output", + "inputs": ["L1"], + }), + ); + let err = res.unwrap_err(); + assert!(err.contains("Batch 'test:empty_output' failed")); + assert!(err.contains("custom message from empty tool")); assert!(s.notices[mark..].iter().any(|n| n.text.contains("1 failed (L1)"))); } @@ -3921,24 +3983,21 @@ mod batch_tests { let ds = Dataset::new("L1", rustgis_data::GeometryType::Point, Vec::new()); s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); - let res = s - .run( - "gp.batch", - &json!({ - "tool": "test:empty_output_multi_msg", - "inputs": ["L1"], - }), - ) - .unwrap(); + let res = s.run( + "gp.batch", + &json!({ + "tool": "test:empty_output_multi_msg", + "inputs": ["L1"], + }), + ); + let err = res.unwrap_err(); + assert!(err.contains("0 features matched criteria")); + } - assert_eq!(res["failed"], 1); - let errs = res["errors"].as_array().unwrap(); - assert_eq!(errs.len(), 1); - assert_eq!(errs[0]["error"], "0 features matched criteria"); - let msgs = errs[0]["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 2); - assert_eq!(msgs[0], "Processed 10 features"); - assert_eq!(msgs[1], "0 features matched criteria"); + fn write_test_shapefile(path: &std::path::Path) { + let mut ds = Dataset::new("test", rustgis_data::GeometryType::Point, Vec::new()); + ds.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(10.0, 20.0))), Vec::new()); + rustgis_data::io::write_path(&ds, path).unwrap(); } #[test] @@ -3950,7 +4009,7 @@ mod batch_tests { std::fs::create_dir_all(&root).unwrap(); let f1 = root.join("rivers.shp"); - std::fs::write(&f1, b"").unwrap(); + write_test_shapefile(&f1); let f2 = root.join("rivers.shp.xml"); std::fs::write(&f2, b"").unwrap(); @@ -3980,21 +4039,24 @@ mod batch_tests { std::fs::create_dir_all(&root).unwrap(); let f1 = root.join("rivers.shp"); - std::fs::write(&f1, b"").unwrap(); + write_test_shapefile(&f1); let f2 = root.join("rivers.shp.xml"); std::fs::write(&f2, b"").unwrap(); let f3 = root.join("rivers.xml"); std::fs::write(&f3, b"").unwrap(); let mut s = Session::new(crate::Project::default()); - let _ = s.run( - "gp.batch", - &json!({ - "tool": "convexHull", - "folder": root.display().to_string(), - }), - ); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "convexHull", + "folder": root.display().to_string(), + }), + ) + .unwrap(); + assert_eq!(res["total"], 1); assert!(s.notices.iter().any(|n| n.text.contains("1 unsupported files skipped"))); let _ = std::fs::remove_dir_all(root); @@ -4594,11 +4656,7 @@ mod batch_tests { )); std::fs::create_dir_all(&root).unwrap(); - std::fs::write(root.join("test.shp"), b"shp").unwrap(); - std::fs::write(root.join("test.dbf"), b"dbf").unwrap(); - std::fs::write(root.join("test.shx"), b"shx").unwrap(); - std::fs::write(root.join("test.prj"), b"prj").unwrap(); - std::fs::write(root.join("test.cpg"), b"cpg").unwrap(); + write_test_shapefile(&root.join("test.shp")); let mut s = Session::new(crate::Project::default()); let mark = s.notices.len(); @@ -4923,25 +4981,93 @@ mod batch_tests { } #[test] - fn batch_run_all_failed_returns_ok_with_errors_array() { + fn batch_run_all_failed_returns_error_with_failure_details() { let mut s = Session::new(crate::Project::default()); + let res = s.run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["NonExistentLayer1", "NonExistentLayer2"], + "params": {"distance": 10} + }), + ); + let err = res.unwrap_err(); + assert!(err.contains("Batch 'buffer' failed")); + assert!(err.contains("NonExistentLayer1")); + assert!(err.contains("NonExistentLayer2")); + assert!(s.notices.iter().any(|n| n.text.contains("2 failed (NonExistentLayer1, NonExistentLayer2)"))); + } + + #[test] + fn batch_run_reader_warnings_aggregated_and_retained_in_history_and_result() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_reader_warn_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + let geojson_content = json!({ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [10.0, 20.0, 30.0] + }, + "properties": {"name": "Test"} + }] + }) + .to_string(); + + let files: Vec<_> = (1..=4) + .map(|i| { + let p = root.join(format!("file_{i}.geojson")); + std::fs::write(&p, &geojson_content).unwrap(); + p.display().to_string() + }) + .collect(); + + let mut s = Session::new(crate::Project::default()); + let mark = s.notices.len(); let res = s .run( "gp.batch", &json!({ "tool": "buffer", - "inputs": ["NonExistentLayer1", "NonExistentLayer2"], - "params": {"distance": 10} + "inputs": files, + "params": {"distance": 5} }), ) .unwrap(); - assert_eq!(res["total"], 2); - assert_eq!(res["succeeded"], 0); - assert_eq!(res["failed"], 2); - let errors = res["errors"].as_array().unwrap(); - assert_eq!(errors.len(), 2); - assert!(s.notices.iter().any(|n| n.text.contains("2 failed (NonExistentLayer1, NonExistentLayer2)"))); + assert_eq!(res["succeeded"], 4); + assert_eq!(res["failed"], 0); + + // Result retained full diagnostics in "warnings" + let warnings = res["warnings"].as_array().unwrap(); + assert_eq!(warnings.len(), 4); + for (i, w) in warnings.iter().enumerate() { + assert!(w["input"].as_str().unwrap().contains(&format!("file_{}", i + 1))); + assert!(w["warning"].as_str().unwrap().contains("Z or M")); + } + + // Notices aggregated into a single notice per warning kind without truncating later files + let notice = s.notices[mark..].iter().find(|n| n.text.contains("Z or M")).unwrap(); + assert!(notice.text.starts_with("gp.batch: 4 inputs (")); + assert!(notice.text.contains("file_1.geojson")); + assert!(notice.text.contains("file_2.geojson")); + assert!(notice.text.contains("file_3.geojson")); + assert!(notice.text.contains("file_4.geojson")); + assert!(!notice.text.contains("and 1 more")); + + // History retains reader warnings for each run + assert_eq!(s.project.gp_history.len(), 4); + for record in &s.project.gp_history { + assert!(record.ok); + assert!(record.messages.iter().any(|m| m.contains("Z or M"))); + } + + let _ = std::fs::remove_dir_all(root); } #[test] diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index e2376fbb7a..39826d1656 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -139,7 +139,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). -Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. At completion, if any items fail, RustGIS emits a single aggregate failure notice without individual error reasons; inspect the returned `"errors"` array in the command result for each input and its reason. A batch run where all inputs fail returns a successful command result with `"succeeded": 0` and the details of each failure in `"errors"`, allowing callers and automated workflows to inspect individual error causes. +Batch runs are fault-tolerant: a failed run on one input continues with the remaining items without aborting the batch. When at least one item succeeds, `gp.batch` returns a successful command result with `"succeeded"` and `"failed"` counts, output layer names, and an `"errors"` array detailing any failed inputs; RustGIS emits a single aggregate failure notice. If no item succeeds (all inputs fail), the command returns an error detailing each failed input and its reason, satisfying 'unusable results are errors'. When reading file inputs, reader loss warnings (such as dropped Z coordinates or approximate coordinate systems) are aggregated by warning kind with counts and affected inputs in session notices, recorded in history for each file run, and retained in the command result's `"warnings"` array. Because `gp.batch` can read datasets from disk in the current working directory, all explicit `inputs` (layer names or file paths) and `folder` specifications are classified as file access (`Reach::Files`), requiring assistant confirmation and restricted in imported tasks without file permissions. A wildcard run (`inputs: "*"`) is file access only when the tool reads files through a `Path` or `VectorLayers` parameter, runs a GeoLibre program, or sets static parameters naming files, paths or connections (matching `PLACE_KEYS`); otherwise it runs over the active map's layers without file access. Tools that write files are refused. From 3bbbf9379c153c9fdce969e8761d6d8ea9550840 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 22:59:17 +0700 Subject: [PATCH 44/56] fix(engine): skip folder fallback on empty wildcard and test raster-only map --- crates/engine/src/cmd/gp.rs | 53 ++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index d6562ee83d..3cdd42a3c6 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1112,12 +1112,13 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 23:22:59 +0700 Subject: [PATCH 45/56] fix(engine): record failed history and preserve reader warnings on setup failures --- crates/engine/src/cmd/gp.rs | 186 +++++++++++++++++++++++++++++++++--- 1 file changed, 175 insertions(+), 11 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 3cdd42a3c6..d0de2d1652 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1415,7 +1415,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { succeeded.push(target.clone()); if !messages.is_empty() { @@ -1532,9 +1532,20 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 500 { + s.project.gp_history.remove(0); } } } @@ -1819,7 +1873,23 @@ fn run_tool_internal( replace_tables, ); } - let inputs = map_inputs(s, rustgis_analysis::tool(&tool), ¶ms)?; + let inputs = match map_inputs(s, rustgis_analysis::tool(&tool), ¶ms) { + Ok(inputs) => inputs, + Err(e) => { + let spec = rustgis_analysis::tool(&tool); + let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); + let millis = now_ms().saturating_sub(t0); + return apply_inner( + s, + Ran { name, id: tool, command, started: t0 }, + recorded(spec, rec_val.clone()), + Err(e), + millis, + notify_messages, + replace_tables, + ); + } + }; let mut res = rustgis_analysis::run_with(&tool, &inputs, ¶ms, control); if let Ok(result) = &mut res { result.messages.extend(inputs.messages.iter().cloned()); @@ -2006,6 +2076,9 @@ fn apply_inner( GpRecord { tool: name, id, params, command, at, ok: false, messages: vec![e.clone()], millis, outputs: Vec::new() } .scrubbed(), ); + if s.project.gp_history.len() > 500 { + s.project.gp_history.remove(0); + } return Err(e); } Ok(r) => r, @@ -5121,6 +5194,97 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[test] + fn batch_run_setup_failure_records_failed_history_and_attaches_reader_warnings() { + let root = std::env::temp_dir().join(format!( + "rustgis_batch_test_setup_fail_{}", + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + + // f1 is inside extent [0, 0, 50, 50] + let f1 = root.join("f1.geojson"); + let content_inside = json!({ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [10.0, 20.0, 30.0] + }, + "properties": {"name": "Inside"} + }] + }) + .to_string(); + std::fs::write(&f1, content_inside).unwrap(); + + // f2 is outside extent [0, 0, 50, 50] + let f2 = root.join("f2.geojson"); + let content_outside = json!({ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [100.0, 100.0, 30.0] + }, + "properties": {"name": "Outside"} + }] + }) + .to_string(); + std::fs::write(&f2, content_outside).unwrap(); + + let mut s = Session::new(crate::Project::default()); + s.project.gp_env.extent = Some(rustgis_geom::Extent::new(0.0, 0.0, 50.0, 50.0)); + + let f1_str = f1.display().to_string(); + let f2_str = f2.display().to_string(); + + let res = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": [f1_str.clone(), f2_str.clone()], + "params": {"distance": 5} + }), + ) + .unwrap(); + + assert_eq!(res["succeeded"], 1); + assert_eq!(res["failed"], 1); + + // Result retained diagnostics for both + let warnings = res["warnings"].as_array().unwrap(); + assert_eq!(warnings.len(), 2); + assert!(warnings[0]["input"].as_str().unwrap().contains("f1.geojson")); + assert!(warnings[1]["input"].as_str().unwrap().contains("f2.geojson")); + + let errors = res["errors"].as_array().unwrap(); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0]["input"], f2_str); + assert!(errors[0]["error"].as_str().unwrap().contains("environment's extent")); + + // History: each run has its own GpRecord + assert_eq!(s.project.gp_history.len(), 2); + + // f1 record + let rec1 = &s.project.gp_history[0]; + assert!(rec1.ok); + assert_eq!(rec1.params["input"], f1_str); + assert!(rec1.messages.iter().any(|m| m.contains("Z or M"))); + assert!(!rec1.messages.iter().any(|m| m.contains("environment's extent"))); + + // f2 record: created for setup failure, contains its own error and reader warning + let rec2 = &s.project.gp_history[1]; + assert!(!rec2.ok); + assert_eq!(rec2.params["input"], f2_str); + assert!(rec2.messages.iter().any(|m| m.contains("environment's extent"))); + assert!(rec2.messages.iter().any(|m| m.contains("Z or M"))); + + let _ = std::fs::remove_dir_all(root); + } + #[test] fn batch_run_wildcard_skips_graphics_with_notice() { let mut s = Session::new(crate::Project::default()); From def7fcb1c9732c43160f614783ec1228e7d678c0 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 23:24:54 +0700 Subject: [PATCH 46/56] docs: update GEOLIBRE_PARITY.md for gp.batch --- GEOLIBRE_PARITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GEOLIBRE_PARITY.md b/GEOLIBRE_PARITY.md index 59bab6793c..e4172e911e 100644 --- a/GEOLIBRE_PARITY.md +++ b/GEOLIBRE_PARITY.md @@ -196,8 +196,8 @@ model-builder.md, segmentation.md, obia.md_ eCognition import. RustGIS has pixel classification. - [ ] **Visual Model Builder canvas**: a node graph with validation and export to code. Partial: ModelBuilder with a node canvas and models from history. -- [ ] **Re-run or copy history as code; batch runs over a folder**. Partial: history copied as - code; batch runs are missing (_Roadmap: solid ground_). +- [x] **Re-run or copy history as code; batch runs over a folder**: history copied as code, batch + runs over layers and folders (`gp.batch`). - [ ] **Viewshed from public terrain tiles (no DEM needed)**. Partial: needs a loaded elevation raster (_Roadmap: analysis_ for a DEM downloader). - [x] **Conversion**: vector to PMTiles/GeoParquet, raster to COG: GeoParquet, COG and vector and From a9d993965dbe00f4375cba0f7327d7adc1cc86be Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 23:37:26 +0700 Subject: [PATCH 47/56] fix(engine): enforce plain output name check for gp.batch in layer_name_left_out --- crates/engine/src/cmd/gp.rs | 6 ++++++ crates/engine/src/cmd/items.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index d0de2d1652..525f43d404 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -3043,6 +3043,12 @@ mod file_permission_tests { let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard_output)); assert!(reaches.is_none()); + // Wildcard batch with non-plain params.output (looks like path / has slashes) touches files. + let p_wildcard_path_output = json!({"tool": "convexHull", "inputs": "*", "params": {"output": "/home/u/.bashrc"}}); + assert!(crate::cmd::items::touches_files("gp.batch", &p_wildcard_path_output)); + let reaches = s.registry.get("gp.batch").and_then(|spec| crate::cmd::plugins::reaches(spec, &p_wildcard_path_output)); + assert!(matches!(reaches, Some(crate::cmd::plugins::Reach::Files))); + // For a tool that is not built-in, or if params.output points to a file, wildcard batch touches files. let p_wildcard_non_builtin = json!({"tool": "nonExistentExternalTool", "inputs": "*", "params": {"output": "some_file.shp"}}); assert!(crate::cmd::items::touches_files("gp.batch", &p_wildcard_non_builtin)); diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 941ef3aae7..01d605c1e3 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -879,6 +879,7 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: let mut modified = false; if built_in() && let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) + && p.get("output").and_then(Value::as_str).is_some_and(plain) && p.remove("output").is_some() { modified = true; From 5b463dee864ce2b65d93d31b19dd96fecf912403 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Fri, 6 Nov 2026 23:53:52 +0700 Subject: [PATCH 48/56] fix(engine): report requested total on cancel and warn on unrecognised params --- crates/engine/src/cmd/gp.rs | 83 +++++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 525f43d404..8174d9776d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1062,6 +1062,28 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = spec.params.iter().map(|param| param.name).collect(); + let mut unknown: Vec<&str> = base_params + .as_object() + .into_iter() + .flat_map(|map| map.keys().map(String::as_str)) + .filter(|key| { + if declared.contains(key) || *key == batch_param { + return false; + } + if spec.id == "joinField" { + !matches!(*key, "input" | "target_layer" | "join" | "source" | "join_table" | "join_field" | "join_key") + } else { + true + } + }) + .collect(); + if !unknown.is_empty() { + unknown.sort(); + let sample = unknown.first().copied().unwrap_or_default(); + s.notify(format!("gp.batch: {} unrecognised parameter(s) ignored in 'params' ({sample})", unknown.len()), false); + } + // Collect inputs: from "inputs" array or single string, wildcard "*", or "folder" directory path. let mut targets: Vec = Vec::new(); let mut dropped_inputs = 0; @@ -1702,7 +1724,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Fri, 6 Nov 2026 23:58:05 +0700 Subject: [PATCH 49/56] fix(engine): preserve outputPattern on file read failure and document table outputs --- crates/engine/src/cmd/gp.rs | 59 +++++++++++++++++++++++++++++++------ docs/user-guide/analysis.md | 2 +- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8174d9776d..82432ca065 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1324,15 +1324,16 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Sat, 7 Nov 2026 00:03:43 +0700 Subject: [PATCH 50/56] fix(engine): avoid cloning parameters when unmodified and document native-only batch file paths --- crates/engine/src/cmd/items.rs | 33 ++++++++++++++------------------- docs/user-guide/analysis.md | 2 +- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 01d605c1e3..e2938caae4 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -875,31 +875,26 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: // Copied only when there is a name to leave out: other commands' parameters (a file's // contents, say) are read as they are. if command == "gp.batch" { + let has_builtin_plain_output = built_in() + && params.get("params").and_then(Value::as_object).and_then(|p| p.get("output")).and_then(Value::as_str).is_some_and(plain); + let has_null_folder = params.get("folder").is_some_and(Value::is_null); + let has_null_inputs = params.get("inputs").is_some_and(Value::is_null); + + if !has_builtin_plain_output && !has_null_folder && !has_null_inputs { + return Cow::Borrowed(params); + } + let mut out = params.clone(); - let mut modified = false; - if built_in() - && let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) - && p.get("output").and_then(Value::as_str).is_some_and(plain) - && p.remove("output").is_some() - { - modified = true; + if has_builtin_plain_output && let Some(p) = out.get_mut("params").and_then(Value::as_object_mut) { + p.remove("output"); } - if out.get("folder").is_some_and(Value::is_null) - && let Some(obj) = out.as_object_mut() - { + if has_null_folder && let Some(obj) = out.as_object_mut() { obj.remove("folder"); - modified = true; } - if out.get("inputs").is_some_and(Value::is_null) - && let Some(obj) = out.as_object_mut() - { + if has_null_inputs && let Some(obj) = out.as_object_mut() { obj.remove("inputs"); - modified = true; } - if modified { - return Cow::Owned(out); - } - return Cow::Borrowed(params); + return Cow::Owned(out); } if command != "gp.run" || !built_in() || !named() { return Cow::Borrowed(params); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 1b243725ef..bc5690dbac 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -133,7 +133,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} ``` - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). -- `"inputs"`: an array of layer names or file paths, or `"*"` to run over every vector layer on the active map (including hidden layers; raster and graphics layers are left out with a notice). +- `"inputs"`: an array of layer names or, on native builds, file paths, or `"*"` to run over every vector layer on the active map (including hidden layers; raster and graphics layers are left out with a notice). In the browser, add a file as a layer first and use its layer name. - `"folder"`: on desktop and the command line, a folder whose vector and tabular files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`, `.geoparquet`, `.kml`, `.kmz`, `.gpx`, `.csv`) are run over in alphabetical order. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. - `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. From f53531de1ab4772fa1b7243c5248259169f2d165 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Sat, 7 Nov 2026 00:12:58 +0700 Subject: [PATCH 51/56] fix(engine): expose joinField aliases from analysis and notify on dropped alias precedence --- crates/analysis/src/lib.rs | 10 +++ crates/analysis/src/tools.rs | 16 ++++ crates/engine/src/cmd/gp.rs | 159 ++++++++++++++++++++++++++++++++--- 3 files changed, 172 insertions(+), 13 deletions(-) diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index 97493c0bd2..464ca550b3 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -380,6 +380,16 @@ pub fn resolve_params(spec: &ToolSpec, params: &Value) -> Result Ok(Value::Object(p)) } +pub use tools::JOIN_FIELD_ALIASES; + +/// Whether a parameter key is declared by `spec` or accepted as a valid parameter alias. +pub fn is_accepted_param(spec: &ToolSpec, key: &str) -> bool { + if spec.params.iter().any(|p| p.name == key) { + return true; + } + if spec.id == "joinField" { JOIN_FIELD_ALIASES.contains(&key) } else { false } +} + /// A built-in tool's on-off parameter as `true` or `false` (text saying so counts). Anything else is /// an error, never a guess at what `"yes"` or `1` meant (#406). A blank value, or null, is the /// default. diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 898b72560e..82fd9f96bc 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -5830,6 +5830,22 @@ pub(crate) fn normalize_params(spec: &ToolSpec, p: &mut serde_json::Map) { let is_blank = |val: Option<&Json>| val.is_none_or(|v| v.is_null() || v.as_str().is_some_and(|s| s.trim().is_empty())); let find_first = diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 82432ca065..22e148c410 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1015,7 +1015,12 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = spec.params.iter().map(|param| param.name).collect(); let mut unknown: Vec<&str> = base_params .as_object() .into_iter() .flat_map(|map| map.keys().map(String::as_str)) - .filter(|key| { - if declared.contains(key) || *key == batch_param { - return false; - } - if spec.id == "joinField" { - !matches!(*key, "input" | "target_layer" | "join" | "source" | "join_table" | "join_field" | "join_key") - } else { - true - } - }) + .filter(|key| !rustgis_analysis::is_accepted_param(spec, key) && *key != batch_param) .collect(); if !unknown.is_empty() { unknown.sort(); @@ -4756,6 +4756,139 @@ mod batch_tests { ); } + #[test] + fn batch_run_alias_precedence_emits_notice() { + let mut s = test_session(); + + // Both param and batch_param + let mark1 = s.notices.len() as u64; + let res1 = s + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "param": "input", + "batch_param": "input", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res1["succeeded"], 1); + let recent1 = s.notices_since(mark1); + assert!( + recent1.iter().any(|n| n.text.contains("gp.batch: 'batch_param' ignored because 'param' is set")), + "expected notice for dropped batch_param: {recent1:?}" + ); + + // Clean call with only param + let mut s_clean1 = test_session(); + let mark_clean1 = s_clean1.notices.len() as u64; + let res_clean1 = s_clean1 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "param": "input", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res_clean1["succeeded"], 1); + let recent_clean1 = s_clean1.notices_since(mark_clean1); + assert!( + !recent_clean1.iter().any(|n| n.text.contains("ignored because")), + "clean call must not emit dropped alias notice: {recent_clean1:?}" + ); + + // Both outputPattern and output_pattern + let mut s2 = test_session(); + let mark2 = s2.notices.len() as u64; + let res2 = s2 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "outputPattern": "{input}_pat1", + "output_pattern": "{input}_pat2", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res2["succeeded"], 1); + let recent2 = s2.notices_since(mark2); + assert!( + recent2.iter().any(|n| n.text.contains("gp.batch: 'output_pattern' ignored because 'outputPattern' is set")), + "expected notice for dropped output_pattern: {recent2:?}" + ); + + // Clean call with only outputPattern + let mut s_clean2 = test_session(); + let mark_clean2 = s_clean2.notices.len() as u64; + let res_clean2 = s_clean2 + .run( + "gp.batch", + &json!({ + "tool": "buffer", + "inputs": ["Cities"], + "outputPattern": "{input}_pat1", + "params": {"distance": 10} + }), + ) + .unwrap(); + assert_eq!(res_clean2["succeeded"], 1); + let recent_clean2 = s_clean2.notices_since(mark_clean2); + assert!( + !recent_clean2.iter().any(|n| n.text.contains("ignored because")), + "clean call must not emit dropped alias notice: {recent_clean2:?}" + ); + } + + #[test] + fn batch_run_join_field_aliases_accepted_without_unrecognised_notice() { + let mut s = Session::new(crate::Project::default()); + let mut ds_target = Dataset::new( + "Tracts", + rustgis_data::GeometryType::Point, + vec![rustgis_data::Field::new("GEOID", rustgis_data::FieldType::Text)], + ); + ds_target.push_next( + Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(0.0, 0.0))), + vec![rustgis_data::Value::Text("01".into())], + ); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds_target)); + + let mut ds_join = + Dataset::new("Census", rustgis_data::GeometryType::None, vec![rustgis_data::Field::new("FIPS", rustgis_data::FieldType::Text)]); + ds_join.push_next(None, vec![rustgis_data::Value::Text("01".into())]); + s.project.add_table(ds_join); + + let mark = s.notices.len() as u64; + let res = s + .run( + "gp.batch", + &json!({ + "tool": "joinField", + "inputs": ["Tracts"], + "params": { + "target_field": "GEOID", + "join_table": "Census", + "join_field": "FIPS", + "duplicate_strategy": "First" + } + }), + ) + .unwrap(); + assert_eq!(res["succeeded"], 1); + let recent = s.notices_since(mark); + assert!( + !recent.iter().any(|n| n.text.contains("unrecognised")), + "joinField aliases must not emit unrecognised parameter notice: {recent:?}" + ); + } + #[test] fn batch_run_failed_item_does_not_reserve_output_name() { let root = std::env::temp_dir().join(format!( From bb9adde045d7ce691aab943f1776611e16d7f29d Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Sat, 7 Nov 2026 00:16:23 +0700 Subject: [PATCH 52/56] fix(engine): preserve early return without history for gp.run missing input --- crates/engine/src/cmd/gp.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 22e148c410..ba72ddb865 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1899,6 +1899,9 @@ fn run_tool_internal( let inputs = match map_inputs(s, rustgis_analysis::tool(&tool), ¶ms) { Ok(inputs) => inputs, Err(e) => { + if command.is_none() { + return Err(e); + } let spec = rustgis_analysis::tool(&tool); let name = spec.map(|t| t.name).unwrap_or("Tool").to_string(); let millis = now_ms().saturating_sub(t0); @@ -5494,6 +5497,20 @@ mod batch_tests { let _ = std::fs::remove_dir_all(root); } + #[test] + fn run_missing_layer_returns_error_without_history_record() { + let mut s = Session::new(crate::Project::default()); + let res = s.run( + "gp.run", + &json!({ + "tool": "buffer", + "params": {"input": "NonExistentLayer", "distance": 10} + }), + ); + assert!(res.is_err()); + assert_eq!(s.project.gp_history.len(), 0); + } + #[test] fn batch_run_read_failure_preserves_output_pattern_in_history() { let root = std::env::temp_dir().join(format!( From dc2dab5471acc1ee2d8ece5dc97a6f1f29806248 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Sat, 7 Nov 2026 00:32:34 +0700 Subject: [PATCH 53/56] fix(engine): drop unnecessary clone in reader warnings and clarify folder scan in docs --- crates/engine/src/cmd/gp.rs | 7 ++++--- crates/engine/src/cmd/items.rs | 4 ++-- docs/user-guide/analysis.md | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 66c60278b2..9e32d16379 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1417,8 +1417,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { for msg in report.messages { - reader_warnings.push((target.clone(), msg.clone())); item_reader_warnings.push(format!("{target}: {msg}")); + reader_warnings.push((target.clone(), msg)); } Some(super::layer::add_dataset(s, dataset)) } @@ -3524,10 +3524,11 @@ mod batch_tests { let symlink_file = root.join("link.geojson"); #[cfg(unix)] - let linked = std::os::unix::fs::symlink(&target_file, &symlink_file).is_ok(); + std::os::unix::fs::symlink(&target_file, &symlink_file).unwrap(); #[cfg(windows)] - let linked = std::os::windows::fs::symlink_file(&target_file, &symlink_file).is_ok(); + let _ = std::os::windows::fs::symlink_file(&target_file, &symlink_file); + let linked = symlink_file.exists(); if linked { let mut s = Session::new(crate::Project::default()); let res = s diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index c4a41b5ed2..703e33f755 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -875,8 +875,6 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: }; let plain = |t: &str| !t.trim().is_empty() && !t.contains(['/', '\\', '.', ':']) && !looks_like_path(t); let named = || params.pointer("/params/output").and_then(Value::as_str).is_some_and(plain); - // Copied only when there is a name to leave out: other commands' parameters (a file's - // contents, say) are read as they are. if command == "gp.batch" { let has_builtin_plain_output = built_in() && params.get("params").and_then(Value::as_object).and_then(|p| p.get("output")).and_then(Value::as_str).is_some_and(plain); @@ -899,6 +897,8 @@ pub(crate) fn layer_name_left_out<'a>(command: &str, params: &'a Value) -> std:: } return Cow::Owned(out); } + // Copied only when there is a name to leave out: other commands' parameters (a file's + // contents, say) are read as they are. if command != "gp.run" || !built_in() || !named() { return Cow::Borrowed(params); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 14f4fd9437..3bc76d7538 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -134,7 +134,7 @@ gp.batch={"tool":"buffer","inputs":["Cities","Rivers"],"output_pattern":"{input} - `"tool"`: the tool's id (such as `"buffer"`, `"simplify"`, or `"convexHull"`). - `"inputs"`: an array of layer names or, on native builds, file paths, or `"*"` to run over every vector layer on the active map (including hidden layers; raster and graphics layers are left out with a notice). In the browser, add a file as a layer first and use its layer name. -- `"folder"`: on desktop and the command line, a folder whose vector and tabular files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`, `.geoparquet`, `.kml`, `.kmz`, `.gpx`, `.csv`) are run over in alphabetical order. +- `"folder"`: on desktop and the command line, a folder whose vector and tabular files (`.shp`, `.geojson`, `.json`, `.gpkg`, `.fgb`, `.parquet`, `.geoparquet`, `.kml`, `.kmz`, `.gpx`, `.csv`) are run over in alphabetical order. Symlink entries and subfolders are skipped. - `"output_pattern"` or `"outputPattern"`: a naming template for resulting layers (default `"{input}_{tool}"`), where `{input}` is replaced by the layer's name or file stem, and `{tool}` by the tool id. Must include `{input}`. This template applies only to tools that honor the batch output parameter, rather than tools that modify layers in place or return selections. - `"params"`: the static parameters shared by each run. If a parameter matching the batch parameter name is included in `"params"` (such as `"input"`), the batch inputs take precedence, replacing it for each item and emitting a notice. - `"param"` or `"batch_param"`: which parameter receives each input (default: auto-detected as the tool's first layer parameter, else `"input"`). From b0665edaa7e4b62e51d3b8dc70331303408fb984 Mon Sep 17 00:00:00 2001 From: quanguet0409 Date: Sat, 7 Nov 2026 00:34:48 +0700 Subject: [PATCH 54/56] fix(engine): accept tool parameter aliases for batch_param validation --- crates/engine/src/cmd/gp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 9e32d16379..b6019bd531 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1143,7 +1143,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Sat, 7 Nov 2026 21:20:25 -0500 Subject: [PATCH 55/56] docs(roadmap): batch runs are done once this merges --- ROADMAP.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 459d0e6a75..c337dd19e9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,8 +38,6 @@ comment on its issue or pull request and open a draft pull request early. [#596](https://github.com/opengeos/RustGIS/issues/596)). - Georeferencer: side-by-side window, transform picker, more formats ([#609](https://github.com/opengeos/RustGIS/issues/609)). -- Batch runs: one tool over many layers or a folder of files - ([#1458](https://github.com/opengeos/RustGIS/pull/1458)). ## Next From be5486f51941695524ea3dbed886000b5c84a541 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 7 Nov 2026 21:27:12 -0500 Subject: [PATCH 56/56] Address review feedback - gp.batch: the browser's folder refusal follows the native gates, so a wildcard keeps its own error and a blank folder is ignored; a blank folder beside inputs no longer says both were given. - gp.batch: a static Join Field alias (`input` for `target`) no longer stands in for each batch input; the analysis crate names a tool parameter's aliases (param_aliases). - gp.batch: every distinct tool message is named in the batch notice (up to 50) and all of them are in the result's `messages`, so a loss reported after the third kind is not hidden. - gp.batch: an item whose input can't be mapped is kept in the history as a failed run, layer inputs included. --- crates/analysis/src/lib.rs | 16 +++++++ crates/engine/src/cmd/gp.rs | 87 ++++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index cb591e8173..ba092a1a4a 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -489,6 +489,22 @@ pub fn is_accepted_param(spec: &ToolSpec, key: &str) -> bool { if spec.id == "joinField" { JOIN_FIELD_ALIASES.contains(&key) } else { false } } +/// The names a tool reads one of its parameters under, `key` among them, or none when it has no +/// other names (Join Field takes its target as `target`, `input` or `target_layer`). +pub fn param_aliases(spec: &ToolSpec, key: &str) -> &'static [&'static str] { + const JOIN_FIELD: &[&[&str]] = &[ + &["target", "input", "target_layer"], + &["joinTable", "join", "source", "join_table"], + &["targetField", "target_field", "key", "target_key"], + &["joinField", "join_field", "join_key"], + &["duplicates", "duplicateStrategy", "duplicate_strategy"], + ]; + if spec.id != "joinField" { + return &[]; + } + JOIN_FIELD.iter().find(|group| group.contains(&key)).copied().unwrap_or(&[]) +} + /// A built-in tool's on-off parameter as `true` or `false` (text saying so counts). Anything else is /// an error, never a guess at what `"yes"` or `1` meant (#406). A blank value, or null, is the /// default. diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 4e6b77407d..ebdfd502f1 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1333,7 +1333,8 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { let sample = distinct_tool_messages .iter() - .take(3) + .take(NAMED) .map(|(m, c)| if *c > 1 { format!("{m} (×{c})") } else { m.clone() }) .collect::>() .join("; "); - let more = if distinct_tool_messages.len() > 3 { format!(" and {} more", distinct_tool_messages.len() - 3) } else { String::new() }; + let more = if distinct_tool_messages.len() > NAMED { + format!(" and {} more, in the result", distinct_tool_messages.len() - NAMED) + } else { + String::new() + }; s.notify(format!("gp.batch: {items_reporting_messages} item(s) reported messages ({sample}{more})"), false); } @@ -1943,6 +1959,7 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc>(), }); if is_cancelled { ret["cancelled"] = json!(true); @@ -2100,7 +2117,7 @@ pub(crate) fn execute( let inputs = match map_inputs(s, Some(spec), params) { Ok(inputs) => inputs, // A batch item that can't read its input is kept in the history as a failed run. - Err(e) if opts.command.is_some() => return apply_with(s, run_of(t0), kept(), Err(e), now_ms().saturating_sub(t0), opts), + Err(e) if opts.record.is_some() => return apply_with(s, run_of(t0), kept(), Err(e), now_ms().saturating_sub(t0), opts), Err(e) => return Err(e), }; let mut res = if let Some(plugin) = pin.plugin() { @@ -5231,6 +5248,66 @@ mod batch_tests { ); } + #[test] + fn batch_input_takes_precedence_over_a_static_alias() { + let mut s = Session::new(crate::Project::default()); + for (i, name) in ["Tracts", "Blocks", "Other"].into_iter().enumerate() { + let mut ds = Dataset::new( + name, + rustgis_data::GeometryType::Point, + vec![rustgis_data::Field::new("GEOID", rustgis_data::FieldType::Text)], + ); + ds.push_next( + Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(0.0, 0.0))), + vec![rustgis_data::Value::Text("01".into())], + ); + s.map_mut().add_layer(rustgis_map::Layer::new(i as u32, ds)); + } + let mut join = + Dataset::new("Census", rustgis_data::GeometryType::None, vec![rustgis_data::Field::new("FIPS", rustgis_data::FieldType::Text)]); + join.push_next(None, vec![rustgis_data::Value::Text("01".into())]); + s.project.add_table(join); + let before = s.project.gp_history.len(); + let res = s + .run( + "gp.batch", + &json!({ + "tool": "joinField", + "inputs": ["Tracts", "Blocks"], + "param": "target_layer", + // `input` names Join Field's target too: it must not stand in for each batch input. + "params": {"input": "Other", "targetField": "GEOID", "joinTable": "Census", "joinField": "FIPS"} + }), + ) + .unwrap(); + assert_eq!(res["succeeded"], 2, "{res}"); + let targets: Vec<_> = s.project.gp_history[before..].iter().map(|r| r.params["target_layer"].clone()).collect(); + assert_eq!(targets, [json!("Tracts"), json!("Blocks")]); + assert!(s.project.gp_history[before..].iter().all(|r| r.params.get("input").is_none())); + let other = s.map().layer_by_name("Other").unwrap(); + assert!(!other.dataset.fields.iter().any(|f| f.name == "FIPS"), "the static alias's layer was joined"); + } + + #[test] + fn batch_item_whose_input_cant_be_read_is_kept_in_the_history() { + let mut s = test_session(); + let before = s.project.gp_history.len(); + // A name that is neither a layer, a table nor a file: the tool's input can't be mapped. + let res = s.run("gp.batch", &json!({"tool": "buffer", "inputs": ["Cities", "Nowhere"], "params": {"distance": 1}})).unwrap(); + assert_eq!(res["failed"], 1, "{res}"); + let failed: Vec<_> = s.project.gp_history[before..].iter().filter(|r| !r.ok).collect(); + assert_eq!(failed.len(), 1); + assert_eq!(failed[0].params["input"], "Nowhere"); + } + + #[test] + fn blank_folder_beside_inputs_is_not_reported() { + let mut s = test_session(); + let mark = s.notices.len() as u64; + s.run("gp.batch", &json!({"tool": "buffer", "inputs": ["Cities"], "folder": " ", "params": {"distance": 1}})).unwrap(); + assert!(!s.notices_since(mark).iter().any(|n| n.text.contains("both 'inputs' and 'folder'"))); + } + #[test] fn batch_run_failed_item_does_not_reserve_output_name() { let root = std::env::temp_dir().join(format!(