From e30829ec04bbe6e75a42fbbb2f251e77e3ee31a8 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 7 Nov 2026 22:04:44 -0500 Subject: [PATCH 1/7] fix(engine): cap gp.batch's inputs and report dropped input aliases Follow-up to #1458, from its last review round: - gp.batch refuses more than 10,000 inputs, with the count, before running any, so a hostile list or folder can't make it unbounded. - gp.batch reports the other names for its input parameter given in `params` (Join Field's `input` for `target`) that each item drops. --- crates/engine/src/cmd/gp.rs | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index ebdfd502f1..8f7a915922 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1172,6 +1172,9 @@ pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { || v.and_then(Value::as_array).is_some_and(|arr| arr.len() == 1 && arr.first().and_then(Value::as_str) == Some("*")) } +/// The most inputs one `gp.batch` runs over. +pub(crate) const MAX_BATCH_ITEMS: usize = 10_000; + /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { use rustgis_analysis::control::Control; @@ -1270,6 +1273,18 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc = rustgis_analysis::param_aliases(spec, batch_param) + .iter() + .copied() + .filter(|alias| *alias != batch_param && base_params.get(*alias).is_some_and(|v| !v.is_null())) + .collect(); + if let Some(first) = shadowed.first() { + s.notify( + format!("gp.batch: {} other name(s) for '{batch_param}' in 'params' replaced by the batch inputs ({first})", shadowed.len()), + false, + ); + } let mut unknown: Vec<&str> = base_params .as_object() @@ -1464,6 +1479,14 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc MAX_BATCH_ITEMS { + return Err(format!( + "{} inputs is more than a batch runs (at most {MAX_BATCH_ITEMS}); split it into smaller batches", + targets.len() + )); + } + let mut succeeded = Vec::new(); let mut failed = Vec::new(); let mut errors = Vec::new(); @@ -5300,6 +5323,47 @@ mod batch_tests { assert_eq!(failed[0].params["input"], "Nowhere"); } + #[test] + fn a_batch_over_too_many_inputs_is_refused() { + let mut s = test_session(); + let inputs = vec![json!("Cities"); super::MAX_BATCH_ITEMS + 1]; + let before = s.map().layers.len(); + let err = s.run("gp.batch", &json!({"tool": "buffer", "inputs": inputs, "params": {"distance": 1}})).unwrap_err(); + assert!(err.contains(&format!("{} inputs", super::MAX_BATCH_ITEMS + 1)), "{err}"); + assert_eq!(s.map().layers.len(), before, "nothing ran"); + } + + #[test] + fn a_static_alias_dropped_from_each_item_is_reported() { + let mut s = Session::new(crate::Project::default()); + let mut ds = Dataset::new( + "Tracts", + rustgis_data::GeometryType::Point, + vec![rustgis_data::Field::new("GEOID", rustgis_data::FieldType::Text)], + ); + ds.push_next(Some(rustgis_data::Geometry::Point(rustgis_geom::Coord::new(0.0, 0.0))), vec![rustgis_data::Value::Text("01".into())]); + s.map_mut().add_layer(rustgis_map::Layer::new(0, ds)); + let mut join = + Dataset::new("Census", rustgis_data::GeometryType::None, vec![rustgis_data::Field::new("FIPS", rustgis_data::FieldType::Text)]); + join.push_next(None, vec![rustgis_data::Value::Text("01".into())]); + s.project.add_table(join); + let shared = json!({"targetField": "GEOID", "joinTable": "Census", "joinField": "FIPS"}); + let mark = s.notices.len() as u64; + s.run("gp.batch", &json!({"tool": "joinField", "inputs": ["Tracts"], "params": shared})).unwrap(); + assert!(!s.notices_since(mark).iter().any(|n| n.text.contains("other name(s)")), "a clean run says nothing"); + let mut with_alias = shared.clone(); + with_alias["input"] = json!("Tracts"); + let mark = s.notices.len() as u64; + s.run("gp.batch", &json!({"tool": "joinField", "inputs": ["Tracts"], "params": with_alias})).unwrap(); + assert!( + s.notices_since(mark) + .iter() + .any(|n| n.text.contains("1 other name(s) for 'target' in 'params' replaced by the batch inputs (input)")), + "{:?}", + s.notices_since(mark) + ); + } + #[test] fn blank_folder_beside_inputs_is_not_reported() { let mut s = test_session(); From cf3d0d561001ac8608d318cbcba378e0543bd54d Mon Sep 17 00:00:00 2001 From: giswqs <5016453+giswqs@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:41:29 +0000 Subject: [PATCH 2/7] fix(engine): count gp.batch notices from the monotonic mark The static-alias notice test took its mark from the kept-notice queue length. That queue is trimmed to 200 entries, so the mark stops matching once it is full. Take the mark from notice_mark() instead, which counts every notice given. Co-Authored-By: Claude Haiku 5.5 Applied by the claude-autofix workflow. --- .bash_profile | 0 .bashrc | 0 .gitconfig | 0 .idea | 0 .profile | 0 .ripgreprc | 0 .vscode | 0 .zprofile | 0 .zshrc | 0 crates/engine/src/cmd/gp.rs | 4 ++-- scripts | 0 11 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 .bash_profile create mode 100644 .bashrc create mode 100644 .gitconfig create mode 100644 .idea create mode 100644 .profile create mode 100644 .ripgreprc create mode 100644 .vscode create mode 100644 .zprofile create mode 100644 .zshrc create mode 100644 scripts diff --git a/.bash_profile b/.bash_profile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.bashrc b/.bashrc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.gitconfig b/.gitconfig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.idea b/.idea new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.profile b/.profile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.ripgreprc b/.ripgreprc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.vscode b/.vscode new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.zprofile b/.zprofile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.zshrc b/.zshrc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8f7a915922..74de04559d 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -5348,12 +5348,12 @@ mod batch_tests { join.push_next(None, vec![rustgis_data::Value::Text("01".into())]); s.project.add_table(join); let shared = json!({"targetField": "GEOID", "joinTable": "Census", "joinField": "FIPS"}); - let mark = s.notices.len() as u64; + let mark = s.notice_mark(); s.run("gp.batch", &json!({"tool": "joinField", "inputs": ["Tracts"], "params": shared})).unwrap(); assert!(!s.notices_since(mark).iter().any(|n| n.text.contains("other name(s)")), "a clean run says nothing"); let mut with_alias = shared.clone(); with_alias["input"] = json!("Tracts"); - let mark = s.notices.len() as u64; + let mark = s.notice_mark(); s.run("gp.batch", &json!({"tool": "joinField", "inputs": ["Tracts"], "params": with_alias})).unwrap(); assert!( s.notices_since(mark) diff --git a/scripts b/scripts new file mode 100644 index 0000000000..e69de29bb2 From eebcb51795bc1db9e137d869a0320282b4088a00 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 7 Nov 2026 23:20:13 -0500 Subject: [PATCH 3/7] Address review feedback - The dropped-alias notice names every alias, not only the first. - A folder batch is refused while it is read once the folder holds more than ten entries per allowed input, and refused with the count before sorting when it has more supported files than a batch runs. - Tests: a batch at exactly the limit is accepted; a folder over it is refused before any item runs. --- crates/engine/src/cmd/gp.rs | 53 ++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8f7a915922..dd249d9766 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1175,6 +1175,10 @@ pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { /// The most inputs one `gp.batch` runs over. pub(crate) const MAX_BATCH_ITEMS: usize = 10_000; +/// The most directory entries a folder batch reads: room for each file's sidecars. +#[cfg(not(target_arch = "wasm32"))] +const MAX_FOLDER_ENTRIES: usize = MAX_BATCH_ITEMS * 10; + /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { use rustgis_analysis::control::Control; @@ -1279,9 +1283,13 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc= MAX_FOLDER_ENTRIES { + return Err(format!( + "folder '{folder}' has more than {MAX_FOLDER_ENTRIES} entries, more than a batch runs; split it into smaller folders" + )); + } match entry { Ok(entry) => match entry.file_type() { Ok(ft) => { @@ -1453,6 +1468,12 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("folder '{folder}': {unreadable} unreadable directory entries skipped"), true); } + if files.len() > MAX_BATCH_ITEMS { + return Err(format!( + "folder '{folder}' has {} supported files, more than a batch runs (at most {MAX_BATCH_ITEMS}); split it into smaller folders", + files.len() + )); + } files.sort(); targets.extend(files); } @@ -5333,6 +5354,32 @@ mod batch_tests { assert_eq!(s.map().layers.len(), before, "nothing ran"); } + #[test] + fn a_batch_at_the_input_limit_is_accepted() { + let mut s = test_session(); + let control = rustgis_analysis::control::Control::new(); + control.cancel(); + let inputs = vec![json!("Cities"); super::MAX_BATCH_ITEMS]; + let res = super::run_batch_with(&mut s, &json!({"tool": "buffer", "inputs": inputs, "params": {"distance": 1}}), &control).unwrap(); + assert_eq!(res["total"], super::MAX_BATCH_ITEMS); + assert_eq!(res["cancelled"], true); + } + + #[test] + fn a_folder_over_the_limit_is_refused_before_any_item_runs() { + let dir = std::env::temp_dir().join(format!("rustgis-batch-limit-{}-{}", std::process::id(), now_ms())); + std::fs::create_dir_all(&dir).unwrap(); + for i in 0..=super::MAX_BATCH_ITEMS { + std::fs::write(dir.join(format!("{i}.csv")), "x,y\n").unwrap(); + } + let mut s = test_session(); + let before = s.map().layers.len(); + let err = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})).unwrap_err(); + let _ = std::fs::remove_dir_all(&dir); + assert!(err.contains(&format!("{} supported files", super::MAX_BATCH_ITEMS + 1)), "{err}"); + assert_eq!(s.map().layers.len(), before); + } + #[test] fn a_static_alias_dropped_from_each_item_is_reported() { let mut s = Session::new(crate::Project::default()); From bb4e5db0bace90e151bed52fb278bfd7b573f032 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 7 Nov 2026 23:21:06 -0500 Subject: [PATCH 4/7] chore: drop empty placeholder files the autofix run committed --- .bash_profile | 0 .bashrc | 0 .gitconfig | 0 .idea | 0 .profile | 0 .ripgreprc | 0 .vscode | 0 .zprofile | 0 .zshrc | 0 scripts | 0 10 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .bash_profile delete mode 100644 .bashrc delete mode 100644 .gitconfig delete mode 100644 .idea delete mode 100644 .profile delete mode 100644 .ripgreprc delete mode 100644 .vscode delete mode 100644 .zprofile delete mode 100644 .zshrc delete mode 100644 scripts diff --git a/.bash_profile b/.bash_profile deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.bashrc b/.bashrc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.gitconfig b/.gitconfig deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.idea b/.idea deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.profile b/.profile deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.ripgreprc b/.ripgreprc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.vscode b/.vscode deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.zprofile b/.zprofile deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.zshrc b/.zshrc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts b/scripts deleted file mode 100644 index e69de29bb2..0000000000 From 73e4d845e7d32766671c6c8da07817014acf6179 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sun, 8 Nov 2026 03:07:42 -0500 Subject: [PATCH 5/7] Address review feedback - gp.batch folder scan holds only files it can run or that may be sidecars, and counts subfolders and other files without holding them, so a folder of many unrelated files runs its data files again instead of being refused. The cap now limits what is held, and its message says the folder has too many data and sidecar files to read. - Test both ways: a folder with one CSV and more other files than the cap runs it and reports them skipped; a folder over the cap in sidecar candidates is refused with no item run or recorded. The cap is smaller under test so the tests write about 10,000 files. --- crates/engine/src/cmd/gp.rs | 156 ++++++++++++++++++++++++------------ 1 file changed, 104 insertions(+), 52 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 2db63e96e1..2a65438148 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1175,9 +1175,25 @@ pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { /// The most inputs one `gp.batch` runs over. pub(crate) const MAX_BATCH_ITEMS: usize = 10_000; -/// The most directory entries a folder batch reads: room for each file's sidecars. -#[cfg(not(target_arch = "wasm32"))] +/// The most data and sidecar files a folder batch holds while it reads the folder: room for +/// each file's sidecars. Other entries are only counted. +#[cfg(all(not(target_arch = "wasm32"), not(test)))] const MAX_FOLDER_ENTRIES: usize = MAX_BATCH_ITEMS * 10; +/// A smaller limit under test, so the tests don't write 100,000 files. +#[cfg(all(not(target_arch = "wasm32"), test))] +const MAX_FOLDER_ENTRIES: usize = MAX_BATCH_ITEMS + 10; + +/// Whether a folder batch runs files with this lowercase extension. +#[cfg(not(target_arch = "wasm32"))] +fn is_folder_target_ext(ext: &str) -> bool { + matches!(ext, "shp" | "geojson" | "json" | "gpkg" | "fgb" | "parquet" | "geoparquet" | "kml" | "kmz" | "gpx" | "csv") +} + +/// Whether a file with this lowercase extension may be a Shapefile or GeoPackage sidecar. +#[cfg(not(target_arch = "wasm32"))] +fn is_folder_sidecar_ext(ext: &str) -> bool { + matches!(ext, "dbf" | "shx" | "prj" | "cpg" | "qpj" | "sbn" | "sbx" | "qix" | "xml" | "gpkg-wal" | "gpkg-shm" | "wal" | "shm") +} /// Runs a geoprocessing tool in batch across multiple layers or files. fn batch(s: &mut Session, p: &Value) -> CmdResult { @@ -1381,39 +1397,49 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc= MAX_FOLDER_ENTRIES { - return Err(format!( - "folder '{folder}' has more than {MAX_FOLDER_ENTRIES} entries, more than a batch runs; split it into smaller folders" - )); - } + for entry in entries { match entry { Ok(entry) => match entry.file_type() { Ok(ft) => { if ft.is_symlink() { symlinks += 1; - } else { - let path = entry.path(); - if path.to_str().is_none() { - non_utf8 += 1; - continue; - } - if ft.is_file() { - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - if ext == "shp" - && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) - { - shp_stems.insert(stem.to_ascii_lowercase()); - } else if ext == "gpkg" - && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) - { - gpkg_stems.insert(stem.to_ascii_lowercase()); - } - } - dir_entries.push((path, ft)); + continue; + } + let path = entry.path(); + if path.to_str().is_none() { + non_utf8 += 1; + continue; } + if ft.is_dir() { + subfolders += 1; + continue; + } + if !ft.is_file() { + unreadable += 1; + continue; + } + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + // Only files that can be run or be a sidecar are held; the rest are + // counted as skipped, so a folder of other files isn't refused. + if !is_folder_target_ext(&ext) && !is_folder_sidecar_ext(&ext) { + skipped += 1; + continue; + } + if dir_entries.len() >= MAX_FOLDER_ENTRIES { + return Err(format!( + "folder '{folder}' has more than {MAX_FOLDER_ENTRIES} data and sidecar files to read; split it into smaller folders" + )); + } + if ext == "shp" + && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) + { + shp_stems.insert(stem.to_ascii_lowercase()); + } else if ext == "gpkg" + && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) + { + gpkg_stems.insert(stem.to_ascii_lowercase()); + } + dir_entries.push((path, ext)); } Err(_) => { unreadable += 1; @@ -1424,30 +1450,19 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc Date: Sun, 8 Nov 2026 03:20:29 -0500 Subject: [PATCH 6/7] Address Claude review feedback - gp.batch folder scan holds an .xml file only when it is named .shp.xml, the one form that can be a Shapefile sidecar, so a folder of many unrelated .xml files runs its data files instead of hitting the entry cap. The many-other-files test now uses .xml files to cover this. - The folder limit tests remove their temporary folder before they check the result, so a failing assertion doesn't leave the files behind. --- crates/engine/src/cmd/gp.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 2a65438148..8f474bf870 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1421,7 +1421,10 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc.shp.xml` can be a Shapefile's metadata sidecar. + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("").to_ascii_lowercase(); + let may_be_sidecar = is_folder_sidecar_ext(&ext) && (ext != "xml" || name.ends_with(".shp.xml")); + if !is_folder_target_ext(&ext) && !may_be_sidecar { skipped += 1; continue; } @@ -5389,8 +5392,9 @@ mod batch_tests { } let mut s = test_session(); let before = s.map().layers.len(); - let err = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})).unwrap_err(); + let res = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})); let _ = std::fs::remove_dir_all(&dir); + let err = res.unwrap_err(); assert!(err.contains(&format!("{} supported files", super::MAX_BATCH_ITEMS + 1)), "{err}"); assert_eq!(s.map().layers.len(), before); } @@ -5401,7 +5405,8 @@ mod batch_tests { std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join("pts.csv"), "x,y\n1,2\n").unwrap(); for i in 0..=super::MAX_FOLDER_ENTRIES { - std::fs::write(dir.join(format!("{i}.txt")), "").unwrap(); + // `.xml` files that aren't `.shp.xml` can't be sidecars, so they aren't held. + std::fs::write(dir.join(format!("{i}.xml")), "").unwrap(); } let mut s = test_session(); let res = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})); @@ -5424,9 +5429,9 @@ mod batch_tests { let layers = s.map().layers.len(); let history = s.project.gp_history.len(); let control = rustgis_analysis::control::Control::new(); - let err = - super::run_batch_with(&mut s, &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}}), &control).unwrap_err(); + let res = super::run_batch_with(&mut s, &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}}), &control); let _ = std::fs::remove_dir_all(&dir); + let err = res.unwrap_err(); assert!(err.contains(&format!("more than {} data and sidecar files", super::MAX_FOLDER_ENTRIES)), "{err}"); assert_eq!(s.map().layers.len(), layers, "no item ran"); assert_eq!(s.project.gp_history.len(), history, "no item was attempted"); From 4daa3dd0deb7c87a8197a93c36b47736c696732e Mon Sep 17 00:00:00 2001 From: giswqs Date: Sun, 8 Nov 2026 03:33:53 -0500 Subject: [PATCH 7/7] Address Claude review feedback - gp.batch reads a folder in two passes instead of holding every possible sidecar: the first pass holds only the files a batch runs and the Shapefile and GeoPackage names, the second counts sidecar-like files that match neither as skipped. Orphan .dbf, .prj and similar files no longer count toward any limit, so the folder entry cap is gone and only the 10,000-file batch limit stays, now checked while the folder is read. - Test a folder with one CSV and text, unrelated .xml and orphan .dbf files: the CSV runs and all of them are reported skipped. The over-limit folder test also checks that no item was attempted. --- crates/engine/src/cmd/gp.rs | 202 +++++++++++++++--------------------- 1 file changed, 84 insertions(+), 118 deletions(-) diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 8f474bf870..ee14d512d0 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1175,14 +1175,6 @@ pub(crate) fn is_wildcard_inputs(v: Option<&Value>) -> bool { /// The most inputs one `gp.batch` runs over. pub(crate) const MAX_BATCH_ITEMS: usize = 10_000; -/// The most data and sidecar files a folder batch holds while it reads the folder: room for -/// each file's sidecars. Other entries are only counted. -#[cfg(all(not(target_arch = "wasm32"), not(test)))] -const MAX_FOLDER_ENTRIES: usize = MAX_BATCH_ITEMS * 10; -/// A smaller limit under test, so the tests don't write 100,000 files. -#[cfg(all(not(target_arch = "wasm32"), test))] -const MAX_FOLDER_ENTRIES: usize = MAX_BATCH_ITEMS + 10; - /// Whether a folder batch runs files with this lowercase extension. #[cfg(not(target_arch = "wasm32"))] fn is_folder_target_ext(ext: &str) -> bool { @@ -1392,84 +1384,80 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc { - let mut dir_entries = Vec::new(); - let mut shp_stems = std::collections::HashSet::new(); - let mut gpkg_stems = std::collections::HashSet::new(); - for entry in entries { - match entry { - Ok(entry) => match entry.file_type() { - Ok(ft) => { - if ft.is_symlink() { - symlinks += 1; - continue; - } - let path = entry.path(); - if path.to_str().is_none() { - non_utf8 += 1; - continue; - } - if ft.is_dir() { - subfolders += 1; - continue; - } - if !ft.is_file() { - unreadable += 1; - continue; - } - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - // Only files that can be run or be a sidecar are held; the rest are - // counted as skipped, so a folder of other files isn't refused. - // Only `.shp.xml` can be a Shapefile's metadata sidecar. - let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("").to_ascii_lowercase(); - let may_be_sidecar = is_folder_sidecar_ext(&ext) && (ext != "xml" || name.ends_with(".shp.xml")); - if !is_folder_target_ext(&ext) && !may_be_sidecar { - skipped += 1; - continue; - } - if dir_entries.len() >= MAX_FOLDER_ENTRIES { - return Err(format!( - "folder '{folder}' has more than {MAX_FOLDER_ENTRIES} data and sidecar files to read; split it into smaller folders" - )); - } - if ext == "shp" - && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) - { - shp_stems.insert(stem.to_ascii_lowercase()); - } else if ext == "gpkg" - && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) - { - gpkg_stems.insert(stem.to_ascii_lowercase()); - } - dir_entries.push((path, ext)); - } - Err(_) => { - unreadable += 1; - } - }, - Err(_) => { - unreadable += 1; - } - } + // The folder is read twice so only the files a batch runs are held, however many other + // entries it has: the first pass finds them and the Shapefile/GeoPackage names, the + // second counts the sidecar-like files that match neither as skipped. + let mut shp_stems = std::collections::HashSet::new(); + let mut gpkg_stems = std::collections::HashSet::new(); + let entries = std::fs::read_dir(dir).map_err(|e| format!("folder '{folder}': {e}"))?; + for entry in entries { + let Ok(entry) = entry else { + unreadable += 1; + continue; + }; + let Ok(ft) = entry.file_type() else { + unreadable += 1; + continue; + }; + if ft.is_symlink() { + symlinks += 1; + continue; + } + let path = entry.path(); + if path.to_str().is_none() { + non_utf8 += 1; + continue; + } + if ft.is_dir() { + subfolders += 1; + continue; + } + if !ft.is_file() { + unreadable += 1; + continue; + } + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if is_folder_target_ext(&ext) { + if files.len() >= MAX_BATCH_ITEMS { + return Err(format!( + "folder '{folder}' has more than {MAX_BATCH_ITEMS} supported files, more than a batch runs; split it into smaller folders" + )); } - for (path, ext) in dir_entries { - if is_folder_target_ext(&ext) { - files.push(path.display().to_string()); - } else { - let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("").to_ascii_lowercase(); - let is_shp_sidecar = (matches!(ext.as_str(), "dbf" | "shx" | "prj" | "cpg" | "qpj" | "sbn" | "sbx" | "qix") - && shp_stems.contains(&stem)) - || (ext == "xml" && stem.strip_suffix(".shp").map(|s| shp_stems.contains(s)).unwrap_or(false)); - let is_gpkg_sidecar = matches!(ext.as_str(), "gpkg-wal" | "gpkg-shm" | "wal" | "shm") - && (gpkg_stems.contains(&stem) || stem.strip_suffix(".gpkg").map(|s| gpkg_stems.contains(s)).unwrap_or(false)); - if !is_shp_sidecar && !is_gpkg_sidecar { - skipped += 1; - } - } + let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("").to_ascii_lowercase(); + if ext == "shp" { + shp_stems.insert(stem); + } else if ext == "gpkg" { + gpkg_stems.insert(stem); } + files.push(path.display().to_string()); + } else if !is_folder_sidecar_ext(&ext) { + skipped += 1; + } + } + // Errors here were already counted by the first pass. + let entries = std::fs::read_dir(dir).map_err(|e| format!("folder '{folder}': {e}"))?; + for entry in entries.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if !ft.is_file() || ft.is_symlink() { + continue; + } + let path = entry.path(); + if path.to_str().is_none() { + continue; + } + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if !is_folder_sidecar_ext(&ext) { + continue; + } + let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("").to_ascii_lowercase(); + let is_shp_sidecar = (matches!(ext.as_str(), "dbf" | "shx" | "prj" | "cpg" | "qpj" | "sbn" | "sbx" | "qix") + && shp_stems.contains(&stem)) + || (ext == "xml" && stem.strip_suffix(".shp").map(|s| shp_stems.contains(s)).unwrap_or(false)); + let is_gpkg_sidecar = matches!(ext.as_str(), "gpkg-wal" | "gpkg-shm" | "wal" | "shm") + && (gpkg_stems.contains(&stem) || stem.strip_suffix(".gpkg").map(|s| gpkg_stems.contains(s)).unwrap_or(false)); + if !is_shp_sidecar && !is_gpkg_sidecar { + skipped += 1; } - Err(e) => return Err(format!("folder '{folder}': {e}")), } if symlinks > 0 { s.notify(format!("folder '{folder}': {symlinks} symlink(s) skipped"), false); @@ -1486,12 +1474,6 @@ pub fn run_batch_with(s: &mut Session, p: &Value, control: &std::sync::Arc 0 { s.notify(format!("folder '{folder}': {unreadable} unreadable directory entries skipped"), true); } - if files.len() > MAX_BATCH_ITEMS { - return Err(format!( - "folder '{folder}' has {} supported files, more than a batch runs (at most {MAX_BATCH_ITEMS}); split it into smaller folders", - files.len() - )); - } files.sort(); targets.extend(files); } @@ -5391,50 +5373,34 @@ mod batch_tests { std::fs::write(dir.join(format!("{i}.csv")), "x,y\n").unwrap(); } let mut s = test_session(); - let before = s.map().layers.len(); + let layers = s.map().layers.len(); + let history = s.project.gp_history.len(); let res = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})); let _ = std::fs::remove_dir_all(&dir); let err = res.unwrap_err(); - assert!(err.contains(&format!("{} supported files", super::MAX_BATCH_ITEMS + 1)), "{err}"); - assert_eq!(s.map().layers.len(), before); + assert!(err.contains(&format!("more than {} supported files", super::MAX_BATCH_ITEMS)), "{err}"); + assert_eq!(s.map().layers.len(), layers, "no item ran"); + assert_eq!(s.project.gp_history.len(), history, "no item was attempted"); } #[test] - fn a_folder_of_many_other_files_still_runs_its_data_files() { + fn a_folder_of_other_and_orphan_sidecar_files_runs_its_data_files() { let dir = std::env::temp_dir().join(format!("rustgis-batch-other-{}-{}", std::process::id(), now_ms())); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join("pts.csv"), "x,y\n1,2\n").unwrap(); - for i in 0..=super::MAX_FOLDER_ENTRIES { - // `.xml` files that aren't `.shp.xml` can't be sidecars, so they aren't held. - std::fs::write(dir.join(format!("{i}.xml")), "").unwrap(); + // Text files, `.xml` files that aren't `.shp.xml` and `.dbf` files with no `.shp` + // are none of them sidecars of a file the batch runs. + for i in 0..50 { + for ext in ["txt", "xml", "dbf"] { + std::fs::write(dir.join(format!("{i}.{ext}")), "").unwrap(); + } } let mut s = test_session(); let res = s.run("gp.batch", &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}})); let _ = std::fs::remove_dir_all(&dir); let res = res.unwrap(); assert_eq!(res["total"], 1, "{res}"); - let skipped = format!("{} unsupported files skipped", super::MAX_FOLDER_ENTRIES + 1); - assert!(s.notices.iter().any(|n| n.text.contains(&skipped)), "{:?}", s.notices); - } - - #[test] - fn a_folder_over_the_entry_limit_is_refused_before_any_item_runs() { - let dir = std::env::temp_dir().join(format!("rustgis-batch-entries-{}-{}", std::process::id(), now_ms())); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join("pts.csv"), "x,y\n1,2\n").unwrap(); - for i in 0..=super::MAX_FOLDER_ENTRIES { - std::fs::write(dir.join(format!("{i}.dbf")), "").unwrap(); - } - let mut s = test_session(); - let layers = s.map().layers.len(); - let history = s.project.gp_history.len(); - let control = rustgis_analysis::control::Control::new(); - let res = super::run_batch_with(&mut s, &json!({"tool": "buffer", "folder": dir, "params": {"distance": 1}}), &control); - let _ = std::fs::remove_dir_all(&dir); - let err = res.unwrap_err(); - assert!(err.contains(&format!("more than {} data and sidecar files", super::MAX_FOLDER_ENTRIES)), "{err}"); - assert_eq!(s.map().layers.len(), layers, "no item ran"); - assert_eq!(s.project.gp_history.len(), history, "no item was attempted"); + assert!(s.notices.iter().any(|n| n.text.contains("150 unsupported files skipped")), "{:?}", s.notices); } #[test]