From 541f79ace2a89aa2a2813872ef5074d89a3e1e33 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 4 Nov 2026 08:02:12 +0100 Subject: [PATCH 1/2] feat(ui): the Analysis pane's single-pass raster tools run in the background (#1291 step 3) Raster Functions, Function Editor chains, Suitability Modeler, Change Detection, Classification Tools (from training polygons), Indices, Process (clip and resample) and a workspace's Build Mosaic now read their inputs on the UI thread and make their raster on a thread of their own, with progress and Cancel, landing through the command again with {"ran": id} as Viewshed and the classifications do. One table in workflows (IN_BACKGROUND) names the commands that run this way, what they are called and what they read; raster_start, made_or_ran, the pane and the recorder all read it. A command's answer is the added layer's (id, size, bands) with what it says besides. Remap, weighted overlay, minimum-distance classification, the change comparison, clip and block-mean resample go through rasterops::per_index: counted, cancellable, rows on several cores. The training classes' means and the mosaic's resampling are counted and cancellable too (the mosaic on several cores). Edit Pixels stays immediate. --- crates/analysis/src/rasterops.rs | 252 ++++++++++++++++---- crates/engine/src/cmd/deeplearning.rs | 2 +- crates/engine/src/cmd/imagery.rs | 326 ++++++++++++++------------ crates/engine/src/cmd/workflows.rs | 207 +++++++++------- crates/engine/src/tests.rs | 67 ++++++ crates/ui-egui/src/analysispane.rs | 11 +- crates/ui-egui/src/lib.rs | 6 +- docs/user-guide/analysis.md | 9 +- 8 files changed, 591 insertions(+), 289 deletions(-) diff --git a/crates/analysis/src/rasterops.rs b/crates/analysis/src/rasterops.rs index 1937c5fecc..5dadf0aff8 100644 --- a/crates/analysis/src/rasterops.rs +++ b/crates/analysis/src/rasterops.rs @@ -93,6 +93,32 @@ fn per_cell_on(threads: Option, r: &Raster, cost: usize, f: impl Fn(i64, Ok(out) } +/// `f` of each of `n` cells by index, `chunk` cells at a time on as many threads as pay off, each +/// with its own `scratch()`: a cell writes only itself. `cost` is a cell's work as in +/// [`per_cell`]; the chunks count toward the run's progress, and a cancelled run stops (#1291). +pub fn per_index( + n: usize, + chunk: usize, + cost: usize, + scratch: impl Fn() -> S + Sync, + f: impl Fn(&mut S, usize) -> f32 + Sync, +) -> Result, String> { + let mut out = new_band(n, 1, NODATA)?; + let chunk = chunk.max(1); + let lines = n.div_ceil(chunk); + let threads = crate::parallel::threads_for_rows(lines, n.saturating_mul(cost), 0); + crate::control::begin(lines); + crate::parallel::for_each_row(out.chunks_mut(chunk), threads, scratch, |s, line, cells: &mut [f32]| { + for (k, slot) in cells.iter_mut().enumerate() { + *slot = f(s, line * chunk + k); + } + })?; + Ok(out) +} + +/// The cells [`weighted_overlay`] makes at a time: its criteria carry no grid, so no rows. +const OVERLAY_CHUNK: usize = 1 << 14; + /// Draw the interior from three borrowed rows; edges and no-data keep the scalar path. fn terrain(r: &Raster, data: &[f32], zf: f64, f: impl Fn(f64, f64) -> f64 + Sync) -> Result, String> { terrain_on(None, r, data, zf, f) @@ -230,19 +256,23 @@ fn band_math_on(threads: Option, r: &Raster, expr: &str) -> Result Result, String> { let d = band(r, b)?; - let mut out = grid(r.width as usize, r.height as usize)?; - out.extend(d.iter().map(|v| { - if !r.is_valid(*v) { - return NODATA; - } - let x = *v as f64; - match ranges.iter().find(|(lo, hi, _)| x >= *lo && x < *hi) { - Some((_, _, new)) => *new as f32, - None if keep => *v, - None => NODATA, - } - })); - Ok(out) + let n = r.width as usize * r.height as usize; + // A lookup in the ranges: about a unit a cell. + per_index( + n, + r.width as usize, + 1, + || (), + |_, i| { + let Some(v) = d.get(i).copied().filter(|v| r.is_valid(*v)) else { return NODATA }; + let x = v as f64; + match ranges.iter().find(|(lo, hi, _)| x >= *lo && x < *hi) { + Some((_, _, new)) => *new as f32, + None if keep => v, + None => NODATA, + } + }, + ) } /// `band`'s values resampled (nearest neighbour) onto `target`'s grid, through longitude/latitude. @@ -277,19 +307,23 @@ pub fn weighted_overlay(criteria: &[(Vec, f64)]) -> Result, String if total <= 0.0 { return Err("the weights must add up to more than zero".into()); } - let mut out = grid(n, 1)?; - out.extend((0..n).map(|i| { - let mut s = 0.0; - for (vals, w) in criteria { - let v = vals.get(i).copied().unwrap_or(NODATA); - if v == NODATA || !v.is_finite() { - return NODATA; + per_index( + n, + OVERLAY_CHUNK, + criteria.len(), + || (), + |_, i| { + let mut s = 0.0; + for (vals, w) in criteria { + let v = vals.get(i).copied().unwrap_or(NODATA); + if v == NODATA || !v.is_finite() { + return NODATA; + } + s += v as f64 * w.max(0.0) / total; } - s += v as f64 * w.max(0.0) / total; - } - s as f32 - })); - Ok(out) + s as f32 + }, + ) } /// Metres between two longitude/latitude points. @@ -671,27 +705,42 @@ pub fn classify_min_distance(r: &Raster, bands: &[usize], means: &[(f32, Vec = Vec::with_capacity(data.len()); - let mut out = grid(r.width as usize, r.height as usize)?; - out.extend((0..n).map(|i| { - p.clear(); - for d in &data { - match d.get(i).copied().filter(|v| r.is_valid(*v)) { - Some(v) => p.push(f64::from(v)), - None => return NODATA, + // A distance to each class's mean in each band. + let cost = means.len().saturating_mul(data.len()).max(1); + per_index( + n, + r.width as usize, + cost, + || Vec::with_capacity(data.len()), + |p: &mut Vec, i| { + p.clear(); + for d in &data { + match d.get(i).copied().filter(|v| r.is_valid(*v)) { + Some(v) => p.push(f64::from(v)), + None => return NODATA, + } } - } - means - .iter() - .map(|(c, m)| (*c, m.iter().zip(&p).map(|(a, b)| (a - b).powi(2)).sum::())) - .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) - .map_or(NODATA, |x| x.0) - })); - Ok(out) + means + .iter() + .map(|(c, m)| (*c, m.iter().zip(p.iter()).map(|(a, b)| (a - b).powi(2)).sum::())) + .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map_or(NODATA, |x| x.0) + }, + ) } /// The mean of each band over the pixels whose centres fall in a polygon (lon/lat rings). pub fn zonal_means(r: &Raster, bands: &[usize], rings: &[Vec]) -> Option<(Vec, usize)> { + means_under(r, bands, rings, || Ok(())).unwrap_or(None) +} + +/// [`zonal_means`], `each_row()` asked before each of the raster's rows: an error stops it. +fn means_under( + r: &Raster, + bands: &[usize], + rings: &[Vec], + mut each_row: impl FnMut() -> Result<(), crate::control::Cancelled>, +) -> Result, usize)>, crate::control::Cancelled> { let mut e = Extent::EMPTY; for ring in rings { for c in ring { @@ -701,6 +750,7 @@ pub fn zonal_means(r: &Raster, bands: &[usize], rings: &[Vec]) -> Option< let mut sums = vec![0.0; bands.len()]; let mut n = 0usize; for row in 0..r.height as i64 { + each_row()?; for col in 0..r.width as i64 { let w = r.transform.to_world(col as f64 + 0.5, row as f64 + 0.5); let Some(ll) = r.transformer().to_lonlat(w) else { continue }; @@ -714,7 +764,84 @@ pub fn zonal_means(r: &Raster, bands: &[usize], rings: &[Vec]) -> Option< } } } - (n > 0).then(|| (sums.into_iter().map(|s| s / n as f64).collect(), n)) + Ok((n > 0).then(|| (sums.into_iter().map(|s| s / n as f64).collect(), n))) +} + +/// Each class's mean of `bands` over the pixels under its training polygons (lon/lat rings, each +/// with its class), and how many pixels: the classes in the order first met, one with no pixel +/// under its polygons left out. Each polygon's rows count toward the run's progress, and a +/// cancelled run stops (#1291). +pub fn class_means(r: &Raster, bands: &[usize], polygons: &[(String, Vec>)]) -> Result, usize)>, String> { + crate::control::begin(polygons.len().saturating_mul(r.height as usize)); + let mut classes: Vec<(String, Vec, usize)> = Vec::new(); + for (key, rings) in polygons { + let counted = || { + crate::control::check()?; + crate::control::advance(1); + Ok(()) + }; + let Some((m, n)) = means_under(r, bands, rings, counted)? else { continue }; + match classes.iter_mut().find(|c| c.0 == *key) { + Some(c) => { + let total = c.2 + n; + c.1 = c.1.iter().zip(&m).map(|(a, b)| (a * c.2 as f64 + b * n as f64) / total as f64).collect(); + c.2 = total; + } + None => classes.push((key.clone(), m, n)), + } + } + Ok(classes) +} + +/// `bands` bands of `images` laid onto a north-up longitude/latitude grid of `cols` × `rows` cells +/// of `cell` degrees from `extent`'s north-west corner: a cell takes the first image's value under +/// its centre, or with `mean` the mean of all of theirs; no-data where none has one. Rows go on +/// several threads and count toward the run's progress, and a cancelled run stops (#1291). +pub fn mosaic( + images: &[&Raster], + extent: &Extent, + cell: f64, + (cols, rows): (usize, usize), + bands: usize, + mean: bool, +) -> Result>, String> { + let mut out = (0..bands).map(|_| new_band(cols, rows, NODATA)).collect::, _>>()?; + if cols == 0 { + return Ok(out); + } + // Each row as a slice of every band, for one thread to fill. + let mut by_band: Vec<_> = out.iter_mut().map(|b| b.chunks_mut(cols)).collect(); + let lines: Vec> = (0..rows).map(|_| by_band.iter_mut().filter_map(Iterator::next).collect()).collect(); + // A cell is placed in each image (some ten units) and read in each band. + let work = cols.saturating_mul(rows).saturating_mul(images.len().max(1)).saturating_mul(10 + bands); + let threads = crate::parallel::threads_for_rows(rows, work, 0); + crate::control::begin(rows); + crate::parallel::for_each_row( + lines.into_iter(), + threads, + || Vec::with_capacity(images.len()), + |at: &mut Vec>, row, mut line: Vec<&mut [f32]>| { + for col in 0..cols { + let c = Coord::new(extent.min_x + (col as f64 + 0.5) * cell, extent.max_y - (row as f64 + 0.5) * cell); + // Where the cell falls in each image, once for all bands. + at.clear(); + at.extend(images.iter().map(|r| r.pixel_at(c))); + for (bi, band) in line.iter_mut().enumerate() { + let vals = images.iter().zip(at.iter()).filter_map(|(r, a)| a.and_then(|(cc, rr)| r.value(bi, cc, rr))); + let v = if mean { + let (sum, k) = vals.fold((0.0f64, 0usize), |(s, k), v| (s + v as f64, k + 1)); + (k > 0).then(|| (sum / k as f64) as f32) + } else { + vals.into_iter().next() + }; + if let (Some(v), Some(slot)) = (v, band.get_mut(col)) { + *slot = v; + } + } + } + }, + )?; + Ok(out) } /// An error encountered while validating or transforming a polygon geometry. @@ -1308,6 +1435,47 @@ mod tests { assert_eq!(kmeans_on(Some(4), &r, &[0, 1, 2], 7, 10).unwrap_err(), crate::control::CANCELLED); } + /// The single passes the Analysis pane runs on a thread stop when their run is cancelled + /// (#1291 step 3), and count their progress otherwise. + #[test] + fn single_passes_stop_when_cancelled() { + let (w, h) = (300u32, 200u32); + let data: Vec = (0..w * h).map(|i| (i % 97) as f32).collect(); + let gt = GeoTransform { origin_x: 10.0, origin_y: 46.0, pixel_width: 0.001, pixel_height: -0.001 }; + let r = Raster::new("r", w, h, vec![data.clone(), data], gt, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap(); + let square = vec![vec![Coord::new(10.01, 45.9), Coord::new(10.05, 45.9), Coord::new(10.05, 45.95), Coord::new(10.01, 45.95)]]; + let polygons = vec![("a".to_string(), square.clone()), ("b".to_string(), square)]; + let means = [(1.0, vec![10.0, 10.0]), (2.0, vec![60.0, 60.0])]; + let ranges = [(0.0, 50.0, 1.0), (50.0, 100.0, 2.0)]; + let passes = |r: &Raster| -> Vec> { + vec![ + remap(r, 0, &ranges, false).map(|v| v.len()), + weighted_overlay(&[(r.bands[0].data.clone(), 1.0)]).map(|v| v.len()), + classify_min_distance(r, &[0, 1], &means).map(|v| v.len()), + class_means(r, &[0, 1], &polygons).map(|c| c.len()), + mosaic(&[r], &r.extent(), 0.001, (w as usize, h as usize), 2, true).map(|b| b.len()), + ] + }; + let run = crate::control::Control::new(); + { + let _installed = crate::control::install(Some(run.clone())); + for (i, done) in passes(&r).into_iter().enumerate() { + assert!(done.is_ok(), "pass {i}: {done:?}"); + let (n, total) = run.progress().unwrap(); + assert!(total > 0 && n == total, "pass {i}: {n} of {total}"); + } + } + run.cancel(); + let _installed = crate::control::install(Some(run)); + for (i, done) in passes(&r).into_iter().enumerate() { + assert_eq!(done, Err(crate::control::CANCELLED.to_string()), "pass {i}"); + } + // The class means equal the zonal means they are made of. + let _none = crate::control::install(None); + let c = class_means(&r, &[0, 1], &polygons[..1]).unwrap(); + assert_eq!(Some((c[0].1.clone(), c[0].2)), zonal_means(&r, &[0, 1], &polygons[0].1)); + } + /// Band math and per-cell rows made on several threads give the same cells, bit for bit, as /// one thread (#1252): no-data, NaN and values that come out non-finite included. #[test] diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index 1ecff7b5af..d7d8a425f2 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -252,7 +252,7 @@ fn free_name(s: &Session, base: &str) -> String { } fn classify(s: &mut Session, p: &Value) -> CmdResult { - super::workflows::made_or_ran(s, p, super::workflows::DEEP_LEARNING, classify_work) + super::workflows::made_or_ran(s, p, "dl.classify") } /// The model a classification uses: one of the project's, or one it trains first. diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 69a905e223..f2e2e524cd 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -49,21 +49,21 @@ pub fn commands() -> Vec { r#"{"points": [[column, row, x, y, on?], …] (on: 1/0 or true/false, default on), "transform"?: "similarity"|"affine"|"projective"|"polynomial2"|"polynomial3"|"spline" (default: similarity under 3 points, else affine), "compare"?: bool (fit every transform and recommend one; transform and leaveOneOut are then ignored), "leaveOneOut"?: bool (default true)}"#, ), C::new("raster.change", "Change Detection", "Analyze › Image Analysis", change) - .params(r#"{"before": raster, "after": raster, "band"?: 1-based, "method"?: "difference"|"ratio"|"percent", "name"?}"#), + .params(r#"{"before": raster, "after": raster, "band"?: 1-based, "method"?: "difference"|"ratio"|"percent", "name"?, "ran"?: (internal: a raster the app has worked out)}"#), C::new("raster.spectral", "Spectral Analysis", "Analyze › Image Analysis", spectral) .view() .params(r#"{"layer": raster, "lon"?, "lat"?, "zone"?: polygon layer (the selected or first polygon)}"#), C::new("raster.classify", "Classification Wizard", "Analyze › Image Classification", |s, p| { - super::workflows::made_or_ran(s, p, super::workflows::CLASSIFY, classify_work) + super::workflows::made_or_ran(s, p, "raster.classify") }) .params(r#"{"layer": raster, "classes"?: 2..50, "bands"?: [1-based], "name"?: output, "ran"?: (internal: classes the app has worked out)}"#), C::new("raster.classifySupervised", "Classification Tools", "Analyze › Image Classification", supervised) - .params(r#"{"layer": raster, "training": polygon layer, "classField": field, "bands"?: [1-based], "name"?: output}"#), + .params(r#"{"layer": raster, "training": polygon layer, "classField": field, "bands"?: [1-based], "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#), C::new("raster.index", "Indices", "Analyze › Imagery Tools", index).params( - r#"{"layer": raster, "index": "ndvi"|"ndwi"|"savi"|"nbr"|"ndbi"|"evi", "red"?, "green"?, "blue"?, "nir"?, "swir"?: band numbers, "name"?: output}"#, + r#"{"layer": raster, "index": "ndvi"|"ndwi"|"savi"|"nbr"|"ndbi"|"evi", "red"?, "green"?, "blue"?, "nir"?, "swir"?: band numbers, "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#, ), C::new("raster.process", "Process", "Analyze › Imagery Tools", process) - .params(r#"{"layer": raster, "operation": "clip"|"resample", "extent"?: [minLon, minLat, maxLon, maxLat] (clip; the view when omitted), "polygon"?: polygon layer (clip to its selected or all polygons; cells outside become no-data), "selectedOnly"?: bool (clip to selected polygons when true, all when false), "factor"?: n (resample: n×n pixels become one), "name"?: output}"#), + .params(r#"{"layer": raster, "operation": "clip"|"resample", "extent"?: [minLon, minLat, maxLon, maxLat] (clip; the view when omitted), "polygon"?: polygon layer (clip to its selected or all polygons; cells outside become no-data), "selectedOnly"?: bool (clip to selected polygons when true, all when false), "factor"?: n (resample: n×n pixels become one), "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#), C::new("raster.mosaic", "Mosaic Rasters", "Analyze › Imagery Tools", raster_mosaic).params( r#"{"inputs": [raster layer or file path, …] (two or more, in one coordinate system and cell size; where they overlap the first wins), "name"?: output, "path"?: GeoTIFF to write the mosaic to}"#, ), @@ -96,7 +96,7 @@ pub fn commands() -> Vec { serde_json::to_value(&*w).map_err(|e| e.to_string()) }) .params(r#"{"name": workspace, "cellSize"?: degrees|null, "method"?: "first"|"mean"}"#), - C::new("imagery.mosaic", "Build Mosaic", "Analyze › Ortho Mapping", mosaic).params(r#"{"name": workspace}"#), + C::new("imagery.mosaic", "Build Mosaic", "Analyze › Ortho Mapping", mosaic).params(r#"{"name": workspace, "ran"?: (internal: a raster the app has worked out)}"#), C::new("imagery.mesh", "Export Mesh", "Analyze › Reality Mapping", mesh) .view() .params(r#"{"layer": elevation raster, "path"?: "*.obj" (the text is returned without one), "step"?: pixels between vertices}"#), @@ -569,30 +569,46 @@ fn gcp_fit(_: &mut Session, p: &Value) -> CmdResult { } fn change(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, "raster.change") +} + +/// What a change detection reads of the session, and the work that compares the rasters. +pub(crate) fn change_work(s: &Session, p: &Value) -> Result { let a = raster_named(s, ps(p, "before").ok_or("before is required")?)?; let bfr = raster_named(s, ps(p, "after").ok_or("after is required")?)?; // The method is checked before any pixels are resampled. - let method = ps(p, "method").unwrap_or("difference"); - if !matches!(method, "difference" | "ratio" | "percent") { + let method = ps(p, "method").unwrap_or("difference").to_string(); + if !matches!(method.as_str(), "difference" | "ratio" | "percent") { return Err(format!("unknown method '{method}'")); } let band = uint(p, "band").unwrap_or(1).saturating_sub(1) as usize; - let av = &a.bands.get(band).ok_or("no such band")?.data; - let bv = rasterops::resample_to(&bfr, band, &a)?; - let values = av.iter().zip(&bv).map(|(x, y)| { - if !a.is_valid(*x) || *y == NODATA { - return NODATA; - } - let v = match method { - "ratio" if *x != 0.0 => y / x, - "percent" if *x != 0.0 => (y - x) / x * 100.0, - "difference" => y - x, - _ => f32::NAN, - }; - if v.is_finite() { v } else { NODATA } - }); - let out = raster::collected(a.width as usize, a.height as usize, values).map_err(|e| e.to_string())?; - add(s, rasterops::derive(&a, &name(p, format!("{}_change", a.name)), vec![out])?, false) + a.bands.get(band).ok_or("no such band")?; + let out_name = name(p, format!("{}_change", a.name)); + Ok(Box::new(move || { + let av = &a.bands.get(band).ok_or("no such band")?.data; + let bv = rasterops::resample_to(&bfr, band, &a)?; + let n = a.width as usize * a.height as usize; + let out = rasterops::per_index( + n, + a.width as usize, + 1, + || (), + |_, i| { + let (Some(x), Some(y)) = (av.get(i).copied(), bv.get(i).copied()) else { return NODATA }; + if !a.is_valid(x) || y == NODATA { + return NODATA; + } + let v = match method.as_str() { + "ratio" if x != 0.0 => y / x, + "percent" if x != 0.0 => (y - x) / x * 100.0, + "difference" => y - x, + _ => f32::NAN, + }; + if v.is_finite() { v } else { NODATA } + }, + )?; + super::workflows::made(rasterops::derive(&a, &out_name, vec![out])?, None) + })) } fn spectral(s: &mut Session, p: &Value) -> CmdResult { @@ -620,6 +636,12 @@ fn spectral(s: &mut Session, p: &Value) -> CmdResult { } fn supervised(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, "raster.classifySupervised") +} + +/// What a classification from training polygons reads of the session (the raster, and each +/// polygon with its class), and the work that classifies it. +pub(crate) fn supervised_work(s: &Session, p: &Value) -> Result { let r = raster(s, p)?; // Training polygons on the map mean nothing on a scan laid there to look at. r.require_coordinates()?; @@ -631,35 +653,33 @@ fn supervised(s: &mut Session, p: &Value) -> CmdResult { .filter(|l| l.geometry_type() == GeometryType::Polygon) .ok_or_else(|| format!("no polygon layer '{tl}'"))?; let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; - // Mean of each class's training pixels. - let mut classes: Vec<(String, Vec, usize)> = Vec::new(); + let mut polygons = Vec::new(); for (row, ft) in l.dataset.features.iter().enumerate() { let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue }; let key = l.dataset.value(row, fi).to_string(); - for rings in polys { - if let Some((m, n)) = rasterops::zonal_means(&r, &bands, rings) { - match classes.iter_mut().find(|c| c.0 == key) { - Some(c) => { - let total = c.2 + n; - c.1 = c.1.iter().zip(&m).map(|(a, b)| (a * c.2 as f64 + b * n as f64) / total as f64).collect(); - c.2 = total; - } - None => classes.push((key.clone(), m, n)), - } - } - } - } - if classes.len() < 2 { - return Err("the training polygons need at least two classes with pixels under them".into()); + polygons.extend(polys.iter().map(|rings| (key.clone(), rings.clone()))); } - let means: Vec<(f32, Vec)> = classes.iter().enumerate().map(|(i, c)| ((i + 1) as f32, c.1.clone())).collect(); - let out = rasterops::classify_min_distance(&r, &bands, &means)?; - let v = add(s, rasterops::derive(&r, &name(p, format!("{}_classified", r.name)), vec![out])?, true)?; - let legend: Vec = classes.iter().enumerate().map(|(i, c)| json!({"value": i + 1, "class": c.0, "pixels": c.2})).collect(); - Ok(json!({"layer": v["layer"], "classes": legend})) + let out = name(p, format!("{}_classified", r.name)); + Ok(Box::new(move || { + // Mean of each class's training pixels. + let classes = rasterops::class_means(&r, &bands, &polygons)?; + if classes.len() < 2 { + return Err("the training polygons need at least two classes with pixels under them".into()); + } + let means: Vec<(f32, Vec)> = classes.iter().enumerate().map(|(i, c)| ((i + 1) as f32, c.1.clone())).collect(); + let pixels = rasterops::classify_min_distance(&r, &bands, &means)?; + let layer = class_layer(rasterops::derive(&r, &out, vec![pixels])?)?; + let legend: Vec = classes.iter().enumerate().map(|(i, c)| json!({"value": i + 1, "class": c.0, "pixels": c.2})).collect(); + Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), None)) + })) } fn index(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, "raster.index") +} + +/// What a spectral index reads of the session, and the work that makes it. +pub(crate) fn index_work(s: &Session, p: &Value) -> Result { let r = raster(s, p)?; let bn = |k: &str, d: u64| uint(p, k).unwrap_or(d); let (blue, green, red, nir, swir) = (bn("blue", 1), bn("green", 2), bn("red", 3), bn("nir", 4), bn("swir", 5)); @@ -673,8 +693,8 @@ fn index(s: &mut Session, p: &Value) -> CmdResult { "evi" => format!("2.5 * (b{nir} - b{red}) / (b{nir} + 6 * b{red} - 7.5 * b{blue} + 1)"), o => return Err(format!("unknown index '{o}'")), }; - let out = rasterops::band_math(&r, &expr)?; - add(s, rasterops::derive(&r, &name(p, format!("{}_{idx}", r.name)), vec![out])?, false) + let out = name(p, format!("{}_{idx}", r.name)); + Ok(Box::new(move || super::workflows::made(rasterops::derive(&r, &out, vec![rasterops::band_math(&r, &expr)?])?, None))) } /// Processes a raster via clipping or resampling. @@ -701,6 +721,12 @@ fn index(s: &mut Session, p: &Value) -> CmdResult { /// - Polygon parts that cannot be projected onto the raster grid are reported separately. /// - If all features fail, the operation fails and reports the specific failure cause of the first feature. fn process(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, "raster.process") +} + +/// What a clip or resample reads of the session (the raster, and the polygons or extent it is +/// clipped to), and the work that makes it. +pub(crate) fn process_work(s: &Session, p: &Value) -> Result { let r = raster(s, p)?; match ps(p, "operation").ok_or("operation is required")? { "clip" if p.get("polygon").is_some_and(|v| !v.is_null()) => { @@ -759,26 +785,25 @@ fn process(s: &mut Session, p: &Value) -> CmdResult { if !failed_features.is_empty() { log::warn!("{pl}: {} feature(s) failed polygon validation during clipping", failed_features.len()); } - let mut failed_parts = Vec::new(); - let (cols, rows, mask) = rasterops::polygon_mask_with_failures(&r, &polygons, &mut failed_parts)?.ok_or_else(|| { - format!("no cell of {} falls inside {pl} ({} polygon part(s) failed raster projection)", r.name, failed_parts.len()) - })?; - let out = clip_window(&r, cols, rows, Some(&mask), name(p, format!("{}_clip", r.name)))?; - let mut result = add(s, out, false)?; - let mut messages = Vec::new(); - if !failed_features.is_empty() { - messages.push(format!("{pl}: skipped {} feature(s) with invalid polygon geometry", failed_features.len())); - } - if !failed_parts.is_empty() { - messages.push(format!( - "{pl}: skipped {} polygon part(s) that could not be projected onto the raster grid; valid parts were retained", - failed_parts.len() - )); - } - if !messages.is_empty() { - result["messages"] = json!(messages); - } - Ok(result) + let (out, pl, skipped) = (name(p, format!("{}_clip", r.name)), pl.to_string(), failed_features.len()); + Ok(Box::new(move || { + let mut failed_parts = Vec::new(); + let (cols, rows, mask) = rasterops::polygon_mask_with_failures(&r, &polygons, &mut failed_parts)?.ok_or_else(|| { + format!("no cell of {} falls inside {pl} ({} polygon part(s) failed raster projection)", r.name, failed_parts.len()) + })?; + let clipped = clip_window(&r, cols, rows, Some(&mask), out)?; + let mut messages = Vec::new(); + if skipped > 0 { + messages.push(format!("{pl}: skipped {skipped} feature(s) with invalid polygon geometry")); + } + if !failed_parts.is_empty() { + messages.push(format!( + "{pl}: skipped {} polygon part(s) that could not be projected onto the raster grid; valid parts were retained", + failed_parts.len() + )); + } + super::workflows::made(clipped, (!messages.is_empty()).then(|| json!({"messages": messages}))) + })) } "clip" => { // Before the extent is placed on the raster's coordinates. @@ -817,8 +842,8 @@ fn process(s: &mut Session, p: &Value) -> CmdResult { if c1 <= c0 || r1 <= r0 { return Err("the extent doesn't overlap the raster".into()); } - let out = clip_window(&r, c0..c1, r0..r1, None, name(p, format!("{}_clip", r.name)))?; - add(s, out, false) + let out = name(p, format!("{}_clip", r.name)); + Ok(Box::new(move || super::workflows::made(clip_window(&r, c0..c1, r0..r1, None, out)?, None))) } "resample" => { // Never coarser than the raster itself; trailing columns and rows short of a full @@ -826,45 +851,50 @@ fn process(s: &mut Session, p: &Value) -> CmdResult { r.require_coordinates()?; let k = (uint(p, "factor").unwrap_or(2).clamp(2, 64) as u32).min(r.width.min(r.height)).max(1); let (w, h) = ((r.width / k).max(1), (r.height / k).max(1)); - // Each output pixel is the mean of the valid pixels it covers. - let bands = r - .bands - .iter() - .map(|b| { - let cells = (0..h).flat_map(|row| (0..w).map(move |col| (row, col))); - raster::collected( - w as usize, - h as usize, - cells.map(|(row, col)| { - let (mut sum, mut n) = (0.0f64, 0u32); - for dy in 0..k { - for dx in 0..k { - let (x, y) = (col * k + dx, row * k + dy); - if x >= r.width || y >= r.height { - continue; - } - if let Some(v) = b.data.get((y * r.width + x) as usize).copied().filter(|v| r.is_valid(*v)) { - sum += v as f64; - n += 1; + let out = name(p, format!("{}_resampled", r.name)); + Ok(Box::new(move || { + // Each output pixel is the mean of the valid pixels it covers. + let bands = r + .bands + .iter() + .map(|b| { + // A cell reads its k × k block; the cells are counted and cancellable. + let cost = (k * k) as usize; + rasterops::per_index( + w as usize * h as usize, + w as usize, + cost, + || (), + |_, i| { + let (row, col) = ((i / w as usize) as u32, (i % w as usize) as u32); + let (mut sum, mut n) = (0.0f64, 0u32); + for dy in 0..k { + for dx in 0..k { + let (x, y) = (col * k + dx, row * k + dy); + if x >= r.width || y >= r.height { + continue; + } + if let Some(v) = b.data.get((y * r.width + x) as usize).copied().filter(|v| r.is_valid(*v)) { + sum += v as f64; + n += 1; + } } } - } - if n > 0 { (sum / n as f64) as f32 } else { NODATA } - }), - ) - }) - .collect::, _>>() - .map_err(|e| e.to_string())?; - let gt = GeoTransform { - pixel_width: r.transform.pixel_width * k as f64, - pixel_height: r.transform.pixel_height * k as f64, - ..r.transform - }; - let nodata = r.nodata.or(Some(NODATA as f64)); - let color = if matches!(r.color, ColorHint::Palette(_)) { ColorHint::Gray } else { r.color.clone() }; - let out = Raster::new(name(p, format!("{}_resampled", r.name)), w, h, bands, gt, r.crs, nodata, color, r.data_type.clone()) - .map_err(|e| e.to_string())?; - add(s, out, false) + if n > 0 { (sum / n as f64) as f32 } else { NODATA } + }, + ) + }) + .collect::, _>>()?; + let gt = GeoTransform { + pixel_width: r.transform.pixel_width * k as f64, + pixel_height: r.transform.pixel_height * k as f64, + ..r.transform + }; + let nodata = r.nodata.or(Some(NODATA as f64)); + let color = if matches!(r.color, ColorHint::Palette(_)) { ColorHint::Gray } else { r.color.clone() }; + let out = Raster::new(out, w, h, bands, gt, r.crs, nodata, color, r.data_type.clone()).map_err(|e| e.to_string())?; + super::workflows::made(out, None) + })) } o => Err(format!("unknown operation '{o}'")), } @@ -1081,22 +1111,27 @@ fn clip_window( let (w, h) = (cols.end.saturating_sub(cols.start), rows.end.saturating_sub(rows.start)); let nodata = if mask.is_some() { r.nodata.or(Some(NODATA as f64)) } else { r.nodata }; let fill = nodata.map(|v| v as f32).unwrap_or(NODATA); + // Each band's cells counted and cancellable, as in a run on another thread. + let (wu, width) = (w as usize, r.width as usize); let bands = r .bands .iter() .map(|b| { - let cells = rows.clone().flat_map(|row| cols.clone().map(move |col| (row, col))).enumerate(); - raster::collected( - w as usize, - h as usize, - cells.map(|(i, (row, col))| match mask.map(|m| m.get(i).copied().unwrap_or(false)) { + rasterops::per_index( + wu * h as usize, + wu, + 1, + || (), + |_, i| match mask.map(|m| m.get(i).copied().unwrap_or(false)) { Some(false) => fill, - _ => b.data.get((row * r.width + col) as usize).copied().unwrap_or(fill), - }), + _ => { + let (row, col) = (rows.start as usize + i / wu.max(1), cols.start as usize + i % wu.max(1)); + b.data.get(row * width + col).copied().unwrap_or(fill) + } + }, ) }) - .collect::, _>>() - .map_err(|e| e.to_string())?; + .collect::, _>>()?; let o = r.transform.to_world(f64::from(cols.start), f64::from(rows.start)); let gt = GeoTransform { origin_x: o.x, origin_y: o.y, ..r.transform }; let mut out = Raster::new(name, w, h, bands, gt, r.crs, nodata, r.color.clone(), r.data_type.clone()).map_err(|e| e.to_string())?; @@ -1356,6 +1391,12 @@ fn new_workspace(s: &mut Session, p: &Value) -> CmdResult { /// Mosaics a workspace's images onto one longitude/latitude grid. fn mosaic(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, "imagery.mosaic") +} + +/// What a workspace's mosaic reads of the session (its images and options), and the work that +/// lays them on one grid. +pub(crate) fn mosaic_work(s: &Session, p: &Value) -> Result { let n = ps(p, "name").ok_or("name is required")?; let w = s.project.workspaces.iter().find(|w| w.name.eq_ignore_ascii_case(n)).cloned().ok_or_else(|| format!("no workspace '{n}'"))?; let imgs: Vec> = w.images.iter().map(|i| raster_named(s, i)).collect::>()?; @@ -1377,42 +1418,27 @@ fn mosaic(s: &mut Session, p: &Value) -> CmdResult { return Err(format!("the mosaic would be {cols} × {rows} cells; set a larger cell size in the workspace options")); } let nb = imgs.iter().map(|r| r.band_count()).min().unwrap_or(1).min(if w.kind == "reality" { 1 } else { 4 }); - let mut bands = (0..nb).map(|_| raster::filled(cols, rows, NODATA)).collect::, _>>().map_err(|e| e.to_string())?; - for row in 0..rows { - for col in 0..cols { - let c = Coord::new(e.min_x + (col as f64 + 0.5) * cell, e.max_y - (row as f64 + 0.5) * cell); - // Where the cell falls in each image, once for all bands. - let at: Vec> = imgs.iter().map(|r| r.pixel_at(c)).collect(); - for (bi, band) in bands.iter_mut().enumerate() { - let vals = imgs.iter().zip(&at).filter_map(|(r, a)| a.and_then(|(cc, rr)| r.value(bi, cc, rr))); - let v = if w.method == "mean" { - let (sum, k) = vals.fold((0.0f64, 0usize), |(s, k), v| (s + v as f64, k + 1)); - (k > 0).then(|| (sum / k as f64) as f32) - } else { - vals.into_iter().next() - }; - if let (Some(v), Some(slot)) = (v, band.get_mut(row * cols + col)) { - *slot = v; - } + Ok(Box::new(move || { + let refs: Vec<&Raster> = imgs.iter().map(|r| &**r).collect(); + let mut bands = rasterops::mosaic(&refs, &e, cell, (cols, rows), nb, w.method == "mean")?; + // Colour only when every image is colour. A mosaic of 8-bit images stays 8-bit (means + // rounded), so it displays as they do; anything else is Float32. + let all_rgb = imgs.iter().all(|r| matches!(r.color, ColorHint::Rgb { .. })); + let color = if matches!(nb, 3 | 4) && all_rgb { ColorHint::Rgb { alpha: nb == 4 } } else { ColorHint::Gray }; + let all_u8 = imgs.iter().all(|r| r.data_type == "UInt8"); + let dt = if matches!(color, ColorHint::Rgb { .. }) && all_u8 { "UInt8" } else { "Float32" }; + if dt == "UInt8" { + for v in bands.iter_mut().flatten().filter(|v| **v != NODATA) { + *v = v.round().clamp(0.0, 255.0); } } - } - // Colour only when every image is colour. A mosaic of 8-bit images stays 8-bit (means - // rounded), so it displays as they do; anything else is Float32. - let all_rgb = imgs.iter().all(|r| matches!(r.color, ColorHint::Rgb { .. })); - let color = if matches!(nb, 3 | 4) && all_rgb { ColorHint::Rgb { alpha: nb == 4 } } else { ColorHint::Gray }; - let all_u8 = imgs.iter().all(|r| r.data_type == "UInt8"); - let dt = if matches!(color, ColorHint::Rgb { .. }) && all_u8 { "UInt8" } else { "Float32" }; - if dt == "UInt8" { - for v in bands.iter_mut().flatten().filter(|v| **v != NODATA) { - *v = v.round().clamp(0.0, 255.0); - } - } - let gt = GeoTransform { origin_x: e.min_x, origin_y: e.max_y, pixel_width: cell, pixel_height: -cell }; - let suffix = if w.kind == "reality" { "DSM" } else { "Orthomosaic" }; - let out = Raster::new(format!("{} {suffix}", w.name), cols as u32, rows as u32, bands, gt, Crs::LonLat, Some(NODATA as f64), color, dt) - .map_err(|e| e.to_string())?; - add(s, out, false) + let gt = GeoTransform { origin_x: e.min_x, origin_y: e.max_y, pixel_width: cell, pixel_height: -cell }; + let suffix = if w.kind == "reality" { "DSM" } else { "Orthomosaic" }; + let out = + Raster::new(format!("{} {suffix}", w.name), cols as u32, rows as u32, bands, gt, Crs::LonLat, Some(NODATA as f64), color, dt) + .map_err(|e| e.to_string())?; + super::workflows::made(out, None) + })) } /// A surface model as a triangle mesh (Wavefront OBJ), in metres from its south-west corner. diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index f1d0d33aa1..cadfc09baf 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -46,12 +46,12 @@ pub fn commands() -> Vec { C::new("script.run", "Run Script", "Analyze › Geoprocessing", run_script) .params(r#"{"script": "one command per line: id {json}; # comments"}"#), C::new("raster.function", "Raster Functions", "Analyze › Raster", raster_function).params( - r#"{"layer": raster, "function": "slope"|"aspect"|"hillshade"|"ndvi"|"bandMath"|"remap"|"threshold"|"resample", "band"?: 1-based, "expression"?: "(b4-b3)/(b4+b3)", "ranges"?: [[lo, hi, value]], "value"?: number, "keep"?: bool (remap: keep unmatched cells), "nir"?, "red"?: band numbers (ndvi; default 4 and 3), "azimuth"?, "altitude"?, "zFactor"?, "target"?: raster (resample onto its grid), "name"?: output}"#, + r#"{"layer": raster, "function": "slope"|"aspect"|"hillshade"|"ndvi"|"bandMath"|"remap"|"threshold"|"resample", "band"?: 1-based, "expression"?: "(b4-b3)/(b4+b3)", "ranges"?: [[lo, hi, value]], "value"?: number, "keep"?: bool (remap: keep unmatched cells), "nir"?, "red"?: band numbers (ndvi; default 4 and 3), "azimuth"?, "altitude"?, "zFactor"?, "target"?: raster (resample onto its grid), "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#, ), C::new("raster.functionChain", "Function Editor", "Analyze › Raster", function_chain) - .params(r#"{"layer": raster, "steps": [{"function", …}], "name"?: output}"#), + .params(r#"{"layer": raster, "steps": [{"function", …}], "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#), C::new("analysis.suitability", "Suitability Modeler", "Analyze › Workflows", suitability).params( - r#"{"criteria": [{"layer": raster, "band"?: 1-based, "ranges": [[lo, hi, score]], "weight": number}], "name"?: output}"#, + r#"{"criteria": [{"layer": raster, "band"?: 1-based, "ranges": [[lo, hi, score]], "weight": number}], "name"?: output, "ran"?: (internal: a raster the app has worked out)}"#, ), C::new("analysis.viewshed", "Visibility Analysis", "Analyze › Workflows", viewshed) .params(r#"{"layer": elevation raster, "lon", "lat", "band"?: 1-based, "height"?: observer metres, "radius"?: metres, "name"?: output, "ran"?: (internal: a viewshed the app has worked out)}"#), @@ -119,11 +119,6 @@ fn point(p: &Value, k: &str) -> Result { if c.is_finite() && c.y.abs() <= 90.0 { Ok(c) } else { Err(format!("{k} is [lon, lat]")) } } -/// Adds a computed raster as a layer (embedded as a GeoTIFF, so it saves with the project). -fn add_raster(s: &mut Session, r: Raster) -> CmdResult { - add_made(s, made(r, None)?) -} - /// A raster a command made, as the layer it adds (its GeoTIFF embedded, so it saves with the /// project), what the command says of it besides its layer, and what else the command does once /// it is added: made on any thread, added by [`add_made`]. @@ -157,29 +152,53 @@ pub(crate) fn made_layer(name: String, layer: rustgis_map::RasterLayer, extra: O Made { name, layer, extra, then, ran: None } } -/// Adds a made raster as a layer: the command's answer is the layer, or the layer and what the -/// command says besides (`extra`), and what it does then. +/// Adds a made raster as a layer: the command's answer is the layer's (its id, size and bands) and +/// what the command says besides (`extra`), and what it does then. fn add_made(s: &mut Session, m: Made) -> CmdResult { let Made { name, layer, extra, then, .. } = m; - let v = super::raster::add_raster(s, &name, layer)?; - let mut v = match extra { - Some(Value::Object(mut more)) => { - more.insert("layer".into(), v["layer"].clone()); - Value::Object(more) - } - _ => v, - }; + let mut v = super::raster::add_raster(s, &name, layer)?; + if let (Some(Value::Object(more)), Some(o)) = (extra, v.as_object_mut()) { + o.extend(more); + } if let Some(then) = then { then(s, &mut v); } Ok(v) } -/// A workflow's raster made here, or the one the app made on a thread ([`raster_start`]) and -/// handed over as `ran`. -/// `name` is the command's, as [`raster_start`] gives it: a raster another command made isn't -/// taken. -pub(crate) fn made_or_ran(s: &mut Session, p: &Value, name: &str, work: fn(&Session, &Value) -> Result) -> CmdResult { +/// What a command that makes its raster on another thread reads of the session, and the work +/// that makes it. +type Start = fn(&Session, &Value) -> Result; + +/// The commands that make a raster on another thread from the app ([`raster_start`]), each with +/// what it is called (in its progress, and a cancelled run's error) and what it reads of the +/// session: their work grows with the raster, and most of their passes spread it over the cores +/// from that thread. Run again with `{"ran": id}`, each adds the raster made (#1291 step 3). +const IN_BACKGROUND: &[(&str, &str, Start)] = &[ + ("analysis.viewshed", "Viewshed", viewshed_work), + ("analysis.interpolate", "Interpolate (IDW)", interpolate_work), + ("analysis.suitability", "Suitability Modeler", suitability_work), + ("raster.function", "Raster Functions", function_work), + ("raster.functionChain", "Function Editor", function_chain_work), + ("raster.classify", "Classification Wizard", super::imagery::classify_work), + ("raster.classifySupervised", "Classification Tools", super::imagery::supervised_work), + ("raster.change", "Change Detection", super::imagery::change_work), + ("raster.index", "Indices", super::imagery::index_work), + ("raster.process", "Process", super::imagery::process_work), + ("imagery.mosaic", "Build Mosaic", super::imagery::mosaic_work), + ("dl.classify", "Deep Learning", super::deeplearning::classify_work), +]; + +/// `command` and what it is called, when it makes its raster on another thread from the app (see +/// [`raster_start`]). +pub fn runs_in_background(command: &str) -> Option<(&'static str, &'static str)> { + IN_BACKGROUND.iter().find(|(c, ..)| *c == command).map(|(c, name, _)| (*c, *name)) +} + +/// `command`'s raster ([`IN_BACKGROUND`]) made here, or the one the app made on a thread +/// ([`raster_start`]) and handed over as `ran`: a raster another command made isn't taken. +pub(crate) fn made_or_ran(s: &mut Session, p: &Value, command: &str) -> CmdResult { + let &(_, name, work) = IN_BACKGROUND.iter().find(|(c, ..)| *c == command).ok_or_else(|| format!("{command} makes no raster"))?; if let Some(ran) = p.get("ran").filter(|v| !v.is_null()) { let id = ran.as_u64().ok_or("ran is the number of a run the app worked out")?; let at = s.raster_results.iter().position(|(k, _)| *k == id).ok_or("that run is no longer kept: run it again")?; @@ -193,9 +212,8 @@ pub(crate) fn made_or_ran(s: &mut Session, p: &Value, name: &str, work: fn(&Sess } #[cfg(not(target_arch = "wasm32"))] -/// A command that makes a raster (`analysis.viewshed`, `analysis.interpolate`, `raster.classify`, -/// `dl.classify`), started to make it on another thread ([`RasterRun::run`]) while the session -/// goes on: its inputs are read now (cheaply: the pixels are shared), and [`raster_land`] hands +/// A command that makes a raster ([`IN_BACKGROUND`]), started to make it on another thread +/// ([`RasterRun::run`]) while the session goes on: its inputs are read now (cheaply: the pixels are shared), and [`raster_land`] hands /// the raster to the command, which adds it with undo and history when run again with `ran` /// (#1291 step 3). pub struct RasterRun { @@ -225,13 +243,9 @@ impl RasterRun { /// it would run, to make its raster on another thread: its parameters checked and its inputs read /// now. The run belongs to this project: [`raster_land`] refuses it once another project is open. pub fn raster_start(s: &mut Session, command: &str, p: &Value) -> Result { - let (name, work): (&'static str, Work) = match command { - "analysis.viewshed" => (VIEWSHED, viewshed_work(s, p)?), - "analysis.interpolate" => (INTERPOLATE, interpolate_work(s, p)?), - "raster.classify" => (CLASSIFY, super::imagery::classify_work(s, p)?), - "dl.classify" => (DEEP_LEARNING, super::deeplearning::classify_work(s, p)?), - _ => return Err(format!("{command} doesn't run in the background")), - }; + let &(_, name, start) = + IN_BACKGROUND.iter().find(|(c, ..)| *c == command).ok_or_else(|| format!("{command} doesn't run in the background"))?; + let work = start(s, p)?; s.next_pending = s.next_pending.wrapping_add(1); let id = s.next_pending; // One runs at a time from the app; a few are kept, apart from the downloads' list, so no @@ -649,8 +663,9 @@ pub fn script_commands(script: &str) -> Result, Stri Ok(out) } -/// One raster function's output pixels on `r`'s grid. -fn apply_function(s: &Session, r: &Raster, p: &Value) -> Result, String> { +/// One raster function's output pixels on `r`'s grid, or a resample's on `target`'s (from +/// [`resample_target`]). +fn apply_function(r: &Raster, p: &Value, target: Option<&Raster>) -> Result, String> { let band = band_of(p, r)?; let zf = f(p, "zFactor").unwrap_or(1.0).clamp(0.0001, 100_000.0); match ps(p, "function").ok_or("function is required")? { @@ -685,63 +700,92 @@ fn apply_function(s: &Session, r: &Raster, p: &Value) -> Result, String }; rasterops::remap(r, band, &ranges, b(p, "keep").unwrap_or(false)) } - "resample" => { - let t = raster_of(s, ps(p, "target").ok_or("target (a raster to match) is required")?)?; - if t.width as usize * t.height as usize > rasterops::max_pixels() { - return Err(format!("the target grid is larger than this computer's {} cells", rasterops::max_pixels())); - } - rasterops::resample_to(r, band, &t) - } + "resample" => rasterops::resample_to(r, band, target.ok_or("target (a raster to match) is required")?), other => Err(format!("unknown raster function '{other}'")), } } +/// The raster a resample step (`p`) takes the grid of: none for another function. +fn resample_target(s: &Session, p: &Value) -> Result>, String> { + if ps(p, "function") != Some("resample") { + return Ok(None); + } + let t = raster_of(s, ps(p, "target").ok_or("target (a raster to match) is required")?)?; + if t.width as usize * t.height as usize > rasterops::max_pixels() { + return Err(format!("the target grid is larger than this computer's {} cells", rasterops::max_pixels())); + } + Ok(Some(t)) +} + fn raster_function(s: &mut Session, p: &Value) -> CmdResult { + made_or_ran(s, p, "raster.function") +} + +/// What a raster function reads of the session, and the work that applies it. +fn function_work(s: &Session, p: &Value) -> Result { let src = raster_of(s, ps(p, "layer").ok_or("layer is required")?)?; - let func = ps(p, "function").unwrap_or(""); - let pixels = apply_function(s, &src, p)?; - let grid = if func == "resample" { raster_of(s, ps(p, "target").unwrap_or(""))? } else { src.clone() }; - let r = rasterops::derive(&grid, &out_name(p, format!("{}_{func}", src.name)), vec![pixels])?; - add_raster(s, r) + let target = resample_target(s, p)?; + let name = out_name(p, format!("{}_{}", src.name, ps(p, "function").unwrap_or(""))); + let p = p.clone(); + Ok(Box::new(move || { + let pixels = apply_function(&src, &p, target.as_deref())?; + // Only a resample takes another raster's grid. + made(rasterops::derive(target.as_deref().unwrap_or(&src), &name, vec![pixels])?, None) + })) } /// Runs raster functions one after another, each on the previous one's output. fn function_chain(s: &mut Session, p: &Value) -> CmdResult { - let mut cur: Raster = (*raster_of(s, ps(p, "layer").ok_or("layer is required")?)?).clone(); - let steps = p.get("steps").and_then(Value::as_array).ok_or("steps is required")?; + made_or_ran(s, p, "raster.functionChain") +} + +/// What a chain of raster functions reads of the session (its raster, and each resample's +/// target), and the work that applies them. +fn function_chain_work(s: &Session, p: &Value) -> Result { + let src = raster_of(s, ps(p, "layer").ok_or("layer is required")?)?; + let steps = p.get("steps").and_then(Value::as_array).ok_or("steps is required")?.clone(); if steps.is_empty() || steps.len() > 20 { return Err("give 1 to 20 steps".into()); } - let mut names = Vec::new(); - for (i, st) in steps.iter().enumerate() { - let func = st.get("function").and_then(Value::as_str).unwrap_or("?").to_string(); - let pixels = apply_function(s, &cur, st).map_err(|e| format!("step {} ({func}): {e}", i + 1))?; - // Only a resample takes another raster's grid; otherwise the grid is the current one. - cur = if func == "resample" { - let target = raster_of(s, st.get("target").and_then(Value::as_str).unwrap_or(""))?; - rasterops::derive(&target, &cur.name, vec![pixels])? - } else { - rasterops::derive(&cur, &cur.name, vec![pixels])? - }; - names.push(func); - } - cur.name = out_name(p, format!("{}_{}", cur.name, names.join("_"))); - add_raster(s, cur) + let func = |st: &Value| st.get("function").and_then(Value::as_str).unwrap_or("?").to_string(); + let targets: Vec<_> = steps + .iter() + .enumerate() + .map(|(i, st)| resample_target(s, st).map_err(|e| format!("step {} ({}): {e}", i + 1, func(st)))) + .collect::>()?; + let named = Some(out_name(p, String::new())).filter(|n| !n.is_empty()); + Ok(Box::new(move || { + let mut cur: Raster = (*src).clone(); + let mut names = Vec::new(); + for (i, (st, target)) in steps.iter().zip(&targets).enumerate() { + let f = func(st); + let pixels = apply_function(&cur, st, target.as_deref()).map_err(|e| format!("step {} ({f}): {e}", i + 1))?; + // Only a resample takes another raster's grid; otherwise the grid is the current one. + cur = rasterops::derive(target.as_deref().unwrap_or(&cur), &cur.name, vec![pixels])?; + names.push(f); + } + cur.name = named.unwrap_or_else(|| format!("{}_{}", cur.name, names.join("_"))); + made(cur, None) + })) } fn suitability(s: &mut Session, p: &Value) -> CmdResult { + made_or_ran(s, p, "analysis.suitability") +} + +/// What a weighted overlay reads of the session (each criterion's raster, band, ranges and +/// weight), and the work that scores and weighs them. +fn suitability_work(s: &Session, p: &Value) -> Result { let crit = p.get("criteria").and_then(Value::as_array).ok_or("criteria is required")?; if crit.is_empty() || crit.len() > 20 { return Err("give 1 to 20 criteria".into()); } let first = raster_of(s, crit.first().and_then(|c| c.get("layer")).and_then(Value::as_str).ok_or("each criterion names a raster layer")?)?; - let mut scored = Vec::new(); + let mut criteria = Vec::new(); for c in crit { let r = raster_of(s, c.get("layer").and_then(Value::as_str).ok_or("each criterion names a raster layer")?)?; let band = band_of(c, &r)?; - let on_grid = rasterops::resample_to(&r, band, &first)?; - let grid = rasterops::derive(&first, "c", vec![on_grid])?; let ranges: Vec<(f64, f64, f64)> = c .get("ranges") .and_then(Value::as_array) @@ -755,24 +799,23 @@ fn suitability(s: &mut Session, p: &Value) -> CmdResult { if ranges.is_empty() { return Err("each criterion has ranges [[lo, hi, score], …]".into()); } - let scores = rasterops::remap(&grid, 0, &ranges, false)?; - scored.push((scores, c.get("weight").and_then(Value::as_f64).unwrap_or(1.0))); + criteria.push((r, band, ranges, c.get("weight").and_then(Value::as_f64).unwrap_or(1.0))); } - let out = rasterops::weighted_overlay(&scored)?; - let r = rasterops::derive(&first, &out_name(p, "Suitability".into()), vec![out])?; - add_raster(s, r) + let name = out_name(p, "Suitability".into()); + Ok(Box::new(move || { + let mut scored = Vec::new(); + for (r, band, ranges, weight) in &criteria { + let on_grid = rasterops::resample_to(r, *band, &first)?; + let grid = rasterops::derive(&first, "c", vec![on_grid])?; + scored.push((rasterops::remap(&grid, 0, ranges, false)?, *weight)); + } + let out = rasterops::weighted_overlay(&scored)?; + made(rasterops::derive(&first, &name, vec![out])?, None) + })) } -/// What a viewshed and an IDW surface are called, in a cancelled run's error and to tell their -/// rasters apart. -const VIEWSHED: &str = "Viewshed"; -const INTERPOLATE: &str = "Interpolate (IDW)"; -/// And the k-means classes and a deep learning classification. -pub(crate) const CLASSIFY: &str = "Classification Wizard"; -pub(crate) const DEEP_LEARNING: &str = "Deep Learning"; - fn viewshed(s: &mut Session, p: &Value) -> CmdResult { - made_or_ran(s, p, VIEWSHED, viewshed_work) + made_or_ran(s, p, "analysis.viewshed") } /// What a viewshed reads of the session, and the work that makes it. @@ -829,7 +872,7 @@ fn line_of_sight(s: &mut Session, p: &Value) -> CmdResult { } fn interpolate(s: &mut Session, p: &Value) -> CmdResult { - made_or_ran(s, p, INTERPOLATE, interpolate_work) + made_or_ran(s, p, "analysis.interpolate") } /// What an IDW surface reads of the session, and the work that makes it. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 8c5db8db44..e373dd3ee7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -25806,6 +25806,73 @@ fn analysis_workflows_run_on_another_thread() { assert!(raster_start(&mut s, "analysis.lineOfSight", &p).is_err()); } +/// The Analysis pane's single-pass raster tools made on another thread (#1291 step 3): each raster +/// handed over (`ran`) is the one its command makes here, bit for bit and with what the command +/// says of it besides, and a cancelled run is an error saying so. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn raster_tools_run_on_another_thread() { + use crate::cmd::workflows::{raster_land, raster_start, runs_in_background}; + use rustgis_analysis::control::Control; + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "dem.tif", "data": dem_b64()})).unwrap(); + let dem = s.map().layers.iter().find(|l| l.is_raster()).unwrap().name.clone(); + s.run("raster.function", &json!({"layer": dem, "function": "bandMath", "expression": "b1 + 10", "name": "dem_later"})).unwrap(); + s.run("layer.template", &json!({"template": "polygons", "name": "Training"})).unwrap(); + for (name, x) in [("low", 10.002), ("high", 10.018)] { + s.run("edit.addFeature", &json!({"layer": "Training", "geometry": {"type": "Polygon", "coordinates": [[[x, 45.018], [x + 0.004, 45.018], [x + 0.004, 45.022], [x, 45.022], [x, 45.018]]]}, "attributes": {"name": name}})).unwrap(); + } + s.run("imagery.workspace", &json!({"kind": "ortho", "name": "Valley", "images": [dem, "dem_later"], "cellSize": 0.002})).unwrap(); + let pixels = |s: &Session, v: &Value| -> Vec { + let l = s.map().layer(v["layer"].as_u64().unwrap() as u32).unwrap(); + l.raster.as_ref().unwrap().data.as_ref().unwrap().bands.iter().flat_map(|b| b.data.iter().map(|x| x.to_bits())).collect() + }; + let ranges = json!([[f64::MIN, 300.0, 1], [300.0, f64::MAX, 10]]); + for (command, p) in [ + ("raster.function", json!({"layer": dem, "function": "hillshade"})), + ("raster.function", json!({"layer": dem, "function": "resample", "target": "dem_later"})), + ("raster.functionChain", json!({"layer": dem, "steps": [{"function": "slope"}, {"function": "threshold", "value": 5}]})), + ( + "analysis.suitability", + json!({"criteria": [{"layer": dem, "ranges": ranges, "weight": 2}, {"layer": "dem_later", "ranges": ranges}]}), + ), + ("raster.change", json!({"before": dem, "after": "dem_later", "method": "percent"})), + ("raster.classifySupervised", json!({"layer": dem, "training": "Training", "classField": "name"})), + ("raster.index", json!({"layer": dem, "index": "ndvi", "nir": 1, "red": 1})), + ("raster.process", json!({"layer": dem, "operation": "clip", "polygon": "Training"})), + ("raster.process", json!({"layer": dem, "operation": "clip", "extent": [10.005, 45.005, 10.02, 45.02]})), + ("raster.process", json!({"layer": dem, "operation": "resample", "factor": 4})), + ("imagery.mosaic", json!({"name": "Valley"})), + ] { + assert!(runs_in_background(command).is_some(), "{command}"); + let here = s.run(command, &p).unwrap(); + let run = raster_start(&mut s, command, &p).unwrap(); + let id = run.id; + let made = std::thread::spawn(move || run.run(&Control::new())).join().unwrap().unwrap(); + assert_eq!(raster_land(&mut s, id, made), Ok(true)); + let mut with = p.clone(); + with["ran"] = json!(id); + let there = s.run(command, &with).unwrap(); + assert_eq!(pixels(&s, &here), pixels(&s, &there), "{command} {p}"); + let besides = |v: &Value| { + v.as_object().map(|o| o.iter().filter(|(k, _)| *k != "layer").map(|(k, v)| (k.clone(), v.clone())).collect::>()) + }; + assert_eq!(besides(&here), besides(&there), "{command} {p}"); + // The answer is the added layer's, whatever else the command says. + assert!(there["width"].is_u64() && there["bands"].is_u64(), "{command}: {there}"); + let run = raster_start(&mut s, command, &p).unwrap(); + let control = Control::new(); + control.cancel(); + let e = run.run(&control).err().unwrap(); + assert!(e.contains("was cancelled"), "{command}: {e}"); + } + // Their inputs are read when they start: a missing one is refused then. + assert!(raster_start(&mut s, "raster.change", &json!({"before": dem, "after": "nowhere"})).err().unwrap().contains("nowhere")); + assert!(raster_start(&mut s, "imagery.mosaic", &json!({"name": "Hill"})).err().unwrap().contains("Hill")); + // Pixels edited by hand stay immediate. + assert!(runs_in_background("raster.editPixels").is_none()); +} + /// k-means classes and a deep learning classification made on another thread, as the Analysis pane /// runs them (#1291 step 3): the classes handed over (`ran`) are the ones the command makes here, a /// model trained on the thread is kept in the project only once its classes are added, and a raster diff --git a/crates/ui-egui/src/analysispane.rs b/crates/ui-egui/src/analysispane.rs index 65bb6d4dd4..d959863a1d 100644 --- a/crates/ui-egui/src/analysispane.rs +++ b/crates/ui-egui/src/analysispane.rs @@ -192,17 +192,14 @@ fn show(app: &mut RustGisApp, r: Result) { } /// The workflows that run on a thread of their own on the desktop, as their command and name: -/// their work grows with the raster and the points, and the parallel helpers spread it over the -/// cores from that thread. +/// Zonal Statistics, and those that make a raster +/// ([`rustgis_engine::cmd::workflows::runs_in_background`]). Their work grows with the raster and +/// the points, and the parallel helpers spread it over the cores from that thread. #[cfg(not(target_arch = "wasm32"))] fn background(id: &str) -> Option<(&'static str, &'static str)> { match id { "gp.zonalStats" => Some(("gp.zonalStats", "Zonal Statistics")), - "analysis.viewshed" => Some(("analysis.viewshed", "Viewshed")), - "analysis.interpolate" => Some(("analysis.interpolate", "Interpolate (IDW)")), - "raster.classify" => Some(("raster.classify", "Classification Wizard")), - "dl.classify" => Some(("dl.classify", "Deep Learning")), - _ => None, + _ => rustgis_engine::cmd::workflows::runs_in_background(id), } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 8737bf90ef..005426262e 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1941,9 +1941,7 @@ impl RustGisApp { let Some(h) = self.session.project.gp_history.last().filter(|h| !h.id.is_empty()) else { return }; finished = json!({"tool": h.id, "params": h.params}); ("gp.run", &finished) - } else if matches!(id, "analysis.viewshed" | "analysis.interpolate" | "raster.classify" | "dl.classify") - && params.get("ran").is_some() - { + } else if rustgis_engine::cmd::workflows::runs_in_background(id).is_some() && params.get("ran").is_some() { // A raster an Analysis workflow made on a thread (`ran`): the step is the command // itself, which a replay runs again from its parameters. let mut again = params.clone(); @@ -4383,7 +4381,7 @@ mod record_tests { #[test] fn a_raster_made_on_a_thread_is_recorded_as_its_command() { let mut app = RustGisApp::with_sample(); - for id in ["analysis.viewshed", "analysis.interpolate", "raster.classify", "dl.classify"] { + for id in ["analysis.viewshed", "analysis.interpolate", "raster.classify", "dl.classify", "raster.function", "imagery.mosaic"] { app.ws.recording = Some(Vec::new()); app.record_step(id, &json!({"layer": "Cities", "ran": 7}), None); let steps = app.ws.recording.as_deref().unwrap_or_default(); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 7da282383c..65fd21ead6 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -68,9 +68,12 @@ where they now are (and says how many are gone), and a result that would replace layer, or one loaded whole meanwhile, is refused. Renaming the map while it runs is fine; after switching to another map, a result that adds to or changes the map is refused (one made only of tables or a file is still made). Opening another project drops the run. The Analysis pane's -**Zonal Statistics**, **Viewshed**, **Interpolate (IDW)**, **Classification Wizard** (k-means) and -**Deep Learning** (training and classifying) run in the background the same way, with their -progress and **Cancel** under the pane's inputs, one at a time; so does the map's right-click +**Zonal Statistics**, **Viewshed**, **Interpolate (IDW)**, **Suitability Modeler**, **Raster +Functions** and the **Function Editor**'s chains, **Change Detection**, **Classification Wizard** +(k-means), **Classification Tools** (from training polygons), **Indices**, **Process** (clip and resample), an ortho or +reality workspace's **Build** (its mosaic) and **Deep Learning** (training and classifying) run in +the background the same way, with their progress and **Cancel** under the pane's inputs, one at a +time; so does the map's right-click **Viewshed … Here**, which opens the pane to show it. A model trained there is kept once its classes are added: a cancelled run keeps none. Scripts, models, the AI Assistant, the command line and the web app wait for each tool to finish. From 859ff8c17c38673db5b56bad8e2955108a8bc67f Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 4 Nov 2026 09:12:44 +0100 Subject: [PATCH 2/2] fix(engine): a change detection checks the after raster's band, and a supervised classification needs a band, before they start (#1416 review) --- crates/engine/src/cmd/imagery.rs | 5 +++++ crates/engine/src/cmd/workflows.rs | 6 +++--- crates/engine/src/tests.rs | 3 +++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index f2e2e524cd..0f7face0ff 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -583,6 +583,7 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result Result CmdResul #[cfg(not(target_arch = "wasm32"))] /// A command that makes a raster ([`IN_BACKGROUND`]), started to make it on another thread -/// ([`RasterRun::run`]) while the session goes on: its inputs are read now (cheaply: the pixels are shared), and [`raster_land`] hands -/// the raster to the command, which adds it with undo and history when run again with `ran` -/// (#1291 step 3). +/// ([`RasterRun::run`]) while the session goes on: its inputs are read now (cheaply: the pixels +/// are shared), and [`raster_land`] hands the raster to the command, which adds it with undo and +/// history when run again with `ran` (#1291 step 3). pub struct RasterRun { pub id: u64, /// What the command is called, for a cancelled run's error. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index e373dd3ee7..f1de0a3d3c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -25869,6 +25869,9 @@ fn raster_tools_run_on_another_thread() { // Their inputs are read when they start: a missing one is refused then. assert!(raster_start(&mut s, "raster.change", &json!({"before": dem, "after": "nowhere"})).err().unwrap().contains("nowhere")); assert!(raster_start(&mut s, "imagery.mosaic", &json!({"name": "Hill"})).err().unwrap().contains("Hill")); + assert!(raster_start(&mut s, "raster.change", &json!({"before": dem, "after": "dem_later", "band": 2})).is_err()); + let none = json!({"layer": dem, "training": "Training", "classField": "name", "bands": []}); + assert!(raster_start(&mut s, "raster.classifySupervised", &none).err().unwrap().contains("at least one band")); // Pixels edited by hand stay immediate. assert!(runs_in_background("raster.editPixels").is_none()); }