diff --git a/crates/analysis/src/neural.rs b/crates/analysis/src/neural.rs index 455d293dfe..af1dacbf3a 100644 --- a/crates/analysis/src/neural.rs +++ b/crates/analysis/src/neural.rs @@ -124,9 +124,39 @@ fn softmax(a: &mut [f32]) { } } +/// The samples of a model's classes under its training polygons, each polygon's outline given +/// with its class and its share of the class's samples, and each class taking up to +/// `max_per_class`: the samples, and how many each class got. The polygons are counted for the +/// run's progress, and a cancelled run stops between rows of a polygon (see [`crate::control`]). +#[allow(clippy::type_complexity)] +pub fn training_samples( + r: &Raster, + bands: &[usize], + polygons: &[(Vec>, usize, usize)], + classes: usize, + max_per_class: usize, +) -> Result<(Vec<(Vec, usize)>, Vec), String> { + crate::control::begin(polygons.len()); + let mut samples: Vec<(Vec, usize)> = Vec::new(); + let mut per_class = vec![0usize; classes]; + for (rings, ci, share) in polygons { + let room = max_per_class.saturating_sub(per_class.get(*ci).copied().unwrap_or(0)).min(*share); + let got = polygon_samples(r, bands, rings, room); + // Stopped part of the way through the polygon: what it got isn't its share. + crate::control::check()?; + if let Some(n) = per_class.get_mut(*ci) { + *n += got.len(); + } + samples.extend(got.into_iter().map(|x| (x, *ci))); + crate::control::advance(1); + } + Ok((samples, per_class)) +} + /// Band values of the pixels inside a polygon (`rings`, longitude/latitude), up to `max`. Only /// the pixels under the polygon's outline are visited, every `step`-th one in each direction when -/// there are many more than `max`, so the samples spread across the polygon. +/// there are many more than `max`, so the samples spread across the polygon. A cancelled run +/// stops between rows, with the pixels found so far. pub fn polygon_samples(r: &Raster, bands: &[usize], rings: &[Vec], max: usize) -> Vec> { let mut out = Vec::new(); let tr = r.transformer(); @@ -155,6 +185,9 @@ pub fn polygon_samples(r: &Raster, bands: &[usize], rings: &[Vec], max: u out.clear(); let mut row = r0; 'scan: while row < r1 { + if crate::control::cancelled() { + return out; + } let mut col = c0; while col < c1 { if out.len() >= max { diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index 2b671da4ce..1ecff7b5af 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -12,7 +12,7 @@ use rustgis_geom::Coord; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use super::imagery::{add, bands_param, name, raster}; +use super::imagery::{bands_param, class_layer, name, raster}; use super::{f, ps}; use crate::{CmdResult, CommandSpec as C, Session}; @@ -56,7 +56,7 @@ pub fn commands() -> Vec { .when(need_raster) .params(r#"{"layer": raster, "training": polygon layer, "classField": field naming each polygon's class, "bands"?: [1-based], "hidden"?: [units per hidden layer] (default [32, 16]), "epochs"?: 1..500 (default 60), "model"?: name}"#), C::new("dl.classify", "Deep Learning Tools", "Analyze › Image Classification", classify).when(need_raster).params( - r#"{"layer": raster, "model"?: a trained model's name, "training"?: polygon layer, "classField"?: field, "bands"?: [1-based], "hidden"?: [units], "epochs"?: 1..500 (without a model: train one first, as dl.train), "name"?: output}"#, + r#"{"layer": raster, "model"?: a trained model's name, "training"?: polygon layer, "classField"?: field, "bands"?: [1-based], "hidden"?: [units], "epochs"?: 1..500 (without a model: train one first, as dl.train), "name"?: output, "ran"?: (internal: classes the app has worked out)}"#, ), C::new("dl.package", "Deep Learning Package", "Publish › Package", |s, p| { let path = ps(p, "path").ok_or("path is required")?; @@ -149,6 +149,42 @@ fn model_index(s: &Session, p: &Value) -> Result { /// Trains a model from the pixels under training polygons. fn train(s: &Session, p: &Value) -> Result { + training(s, p)?.fit() +} + +/// A model to train: the raster, the training polygons and their classes, read of the session, and +/// how to train it ([`Training::fit`], which reads nothing more and runs on any thread: the pixels +/// under the polygons are sampled there, as the network learns). +struct Training { + raster: std::sync::Arc, + bands: Vec, + classes: Vec, + /// Each polygon's outline, its class and its share of the class's samples. + polygons: Vec<(Vec>, usize, usize)>, + hidden: Vec, + epochs: usize, + name: String, + /// No name was given: the name's base, for a name of its own when the model is kept (a model + /// kept meanwhile may have taken this one). + auto: Option, +} + +impl Training { + fn fit(self) -> Result { + let Training { raster, bands, classes, polygons, hidden, epochs, name, .. } = self; + let (samples, per_class) = neural::training_samples(&raster, &bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?; + // Every class needs pixels of its own (its polygons may miss the raster or cover no data). + if let Some((c, _)) = classes.iter().zip(&per_class).find(|(_, n)| **n < 5) { + return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons")); + } + let t = neural::train(&bands, classes, &samples, &hidden, epochs)?; + Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs }) + } +} + +/// What training reads of the session: the training polygons and their classes, and the model's +/// name. +fn training(s: &Session, p: &Value) -> Result { let r = raster(s, p)?; r.require_coordinates()?; let bands = bands_param(p, &r)?; @@ -160,8 +196,7 @@ fn train(s: &Session, p: &Value) -> Result { .ok_or_else(|| format!("no polygon layer '{tl}'"))?; let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; let mut classes: Vec = Vec::new(); - let mut samples: Vec<(Vec, usize)> = Vec::new(); - let mut per_class: Vec = Vec::new(); + let mut parts: Vec<(Vec>, usize, usize)> = Vec::new(); // Polygons per class, so each polygon of a class gets a fair share of its samples. let mut polygons: HashMap = HashMap::new(); for (row, ft) in l.dataset.features.iter().enumerate() { @@ -182,73 +217,96 @@ fn train(s: &Session, p: &Value) -> Result { Some(i) => i, None if classes.len() < neural::MAX_CLASSES => { classes.push(key); - per_class.push(0); classes.len() - 1 } None => return Err(format!("training holds at most {} classes", neural::MAX_CLASSES)), }; - for rings in polys { - let room = MAX_SAMPLES_PER_CLASS.saturating_sub(per_class.get(ci).copied().unwrap_or(0)).min(share); - let got = neural::polygon_samples(&r, &bands, rings, room); - if let Some(n) = per_class.get_mut(ci) { - *n += got.len(); - } - samples.extend(got.into_iter().map(|x| (x, ci))); - } - } - // Every class needs pixels of its own (its polygons may miss the raster or cover no data). - if let Some((c, _)) = classes.iter().zip(&per_class).find(|(_, n)| **n < 5) { - return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons")); + parts.extend(polys.iter().map(|rings| (rings.clone(), ci, share))); } let hidden: Vec = match p.get("hidden").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_u64).map(|u| u as usize).take(neural::MAX_LAYERS).collect(), None => vec![32, 16], }; let epochs = f(p, "epochs").unwrap_or(60.0).clamp(1.0, neural::MAX_EPOCHS as f64) as usize; - let t = neural::train(&bands, classes, &samples, &hidden, epochs)?; // Without a name given, a new model never replaces one: it gets a name of its own. - let name = match ps(p, "model") { - Some(_) => name(&json!({"name": p.get("model")}), format!("{}_model", r.name)), + let (name, auto) = match ps(p, "model") { + Some(_) => (name(&json!({"name": p.get("model")}), format!("{}_model", r.name)), None), None => { let base = format!("{}_model", r.name); - let taken = |n: &str| s.project.dl_models.iter().any(|x| x.name.eq_ignore_ascii_case(n)); - let mut n = base.clone(); - let mut k = 1; - while taken(&n) { - k += 1; - n = format!("{base} {k}"); - } - n + (free_name(s, &base), Some(base)) } }; - Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs }) + Ok(Training { raster: r, bands, classes, polygons: parts, hidden, epochs, name, auto }) +} + +/// `base`, or `base 2`, `base 3`… : the first no model of the project has (whatever its case). +fn free_name(s: &Session, base: &str) -> String { + let taken = |n: &str| s.project.dl_models.iter().any(|x| x.name.eq_ignore_ascii_case(n)); + let mut n = base.to_string(); + let mut k = 1; + while taken(&n) { + k += 1; + n = format!("{base} {k}"); + } + n } fn classify(s: &mut Session, p: &Value) -> CmdResult { + super::workflows::made_or_ran(s, p, super::workflows::DEEP_LEARNING, classify_work) +} + +/// The model a classification uses: one of the project's, or one it trains first. +enum Model { + Saved(DlModel), + Train(Training), +} + +/// What a deep learning classification reads of the session (the raster, and the model or what +/// trains one), and the work that trains the model if need be and classifies the pixels. A model +/// trained is kept in the project once the classes are added. +pub(crate) fn classify_work(s: &Session, p: &Value) -> Result { let r = raster(s, p)?; r.require_coordinates()?; - let (m, trained) = if p.get("training").is_some() { - (train(s, p)?, true) + let model = if p.get("training").is_some() { + Model::Train(training(s, p)?) } else { let i = model_index(s, p)?; - (s.project.dl_models.get(i).cloned().ok_or("no model")?, false) + Model::Saved(s.project.dl_models.get(i).cloned().ok_or("no model")?) }; - let (class, prob) = neural::classify(&r, &m.model, rasterops::NODATA)?; - let mut counts = vec![0usize; m.model.classes.len()]; - for v in class.iter().filter(|v| **v >= 1.0) { - if let Some(c) = counts.get_mut(*v as usize - 1) { - *c += 1; + let out = name(p, format!("{}_dl", r.name)); + Ok(Box::new(move || { + let (m, trained) = match model { + Model::Saved(m) => (m, None), + Model::Train(t) => { + let auto = t.auto.clone(); + (t.fit()?, Some(auto)) + } + }; + let (class, prob) = neural::classify(&r, &m.model, rasterops::NODATA)?; + let mut counts = vec![0usize; m.model.classes.len()]; + for v in class.iter().filter(|v| **v >= 1.0) { + if let Some(c) = counts.get_mut(*v as usize - 1) { + *c += 1; + } } - } - let out = rasterops::derive(&r, &name(p, format!("{}_dl", r.name)), vec![class, prob])?; - let v = add(s, out, true)?; - let legend: Vec = - m.model.classes.iter().zip(&counts).enumerate().map(|(i, (c, n))| json!({"value": i + 1, "class": c, "pixels": n})).collect(); - let mut r = json!({"layer": v["layer"], "model": m.name, "accuracy": m.accuracy, "classes": legend}); - if trained { - r["replaced"] = json!(store(s, m)); - } - Ok(r) + let layer = class_layer(rasterops::derive(&r, &out, vec![class, prob])?)?; + let legend: Vec = + m.model.classes.iter().zip(&counts).enumerate().map(|(i, (c, n))| json!({"value": i + 1, "class": c, "pixels": n})).collect(); + let extra = json!({"model": m.name, "accuracy": m.accuracy, "classes": legend}); + // Kept once its classes are added. One that was given no name is named then, so it never + // replaces a model kept while it trained. + let then: Option = trained.map(|auto| { + Box::new(move |s: &mut Session, v: &mut Value| { + let mut m = m; + if let Some(base) = auto { + m.name = free_name(s, &base); + v["model"] = json!(m.name); + } + v["replaced"] = json!(store(s, m)); + }) as super::workflows::Then + }); + Ok(super::workflows::made_layer(out, layer, Some(extra), then)) + })) } /// A deep learning package: the model and a description of its inputs and classes. diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 66d57731de..69a905e223 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -54,19 +54,9 @@ pub fn commands() -> Vec { .view() .params(r#"{"layer": raster, "lon"?, "lat"?, "zone"?: polygon layer (the selected or first polygon)}"#), C::new("raster.classify", "Classification Wizard", "Analyze › Image Classification", |s, p| { - let r = raster(s, p)?; - let bands = bands_param(p, &r)?; - if bands.is_empty() { - return Err("give at least one band".into()); - } - let k = match p.get("classes") { - None => 5, - Some(_) => uint(p, "classes").filter(|k| (2..=50).contains(k)).ok_or("classes is 2 to 50")? as usize, - }; - let classes = rasterops::kmeans(&r, &bands, k, 20)?; - add(s, rasterops::derive(&r, &name(p, format!("{}_classes", r.name)), vec![classes])?, true) + super::workflows::made_or_ran(s, p, super::workflows::CLASSIFY, classify_work) }) - .params(r#"{"layer": raster, "classes"?: 2..50, "bands"?: [1-based], "name"?: output}"#), + .params(r#"{"layer": raster, "classes"?: 2..50, "bands"?: [1-based], "name"?: output, "ran"?: (internal: classes the app has worked out)}"#), C::new("raster.classifySupervised", "Classification Tools", "Analyze › Image Classification", supervised) .params(r#"{"layer": raster, "training": polygon layer, "classField": field, "bands"?: [1-based], "name"?: output}"#), C::new("raster.index", "Indices", "Analyze › Imagery Tools", index).params( @@ -151,11 +141,37 @@ fn layer_of(r: Raster) -> Result { /// Adds a raster layer; classes get unique-value colours. pub(crate) fn add(s: &mut Session, r: Raster, classes: bool) -> CmdResult { let n = r.name.clone(); + let layer = if classes { class_layer(r)? } else { layer_of(r)? }; + super::raster::add_raster(s, &n, layer) +} + +/// A raster of classes as a layer, each class a colour of its own. +pub(crate) fn class_layer(r: Raster) -> Result { let mut layer = layer_of(r)?; - if classes && let Some(d) = layer.data.clone() { + if let Some(d) = layer.data.clone() { layer.style.renderer = rustgis_map::raster::unique_values(&d, 0, rustgis_map::Ramp::Harbour); } - super::raster::add_raster(s, &n, layer) + Ok(layer) +} + +/// What the Classification Wizard (k-means) reads of the session, and the work that makes its +/// classes. +pub(crate) fn classify_work(s: &Session, p: &Value) -> Result { + let r = raster(s, p)?; + let bands = bands_param(p, &r)?; + if bands.is_empty() { + return Err("give at least one band".into()); + } + let k = match p.get("classes") { + None => 5, + Some(_) => uint(p, "classes").filter(|k| (2..=50).contains(k)).ok_or("classes is 2 to 50")? as usize, + }; + let out = name(p, format!("{}_classes", r.name)); + Ok(Box::new(move || { + let classes = rasterops::kmeans(&r, &bands, k, 20)?; + let layer = class_layer(rasterops::derive(&r, &out, vec![classes])?)?; + Ok(super::workflows::made_layer(out, layer, None, None)) + })) } /// Replaces a raster layer's pixels, keeping its look. diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 7d9362a574..f1d0d33aa1 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -124,46 +124,62 @@ fn add_raster(s: &mut Session, r: Raster) -> CmdResult { add_made(s, made(r, None)?) } -/// A raster a workflow made, with its GeoTIFF (so it saves with the project) and what the command -/// says of it besides its layer: made on any thread, added by [`add_made`]. +/// A raster a command made, as the layer it adds (its GeoTIFF embedded, so it saves with the +/// project), what the command says of it besides its layer, and what else the command does once +/// it is added: made on any thread, added by [`add_made`]. pub struct Made { - raster: Raster, - bytes: Vec, + name: String, + layer: rustgis_map::RasterLayer, extra: Option, + then: Option, /// Made on another thread: the command's name and the map it was started on. ran: Option<(&'static str, u64)>, } /// A workflow's work once its inputs are read: it reads nothing more of the session, so it runs on /// any thread. -type Work = Box Result + Send>; +pub(crate) type Work = Box Result + Send>; -fn made(raster: Raster, extra: Option) -> Result { +/// What a command does to the session once its raster is added (a trained model kept), and what it +/// adds to its answer. +pub(crate) type Then = Box; + +/// A raster embedded as a GeoTIFF, as a layer of its own. +pub(crate) fn made(raster: Raster, extra: Option) -> Result { let bytes = rustgis_data::raster::geotiff::write(&raster, 0).map_err(|e| e.to_string())?; - Ok(Made { raster, bytes, extra, ran: None }) + let name = raster.name.clone(); + let layer = rustgis_map::RasterLayer::from_bytes(raster, format!("{name}.tif"), bytes); + Ok(made_layer(name, layer, extra, None)) +} + +/// A layer made (styled, say), to be added as `name`. +pub(crate) fn made_layer(name: String, layer: rustgis_map::RasterLayer, extra: Option, then: Option) -> Made { + Made { name, layer, extra, then, ran: None } } /// Adds a made raster as a layer: the command's answer is the layer, or the layer and what the -/// command says besides (`extra`). +/// command says besides (`extra`), and what it does then. fn add_made(s: &mut Session, m: Made) -> CmdResult { - let Made { raster, bytes, extra, .. } = m; - let name = raster.name.clone(); - let layer = rustgis_map::RasterLayer::from_bytes(raster, format!("{name}.tif"), bytes); + let Made { name, layer, extra, then, .. } = m; let v = super::raster::add_raster(s, &name, layer)?; - Ok(match extra { + let mut v = match extra { Some(Value::Object(mut more)) => { more.insert("layer".into(), v["layer"].clone()); Value::Object(more) } _ => v, - }) + }; + if let Some(then) = then { + then(s, &mut v); + } + Ok(v) } /// A workflow's raster made here, or the one the app made on a thread ([`raster_start`]) and /// handed over as `ran`. /// `name` is the command's, as [`raster_start`] gives it: a raster another command made isn't /// taken. -fn made_or_ran(s: &mut Session, p: &Value, name: &str, work: fn(&Session, &Value) -> Result) -> CmdResult { +pub(crate) fn made_or_ran(s: &mut Session, p: &Value, name: &str, work: fn(&Session, &Value) -> Result) -> CmdResult { if let Some(ran) = p.get("ran").filter(|v| !v.is_null()) { let id = ran.as_u64().ok_or("ran is the number of a run the app worked out")?; let at = s.raster_results.iter().position(|(k, _)| *k == id).ok_or("that run is no longer kept: run it again")?; @@ -177,10 +193,11 @@ fn made_or_ran(s: &mut Session, p: &Value, name: &str, work: fn(&Session, &Value } #[cfg(not(target_arch = "wasm32"))] -/// A workflow that makes a raster (`analysis.viewshed`, `analysis.interpolate`), started to make it -/// on another thread ([`RasterRun::run`]) while the session goes on: its inputs are read now -/// (cheaply: the pixels are shared), and [`raster_land`] hands the raster to the command, which -/// adds it with undo and history when run again with `ran` (#1291 step 3). +/// A command that makes a raster (`analysis.viewshed`, `analysis.interpolate`, `raster.classify`, +/// `dl.classify`), started to make it on another thread ([`RasterRun::run`]) while the session +/// goes on: its inputs are read now (cheaply: the pixels are shared), and [`raster_land`] hands +/// the raster to the command, which adds it with undo and history when run again with `ran` +/// (#1291 step 3). pub struct RasterRun { pub id: u64, /// What the command is called, for a cancelled run's error. @@ -204,13 +221,15 @@ impl RasterRun { } #[cfg(not(target_arch = "wasm32"))] -/// Starts `command` (`analysis.viewshed` or `analysis.interpolate`) with `p` on the active map, as +/// Starts `command` (one of [`RasterRun`]'s) with `p` on the active map, as /// it would run, to make its raster on another thread: its parameters checked and its inputs read /// now. The run belongs to this project: [`raster_land`] refuses it once another project is open. pub fn raster_start(s: &mut Session, command: &str, p: &Value) -> Result { let (name, work): (&'static str, Work) = match command { "analysis.viewshed" => (VIEWSHED, viewshed_work(s, p)?), "analysis.interpolate" => (INTERPOLATE, interpolate_work(s, p)?), + "raster.classify" => (CLASSIFY, super::imagery::classify_work(s, p)?), + "dl.classify" => (DEEP_LEARNING, super::deeplearning::classify_work(s, p)?), _ => return Err(format!("{command} doesn't run in the background")), }; s.next_pending = s.next_pending.wrapping_add(1); @@ -748,6 +767,9 @@ fn suitability(s: &mut Session, p: &Value) -> CmdResult { /// rasters apart. const VIEWSHED: &str = "Viewshed"; const INTERPOLATE: &str = "Interpolate (IDW)"; +/// And the k-means classes and a deep learning classification. +pub(crate) const CLASSIFY: &str = "Classification Wizard"; +pub(crate) const DEEP_LEARNING: &str = "Deep Learning"; fn viewshed(s: &mut Session, p: &Value) -> CmdResult { made_or_ran(s, p, VIEWSHED, viewshed_work) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 24de8b1fe5..b30d3f63c5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -25799,3 +25799,87 @@ fn analysis_workflows_run_on_another_thread() { assert_eq!(raster_land(&mut s, id, made), Ok(false)); assert!(raster_start(&mut s, "analysis.lineOfSight", &p).is_err()); } + +/// k-means classes and a deep learning classification made on another thread, as the Analysis pane +/// runs them (#1291 step 3): the classes handed over (`ran`) are the ones the command makes here, a +/// model trained on the thread is kept in the project only once its classes are added, and a raster +/// handed to the other command isn't taken. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn raster_classes_run_on_another_thread() { + use crate::cmd::workflows::{raster_land, raster_start}; + use rustgis_analysis::control::Control; + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + let training = json!({"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"class": "water"}, "geometry": {"type": "Polygon", "coordinates": [[[10.002, 45.002], [10.015, 45.002], [10.015, 45.035], [10.002, 45.035], [10.002, 45.002]]]}}, + {"type": "Feature", "properties": {"class": "field"}, "geometry": {"type": "Polygon", "coordinates": [[[10.025, 45.002], [10.038, 45.002], [10.038, 45.035], [10.025, 45.035], [10.025, 45.002]]]}}]}); + s.run("layer.addDataText", &json!({"fileName": "training.geojson", "text": training.to_string()})).unwrap(); + let pixels = |s: &Session, v: &Value| -> Vec { + let l = s.map().layer(v["layer"].as_u64().unwrap() as u32).unwrap(); + l.raster.as_ref().unwrap().data.as_ref().unwrap().bands.iter().flat_map(|b| b.data.iter().map(|x| x.to_bits())).collect() + }; + let dl = json!({"layer": "scene", "training": "training", "classField": "class", "bands": [1, 2], "epochs": 20, "model": "landcover"}); + for (command, p) in [("raster.classify", json!({"layer": "scene", "classes": 3})), ("dl.classify", dl)] { + let here = s.run(command, &p).unwrap(); + s.project.dl_models.clear(); + let run = raster_start(&mut s, command, &p).unwrap(); + let id = run.id; + let made = std::thread::spawn(move || run.run(&Control::new())).join().unwrap().unwrap(); + assert_eq!(raster_land(&mut s, id, made), Ok(true)); + // Trained on the thread, kept only when its classes are added. + assert!(s.project.dl_models.is_empty(), "{command}"); + let other = if command == "raster.classify" { "dl.classify" } else { "raster.classify" }; + let mut with = p.clone(); + with["ran"] = json!(id); + assert!(s.run(other, &with).unwrap_err().contains("another command"), "{command}"); + let there = s.run(command, &with).unwrap(); + assert_eq!(pixels(&s, &here), pixels(&s, &there), "{command}"); + let class_style = |v: &Value| { + format!("{:?}", s.map().layer(v["layer"].as_u64().unwrap() as u32).unwrap().raster.as_ref().unwrap().style.renderer) + }; + assert_eq!(class_style(&here), class_style(&there), "{command}"); + if command == "dl.classify" { + assert_eq!((&here["classes"], &here["accuracy"], &here["model"]), (&there["classes"], &there["accuracy"], &there["model"])); + assert_eq!(there["replaced"], false); + assert_eq!(s.project.dl_models.iter().map(|m| m.name.as_str()).collect::>(), ["landcover"]); + } + } + // A training cancelled once under way keeps no model, and says how far it came. + s.project.dl_models.clear(); + let p = json!({"layer": "scene", "training": "training", "classField": "class", "epochs": 500, "hidden": [64, 64]}); + let run = raster_start(&mut s, "dl.classify", &p).unwrap(); + let control = Control::new(); + let ctl = control.clone(); + let worker = std::thread::spawn(move || run.run(&ctl)); + let end = std::time::Instant::now() + std::time::Duration::from_secs(60); + while control.progress().is_none() { + assert!(std::time::Instant::now() < end, "the run never counted its work"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control.cancel(); + let e = worker.join().unwrap().err().unwrap_or_default(); + assert!(e.contains("was cancelled") && !e.contains("before it started"), "{e}"); + assert!(s.project.dl_models.is_empty()); + // Sampling the training polygons is part of the work: a class whose polygons miss the raster + // is said when the run ends, not when it starts. + let lost = json!({"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"class": "water"}, "geometry": {"type": "Polygon", "coordinates": [[[10.002, 45.002], [10.015, 45.002], [10.015, 45.035], [10.002, 45.035], [10.002, 45.002]]]}}, + {"type": "Feature", "properties": {"class": "lost"}, "geometry": {"type": "Polygon", "coordinates": [[[-60.0, -10.0], [-59.0, -10.0], [-59.0, -9.0], [-60.0, -9.0], [-60.0, -10.0]]]}}]}); + s.run("layer.addDataText", &json!({"fileName": "lost.geojson", "text": lost.to_string()})).unwrap(); + let run = raster_start(&mut s, "dl.classify", &json!({"layer": "scene", "training": "lost", "classField": "class"})).unwrap(); + assert!(run.run(&Control::new()).err().is_some_and(|e| e.contains("'lost'"))); + // A model given no name doesn't replace one kept while it trained: it's named when it is kept. + s.project.dl_models.clear(); + let p = json!({"layer": "scene", "training": "training", "classField": "class", "bands": [1, 2], "epochs": 5}); + let run = raster_start(&mut s, "dl.classify", &p).unwrap(); + let id = run.id; + assert_eq!(raster_land(&mut s, id, run.run(&Control::new()).unwrap()), Ok(true)); + let trained = s.run("dl.train", &json!({"layer": "scene", "training": "training", "classField": "class", "epochs": 5})).unwrap(); + assert_eq!(trained["model"], "scene_model"); + let mut with = p.clone(); + with["ran"] = json!(id); + let r = s.run("dl.classify", &with).unwrap(); + assert_eq!((&r["model"], &r["replaced"]), (&json!("scene_model 2"), &json!(false)), "{r}"); + assert_eq!(s.project.dl_models.iter().map(|m| m.name.as_str()).collect::>(), ["scene_model", "scene_model 2"]); +} diff --git a/crates/ui-egui/src/analysispane.rs b/crates/ui-egui/src/analysispane.rs index a2bd1c1015..65bb6d4dd4 100644 --- a/crates/ui-egui/src/analysispane.rs +++ b/crates/ui-egui/src/analysispane.rs @@ -200,6 +200,8 @@ fn background(id: &str) -> Option<(&'static str, &'static str)> { "gp.zonalStats" => Some(("gp.zonalStats", "Zonal Statistics")), "analysis.viewshed" => Some(("analysis.viewshed", "Viewshed")), "analysis.interpolate" => Some(("analysis.interpolate", "Interpolate (IDW)")), + "raster.classify" => Some(("raster.classify", "Classification Wizard")), + "dl.classify" => Some(("dl.classify", "Deep Learning")), _ => None, } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bed678da10..8737bf90ef 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1941,7 +1941,9 @@ impl RustGisApp { let Some(h) = self.session.project.gp_history.last().filter(|h| !h.id.is_empty()) else { return }; finished = json!({"tool": h.id, "params": h.params}); ("gp.run", &finished) - } else if matches!(id, "analysis.viewshed" | "analysis.interpolate") && params.get("ran").is_some() { + } else if matches!(id, "analysis.viewshed" | "analysis.interpolate" | "raster.classify" | "dl.classify") + && params.get("ran").is_some() + { // A raster an Analysis workflow made on a thread (`ran`): the step is the command // itself, which a replay runs again from its parameters. let mut again = params.clone(); @@ -4376,6 +4378,22 @@ mod record_tests { assert_eq!(app.ws.recording.as_ref().map(Vec::len), Some(1)); } + /// A raster a workflow made on a thread is recorded as its command without `ran`, so a replay + /// works it out again (#1291 step 3). + #[test] + fn a_raster_made_on_a_thread_is_recorded_as_its_command() { + let mut app = RustGisApp::with_sample(); + for id in ["analysis.viewshed", "analysis.interpolate", "raster.classify", "dl.classify"] { + app.ws.recording = Some(Vec::new()); + app.record_step(id, &json!({"layer": "Cities", "ran": 7}), None); + let steps = app.ws.recording.as_deref().unwrap_or_default(); + assert_eq!( + steps.iter().map(|s| (s.command.as_str(), s.params.clone())).collect::>(), + [(id, json!({"layer": "Cities"}))] + ); + } + } + /// A background run's `gp.finish` is recorded as the `gp.run` it finished. #[test] fn a_tool_finished_in_the_background_is_recorded_as_its_run() { diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index b2731efa4c..53dda892e4 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -68,9 +68,11 @@ where they now are (and says how many are gone), and a result that would replace layer, or one loaded whole meanwhile, is refused. Renaming the map while it runs is fine; after switching to another map, a result that adds to or changes the map is refused (one made only of tables or a file is still made). Opening another project drops the run. The Analysis pane's -**Zonal Statistics**, **Viewshed** and **Interpolate (IDW)** run in the background the same way, -with their progress and **Cancel** under the pane's inputs, one at a time; so does the map's -right-click **Viewshed … Here**, which opens the pane to show it. Scripts, models, the AI +**Zonal Statistics**, **Viewshed**, **Interpolate (IDW)**, **Classification Wizard** (k-means) and +**Deep Learning** (training and classifying) run in the background the same way, with their +progress and **Cancel** under the pane's inputs, one at a time; so does the map's right-click +**Viewshed … Here**, which opens the pane to show it. A model trained there is kept once its classes +are added: a cancelled run keeps none. Scripts, models, the AI Assistant, the command line and the web app wait for each tool to finish. **Viewshed**, **IDW Interpolation** and **Kriging** refuse a run that would take more than a second