diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index f28008829f..625593603f 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -48,7 +48,8 @@ fn need_model(s: &Session) -> Option<&'static str> { pub fn commands() -> Vec { vec![ C::new("dl.train", "Train Deep Learning Model", "Analyze › Image Classification", |s, p| { - let m = train(s, p)?; + let (m, unread) = train(s, p)?; + unread.say(s); let mut r = json!({"model": m.name, "accuracy": m.accuracy, "classes": m.model.classes, "epochs": m.epochs}); r["replaced"] = json!(store(s, m)); Ok(r) @@ -144,10 +145,15 @@ fn model_index(input: &impl super::InputSource, p: &Value) -> Result Result { +/// Trains a model from the pixels under training polygons, with the rows of the training layer's +/// source that couldn't be read. Where a person at a terminal can stop it with Ctrl-C +/// ([`Session::stoppable`](crate::Session::stoppable)), it takes no work cap, as from the pane. +/// It runs where it is called, reading of a layer read in place, training and all: the app's +/// Deep Learning Tools train through `dl.classify`, which runs on a thread of its own; this is +/// the scripts', command line's and assistant's way to train without classifying. Called on the +/// app's thread (by the assistant), it holds the window while it reads: a training layer read +/// from a database or a web address is read whole first, for as long as that takes. +fn train(s: &Session, p: &Value) -> Result<(DlModel, super::whole::Unread), String> { let t = training(s, p)?; if s.stoppable { rustgis_analysis::run_under("Train Deep Learning Model", &rustgis_analysis::control::Control::stoppable(), || t.fit()) @@ -156,15 +162,17 @@ fn train(s: &Session, p: &Value) -> Result { } } -/// A model to train: the raster, the training polygons and their classes, read of the session, and -/// how to train it ([`Training::fit`], which reads nothing more and runs on any thread: the pixels -/// under the polygons are sampled there, as the network learns). +/// A model to train: the raster and the training layer, taken of the session, and how to train it +/// ([`Training::fit`], which reads nothing more of the session and runs on any thread: the +/// training layer is read whole and the pixels under its polygons sampled there, as the network +/// learns). struct Training { raster: std::sync::Arc, bands: Vec, - classes: Vec, - /// Each polygon's outline, its class and its share of the class's samples. - polygons: Vec<(Vec>, usize, usize)>, + /// The training layer's rows, read whole in [`Training::fit`]. + rows: super::whole::Whole, + /// The class field's index. + field: usize, hidden: Vec, epochs: usize, name: String, @@ -174,44 +182,53 @@ struct Training { } impl Training { - fn fit(self) -> Result { - let Training { raster, bands, classes, polygons, hidden, epochs, name, .. } = self; - let (samples, per_class) = neural::training_samples(&raster, &bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?; + /// Reads the training layer whole and trains the model: the model, with the rows of the + /// training layer's source that couldn't be read (named in an error, as they may be its + /// cause). + fn fit(self) -> Result<(DlModel, super::whole::Unread), String> { + let Training { raster, bands, rows, field, hidden, epochs, name, .. } = self; + let (ds, unread) = rows.read()?; + let model = Self::learn(&ds, field, &raster, &bands, &hidden, epochs, name).map_err(|e| unread.explain(e))?; + Ok((model, unread)) + } + + /// The model learnt from the polygons of `ds` (their classes in field `field`) over `raster`. + fn learn( + ds: &rustgis_data::Dataset, + field: usize, + raster: &rustgis_data::raster::Raster, + bands: &[usize], + hidden: &[usize], + epochs: usize, + name: String, + ) -> Result { + let (classes, polygons) = training_polygons(ds, field)?; + let (samples, per_class) = neural::training_samples(raster, bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?; // Every class needs pixels of its own (its polygons may miss the raster or cover no data). if let Some((c, _)) = classes.iter().zip(&per_class).find(|(_, n)| **n < 5) { return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons")); } - let t = neural::train(&bands, classes, &samples, &hidden, epochs)?; + let t = neural::train(bands, classes, &samples, hidden, epochs)?; Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs }) } } -/// What training reads of the session: the training polygons and their classes, and the model's -/// name. -fn training(s: &Session, p: &Value) -> Result { - let r = raster(s, p)?; - r.require_coordinates()?; - let bands = bands_param(p, &r)?; - let tl = super::imagery::training_name(p)?; - let l = s - .map() - .layer_by_name(tl) - .filter(|l| l.geometry_type() == GeometryType::Polygon) - .ok_or_else(|| format!("no polygon layer '{tl}'"))?; - let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; +/// The classes of a training layer's polygons, in the order they come, and each polygon's outline +/// with its class and its share of the class's samples. A polygon with no class value trains +/// nothing. +fn training_polygons(ds: &rustgis_data::Dataset, fi: usize) -> Result<(Vec, Vec<(Vec>, usize, usize)>), String> { let mut classes: Vec = Vec::new(); let mut parts: Vec<(Vec>, usize, usize)> = Vec::new(); // Polygons per class, so each polygon of a class gets a fair share of its samples. let mut polygons: HashMap = HashMap::new(); - for (row, ft) in l.dataset.features.iter().enumerate() { + for (row, ft) in ds.features.iter().enumerate() { if let Some(Geometry::Polygon(polys)) = &ft.geometry { - *polygons.entry(l.dataset.value(row, fi).to_string()).or_default() += polys.len(); + *polygons.entry(ds.value(row, fi).to_string()).or_default() += polys.len(); } } - for (row, ft) in l.dataset.features.iter().enumerate() { + for (row, ft) in ds.features.iter().enumerate() { let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue }; - // A polygon with no class value trains nothing. - let v = l.dataset.value(row, fi); + let v = ds.value(row, fi); if matches!(v, rustgis_data::Value::Null) || v.to_string().trim().is_empty() { continue; } @@ -227,6 +244,23 @@ fn training(s: &Session, p: &Value) -> Result { }; parts.extend(polys.iter().map(|rings| (rings.clone(), ci, share))); } + Ok((classes, parts)) +} + +/// What training takes of the session: the raster, the training layer (read whole when the model +/// is fitted), and the model's name. +fn training(s: &Session, p: &Value) -> Result { + let r = raster(s, p)?; + r.require_coordinates()?; + let bands = bands_param(p, &r)?; + let tl = super::imagery::training_name(p)?; + let l = s + .map() + .layer_by_name(tl) + .filter(|l| l.geometry_type() == GeometryType::Polygon) + .ok_or_else(|| format!("no polygon layer '{tl}'"))?; + let field = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; + let rows = super::whole::Whole::of(l); let hidden: Vec = match p.get("hidden").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_u64).map(|u| u as usize).take(neural::MAX_LAYERS).collect(), None => vec![32, 16], @@ -240,7 +274,7 @@ fn training(s: &Session, p: &Value) -> Result { (free_name(s, &base), Some(base)) } }; - Ok(Training { raster: r, bands, classes, polygons: parts, hidden, epochs, name, auto }) + Ok(Training { raster: r, bands, rows, field, hidden, epochs, name, auto }) } /// `base`, or `base 2`, `base 3`… : the first no model of the project has (whatever its case). @@ -283,7 +317,8 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result (m, None), Model::Train(t) => { let auto = t.auto.clone(); - (t.fit()?, Some(auto)) + let (m, unread) = t.fit()?; + (m, Some((auto, unread))) } }; let (class, prob) = neural::classify(&r, &m.model, rasterops::NODATA)?; @@ -299,8 +334,9 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result = trained.map(|auto| { + let then: Option = trained.map(|(auto, unread)| { Box::new(move |s: &mut Session, v: &mut Value| { + unread.say(s); let mut m = m; if let Some(base) = auto { m.name = free_name(s, &base); diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 555e6d3b8e..2c818b12c5 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -1001,24 +1001,29 @@ pub(crate) fn supervised_work(s: &Session, p: &Value) -> Result)> = classes.iter().enumerate().map(|(i, c)| ((i + 1) as f32, c.1.clone())).collect(); let pixels = rasterops::classify_min_distance(&r, &bands, &means)?; let layer = class_layer(rasterops::derive(&r, &out, vec![pixels])?)?; let legend: Vec = classes.iter().enumerate().map(|(i, c)| json!({"value": i + 1, "class": c.0, "pixels": c.2})).collect(); - Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), None)) + let then: Option = Some(Box::new(move |s: &mut Session, _: &mut Value| unread.say(s))); + Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), then)) })) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index b56af5908c..58ac0d105e 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2911,7 +2911,7 @@ fn load_all(s: &mut Session, p: &Value) -> CmdResult { ds.name = l.dataset.name.clone(); let (n, name) = (ds.features.len(), l.name.clone()); if unread > 0 { - s.warn(format!("{name}: {unread} rows couldn't be read and were left out")); + s.warn(super::whole::unread_notice(&name, unread)); } let l = s.map_mut().layer_mut(id).ok_or("no such layer")?; l.dataset = ds; diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 2358b59cb4..3b163efc13 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -56,6 +56,7 @@ pub mod tiles; pub mod time; pub mod view; pub mod webdata; +pub(crate) mod whole; pub mod workflows; pub mod workspace; pub mod zarr; diff --git a/crates/engine/src/cmd/whole.rs b/crates/engine/src/cmd/whole.rs new file mode 100644 index 0000000000..82ab5cb48b --- /dev/null +++ b/crates/engine/src/cmd/whole.rs @@ -0,0 +1,99 @@ +//! Every feature of a layer, for a command whose result stands for the whole layer (a model +//! trained from it, a classification): a layer read in place holds only the features of the +//! last view, so its source is read whole, as Load All Features reads it, and the rows that +//! couldn't be read are said. As for an in-memory layer, every row is read, whatever the layer's +//! definition query. + +use rustgis_data::Dataset; +use rustgis_map::Layer; + +use crate::Session; + +/// A layer's rows to read whole, taken when the command starts and read where its work runs +/// (on another thread for one the app runs in the background). +pub(crate) struct Whole { + /// The layer's name, for what is said of the read. + name: String, + rows: Rows, +} + +enum Rows { + /// A layer in memory: its features, which are all of them (shared, not copied). + Held(Dataset), + /// A layer read in place: a copy of it, its own read of the source to make. + Source(Box), +} + +/// The rows of a layer's source that a whole read couldn't read and left out. +pub(crate) struct Unread { + layer: String, + rows: usize, +} + +impl Whole { + /// `l`'s rows, to read with [`Whole::read`]. + pub(crate) fn of(l: &Layer) -> Self { + let rows = if let Some(source) = &l.source { + // What a whole read needs of the layer (its source, and the name, fields, type and + // marks the read keeps), not the view's features and caches. + let ds = &l.dataset; + let mut header = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone()); + header.zm_dropped = ds.zm_dropped; + header.source = ds.source.clone(); + let mut copy = Layer::new(l.id, header); + let mut source = source.clone(); + // Its own read: the live layer keeps the one under way for its view. + source.detach_read(); + copy.source = Some(source); + Rows::Source(Box::new(copy)) + } else { + Rows::Held(l.dataset.clone()) + }; + Whole { name: l.name.clone(), rows } + } + + /// Every feature, with the fields the layer lists (so a field's index in the layer is its + /// index here), and the rows of its source that couldn't be read. A source read a chunk at a + /// time looks at Cancel between chunks; one read at once (a GeoJSON file, a database table) + /// is a single step that doesn't, as for Load All Features. + pub(crate) fn read(self) -> Result<(Dataset, Unread), String> { + let Whole { name, rows } = self; + let (ds, rows) = match rows { + Rows::Held(ds) => (ds, 0), + Rows::Source(l) => l.load_all_counted().map_err(|e| format!("{name}: {e}"))?, + }; + Ok((ds, Unread { layer: name, rows })) + } +} + +impl Unread { + /// Says the rows left out, as Load All Features says them (nothing when none were). + pub(crate) fn say(&self, s: &mut Session) { + if self.rows > 0 { + s.warn(unread_notice(&self.layer, self.rows)); + } + } + + /// `error`, with the rows left out when there were any: they may be why it failed. + pub(crate) fn explain(&self, error: String) -> String { + if self.rows == 0 { error } else { format!("{error} ({})", unread_notice(&self.layer, self.rows)) } + } +} + +/// The notice for `unread` rows of `layer`'s source that couldn't be read. +pub(crate) fn unread_notice(layer: &str, unread: usize) -> String { + match unread { + 1 => format!("{layer}: 1 row couldn't be read and was left out"), + n => format!("{layer}: {n} rows couldn't be read and were left out"), + } +} + +/// The layers read in place that `command` with `params` reads whole from their sources (for its +/// permission effects, which then reach those sources as Load All Features does). +pub(crate) fn layers_read_whole<'a>(s: &super::InputContext<'a>, command: &str, params: &serde_json::Value) -> Vec<&'a Layer> { + let names: Vec<&str> = match command { + "dl.train" | "dl.classify" | "raster.classifySupervised" => super::ps(params, "training").into_iter().collect(), + _ => Vec::new(), + }; + names.into_iter().filter_map(|n| s.map().layer_by_name(n)).filter(|l| l.is_streamed()).collect() +} diff --git a/crates/engine/src/policy/effects.rs b/crates/engine/src/policy/effects.rs index c4b5a17be4..819b790204 100644 --- a/crates/engine/src/policy/effects.rs +++ b/crates/engine/src/policy/effects.rs @@ -116,6 +116,9 @@ pub(super) fn command(s: &crate::cmd::InputContext<'_>, spec: &CommandSpec, para { effects = effects.union(source_reach(source)); } + for source in crate::cmd::whole::layers_read_whole(s, spec.id, params).into_iter().filter_map(|l| l.source.as_ref()) { + effects = effects.union(source_reach(source)); + } if crate::cmd::time::follows_sources(spec.id) { for (_, _, source) in crate::cmd::time::following_sources(s) { effects = effects.with(Effect::Files); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f13184a141..589ea57e6b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -11696,6 +11696,68 @@ fn two_class_b64() -> String { base64::engine::general_purpose::STANDARD.encode(geotiff::write(&r, 0).unwrap()) } +/// Training read in place a view at a time: the view holds none of the training polygons, yet +/// a model and a supervised classification learn from all of them, read from the source, and the +/// rows the source can't give are said once. +#[test] +fn training_layers_read_in_place_train_from_their_whole_source() { + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let dir = Dir(std::env::temp_dir().join(format!("rustgis-whole-training-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(&dir.0); + std::fs::create_dir_all(&dir.0).unwrap(); + let dir = &dir.0; + let square = |class: &str, x: f64| json!({"type": "Feature", "properties": {"class": class}, "geometry": {"type": "Polygon", "coordinates": [[[x, 45.002], [x + 0.013, 45.002], [x + 0.013, 45.035], [x, 45.035], [x, 45.002]]]}}); + let bad = json!({"type": "Feature", "properties": {"class": "field"}, "geometry": {"type": "Polygon", "coordinates": "x"}}); + // The notice for the one row of the training layer its source can't give. + let one_unread = crate::cmd::whole::unread_notice("training", 1); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text == one_unread).count(); + for (file, extra) in [("clean.geojson", None), ("bad.geojson", Some(bad.clone()))] { + let mut features = vec![square("water", 10.002), square("field", 10.025)]; + features.extend(extra.clone()); + let path = dir.join(file); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": features}).to_string()).unwrap(); + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "training"})).unwrap()["layer"] + .as_u64() + .unwrap() as u32; + // The view is far from the polygons: the layer holds none of them. + s.map_mut().sync_sources(&rustgis_geom::Extent::new(-61.0, -11.0, -59.0, -9.0), 1000.0, true); + let l = s.map().layer(id).unwrap(); + assert!(l.is_streamed() && l.dataset.features.is_empty(), "{file}: a window without the polygons"); + let said = usize::from(extra.is_some()); + let train = json!({"layer": "scene", "training": "training", "classField": "class", "bands": [1, 2], "epochs": 5}); + let r = s.run("dl.train", &train).unwrap_or_else(|e| panic!("{file}: {e}")); + assert_eq!(r["classes"], json!(["water", "field"]), "{file}: {r}"); + assert_eq!(unread(&s), said, "{file}: dl.train"); + let r = s.run("dl.classify", &train).unwrap_or_else(|e| panic!("{file}: {e}")); + assert_eq!(r["classes"].as_array().unwrap().len(), 2, "{file}: {r}"); + assert_eq!(unread(&s), 2 * said, "{file}: dl.classify"); + let r = s.run("raster.classifySupervised", &json!({"layer": "scene", "training": "training", "classField": "class"})).unwrap(); + assert_eq!(r["classes"].as_array().unwrap().len(), 2, "{file}: {r}"); + assert_eq!(unread(&s), 3 * said, "{file}: raster.classifySupervised"); + // The live layer still reads its view: nothing was loaded into it. + let l = s.map().layer(id).unwrap(); + assert!(l.is_streamed() && l.dataset.features.is_empty(), "{file}"); + } + // A class whose only polygon can't be read: the error says the rows left out, its likely cause. + let path = dir.join("lost-class.geojson"); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": [square("water", 10.002), bad]}).to_string()).unwrap(); + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "training"})).unwrap(); + for command in ["dl.train", "raster.classifySupervised"] { + let err = s.run(command, &json!({"layer": "scene", "training": "training", "classField": "class", "epochs": 5})).unwrap_err(); + assert!(err.contains(&one_unread), "{command}: {err}"); + } +} + #[test] fn deep_learning_hosted_imagery_and_sharing() { let dir = std::env::temp_dir().join(format!("rustgis-share-{}", std::process::id())); @@ -35586,7 +35648,7 @@ fn reads_made_ahead_are_taken_by_the_command() { assert_eq!(crate::cmd::reads::read_land(&mut s, read).unwrap()["features"], 2); assert!(s.read_ahead.is_none()); assert!(!s.map().layer(id as u32).unwrap().is_streamed()); - let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains("couldn't be read and were left out")).count(); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains(" couldn't be read and w")).count(); assert_eq!(unread(&s), 0, "every row read: nothing said"); // A row that can't be read is left out and said, once, read ahead or here. diff --git a/crates/engine/src/tests/permission_policy.rs b/crates/engine/src/tests/permission_policy.rs index 8a0b1e3c24..0b70be50d8 100644 --- a/crates/engine/src/tests/permission_policy.rs +++ b/crates/engine/src/tests/permission_policy.rs @@ -1688,6 +1688,30 @@ fn permission_policy_renamed_streamed_inputs_keep_their_reach() { assert!(outcomes.iter().all(|(_, asks, shown)| *asks && *shown), "{outcomes:?}"); } +/// Training read whole from a layer read in place reaches its source, as Load All does; the +/// same layer in memory reaches nothing. +#[test] +fn permission_policy_training_read_whole_reaches_its_source() { + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName":"scene.tif","data":two_class_b64()})).unwrap(); + let (scope, _) = assistant(3602); + let calls = [ + Call::command("dl.train", json!({"layer":"scene","training":"Countries","classField":"name"})), + Call::command("dl.classify", json!({"layer":"scene","training":"Countries","classField":"name"})), + Call::command("raster.classifySupervised", json!({"layer":"scene","training":"Countries","classField":"name"})), + ]; + let reaches = |s: &Session, call: &Call| matches!(s.review(&scope, call), Outcome::NeedsApproval(_)); + assert!(calls.iter().all(|call| !reaches(&s, call)), "in memory: nothing to approve"); + let id = s.map().layer_by_name("Countries").unwrap().id; + let mut source = rustgis_map::source::SourceLayer::default(); + source.path = Some("https://example.invalid/countries.fgb".into()); + s.map_mut().layer_mut(id).unwrap().source = Some(source); + for call in &calls { + let approved = pending(&s, &scope, call); + assert!(approved.review().why.contains("internet"), "{:?}", approved.review().why); + } +} + /// In-memory range reads need Network, while local paths and external drivers retain Files. #[test] fn permission_policy_remote_range_reads_need_only_network() {