From 3e07006c03bf1634c5b4a06e10e0471f47edf66f Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 00:38:31 -0600 Subject: [PATCH 1/2] fix(engine): train from all of a layer read in place, not its view dl.train, dl.classify given training polygons and raster.classifySupervised took their polygons from the features a layer read in place holds for the current view. They now read the training layer's whole source, as Load All Features does, where their work runs; rows the source can't read are said, and named in an error they may have caused. Their permission effects reach the source they read, as layer.loadAll's do. --- crates/engine/src/cmd/deeplearning.rs | 106 ++++++++++++------- crates/engine/src/cmd/imagery.rs | 23 ++-- crates/engine/src/cmd/layer.rs | 2 +- crates/engine/src/cmd/mod.rs | 1 + crates/engine/src/cmd/whole.rs | 90 ++++++++++++++++ crates/engine/src/policy/effects.rs | 3 + crates/engine/src/tests.rs | 53 ++++++++++ crates/engine/src/tests/permission_policy.rs | 24 +++++ 8 files changed, 256 insertions(+), 46 deletions(-) create mode 100644 crates/engine/src/cmd/whole.rs diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index f28008829f..96c127df9c 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -48,7 +48,8 @@ fn need_model(s: &Session) -> Option<&'static str> { pub fn commands() -> Vec { vec![ C::new("dl.train", "Train Deep Learning Model", "Analyze › Image Classification", |s, p| { - let m = train(s, p)?; + let (m, unread) = train(s, p)?; + unread.say(s); let mut r = json!({"model": m.name, "accuracy": m.accuracy, "classes": m.model.classes, "epochs": m.epochs}); r["replaced"] = json!(store(s, m)); Ok(r) @@ -144,10 +145,13 @@ fn model_index(input: &impl super::InputSource, p: &Value) -> Result Result { +/// Trains a model from the pixels under training polygons, with the rows of the training layer's +/// source that couldn't be read. Where a person at a terminal can stop it with Ctrl-C +/// ([`Session::stoppable`](crate::Session::stoppable)), it takes no work cap, as from the pane. +/// It runs where it is called, reading of a layer read in place, training and all: the app's +/// Deep Learning Tools train through `dl.classify`, which runs on a thread of its own; this is +/// the scripts', command line's and assistant's way to train without classifying. +fn train(s: &Session, p: &Value) -> Result<(DlModel, super::whole::Unread), String> { let t = training(s, p)?; if s.stoppable { rustgis_analysis::run_under("Train Deep Learning Model", &rustgis_analysis::control::Control::stoppable(), || t.fit()) @@ -156,15 +160,17 @@ fn train(s: &Session, p: &Value) -> Result { } } -/// A model to train: the raster, the training polygons and their classes, read of the session, and -/// how to train it ([`Training::fit`], which reads nothing more and runs on any thread: the pixels -/// under the polygons are sampled there, as the network learns). +/// A model to train: the raster and the training layer, taken of the session, and how to train it +/// ([`Training::fit`], which reads nothing more of the session and runs on any thread: the +/// training layer is read whole and the pixels under its polygons sampled there, as the network +/// learns). struct Training { raster: std::sync::Arc, bands: Vec, - classes: Vec, - /// Each polygon's outline, its class and its share of the class's samples. - polygons: Vec<(Vec>, usize, usize)>, + /// The training layer's rows, read whole in [`Training::fit`]. + rows: super::whole::Whole, + /// The class field's index. + field: usize, hidden: Vec, epochs: usize, name: String, @@ -174,44 +180,53 @@ struct Training { } impl Training { - fn fit(self) -> Result { - let Training { raster, bands, classes, polygons, hidden, epochs, name, .. } = self; - let (samples, per_class) = neural::training_samples(&raster, &bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?; + /// Reads the training layer whole and trains the model: the model, with the rows of the + /// training layer's source that couldn't be read (named in an error, as they may be its + /// cause). + fn fit(self) -> Result<(DlModel, super::whole::Unread), String> { + let Training { raster, bands, rows, field, hidden, epochs, name, .. } = self; + let (ds, unread) = rows.read()?; + let model = Self::learn(&ds, field, &raster, &bands, &hidden, epochs, name).map_err(|e| unread.explain(e))?; + Ok((model, unread)) + } + + /// The model learnt from the polygons of `ds` (their classes in field `field`) over `raster`. + fn learn( + ds: &rustgis_data::Dataset, + field: usize, + raster: &rustgis_data::raster::Raster, + bands: &[usize], + hidden: &[usize], + epochs: usize, + name: String, + ) -> Result { + let (classes, polygons) = training_polygons(ds, field)?; + let (samples, per_class) = neural::training_samples(raster, bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?; // Every class needs pixels of its own (its polygons may miss the raster or cover no data). if let Some((c, _)) = classes.iter().zip(&per_class).find(|(_, n)| **n < 5) { return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons")); } - let t = neural::train(&bands, classes, &samples, &hidden, epochs)?; + let t = neural::train(bands, classes, &samples, hidden, epochs)?; Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs }) } } -/// What training reads of the session: the training polygons and their classes, and the model's -/// name. -fn training(s: &Session, p: &Value) -> Result { - let r = raster(s, p)?; - r.require_coordinates()?; - let bands = bands_param(p, &r)?; - let tl = super::imagery::training_name(p)?; - let l = s - .map() - .layer_by_name(tl) - .filter(|l| l.geometry_type() == GeometryType::Polygon) - .ok_or_else(|| format!("no polygon layer '{tl}'"))?; - let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; +/// The classes of a training layer's polygons, in the order they come, and each polygon's outline +/// with its class and its share of the class's samples. A polygon with no class value trains +/// nothing. +fn training_polygons(ds: &rustgis_data::Dataset, fi: usize) -> Result<(Vec, Vec<(Vec>, usize, usize)>), String> { let mut classes: Vec = Vec::new(); let mut parts: Vec<(Vec>, usize, usize)> = Vec::new(); // Polygons per class, so each polygon of a class gets a fair share of its samples. let mut polygons: HashMap = HashMap::new(); - for (row, ft) in l.dataset.features.iter().enumerate() { + for (row, ft) in ds.features.iter().enumerate() { if let Some(Geometry::Polygon(polys)) = &ft.geometry { - *polygons.entry(l.dataset.value(row, fi).to_string()).or_default() += polys.len(); + *polygons.entry(ds.value(row, fi).to_string()).or_default() += polys.len(); } } - for (row, ft) in l.dataset.features.iter().enumerate() { + for (row, ft) in ds.features.iter().enumerate() { let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue }; - // A polygon with no class value trains nothing. - let v = l.dataset.value(row, fi); + let v = ds.value(row, fi); if matches!(v, rustgis_data::Value::Null) || v.to_string().trim().is_empty() { continue; } @@ -227,6 +242,23 @@ fn training(s: &Session, p: &Value) -> Result { }; parts.extend(polys.iter().map(|rings| (rings.clone(), ci, share))); } + Ok((classes, parts)) +} + +/// What training takes of the session: the raster, the training layer (read whole when the model +/// is fitted), and the model's name. +fn training(s: &Session, p: &Value) -> Result { + let r = raster(s, p)?; + r.require_coordinates()?; + let bands = bands_param(p, &r)?; + let tl = super::imagery::training_name(p)?; + let l = s + .map() + .layer_by_name(tl) + .filter(|l| l.geometry_type() == GeometryType::Polygon) + .ok_or_else(|| format!("no polygon layer '{tl}'"))?; + let field = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?; + let rows = super::whole::Whole::of(l); let hidden: Vec = match p.get("hidden").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_u64).map(|u| u as usize).take(neural::MAX_LAYERS).collect(), None => vec![32, 16], @@ -240,7 +272,7 @@ fn training(s: &Session, p: &Value) -> Result { (free_name(s, &base), Some(base)) } }; - Ok(Training { raster: r, bands, classes, polygons: parts, hidden, epochs, name, auto }) + Ok(Training { raster: r, bands, rows, field, hidden, epochs, name, auto }) } /// `base`, or `base 2`, `base 3`… : the first no model of the project has (whatever its case). @@ -283,7 +315,8 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result (m, None), Model::Train(t) => { let auto = t.auto.clone(); - (t.fit()?, Some(auto)) + let (m, unread) = t.fit()?; + (m, Some((auto, unread))) } }; let (class, prob) = neural::classify(&r, &m.model, rasterops::NODATA)?; @@ -299,8 +332,9 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result = trained.map(|auto| { + let then: Option = trained.map(|(auto, unread)| { Box::new(move |s: &mut Session, v: &mut Value| { + unread.say(s); let mut m = m; if let Some(base) = auto { m.name = free_name(s, &base); diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 555e6d3b8e..2c818b12c5 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -1001,24 +1001,29 @@ pub(crate) fn supervised_work(s: &Session, p: &Value) -> Result)> = classes.iter().enumerate().map(|(i, c)| ((i + 1) as f32, c.1.clone())).collect(); let pixels = rasterops::classify_min_distance(&r, &bands, &means)?; let layer = class_layer(rasterops::derive(&r, &out, vec![pixels])?)?; let legend: Vec = classes.iter().enumerate().map(|(i, c)| json!({"value": i + 1, "class": c.0, "pixels": c.2})).collect(); - Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), None)) + let then: Option = Some(Box::new(move |s: &mut Session, _: &mut Value| unread.say(s))); + Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), then)) })) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index b56af5908c..58ac0d105e 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2911,7 +2911,7 @@ fn load_all(s: &mut Session, p: &Value) -> CmdResult { ds.name = l.dataset.name.clone(); let (n, name) = (ds.features.len(), l.name.clone()); if unread > 0 { - s.warn(format!("{name}: {unread} rows couldn't be read and were left out")); + s.warn(super::whole::unread_notice(&name, unread)); } let l = s.map_mut().layer_mut(id).ok_or("no such layer")?; l.dataset = ds; diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 2358b59cb4..3b163efc13 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -56,6 +56,7 @@ pub mod tiles; pub mod time; pub mod view; pub mod webdata; +pub(crate) mod whole; pub mod workflows; pub mod workspace; pub mod zarr; diff --git a/crates/engine/src/cmd/whole.rs b/crates/engine/src/cmd/whole.rs new file mode 100644 index 0000000000..1361117e0f --- /dev/null +++ b/crates/engine/src/cmd/whole.rs @@ -0,0 +1,90 @@ +//! Every feature of a layer, for a command whose result stands for the whole layer (a model +//! trained from it, a classification): a layer read in place holds only the features of the +//! last view, so its source is read whole, as Load All Features reads it, and the rows that +//! couldn't be read are said. As for an in-memory layer, every row is read, whatever the layer's +//! definition query. + +use rustgis_data::Dataset; +use rustgis_map::Layer; + +use crate::Session; + +/// A layer's rows to read whole, taken when the command starts and read where its work runs +/// (on another thread for one the app runs in the background). +pub(crate) struct Whole { + /// The layer's name, for what is said of the read. + name: String, + rows: Rows, +} + +enum Rows { + /// A layer in memory: its features, which are all of them (shared, not copied). + Held(Dataset), + /// A layer read in place: a copy of it, its own read of the source to make. + Source(Box), +} + +/// The rows of a layer's source that a whole read couldn't read and left out. +pub(crate) struct Unread { + layer: String, + rows: usize, +} + +impl Whole { + /// `l`'s rows, to read with [`Whole::read`]. + pub(crate) fn of(l: &Layer) -> Self { + let rows = if l.is_streamed() { + let mut copy = l.clone(); + // Its own read: the live layer keeps the one under way for its view. + if let Some(source) = copy.source.as_mut() { + source.detach_read(); + } + Rows::Source(Box::new(copy)) + } else { + Rows::Held(l.dataset.clone()) + }; + Whole { name: l.name.clone(), rows } + } + + /// Every feature, with the fields the layer lists (so a field's index in the layer is its + /// index here), and the rows of its source that couldn't be read. A source read a chunk at a + /// time looks at Cancel between chunks; one read at once (a GeoJSON file, a database table) + /// is a single step that doesn't, as for Load All Features. + pub(crate) fn read(self) -> Result<(Dataset, Unread), String> { + let Whole { name, rows } = self; + let (ds, rows) = match rows { + Rows::Held(ds) => (ds, 0), + Rows::Source(l) => l.load_all_counted().map_err(|e| format!("{name}: {e}"))?, + }; + Ok((ds, Unread { layer: name, rows })) + } +} + +impl Unread { + /// Says the rows left out, as Load All Features says them (nothing when none were). + pub(crate) fn say(&self, s: &mut Session) { + if self.rows > 0 { + s.warn(unread_notice(&self.layer, self.rows)); + } + } + + /// `error`, with the rows left out when there were any: they may be why it failed. + pub(crate) fn explain(&self, error: String) -> String { + if self.rows == 0 { error } else { format!("{error} ({})", unread_notice(&self.layer, self.rows)) } + } +} + +/// The notice for `unread` rows of `layer`'s source that couldn't be read. +pub(crate) fn unread_notice(layer: &str, unread: usize) -> String { + format!("{layer}: {unread} rows couldn't be read and were left out") +} + +/// The layers read in place that `command` with `params` reads whole from their sources (for its +/// permission effects, which then reach those sources as Load All Features does). +pub(crate) fn layers_read_whole<'a>(s: &super::InputContext<'a>, command: &str, params: &serde_json::Value) -> Vec<&'a Layer> { + let names: Vec<&str> = match command { + "dl.train" | "dl.classify" | "raster.classifySupervised" => super::ps(params, "training").into_iter().collect(), + _ => Vec::new(), + }; + names.into_iter().filter_map(|n| s.map().layer_by_name(n)).filter(|l| l.is_streamed()).collect() +} diff --git a/crates/engine/src/policy/effects.rs b/crates/engine/src/policy/effects.rs index c4b5a17be4..819b790204 100644 --- a/crates/engine/src/policy/effects.rs +++ b/crates/engine/src/policy/effects.rs @@ -116,6 +116,9 @@ pub(super) fn command(s: &crate::cmd::InputContext<'_>, spec: &CommandSpec, para { effects = effects.union(source_reach(source)); } + for source in crate::cmd::whole::layers_read_whole(s, spec.id, params).into_iter().filter_map(|l| l.source.as_ref()) { + effects = effects.union(source_reach(source)); + } if crate::cmd::time::follows_sources(spec.id) { for (_, _, source) in crate::cmd::time::following_sources(s) { effects = effects.with(Effect::Files); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f13184a141..ae3cc2659b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -11696,6 +11696,59 @@ fn two_class_b64() -> String { base64::engine::general_purpose::STANDARD.encode(geotiff::write(&r, 0).unwrap()) } +/// Training read in place a view at a time: the view holds none of the training polygons, yet +/// a model and a supervised classification learn from all of them, read from the source, and the +/// rows the source can't give are said once. +#[test] +fn training_layers_read_in_place_train_from_their_whole_source() { + let dir = std::env::temp_dir().join(format!("rustgis-whole-training-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let square = |class: &str, x: f64| json!({"type": "Feature", "properties": {"class": class}, "geometry": {"type": "Polygon", "coordinates": [[[x, 45.002], [x + 0.013, 45.002], [x + 0.013, 45.035], [x, 45.035], [x, 45.002]]]}}); + let bad = json!({"type": "Feature", "properties": {"class": "field"}, "geometry": {"type": "Polygon", "coordinates": "x"}}); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains("couldn't be read and were left out")).count(); + for (file, extra) in [("clean.geojson", None), ("bad.geojson", Some(bad.clone()))] { + let mut features = vec![square("water", 10.002), square("field", 10.025)]; + features.extend(extra.clone()); + let path = dir.join(file); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": features}).to_string()).unwrap(); + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "training"})).unwrap()["layer"] + .as_u64() + .unwrap() as u32; + // The view is far from the polygons: the layer holds none of them. + s.map_mut().sync_sources(&rustgis_geom::Extent::new(-61.0, -11.0, -59.0, -9.0), 1000.0, true); + let l = s.map().layer(id).unwrap(); + assert!(l.is_streamed() && l.dataset.features.is_empty(), "{file}: a window without the polygons"); + let said = usize::from(extra.is_some()); + let train = json!({"layer": "scene", "training": "training", "classField": "class", "bands": [1, 2], "epochs": 5}); + let r = s.run("dl.train", &train).unwrap_or_else(|e| panic!("{file}: {e}")); + assert_eq!(r["classes"], json!(["water", "field"]), "{file}: {r}"); + assert_eq!(unread(&s), said, "{file}: dl.train"); + let r = s.run("dl.classify", &train).unwrap_or_else(|e| panic!("{file}: {e}")); + assert_eq!(r["classes"].as_array().unwrap().len(), 2, "{file}: {r}"); + assert_eq!(unread(&s), 2 * said, "{file}: dl.classify"); + let r = s.run("raster.classifySupervised", &json!({"layer": "scene", "training": "training", "classField": "class"})).unwrap(); + assert_eq!(r["classes"].as_array().unwrap().len(), 2, "{file}: {r}"); + assert_eq!(unread(&s), 3 * said, "{file}: raster.classifySupervised"); + // The live layer still reads its view: nothing was loaded into it. + let l = s.map().layer(id).unwrap(); + assert!(l.is_streamed() && l.dataset.features.is_empty(), "{file}"); + } + // A class whose only polygon can't be read: the error says the rows left out, its likely cause. + let path = dir.join("lost-class.geojson"); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": [square("water", 10.002), bad]}).to_string()).unwrap(); + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "scene.tif", "data": two_class_b64()})).unwrap(); + s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "training"})).unwrap(); + for command in ["dl.train", "raster.classifySupervised"] { + let err = s.run(command, &json!({"layer": "scene", "training": "training", "classField": "class", "epochs": 5})).unwrap_err(); + assert!(err.contains("training: 1 rows couldn't be read and were left out"), "{command}: {err}"); + } + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn deep_learning_hosted_imagery_and_sharing() { let dir = std::env::temp_dir().join(format!("rustgis-share-{}", std::process::id())); diff --git a/crates/engine/src/tests/permission_policy.rs b/crates/engine/src/tests/permission_policy.rs index 8a0b1e3c24..0b70be50d8 100644 --- a/crates/engine/src/tests/permission_policy.rs +++ b/crates/engine/src/tests/permission_policy.rs @@ -1688,6 +1688,30 @@ fn permission_policy_renamed_streamed_inputs_keep_their_reach() { assert!(outcomes.iter().all(|(_, asks, shown)| *asks && *shown), "{outcomes:?}"); } +/// Training read whole from a layer read in place reaches its source, as Load All does; the +/// same layer in memory reaches nothing. +#[test] +fn permission_policy_training_read_whole_reaches_its_source() { + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName":"scene.tif","data":two_class_b64()})).unwrap(); + let (scope, _) = assistant(3602); + let calls = [ + Call::command("dl.train", json!({"layer":"scene","training":"Countries","classField":"name"})), + Call::command("dl.classify", json!({"layer":"scene","training":"Countries","classField":"name"})), + Call::command("raster.classifySupervised", json!({"layer":"scene","training":"Countries","classField":"name"})), + ]; + let reaches = |s: &Session, call: &Call| matches!(s.review(&scope, call), Outcome::NeedsApproval(_)); + assert!(calls.iter().all(|call| !reaches(&s, call)), "in memory: nothing to approve"); + let id = s.map().layer_by_name("Countries").unwrap().id; + let mut source = rustgis_map::source::SourceLayer::default(); + source.path = Some("https://example.invalid/countries.fgb".into()); + s.map_mut().layer_mut(id).unwrap().source = Some(source); + for call in &calls { + let approved = pending(&s, &scope, call); + assert!(approved.review().why.contains("internet"), "{:?}", approved.review().why); + } +} + /// In-memory range reads need Network, while local paths and external drivers retain Files. #[test] fn permission_policy_remote_range_reads_need_only_network() { From 77b9fef57393ad96768c31790c1784c7dc7cd362 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 03:51:24 -0600 Subject: [PATCH 2/2] fix(engine): read a training layer's source without copying its view, and say one row in the singular Whole::of copies only the source and the dataset's header, not the view's features and caches. The rows-left-out notice (shared with Load All) says one row in the singular. dl.train's doc says it holds the caller while it reads. The training test removes its folder however it ends and checks the notice the command makes. --- crates/engine/src/cmd/deeplearning.rs | 4 +++- crates/engine/src/cmd/whole.rs | 21 +++++++++++++++------ crates/engine/src/tests.rs | 23 ++++++++++++++++------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index 96c127df9c..625593603f 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -150,7 +150,9 @@ fn model_index(input: &impl super::InputSource, p: &Value) -> Result Result<(DlModel, super::whole::Unread), String> { let t = training(s, p)?; if s.stoppable { diff --git a/crates/engine/src/cmd/whole.rs b/crates/engine/src/cmd/whole.rs index 1361117e0f..82ab5cb48b 100644 --- a/crates/engine/src/cmd/whole.rs +++ b/crates/engine/src/cmd/whole.rs @@ -33,12 +33,18 @@ pub(crate) struct Unread { impl Whole { /// `l`'s rows, to read with [`Whole::read`]. pub(crate) fn of(l: &Layer) -> Self { - let rows = if l.is_streamed() { - let mut copy = l.clone(); + let rows = if let Some(source) = &l.source { + // What a whole read needs of the layer (its source, and the name, fields, type and + // marks the read keeps), not the view's features and caches. + let ds = &l.dataset; + let mut header = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone()); + header.zm_dropped = ds.zm_dropped; + header.source = ds.source.clone(); + let mut copy = Layer::new(l.id, header); + let mut source = source.clone(); // Its own read: the live layer keeps the one under way for its view. - if let Some(source) = copy.source.as_mut() { - source.detach_read(); - } + source.detach_read(); + copy.source = Some(source); Rows::Source(Box::new(copy)) } else { Rows::Held(l.dataset.clone()) @@ -76,7 +82,10 @@ impl Unread { /// The notice for `unread` rows of `layer`'s source that couldn't be read. pub(crate) fn unread_notice(layer: &str, unread: usize) -> String { - format!("{layer}: {unread} rows couldn't be read and were left out") + match unread { + 1 => format!("{layer}: 1 row couldn't be read and was left out"), + n => format!("{layer}: {n} rows couldn't be read and were left out"), + } } /// The layers read in place that `command` with `params` reads whole from their sources (for its diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ae3cc2659b..589ea57e6b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -11701,12 +11701,22 @@ fn two_class_b64() -> String { /// rows the source can't give are said once. #[test] fn training_layers_read_in_place_train_from_their_whole_source() { - let dir = std::env::temp_dir().join(format!("rustgis-whole-training-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let dir = Dir(std::env::temp_dir().join(format!("rustgis-whole-training-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(&dir.0); + std::fs::create_dir_all(&dir.0).unwrap(); + let dir = &dir.0; let square = |class: &str, x: f64| json!({"type": "Feature", "properties": {"class": class}, "geometry": {"type": "Polygon", "coordinates": [[[x, 45.002], [x + 0.013, 45.002], [x + 0.013, 45.035], [x, 45.035], [x, 45.002]]]}}); let bad = json!({"type": "Feature", "properties": {"class": "field"}, "geometry": {"type": "Polygon", "coordinates": "x"}}); - let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains("couldn't be read and were left out")).count(); + // The notice for the one row of the training layer its source can't give. + let one_unread = crate::cmd::whole::unread_notice("training", 1); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text == one_unread).count(); for (file, extra) in [("clean.geojson", None), ("bad.geojson", Some(bad.clone()))] { let mut features = vec![square("water", 10.002), square("field", 10.025)]; features.extend(extra.clone()); @@ -11744,9 +11754,8 @@ fn training_layers_read_in_place_train_from_their_whole_source() { s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "training"})).unwrap(); for command in ["dl.train", "raster.classifySupervised"] { let err = s.run(command, &json!({"layer": "scene", "training": "training", "classField": "class", "epochs": 5})).unwrap_err(); - assert!(err.contains("training: 1 rows couldn't be read and were left out"), "{command}: {err}"); + assert!(err.contains(&one_unread), "{command}: {err}"); } - let _ = std::fs::remove_dir_all(&dir); } #[test] @@ -35639,7 +35648,7 @@ fn reads_made_ahead_are_taken_by_the_command() { assert_eq!(crate::cmd::reads::read_land(&mut s, read).unwrap()["features"], 2); assert!(s.read_ahead.is_none()); assert!(!s.map().layer(id as u32).unwrap().is_streamed()); - let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains("couldn't be read and were left out")).count(); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text.contains(" couldn't be read and w")).count(); assert_eq!(unread(&s), 0, "every row read: nothing said"); // A row that can't be read is left out and said, once, read ahead or here.