Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 72 additions & 36 deletions crates/engine/src/cmd/deeplearning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ fn need_model(s: &Session) -> Option<&'static str> {
pub fn commands() -> Vec<C> {
vec![
C::new("dl.train", "Train Deep Learning Model", "Analyze › Image Classification", |s, p| {
let m = train(s, p)?;
let (m, unread) = train(s, p)?;
unread.say(s);
let mut r = json!({"model": m.name, "accuracy": m.accuracy, "classes": m.model.classes, "epochs": m.epochs});
r["replaced"] = json!(store(s, m));
Ok(r)
Expand Down Expand Up @@ -144,10 +145,15 @@ fn model_index(input: &impl super::InputSource, p: &Value) -> Result<usize, Stri
}
}

/// Trains a model from the pixels under training polygons. Where a person at a terminal can stop
/// it with Ctrl-C ([`Session::stoppable`](crate::Session::stoppable)), it takes no work cap, as
/// from the pane.
fn train(s: &Session, p: &Value) -> Result<DlModel, String> {
/// Trains a model from the pixels under training polygons, with the rows of the training layer's
/// source that couldn't be read. Where a person at a terminal can stop it with Ctrl-C
/// ([`Session::stoppable`](crate::Session::stoppable)), it takes no work cap, as from the pane.
/// It runs where it is called, reading of a layer read in place, training and all: the app's
/// Deep Learning Tools train through `dl.classify`, which runs on a thread of its own; this is
/// the scripts', command line's and assistant's way to train without classifying. Called on the
/// app's thread (by the assistant), it holds the window while it reads: a training layer read
/// from a database or a web address is read whole first, for as long as that takes.
fn train(s: &Session, p: &Value) -> Result<(DlModel, super::whole::Unread), String> {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Medium (confidence: medium): dl.train isn't in workflows::IN_BACKGROUND, so when it's run from the desktop UI it runs on the UI thread. This PR adds a whole-source read to that path: Whole::read → load_all_counted → read_all → materialize, which for a web-address source fetches the whole file, and for a database table reads it all. Before, the UI thread only trained. The doc comment explains the thread choice, but not that the new read can hold the window for as long as the fetch takes. Either route dl.train through a background path, or state in the doc comment that reading a remote or database source blocks the window.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in 77b9fef. The app never runs dl.train from a click (its Deep Learning dialog trains through dl.classify, which runs on a thread of its own). It's run by scripts, the command line and the assistant. Its doc comment now says that, called on the app's thread by the assistant, it holds the window while a training layer read from a database or web address is read whole.

let t = training(s, p)?;
if s.stoppable {
rustgis_analysis::run_under("Train Deep Learning Model", &rustgis_analysis::control::Control::stoppable(), || t.fit())
Expand All @@ -156,15 +162,17 @@ fn train(s: &Session, p: &Value) -> Result<DlModel, String> {
}
}

/// A model to train: the raster, the training polygons and their classes, read of the session, and
/// how to train it ([`Training::fit`], which reads nothing more and runs on any thread: the pixels
/// under the polygons are sampled there, as the network learns).
/// A model to train: the raster and the training layer, taken of the session, and how to train it
/// ([`Training::fit`], which reads nothing more of the session and runs on any thread: the
/// training layer is read whole and the pixels under its polygons sampled there, as the network
/// learns).
struct Training {
raster: std::sync::Arc<rustgis_data::raster::Raster>,
bands: Vec<usize>,
classes: Vec<String>,
/// Each polygon's outline, its class and its share of the class's samples.
polygons: Vec<(Vec<Vec<Coord>>, usize, usize)>,
/// The training layer's rows, read whole in [`Training::fit`].
rows: super::whole::Whole,
/// The class field's index.
field: usize,
hidden: Vec<usize>,
epochs: usize,
name: String,
Expand All @@ -174,44 +182,53 @@ struct Training {
}

impl Training {
fn fit(self) -> Result<DlModel, String> {
let Training { raster, bands, classes, polygons, hidden, epochs, name, .. } = self;
let (samples, per_class) = neural::training_samples(&raster, &bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?;
/// Reads the training layer whole and trains the model: the model, with the rows of the
/// training layer's source that couldn't be read (named in an error, as they may be its
/// cause).
fn fit(self) -> Result<(DlModel, super::whole::Unread), String> {
let Training { raster, bands, rows, field, hidden, epochs, name, .. } = self;
let (ds, unread) = rows.read()?;
let model = Self::learn(&ds, field, &raster, &bands, &hidden, epochs, name).map_err(|e| unread.explain(e))?;
Ok((model, unread))
}

/// The model learnt from the polygons of `ds` (their classes in field `field`) over `raster`.
fn learn(
ds: &rustgis_data::Dataset,
field: usize,
raster: &rustgis_data::raster::Raster,
bands: &[usize],
hidden: &[usize],
epochs: usize,
name: String,
) -> Result<DlModel, String> {
let (classes, polygons) = training_polygons(ds, field)?;
let (samples, per_class) = neural::training_samples(raster, bands, &polygons, classes.len(), MAX_SAMPLES_PER_CLASS)?;
// Every class needs pixels of its own (its polygons may miss the raster or cover no data).
if let Some((c, _)) = classes.iter().zip(&per_class).find(|(_, n)| **n < 5) {
return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons"));
}
let t = neural::train(&bands, classes, &samples, &hidden, epochs)?;
let t = neural::train(bands, classes, &samples, hidden, epochs)?;
Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs })
}
}

/// What training reads of the session: the training polygons and their classes, and the model's
/// name.
fn training(s: &Session, p: &Value) -> Result<Training, String> {
let r = raster(s, p)?;
r.require_coordinates()?;
let bands = bands_param(p, &r)?;
let tl = super::imagery::training_name(p)?;
let l = s
.map()
.layer_by_name(tl)
.filter(|l| l.geometry_type() == GeometryType::Polygon)
.ok_or_else(|| format!("no polygon layer '{tl}'"))?;
let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?;
/// The classes of a training layer's polygons, in the order they come, and each polygon's outline
/// with its class and its share of the class's samples. A polygon with no class value trains
/// nothing.
fn training_polygons(ds: &rustgis_data::Dataset, fi: usize) -> Result<(Vec<String>, Vec<(Vec<Vec<Coord>>, usize, usize)>), String> {
let mut classes: Vec<String> = Vec::new();
let mut parts: Vec<(Vec<Vec<Coord>>, usize, usize)> = Vec::new();
// Polygons per class, so each polygon of a class gets a fair share of its samples.
let mut polygons: HashMap<String, usize> = HashMap::new();
for (row, ft) in l.dataset.features.iter().enumerate() {
for (row, ft) in ds.features.iter().enumerate() {
if let Some(Geometry::Polygon(polys)) = &ft.geometry {
*polygons.entry(l.dataset.value(row, fi).to_string()).or_default() += polys.len();
*polygons.entry(ds.value(row, fi).to_string()).or_default() += polys.len();
}
}
for (row, ft) in l.dataset.features.iter().enumerate() {
for (row, ft) in ds.features.iter().enumerate() {
let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue };
// A polygon with no class value trains nothing.
let v = l.dataset.value(row, fi);
let v = ds.value(row, fi);
if matches!(v, rustgis_data::Value::Null) || v.to_string().trim().is_empty() {
continue;
}
Expand All @@ -227,6 +244,23 @@ fn training(s: &Session, p: &Value) -> Result<Training, String> {
};
parts.extend(polys.iter().map(|rings| (rings.clone(), ci, share)));
}
Ok((classes, parts))
}

/// What training takes of the session: the raster, the training layer (read whole when the model
/// is fitted), and the model's name.
fn training(s: &Session, p: &Value) -> Result<Training, String> {
let r = raster(s, p)?;
r.require_coordinates()?;
let bands = bands_param(p, &r)?;
let tl = super::imagery::training_name(p)?;
let l = s
.map()
.layer_by_name(tl)
.filter(|l| l.geometry_type() == GeometryType::Polygon)
.ok_or_else(|| format!("no polygon layer '{tl}'"))?;
let field = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?;
let rows = super::whole::Whole::of(l);
let hidden: Vec<usize> = match p.get("hidden").and_then(Value::as_array) {
Some(a) => a.iter().filter_map(Value::as_u64).map(|u| u as usize).take(neural::MAX_LAYERS).collect(),
None => vec![32, 16],
Expand All @@ -240,7 +274,7 @@ fn training(s: &Session, p: &Value) -> Result<Training, String> {
(free_name(s, &base), Some(base))
}
};
Ok(Training { raster: r, bands, classes, polygons: parts, hidden, epochs, name, auto })
Ok(Training { raster: r, bands, rows, field, hidden, epochs, name, auto })
}

/// `base`, or `base 2`, `base 3`… : the first no model of the project has (whatever its case).
Expand Down Expand Up @@ -283,7 +317,8 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result<super::workflows::
Model::Saved(m) => (m, None),
Model::Train(t) => {
let auto = t.auto.clone();
(t.fit()?, Some(auto))
let (m, unread) = t.fit()?;
(m, Some((auto, unread)))
}
};
let (class, prob) = neural::classify(&r, &m.model, rasterops::NODATA)?;
Expand All @@ -299,8 +334,9 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result<super::workflows::
let extra = json!({"model": m.name, "accuracy": m.accuracy, "classes": legend});
// Kept once its classes are added. One that was given no name is named then, so it never
// replaces a model kept while it trained.
let then: Option<super::workflows::Then> = trained.map(|auto| {
let then: Option<super::workflows::Then> = trained.map(|(auto, unread)| {
Box::new(move |s: &mut Session, v: &mut Value| {
unread.say(s);
let mut m = m;
if let Some(base) = auto {
m.name = free_name(s, &base);
Expand Down
23 changes: 14 additions & 9 deletions crates/engine/src/cmd/imagery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1001,24 +1001,29 @@ pub(crate) fn supervised_work(s: &Session, p: &Value) -> Result<super::workflows
.filter(|l| l.geometry_type() == GeometryType::Polygon)
.ok_or_else(|| format!("no polygon layer '{tl}'"))?;
let fi = l.dataset.field_index(ps(p, "classField").ok_or("classField is required")?).ok_or("no such class field")?;
let mut polygons = Vec::new();
for (row, ft) in l.dataset.features.iter().enumerate() {
let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue };
let key = l.dataset.value(row, fi).to_string();
polygons.extend(polys.iter().map(|rings| (key.clone(), rings.clone())));
}
// All the training polygons: a layer read in place is read whole as the work runs.
let rows = super::whole::Whole::of(l);
let out = name(p, format!("{}_classified", r.name));
Ok(Box::new(move || {
let (ds, unread) = rows.read()?;
let explain = |e: String| unread.explain(e);
let mut polygons = Vec::new();
for (row, ft) in ds.features.iter().enumerate() {
let Some(Geometry::Polygon(polys)) = &ft.geometry else { continue };
let key = ds.value(row, fi).to_string();
polygons.extend(polys.iter().map(|rings| (key.clone(), rings.clone())));
}
// Mean of each class's training pixels.
let classes = rasterops::class_means(&r, &bands, &polygons)?;
let classes = rasterops::class_means(&r, &bands, &polygons).map_err(explain)?;
if classes.len() < 2 {
return Err("the training polygons need at least two classes with pixels under them".into());
return Err(explain("the training polygons need at least two classes with pixels under them".into()));
}
let means: Vec<(f32, Vec<f64>)> = classes.iter().enumerate().map(|(i, c)| ((i + 1) as f32, c.1.clone())).collect();
let pixels = rasterops::classify_min_distance(&r, &bands, &means)?;
let layer = class_layer(rasterops::derive(&r, &out, vec![pixels])?)?;
let legend: Vec<Value> = classes.iter().enumerate().map(|(i, c)| json!({"value": i + 1, "class": c.0, "pixels": c.2})).collect();
Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), None))
let then: Option<super::workflows::Then> = Some(Box::new(move |s: &mut Session, _: &mut Value| unread.say(s)));
Ok(super::workflows::made_layer(out, layer, Some(json!({"classes": legend})), then))
}))
}

Expand Down
2 changes: 1 addition & 1 deletion crates/engine/src/cmd/layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2911,7 +2911,7 @@ fn load_all(s: &mut Session, p: &Value) -> CmdResult {
ds.name = l.dataset.name.clone();
let (n, name) = (ds.features.len(), l.name.clone());
if unread > 0 {
s.warn(format!("{name}: {unread} rows couldn't be read and were left out"));
s.warn(super::whole::unread_notice(&name, unread));
}
let l = s.map_mut().layer_mut(id).ok_or("no such layer")?;
l.dataset = ds;
Expand Down
1 change: 1 addition & 0 deletions crates/engine/src/cmd/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ pub mod tiles;
pub mod time;
pub mod view;
pub mod webdata;
pub(crate) mod whole;
pub mod workflows;
pub mod workspace;
pub mod zarr;
Expand Down
99 changes: 99 additions & 0 deletions crates/engine/src/cmd/whole.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
//! Every feature of a layer, for a command whose result stands for the whole layer (a model
//! trained from it, a classification): a layer read in place holds only the features of the
//! last view, so its source is read whole, as Load All Features reads it, and the rows that
//! couldn't be read are said. As for an in-memory layer, every row is read, whatever the layer's
//! definition query.

use rustgis_data::Dataset;
use rustgis_map::Layer;

use crate::Session;

/// A layer's rows to read whole, taken when the command starts and read where its work runs
/// (on another thread for one the app runs in the background).
pub(crate) struct Whole {
/// The layer's name, for what is said of the read.
name: String,
rows: Rows,
}

enum Rows {
/// A layer in memory: its features, which are all of them (shared, not copied).
Held(Dataset),
/// A layer read in place: a copy of it, its own read of the source to make.
Source(Box<Layer>),
}

/// The rows of a layer's source that a whole read couldn't read and left out.
pub(crate) struct Unread {
layer: String,
rows: usize,
}

impl Whole {
/// `l`'s rows, to read with [`Whole::read`].
pub(crate) fn of(l: &Layer) -> Self {
let rows = if let Some(source) = &l.source {
// What a whole read needs of the layer (its source, and the name, fields, type and
// marks the read keeps), not the view's features and caches.
let ds = &l.dataset;
let mut header = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone());
header.zm_dropped = ds.zm_dropped;
header.source = ds.source.clone();
let mut copy = Layer::new(l.id, header);
let mut source = source.clone();
// Its own read: the live layer keeps the one under way for its view.
source.detach_read();
copy.source = Some(source);
Rows::Source(Box::new(copy))
} else {
Rows::Held(l.dataset.clone())
};
Whole { name: l.name.clone(), rows }
}

/// Every feature, with the fields the layer lists (so a field's index in the layer is its
/// index here), and the rows of its source that couldn't be read. A source read a chunk at a
/// time looks at Cancel between chunks; one read at once (a GeoJSON file, a database table)
/// is a single step that doesn't, as for Load All Features.
pub(crate) fn read(self) -> Result<(Dataset, Unread), String> {
let Whole { name, rows } = self;
let (ds, rows) = match rows {
Rows::Held(ds) => (ds, 0),
Rows::Source(l) => l.load_all_counted().map_err(|e| format!("{name}: {e}"))?,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low (confidence: medium-low): load_all_counted returns only report.skipped and drops ReadReport::messages, the source's own notes (for example the {name}: {note} lines materialize builds from info.notes). Load All already drops them. This PR routes training and classification of streamed layers through the same read, so those notes are lost there too, without a notice. I didn't confirm whether the same notes are already shown when the source opens, so this may be covered elsewhere. If it isn't, pass report.messages through Unread.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The source's notes are said when the layer is added (layer.rs add_source_with, ~L2142, and add_remote, ~L3684, both notify each info.notes line), so re-saying them on every whole read would repeat them. The Z/M mark is carried (load_all_counted ORs zm_dropped). Load All keeps the same behaviour; no change here.

};
Ok((ds, Unread { layer: name, rows }))
}
}

impl Unread {
/// Says the rows left out, as Load All Features says them (nothing when none were).
pub(crate) fn say(&self, s: &mut Session) {
if self.rows > 0 {
s.warn(unread_notice(&self.layer, self.rows));
}
}

/// `error`, with the rows left out when there were any: they may be why it failed.
pub(crate) fn explain(&self, error: String) -> String {
if self.rows == 0 { error } else { format!("{error} ({})", unread_notice(&self.layer, self.rows)) }
}
}

/// The notice for `unread` rows of `layer`'s source that couldn't be read.
pub(crate) fn unread_notice(layer: &str, unread: usize) -> String {
match unread {
1 => format!("{layer}: 1 row couldn't be read and was left out"),
n => format!("{layer}: {n} rows couldn't be read and were left out"),
}
}

/// The layers read in place that `command` with `params` reads whole from their sources (for its
/// permission effects, which then reach those sources as Load All Features does).
pub(crate) fn layers_read_whole<'a>(s: &super::InputContext<'a>, command: &str, params: &serde_json::Value) -> Vec<&'a Layer> {
let names: Vec<&str> = match command {
"dl.train" | "dl.classify" | "raster.classifySupervised" => super::ps(params, "training").into_iter().collect(),
_ => Vec::new(),
};
names.into_iter().filter_map(|n| s.map().layer_by_name(n)).filter(|l| l.is_streamed()).collect()
}
3 changes: 3 additions & 0 deletions crates/engine/src/policy/effects.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ pub(super) fn command(s: &crate::cmd::InputContext<'_>, spec: &CommandSpec, para
{
effects = effects.union(source_reach(source));
}
for source in crate::cmd::whole::layers_read_whole(s, spec.id, params).into_iter().filter_map(|l| l.source.as_ref()) {
effects = effects.union(source_reach(source));
}
if crate::cmd::time::follows_sources(spec.id) {
for (_, _, source) in crate::cmd::time::following_sources(s) {
effects = effects.with(Effect::Files);
Expand Down
Loading
Loading