diff --git a/crates/analysis/src/neural.rs b/crates/analysis/src/neural.rs index 1e3b26d265..6840a551ba 100644 --- a/crates/analysis/src/neural.rs +++ b/crates/analysis/src/neural.rs @@ -221,6 +221,31 @@ pub struct Trained { /// The epochs run (fewer than asked when the work would pass [`MAX_WORK`] in a run that can't /// be cancelled). pub epochs: usize, + /// The epochs asked (within 1 to [`MAX_EPOCHS`]). + pub epochs_asked: usize, + /// The training samples fitted each epoch (fewer than [`Trained::training`] when they would + /// pass [`MAX_WORK`] in a run that can't be cancelled: a random subset). + pub fitted: usize, + /// The samples kept for training (the rest are held back to measure the accuracy). + pub training: usize, +} + +impl Trained { + /// What a run that can't be cancelled left out to stay within its work limit, to be said: `None` + /// when it fitted every training sample for every epoch asked. + pub fn limited(&self) -> Option { + if self.fitted >= self.training && self.epochs >= self.epochs_asked { + return None; + } + Some(format!( + "trained on {} of {} training samples for {} of {} epochs, the work limit of a run that can't be cancelled{}", + self.fitted, + self.training, + self.epochs, + self.epochs_asked, + crate::parallel::UNCAPPED_HINT + )) + } } /// Trains a classifier. `samples` are (band values, class index); `hidden` the units per hidden @@ -292,9 +317,14 @@ pub fn train( let per_sample = 3.0 * layers.iter().map(|l| l.w.len() + l.b.len()).sum::() as f64; let max_work = if crate::control::uncapped() { f64::INFINITY } else { MAX_WORK }; let mut fit = fit; + let training = fit.len(); fit.truncate(((max_work / per_sample).min(usize::MAX as f64) as usize).max(1)); let per_epoch = per_sample * fit.len().max(1) as f64; - let epochs = epochs.clamp(1, MAX_EPOCHS).min(((max_work / per_epoch).min(usize::MAX as f64) as usize).max(1)); + if !(1..=MAX_EPOCHS).contains(&epochs) { + return Err(format!("epochs is 1 to {MAX_EPOCHS}, not {epochs}")); + } + let epochs_asked = epochs; + let epochs = epochs_asked.min(((max_work / per_epoch).min(usize::MAX as f64) as usize).max(1)); let (lr, momentum, batch) = (0.05f32, 0.9f32, 32usize); let mut vel: Vec<(Vec, Vec)> = layers.iter().map(|l| (vec![0.0; l.w.len()], vec![0.0; l.b.len()])).collect(); crate::control::begin(epochs); @@ -383,7 +413,7 @@ pub fn train( let model = Mlp { bands: bands.to_vec(), mean, std, layers, classes }; let correct = test.iter().filter(|&&i| samples.get(i).is_some_and(|(x, c)| model.predict(x).0 == *c)).count(); let accuracy = if test.is_empty() { 1.0 } else { correct as f64 / test.len() as f64 }; - Ok(Trained { model, accuracy, samples: samples.len(), epochs }) + Ok(Trained { model, accuracy, samples: samples.len(), epochs, epochs_asked, fitted: fit.len(), training }) } /// Classifies every pixel: class numbers (1-based, `nodata` where a band has no data) and each @@ -534,6 +564,21 @@ mod tests { // The largest network for every epoch would pass the work cap: fewer epochs run. let big = train(&[0, 1], vec!["out".into(), "in".into()], &samples, &[MAX_UNITS; MAX_LAYERS], MAX_EPOCHS).unwrap(); assert!(big.epochs < MAX_EPOCHS, "{} epochs", big.epochs); + // ...and it says so, with what it ran of what was asked. + let said = big.limited().expect("a capped run says what it left out"); + assert!(said.contains(&format!("for {} of {MAX_EPOCHS} epochs", big.epochs)), "{said}"); + assert_eq!(t.limited(), None, "a run within the limit says nothing"); + // Too many samples for even one epoch of the largest network: a random subset is fitted, + // and said with its count. + let many: Vec<(Vec, usize)> = (0..12_000).map(|i| (vec![(i % 97) as f32, (i % 89) as f32], i % 2)).collect(); + let cut = train(&[0, 1], vec!["a".into(), "b".into()], &many, &[MAX_UNITS; MAX_LAYERS], 1).unwrap(); + assert!(cut.fitted < cut.training && cut.epochs == 1, "{} of {}", cut.fitted, cut.training); + let said = cut.limited().expect("a cut sample set is said"); + assert!(said.contains(&format!("on {} of {} training samples for 1 of 1 epochs", cut.fitted, cut.training)), "{said}"); + // Run so it can be cancelled, it fits every sample for every epoch and says nothing. + let control = crate::control::Control::stoppable(); + let whole = crate::run_under("Train", &control, || train(&[0, 1], vec!["out".into(), "in".into()], &samples, &[8], 3)).unwrap(); + assert_eq!((whole.fitted, whole.epochs, whole.limited()), (whole.training, 3, None)); assert!(train(&[0, 1], vec!["a".into(), "b".into()], &samples[..2], &[4], 1).is_err(), "too few"); } } diff --git a/crates/engine/src/cmd/deeplearning.rs b/crates/engine/src/cmd/deeplearning.rs index 625593603f..578b6bc58b 100644 --- a/crates/engine/src/cmd/deeplearning.rs +++ b/crates/engine/src/cmd/deeplearning.rs @@ -35,6 +35,21 @@ pub struct DlModel { /// Epochs the last training ran (not saved). #[serde(skip)] pub epochs: usize, + /// What the last training left out to stay within the work limit of a run that can't be + /// cancelled ([`neural::Trained::limited`]), said when the model is kept (not saved). + #[serde(skip)] + pub limited: Option, +} + +impl DlModel { + /// Says what its training left out to stay within the work limit, if anything (AGENTS.md › + /// Never silent), and puts it in the command's answer. + fn say_limited(&self, s: &mut Session, v: &mut Value) { + if let Some(what) = &self.limited { + s.warn(format!("{}: {what}", self.name)); + v["limited"] = json!(what); + } + } } fn need_raster(s: &Session) -> Option<&'static str> { @@ -51,6 +66,7 @@ pub fn commands() -> Vec { let (m, unread) = train(s, p)?; unread.say(s); let mut r = json!({"model": m.name, "accuracy": m.accuracy, "classes": m.model.classes, "epochs": m.epochs}); + m.say_limited(s, &mut r); r["replaced"] = json!(store(s, m)); Ok(r) }).inputs(training_inputs).roles(&[]).effects(crate::policy::Effects::NONE) @@ -209,7 +225,8 @@ impl Training { return Err(format!("the class '{c}' has fewer than 5 pixels with data under its polygons")); } let t = neural::train(bands, classes, &samples, hidden, epochs)?; - Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs }) + let limited = t.limited(); + Ok(DlModel { name, model: t.model, accuracy: t.accuracy, epochs: t.epochs, limited }) } } @@ -265,7 +282,16 @@ fn training(s: &Session, p: &Value) -> Result { Some(a) => a.iter().filter_map(Value::as_u64).map(|u| u as usize).take(neural::MAX_LAYERS).collect(), None => vec![32, 16], }; - let epochs = f(p, "epochs").unwrap_or(60.0).clamp(1.0, neural::MAX_EPOCHS as f64) as usize; + // A count given that isn't a whole number from 1 to 500 is refused, not cut to one or left + // for the default: the training would say nothing of the epochs left out. + let epochs = match p.get("epochs").filter(|v| !v.is_null()) { + None => 60.0, + // Read as other counts are (a numeric string too), then checked. + Some(v) => f(p, "epochs") + .filter(|e| (1.0..=neural::MAX_EPOCHS as f64).contains(e) && e.fract() == 0.0) + .ok_or_else(|| format!("epochs is 1 to {}, not {v}", neural::MAX_EPOCHS))?, + }; + let epochs = epochs as usize; // Without a name given, a new model never replaces one: it gets a name of its own. let (name, auto) = match ps(p, "model") { Some(_) => (name(&json!({"name": p.get("model")}), format!("{}_model", r.name)), None), @@ -342,6 +368,7 @@ pub(crate) fn classify_work(s: &Session, p: &Value) -> Result().unwrap_or(60.0); - run( - app, - "dl.classify", - json!({"layer": app.workflow.layer, "training": app.workflow.layer2, "classField": app.workflow.field, "epochs": epochs}), - ); + // A blank box is the default; what is typed goes as it is, for the command to + // read or refuse (a fraction or text isn't cut or replaced without a word). + let mut p = json!({"layer": app.workflow.layer, "training": app.workflow.layer2, "classField": app.workflow.field}); + let typed = app.workflow.number.trim(); + if !typed.is_empty() { + p["epochs"] = json!(typed); + } + run(app, "dl.classify", p); } }); let models: Vec<(String, f64)> = app.session.project.dl_models.iter().map(|m| (m.name.clone(), m.accuracy)).collect(); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 05685b0178..eae4f74460 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -94,8 +94,8 @@ run` called from another program, such as an agent or a CI job, which may wait w Viewshed, IDW Interpolation and Kriging refuse a run that would take more than a second or two of work on one thread (a ray step, a cell-point pair), Kernel Density one whose points' search windows (the square around each point's radius) cover more than 50 million cells, and Deep Learning a model -too large for its raster; its training fits a share of the sampled pixels, or fewer epochs, instead -(`dl.train` reports the epochs it ran). A task or model recorded from a pane run keeps these limits +too large for its raster; its training fits a share of the sampled pixels, or fewer epochs, instead, +and says how many of each it trained on (`"limited"` in `dl.train`'s and `dl.classify`'s answer). A task or model recorded from a pane run keeps these limits when it runs again in the app, so a step the pane took on can be refused there, and a model trained again can differ. On the desktop these limits grow with the threads the run takes (the cores, up to 16 unless [`threads`](#analysis-workflows) says otherwise), as many