Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion apps/rustgis-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -849,8 +849,11 @@ fn main() -> ExitCode {
}
"run" => {
// Nothing waits on `run` but the shell (an assistant or an MCP client does, and has a
// timeout): long runs take the app's bounds, their progress on stderr.
// timeout): long runs take the app's bounds, their progress on stderr. A person at the
// terminal can stop a run with Ctrl-C, so there tools take no work cap; a program
// calling it may have a timeout, and keeps the caps.
s.long_runs = Some(progress);
s.stoppable = std::io::stdin().is_terminal() && std::io::stderr().is_terminal();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The terminal check doesn't separate a person from a program with a pseudo-terminal (medium-low confidence). is_terminal() on stdin and stderr is true for any caller that runs the CLI under a pty: script, tmux or screen panes, IDE and CI terminal runners, and some agent shells. Such a caller can wait with a timeout, and it now gets no refusal past the caps, so a timed-out call is killed with its --save lost. This is the timeout case the earlier thread raised, narrowed but not gone. Options: an explicit opt-in flag on run (for example --stoppable) that a person types, with the TTY check kept only for the default, or a note in the guide that a pty-attached caller is uncapped too.

open_inputs(&mut s, &args)?;
for (i, a) in args.iter().enumerate() {
if a == "--cmd"
Expand Down
56 changes: 56 additions & 0 deletions crates/analysis/src/control.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
//! the tool has come. The run's control is the current one of the thread the tool runs on, so a
//! tool asks [`check`] in its long loops without passing it along; the threads of
//! [`crate::parallel`] take it over from the thread that starts them.
//!
//! A run someone can stop while it goes ([`Control::stoppable`]: the app's panes, `rustgis-cli
//! run`) takes no work cap ([`uncapped`]): the caps, a second or two of work, are for the calls
//! that wait on a tool with no way to stop it (a script, the assistant, the web app).

use std::cell::RefCell;
use std::sync::Arc;
Expand All @@ -15,13 +19,21 @@ pub struct Control {
cancel: AtomicBool,
done: AtomicU64,
total: AtomicU64,
/// Someone can stop the run while it goes ([`Control::stoppable`]).
stoppable: bool,
}

impl Control {
pub fn new() -> Arc<Control> {
Arc::new(Control::default())
}

/// A control for a run someone can stop while it goes, with a Cancel button or Ctrl-C: its
/// tools take no work cap ([`uncapped`]). Not in the browser, whose one thread a run holds.
pub fn stoppable() -> Arc<Control> {
Arc::new(Control { stoppable: !cfg!(target_arch = "wasm32"), ..Control::default() })
}

/// Asks the tool to stop at its next check.
pub fn cancel(&self) {
self.cancel.store(true, Ordering::Relaxed);
Expand Down Expand Up @@ -89,6 +101,33 @@ pub(crate) fn cancelled() -> bool {
CURRENT.with(|c| c.borrow().as_ref().is_some_and(|c| c.is_cancelled()))
}

/// Whether the run on this thread can be stopped while it goes ([`Control::stoppable`]), and so
/// takes no work cap.
pub(crate) fn uncapped() -> bool {
CURRENT.with(|c| c.borrow().as_ref().is_some_and(|c| c.stoppable))
}

/// The work units (a distance, a ray step, a multiply-add) between two looks at the cancel flag
/// inside an item of a loop ([`Pace`]): a few milliseconds.
const PACE_WORK: usize = 1 << 22;

/// Looks at the cancel flag about every [`PACE_WORK`] units of work, within an item that may
/// take long on its own (a row of a wide grid without a work cap), so Cancel stops the run soon.
#[derive(Default)]
pub(crate) struct Pace(usize);

impl Pace {
/// Whether to stop after `work` more units: once the run was asked to.
pub(crate) fn stop(&mut self, work: usize) -> bool {
self.0 = self.0.saturating_add(work);
if self.0 < PACE_WORK {
return false;
}
self.0 = 0;
cancelled()
}
}

/// Stops the tool when its run was asked to: call it between rows or features of a long loop.
pub(crate) fn check() -> Result<(), Cancelled> {
if cancelled() { Err(Cancelled) } else { Ok(()) }
Expand Down Expand Up @@ -159,6 +198,23 @@ mod tests {
assert!(current().is_none() && check().is_ok());
}

#[test]
fn only_a_stoppable_run_is_uncapped_and_pace_sees_its_cancel() {
assert!(!uncapped(), "no run");
{
let _g = install(Some(Control::new()));
assert!(!uncapped(), "a run nobody can stop keeps its caps");
}
let c = Control::stoppable();
let _g = install(Some(c.clone()));
assert!(uncapped());
let mut pace = Pace::default();
assert!(!pace.stop(PACE_WORK), "not cancelled");
c.cancel();
assert!(!pace.stop(1), "not looked at again before PACE_WORK more");
assert!(pace.stop(PACE_WORK));
}

#[test]
fn no_progress_before_a_step_is_counted() {
let c = Control::new();
Expand Down
24 changes: 18 additions & 6 deletions crates/analysis/src/density.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ pub fn max_density_cells() -> usize {
}

/// Upper limit on aggregate cell visits across all points on one thread, to prevent CPU
/// exhaustion; a run on several threads may take as many times more
/// ([`crate::parallel::work_cap`]).
/// exhaustion; a run on several threads may take as many times more, and one that can be
/// cancelled any amount ([`crate::parallel::work_cap`]).
pub const MAX_DENSITY_WORK: usize = 50_000_000;

/// The points between two counts of a strip's progress and checks for a cancel.
Expand Down Expand Up @@ -198,8 +198,8 @@ where
crate::parallel::for_each_row(
grid.chunks_mut(w * strip_rows),
threads,
|| (),
|_, s, strip: &mut [f64]| {
crate::control::Pace::default,
|pace, s, strip: &mut [f64]| {
let first = s * strip_rows;
let last = first + strip.len() / w - 1;
let (mut counted, mut cells) = (0, 0usize);
Expand Down Expand Up @@ -232,6 +232,11 @@ where
}
let rem_sq = h_sq - dy_sq;
let row_offset = (row - first) * w;
// One point's window can span the grid without a cap: Cancel is looked at
// within it.
if pace.stop(col_max + 1 - col_min) {
return;
}

for col in col_min..=col_max {
let x_cell = origin_x + (col as f64 + 0.5) * cell_size;
Expand Down Expand Up @@ -335,8 +340,9 @@ fn kernel_density_on(
let cap = crate::parallel::work_cap(MAX_DENSITY_WORK, threads);
if total_work > cap {
return Err(format!(
"{w} × {h_rows} cells from {} points is too much work, more than the {cap} cell visits this computer takes on {threads} thread(s); choose a larger pixel size or a smaller search radius",
points.len()
"{w} × {h_rows} cells from {} points is too much work, more than the {cap} cell visits this computer takes on {threads} thread(s); choose a larger pixel size or a smaller search radius{}",
points.len(),
crate::parallel::UNCAPPED_HINT
));
}

Expand Down Expand Up @@ -552,6 +558,12 @@ mod tests {
err.contains("too much work") && err.contains(&format!("the {MAX_DENSITY_WORK} cell visits")) && err.contains("1 thread(s)"),
"{err}"
);
// A run that can be cancelled takes it on: cancelled before it starts, it says so.
let run = crate::control::Control::stoppable();
run.cancel();
let _installed = crate::control::install(Some(run));
let err = kernel_density_on(Some(1), &pts, Some(&ext), 10.0, 1000.0, KernelType::Quartic, "huge", Crs::WebMercator).unwrap_err();
assert_eq!(err, crate::control::CANCELLED);
}

#[test]
Expand Down
34 changes: 29 additions & 5 deletions crates/analysis/src/kriging.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ use rustgis_geom::{Coord, Extent};
use crate::rasterops::{NODATA, max_pixels};

/// Maximum cell-point pairs Kriging computes on one thread (about a second's work); a run on several
/// threads may take as many times more ([`crate::parallel::work_cap`]).
/// threads may take as many times more, and one that can be cancelled any amount
/// ([`crate::parallel::work_cap`]).
pub const MAX_KRIGING_WORK: usize = 40_000_000;

/// Theoretical semivariogram models for spatial autocorrelation.
Expand Down Expand Up @@ -353,17 +354,22 @@ fn kriging_on(
let cap = crate::parallel::work_cap(MAX_KRIGING_WORK, threads);
if work > cap {
return Err(format!(
"{w} × {h} cells from {} points is too much work, more than the {cap} this computer takes on {threads} thread(s); use a larger cell size",
points.len()
"{w} × {h} cells from {} points is too much work, more than the {cap} this computer takes on {threads} thread(s); use a larger cell size{}",
points.len(),
crate::parallel::UNCAPPED_HINT
));
}
let scratch = || (Vec::<(f64, f64, Coord)>::with_capacity(points.len()), KrigingBuffers::new());
let scratch = || (Vec::<(f64, f64, Coord)>::with_capacity(points.len()), KrigingBuffers::new(), crate::control::Pace::default());
let (mut pred_data, mut var_data) = (crate::rasterops::new_band(w, h, NODATA)?, crate::rasterops::new_band(w, h, NODATA)?);
let rows = pred_data.chunks_mut(w).zip(var_data.chunks_mut(w));
crate::control::begin(h);
crate::parallel::for_each_row(rows, threads, scratch, |(near, buffers), row, (pred, var): (&mut [f32], &mut [f32])| {
crate::parallel::for_each_row(rows, threads, scratch, |(near, buffers, pace), row, (pred, var): (&mut [f32], &mut [f32])| {
let kx = (extent.max_y - (row as f64 + 0.5) * cell_deg).to_radians().cos().abs().max(1e-6);
for (col, (pred, var)) in pred.iter_mut().zip(var.iter_mut()).enumerate() {
// A wide row of many points takes long without a cap: Cancel is looked at within it.
if pace.stop(work_per_cell) {
return;
}
let c = Coord::new(extent.min_x + (col as f64 + 0.5) * cell_deg, extent.max_y - (row as f64 + 0.5) * cell_deg);
near.clear();
near.extend(points.iter().map(|(p, v)| {
Expand Down Expand Up @@ -393,6 +399,24 @@ fn kriging_on(
mod tests {
use super::*;

/// A run past one thread's work is refused where nothing can stop it, and taken on by a run
/// that can be cancelled (#1291 step 2).
#[test]
fn kriging_past_the_cap_runs_only_when_it_can_be_cancelled() {
// 500 × 500 cells from 300 points (and 12 neighbours): about 220 million units, past one
// thread's 40 million.
let pts: Vec<(Coord, f64)> =
(0..300).map(|i| (Coord::new((i % 20) as f64 / 2.0, 40.0 + (i / 20) as f64 / 1.5), i as f64)).collect();
let extent = Extent::new(0.0, 40.0, 10.0, 50.0);
let params = auto_variogram(&pts, &extent, VariogramModel::Spherical);
let e = kriging_on(Some(1), &pts, &extent, 0.02, &params, 12, "k").unwrap_err();
assert!(e.contains("too much work") && e.contains("on 1 thread(s)"), "{e}");
let run = crate::control::Control::stoppable();
run.cancel();
let _installed = crate::control::install(Some(run));
assert_eq!(kriging_on(Some(1), &pts, &extent, 0.02, &params, 12, "k").unwrap_err(), crate::control::CANCELLED);
}

#[test]
fn test_exact_interpolation_at_sample_points() {
let sample_pts = vec![
Expand Down
64 changes: 50 additions & 14 deletions crates/analysis/src/neural.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ pub const MAX_EPOCHS: usize = 500;
/// Most classes and input bands.
pub const MAX_CLASSES: usize = 64;
pub const MAX_INPUTS: usize = 64;
/// Most multiply-adds one training may do (seconds, not minutes): epochs are reduced to fit.
/// Most multiply-adds one training may do (seconds, not minutes): epochs are reduced to fit. A
/// run that can be cancelled takes any amount ([`crate::control::uncapped`]).
pub const MAX_WORK: f64 = 5e9;

/// One dense layer: `out × inp` weights (row per output) and `out` biases.
Expand Down Expand Up @@ -217,7 +218,8 @@ pub struct Trained {
pub model: Mlp,
pub accuracy: f64,
pub samples: usize,
/// The epochs run (fewer than asked when the work would pass [`MAX_WORK`]).
/// The epochs run (fewer than asked when the work would pass [`MAX_WORK`] in a run that can't
/// be cancelled).
pub epochs: usize,
}

Expand Down Expand Up @@ -285,25 +287,28 @@ pub fn train(
let inputs: Vec<Vec<f32>> = samples.iter().map(|(x, _)| scaled(x)).collect();
// Mini-batch gradient descent with momentum.
// Each sample costs about three passes over the weights (forward, backward, update).
// Too many samples for even one epoch: a random subset (they're shuffled) is fitted.
// Too many samples for even one epoch: a random subset (they're shuffled) is fitted. A run
// that can be cancelled fits them all, for every epoch asked.
let per_sample = 3.0 * layers.iter().map(|l| l.w.len() + l.b.len()).sum::<usize>() as f64;
let max_work = if crate::control::uncapped() { f64::INFINITY } else { MAX_WORK };
let mut fit = fit;
fit.truncate(((MAX_WORK / per_sample) as usize).max(1));
fit.truncate(((max_work / per_sample).min(usize::MAX as f64) as usize).max(1));
let per_epoch = per_sample * fit.len().max(1) as f64;
let epochs = epochs.clamp(1, MAX_EPOCHS).min(((MAX_WORK / per_epoch) as usize).max(1));
let epochs = epochs.clamp(1, MAX_EPOCHS).min(((max_work / per_epoch).min(usize::MAX as f64) as usize).max(1));
let (lr, momentum, batch) = (0.05f32, 0.9f32, 32usize);
let mut vel: Vec<(Vec<f32>, Vec<f32>)> = layers.iter().map(|l| (vec![0.0; l.w.len()], vec![0.0; l.b.len()])).collect();
crate::control::begin(epochs);
for epoch in 0..epochs {
if crate::control::cancelled() {
break;
}
'epochs: for epoch in 0..epochs {
crate::control::reached(epoch);
for i in (1..fit.len()).rev() {
let j = rng.below(i + 1);
fit.swap(i, j);
}
for chunk in fit.chunks(batch) {
// An epoch of every sample takes long without a cap: Cancel is looked at each batch.
if crate::control::cancelled() {
break 'epochs;
}
let mut grads: Vec<(Vec<f32>, Vec<f32>)> = layers.iter().map(|l| (vec![0.0; l.w.len()], vec![0.0; l.b.len()])).collect();
for &si in chunk {
let (Some(x), Some((_, class))) = (inputs.get(si), samples.get(si)) else { continue };
Expand Down Expand Up @@ -370,6 +375,8 @@ pub fn train(
}
}
}
// A cancelled run makes no model: the accuracy check on held-back samples can take long too.
crate::control::check()?;
if !layers.iter().all(|l| l.w.iter().chain(&l.b).all(|v| v.is_finite())) {
return Err("training diverged; try fewer epochs or smaller layers".into());
}
Expand All @@ -389,10 +396,14 @@ pub fn classify(r: &Raster, model: &Mlp, nodata: f32) -> Result<(Vec<f32>, Vec<f
/// [`classify`] on `threads` threads (none: as many as pay off).
fn classify_on(r: &Raster, model: &Mlp, nodata: f32, threads: Option<usize>) -> Result<(Vec<f32>, Vec<f32>), String> {
model.validate()?;
// A large model on a large raster is refused rather than run for minutes.
// A large model on a large raster is refused rather than run for minutes, unless the run can
// be cancelled.
let weights: usize = model.layers.iter().map(|l| l.w.len() + l.b.len()).sum();
if f64::from(r.width) * f64::from(r.height) * weights as f64 > 4.0 * MAX_WORK {
return Err("that model is too large for this raster; use a smaller model or clip the raster first".into());
if !crate::control::uncapped() && f64::from(r.width) * f64::from(r.height) * weights as f64 > 4.0 * MAX_WORK {
return Err(format!(
"that model is too large for this raster; use a smaller model or clip the raster first{}",
crate::parallel::UNCAPPED_HINT
));
}
if let Some(b) = model.bands.iter().find(|b| **b >= r.bands.len()) {
return Err(format!("{} has no band {}", r.name, b + 1));
Expand All @@ -408,9 +419,14 @@ fn classify_on(r: &Raster, model: &Mlp, nodata: f32, threads: Option<usize>) ->
crate::parallel::for_each_row(
class.chunks_mut(w.max(1)).zip(prob.chunks_mut(w.max(1))),
threads,
|| vec![0.0f32; model.bands.len()],
|x: &mut Vec<f32>, row, (class, prob): (&mut [f32], &mut [f32])| {
|| (vec![0.0f32; model.bands.len()], crate::control::Pace::default()),
|(x, pace): &mut (Vec<f32>, crate::control::Pace), row, (class, prob): (&mut [f32], &mut [f32])| {
for (col, (c_out, p_out)) in class.iter_mut().zip(prob.iter_mut()).enumerate() {
// A large model on a wide row takes long without a cap: Cancel is looked at
// within it.
if pace.stop(weights) {
return;
}
// A non-finite value is treated as no data.
let ok = model.bands.iter().zip(x.iter_mut()).all(|(b, xv)| {
match r.value(*b, col as i64, row as i64).filter(|v| v.is_finite()) {
Expand Down Expand Up @@ -472,6 +488,26 @@ mod tests {
assert_eq!((c, p), (c1, p1));
}

/// A model too large for a raster is refused where nothing can stop the run, and taken on by
/// a run that can be cancelled (#1291 step 2).
#[test]
fn a_large_model_on_a_large_raster_runs_only_when_it_can_be_cancelled() {
use rustgis_data::raster::{ColorHint, Crs, GeoTransform};
let samples: Vec<(Vec<f32>, usize)> = (0..40).map(|i| (vec![i as f32], usize::from(i >= 20))).collect();
// About 5,500 weights on 2000 × 2000 pixels: past four times MAX_WORK.
let model = train(&[0], vec!["a".into(), "b".into()], &samples, &[80, 64], 1).unwrap().model;
let (w, h) = (2000u32, 2000u32);
let transform = GeoTransform { origin_x: 0.0, origin_y: 45.0, pixel_width: 0.001, pixel_height: -0.001 };
let r =
Raster::new("img", w, h, vec![vec![1.0; (w * h) as usize]], transform, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap();
let e = classify_on(&r, &model, -1.0, Some(1)).unwrap_err();
assert!(e.starts_with("that model is too large for this raster"), "{e}");
let run = crate::control::Control::stoppable();
run.cancel();
let _installed = crate::control::install(Some(run));
assert_eq!(classify_on(&r, &model, -1.0, Some(1)).unwrap_err(), crate::control::CANCELLED);
}

#[test]
fn learns_a_nonlinear_boundary_and_validates() {
// Class 1 inside a ring, class 0 outside: not separable by a line.
Expand Down
13 changes: 11 additions & 2 deletions crates/analysis/src/parallel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,20 @@ pub(crate) fn threads_for_rows(rows: usize, work: usize, scratch_bytes: usize) -

/// A work cap tuned for one thread (a second or two of its work) for a run on `threads` threads:
/// as many times larger, since the threads share the work and the run takes about as long
/// (#1291 step 2). On one thread, and so in the browser, the cap as it is.
/// (#1291 step 2). On one thread, and so in the browser, the cap as it is. None for a run that
/// can be stopped while it goes ([`crate::control::uncapped`]): its progress shows, and Cancel
/// stops it.
pub(crate) fn work_cap(one_thread: usize, threads: usize) -> usize {
one_thread.saturating_mul(threads.max(1))
if crate::control::uncapped() { usize::MAX } else { one_thread.saturating_mul(threads.max(1)) }
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

/// What a refusal past [`work_cap`] adds: where the work isn't capped. Nothing in the browser.
pub(crate) const UNCAPPED_HINT: &str = if cfg!(target_arch = "wasm32") {
""
} else {
Comment thread
pick2510 marked this conversation as resolved.
Resolved
" (or run it from its pane in the desktop app, which takes any amount of work and can be cancelled)"
};

/// The cores to use, at most [`MAX_THREADS`].
fn cores() -> usize {
std::thread::available_parallelism().map_or(1, |n| n.get()).min(MAX_THREADS)
Expand Down
Loading
Loading