diff --git a/crates/analysis/src/density.rs b/crates/analysis/src/density.rs index 298fcdddde..8a3cf9df49 100644 --- a/crates/analysis/src/density.rs +++ b/crates/analysis/src/density.rs @@ -35,8 +35,16 @@ impl KernelType { } } -/// Upper limit on total output raster cells to prevent memory exhaustion (64 MB for f32). -pub const MAX_DENSITY_CELLS: usize = 16_000_000; +/// The fewest output cells [`max_density_cells`] allows, and its limit in the browser. +pub const DENSITY_CELLS_FLOOR: usize = 16_000_000; + +/// Upper limit on total output raster cells, to prevent memory exhaustion: on the desktop as many as +/// a quarter of the computer's memory holds at 16 bytes a cell (the sums, the output and the +/// GeoTIFF written from it), from [`DENSITY_CELLS_FLOOR`] up to the most any tool's output raster +/// takes ([`crate::rasterops::PIXELS_CEILING`]) (#1291 step 2). +pub fn max_density_cells() -> usize { + rustgis_data::memory::scaled(rustgis_data::memory::physical(), DENSITY_CELLS_FLOOR, crate::rasterops::PIXELS_CEILING, 16) +} /// Upper limit on aggregate cell visits across all points on one thread, to prevent CPU /// exhaustion; a run on several threads may take as many times more @@ -308,8 +316,9 @@ fn kernel_density_on( let h_rows = snap_cells(base_extent.height()); let total_cells = w.checked_mul(h_rows).ok_or("grid dimensions overflow")?; - if total_cells > MAX_DENSITY_CELLS { - return Err(format!("{w} × {h_rows} cells is too large; choose a larger pixel size")); + let max_cells = max_density_cells(); + if total_cells > max_cells { + return Err(format!("{w} × {h_rows} cells is too large, more than this computer's {max_cells} cells; choose a larger pixel size")); } // Estimate total cell visits bounded by the point kernel window and output grid @@ -534,7 +543,7 @@ mod tests { #[test] fn excessive_workload_is_rejected() { - // 50,000 points in a 100x100 grid (10,000 cells <= MAX_DENSITY_CELLS) with large radius: + // 50,000 points in a 100x100 grid (10,000 cells <= DENSITY_CELLS_FLOOR) with large radius: // 500 million cell visits, more than one thread takes (several may take them). let pts: Vec<(Coord, f64)> = (0..50_000).map(|_| (Coord::new(500.0, 500.0), 1.0)).collect(); let ext = Extent::new(0.0, 0.0, 1000.0, 1000.0); diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index 751f0622a5..0436e57e11 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -36,11 +36,12 @@ pub mod warp; pub use attribute_join::{DuplicateStrategy, JoinFieldOptions, JoinFieldReport, join_field}; pub use density::{ - KernelType, MAX_DENSITY_CELLS, MAX_DENSITY_WORK, auto_cell_size, kernel_density, points_extent, resolve_grid_extent, - silverman_bandwidth, + DENSITY_CELLS_FLOOR, KernelType, MAX_DENSITY_WORK, auto_cell_size, kernel_density, max_density_cells, points_extent, + resolve_grid_extent, silverman_bandwidth, }; +pub use parallel::scratch_budget; pub use parameters::StatisticList; -pub use tools::{TOOLS, find_field_case_insensitive, holds_coordinates, param_applies, simplify_geometry, zonal_bands}; +pub use tools::{TOOLS, find_field_case_insensitive, holds_coordinates, max_buffer_bytes, param_applies, simplify_geometry, zonal_bands}; use rustgis_data::Dataset; use serde_json::Value; diff --git a/crates/analysis/src/parallel.rs b/crates/analysis/src/parallel.rs index e8c8467642..abbaba511f 100644 --- a/crates/analysis/src/parallel.rs +++ b/crates/analysis/src/parallel.rs @@ -18,8 +18,18 @@ const STACK_BYTES: usize = 8 << 20; /// cell-point pairs for IDW (about 30 ns each, so some 5 ms of it), the same plus each cell's /// system for kriging, ray steps for a viewshed. Starting a thread costs tens of microseconds. pub(crate) const MIN_ROW_WORK: usize = 200_000; -/// The scratch memory a raster tool's threads may take together, beyond one thread's. -const SCRATCH_BUDGET: usize = 256 << 20; +/// The scratch memory a tool's threads may take together beyond one thread's, in the browser and +/// at least on the desktop ([`scratch_budget`]). +const SCRATCH_FLOOR: usize = 256 << 20; +/// The most [`scratch_budget`] allows, however much memory the computer has. +const SCRATCH_CEILING: usize = 4 << 30; + +/// The scratch memory a tool's threads may take together beyond one thread's: on the desktop a +/// sixteenth of the computer's memory, on top of the quarter its input and output limits plan for, +/// from [`SCRATCH_FLOOR`] up to [`SCRATCH_CEILING`] (#1291 step 2). +pub fn scratch_budget() -> usize { + rustgis_data::memory::scaled(rustgis_data::memory::physical().map(|m| m / 4), SCRATCH_FLOOR, SCRATCH_CEILING, 1) +} /// How many threads to make `items` items on: one below `min_items`, always one in the browser, /// and no more than there are batches. @@ -31,20 +41,20 @@ pub(crate) fn threads_for(items: usize, min_items: usize) -> usize { } /// [`threads_for`], each thread holding up to `scratch_bytes` at its peak (a feature's raster -/// masks, say): beyond one thread's, no more than [`SCRATCH_BUDGET`] holds. +/// masks, say): beyond one thread's, no more than [`scratch_budget`] holds. pub(crate) fn threads_for_scratch(items: usize, min_items: usize, scratch_bytes: usize) -> usize { - threads_for(items, min_items).min(1 + SCRATCH_BUDGET / scratch_bytes.max(1)) + threads_for(items, min_items).min(1 + scratch_budget() / scratch_bytes.max(1)) } /// How many threads to make `rows` raster rows on, `work` in all, each thread taking /// `scratch_bytes` of scratch: one below [`MIN_ROW_WORK`] and in the browser, no more than there -/// are rows, and, beyond one thread's scratch, no more than [`SCRATCH_BUDGET`] holds (a scratch per point, for millions of +/// are rows, and, beyond one thread's scratch, no more than [`scratch_budget`] holds (a scratch per point, for millions of /// points, is large). pub(crate) fn threads_for_rows(rows: usize, work: usize, scratch_bytes: usize) -> usize { if cfg!(target_arch = "wasm32") || work < MIN_ROW_WORK { return 1; } - let affordable = 1 + SCRATCH_BUDGET / scratch_bytes.max(1); + let affordable = 1 + scratch_budget() / scratch_bytes.max(1); cores().min(rows).min(affordable).max(1) } @@ -389,7 +399,7 @@ mod tests { assert_eq!(threads_for_rows(100, MIN_ROW_WORK - 1, 0), 1); assert!(threads_for_rows(3, usize::MAX, 0) <= 3); // Scratch larger than the budget: one thread more than it holds, none beyond. - assert!(threads_for_rows(100, usize::MAX, SCRATCH_BUDGET) <= 2); + assert!(threads_for_rows(100, usize::MAX, scratch_budget()) <= 2); assert_eq!(threads_for_rows(100, usize::MAX, usize::MAX), 1); } diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 6c413f9a47..991b2cb965 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1245,8 +1245,11 @@ fn union_all<'a>(polys: impl IntoIterator>) -> Mult /// Two million coordinates occupy roughly 32 MB before containers and overlay scratch. const MAX_BUFFER_VERTICES: usize = 2_000_000; /// Retained ordinary output/selector storage, separate from overlay complexity. Temporary -/// geometry on each worker and the already loaded input are additional to this allowance. +/// geometry on each worker and the already loaded input are additional to this allowance. The +/// browser's budget, and the fewest bytes [`max_buffer_bytes`] allows on the desktop. const MAX_BUFFER_BYTES: usize = if cfg!(target_arch = "wasm32") { 128 << 20 } else { 512 << 20 }; +/// The most bytes [`max_buffer_bytes`] allows, however much memory the computer has. +const BUFFER_BYTES_CEILING: usize = 8 << 30; /// Drain results regularly instead of keeping a second whole-layer result array. const BUFFER_BATCH: usize = 256; /// Bound actual candidate checks across all targets for large distance-selector sets, with @@ -1254,6 +1257,18 @@ const BUFFER_BATCH: usize = 256; const MAX_SELECTOR_CHECKS: usize = 50_000_000; const BUFFER_STORAGE_ERROR: &str = "Buffer output or distance selectors exceed their memory budget; select fewer features"; +/// Buffer's retained-storage budget on this computer: on the desktop a quarter of its memory, from +/// [`MAX_BUFFER_BYTES`] up to [`BUFFER_BYTES_CEILING`] (#1291 step 2); fixed in the browser. +pub fn max_buffer_bytes() -> usize { + rustgis_data::memory::scaled(rustgis_data::memory::physical(), MAX_BUFFER_BYTES, BUFFER_BYTES_CEILING, 1) +} + +/// [`BUFFER_STORAGE_ERROR`] with the budget it ran out of, so a run refused here but not on a +/// larger computer says why. +fn with_budget(e: String, bytes: usize) -> String { + if e == BUFFER_STORAGE_ERROR { format!("{e} (this computer's budget is {} MiB)", bytes >> 20) } else { e } +} + /// Shared retained-storage accounting; workers charge payloads before handing them back. struct BufferStorage(std::sync::atomic::AtomicUsize); @@ -1346,7 +1361,8 @@ fn buffer_rows( /// Buffer with the output budget shared by all selected input features. fn buffer(inp: &dyn Inputs, p: &Json) -> Result { - buffer_with_limits(inp, p, MAX_BUFFER_VERTICES, MAX_BUFFER_BYTES) + let bytes = max_buffer_bytes(); + buffer_with_limits(inp, p, MAX_BUFFER_VERTICES, bytes).map_err(|e| with_budget(e, bytes)) } /// Independent complexity and retained-output budgets; injectable for small boundary tests. @@ -2451,7 +2467,8 @@ fn selector_heap_bytes(g: &geo::Geometry) -> usize { /// Select against nearby selectors, indexed in the projection used by the predicate and after /// any distance buffer. Inversion and input row order keep the exhaustive selection's semantics. fn select_by_location(inp: &dyn Inputs, p: &Json) -> Result { - select_by_location_with_limits(inp, p, MAX_BUFFER_VERTICES, MAX_BUFFER_BYTES) + let bytes = max_buffer_bytes(); + select_by_location_with_limits(inp, p, MAX_BUFFER_VERTICES, bytes).map_err(|e| with_budget(e, bytes)) } /// Bounds each distance-buffered selector's complexity and their combined retained storage. @@ -6632,6 +6649,16 @@ mod tests { assert!((pred_inter.bands[0].data[0] - 20.0).abs() < 1e-4, "expected 20.0 at (0.5, 0.5), got {}", pred_inter.bands[0].data[0]); } + /// A Buffer refused for memory names the budget on this computer (#1291 step 2), which on the + /// desktop follows its memory and is never below the fixed one; other errors are left as they are. + #[test] + fn buffer_budget_follows_the_computer_and_is_named() { + assert!(max_buffer_bytes() >= MAX_BUFFER_BYTES && max_buffer_bytes() <= BUFFER_BYTES_CEILING.max(MAX_BUFFER_BYTES)); + let e = with_budget(BUFFER_STORAGE_ERROR.to_string(), 512 << 20); + assert!(e.starts_with(BUFFER_STORAGE_ERROR) && e.contains("512 MiB"), "{e}"); + assert_eq!(with_budget("Distance must be a number".into(), 512 << 20), "Distance must be a number"); + } + #[test] fn kernel_density_analysis_tool_tests() { // Create point dataset with UTM projection diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index f5f49a5c41..d0331a5da7 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -180,6 +180,9 @@ fn environments(s: &mut Session, p: &Value) -> CmdResult { "rasterSamples": rustgis_data::raster::max_samples(), "outputPixels": rasterops::max_pixels(), "pagedFeatures": rustgis_data::ogc::max_paged_features(), + "densityCells": rustgis_analysis::max_density_cells(), + "bufferBytes": rustgis_analysis::max_buffer_bytes(), + "threadScratchBytes": rustgis_analysis::scratch_budget(), }); Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size, "limits": limits})) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a8e2fe7abd..523d9fa226 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6617,6 +6617,8 @@ fn analysis_workflows_models_scripts_and_raster_functions() { assert!(limit("rasterSamples") >= rustgis_data::raster::SAMPLES_FLOOR, "{env}"); assert!(limit("outputPixels") >= rustgis_analysis::rasterops::PIXELS_FLOOR, "{env}"); assert!(limit("pagedFeatures") >= rustgis_data::ogc::PAGED_FEATURES_FLOOR, "{env}"); + assert!(limit("densityCells") >= rustgis_analysis::DENSITY_CELLS_FLOOR, "{env}"); + assert!(limit("bufferBytes") >= 512 << 20 && limit("threadScratchBytes") >= 256 << 20, "{env}"); // Network route along the rivers: two points on the same river. let rivers = s.map().layer_by_name("Rivers").unwrap(); let (a, b2) = match rivers.dataset.features.iter().filter_map(|f| f.geometry.as_ref()).find_map(|g| match g { diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 80049b20d9..f6cdc48f4d 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -250,7 +250,8 @@ to 0.031 % of the distance inside the true curve in the projection plane, and up polygon corners and line joins, in addition to the projection error. Negative Buffer distances shrink polygons. Buffer and Select Layer By Location limit distances to 5,000 km. Ordinary Buffer limits each feature to 2,000,000 outline vertices. Its retained output has a -separate memory budget: 512 MiB on desktop, 128 MiB in the browser, including geometry +separate memory budget: on the desktop a quarter of the computer's memory, from 512 MiB up to 8 GiB +(`bufferBytes` in `gp.env`), and 128 MiB in the browser, including geometry allocations, copied attributes and spare row capacity. Features are processed in bounded batches, in input order. Many small buffers can therefore exceed 2,000,000 vertices together. Dissolve All retains the shared 2,000,000-vertex limit before union and checks the dissolved @@ -875,6 +876,9 @@ below the web app's fixed limits and never above a ceiling. | `rasterSamples`: samples a raster read holds (cells times bands) | 120 million (32 million in the web app) | 8 bytes a sample | 1 billion | | `outputPixels`: pixels of a raster a tool makes | 64 million | 16 bytes a pixel | 512 million | | `pagedFeatures`: features read from a service page by page | 100,000 | 20 KB a feature (about 100,000 for every 8 GB) | 1 million | +| `densityCells`: cells of a Kernel Density raster | 16 million | 16 bytes a cell | 512 million | +| `bufferBytes`: memory for Buffer's output and Select Layer By Location's buffered selectors | 512 MiB (128 MiB in the web app) | the bytes themselves | 8 GiB | +| `threadScratchBytes`: working memory a tool's threads take together, beyond one thread's | 256 MiB | a sixteenth of memory, on top of the quarter | 4 GiB | So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster