From 2c2a1f020530011994a3490cfa80b29d497c4d64 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 11 Nov 2026 08:47:49 +0100 Subject: [PATCH 1/5] perf(analysis): Clip on several cores (#1252) --- crates/analysis/src/tools.rs | 260 ++++++++++++++++++++++++++--------- 1 file changed, 197 insertions(+), 63 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index ef9f6d6a5c..5e271ee352 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1641,33 +1641,101 @@ fn part(g: &Geometry, mask: &MultiPolygon, inside: bool) -> Option Result { + clip_on(inp, p, None, None) +} + +/// A clip mask feature as [`clip_on`] makes it: its polygons unioned, with its extent, or why it +/// is left out. +enum ClipMask { + Mask(MultiPolygon, Extent), + NoPolygon, + NoArea, +} + +/// The most multi-mask unions one thread of Clip keeps for features that meet the same masks. +const MAX_CACHED_MASKS: usize = 8; + +/// One thread's recent multi-mask unions for Clip, least recently used first, with the bytes +/// their coordinates take, kept within `cap` bytes. A union is the same however often it is made, +/// so what the cache keeps changes the time, not the output. +struct MaskCache { + entries: std::collections::VecDeque<(Vec, MultiPolygon, usize)>, + bytes: usize, + cap: usize, +} + +impl MaskCache { + fn new(cap: usize) -> Self { + MaskCache { entries: std::collections::VecDeque::new(), bytes: 0, cap } + } + + /// The union of the masks `near` names, from the cache, or made and kept there; one too big to + /// keep within the cap is made and handed over without being kept. + fn union_of(&mut self, near: &[usize], masks: &[(MultiPolygon, Extent)]) -> std::borrow::Cow<'_, MultiPolygon> { + let make = || union_all(near.iter().filter_map(|i| masks.get(*i)).flat_map(|(m, _)| m.0.iter())); + if let Some(pos) = self.entries.iter().position(|(ids, ..)| ids.as_slice() == near) { + if let Some(entry) = self.entries.remove(pos) { + self.entries.push_back(entry); + } + } else { + let mask = make(); + let coords: usize = mask.0.iter().map(|p| p.exterior().0.len() + p.interiors().iter().map(|r| r.0.len()).sum::()).sum(); + let bytes = coords.saturating_mul(std::mem::size_of::>()); + if bytes > self.cap { + return std::borrow::Cow::Owned(mask); + } + while self.entries.len() >= MAX_CACHED_MASKS || self.bytes.saturating_add(bytes) > self.cap { + let Some((.., b)) = self.entries.pop_front() else { break }; + self.bytes = self.bytes.saturating_sub(b); + } + self.bytes = self.bytes.saturating_add(bytes); + self.entries.push_back((near.to_vec(), mask, bytes)); + } + match self.entries.back() { + Some((_, mask, _)) => std::borrow::Cow::Borrowed(mask), + None => std::borrow::Cow::Owned(make()), + } + } +} + +/// [`clip`] on `threads` threads (by the number of items in each step when `None`): the masks are +/// unioned, and each input feature clipped, on their own (#1252), and the rows come back in input +/// order, as on one thread. Each thread keeps its own recent unions of several masks, all of them +/// within `cache_bytes` (by default [`crate::parallel::scratch_budget`]). +fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Option) -> Result { let ds = input(inp, p, "input")?; let clip_ds = input(inp, p, "clip")?; if clip_ds.geometry_type != GeometryType::Polygon { return Err(format!("'{}' is not a polygon layer", clip_ds.name)); } + let threads_for = |n: usize| threads.unwrap_or_else(|| crate::parallel::threads_for(n, PARALLEL_FEATURES)); + let mask_rows = each_once(rows(inp, p, "clip", clip_ds)); + // Each mask is a union: Cancel is looked at, and progress counted, before each. + crate::control::begin(mask_rows.len()); + let made = crate::parallel::try_map_on( + &mask_rows, + threads_for(mask_rows.len()), + || (), + |_, _, &r| { + let Some(m) = clip_ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(polygon_of) else { + return Ok::<_, String>(ClipMask::NoPolygon); + }; + let ext = merc_extent(&m); + // Union even one polygon: collapsed rings have no filled area. Keep the original + // bounds too: rounding an edge must not hide a line the boolean overlay clips to it. + let mask = union_all(m.0.iter()); + let ext = ext.union(&merc_extent(&mask)); + Ok(if mask.0.is_empty() { ClipMask::NoArea } else { ClipMask::Mask(mask, ext) }) + }, + )?; let mut masks = Vec::new(); // Mask features that clip nothing, left out and counted: no polygon, or one with no area. let mut left = LeftOut::default(); - let mask_rows = each_once(rows(inp, p, "clip", clip_ds)); - crate::control::begin(mask_rows.len()); - for (done, r) in mask_rows.into_iter().enumerate() { - // Each mask is a union: Cancel is looked at, and progress counted, before each. - crate::control::check()?; - crate::control::reached(done); - let Some(m) = clip_ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(polygon_of) else { - left.no_polygon += 1; - continue; - }; - let ext = merc_extent(&m); - // Union even one polygon: collapsed rings have no filled area. Keep the original - // bounds too: rounding an edge must not hide a line the boolean overlay clips to it. - let mask = union_all(m.0.iter()); - let ext = ext.union(&merc_extent(&mask)); - if mask.0.is_empty() { - left.no_area += 1; - } else { - masks.push((mask, ext)); + for m in made { + match m { + ClipMask::Mask(mask, ext) => masks.push((mask, ext)), + ClipMask::NoPolygon => left.no_polygon += 1, + ClipMask::NoArea => left.no_area += 1, } } if masks.is_empty() && left.no_polygon + left.no_area > 0 { @@ -1676,54 +1744,46 @@ fn clip(inp: &dyn Inputs, p: &Json) -> Result { let index = ExtentIndex::new(masks.iter().map(|(_, ext)| *ext)); // The masks reaching each pole: a point at one is in them at any longitude (#1033). let at_poles = AtPoles::new(masks.iter().map(|(m, _)| plane_poles(m))); - // Single candidates reuse their normalized mask. Retain eight recent multi-mask unions - // so alternating neighbourhoods can reuse work too; evict the least recently used. - const MAX_CACHED_MASKS: usize = 8; - let mut cached_masks: std::collections::VecDeque<(Vec, MultiPolygon)> = std::collections::VecDeque::new(); let mut out = copy_schema(ds, out_name(p, format!("{}_Clip", ds.name)), ds.geometry_type); let input_rows = rows(inp, p, "input", ds); crate::control::begin(input_rows.len()); - for (done, r) in input_rows.into_iter().enumerate() { - crate::control::check()?; - crate::control::reached(done); - let Some(f) = ds.features.get(r) else { continue }; - let Some(g) = f.geometry.as_ref() else { continue }; - let ext = projected_extent(g); - // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps - // for normalization and clipping: a line just outside an edge can snap onto it. - // This widens only the candidate search; the boolean overlay still decides the result. - let domain = index.total.union(&ext); - let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); - let ext = ext.padded(0.0, margin); - let mut near: Vec<_> = - index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); - if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { - at_poles.add_to(poles(&g.extent()), &mut near); - } - if near.is_empty() { - continue; - } - let mask = if let [i] = near.as_slice() { - let Some((m, _)) = masks.get(*i) else { continue }; - m - } else { - if let Some(pos) = cached_masks.iter().position(|(ids, _)| *ids == near) { - if let Some(entry) = cached_masks.remove(pos) { - cached_masks.push_back(entry); - } - } else { - if cached_masks.len() == MAX_CACHED_MASKS { - cached_masks.pop_front(); - } - let mask = union_all(near.iter().filter_map(|i| masks.get(*i)).flat_map(|(m, _)| m.0.iter())); - cached_masks.push_back((near, mask)); + let threads = threads_for(input_rows.len()); + // Single candidates reuse their normalized mask. Each thread keeps eight recent multi-mask + // unions, so alternating neighbourhoods can reuse work too, the least recently used going + // first; the threads' unions together stay within the scratch memory work may take. + let cache_cap = cache_bytes.unwrap_or_else(crate::parallel::scratch_budget) / threads.max(1); + let clipped = crate::parallel::try_map_on( + &input_rows, + threads, + || MaskCache::new(cache_cap), + |cache, _, &r| { + let Some(g) = ds.features.get(r).and_then(|f| f.geometry.as_ref()) else { return Ok::<_, String>(None) }; + let ext = projected_extent(g); + // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps + // for normalization and clipping: a line just outside an edge can snap onto it. + // This widens only the candidate search; the boolean overlay still decides the result. + let domain = index.total.union(&ext); + let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); + let ext = ext.padded(0.0, margin); + let mut near: Vec<_> = + index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); + if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { + at_poles.add_to(poles(&g.extent()), &mut near); } - let Some((_, mask)) = cached_masks.back() else { continue }; - mask - }; - if let Some(c) = part(&dissolved(g), mask, true) { - out.push(Some(c), f.attrs.clone()); - } + if near.is_empty() { + return Ok(None); + } + let shape = dissolved(g); + Ok(if let [i] = near.as_slice() { + masks.get(*i).and_then(|(m, _)| part(&shape, m, true)) + } else { + part(&shape, &cache.union_of(&near, &masks), true) + }) + }, + )?; + for (r, c) in input_rows.into_iter().zip(clipped) { + let (Some(f), Some(c)) = (ds.features.get(r), c) else { continue }; + out.push(Some(c), f.attrs.clone()); } let n = out.features.len(); let mut r = result(out, format!("Clip: {n} features kept"))?; @@ -11920,6 +11980,80 @@ mod tests { } } + /// Clip gives the same rows, bit for bit, on several threads as on one (#1252), and whatever its + /// threads' mask caches can keep (none, a few unions, the default): polygons of one and of + /// several parts, lines and points, features without geometry, masks that overlap so features + /// meet several (the same ones in a row, so a union comes from the cache), and masks left out + /// for no polygon. + #[test] + fn clip_on_several_threads_and_any_cache_is_on_one() { + let mut input = Dataset::new("input", GeometryType::Polygon, vec![Field::new("v", FieldType::Integer)]); + let mut lines = Dataset::new("lines", GeometryType::Polyline, vec![Field::new("v", FieldType::Integer)]); + let mut points = Dataset::new("points", GeometryType::Point, vec![Field::new("v", FieldType::Integer)]); + let mut masks = Dataset::new("masks", GeometryType::Polygon, vec![Field::new("v", FieldType::Integer)]); + for i in 0..600i64 { + let (x, y) = ((i * 37 % 300) as f64 * 0.1, (i * 53 % 120) as f64 * 0.1); + let none = i % 97 == 0; + let poly = if i % 7 == 0 { + let ring = |x: f64| { + vec![Coord::new(x, y), Coord::new(x + 0.4, y), Coord::new(x + 0.4, y + 0.4), Coord::new(x, y + 0.4), Coord::new(x, y)] + }; + Geometry::Polygon(vec![vec![ring(x)], vec![ring(x + 0.25)]]) + } else { + sq(x, y, 0.35) + }; + input.push(if none { None } else { Some(poly) }, vec![Value::Int(i)]); + let line = Geometry::Line(vec![vec![Coord::new(x, y), Coord::new(x + 0.9, y + 0.3)]]); + lines.push(if none { None } else { Some(line) }, vec![Value::Int(i)]); + points.push(if none { None } else { Some(Geometry::Point(Coord::new(x + 0.05, y + 0.05))) }, vec![Value::Int(i)]); + } + for i in 0..200i64 { + let (x, y) = ((i * 71 % 290) as f64 * 0.1, (i * 29 % 110) as f64 * 0.1); + masks.push(if i % 53 == 0 { None } else { Some(sq(x, y, 0.6)) }, vec![Value::Int(1000 + i)]); + } + // Away from the rest, two overlapping masks and five features in a row meeting both, so + // a thread's cache gives back the union it made for the one before. + masks.push(Some(sq(40.0, 0.0, 0.6)), vec![Value::Int(2000)]); + masks.push(Some(sq(40.3, 0.0, 0.6)), vec![Value::Int(2001)]); + for i in 600..605i64 { + input.push(Some(sq(40.2, 0.1, 0.35)), vec![Value::Int(i)]); + lines.push(Some(Geometry::Line(vec![vec![Coord::new(40.1, 0.1), Coord::new(40.8, 0.3)]])), vec![Value::Int(i)]); + points.push(Some(Geometry::Point(Coord::new(40.4, 0.2))), vec![Value::Int(i)]); + } + let m = Map(HashMap::from([ + ("input".to_string(), input), + ("lines".to_string(), lines), + ("points".to_string(), points), + ("masks".to_string(), masks), + ])); + for name in ["input", "lines", "points"] { + let p = serde_json::json!({"input": name, "clip": "masks"}); + let one = clip_on(&m, &p, Some(1), None).unwrap(); + let one_messages = one.messages.clone(); + assert!(one_messages.iter().any(|m| m.contains("left out")), "{name}: {one_messages:?}"); + let one = layer(one); + assert!(one.features.len() > 40, "{name}: {}", one.features.len()); + // The five features meeting both masks at the end are kept. + let last: Vec<_> = one.features.iter().rev().take(5).map(|f| f.attrs.first().cloned()).collect(); + assert_eq!(last, (600..605).rev().map(|i| Some(Value::Int(i))).collect::>(), "{name}"); + for threads in [1, 2, 7, 16] { + for cache in [Some(0), Some(4_000), None] { + let many = clip_on(&m, &p, Some(threads), cache).unwrap(); + assert_eq!(many.messages, one_messages, "{name}, {threads} threads, cache {cache:?}"); + let many = layer(many); + assert_eq!(many.features.len(), one.features.len(), "{name}, {threads} threads, cache {cache:?}"); + for (a, b) in one.features.iter().zip(&many.features) { + assert_eq!( + format!("{:?}", (&a.geometry, &a.attrs)), + format!("{:?}", (&b.geometry, &b.attrs)), + "{name}, {threads} threads, cache {cache:?}" + ); + } + } + } + } + } + /// Spatial Join (every match mode, one to one with merge rules, one to many, and only the /// matching targets), Summarize Within (with statistics) and Simplify give the same rows, bit /// for bit, on several threads as on one (#1252), with features without geometry among them From 63b1978559cf129d2dce91fa4c29cf77a7dc16b0 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 11 Nov 2026 09:35:26 +0100 Subject: [PATCH 2/5] fix(analysis): Clip bounds its threads by memory, adds rows batch by batch, and says which features have no geometry --- crates/analysis/src/tools.rs | 200 +++++++++++++++++++++++++---------- 1 file changed, 144 insertions(+), 56 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 5e271ee352..5035a53fd2 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -12,7 +12,7 @@ use rustgis_data::{Coord, Dataset, Feature, Field, FieldType, Geometry, Geometry use rustgis_geom::Extent; use serde_json::Value as Json; use std::borrow::Cow; -use std::collections::{HashMap, HashSet, TryReserveError}; +use std::collections::{HashMap, HashSet, TryReserveError, VecDeque}; use std::ops::ControlFlow; use crate::convert::{from_line, from_multipolygon, from_point, reaches_poles, to_geo, to_multiline, to_multipolygon}; @@ -1652,37 +1652,78 @@ enum ClipMask { NoArea, } +/// The input features each thread of Clip takes per batch: their clipped parts are held until the +/// batch's rows are added to the output, and a thread's recent unions are made afresh for each +/// batch. Small in tests, so they cross from one batch to the next. +const CLIP_BATCH: usize = if cfg!(test) { 16 } else { 4096 }; + /// The most multi-mask unions one thread of Clip keeps for features that meet the same masks. const MAX_CACHED_MASKS: usize = 8; -/// One thread's recent multi-mask unions for Clip, least recently used first, with the bytes -/// their coordinates take, kept within `cap` bytes. A union is the same however often it is made, -/// so what the cache keeps changes the time, not the output. +/// The memory a union kept by Clip holds, about: its coordinates, its rings' and polygons' own +/// parts, and the key naming its masks. +fn kept_union_bytes(key: &[usize], m: &MultiPolygon) -> usize { + use std::mem::size_of; + let coords = |r: &geo::LineString| r.0.capacity().saturating_mul(size_of::>()); + // A polygon's own size holds its outer ring's and its holes' headers; each hole's header is + // in the holes' list. + let polygon = |p: &geo::Polygon| { + p.interiors() + .iter() + .map(|r| coords(r).saturating_add(size_of::>())) + .fold(coords(p.exterior()), usize::saturating_add) + }; + m.0.iter() + .map(polygon) + .fold(m.0.capacity().saturating_mul(size_of::>()), usize::saturating_add) + .saturating_add(std::mem::size_of_val(key)) + .saturating_add(size_of::>()) +} + +/// The memory a feature's coordinates take, about, as a mask or an overlay holds them. +fn coord_bytes(g: &Geometry) -> usize { + g.coord_count().saturating_mul(std::mem::size_of::>()) +} + +/// One thread's recent multi-mask unions for Clip, least recently used first, with the memory +/// each holds ([`kept_union_bytes`]), kept within `cap` bytes; and the latest union too big for +/// that, kept on its own while the features next to each other meet it, so they share it. That +/// one is outside the cap, as the union a feature is clipped by on one thread is: a thread holds +/// one at a time. A union is the same however often it is made, so what the cache keeps changes +/// the time, not the output. struct MaskCache { - entries: std::collections::VecDeque<(Vec, MultiPolygon, usize)>, + entries: VecDeque<(Vec, MultiPolygon, usize)>, bytes: usize, cap: usize, + big: Option<(Vec, MultiPolygon)>, } impl MaskCache { fn new(cap: usize) -> Self { - MaskCache { entries: std::collections::VecDeque::new(), bytes: 0, cap } + MaskCache { entries: VecDeque::new(), bytes: 0, cap, big: None } } - /// The union of the masks `near` names, from the cache, or made and kept there; one too big to - /// keep within the cap is made and handed over without being kept. - fn union_of(&mut self, near: &[usize], masks: &[(MultiPolygon, Extent)]) -> std::borrow::Cow<'_, MultiPolygon> { + /// The union of the masks `near` names, from the cache, or made and kept there. + fn union_of(&mut self, near: &[usize], masks: &[(MultiPolygon, Extent)]) -> Cow<'_, MultiPolygon> { let make = || union_all(near.iter().filter_map(|i| masks.get(*i)).flat_map(|(m, _)| m.0.iter())); + if self.big.as_ref().is_some_and(|(ids, _)| ids.as_slice() == near) { + return match &self.big { + Some((_, mask)) => Cow::Borrowed(mask), + None => Cow::Owned(make()), + }; + } if let Some(pos) = self.entries.iter().position(|(ids, ..)| ids.as_slice() == near) { if let Some(entry) = self.entries.remove(pos) { self.entries.push_back(entry); } } else { + // The big one goes first, so no more than one is held while the next is made. + self.big = None; let mask = make(); - let coords: usize = mask.0.iter().map(|p| p.exterior().0.len() + p.interiors().iter().map(|r| r.0.len()).sum::()).sum(); - let bytes = coords.saturating_mul(std::mem::size_of::>()); + let bytes = kept_union_bytes(near, &mask); if bytes > self.cap { - return std::borrow::Cow::Owned(mask); + let (_, mask) = self.big.insert((near.to_vec(), mask)); + return Cow::Borrowed(mask); } while self.entries.len() >= MAX_CACHED_MASKS || self.bytes.saturating_add(bytes) > self.cap { let Some((.., b)) = self.entries.pop_front() else { break }; @@ -1691,30 +1732,41 @@ impl MaskCache { self.bytes = self.bytes.saturating_add(bytes); self.entries.push_back((near.to_vec(), mask, bytes)); } + // The entry asked for is the last, as just moved or put there; `back` can't be empty + // here, but without a way to say so that can't panic, an empty one is made again. match self.entries.back() { - Some((_, mask, _)) => std::borrow::Cow::Borrowed(mask), - None => std::borrow::Cow::Owned(make()), + Some((_, mask, _)) => Cow::Borrowed(mask), + None => Cow::Owned(make()), } } } -/// [`clip`] on `threads` threads (by the number of items in each step when `None`): the masks are -/// unioned, and each input feature clipped, on their own (#1252), and the rows come back in input -/// order, as on one thread. Each thread keeps its own recent unions of several masks, all of them -/// within `cache_bytes` (by default [`crate::parallel::scratch_budget`]). +/// [`clip`] on `threads` threads (by the number of items in each step, and the memory each +/// takes, when `None`): the masks are unioned, and each input feature clipped, on their own +/// (#1252), in batches of [`CLIP_BATCH`] features a thread, and the rows come back in input +/// order, as on one thread. A thread's work in progress takes about twice its largest mask's and +/// input feature's coordinates; the threads are as many as half the scratch memory work may take +/// holds of that, and their recent unions of several masks share `cache_bytes` (by default the +/// other half of [`crate::parallel::scratch_budget`]). fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Option) -> Result { let ds = input(inp, p, "input")?; let clip_ds = input(inp, p, "clip")?; if clip_ds.geometry_type != GeometryType::Polygon { return Err(format!("'{}' is not a polygon layer", clip_ds.name)); } - let threads_for = |n: usize| threads.unwrap_or_else(|| crate::parallel::threads_for(n, PARALLEL_FEATURES)); + let threads_for = |n: usize, scratch: usize| { + threads.unwrap_or_else(|| crate::parallel::threads_for_scratch(n, PARALLEL_FEATURES, scratch.saturating_mul(2))) + }; let mask_rows = each_once(rows(inp, p, "clip", clip_ds)); + let largest = |ds: &Dataset, rows: &[usize]| { + rows.iter().filter_map(|&r| ds.features.get(r)?.geometry.as_ref()).map(coord_bytes).max().unwrap_or(0) + }; + let largest_mask = largest(clip_ds, &mask_rows); // Each mask is a union: Cancel is looked at, and progress counted, before each. crate::control::begin(mask_rows.len()); let made = crate::parallel::try_map_on( &mask_rows, - threads_for(mask_rows.len()), + threads_for(mask_rows.len(), largest_mask.saturating_mul(2)), || (), |_, _, &r| { let Some(m) = clip_ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(polygon_of) else { @@ -1746,47 +1798,59 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti let at_poles = AtPoles::new(masks.iter().map(|(m, _)| plane_poles(m))); let mut out = copy_schema(ds, out_name(p, format!("{}_Clip", ds.name)), ds.geometry_type); let input_rows = rows(inp, p, "input", ds); + // Input features without geometry: nothing of them can be kept. + let without = input_rows.iter().filter(|&&r| ds.features.get(r).is_some_and(|f| f.geometry.is_none())).count(); crate::control::begin(input_rows.len()); - let threads = threads_for(input_rows.len()); + let work = largest(ds, &input_rows).saturating_add(largest_mask).saturating_mul(2); + let clip_threads = threads_for(input_rows.len(), work); // Single candidates reuse their normalized mask. Each thread keeps eight recent multi-mask // unions, so alternating neighbourhoods can reuse work too, the least recently used going - // first; the threads' unions together stay within the scratch memory work may take. - let cache_cap = cache_bytes.unwrap_or_else(crate::parallel::scratch_budget) / threads.max(1); - let clipped = crate::parallel::try_map_on( - &input_rows, - threads, - || MaskCache::new(cache_cap), - |cache, _, &r| { - let Some(g) = ds.features.get(r).and_then(|f| f.geometry.as_ref()) else { return Ok::<_, String>(None) }; - let ext = projected_extent(g); - // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps - // for normalization and clipping: a line just outside an edge can snap onto it. - // This widens only the candidate search; the boolean overlay still decides the result. - let domain = index.total.union(&ext); - let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); - let ext = ext.padded(0.0, margin); - let mut near: Vec<_> = - index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); - if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { - at_poles.add_to(poles(&g.extent()), &mut near); - } - if near.is_empty() { - return Ok(None); - } - let shape = dissolved(g); - Ok(if let [i] = near.as_slice() { - masks.get(*i).and_then(|(m, _)| part(&shape, m, true)) - } else { - part(&shape, &cache.union_of(&near, &masks), true) - }) - }, - )?; - for (r, c) in input_rows.into_iter().zip(clipped) { - let (Some(f), Some(c)) = (ds.features.get(r), c) else { continue }; - out.push(Some(c), f.attrs.clone()); + // first; the threads' unions together stay within their share of the scratch memory. + let cache_cap = cache_bytes.unwrap_or_else(|| crate::parallel::scratch_budget() / 2) / clip_threads.max(1); + // In batches, so the clipped features held before they're added to the output stay few. + for batch in input_rows.chunks(clip_threads.max(1).saturating_mul(CLIP_BATCH)) { + let clipped = crate::parallel::try_map_on( + batch, + clip_threads, + || MaskCache::new(cache_cap), + |cache, _, &r| { + let Some(g) = ds.features.get(r).and_then(|f| f.geometry.as_ref()) else { return Ok::<_, String>(None) }; + let ext = projected_extent(g); + // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps + // for normalization and clipping: a line just outside an edge can snap onto it. + // This widens only the candidate search; the boolean overlay still decides the result. + let domain = index.total.union(&ext); + let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); + let ext = ext.padded(0.0, margin); + let mut near: Vec<_> = + index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); + if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { + at_poles.add_to(poles(&g.extent()), &mut near); + } + if near.is_empty() { + cache.big = None; + return Ok(None); + } + let shape = dissolved(g); + Ok(if let [i] = near.as_slice() { + // A big union is kept only while the features next to each other need it. + cache.big = None; + masks.get(*i).and_then(|(m, _)| part(&shape, m, true)) + } else { + part(&shape, &cache.union_of(&near, &masks), true) + }) + }, + )?; + for (&r, c) in batch.iter().zip(clipped) { + let (Some(f), Some(c)) = (ds.features.get(r), c) else { continue }; + out.push(Some(c), f.attrs.clone()); + } } let n = out.features.len(); let mut r = result(out, format!("Clip: {n} features kept"))?; + if without > 0 { + r.messages.push(format!("Clip: {without} feature(s) of '{}' left out: they have no geometry to clip", ds.name)); + } r.messages.extend(left.notes("Clip", &clip_ds.name)); Ok(r) } @@ -11980,6 +12044,29 @@ mod tests { } } + /// Clip says how many input features it left out for having no geometry, and says nothing of + /// it when all have one. + #[test] + fn clip_says_which_input_features_have_no_geometry() { + let mut input = Dataset::new("input", GeometryType::Polygon, vec![]); + input.push(Some(sq(0.0, 0.0, 1.0)), vec![]); + input.push(None, vec![]); + input.push(None, vec![]); + let mut clean = Dataset::new("clean", GeometryType::Polygon, vec![]); + clean.push(Some(sq(0.0, 0.0, 1.0)), vec![]); + let mut masks = Dataset::new("masks", GeometryType::Polygon, vec![]); + masks.push(Some(sq(0.5, 0.5, 1.0)), vec![]); + let m = Map(HashMap::from([("input".to_string(), input), ("clean".to_string(), clean), ("masks".to_string(), masks)])); + let lossy = clip(&m, &serde_json::json!({"input": "input", "clip": "masks"})).unwrap(); + assert!( + lossy.messages.iter().any(|m| m == "Clip: 2 feature(s) of 'input' left out: they have no geometry to clip"), + "{:?}", + lossy.messages + ); + let clean = clip(&m, &serde_json::json!({"input": "clean", "clip": "masks"})).unwrap(); + assert!(!clean.messages.iter().any(|m| m.contains("no geometry")), "{:?}", clean.messages); + } + /// Clip gives the same rows, bit for bit, on several threads as on one (#1252), and whatever its /// threads' mask caches can keep (none, a few unions, the default): polygons of one and of /// several parts, lines and points, features without geometry, masks that overlap so features @@ -12012,7 +12099,8 @@ mod tests { masks.push(if i % 53 == 0 { None } else { Some(sq(x, y, 0.6)) }, vec![Value::Int(1000 + i)]); } // Away from the rest, two overlapping masks and five features in a row meeting both, so - // a thread's cache gives back the union it made for the one before. + // the cache gives back the union it made for the one before (on one thread; on several, + // where the five fall in one thread's share). masks.push(Some(sq(40.0, 0.0, 0.6)), vec![Value::Int(2000)]); masks.push(Some(sq(40.3, 0.0, 0.6)), vec![Value::Int(2001)]); for i in 600..605i64 { From e18821632c3e261d7c34956c3dfa98f61a7cc1f6 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 11 Nov 2026 10:08:25 +0100 Subject: [PATCH 3/5] fix(analysis): Clip counts the union of the masks a feature meets in its threads' memory --- crates/analysis/src/tools.rs | 103 +++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 36 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 5035a53fd2..fa8202bcbd 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1687,10 +1687,10 @@ fn coord_bytes(g: &Geometry) -> usize { /// One thread's recent multi-mask unions for Clip, least recently used first, with the memory /// each holds ([`kept_union_bytes`]), kept within `cap` bytes; and the latest union too big for -/// that, kept on its own while the features next to each other meet it, so they share it. That -/// one is outside the cap, as the union a feature is clipped by on one thread is: a thread holds -/// one at a time. A union is the same however often it is made, so what the cache keeps changes -/// the time, not the output. +/// that, kept on its own until a feature wants another union, or meets one mask or none, so +/// the features next to each other that meet it share it. That one is outside the cap: it is +/// counted as the thread's work in progress, as the union a feature is clipped by is. A union is +/// the same however often it is made, so what the cache keeps changes the time, not the output. struct MaskCache { entries: VecDeque<(Vec, MultiPolygon, usize)>, bytes: usize, @@ -1712,13 +1712,14 @@ impl MaskCache { None => Cow::Owned(make()), }; } + // Another union is wanted: the big one goes, so no more than one is held while the next + // is made, and none is kept for features that no longer meet it. + self.big = None; if let Some(pos) = self.entries.iter().position(|(ids, ..)| ids.as_slice() == near) { if let Some(entry) = self.entries.remove(pos) { self.entries.push_back(entry); } } else { - // The big one goes first, so no more than one is held while the next is made. - self.big = None; let mask = make(); let bytes = kept_union_bytes(near, &mask); if bytes > self.cap { @@ -1744,16 +1745,19 @@ impl MaskCache { /// [`clip`] on `threads` threads (by the number of items in each step, and the memory each /// takes, when `None`): the masks are unioned, and each input feature clipped, on their own /// (#1252), in batches of [`CLIP_BATCH`] features a thread, and the rows come back in input -/// order, as on one thread. A thread's work in progress takes about twice its largest mask's and -/// input feature's coordinates; the threads are as many as half the scratch memory work may take -/// holds of that, and their recent unions of several masks share `cache_bytes` (by default the -/// other half of [`crate::parallel::scratch_budget`]). +/// order, as on one thread. A thread's work in progress takes about twice the largest mask's +/// coordinates while the masks are unioned, then twice the largest input feature's plus the +/// largest union of the masks one feature meets; the threads are as many as half the scratch +/// memory work may take holds of that, and their recent unions of several masks share +/// `cache_bytes` (by default the other half of [`crate::parallel::scratch_budget`]). fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Option) -> Result { let ds = input(inp, p, "input")?; let clip_ds = input(inp, p, "clip")?; if clip_ds.geometry_type != GeometryType::Polygon { return Err(format!("'{}' is not a polygon layer", clip_ds.name)); } + // A thread's work in progress is `scratch`; doubled here, so the threads' work takes at most + // half the scratch memory (`threads_for_scratch` divides all of it), the caches the other half. let threads_for = |n: usize, scratch: usize| { threads.unwrap_or_else(|| crate::parallel::threads_for_scratch(n, PARALLEL_FEATURES, scratch.saturating_mul(2))) }; @@ -1762,7 +1766,8 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti rows.iter().filter_map(|&r| ds.features.get(r)?.geometry.as_ref()).map(coord_bytes).max().unwrap_or(0) }; let largest_mask = largest(clip_ds, &mask_rows); - // Each mask is a union: Cancel is looked at, and progress counted, before each. + // Each mask is a union: Cancel is looked at, and progress counted, before each. A thread + // holds a mask's polygons and their union. crate::control::begin(mask_rows.len()); let made = crate::parallel::try_map_on( &mask_rows, @@ -1800,44 +1805,70 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti let input_rows = rows(inp, p, "input", ds); // Input features without geometry: nothing of them can be kept. let without = input_rows.iter().filter(|&&r| ds.features.get(r).is_some_and(|f| f.geometry.is_none())).count(); - crate::control::begin(input_rows.len()); - let work = largest(ds, &input_rows).saturating_add(largest_mask).saturating_mul(2); - let clip_threads = threads_for(input_rows.len(), work); - // Single candidates reuse their normalized mask. Each thread keeps eight recent multi-mask - // unions, so alternating neighbourhoods can reuse work too, the least recently used going - // first; the threads' unions together stay within their share of the scratch memory. - let cache_cap = cache_bytes.unwrap_or_else(|| crate::parallel::scratch_budget() / 2) / clip_threads.max(1); + // The masks whose extents meet a feature's, and the poles' for a point at one. + let near_of = |g: &Geometry| { + let ext = projected_extent(g); + // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps + // for normalization and clipping: a line just outside an edge can snap onto it. + // This widens only the candidate search; the boolean overlay still decides the result. + let domain = index.total.union(&ext); + let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); + let ext = ext.padded(0.0, margin); + let mut near: Vec<_> = + index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); + if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { + at_poles.add_to(poles(&g.extent()), &mut near); + } + near + }; + // A thread's work in progress: a feature, its clipped part, and the union of the masks it + // meets when they are several, which can be far larger than any one mask (at most all of + // them). + let mask_bytes: Vec = masks.iter().map(|(m, _)| kept_union_bytes(&[], m)).collect(); + let union_bytes = |near: &[usize]| match near { + [] | [_] => 0, + _ => near.iter().filter_map(|&i| mask_bytes.get(i)).fold(0usize, |a, &b| a.saturating_add(b)), + }; + let feature_work = largest(ds, &input_rows).saturating_mul(2); + // As many threads as there could be, if no feature met several masks: they size the batches. + let most_threads = threads_for(input_rows.len(), feature_work.saturating_add(largest_mask)).max(1); + // Each feature is looked at twice: for the masks it meets, then to be clipped. + crate::control::begin(input_rows.len().saturating_mul(2)); // In batches, so the clipped features held before they're added to the output stay few. - for batch in input_rows.chunks(clip_threads.max(1).saturating_mul(CLIP_BATCH)) { - let clipped = crate::parallel::try_map_on( + for batch in input_rows.chunks(most_threads.saturating_mul(CLIP_BATCH)) { + let nears = crate::parallel::try_map_on( batch, + most_threads, + || (), + |_, _, &r| Ok::<_, String>(ds.features.get(r).and_then(|f| f.geometry.as_ref()).map(near_of)), + )?; + // The batch's threads, by its largest union too. + let largest_union = nears.iter().flatten().map(|near| union_bytes(near)).max().unwrap_or(0); + let clip_threads = threads_for(batch.len(), feature_work.saturating_add(largest_mask.max(largest_union))).min(most_threads); + // Single candidates reuse their normalized mask. Each thread keeps eight recent multi-mask + // unions, so alternating neighbourhoods can reuse work too, the least recently used going + // first; the threads' unions together stay within their share of the scratch memory. + let cache_cap = cache_bytes.unwrap_or_else(|| crate::parallel::scratch_budget() / 2) / clip_threads.max(1); + let work: Vec<(usize, Option>)> = batch.iter().copied().zip(nears).collect(); + let clipped = crate::parallel::try_map_on( + &work, clip_threads, || MaskCache::new(cache_cap), - |cache, _, &r| { - let Some(g) = ds.features.get(r).and_then(|f| f.geometry.as_ref()) else { return Ok::<_, String>(None) }; - let ext = projected_extent(g); - // i_overlay uses 29 integer bits about the domain centre. Allow several grid steps - // for normalization and clipping: a line just outside an edge can snap onto it. - // This widens only the candidate search; the boolean overlay still decides the result. - let domain = index.total.union(&ext); - let margin = domain.width().max(domain.height()) / f64::from(1_u32 << 27); - let ext = ext.padded(0.0, margin); - let mut near: Vec<_> = - index.candidates(&ext).into_iter().filter(|i| masks.get(*i).is_some_and(|(_, e)| e.intersects(&ext))).collect(); - if matches!(g, Geometry::Point(_) | Geometry::MultiPoint(_)) { - at_poles.add_to(poles(&g.extent()), &mut near); - } + |cache, _, (r, near)| { + let (Some(g), Some(near)) = (ds.features.get(*r).and_then(|f| f.geometry.as_ref()), near) else { + return Ok::<_, String>(None); + }; if near.is_empty() { + // A big union is kept only while the features next to each other meet it. cache.big = None; return Ok(None); } let shape = dissolved(g); Ok(if let [i] = near.as_slice() { - // A big union is kept only while the features next to each other need it. cache.big = None; masks.get(*i).and_then(|(m, _)| part(&shape, m, true)) } else { - part(&shape, &cache.union_of(&near, &masks), true) + part(&shape, &cache.union_of(near, &masks), true) }) }, )?; From 10771a28f95769fbff1129700e80165e27e00915 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 11 Nov 2026 10:40:41 +0100 Subject: [PATCH 4/5] fix(analysis): Clip keeps only union sizes between its passes and counts empty input geometries --- crates/analysis/src/tools.rs | 43 +++++++++++++++++++++--------------- docs/user-guide/analysis.md | 3 ++- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index fa8202bcbd..807c3c1d6b 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1746,8 +1746,9 @@ impl MaskCache { /// takes, when `None`): the masks are unioned, and each input feature clipped, on their own /// (#1252), in batches of [`CLIP_BATCH`] features a thread, and the rows come back in input /// order, as on one thread. A thread's work in progress takes about twice the largest mask's -/// coordinates while the masks are unioned, then twice the largest input feature's plus the -/// largest union of the masks one feature meets; the threads are as many as half the scratch +/// coordinates while the masks are unioned, then twice the largest input feature's, a list of +/// the masks, and the larger of the largest mask and the largest union of the masks one feature +/// meets (found for each batch first); the threads are as many as half the scratch /// memory work may take holds of that, and their recent unions of several masks share /// `cache_bytes` (by default the other half of [`crate::parallel::scratch_budget`]). fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Option) -> Result { @@ -1804,7 +1805,11 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti let mut out = copy_schema(ds, out_name(p, format!("{}_Clip", ds.name)), ds.geometry_type); let input_rows = rows(inp, p, "input", ds); // Input features without geometry: nothing of them can be kept. - let without = input_rows.iter().filter(|&&r| ds.features.get(r).is_some_and(|f| f.geometry.is_none())).count(); + // Input features without geometry, or with an empty one: nothing of them can be kept. + let without = input_rows + .iter() + .filter(|&&r| ds.features.get(r).is_some_and(|f| f.geometry.as_ref().is_none_or(|g| g.coord_count() == 0))) + .count(); // The masks whose extents meet a feature's, and the poles' for a point at one. let near_of = |g: &Geometry| { let ext = projected_extent(g); @@ -1829,35 +1834,36 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti [] | [_] => 0, _ => near.iter().filter_map(|&i| mask_bytes.get(i)).fold(0usize, |a, &b| a.saturating_add(b)), }; - let feature_work = largest(ds, &input_rows).saturating_mul(2); + // A thread's work in progress: a feature and its clipped part, the list of the masks it meets + // (at most all of them), and the larger of a mask and the union of the masks it meets. + let feature_work = largest(ds, &input_rows).saturating_mul(2).saturating_add(masks.len().saturating_mul(std::mem::size_of::())); // As many threads as there could be, if no feature met several masks: they size the batches. let most_threads = threads_for(input_rows.len(), feature_work.saturating_add(largest_mask)).max(1); - // Each feature is looked at twice: for the masks it meets, then to be clipped. + // Each feature is looked at twice: for the size of the union of the masks it meets, then to + // be clipped. Only those sizes are kept between the two, not the masks. crate::control::begin(input_rows.len().saturating_mul(2)); // In batches, so the clipped features held before they're added to the output stay few. for batch in input_rows.chunks(most_threads.saturating_mul(CLIP_BATCH)) { - let nears = crate::parallel::try_map_on( + let unions = crate::parallel::try_map_on( batch, most_threads, || (), - |_, _, &r| Ok::<_, String>(ds.features.get(r).and_then(|f| f.geometry.as_ref()).map(near_of)), + |_, _, &r| Ok::<_, String>(ds.features.get(r).and_then(|f| f.geometry.as_ref()).map_or(0, |g| union_bytes(&near_of(g)))), )?; // The batch's threads, by its largest union too. - let largest_union = nears.iter().flatten().map(|near| union_bytes(near)).max().unwrap_or(0); + let largest_union = unions.into_iter().max().unwrap_or(0); let clip_threads = threads_for(batch.len(), feature_work.saturating_add(largest_mask.max(largest_union))).min(most_threads); // Single candidates reuse their normalized mask. Each thread keeps eight recent multi-mask // unions, so alternating neighbourhoods can reuse work too, the least recently used going // first; the threads' unions together stay within their share of the scratch memory. let cache_cap = cache_bytes.unwrap_or_else(|| crate::parallel::scratch_budget() / 2) / clip_threads.max(1); - let work: Vec<(usize, Option>)> = batch.iter().copied().zip(nears).collect(); let clipped = crate::parallel::try_map_on( - &work, + batch, clip_threads, || MaskCache::new(cache_cap), - |cache, _, (r, near)| { - let (Some(g), Some(near)) = (ds.features.get(*r).and_then(|f| f.geometry.as_ref()), near) else { - return Ok::<_, String>(None); - }; + |cache, _, &r| { + let Some(g) = ds.features.get(r).and_then(|f| f.geometry.as_ref()) else { return Ok::<_, String>(None) }; + let near = near_of(g); if near.is_empty() { // A big union is kept only while the features next to each other meet it. cache.big = None; @@ -1868,7 +1874,7 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti cache.big = None; masks.get(*i).and_then(|(m, _)| part(&shape, m, true)) } else { - part(&shape, &cache.union_of(near, &masks), true) + part(&shape, &cache.union_of(&near, &masks), true) }) }, )?; @@ -12075,14 +12081,15 @@ mod tests { } } - /// Clip says how many input features it left out for having no geometry, and says nothing of - /// it when all have one. + /// Clip says how many input features it left out for having no geometry, or an empty one, and + /// says nothing of it when all have one. #[test] fn clip_says_which_input_features_have_no_geometry() { let mut input = Dataset::new("input", GeometryType::Polygon, vec![]); input.push(Some(sq(0.0, 0.0, 1.0)), vec![]); input.push(None, vec![]); input.push(None, vec![]); + input.push(Some(Geometry::Polygon(vec![])), vec![]); let mut clean = Dataset::new("clean", GeometryType::Polygon, vec![]); clean.push(Some(sq(0.0, 0.0, 1.0)), vec![]); let mut masks = Dataset::new("masks", GeometryType::Polygon, vec![]); @@ -12090,7 +12097,7 @@ mod tests { let m = Map(HashMap::from([("input".to_string(), input), ("clean".to_string(), clean), ("masks".to_string(), masks)])); let lossy = clip(&m, &serde_json::json!({"input": "input", "clip": "masks"})).unwrap(); assert!( - lossy.messages.iter().any(|m| m == "Clip: 2 feature(s) of 'input' left out: they have no geometry to clip"), + lossy.messages.iter().any(|m| m == "Clip: 3 feature(s) of 'input' left out: they have no geometry to clip"), "{:?}", lossy.messages ); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 12a89586e2..7efd4ddc49 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -623,7 +623,8 @@ feature of the clip, erase, overlay, update or split layer with no polygon, and without geometry, a point or line in the polygon layer (as Minimum Bounding Geometry's flat hulls are), or an empty polygon. Clip also leaves out a mask whose polygons, unioned, have no area, and the others a feature of several parts that, unioned, come to nothing (Update writes it as it is), -saying how many. When none of the features of that layer is left, Clip, Intersect and Split, whose +saying how many. Clip also says how many input features it leaves out for having no geometry, or +an empty one. When none of the features of that layer is left, Clip, Intersect and Split, whose result would be empty, stop with an error. In all of them, a feature selected twice in that layer is used once. From 2b4824aba54db08b8474b3d57e89bc21d06e9c5a Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Wed, 11 Nov 2026 11:01:22 +0100 Subject: [PATCH 5/5] fix(analysis): Clip charges the index's references a lookup collects to its threads' memory --- crates/analysis/src/tools.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 807c3c1d6b..c35966051c 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1747,8 +1747,9 @@ impl MaskCache { /// (#1252), in batches of [`CLIP_BATCH`] features a thread, and the rows come back in input /// order, as on one thread. A thread's work in progress takes about twice the largest mask's /// coordinates while the masks are unioned, then twice the largest input feature's, a list of -/// the masks, and the larger of the largest mask and the largest union of the masks one feature -/// meets (found for each batch first); the threads are as many as half the scratch +/// the masks (or the index's references to them), and the larger of the largest mask and the +/// largest size of the masks one feature meets, together (found for each batch first, standing +/// for their union); the threads are as many as half the scratch /// memory work may take holds of that, and their recent unions of several masks share /// `cache_bytes` (by default the other half of [`crate::parallel::scratch_budget`]). fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Option) -> Result { @@ -1804,7 +1805,6 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti let at_poles = AtPoles::new(masks.iter().map(|(m, _)| plane_poles(m))); let mut out = copy_schema(ds, out_name(p, format!("{}_Clip", ds.name)), ds.geometry_type); let input_rows = rows(inp, p, "input", ds); - // Input features without geometry: nothing of them can be kept. // Input features without geometry, or with an empty one: nothing of them can be kept. let without = input_rows .iter() @@ -1826,20 +1826,22 @@ fn clip_on(inp: &dyn Inputs, p: &Json, threads: Option, cache_bytes: Opti } near }; - // A thread's work in progress: a feature, its clipped part, and the union of the masks it - // meets when they are several, which can be far larger than any one mask (at most all of - // them). + // The union of the masks a feature meets, when they are several, can be far larger than any + // one mask (at most all of them). Its size is taken as theirs together, which it doesn't + // exceed by much (an overlay adds a vertex where edges cross, and takes away the overlaps). let mask_bytes: Vec = masks.iter().map(|(m, _)| kept_union_bytes(&[], m)).collect(); let union_bytes = |near: &[usize]| match near { [] | [_] => 0, _ => near.iter().filter_map(|&i| mask_bytes.get(i)).fold(0usize, |a, &b| a.saturating_add(b)), }; // A thread's work in progress: a feature and its clipped part, the list of the masks it meets - // (at most all of them), and the larger of a mask and the union of the masks it meets. - let feature_work = largest(ds, &input_rows).saturating_mul(2).saturating_add(masks.len().saturating_mul(std::mem::size_of::())); + // (at most all of them, or all the grid's references before each is kept once), and the + // larger of a mask and the union of the masks it meets. + let candidates = masks.len().max(index.references); + let feature_work = largest(ds, &input_rows).saturating_mul(2).saturating_add(candidates.saturating_mul(std::mem::size_of::())); // As many threads as there could be, if no feature met several masks: they size the batches. let most_threads = threads_for(input_rows.len(), feature_work.saturating_add(largest_mask)).max(1); - // Each feature is looked at twice: for the size of the union of the masks it meets, then to + // Each feature is looked at twice: for the size of the masks it meets together, then to // be clipped. Only those sizes are kept between the two, not the masks. crate::control::begin(input_rows.len().saturating_mul(2)); // In batches, so the clipped features held before they're added to the output stay few. @@ -5940,6 +5942,9 @@ struct ExtentIndex { grid: Option>>, /// How many features there are. len: usize, + /// How many cell references the grid holds (0 without one): the most a query can collect + /// before each feature is kept once. + references: usize, } impl ExtentIndex { @@ -5959,6 +5964,7 @@ impl ExtentIndex { cell_h: (total.height() / n_cells).max(1e-9), grid: None, len: extents.len(), + references: 0, }; if extents.len() <= Self::MAX_LINEAR_FEATURES || total.is_empty() { return index; @@ -5981,6 +5987,7 @@ impl ExtentIndex { } } index.grid = Some(grid); + index.references = references; index }