From 33fed9ccfad69d1d1cd744cd2a5728f42ac5012b Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Tue, 10 Nov 2026 20:39:35 +0100 Subject: [PATCH] perf(analysis): Erase and Update work on several cores (#1252) #1252 step 2, Erase and Update. Their erase/update zones come from `overlay_zones_on` (on several cores since the overlay tools), and each input feature's part outside the zones is now cut on its own on `parallel::try_map_on`, in `outside_each`. The output rows are made in order on the calling thread, so the output is the same as on one thread, bit for bit. Cancel is looked at, and progress counted, feature by feature as before; one thread in the browser and below the usual threshold of features. --- crates/analysis/src/tools.rs | 120 +++++++++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 5bd4030b0e..59773e0339 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -1662,22 +1662,26 @@ fn clip(inp: &dyn Inputs, p: &Json) -> Result { /// meets is kept as it is. Features left with nothing, or without geometry, are left out; the /// input must have geometry. fn erase(inp: &dyn Inputs, p: &Json) -> Result { + erase_on(inp, p, None) +} + +/// [`erase`] on `threads` threads (by the number of items in each step when `None`): each input +/// feature is cut on its own (#1252), and the rows come back in input order, as on one thread. +fn erase_on(inp: &dyn Inputs, p: &Json, threads: Option) -> Result { let ds = input(inp, p, "input")?; if ds.geometry_type == GeometryType::None { return Err(format!("'{}' has no geometry", ds.name)); } - let (er, zones) = overlay_zones(inp, p, "erase")?; + let (er, zones) = overlay_zones_on(inp, p, "erase", threads)?; let index = ExtentIndex::new(zones.iter().map(|z| z.ext)); let at_poles = AtPoles::new(zones.iter().map(|z| poles(&z.ext))); let mut out = copy_schema(ds, out_name(p, format!("{}_Erase", ds.name)), ds.geometry_type); keep_source_crs(&mut out, ds, er); let input_rows = rows(inp, p, "input", ds); - crate::control::begin(input_rows.len()); - for (done, r) in input_rows.into_iter().enumerate() { - crate::control::check()?; - crate::control::reached(done); + let rests = outside_each(ds, &input_rows, &zones, (&index, &at_poles), threads)?; + for (r, rest) in input_rows.into_iter().zip(rests) { let Some(f) = ds.features.get(r) else { continue }; - if let Some(rest) = f.geometry.as_ref().and_then(|g| outside(g, &zones, (&index, &at_poles))) { + if let Some(Some(rest)) = rest { out.push_next(Some(rest), f.attrs.clone()); } } @@ -1685,6 +1689,27 @@ fn erase(inp: &dyn Inputs, p: &Json) -> Result { result(out, format!("Erase: {n} features remain")) } +/// Each of `input_rows`' parts outside every zone, as [`outside`] gives it, in their order: `None` +/// for a row with no feature or no geometry, `Some(None)` for one left with nothing. Made on +/// `threads` threads (by the number of rows when `None`), each feature on its own (#1252); Cancel +/// is looked at, and progress counted, between features. +fn outside_each( + ds: &Dataset, + input_rows: &[usize], + zones: &[Zone], + near: (&ExtentIndex, &AtPoles), + threads: Option, +) -> Result>>, String> { + crate::control::begin(input_rows.len()); + let threads = threads.unwrap_or_else(|| crate::parallel::threads_for(input_rows.len(), PARALLEL_FEATURES)); + crate::parallel::try_map_on( + input_rows, + threads, + || (), + |_, _, &r| Ok::<_, String>(ds.features.get(r).and_then(|f| f.geometry.as_ref()).map(|g| outside(g, zones, near))), + ) +} + /// An overlay polygon: its row in the overlay layer, its extent and its polygons in the analysis /// plane, oriented for the overlay, its parts unioned when it has several (as Clip's mask is). struct Zone { @@ -2046,11 +2071,17 @@ fn converted(v: &Value, ty: FieldType) -> Value { /// update polygons that overlap each other each come out whole. An input field that no update /// field matches exactly, but two match ignoring case, is an error rather than a guess. fn update(inp: &dyn Inputs, p: &Json) -> Result { + update_on(inp, p, None) +} + +/// [`update`] on `threads` threads (by the number of items in each step when `None`): each input +/// polygon is cut on its own (#1252), and the rows come back in input order, as on one thread. +fn update_on(inp: &dyn Inputs, p: &Json, threads: Option) -> Result { let ds = input(inp, p, "input")?; if ds.geometry_type != GeometryType::Polygon { return Err(format!("'{}' is not a polygon layer", ds.name)); } - let (up, zones) = overlay_zones(inp, p, "update")?; + let (up, zones) = overlay_zones_on(inp, p, "update", threads)?; // Each input field's field in the update layer, if it has one: an ambiguous one fails here, // before any geometry work. let from = ds @@ -2069,20 +2100,13 @@ fn update(inp: &dyn Inputs, p: &Json) -> Result { let mut out = copy_schema(ds, out_name(p, format!("{}_Update", ds.name)), GeometryType::Polygon); keep_source_crs(&mut out, ds, up); let input_rows = rows(inp, p, "input", ds); - crate::control::begin(input_rows.len()); - for (done, r) in input_rows.into_iter().enumerate() { - crate::control::check()?; - crate::control::reached(done); + let rests = outside_each(ds, &input_rows, &zones, (&index, &at_poles), threads)?; + for (r, rest) in input_rows.into_iter().zip(rests) { let Some(f) = ds.features.get(r) else { continue }; - let rest = match &f.geometry { - None => { - out.push_next(None, f.attrs.clone()); - continue; - } - Some(g) => outside(g, &zones, (&index, &at_poles)), - }; - if let Some(rest) = rest { - out.push_next(Some(rest), f.attrs.clone()); + match rest { + None => out.push_next(None, f.attrs.clone()), + Some(Some(rest)) => out.push_next(Some(rest), f.attrs.clone()), + Some(None) => {} } } let kept = out.features.len(); @@ -11228,6 +11252,62 @@ mod tests { } } + /// Erase and Update give the same rows, bit for bit, on several threads as on one (#1252): + /// overlapping squares of two layers, erase and update features of two parts, input features + /// without geometry, and (for Erase) points and lines. + #[test] + fn erase_and_update_on_several_threads_are_on_one() { + let mut input = Dataset::new("input", GeometryType::Polygon, vec![Field::new("v", FieldType::Integer)]); + let mut lines = Dataset::new("lines", GeometryType::Polyline, vec![Field::new("v", FieldType::Integer)]); + let mut points = Dataset::new("points", GeometryType::Point, vec![Field::new("v", FieldType::Integer)]); + let mut over = Dataset::new("over", GeometryType::Polygon, vec![Field::new("v", FieldType::Integer)]); + for i in 0..600i64 { + let (x, y) = ((i * 37 % 300) as f64 * 0.1, (i * 53 % 120) as f64 * 0.1); + let none = i % 97 == 0; + input.push(if none { None } else { Some(sq(x, y, 0.35)) }, vec![Value::Int(i)]); + let line = Geometry::Line(vec![vec![Coord::new(x, y), Coord::new(x + 0.7, y + 0.2)]]); + lines.push(if none { None } else { Some(line) }, vec![Value::Int(i)]); + points.push(if none { None } else { Some(Geometry::Point(Coord::new(x, y))) }, vec![Value::Int(i)]); + } + for i in 0..200i64 { + let (x, y) = ((i * 71 % 290) as f64 * 0.1, (i * 29 % 110) as f64 * 0.1); + // Every fifth of two overlapping parts, so its zone is a union. + let ring = |x: f64| { + vec![Coord::new(x, y), Coord::new(x + 0.5, y), Coord::new(x + 0.5, y + 0.5), Coord::new(x, y + 0.5), Coord::new(x, y)] + }; + let g = if i % 5 == 0 { Geometry::Polygon(vec![vec![ring(x)], vec![ring(x + 0.3)]]) } else { sq(x, y, 0.5) }; + over.push(Some(g), vec![Value::Int(1000 + i)]); + } + let m = Map(HashMap::from([ + ("input".to_string(), input), + ("lines".to_string(), lines), + ("points".to_string(), points), + ("over".to_string(), over), + ])); + type On = fn(&dyn Inputs, &Json, Option) -> Result; + let runs: [(&str, On, Json); 4] = [ + ("erase", erase_on, serde_json::json!({"input": "input", "erase": "over"})), + ("erase lines", erase_on, serde_json::json!({"input": "lines", "erase": "over"})), + ("erase points", erase_on, serde_json::json!({"input": "points", "erase": "over"})), + ("update", update_on, serde_json::json!({"input": "input", "update": "over"})), + ]; + for (name, run, p) in runs { + let one = layer(run(&m, &p, Some(1)).unwrap()); + assert!(one.features.len() > 100, "{name}: {}", one.features.len()); + for threads in [2, 7, 16] { + let many = layer(run(&m, &p, Some(threads)).unwrap()); + assert_eq!(many.features.len(), one.features.len(), "{name}, {threads} threads"); + for (a, b) in one.features.iter().zip(&many.features) { + assert_eq!( + format!("{:?}", (&a.geometry, &a.attrs)), + format!("{:?}", (&b.geometry, &b.attrs)), + "{name}, {threads} threads" + ); + } + } + } + } + /// Features searched on several threads find the same nearest features, distances and points /// as on one thread, in input order (#1252), the same layer as both included. #[test]