diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index 97493c0bd2..2253cc3ba9 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -25,6 +25,7 @@ pub mod neural; mod parallel; mod parameters; pub mod rasterops; +mod repair; pub mod selection; pub mod sketch_edit; mod stats; diff --git a/crates/analysis/src/repair.rs b/crates/analysis/src/repair.rs new file mode 100644 index 0000000000..a921ac3db3 --- /dev/null +++ b/crates/analysis/src/repair.rs @@ -0,0 +1,918 @@ +//! Geometry repair in stored coordinates: a valid geometry, after the OGC Simple Features rules +//! (OGC 06-103r4, ISO 19125-1: 6.1.11.1 Polygon, 6.1.12 MultiPolygon), for one in which +//! [`validity::check`] finds problems. Repair Geometry writes the results. +//! +//! - Vertex-level problems get fixes that move no coordinate: an unclosed ring is closed, +//! repeated vertices are dropped, and a ring with fewer than 3 distinct points (a line part with +//! fewer than 2) is dropped, with its part's holes when it is the part's exterior ring. +//! - Polygon-level problems (a ring crossing, touching or running back over itself, a hole not +//! inside its exterior ring, rings or parts that overlap or touch along a line, an interior split +//! by rings touching at points) are repaired by area. Each ring stands for the area it winds +//! around (non-zero winding: both lobes of a bowtie count, and the direction a ring is wound in +//! doesn't matter); a part is its exterior ring's area less the union of its holes' areas, so a +//! hole outside its exterior ring disappears and holes that overlap merge; and the feature is +//! the union of its parts, so parts that overlap merge. geo's boolean overlay (i_overlay) does +//! the arithmetic on an integer grid, which moves coordinates a little ([`grid_step`], +//! [`GRID_STEPS_MOVED`]), and writes each piece of a split interior as a part of its own. A hole +//! or part narrower than a step of that grid in any direction (up to √2 steps across a diagonal, +//! [`narrower_than`]) may be left no area by it, so it is dropped before the overlay and counted +//! ([`Repaired::narrow`]). +//! - A repaired geometry is checked again and written only when it is valid. A feature too busy +//! to check or overlay within the work bound, or without the memory for it, or one the overlay +//! fails on, is left as it was ([`Kept`]): never dropped. +//! - Lines may cross themselves (OGC doesn't require a simple line), and points have nothing to +//! repair. Missing geometry and non-finite coordinates can't be repaired ([`Gone`]). +//! +//! In the browser a panic in the overlay can't be caught: it stops the app there, as it would in +//! the other tools that run the overlay (Clip, Dissolve, Buffer). So its input is bounded by the +//! work charge, has finite coordinates on a grid i_overlay can represent, and its memory is asked +//! of the browser before it runs. Natively a panic is caught, and the feature is left as it was. + +use crate::validity::{self, At, Checked, Code, Limit, Problem, Room}; +use geo::algorithm::bool_ops::{BooleanOps, FillRule, OpType}; +use geo::orient::{Direction, Orient}; +use rustgis_data::{Coord, Geometry}; + +/// Work per vertex the overlay takes in or makes, against the same bound as a check +/// ([`validity::MAX_WORK`]): at 25, up to 2 million vertices a feature, as Buffer allows. +const OVERLAY_COST: u64 = 25; +/// The memory the overlay holds per vertex, as estimated: geo's and i_overlay's copies of the +/// rings in floats and on the grid, their segments and sort, the overlay's graph and its output. +const OVERLAY_BYTES: u64 = 160; +/// How far a coordinate may move through the overlays of one repair, in steps of the grid: a +/// vertex is rounded to the grid in each of up to three overlays (a ring's own area, a part's holes +/// taken out, the union of the parts), each time by at most half a step on each axis, and a vertex +/// the overlay adds where rings cross lies on the rounded segments, so within as much again of the +/// input's boundary. Measured in the tests, the largest move is well under one step. +pub(crate) const GRID_STEPS_MOVED: f64 = 5.0; + +/// Why a feature is left as it was. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kept { + /// Checking or overlaying it needs more work than the bound. + Work, + /// The memory for it couldn't be had. + Memory, + /// geo failed on it: its relate when checking, or the overlay (a panic, natively; + /// coordinates on no grid it can represent; or a result that isn't valid). + Failed, +} + +impl Kept { + /// The code written for it (data: never translated). + pub(crate) fn as_str(self) -> &'static str { + match self { + Kept::Work | Kept::Memory => Code::TooComplex.as_str(), + Kept::Failed => "repair_failed", + } + } +} + +/// Why a feature can't be repaired: it is written without geometry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Gone { + Null, + NonFinite, + /// Every part has too few distinct points. + TooFewPoints, + /// Nothing with an area is left once repaired: rings that run back over themselves, or holes + /// covering their whole part. + NoArea, +} + +impl Gone { + pub(crate) const ALL: [Gone; 4] = [Gone::Null, Gone::NonFinite, Gone::TooFewPoints, Gone::NoArea]; + + /// The code written for it (data: never translated). + pub(crate) fn as_str(self) -> &'static str { + match self { + Gone::Null => Code::NullGeometry.as_str(), + Gone::NonFinite => Code::NonFiniteCoordinate.as_str(), + Gone::TooFewPoints => Code::TooFewPoints.as_str(), + Gone::NoArea => "no_area", + } + } +} + +/// A repaired geometry, and what was done to it. +#[derive(Debug, PartialEq)] +pub(crate) struct Repaired { + pub geometry: Geometry, + /// What was wrong with it, each once, in [`Code`] order. + pub codes: Vec, + /// Rings and parts left out: too few distinct points, or no area. + pub dropped: usize, + /// Holes and parts left out of the overlay as narrower than a step of its grid in any + /// direction, where they would have no area ([`narrower_than`]); written as `no_area` after + /// the codes. + pub narrow: usize, + /// When the overlay rebuilt it, how far its coordinates may have moved, in their units (degrees, + /// as coordinates are stored). + pub moved: Option, +} + +/// What [`repair`] made of one geometry. +#[derive(Debug, PartialEq)] +pub(crate) enum Outcome { + /// Nothing is wrong: written as it is. + Valid, + Repaired(Repaired), + Gone(Gone), + Kept(Kept), +} + +/// Repairs one geometry, spending at most [`validity::MAX_WORK`] on each check and on the +/// overlay, and asking `has_room` whether the browser can give the overlay's memory before it runs +/// (see `web_has_room` in the tools). +pub(crate) fn repair(g: Option<&Geometry>, has_room: &dyn Fn(u64) -> bool) -> Outcome { + repair_with(g, validity::MAX_WORK, &|_| true, has_room) +} + +/// [`repair`] with a work `budget`, asking `allow` before each reservation of a buffer whether to +/// try it, as [`validity::check_with`] does: tests refuse some to stand in for running out of +/// memory. +pub(crate) fn repair_with(g: Option<&Geometry>, budget: u64, allow: &dyn Fn(usize) -> bool, has_room: &dyn Fn(u64) -> bool) -> Outcome { + let first = validity::check_with(g, budget, allow); + if first.problems.is_empty() { + return Outcome::Valid; + } + let Some(g) = g else { return Outcome::Gone(Gone::Null) }; + if let Some(gone) = gone(&first) { + return Outcome::Gone(gone); + } + if let Some(kept) = kept(&first) { + return Outcome::Kept(kept); + } + let room = Room(allow); + match g { + Geometry::Line(parts) => match lines(parts, room) { + Err(limit) => Outcome::Kept(kept_for(limit)), + Ok((parts, _)) if parts.is_empty() => Outcome::Gone(Gone::TooFewPoints), + Ok((parts, dropped)) => { + verified(Repaired { geometry: Geometry::Line(parts), codes: first.codes(), dropped, narrow: 0, moved: None }, budget, allow) + } + }, + Geometry::Polygon(polys) => polygon(polys, &first, budget, allow, has_room), + // A point's only problems, a missing or non-finite coordinate, are handled above. + Geometry::Point(_) | Geometry::MultiPoint(_) => Outcome::Kept(Kept::Failed), + } +} + +/// Why a checked feature can't be repaired, if it can't. +fn gone(c: &Checked) -> Option { + c.problems.iter().find_map(|p| match p { + Problem::Null { .. } => Some(Gone::Null), + Problem::NonFinite { .. } => Some(Gone::NonFinite), + _ => None, + }) +} + +/// Why a feature that wasn't checked fully is left as it was, if it wasn't. +fn kept(c: &Checked) -> Option { + c.problems.iter().find_map(|p| match p { + Problem::TooComplex { limit } => Some(kept_for(*limit)), + _ => None, + }) +} + +fn kept_for(limit: Limit) -> Kept { + match limit { + Limit::Work => Kept::Work, + Limit::Memory => Kept::Memory, + Limit::Relate => Kept::Failed, + } +} + +/// `r`, once a check finds nothing wrong with its geometry; left as it was otherwise. +fn verified(r: Repaired, budget: u64, allow: &dyn Fn(usize) -> bool) -> Outcome { + let c = validity::check_with(Some(&r.geometry), budget, allow); + if let Some(k) = kept(&c) { + return Outcome::Kept(k); + } + if c.problems.is_empty() { Outcome::Repaired(r) } else { Outcome::Kept(Kept::Failed) } +} + +/// `pts` without repeated consecutive vertices, closed when `ring`, in a vector reserved fallibly. +fn cleaned(pts: &[Coord], ring: bool, room: Room) -> Result, Limit> { + let mut v: Vec = room.vec(pts.len().saturating_add(usize::from(ring)))?; + for c in pts { + if v.last() != Some(c) { + v.push(*c); + } + } + if ring + && let (Some(first), Some(last)) = (v.first().copied(), v.last()) + && first != *last + { + v.push(first); + } + Ok(v) +} + +/// A line's parts without repeated vertices, and without the parts that have fewer than 2 +/// distinct points; how many were dropped. +fn lines(parts: &[Vec], room: Room) -> Result<(Vec>, usize), Limit> { + let mut out = room.vec(parts.len())?; + let mut dropped = 0usize; + for part in parts { + let part = cleaned(part, false, room)?; + // Consecutive vertices differ now: two of them are two distinct points. + if part.len() < 2 { + dropped += 1; + } else { + out.push(part); + } + } + Ok((out, dropped)) +} + +/// A polygon's rings closed and without repeated vertices, and without the rings that have fewer +/// than 3 distinct points (an exterior ring's part goes with it, holes and all), nor parts without +/// rings; how many rings and parts were dropped, and whether anything changed. +fn rings(polys: &[Vec>], room: Room) -> Result<(Vec>>, usize, bool), Limit> { + let mut out = room.vec(polys.len())?; + let (mut dropped, mut changed) = (0usize, false); + for part in polys { + let mut kept: Vec> = room.vec(part.len())?; + for (r, ring) in part.iter().enumerate() { + let fixed = cleaned(ring, true, room)?; + if validity::distinct_up_to(&fixed, 3) < 3 { + dropped += 1; + changed = true; + if r == 0 { + break; + } + continue; + } + changed |= fixed != *ring; + kept.push(fixed); + } + if part.is_empty() { + dropped += 1; + changed = true; + } + // An exterior ring dropped leaves nothing: its holes went with it. + if !kept.is_empty() { + out.push(kept); + } + } + Ok((out, dropped, changed)) +} + +/// A polygon feature: the fixes that move no coordinate, then, when the polygon-level checks +/// still find something, the rebuild by area. +fn polygon( + polys: &[Vec>], + first: &Checked, + budget: u64, + allow: &dyn Fn(usize) -> bool, + has_room: &dyn Fn(u64) -> bool, +) -> Outcome { + let room = Room(allow); + let (fixed, dropped, changed) = match rings(polys, room) { + Ok(fixed) => fixed, + Err(limit) => return Outcome::Kept(kept_for(limit)), + }; + if fixed.is_empty() { + return Outcome::Gone(Gone::TooFewPoints); + } + let fixed = Geometry::Polygon(fixed); + // Checked again when something changed: rings and parts left unrelated by a ring's problem + // (see `Checked::unrelated`) are related now. + let rechecked; + let second = if changed { + rechecked = validity::check_with(Some(&fixed), budget, allow); + &rechecked + } else { + first + }; + if let Some(kept) = kept(second) { + return Outcome::Kept(kept); + } + let mut codes = first.codes(); + for code in second.codes() { + if !codes.contains(&code) { + codes.push(code); + } + } + codes.sort_unstable(); + if second.problems.is_empty() { + return Outcome::Repaired(Repaired { geometry: fixed, codes, dropped, narrow: 0, moved: None }); + } + let Geometry::Polygon(parts) = &fixed else { return Outcome::Kept(Kept::Failed) }; + if gone(second).is_some() { + return Outcome::Kept(Kept::Failed); + } + + let coords = parts.iter().flatten().map(Vec::len).fold(0usize, usize::saturating_add); + let size = u64::try_from(coords).unwrap_or(u64::MAX).saturating_add(second.contacts.saturating_mul(2)); + if size.saturating_mul(OVERLAY_COST) > budget { + return Outcome::Kept(Kept::Work); + } + if !has_room(size.saturating_mul(OVERLAY_BYTES)) { + return Outcome::Kept(Kept::Memory); + } + let Some(step) = grid_step(parts.iter().flatten().flatten().map(|c| (c.x, c.y))) else { return Outcome::Kept(Kept::Failed) }; + // The rings crossing themselves, by part and ring: in order, as the problems are. + let crossing = second.problems.iter().filter_map(|p| match p { + Problem::SelfIntersection { at: At::Ring(p, r), .. } => Some((*p, *r)), + _ => None, + }); + let crossing = match room.collect(crossing.clone().count(), crossing) { + Ok(c) => c, + Err(limit) => return Outcome::Kept(kept_for(limit)), + }; + let rebuilt = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| by_area(parts, &crossing, room))); + let Rebuilt { polys, no_area, narrow, found } = match rebuilt { + Ok(Ok(rebuilt)) => rebuilt, + Ok(Err(kept)) => return Outcome::Kept(kept), + Err(_) => return Outcome::Kept(Kept::Failed), + }; + if polys.is_empty() { + return Outcome::Gone(Gone::NoArea); + } + for code in found { + if !codes.contains(&code) { + codes.push(code); + } + } + codes.sort_unstable(); + let moved = Some(step * GRID_STEPS_MOVED); + verified(Repaired { geometry: Geometry::Polygon(polys), codes, dropped: dropped.saturating_add(no_area), narrow, moved }, budget, allow) +} + +/// The step of the integer grid i_overlay rounds the points `pts` to: it takes half the larger +/// side of their extent, `h`, to about 2^29 grid units (2^(29 - round(log2 h)) units a unit of +/// the coordinates). `None` when there is no such grid: an extent that is empty, not finite, or +/// so large or so small that the scale or the step isn't a finite number. +pub(crate) fn grid_step(pts: impl Iterator) -> Option { + let (mut lo_x, mut lo_y, mut hi_x, mut hi_y) = (f64::INFINITY, f64::INFINITY, f64::NEG_INFINITY, f64::NEG_INFINITY); + for (x, y) in pts { + (lo_x, lo_y, hi_x, hi_y) = (lo_x.min(x), lo_y.min(y), hi_x.max(x), hi_y.max(y)); + } + let half = (hi_x - lo_x).max(hi_y - lo_y) * 0.5; + if !(half.is_finite() && half > 0.0) { + return None; + } + let e = 29.0 - half.log2().round(); + let (scale, step) = (e.exp2(), (-e).exp2()); + (scale.is_normal() && step.is_normal()).then_some(step) +} + +/// The step of the grid the overlay rounds `rings` to ([`grid_step`]); an error when there is +/// none, and the overlay can't run on them. +fn on_grid<'a>(rings: impl Iterator>) -> Result { + grid_step(rings.flat_map(|r| r.0.iter().map(|c| (c.x, c.y)))).ok_or(Kept::Failed) +} + +/// Every ring of `polys`, as the overlay takes them in. +fn rings_of(polys: &[geo::Polygon]) -> impl Iterator> { + polys.iter().flat_map(|p| std::iter::once(p.exterior()).chain(p.interiors())) +} + +/// Whether `poly` is narrower than `step`, the step of the overlay's grid, in some direction: +/// rounded to that grid, its points may then all fall on one line, which leaves it no area. +/// Whether they do depends on where it lies between the grid's lines, so such a polygon is dropped +/// before the overlay, and counted, rather than left to the overlay to drop unsaid. +/// +/// Rounding moves a point by at most half a step on each axis, so across a line with unit normal +/// `(nx, ny)` by at most `(|nx| + |ny|) / 2` steps. Points rounded onto one such line were within +/// `|nx| + |ny|` steps of each other across it: one step across the axes, up to √2 steps at 45°. So +/// `poly` is narrower when, in some direction, its width is less than `|nx| + |ny|` steps. Between +/// the axes and the normals of its convex hull's edges, width and allowance are each a sinusoid of +/// the direction, and their ratio is monotonic, so the least ratio is found in one of those +/// directions: the axes, by the extent, and each hull edge, by the hull vertex farthest from its +/// line, which rotating calipers find in one turn around the hull (M. I. Shamos, "Computational +/// Geometry", PhD thesis, Yale University, 1978; G. T. Toussaint, "Solving geometric problems with +/// the rotating calipers", Proc. IEEE MELECON '83, 1983). Only the exterior ring makes the hull. +/// +/// Only thin polygons pay for the hull. A polygon of least width `w` lies in a strip `w` wide and +/// as long as its diameter, at most the diagonal of its extent, so its area is at most `w` times +/// that diagonal; its rings here are simple (the overlay's, or ones the check found not to cross +/// themselves), so the exterior ring's area is the area it covers. An area of twice the step times +/// the diagonal or more means a width of more than √2 steps, wide enough in every direction (twice, +/// so that the rounding of the area's sum can't tip it). The hull sorts the vertices the overlay +/// is charged for ([`OVERLAY_COST`]), in buffers reserved fallibly. +fn narrower_than(poly: &geo::Polygon, step: f64, room: Room) -> Result { + let ring = &poly.exterior().0; + let (mut lo_x, mut lo_y, mut hi_x, mut hi_y) = (f64::INFINITY, f64::INFINITY, f64::NEG_INFINITY, f64::NEG_INFINITY); + for c in ring { + (lo_x, lo_y, hi_x, hi_y) = (lo_x.min(c.x), lo_y.min(c.y), hi_x.max(c.x), hi_y.max(c.y)); + } + let (wide, high) = (hi_x - lo_x, hi_y - lo_y); + if wide.min(high) < step { + return Ok(true); + } + // Twice the area, from the first vertex, so that far from the origin no precision is lost. + let Some(o) = ring.first() else { return Ok(true) }; + let twice = ring.windows(2).fold(0.0, |s, w| match w { + [a, b] => s + (a.x - o.x) * (b.y - o.y) - (b.x - o.x) * (a.y - o.y), + _ => s, + }); + if twice.abs() >= 4.0 * step * wide.hypot(high) { + return Ok(false); + } + let hull = hull(ring, room)?; + let n = hull.len(); + if n < 3 { + // All on one line. + return Ok(true); + } + // Twice the area of the triangle of edge `i` and vertex `k`: the vertex's distance from the + // edge's line times the edge's length; positive inside, as the hull turns anticlockwise. + let across = |i: usize, k: usize| -> Option { + let (a, b, p) = (hull.get(i % n)?, hull.get((i + 1) % n)?, hull.get(k % n)?); + Some((b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x)) + }; + // The calipers turn once around the hull; the bound on their steps only guards the loop. + let (mut far, mut turns) = (1usize, 0usize); + for i in 0..n { + while turns < n.saturating_mul(3) { + match (across(i, far), across(i, far + 1)) { + (Some(here), Some(next)) if next > here => { + far = (far + 1) % n; + turns += 1; + } + _ => break, + } + } + let (Some(a), Some(b), Some(d)) = (hull.get(i), hull.get((i + 1) % n), across(i, far)) else { return Ok(false) }; + // The farthest vertex's distance, `d / |e|`, against the allowance `(|ex| + |ey|) / |e|` steps. + if d < step * ((b.x - a.x).abs() + (b.y - a.y).abs()) { + return Ok(true); + } + } + Ok(false) +} + +/// The convex hull of `pts`, anticlockwise, without collinear points or a closing repeat, by +/// Andrew's monotone chain (A. M. Andrew, "Another efficient algorithm for convex hulls in two +/// dimensions", Information Processing Letters 9(5), 1979), in buffers reserved fallibly. +fn hull(pts: &[geo::Coord], room: Room) -> Result>, Limit> { + let mut pts = room.collect(pts.len(), pts.iter().copied())?; + pts.sort_unstable_by(|a, b| a.x.total_cmp(&b.x).then(a.y.total_cmp(&b.y))); + pts.dedup(); + let turns_left = |h: &[geo::Coord], p: geo::Coord| match h { + [.., a, b] => (b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x) > 0.0, + _ => true, + }; + let mut hull = room.vec(pts.len().saturating_add(1))?; + // The lower chain, left to right, then the upper, right to left, on top of it. + for &p in &pts { + while !turns_left(&hull, p) { + hull.pop(); + } + room.grow(&mut hull)?; + hull.push(p); + } + let lower = hull.len(); + for &p in pts.iter().rev().skip(1) { + while hull.len() > lower && !turns_left(&hull, p) { + hull.pop(); + } + room.grow(&mut hull)?; + hull.push(p); + } + // The upper chain ends where the lower began. + if hull.len() > 1 { + hull.pop(); + } + Ok(hull) +} + +/// `polys` without those narrower than `step` ([`narrower_than`]), adding how many were left out +/// to `narrow`. +fn wider(mut polys: Vec>, step: f64, narrow: &mut usize, room: Room) -> Result>, Kept> { + let before = polys.len(); + let mut failed = None; + polys.retain(|p| match narrower_than(p, step, room) { + Ok(thin) => !thin, + Err(limit) => { + failed.get_or_insert(limit); + true + } + }); + if let Some(limit) = failed { + return Err(kept_for(limit)); + } + *narrow = narrow.saturating_add(before - polys.len()); + Ok(polys) +} + +/// What the rebuild by area found that the checks may have missed: a part with a ring crossing +/// itself isn't related to its other rings or to other parts (see `Checked::unrelated`). +type Found = Vec; + +/// What [`by_area`] made of a feature's parts. +struct Rebuilt { + /// The rings of the result. + polys: Vec>>, + /// Rings and parts left without area. + no_area: usize, + /// Holes and parts dropped as narrower than a step of the overlay's grid in any direction. + narrow: usize, + found: Found, +} + +/// The rebuild by area (see the module doc) of `parts`, closed rings each with 3 distinct points +/// or more, of which those in `crossing` (part, ring; sorted) cross, touch or run back over +/// themselves. The rings of the result, how many rings and parts were left without area, how many +/// holes and parts were dropped as narrower, in any direction, than a step of the grid of the +/// overlay that takes them out or merges them, and what it found: [`Code::HoleOutsideShell`] when some of the holes of a +/// part with such a ring lie outside its exterior ring's area, [`Code::RingsIntersect`] when holes +/// of such a part merged, and [`Code::PartsOverlap`] when parts merged, which each union tells by +/// making fewer polygons than it was given (pieces that only touch at points stay apart; the narrow +/// ones are dropped first, so their vanishing isn't taken for a merge). +fn by_area(parts: &[Vec>], crossing: &[(usize, usize)], room: Room) -> Result { + let empty = geo::MultiPolygon::::new(vec![]); + let area = |ring: &Vec, at: (usize, usize)| -> Result, Kept> { + let ring = geo::LineString(room.collect(ring.len(), ring.iter().map(|c| geo::Coord { x: c.x, y: c.y })).map_err(kept_for)?); + let poly = geo::Polygon::new(ring, vec![]); + if crossing.binary_search(&at).is_err() { + // A simple ring: its area as it is, wound as the overlay expects. + return Ok(geo::MultiPolygon::new(vec![poly.orient(Direction::Default)])); + } + on_grid(std::iter::once(poly.exterior()))?; + Ok(poly.boolean_op_with_fill_rule(&empty, OpType::Union, FillRule::NonZero)) + }; + let mut all: Vec> = Vec::new(); + let mut no_area = 0usize; + let mut narrow = 0usize; + let mut found = Found::new(); + for (p, rings) in parts.iter().enumerate() { + let Some((shell, holes)) = rings.split_first() else { continue }; + let shell = area(shell, (p, 0))?; + if shell.0.is_empty() { + // Its holes go with it. + no_area += 1; + continue; + } + let mut cut: Vec> = Vec::new(); + for (h, hole) in holes.iter().enumerate() { + let hole = area(hole, (p, h + 1))?; + if hole.0.is_empty() { + no_area += 1; + } + for piece in hole.0 { + room.grow(&mut cut).map_err(kept_for)?; + cut.push(piece); + } + } + // Holes narrower than a step of the grid the difference below rounds to, in any direction, + // are dropped first. + if !cut.is_empty() { + let step = on_grid(rings_of(&shell.0).chain(rings_of(&cut)))?; + cut = wider(cut, step, &mut narrow, room)?; + } + let part = if cut.is_empty() { + shell + } else { + // The holes' areas, each wound the same way, make their union under non-zero winding. + let cut = geo::MultiPolygon::new(cut); + let related = (0..rings.len()).all(|r| crossing.binary_search(&(p, r)).is_err()); + if !related + && !found.contains(&Code::HoleOutsideShell) + && !cut.boolean_op_with_fill_rule(&shell, OpType::Difference, FillRule::NonZero).0.is_empty() + { + found.push(Code::HoleOutsideShell); + } + // Holes that overlap or touch along a line merge, which their union tells by making + // fewer polygons than it was given, as the parts' union does below. + if !related + && cut.0.len() > 1 + && !found.contains(&Code::RingsIntersect) + && on_grid(rings_of(&cut.0)).is_ok() + && cut.boolean_op_with_fill_rule(&empty, OpType::Union, FillRule::NonZero).0.len() < cut.0.len() + { + found.push(Code::RingsIntersect); + } + shell.boolean_op_with_fill_rule(&cut, OpType::Difference, FillRule::NonZero) + }; + if part.0.is_empty() { + no_area += 1; + } + for piece in part.0 { + room.grow(&mut all).map_err(kept_for)?; + all.push(piece); + } + } + if parts.len() > 1 && all.len() > 1 { + // Parts narrower than a step of the union's grid, in any direction, are dropped first. + let step = on_grid(rings_of(&all))?; + all = wider(all, step, &mut narrow, room)?; + } + let all = geo::MultiPolygon::new(all); + let merged = if parts.len() > 1 && all.0.len() > 1 { + let merged = all.boolean_op_with_fill_rule(&empty, OpType::Union, FillRule::NonZero); + if merged.0.len() < all.0.len() { + found.push(Code::PartsOverlap); + } + merged + } else { + all + }; + Ok(Rebuilt { polys: polygons(&merged, room).map_err(kept_for)?, no_area, narrow, found }) +} + +/// The overlay's polygons as rings, copied fallibly. +fn polygons(m: &geo::MultiPolygon, room: Room) -> Result>>, Limit> { + let mut out = room.vec(m.0.len())?; + for poly in &m.0 { + // The overlay writes closed rings of 3 points or more; anything shorter has no area. + if poly.exterior().0.len() < 4 { + continue; + } + let mut rings = room.vec(poly.interiors().len().saturating_add(1))?; + for ring in std::iter::once(poly.exterior()).chain(poly.interiors()).filter(|r| r.0.len() >= 4) { + rings.push(room.collect(ring.0.len(), ring.0.iter().map(|c| Coord::new(c.x, c.y)))?); + } + out.push(rings); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + type Ring<'a> = &'a [(f64, f64)]; + + /// Every reservation tried. + const ANY: Room<'static> = Room(&|_| true); + + fn ring(pts: Ring) -> Vec { + pts.iter().map(|&(x, y)| Coord::new(x, y)).collect() + } + fn multi(parts: &[&[Ring]]) -> Geometry { + Geometry::Polygon(parts.iter().map(|rings| rings.iter().map(|r| ring(r)).collect()).collect()) + } + fn poly(rings: &[Ring]) -> Geometry { + multi(&[rings]) + } + fn fix(g: &Geometry) -> Outcome { + repair(Some(g), &|_| true) + } + fn repaired(g: &Geometry) -> Repaired { + match fix(g) { + Outcome::Repaired(r) => r, + other => panic!("{g:?} gave {other:?}"), + } + } + /// The area of a polygon geometry, by the shoelace formula over every ring (holes wound + /// against their shell count negative, so only the magnitude per ring is used: shell minus holes). + fn area(g: &Geometry) -> f64 { + let ring_area = |r: &[Coord]| r.windows(2).map(|w| w[0].x * w[1].y - w[1].x * w[0].y).sum::().abs() / 2.0; + let Geometry::Polygon(parts) = g else { panic!("a polygon") }; + parts.iter().map(|rings| ring_area(&rings[0]) - rings[1..].iter().map(|h| ring_area(h)).sum::()).sum() + } + + const SQUARE: Ring = &[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 0.)]; + const BOWTIE: Ring = &[(0., 0.), (2., 2.), (2., 0.), (0., 2.), (0., 0.)]; + + #[test] + fn cleaned_drops_repeats_and_closes_rings() { + let pts = ring(&[(0., 0.), (0., 0.), (1., 0.), (1., 0.), (1., 1.)]); + assert_eq!(cleaned(&pts, true, ANY).unwrap(), ring(&[(0., 0.), (1., 0.), (1., 1.), (0., 0.)])); + assert_eq!(cleaned(&pts, false, ANY).unwrap(), ring(&[(0., 0.), (1., 0.), (1., 1.)])); + // A duplicated closing point is one repeat; a closed ring isn't closed again. + assert_eq!(cleaned(&ring(&[(0., 0.), (1., 0.), (1., 1.), (0., 0.), (0., 0.)]), true, ANY).unwrap().len(), 4); + assert_eq!(cleaned(&[], true, ANY).unwrap(), []); + } + + #[test] + fn degenerate_rings_and_parts_are_dropped_and_counted() { + let flat: Ring = &[(0., 0.), (1., 0.), (0., 0.)]; + let Geometry::Polygon(polys) = multi(&[&[SQUARE, flat], &[flat, SQUARE], &[], &[SQUARE]]) else { panic!("a polygon") }; + let (out, dropped, changed) = rings(&polys, ANY).unwrap(); + // The hole of part 1; part 2, whose exterior ring goes with its hole; part 3, without rings. + assert_eq!(dropped, 3); + assert!(changed); + assert_eq!(out, vec![vec![ring(SQUARE)], vec![ring(SQUARE)]]); + let Geometry::Polygon(clean) = poly(&[SQUARE]) else { panic!("a polygon") }; + assert_eq!(rings(&clean, ANY).unwrap(), (clean.clone(), 0, false)); + // The same length as before but not the same ring: a repeat dropped and the ring closed. + let Geometry::Polygon(open) = poly(&[&[(0., 0.), (10., 0.), (10., 0.), (10., 10.)]]) else { panic!("a polygon") }; + assert!(rings(&open, ANY).unwrap().2); + + let (out, dropped) = lines(&[ring(&[(0., 0.), (0., 0.)]), ring(&[(0., 0.), (1., 1.), (1., 1.)]), vec![]], ANY).unwrap(); + assert_eq!((out, dropped), (vec![ring(&[(0., 0.), (1., 1.)])], 2)); + } + + #[test] + fn a_ring_stands_for_the_area_it_winds_around() { + // A bowtie keeps both lobes, whichever way it is wound. + for bowtie in [BOWTIE.to_vec(), BOWTIE.iter().rev().copied().collect()] { + let r = repaired(&poly(&[&bowtie])); + let Geometry::Polygon(parts) = &r.geometry else { panic!() }; + assert_eq!(parts.len(), 2); + assert_eq!(area(&r.geometry), 2.0); + assert_eq!(r.codes, [Code::SelfIntersection]); + } + // A loop inside the ring, touching it at a vertex: wound against the ring, it is outside + // (an "inverted" hole, as some formats write one); wound with it, it is inside. + let against: Ring = &[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 0.), (5., 8.), (8., 5.), (5., 2.), (0., 0.)]; + let with: Ring = &[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 0.), (5., 2.), (8., 5.), (5., 8.), (0., 0.)]; + let r = repaired(&poly(&[against])); + assert_eq!(area(&r.geometry), 100.0 - 24.0); + let Geometry::Polygon(parts) = &r.geometry else { panic!() }; + assert_eq!((parts.len(), parts[0].len()), (1, 2), "one polygon with a hole touching its shell at a point"); + assert_eq!(area(&repaired(&poly(&[with])).geometry), 100.0); + } + + #[test] + fn grid_steps_follow_i_overlays_scale() { + let step = |pts: &[(f64, f64)]| grid_step(pts.iter().copied()); + // Half of 10 is 5: log2 5 rounds to 2, so 2^27 units a unit. + assert_eq!(step(&[(0., 0.), (10., 3.)]), Some((-27f64).exp2())); + assert_eq!(step(&[(1e5, 0.), (1e5 + 10.7, 10.)]), Some((-27f64).exp2())); + assert_eq!(step(&[(-180., -90.), (180., 90.)]), Some((-22f64).exp2())); + assert_eq!(step(&[(1., 1.), (1., 1.)]), None, "no extent"); + assert_eq!(step(&[(-1e308, 0.), (1e308, 0.)]), None, "an extent past the largest float"); + assert_eq!(step(&[(0., 0.), (1e-300, 1e-300)]), None, "a scale past the largest float"); + assert_eq!(step(&[]), None); + // The overlay runs only on rings with a grid: in the browser, where its failures can't be + // caught, it never sees one it can't scale. + let ls = |pts: &[(f64, f64)]| geo::LineString::from(pts.to_vec()); + assert_eq!(on_grid([ls(&[(0., 0.), (1., 0.), (0., 1.), (0., 0.)])].iter()), Ok((-30f64).exp2())); + assert_eq!(on_grid([ls(&[(0., 0.), (1e-310, 0.), (0., 1e-310), (0., 0.)])].iter()), Err(Kept::Failed)); + assert_eq!(on_grid([ls(&[(-1e308, 0.), (1e308, 0.), (0., 1.), (-1e308, 0.)])].iter()), Err(Kept::Failed)); + } + + #[test] + fn widths_are_measured_in_every_direction() { + let p = |pts: &[(f64, f64)]| geo::Polygon::new(geo::LineString::from(pts.to_vec()), vec![]); + let thin = |pts: &[(f64, f64)], step: f64| narrower_than(&p(pts), step, ANY); + // A strip at 45°, 100 long and 1 wide: 70 across and up, but within √2 steps across only + // at a step of more than 1 / √2. + let a = 0.5f64.sqrt(); + let strip = [(0., 0.), (70.7, 70.7), (70.7 - a, 70.7 + a), (-a, a), (0., 0.)]; + assert_eq!(thin(&strip, 0.70), Ok(false)); + assert_eq!(thin(&strip, 0.71), Ok(true)); + // On the axes, a step is enough, and the bound is strict. + let wide = [(0., 0.), (100., 0.), (100., 1.), (0., 1.), (0., 0.)]; + assert_eq!(thin(&wide, 1.0), Ok(false)); + assert_eq!(thin(&wide, 1.0 + 1e-9), Ok(true)); + // A triangle's least width is its smallest height: here 4.8, from the right angle to the + // hypotenuse of a 6, 8, 10 triangle at 36.87°, where rounding moves a point across by up + // to 0.7 steps (cos + sin over 2); its legs, 6 and 8, lie on the axes. + let triangle = [(0., 0.), (8., 0.), (0., 6.), (0., 0.)]; + assert_eq!(thin(&triangle, 4.8 / 1.4 - 1e-9), Ok(false)); + assert_eq!(thin(&triangle, 4.8 / 1.4 + 1e-9), Ok(true)); + // Many vertices on a thin lens, far from the origin, wound either way. + let lens: Vec<(f64, f64)> = (0..=2000) + .map(|k| { + let t = std::f64::consts::TAU * f64::from(k) / 2000.0; + (1e6 + 50.0 * t.cos() - 0.2 * t.sin(), 1e6 + 50.0 * t.cos() + 0.2 * t.sin()) + }) + .collect(); + let back: Vec<(f64, f64)> = lens.iter().rev().copied().collect(); + // 0.4 √2 wide across its diagonal: thin at a step over 0.4. + for l in [&lens, &back] { + assert_eq!(thin(l, 0.399), Ok(false)); + assert_eq!(thin(l, 0.401), Ok(true)); + } + // Points in a row, all one point, nothing, or not numbers: thin or not, but never a panic. + assert_eq!(thin(&[(0., 0.), (5., 5.), (10., 10.), (0., 0.)], 1e-9), Ok(true)); + assert_eq!(thin(&[(3., 3.), (3., 3.), (3., 3.), (3., 3.)], 1e-9), Ok(true)); + assert_eq!(thin(&[], 1e-9), Ok(true)); + let _ = thin(&[(f64::NAN, 0.), (1., f64::NAN), (2., 2.), (f64::NAN, 0.)], 1e-9); + let _ = thin(&[(f64::INFINITY, 0.), (1., 1.), (0., f64::NEG_INFINITY), (f64::INFINITY, 0.)], 1e-9); + // The hull's buffers are asked for first. + assert_eq!(narrower_than(&p(&strip), 0.70, Room(&|_| false)), Err(Limit::Memory)); + // The hull: anticlockwise, without the points inside or on its edges, or a repeat. + let hull_of = |pts: &[(f64, f64)]| -> Vec<(f64, f64)> { + let pts: Vec> = pts.iter().map(|&(x, y)| geo::Coord { x, y }).collect(); + hull(&pts, ANY).unwrap().iter().map(|c| (c.x, c.y)).collect() + }; + assert_eq!( + hull_of(&[(0., 0.), (2., 0.), (1., 1.), (2., 2.), (1., 0.), (0., 2.), (0., 0.)]), + [(0., 0.), (2., 0.), (2., 2.), (0., 2.)] + ); + assert_eq!(hull_of(&[(0., 0.), (1., 1.), (2., 2.)]), [(0., 0.), (2., 2.)]); + assert_eq!(hull_of(&[(1., 1.)]), [(1., 1.)]); + assert_eq!(hull_of(&[]), []); + } + + #[test] + fn what_can_not_be_repaired_has_its_reason() { + assert_eq!(repair(None, &|_| true), Outcome::Gone(Gone::Null)); + assert_eq!(fix(&Geometry::Polygon(vec![])), Outcome::Gone(Gone::Null)); + assert_eq!(fix(&Geometry::Point(Coord::new(f64::NAN, 0.))), Outcome::Gone(Gone::NonFinite)); + assert_eq!(fix(&poly(&[SQUARE, &[(1., 1.), (f64::INFINITY, 1.), (2., 2.), (1., 1.)]])), Outcome::Gone(Gone::NonFinite)); + assert_eq!(fix(&poly(&[&[(0., 0.), (1., 0.), (0., 0.)]])), Outcome::Gone(Gone::TooFewPoints)); + assert_eq!(fix(&Geometry::Line(vec![ring(&[(0., 0.), (0., 0.)])])), Outcome::Gone(Gone::TooFewPoints)); + // Three distinct points in a row: a ring with no area. + assert_eq!(fix(&poly(&[&[(0., 0.), (1., 0.), (2., 0.), (0., 0.)]])), Outcome::Gone(Gone::NoArea)); + // A hole covering its whole shell. + assert_eq!(fix(&poly(&[SQUARE, SQUARE])), Outcome::Gone(Gone::NoArea)); + assert_eq!(fix(&Geometry::Point(Coord::new(1., 0.))), Outcome::Valid); + } + + /// A pentagram: 5 segments, each crossing two others. + fn pentagram() -> Geometry { + let at = |k: f64| { + let a = (90.0 + 144.0 * k).to_radians(); + Coord::new((10.0 * a.cos() * 1e6).round() / 1e6, (10.0 * a.sin() * 1e6).round() / 1e6) + }; + Geometry::Polygon(vec![vec![(0..=5).map(|k| if k == 5 { at(0.0) } else { at(f64::from(k)) }).collect()]]) + } + + #[test] + fn the_work_bound_and_the_overlays_memory_leave_a_feature_as_it_was() { + let star = pentagram(); + let first = validity::check(Some(&star), validity::MAX_WORK); + assert_eq!(first.contacts, 5); + // Its 6 coordinates and up to 2 vertices a contact, at 25 each: more than its checks take. + let need = (6 + 2 * 5) * OVERLAY_COST; + assert_eq!(kept(&validity::check(Some(&star), need - 1)), None, "the checks fit"); + let r = repaired(&star); + let Geometry::Polygon(parts) = &r.geometry else { panic!("a polygon") }; + assert_eq!((parts.len(), parts[0].len(), parts[0][0].len()), (1, 1, 11), "the star's outline, its centre filled"); + assert!(matches!(repair_with(Some(&star), need, &|_| true, &|_| true), Outcome::Repaired(_))); + assert_eq!(repair_with(Some(&star), need - 1, &|_| true, &|_| true), Outcome::Kept(Kept::Work)); + // The browser is asked for the overlay's memory, and refusing it leaves the feature as it was. + let asked = std::cell::Cell::new(0); + let r = repair_with(Some(&star), validity::MAX_WORK, &|_| true, &|need| { + asked.set(need); + false + }); + assert_eq!(r, Outcome::Kept(Kept::Memory)); + assert_eq!(asked.get(), (6 + 2 * 5) * OVERLAY_BYTES); + // Without an overlay nothing is asked of the browser. + let unclosed = poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.)]]); + assert!(matches!(repair(Some(&unclosed), &|_| false), Outcome::Repaired(_))); + } + + /// A star of 2001 points, each joined to the one 1000 on: about 2 million crossings, which the + /// checks find within the bound (they count a ring's crossings once), but which the overlay + /// would cut into 4 million vertices. + #[test] + fn crossings_are_charged_to_the_overlay() { + let (k, m) = (2001u32, 1000u32); + let star: Vec = (0..=k) + .map(|i| { + let a = std::f64::consts::TAU * f64::from((i * m) % k) / f64::from(k); + Coord::new(a.cos(), a.sin()) + }) + .collect(); + let g = Geometry::Polygon(vec![vec![star]]); + let c = validity::check(Some(&g), validity::MAX_WORK); + assert_eq!(c.codes(), [Code::SelfIntersection], "checked fully"); + assert!(c.contacts > 1_900_000, "{}", c.contacts); + assert_eq!(fix(&g), Outcome::Kept(Kept::Work)); + } + + /// Every reservation refused in turn: the feature is then left as it was, never dropped, + /// taken for valid, or written half repaired. + #[test] + fn a_reservation_refused_leaves_the_feature_as_it_was() { + let hole_out: Ring = &[(20., 20.), (21., 20.), (21., 21.), (20., 20.)]; + // A strip at 45°, 1.45 steps of the union's grid wide: only its hull tells it is wide enough. + let a = 1.45 * (-20f64).exp2() / 2f64.sqrt(); + let strip: Ring = &[(1000., 1000.), (1005., 1005.), (1005. - a, 1005. + a), (1000. - a, 1000. + a), (1000., 1000.)]; + for g in [ + poly(&[BOWTIE]), + multi(&[&[SQUARE], &[&[(5., 5.), (15., 5.), (15., 15.), (5., 15.), (5., 5.)]], &[strip]]), + multi(&[&[SQUARE, hole_out], &[&[(5., 5.), (15., 5.), (15., 15.), (5., 15.), (5., 5.)]]]), + poly(&[&[(0., 0.), (10., 0.), (10., 0.), (10., 10.)]]), + Geometry::Line(vec![ring(&[(0., 0.), (1., 1.), (1., 1.)])]), + ] { + let full = repaired(&g); + let tried = std::cell::Cell::new(0usize); + let _ = repair_with(Some(&g), validity::MAX_WORK, &|_| tried.replace(tried.get() + 1) < usize::MAX, &|_| true); + let n = tried.get(); + assert!(n >= 2, "{g:?}"); + for k in 0..n { + let tried = std::cell::Cell::new(0usize); + let allow = |_| { + let i = tried.get(); + tried.set(i + 1); + i != k + }; + assert_eq!( + repair_with(Some(&g), validity::MAX_WORK, &allow, &|_| true), + Outcome::Kept(Kept::Memory), + "{g:?}, refusing {k}" + ); + } + assert_eq!(repair_with(Some(&g), validity::MAX_WORK, &|_| true, &|_| true), Outcome::Repaired(full)); + } + } + + #[test] + fn a_feature_the_overlay_can_not_represent_is_left_as_it_was() { + // A bowtie a hundred-billionth of the smallest normal float across: no grid holds it. + let tiny: Vec<(f64, f64)> = BOWTIE.iter().map(|&(x, y)| (x * 1e-310, y * 1e-310)).collect(); + let g = poly(&[&tiny]); + assert_eq!(validity::check(Some(&g), validity::MAX_WORK).codes(), [Code::SelfIntersection]); + assert_eq!(fix(&g), Outcome::Kept(Kept::Failed)); + } + + #[test] + fn a_repair_that_is_not_valid_is_not_kept() { + let bad = Repaired { geometry: poly(&[BOWTIE]), codes: vec![Code::SelfIntersection], dropped: 0, narrow: 0, moved: None }; + assert_eq!(verified(bad, validity::MAX_WORK, &|_| true), Outcome::Kept(Kept::Failed)); + let busy = Repaired { geometry: poly(&[BOWTIE]), codes: vec![], dropped: 0, narrow: 0, moved: None }; + assert_eq!(verified(busy, 0, &|_| true), Outcome::Kept(Kept::Work)); + } +} diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 898b72560e..d1b3a3ef7b 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2,6 +2,7 @@ mod check_geometry; mod geometry; +mod repair_geometry; use geo::{ BooleanOps, Contains, ConvexHull, CoordsIter, InteriorPoint, Intersects, MultiPolygon, Orient, Relate, Simplify, orient::Direction, @@ -93,6 +94,17 @@ pub static TOOLS: &[ToolSpec] = &[ ], run: check_geometry::run, }, + ToolSpec { + id: "repairGeometry", + name: "Repair Geometry", + toolbox: "Data Management › Features", + summary: "Writes every feature with its geometry made valid: rings closed, repeated vertices and degenerate parts dropped, self-intersections, overlapping parts and holes outside their shell resolved. Says what was repaired in each.", + params: &[ + P { name: "input", label: "Input Features", kind: K::VectorLayer, required: true, default: "", help: "" }, + P { name: "output", label: "Output Name", kind: K::Text, required: false, default: "", help: "" }, + ], + run: repair_geometry::run, + }, ToolSpec { id: "buffer", name: "Buffer", diff --git a/crates/analysis/src/tools/check_geometry.rs b/crates/analysis/src/tools/check_geometry.rs index 6249ec74a1..11f7998ead 100644 --- a/crates/analysis/src/tools/check_geometry.rs +++ b/crates/analysis/src/tools/check_geometry.rs @@ -118,7 +118,11 @@ fn run_with_room( if n > 0 { let need = found .iter() - .filter_map(|item| ds.features.get(item.row).map(|f| row_bytes(f, width, out.fields.len(), item))) + .filter_map(|item| { + let f = ds.features.get(item.row)?; + let geometry = f.geometry.as_ref().filter(|_| !item.non_finite); + Some(row_bytes(&f.attrs, width, out.fields.len(), &[&item.problem, &item.detail], geometry)) + }) .fold(0u64, u64::saturating_add); if !has_room(need) { return Err(format!( @@ -155,15 +159,15 @@ fn run_with_room( Ok(ToolResult { output: Output::Layer(out), messages }) } -/// The memory an output row copying `f` (with `width` of its values, of `out_width` in the -/// output) takes: what [`point_bytes`] counts for a row, the two text fields, and the geometry, -/// held twice (the layer's copy and the map's projected one for drawing), each allocation with its -/// overhead. -fn row_bytes(f: &Feature, width: usize, out_width: usize, item: &Found) -> u64 { +/// The memory an output row copying `attrs` (`width` of them, of `out_width` values in the +/// output) takes, with the text fields `texts` added and `geometry`: what [`point_bytes`] counts for +/// a row, the texts, and the geometry, held twice (the layer's copy and the map's projected one for +/// drawing), each allocation with its overhead. +pub(super) fn row_bytes(attrs: &[Value], width: usize, out_width: usize, texts: &[&str], geometry: Option<&Geometry>) -> u64 { let u = |x: usize| u64::try_from(x).unwrap_or(u64::MAX); - let own = f.attrs.get(..width).unwrap_or(&f.attrs); - let text = u(item.problem.len()).saturating_add(u(item.detail.len())).saturating_add(2 * ALLOCATION_OVERHEAD); - let geometry = match f.geometry.as_ref().filter(|_| !item.non_finite) { + let own = attrs.get(..width).unwrap_or(attrs); + let text = texts.iter().fold(0u64, |n, t| n.saturating_add(u(t.len())).saturating_add(ALLOCATION_OVERHEAD)); + let geometry = match geometry { None => 0, Some(g) => { let lists = match g { @@ -229,7 +233,7 @@ fn notices(t: &Tally, problem: &str) -> Vec { } /// "1 feature", "2 features". -fn features(n: usize) -> String { +pub(super) fn features(n: usize) -> String { format!("{n} feature{}", if n == 1 { "" } else { "s" }) } @@ -258,7 +262,7 @@ fn details(problems: &[validity::Problem]) -> Result { /// `parts` joined with `sep`, in a string allocated fallibly at its exact length: it is kept until /// the output is copied. -fn joined>(parts: &[S], sep: &str) -> Result { +pub(super) fn joined>(parts: &[S], sep: &str) -> Result { let seps = sep.len().saturating_mul(parts.len().saturating_sub(1)); let len = parts.iter().map(|p| p.as_ref().len()).fold(seps, usize::saturating_add); let mut s = String::new(); diff --git a/crates/analysis/src/tools/repair_geometry.rs b/crates/analysis/src/tools/repair_geometry.rs new file mode 100644 index 0000000000..e6b5a2f363 --- /dev/null +++ b/crates/analysis/src/tools/repair_geometry.rs @@ -0,0 +1,450 @@ +//! Repair Geometry: a new layer of the features (the selected ones, or all) with their geometry +//! made valid after the OGC Simple Features rules ([`crate::repair`]), each saying what was +//! repaired. +use super::check_geometry::{features, joined, row_bytes}; +use super::*; +use crate::repair::{self, Gone, Kept, Outcome}; +use crate::validity::Code; + +const TOOL: &str = "Repair Geometry"; + +/// Writes every processed feature with its id, attributes and a text field of what was repaired: +/// valid ones as stored, repaired ones with their new geometry, those that can't be repaired +/// without geometry, and those left as they were (too complex, or the repair failed) as stored. +pub(super) fn run(inp: &dyn Inputs, p: &Json) -> Result { + run_with_room(inp, p, &web_has_room, |g| repair::repair(g, &web_has_room)) +} + +/// A feature's geometry in the output. +enum Written { + /// As stored. + Same, + /// None: it can't be repaired. + Without, + Repaired(Geometry), +} + +/// A processed feature, before it is copied to the output. +struct Item { + row: usize, + /// Its `repairs` value. + repairs: String, + geometry: Written, +} + +/// What the repairs did, by feature, for the notices. +#[derive(Default)] +struct Tally { + processed: usize, + repaired: usize, + /// Repaired features with each code, in `Code::ALL` order. + counts: [usize; Code::ALL.len()], + /// Features written without geometry, by reason, in `Gone::ALL` order. + gone: [usize; Gone::ALL.len()], + /// Features left as they were, because they needed more than the work bound, because the + /// memory for their repair couldn't be had, and because geo failed on them. + work: usize, + memory: usize, + failed: usize, + /// Rings and parts dropped (too few points, no area), and from how many features. + dropped: usize, + dropped_features: usize, + /// Holes and parts dropped as narrower than a step of the overlay's grid in any direction, and + /// from how many features (each with `no_area` in its repairs). + narrow: usize, + narrow_features: usize, + /// Features rebuilt by the overlay, and how far their coordinates may have moved at most. + overlaid: usize, + moved: f64, + /// Features with more attribute values than the layer has fields. + extra: usize, +} + +impl Tally { + fn count(&self, code: Code) -> usize { + Code::ALL.iter().position(|c| *c == code).and_then(|k| self.counts.get(k)).copied().unwrap_or(0) + } +} + +/// [`run`], asking `has_room` whether the browser can give the output's memory (see +/// [`web_has_room`]) before any feature is copied, and repairing each geometry with `repair`. +fn run_with_room( + inp: &dyn Inputs, + p: &Json, + has_room: &dyn Fn(u64) -> bool, + repair: impl Fn(Option<&Geometry>) -> Outcome, +) -> Result { + let ds = input(inp, p, "input")?; + if ds.geometry_type == GeometryType::None { + return Err(format!("{TOOL} needs a layer with geometry: '{}' is a table", ds.name)); + } + let mut rows = rows(inp, p, "input", ds); + rows.sort_unstable(); + rows.dedup(); + let width = ds.fields.len(); + let mut out = copy_schema(ds, out_name(p, format!("{}_repaired", ds.name)), ds.geometry_type); + out.crs = ds.crs; + out.crs_incomplete = ds.crs_incomplete; + out.zm_dropped = ds.zm_dropped; + let mut notes = Vec::new(); + let field = unique_field(&out, "repairs"); + if field != "repairs" { + notes.push(format!("{TOOL}: '{}' already has a field named repairs, so the output's repairs field is {field}", ds.name)); + } + out.fields.push(Field::new(&field, FieldType::Text)); + + // The repairs first, keeping what the output needs of each feature in memory allocated + // fallibly, as the copies below are. + let mut tally = Tally { processed: rows.len(), ..Tally::default() }; + let no_memory = |e: TryReserveError| format!("{TOOL}: not enough memory for the repairs of {}: {e}", features(rows.len())); + let mut items: Vec = Vec::new(); + items.try_reserve_exact(rows.len()).map_err(no_memory)?; + crate::control::begin(rows.len()); + for (i, row) in rows.iter().enumerate() { + crate::control::check()?; + crate::control::reached(i); + let Some(f) = ds.features.get(*row) else { continue }; + tally.extra += usize::from(f.attrs.len() > width); + let (repairs, geometry) = match repair(f.geometry.as_ref()) { + Outcome::Valid => (String::new(), Written::Same), + Outcome::Repaired(r) => { + tally.repaired += 1; + for code in &r.codes { + if let Some(n) = Code::ALL.iter().position(|c| c == code).and_then(|k| tally.counts.get_mut(k)) { + *n += 1; + } + } + tally.dropped += r.dropped; + tally.dropped_features += usize::from(r.dropped > 0); + tally.narrow += r.narrow; + tally.narrow_features += usize::from(r.narrow > 0); + if let Some(moved) = r.moved { + tally.overlaid += 1; + tally.moved = tally.moved.max(moved); + } + // A hole or part with no area on the overlay's grid is said after the codes. + let narrow = (r.narrow > 0).then_some(Gone::NoArea.as_str()); + let names: Vec<&str> = r.codes.iter().map(|c| c.as_str()).chain(narrow).collect(); + (joined(&names, ", ").map_err(no_memory)?, Written::Repaired(r.geometry)) + } + Outcome::Gone(why) => { + if let Some(n) = Gone::ALL.iter().position(|g| *g == why).and_then(|k| tally.gone.get_mut(k)) { + *n += 1; + } + (joined(&[why.as_str()], "").map_err(no_memory)?, Written::Without) + } + Outcome::Kept(why) => { + match why { + Kept::Work => tally.work += 1, + Kept::Memory => tally.memory += 1, + Kept::Failed => tally.failed += 1, + } + (joined(&[why.as_str()], "").map_err(no_memory)?, Written::Same) + } + }; + // Within the room reserved for one item a row. + items.push(Item { row: *row, repairs, geometry }); + } + + // Then the copies, once the memory for them is known to be there: in the browser, whose memory + // has a ceiling, it is asked for before anything is copied, and every copy is allocated + // fallibly, so running out of memory stops the tool with a message. + let n = items.len(); + if n > 0 { + let need = items + .iter() + .filter_map(|item| { + let f = ds.features.get(item.row)?; + let geometry = match &item.geometry { + Written::Same => f.geometry.as_ref(), + Written::Without => None, + Written::Repaired(g) => Some(g), + }; + Some(row_bytes(&f.attrs, width, out.fields.len(), &[&item.repairs], geometry)) + }) + .fold(0u64, u64::saturating_add); + if !has_room(need) { + return Err(format!( + "{TOOL}: the {} would need about {} MiB, more than the browser can give now: select fewer features", + features(n), + need.div_ceil(1 << 20) + )); + } + } + let no_memory = |e: TryReserveError| format!("{TOOL}: not enough memory for the {}: {e}", features(n)); + let mut list: Vec = Vec::new(); + list.try_reserve_exact(n).map_err(no_memory)?; + for item in items { + // The copies can take a while for many large features: a cancel still stops them. + crate::control::check()?; + let Some(f) = ds.features.get(item.row) else { continue }; + let mut attrs = Vec::new(); + attrs.try_reserve_exact(width.saturating_add(1)).map_err(no_memory)?; + for v in f.attrs.iter().take(width) { + attrs.push(try_clone_value(v).map_err(no_memory)?); + } + attrs.resize(width, Value::Null); + attrs.push(Value::Text(item.repairs)); + let geometry = match item.geometry { + Written::Same => f.geometry.as_ref().map(crate::merge::clone_geometry).transpose().map_err(no_memory)?, + Written::Without => None, + Written::Repaired(g) => Some(g), + }; + // The input's id, so a row leads back to its feature. + list.push(Feature { id: f.id, geometry, attrs }); + } + out.features = list.into(); + + let mut messages = notices(&tally, &field); + messages.extend(notes); + Ok(ToolResult { output: Output::Layer(out), messages }) +} + +/// The run's notices: the result in one line, then one notice per kind of feature not repaired, +/// and per kind of change beyond the codes: pieces dropped (with too few points or no area, or +/// narrower than a step of the overlay's grid in any direction), holes taken out, parts merged or +/// split, coordinates moved. `field` is the output's repairs field. +fn notices(t: &Tally, field: &str) -> Vec { + let mut messages = vec![summary(t)]; + let is = |n: usize| if n == 1 { "is" } else { "are" }; + let was = |n: usize| if n == 1 { "was" } else { "were" }; + let gone: usize = t.gone.iter().sum(); + if gone > 0 { + let by: Vec = Gone::ALL.iter().zip(t.gone).filter(|(_, n)| *n > 0).map(|(g, n)| format!("{}: {n}", g.as_str())).collect(); + messages.push(format!( + "{TOOL}: {} couldn't be repaired and {} written without geometry ({})", + features(gone), + is(gone), + by.join(", ") + )); + } + if t.work > 0 { + messages.push(format!( + "{TOOL}: {} {} too complex to repair within the work limit and {} written unchanged, with too_complex in {field}", + features(t.work), + was(t.work), + is(t.work) + )); + } + if t.memory > 0 { + messages.push(format!( + "{TOOL}: {} couldn't be repaired, because there wasn't enough memory to repair {}, and {} written unchanged, with too_complex in {field}", + features(t.memory), + if t.memory == 1 { "it" } else { "them" }, + is(t.memory) + )); + } + if t.failed > 0 { + messages.push(format!( + "{TOOL}: {} couldn't be repaired, because geo's geometry operations failed on {}, and {} written unchanged, with repair_failed in {field}", + features(t.failed), + if t.failed == 1 { "it" } else { "them" }, + is(t.failed) + )); + } + if t.dropped > 0 { + messages.push(format!( + "{TOOL}: {} {} with too few distinct points or no area {} dropped from {}", + t.dropped, + if t.dropped == 1 { "ring or part" } else { "rings or parts" }, + was(t.dropped), + features(t.dropped_features) + )); + } + if t.narrow > 0 { + messages.push(format!( + "{TOOL}: {} {} narrower than a step of the overlay's grid, which would leave {} no area, {} dropped from {}, with {} in {field}", + t.narrow, + if t.narrow == 1 { "hole or part" } else { "holes or parts" }, + if t.narrow == 1 { "it" } else { "them" }, + was(t.narrow), + features(t.narrow_features), + Gone::NoArea.as_str() + )); + } + let holes = t.count(Code::HoleOutsideShell); + if holes > 0 { + messages.push(format!( + "{TOOL}: holes not inside their exterior ring were taken out of {}; what of a hole is inside its exterior ring stays a hole", + features(holes) + )); + } + let merged = t.count(Code::PartsOverlap); + if merged > 0 { + messages.push(format!("{TOOL}: the parts that overlapped or touched along a line were merged in {}", features(merged))); + } + let split = t.count(Code::DisconnectedInterior); + if split > 0 { + messages.push(format!( + "{TOOL}: {} whose interior was cut into pieces by rings touching at points {} written with each piece as a part of its own", + features(split), + is(split) + )); + } + if t.overlaid > 0 { + messages.push(format!( + "{TOOL}: {} {} rebuilt by geo's boolean overlay, which rounds coordinates to a fine grid: they may have moved by up to {} degrees of longitude or latitude, and vertices in the middle of straight edges were left out", + features(t.overlaid), + was(t.overlaid), + rounded_up(t.moved) + )); + } + if t.extra > 0 { + messages.push(format!( + "{TOOL}: {} had more attribute values than the layer has fields; the extra values aren't in the output", + features(t.extra) + )); + } + messages +} + +/// `v`, positive, to two significant digits rounded up, so a bound stays one: "7.5e-8". +fn rounded_up(v: f64) -> String { + let mut e = v.log10().floor(); + let mut m = (v / 10f64.powf(e) * 10.0 - 1e-9).ceil() / 10.0; + if m >= 10.0 { + m /= 10.0; + e += 1.0; + } + format!("{m:.1}e{e}") +} + +/// The run's result in one line, with the repaired features having each code. +fn summary(t: &Tally) -> String { + if t.processed == 0 { + return format!("{TOOL}: no features to repair"); + } + let others = t.gone.iter().sum::() + t.work + t.memory + t.failed; + if t.repaired == 0 && others == 0 { + return format!("{TOOL}: no repairs needed in {}", features(t.processed)); + } + let mut by_code: Vec = + Code::ALL.iter().zip(t.counts).filter(|(_, n)| *n > 0).map(|(c, n)| format!("{}: {n}", c.as_str())).collect(); + if t.narrow_features > 0 { + by_code.push(format!("{}: {}", Gone::NoArea.as_str(), t.narrow_features)); + } + let codes = if by_code.is_empty() { String::new() } else { format!(" ({})", by_code.join(", ")) }; + format!("{TOOL}: repaired {} of {}{codes}", t.repaired, features(t.processed)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The inputs: one layer, with nothing selected. + struct One(Dataset); + impl Inputs for One { + fn dataset(&self, _: &str) -> Option<&Dataset> { + Some(&self.0) + } + fn selection(&self, _: &str) -> &[usize] { + &[] + } + } + + fn ring(pts: &[(f64, f64)]) -> Vec { + pts.iter().map(|&(x, y)| Coord::new(x, y)).collect() + } + + /// The output's memory is asked for before any feature is copied, and a browser that can't + /// give it refuses the run with how much it needed. + #[test] + fn the_output_is_refused_when_the_browser_has_no_room_for_it() { + let mut ds = Dataset::new("input", GeometryType::Polygon, vec![Field::new("name", FieldType::Text)]); + let open = ring(&[(0., 0.), (2., 0.), (2., 2.), (0., 2.)]); + ds.push_next(Some(Geometry::Polygon(vec![vec![open]])), vec![Value::Text("x".repeat(1 << 21))]); + ds.push_next(None, vec![Value::Text("none".into())]); + let inp = One(ds); + let p = serde_json::json!({"input": "input"}); + let asked = std::cell::Cell::new(0); + let r = run_with_room( + &inp, + &p, + &|need| { + asked.set(need); + true + }, + |g| repair::repair(g, &|_| true), + ) + .unwrap(); + let Output::Layer(out) = &r.output else { panic!("a layer") }; + assert_eq!(out.features.len(), 2); + // The rows with their copied text and codes, and the repaired geometry twice: its five + // coordinates and two lists, each with an allocation's overhead. The row without geometry + // has none. + let closed = Geometry::Polygon(vec![vec![ring(&[(0., 0.), (2., 0.), (2., 2.), (0., 2.), (0., 0.)])]]); + let geometry = rustgis_data::source::chunk_memory::geometry_bytes(&closed) as u64 + 3 * ALLOCATION_OVERHEAD; + let first = point_bytes(&inp.0.features[0].attrs, 2) + "ring_not_closed".len() as u64 + ALLOCATION_OVERHEAD + 2 * geometry; + let second = point_bytes(&inp.0.features[1].attrs, 2) + "null_geometry".len() as u64 + ALLOCATION_OVERHEAD; + assert_eq!(asked.get(), first + second); + let e = run_with_room(&inp, &p, &|_| false, |g| repair::repair(g, &|_| true)).err().unwrap(); + assert_eq!(e, "Repair Geometry: the 2 features would need about 3 MiB, more than the browser can give now: select fewer features"); + } + + /// Each reason a feature is left as it was has a notice of its own, with its count: geo + /// failing natively can't be brought about on purpose, so its notice is checked here. + #[test] + fn features_left_as_they_were_are_counted_by_reason() { + let t = Tally { processed: 9, failed: 1, ..Tally::default() }; + assert_eq!( + notices(&t, "repairs"), + [ + "Repair Geometry: repaired 0 of 9 features", + "Repair Geometry: 1 feature couldn't be repaired, because geo's geometry operations failed on it, and is written unchanged, with repair_failed in repairs", + ] + ); + let t = Tally { processed: 9, work: 2, memory: 3, failed: 2, ..Tally::default() }; + assert_eq!( + notices(&t, "repairs_2")[1..], + [ + "Repair Geometry: 2 features were too complex to repair within the work limit and are written unchanged, with too_complex in repairs_2", + "Repair Geometry: 3 features couldn't be repaired, because there wasn't enough memory to repair them, and are written unchanged, with too_complex in repairs_2", + "Repair Geometry: 2 features couldn't be repaired, because geo's geometry operations failed on them, and are written unchanged, with repair_failed in repairs_2", + ] + ); + assert_eq!(notices(&Tally::default(), "repairs"), ["Repair Geometry: no features to repair"]); + } + + /// A cancel stops the repairs at the next feature, and the copies at the next row. + #[test] + fn a_cancel_stops_the_repairs_and_the_copies() { + let mut ds = Dataset::new("input", GeometryType::Polygon, vec![]); + for _ in 0..3 { + ds.push_next(Some(Geometry::Polygon(vec![vec![ring(&[(0., 0.), (2., 0.), (2., 2.), (0., 2.)])]])), vec![]); + } + let inp = One(ds); + let p = serde_json::json!({"input": "input"}); + let control = crate::control::Control::new(); + let _installed = crate::control::install(Some(control.clone())); + let repaired = std::cell::Cell::new(0); + let r = run_with_room(&inp, &p, &|_| true, |g| { + repaired.set(repaired.get() + 1); + control.cancel(); + repair::repair(g, &|_| true) + }); + assert_eq!(r.err().as_deref(), Some(crate::control::CANCELLED)); + assert_eq!(repaired.get(), 1, "stopped before the second feature"); + + let control = crate::control::Control::new(); + let _installed = crate::control::install(Some(control.clone())); + let r = run_with_room( + &inp, + &p, + &|_| { + control.cancel(); + true + }, + |g| repair::repair(g, &|_| true), + ); + assert_eq!(r.err().as_deref(), Some(crate::control::CANCELLED), "stopped in the copies"); + } + + #[test] + fn bounds_are_rounded_up() { + assert_eq!(rounded_up(5.0 * (-26f64).exp2()), "7.5e-8"); + assert_eq!(rounded_up(5.0 * (-29f64).exp2()), "9.4e-9"); + assert_eq!(rounded_up(2.5e-3), "2.5e-3"); + assert_eq!(rounded_up(9.96), "1.0e1"); + assert_eq!(rounded_up(1.0), "1.0e0"); + } +} diff --git a/crates/analysis/src/validity.rs b/crates/analysis/src/validity.rs index 72349bf494..29fbb626ca 100644 --- a/crates/analysis/src/validity.rs +++ b/crates/analysis/src/validity.rs @@ -1,6 +1,6 @@ //! Geometry validity in stored coordinates, after the OGC Simple Features rules (OGC 06-103r4, //! ISO 19125-1: 6.1.11.1 Polygon, 6.1.12 MultiPolygon): what is wrong with one geometry, as a list -//! of [`Problem`]s. Check Geometry reports them; a repair can start from the same list. +//! of [`Problem`]s. Check Geometry reports them; Repair Geometry starts from them ([`crate::repair`]). //! //! - Rings are read as stored: an unclosed ring is reported, and then checked as if it were closed. //! Repeated vertices are reported, and then left out of the other checks. @@ -273,6 +273,10 @@ pub(crate) struct Checked { /// Some rings or parts weren't checked against each other: one of them has too few points, /// non-finite coordinates or crosses itself, or (for parts) has a ring problem of its own. pub unrelated: bool, + /// The pairs of ring segments the sweep found meeting, apart from neighbours at their shared + /// vertex: each adds at most two vertices where an overlay cuts the rings (see + /// [`crate::repair`]). + pub contacts: u64, } impl Checked { @@ -374,7 +378,7 @@ fn vertex_checks(problems: &mut Vec, at: At, pts: &[Coord], ring: bool) } /// How many distinct points `pts` has, counting up to `limit`. -fn distinct_up_to(pts: &[Coord], limit: usize) -> usize { +pub(crate) fn distinct_up_to(pts: &[Coord], limit: usize) -> usize { let mut seen: Vec = Vec::with_capacity(limit); for c in pts { if seen.len() >= limit { @@ -394,11 +398,11 @@ fn repeated_vertices(pts: &[Coord]) -> usize { /// Reserves a feature's buffers fallibly, asking first whether to try (see [`check_with`]). #[derive(Clone, Copy)] -struct Room<'a>(&'a dyn Fn(usize) -> bool); +pub(crate) struct Room<'a>(pub(crate) &'a dyn Fn(usize) -> bool); impl Room<'_> { /// An empty vector with room for `n` elements. - fn vec(self, n: usize) -> Result, Limit> { + pub(crate) fn vec(self, n: usize) -> Result, Limit> { let mut v = Vec::new(); if !(self.0)(n) { return Err(Limit::Memory); @@ -408,7 +412,7 @@ impl Room<'_> { } /// At most `n` of `items`, in a vector reserved for `n`, so collecting them allocates nothing more. - fn collect(self, n: usize, items: impl Iterator) -> Result, Limit> { + pub(crate) fn collect(self, n: usize, items: impl Iterator) -> Result, Limit> { let mut v = self.vec(n)?; v.extend(items.take(n)); Ok(v) @@ -437,7 +441,7 @@ impl Room<'_> { } /// Room for one more element in `v`, growing it as `push` would. - fn grow(self, v: &mut Vec) -> Result<(), Limit> { + pub(crate) fn grow(self, v: &mut Vec) -> Result<(), Limit> { if v.len() < v.capacity() { return Ok(()); } @@ -650,6 +654,7 @@ fn polygon(out: &mut Checked, polys: &[Vec>], work: &mut Work) { if contact == Contact::Touch && s.next_to(t) { continue; } + out.contacts = out.contacts.saturating_add(1); if !selfx.contains_key(&(s.part, s.ring)) && room.map(&mut selfx).is_err() { out.problems.push(Problem::TooComplex { limit: Limit::Memory }); return; @@ -660,6 +665,7 @@ fn polygon(out: &mut Checked, polys: &[Vec>], work: &mut Work) { *worst = (*worst).max(contact); new } else { + out.contacts = out.contacts.saturating_add(1); let meet = if s.part == t.part { let key = (s.part, s.ring.min(t.ring), s.ring.max(t.ring)); if !rings_meet.contains_key(&key) && room.map(&mut rings_meet).is_err() { diff --git a/crates/analysis/tests/repair_geometry.rs b/crates/analysis/tests/repair_geometry.rs new file mode 100644 index 0000000000..08709f3836 --- /dev/null +++ b/crates/analysis/tests/repair_geometry.rs @@ -0,0 +1,920 @@ +//! Repair Geometry through the public tool dispatch: each repair with its exact result and code, +//! several in one feature, valid features copied as they are, both ring windings, lines and +//! points, rows that can't be repaired, the work bound, selections, attributes and CRS, the +//! repairs field's name, how far the overlay moves coordinates, and input read by the real +//! GeoJSON and Shapefile readers. Every repaired geometry passes Check Geometry. +use rustgis_analysis::{Inputs, Output, ToolResult}; +use rustgis_data::{Coord, Dataset, Feature, Field, FieldType, Geometry, GeometryType, Value}; +use serde_json::json; + +struct Layer { + ds: Dataset, + selected: Vec, +} +impl Inputs for Layer { + fn dataset(&self, name: &str) -> Option<&Dataset> { + (name == "input").then_some(&self.ds) + } + fn selection(&self, _: &str) -> &[usize] { + &self.selected + } +} + +type Ring<'a> = &'a [(f64, f64)]; + +fn layer(ty: GeometryType) -> Layer { + Layer { ds: Dataset::new("input", ty, vec![Field::new("name", FieldType::Text)]), selected: vec![] } +} +fn pts(r: Ring) -> Vec { + r.iter().map(|&(x, y)| Coord::new(x, y)).collect() +} +fn multi(parts: &[&[Ring]]) -> Geometry { + Geometry::Polygon(parts.iter().map(|rings| rings.iter().map(|r| pts(r)).collect()).collect()) +} +fn poly(rings: &[Ring]) -> Geometry { + multi(&[rings]) +} +fn line(parts: &[Ring]) -> Geometry { + Geometry::Line(parts.iter().map(|r| pts(r)).collect()) +} +fn add(l: &mut Layer, g: Option, name: &str) { + l.ds.push_next(g, vec![Value::Text(name.into())]); +} +fn run(l: &Layer) -> ToolResult { + rustgis_analysis::run("repairGeometry", l, &json!({"input": "input"})).unwrap() +} +fn dataset(r: &ToolResult) -> &Dataset { + let Output::Layer(ds) = &r.output else { panic!("expected a layer") }; + ds +} +fn text(ds: &Dataset, row: usize, field: &str) -> String { + match ds.value(row, ds.field_index(field).unwrap()) { + Value::Text(t) => t.clone(), + v => panic!("{field} is {v:?}"), + } +} +/// The output's geometries and repairs, row by row. +fn rows(r: &ToolResult) -> Vec<(Option, String)> { + let ds = dataset(r); + (0..ds.features.len()).map(|i| (ds.features[i].geometry.clone(), text(ds, i, "repairs"))).collect() +} +/// The geometry and repairs of a single feature of type `ty` holding `g`. +fn one(ty: GeometryType, g: Option) -> (Option, String) { + let mut l = layer(ty); + add(&mut l, g, "x"); + let r = run(&l); + assert_eq!(dataset(&r).features.len(), 1, "{:?}", r.messages); + valid(dataset(&r)); + rows(&r).remove(0) +} + +/// Every geometry of `ds` passes Check Geometry: its output is empty but for rows without geometry. +fn valid(ds: &Dataset) { + let mut copy = ds.clone(); + copy.name = "input".into(); + let l = Layer { ds: copy, selected: vec![] }; + let r = rustgis_analysis::run("checkGeometry", &l, &json!({"input": "input"})).unwrap(); + let out = dataset(&r); + let bad: Vec<(String, Option)> = (0..out.features.len()) + .filter(|&i| text(out, i, "problem") != "null_geometry") + .map(|i| (text(out, i, "detail"), out.features[i].geometry.clone())) + .collect(); + assert!(bad.is_empty(), "not valid: {bad:?}"); +} + +/// A polygon geometry with each ring's closing point left out and the ring started at its +/// smallest vertex (by x, then y), and the parts in order of their first vertex, so geometries +/// that differ only in where rings start compare equal. Rings keep their direction. +fn norm(g: &Geometry) -> Geometry { + let Geometry::Polygon(parts) = g else { return g.clone() }; + let ring = |r: &Vec| { + let mut v: Vec = r.clone(); + if v.len() > 1 && v.first() == v.last() { + v.pop(); + } + let min = (0..v.len()).min_by(|&a, &b| (v[a].x, v[a].y).partial_cmp(&(v[b].x, v[b].y)).unwrap()).unwrap_or(0); + v.rotate_left(min); + v + }; + let mut parts: Vec>> = parts + .iter() + .map(|rings| { + let mut rings: Vec> = rings.iter().map(ring).collect(); + rings[1..].sort_by(|a, b| (a[0].x, a[0].y).partial_cmp(&(b[0].x, b[0].y)).unwrap()); + rings + }) + .collect(); + parts.sort_by(|a, b| (a[0][0].x, a[0][0].y).partial_cmp(&(b[0][0].x, b[0][0].y)).unwrap()); + Geometry::Polygon(parts) +} +/// Every coordinate of `g`, in order. +fn coords(g: &Geometry) -> Vec { + match g { + Geometry::Point(c) => vec![*c], + Geometry::MultiPoint(v) => v.clone(), + Geometry::Line(parts) => parts.iter().flatten().copied().collect(), + Geometry::Polygon(parts) => parts.iter().flatten().flatten().copied().collect(), + } +} +/// Twice the signed area of a ring (positive counter-clockwise). +fn signed(r: &[Coord]) -> f64 { + r.windows(2).map(|w| w[0].x * w[1].y - w[1].x * w[0].y).sum() +} +/// A polygon geometry's area: each part's exterior ring less its holes. +fn area(g: &Geometry) -> f64 { + let Geometry::Polygon(parts) = g else { panic!("a polygon: {g:?}") }; + parts.iter().map(|rings| signed(&rings[0]).abs() / 2.0 - rings[1..].iter().map(|h| signed(h).abs() / 2.0).sum::()).sum() +} +/// Every ring reversed, as the other of GeoJSON's and Shapefile's windings has it. +fn reversed(g: &Geometry) -> Geometry { + let Geometry::Polygon(parts) = g else { return g.clone() }; + Geometry::Polygon(parts.iter().map(|rings| rings.iter().map(|r| r.iter().rev().copied().collect()).collect()).collect()) +} + +const SQUARE: Ring = &[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 0.)]; +const HOLE: Ring = &[(2., 2.), (4., 2.), (4., 4.), (2., 4.), (2., 2.)]; +const BOWTIE: Ring = &[(0., 0.), (2., 2.), (2., 0.), (0., 2.), (0., 0.)]; +const OUTSIDE: Ring = &[(20., 20.), (21., 20.), (21., 21.), (20., 20.)]; +/// A hole touching the shell at (0, 5) and (10, 5), splitting the square in two. +const ACROSS: Ring = &[(0., 5.), (5., 2.), (10., 5.), (5., 8.), (0., 5.)]; +const OVERLAPPING: Ring = &[(5., 5.), (15., 5.), (15., 15.), (5., 15.), (5., 5.)]; +/// Three holes each touching the next at a vertex, enclosing a triangle of the interior. +const LOOP: [Ring; 3] = + [&[(3., 3.), (5., 6.), (1., 6.), (3., 3.)], &[(3., 3.), (5., 1.), (7., 3.), (3., 3.)], &[(7., 3.), (9., 6.), (5., 6.), (7., 3.)]]; + +/// The fixtures the overlay repairs, by name, with the area each comes out with. +fn rebuilt() -> Vec<(&'static str, Geometry, f64)> { + vec![ + ("bowtie", poly(&[BOWTIE]), 2.0), + ("touching itself", poly(&[&[(0., 0.), (4., 0.), (2., 2.), (4., 4.), (0., 4.), (2., 2.), (0., 0.)]]), 8.0), + ("hole outside", poly(&[SQUARE, OUTSIDE]), 100.0), + ("hole half outside", poly(&[SQUARE, &[(8., 4.), (12., 4.), (12., 6.), (8., 6.), (8., 4.)]]), 96.0), + ("holes overlapping", poly(&[SQUARE, HOLE, &[(3., 3.), (5., 3.), (5., 5.), (3., 5.), (3., 3.)]]), 93.0), + ("hole along the shell", poly(&[SQUARE, &[(0., 2.), (3., 2.), (3., 4.), (0., 4.), (0., 2.)]]), 94.0), + ("split interior", poly(&[SQUARE, ACROSS]), 70.0), + ("holes in a loop", poly(&[SQUARE, LOOP[0], LOOP[1], LOOP[2]]), 84.0), + ("parts overlapping", multi(&[&[SQUARE], &[OVERLAPPING]]), 175.0), + ("parts along a line", multi(&[&[SQUARE], &[&[(10., 0.), (20., 0.), (20., 10.), (10., 10.), (10., 0.)]]]), 200.0), + ("hole filled by another part", multi(&[&[SQUARE, HOLE], &[&[(1., 1.), (5., 1.), (5., 5.), (1., 5.), (1., 1.)]]]), 100.0), + ("bowtie part over a square", multi(&[&[SQUARE], &[&[(5., 5.), (15., 15.), (15., 5.), (5., 15.), (5., 5.)]]]), 137.5), + ] +} + +#[test] +fn each_repair_on_its_own() { + use GeometryType::{Point, Polygon, Polyline}; + let fixed = |g: Geometry, code: &str| (Some(g), code.to_string()); + let gone = |code: &str| (None, code.to_string()); + let cases: Vec<(GeometryType, Option, (Option, String))> = vec![ + // The fixes that move no coordinate. + (Polygon, Some(poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.)]])), fixed(poly(&[SQUARE]), "ring_not_closed")), + ( + Polygon, + Some(poly(&[SQUARE, &[(2., 2.), (4., 2.), (4., 2.), (4., 2.), (4., 4.), (2., 4.), (2., 2.)]])), + fixed(poly(&[SQUARE, HOLE]), "repeated_vertex"), + ), + ( + Polygon, + Some(poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 0.), (0., 0.)]])), + fixed(poly(&[SQUARE]), "repeated_vertex"), + ), + (Polygon, Some(poly(&[SQUARE, &[(2., 2.), (3., 3.), (2., 2.)]])), fixed(poly(&[SQUARE]), "too_few_points")), + (Polygon, Some(multi(&[&[SQUARE], &[&[(20., 0.), (21., 0.), (20., 0.)], HOLE]])), fixed(poly(&[SQUARE]), "too_few_points")), + (Polygon, Some(multi(&[&[SQUARE], &[]])), fixed(poly(&[SQUARE]), "too_few_points")), + ( + Polyline, + Some(line(&[&[(0., 0.), (1., 1.), (1., 1.), (2., 0.)]])), + fixed(line(&[&[(0., 0.), (1., 1.), (2., 0.)]]), "repeated_vertex"), + ), + (Polyline, Some(line(&[&[(0., 0.), (1., 1.)], &[(5., 5.)]])), fixed(line(&[&[(0., 0.), (1., 1.)]]), "too_few_points")), + ( + Polyline, + Some(line(&[&[(0., 0.), (1., 1.)], &[(5., 5.), (5., 5.)]])), + fixed(line(&[&[(0., 0.), (1., 1.)]]), "too_few_points, repeated_vertex"), + ), + // The rebuild by area. + ( + Polygon, + Some(poly(&[BOWTIE])), + fixed(multi(&[&[&[(0., 0.), (1., 1.), (0., 2.)]], &[&[(1., 1.), (2., 0.), (2., 2.)]]]), "self_intersection"), + ), + ( + Polygon, + Some(poly(&[&[(0., 0.), (4., 0.), (2., 2.), (4., 4.), (0., 4.), (2., 2.), (0., 0.)]])), + fixed(multi(&[&[&[(0., 0.), (4., 0.), (2., 2.)]], &[&[(0., 4.), (2., 2.), (4., 4.)]]]), "self_intersection"), + ), + (Polygon, Some(poly(&[SQUARE, OUTSIDE])), fixed(poly(&[SQUARE]), "hole_outside_shell")), + ( + Polygon, + Some(poly(&[SQUARE, HOLE, &[(3., 3.), (5., 3.), (5., 5.), (3., 5.), (3., 3.)]])), + fixed(poly(&[SQUARE, &[(2., 2.), (2., 4.), (3., 4.), (3., 5.), (5., 5.), (5., 3.), (4., 3.), (4., 2.)]]), "rings_intersect"), + ), + ( + Polygon, + Some(poly(&[SQUARE, &[(0., 2.), (3., 2.), (3., 4.), (0., 4.), (0., 2.)]])), + fixed(poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.), (0., 4.), (3., 4.), (3., 2.), (0., 2.)]]), "rings_intersect"), + ), + ( + Polygon, + Some(poly(&[SQUARE, ACROSS])), + fixed( + multi(&[ + &[&[(0., 0.), (10., 0.), (10., 5.), (5., 2.), (0., 5.)]], + &[&[(0., 5.), (5., 8.), (10., 5.), (10., 10.), (0., 10.)]], + ]), + "disconnected_interior", + ), + ), + ( + Polygon, + Some(poly(&[SQUARE, LOOP[0], LOOP[1], LOOP[2]])), + fixed( + multi(&[&[SQUARE, &[(1., 6.), (9., 6.), (7., 3.), (5., 1.), (3., 3.)]], &[&[(3., 3.), (7., 3.), (5., 6.)]]]), + "disconnected_interior", + ), + ), + ( + Polygon, + Some(multi(&[&[SQUARE], &[OVERLAPPING]])), + fixed(poly(&[&[(0., 0.), (10., 0.), (10., 5.), (15., 5.), (15., 15.), (5., 15.), (5., 10.), (0., 10.)]]), "parts_overlap"), + ), + ( + Polygon, + Some(multi(&[&[SQUARE], &[&[(10., 0.), (20., 0.), (20., 10.), (10., 10.), (10., 0.)]]])), + fixed(poly(&[&[(0., 0.), (20., 0.), (20., 10.), (0., 10.)]]), "parts_overlap"), + ), + // What can't be repaired is written without geometry. + (Polygon, None, gone("null_geometry")), + (Polygon, Some(Geometry::Polygon(vec![])), gone("null_geometry")), + (Polyline, Some(Geometry::Line(vec![vec![]])), gone("null_geometry")), + (Point, Some(Geometry::MultiPoint(vec![])), gone("null_geometry")), + (Point, Some(Geometry::Point(Coord::new(f64::NAN, 1.))), gone("non_finite_coordinate")), + (Point, Some(Geometry::MultiPoint(pts(&[(0., 0.), (f64::INFINITY, 1.)]))), gone("non_finite_coordinate")), + (Polygon, Some(poly(&[SQUARE, &[(2., 2.), (f64::NAN, 2.), (4., 4.), (2., 2.)]])), gone("non_finite_coordinate")), + (Polyline, Some(line(&[&[(0., 0.), (f64::NEG_INFINITY, 1.)]])), gone("non_finite_coordinate")), + (Polygon, Some(poly(&[&[(0., 0.), (1., 0.), (0., 0.)]])), gone("too_few_points")), + (Polyline, Some(line(&[&[(1., 1.), (1., 1.)], &[(2., 2.)]])), gone("too_few_points")), + (Polygon, Some(poly(&[&[(0., 0.), (1., 0.), (2., 0.), (0., 0.)]])), gone("no_area")), + (Polygon, Some(poly(&[SQUARE, SQUARE])), gone("no_area")), + ]; + for (ty, g, (want, code)) in cases { + let (got, repairs) = one(ty, g.clone()); + assert_eq!(repairs, code, "{g:?}"); + assert_eq!(got.as_ref().map(norm), want.as_ref().map(norm), "{g:?}"); + } +} + +/// The rebuilt rings wind counter-clockwise, holes clockwise; the other fixes keep the input's +/// winding. +#[test] +fn rebuilt_rings_wind_as_ogc_says() { + for (name, g, _) in rebuilt() { + let (Some(Geometry::Polygon(parts)), _) = one(GeometryType::Polygon, Some(g)) else { panic!("{name}") }; + for rings in &parts { + assert!(signed(&rings[0]) > 0.0, "{name}"); + assert!(rings[1..].iter().all(|h| signed(h) < 0.0), "{name}"); + } + } + let cw = reversed(&poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.)]])); + assert_eq!(one(GeometryType::Polygon, Some(cw)).0, Some(poly(&[&[(0., 10.), (10., 10.), (10., 0.), (0., 0.), (0., 10.)]]))); +} + +/// GeoJSON winds exterior rings counter-clockwise and Shapefiles clockwise: the rebuild gives +/// the same geometry, coordinate for coordinate, whichever way the rings are wound, and when one +/// part or ring is wound against the rest. +#[test] +fn both_windings_give_the_same_result() { + for (name, g, want_area) in rebuilt() { + let (want, code) = one(GeometryType::Polygon, Some(g.clone())); + let want = want.unwrap(); + assert!((area(&want) - want_area).abs() < 1e-9, "{name}: {}", area(&want)); + assert_eq!(one(GeometryType::Polygon, Some(reversed(&g))), (Some(want.clone()), code.clone()), "{name} reversed"); + // Each ring on its own reversed. + let Geometry::Polygon(parts) = &g else { panic!() }; + for (p, rings) in parts.iter().enumerate() { + for r in 0..rings.len() { + let mut mixed = parts.clone(); + mixed[p][r].reverse(); + assert_eq!( + one(GeometryType::Polygon, Some(Geometry::Polygon(mixed))).0.as_ref(), + Some(&want), + "{name}: ring {r} of part {p} reversed" + ); + } + } + } +} + +#[test] +fn several_problems_in_one_feature() { + // An unclosed bowtie with a repeated vertex, a hole outside it, and two overlapping parts. + let g = multi(&[&[&[(0., 0.), (2., 2.), (2., 2.), (2., 0.), (0., 2.)], OUTSIDE], &[SQUARE], &[OVERLAPPING]]); + let (got, repairs) = one(GeometryType::Polygon, Some(g)); + assert_eq!(repairs, "ring_not_closed, repeated_vertex, self_intersection, hole_outside_shell, parts_overlap"); + assert_eq!(area(got.as_ref().unwrap()), 175.0, "the bowtie inside the square, the hole gone"); + // A ring with too few points keeps the parts from being checked against each other; once it + // is dropped, they are, and their overlap is repaired too. + let g = multi(&[&[SQUARE, &[(2., 2.), (3., 3.), (2., 2.)]], &[OVERLAPPING]]); + let (got, repairs) = one(GeometryType::Polygon, Some(g)); + assert_eq!(repairs, "too_few_points, parts_overlap"); + assert_eq!( + got.map(|g| norm(&g)), + Some(norm(&poly(&[&[(0., 0.), (10., 0.), (10., 5.), (15., 5.), (15., 15.), (5., 15.), (5., 10.), (0., 10.)]]))) + ); + // A part crossing itself isn't checked against the other parts or its holes: what the rebuild + // finds is said all the same. + let g = + multi(&[&[SQUARE], &[&[(5., 5.), (15., 15.), (15., 5.), (5., 15.), (5., 5.)], &[(30., 30.), (31., 30.), (31., 31.), (30., 30.)]]]); + let (got, repairs) = one(GeometryType::Polygon, Some(g)); + assert_eq!(repairs, "self_intersection, hole_outside_shell, parts_overlap"); + let got = got.unwrap(); + assert_eq!(area(&got), 137.5); + let Geometry::Polygon(parts) = &got else { panic!() }; + assert_eq!(parts.len(), 2, "the square with the bowtie's left half, and its right half touching it at a corner"); + // Without a hole outside, none is said to be. + let g = multi(&[&[SQUARE], &[&[(5., 5.), (15., 15.), (15., 5.), (5., 15.), (5., 5.)], &[(12., 8.), (13., 8.), (13., 9.), (12., 8.)]]]); + assert_eq!(one(GeometryType::Polygon, Some(g)).1, "self_intersection, parts_overlap"); + // Holes of a part crossing itself that overlap are merged, and said to intersect. + let bowtie: Ring = &[(0., 0.), (20., 20.), (20., 0.), (0., 20.), (0., 0.)]; + let a: Ring = &[(1., 8.), (4., 8.), (4., 12.), (1., 12.), (1., 8.)]; + let b: Ring = &[(2., 9.), (5., 9.), (5., 11.), (2., 11.), (2., 9.)]; + let (got, repairs) = one(GeometryType::Polygon, Some(poly(&[bowtie, a, b]))); + assert_eq!(repairs, "self_intersection, rings_intersect"); + assert_eq!(area(got.as_ref().unwrap()), 200.0 - 14.0, "the holes' union taken out"); + // Holes apart, or only touching at a point, aren't. + let c: Ring = &[(1., 9.), (3., 9.), (3., 11.), (1., 11.), (1., 9.)]; + let d: Ring = &[(4., 9.), (6., 9.), (6., 11.), (4., 11.), (4., 9.)]; + let e: Ring = &[(3., 11.), (4., 11.), (4., 12.), (3., 12.), (3., 11.)]; + assert_eq!(one(GeometryType::Polygon, Some(poly(&[bowtie, c, d]))).1, "self_intersection"); + assert_eq!(one(GeometryType::Polygon, Some(poly(&[bowtie, c, e]))).1, "self_intersection"); + // Nor parts merged that only touch at a point. + let g = multi(&[&[BOWTIE], &[&[(2., 2.), (4., 2.), (4., 4.), (2., 2.)]]]); + assert_eq!(one(GeometryType::Polygon, Some(g)).1, "self_intersection"); +} + +/// Features with only vertex-level problems keep every other coordinate exactly, even those the +/// overlay would round or leave out (a vertex in the middle of an edge). +#[test] +fn vertex_fixes_move_no_coordinate() { + let open: Ring = &[ + (500_000.123, 5_600_000.789), + (500_005.5, 5_600_000.789), + (500_010.321, 5_600_000.789), + (500_010.321, 5_600_010.1), + (500_000.123, 5_600_010.1), + ]; + let mut closed = open.to_vec(); + closed.push(open[0]); + let hole: Ring = + &[(500_002.1, 5_600_002.2), (500_002.1, 5_600_002.2), (500_002.1, 5_600_004.4), (500_004.3, 5_600_004.4), (500_002.1, 5_600_002.2)]; + let (got, repairs) = one(GeometryType::Polygon, Some(poly(&[open, hole]))); + assert_eq!(repairs, "ring_not_closed, repeated_vertex"); + assert_eq!(got, Some(poly(&[&closed, &hole[1..]]))); +} + +/// A feature with nothing wrong is written exactly as stored, with an empty repairs value. +#[test] +fn valid_features_are_copied_exactly() { + let odd: Ring = &[(0.1, 0.2), (5.000_000_000_1, 0.2), (10.3, 0.2), (10.3, 10.7), (0.1, 10.7), (0.1, 0.2)]; + let cases = [ + (GeometryType::Polygon, poly(&[odd])), + (GeometryType::Polygon, reversed(&poly(&[SQUARE, HOLE]))), + ( + GeometryType::Polygon, + multi(&[ + &[SQUARE, HOLE], + &[&[(2.5, 2.5), (3.5, 2.5), (3.5, 3.5), (2.5, 2.5)]], + &[&[(20., 0.), (30., 0.), (30., 10.), (20., 0.)]], + ]), + ), + (GeometryType::Polygon, poly(&[SQUARE, &[(0., 5.), (3., 4.), (3., 6.), (0., 5.)]])), + (GeometryType::Polyline, line(&[BOWTIE, &[(0.1, 0.1), (0.1, 0.1 + 1e-12)]])), + (GeometryType::Point, Geometry::Point(Coord::new(-0.0, 1e-300))), + (GeometryType::Point, Geometry::MultiPoint(pts(&[(1., 2.), (1., 2.)]))), + ]; + for (ty, g) in cases { + let mut l = layer(ty); + add(&mut l, Some(g.clone()), "ok"); + let r = run(&l); + assert_eq!(r.messages, ["Repair Geometry: no repairs needed in 1 feature"], "{g:?}"); + let (got, repairs) = rows(&r).remove(0); + assert_eq!(repairs, ""); + let bits = |g: &Geometry| coords(g).iter().map(|c| (c.x.to_bits(), c.y.to_bits())).collect::>(); + assert_eq!(got.as_ref().map(bits), Some(bits(&g)), "{g:?}"); + assert_eq!(got, Some(g)); + } +} + +#[test] +fn a_mixed_layer_says_what_was_done_with_counts() { + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(poly(&[SQUARE])), "ok"); + add(&mut l, Some(poly(&[BOWTIE])), "bowtie"); + add(&mut l, Some(poly(&[&[(0., 0.), (10., 0.), (10., 10.), (0., 10.)]])), "open"); + add(&mut l, None, "null"); + add(&mut l, Some(poly(&[SQUARE, OUTSIDE])), "hole outside"); + add(&mut l, Some(multi(&[&[SQUARE], &[OVERLAPPING]])), "overlapping"); + add(&mut l, Some(poly(&[SQUARE, ACROSS])), "split"); + add(&mut l, Some(poly(&[SQUARE, &[(2., 2.), (3., 3.), (2., 2.)]])), "degenerate hole"); + add(&mut l, Some(poly(&[&[(0., 0.), (1., 0.), (2., 0.), (0., 0.)]])), "flat"); + add(&mut l, Some(Geometry::Point(Coord::new(f64::NAN, 0.))), "nan"); + let r = run(&l); + let ds = dataset(&r); + valid(ds); + let repairs: Vec<(String, String)> = (0..ds.features.len()).map(|i| (text(ds, i, "name"), text(ds, i, "repairs"))).collect(); + let row = |n: &str, c: &str| (n.to_string(), c.to_string()); + assert_eq!( + repairs, + [ + row("ok", ""), + row("bowtie", "self_intersection"), + row("open", "ring_not_closed"), + row("null", "null_geometry"), + row("hole outside", "hole_outside_shell"), + row("overlapping", "parts_overlap"), + row("split", "disconnected_interior"), + row("degenerate hole", "too_few_points"), + row("flat", "no_area"), + row("nan", "non_finite_coordinate"), + ] + ); + let without: Vec = ds.features.iter().map(|f| f.geometry.is_none()).collect(); + assert_eq!(without, [false, false, false, true, false, false, false, false, true, true]); + assert_eq!( + r.messages, + [ + "Repair Geometry: repaired 6 of 10 features (too_few_points: 1, ring_not_closed: 1, self_intersection: 1, hole_outside_shell: 1, disconnected_interior: 1, parts_overlap: 1)", + "Repair Geometry: 3 features couldn't be repaired and are written without geometry (null_geometry: 1, non_finite_coordinate: 1, no_area: 1)", + "Repair Geometry: 1 ring or part with too few distinct points or no area was dropped from 1 feature", + "Repair Geometry: holes not inside their exterior ring were taken out of 1 feature; what of a hole is inside its exterior ring stays a hole", + "Repair Geometry: the parts that overlapped or touched along a line were merged in 1 feature", + "Repair Geometry: 1 feature whose interior was cut into pieces by rings touching at points is written with each piece as a part of its own", + // The hole outside makes the largest extent, 21 across: a grid step of 2^-26, five of them. + "Repair Geometry: 4 features were rebuilt by geo's boolean overlay, which rounds coordinates to a fine grid: they may have moved by up to 7.5e-8 degrees of longitude or latitude, and vertices in the middle of straight edges were left out", + ] + ); +} + +#[test] +fn plural_counts_of_pieces_and_holes() { + let mut l = layer(GeometryType::Polygon); + add( + &mut l, + Some(multi(&[&[SQUARE, OUTSIDE, &[(30., 30.), (31., 30.), (31., 31.), (30., 30.)]], &[&[(40., 0.), (41., 0.), (40., 0.)]]])), + "a", + ); + add(&mut l, Some(Geometry::Line(vec![pts(&[(0., 0.), (1., 1.)]), pts(&[(2., 2.)]), vec![]])), "b"); + add( + &mut l, + Some(multi(&[ + &[SQUARE], + &[&[(10., 0.), (20., 0.), (20., 10.), (10., 10.), (10., 0.)]], + &[&[(20., 0.), (30., 0.), (30., 10.), (20., 10.), (20., 0.)]], + ])), + "c", + ); + let r = run(&l); + valid(dataset(&r)); + assert_eq!(text(dataset(&r), 0, "repairs"), "too_few_points, hole_outside_shell"); + assert_eq!(area(dataset(&r).features[2].geometry.as_ref().unwrap()), 300.0); + assert_eq!( + r.messages[1..4], + [ + "Repair Geometry: 3 rings or parts with too few distinct points or no area were dropped from 2 features", + "Repair Geometry: holes not inside their exterior ring were taken out of 1 feature; what of a hole is inside its exterior ring stays a hole", + "Repair Geometry: the parts that overlapped or touched along a line were merged in 1 feature", + ] + ); +} + +/// A closed ring around a rectangle `w` by `h` with its lower left corner at `(x, y)`. +fn rect(x: f64, y: f64, w: f64, h: f64) -> Vec<(f64, f64)> { + vec![(x, y), (x + w, y), (x + w, y + h), (x, y + h), (x, y)] +} +/// A closed ring around a strip `len` long and `w` wide at `deg` degrees from the x axis, with a +/// corner at `(x, y)`. +fn strip(x: f64, y: f64, len: f64, deg: f64, w: f64) -> Vec<(f64, f64)> { + let (s, c) = deg.to_radians().sin_cos(); + vec![(x, y), (x + len * c, y + len * s), (x + len * c - w * s, y + len * s + w * c), (x - w * s, y + w * c), (x, y)] +} +/// A strip at 45°, 5 across and up from a corner placed 0.45 of a step of the grid of the parts' +/// union (2^-20, as below) right of (1000, 1000), and `w` steps wide: one step wide, the overlay +/// rounds all its points onto one diagonal line of its grid. +fn diagonal(w: f64) -> Vec<(f64, f64)> { + let step = (-20f64).exp2(); + let (x, y, a) = (1000. + 0.45 * step, 1000., w * step / 2f64.sqrt()); + vec![(x, y), (x + 5., y + 5.), (x + 5. - a, y + 5. + a), (x - a, y + a), (x, y)] +} +/// A bowtie 20 across, whose left lobe holds the holes at x 1 to 4 and whose right lobe holds +/// those at x 18. +const BIG_BOWTIE: Ring = &[(0., 0.), (20., 20.), (20., 0.), (0., 20.), (0., 0.)]; + +/// A part far from the others, or a hole far from the other holes, narrower than a step of the +/// overlay's grid would be rounded to no area by the union or the difference, which would also +/// take its vanishing for a merge: it is dropped first, counted, and said with no_area. +#[test] +fn a_part_or_hole_narrower_than_the_overlays_grid_is_dropped_and_said() { + // Parts 1000 apart: half the extent is about 500, so the union's grid step is 2^-20; the far + // part is three quarters of a step across. + let tiny = rect(1000., 1000., 0.75 * (-20f64).exp2(), 0.75 * (-20f64).exp2()); + let overlapping = multi(&[&[SQUARE], &[OVERLAPPING], &[&tiny]]); + // Parts apart, rebuilt for the bowtie: without the narrow part dropped first, its vanishing + // would have been taken for parts that overlap. + let apart = multi(&[&[BOWTIE], &[&rect(5., 5., 2., 2.)], &[&tiny]]); + // A hole three quarters of a step of the difference's grid across (2^-26: the bowtie is 20 + // across), far from the other hole: their union would have taken its vanishing for holes + // that overlap. + let tiny_hole = rect(18., 10., 0.75 * (-26f64).exp2(), 0.75 * (-26f64).exp2()); + let holes = poly(&[BIG_BOWTIE, &rect(1., 8., 3., 4.), &tiny_hole]); + + let (got, repairs) = one(GeometryType::Polygon, Some(overlapping.clone())); + assert_eq!(repairs, "parts_overlap, no_area"); + let got = got.unwrap(); + let Geometry::Polygon(parts) = &got else { panic!() }; + assert_eq!(parts.len(), 1, "the squares merged, the narrow part dropped"); + assert_eq!(area(&got), 175.0); + + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(overlapping), "overlapping"); + add(&mut l, Some(apart), "apart"); + add(&mut l, Some(holes), "holes"); + let r = run(&l); + let ds = dataset(&r); + valid(ds); + let repairs: Vec = (0..3).map(|i| text(ds, i, "repairs")).collect(); + assert_eq!(repairs, ["parts_overlap, no_area", "self_intersection, no_area", "self_intersection, no_area"]); + let Some(Geometry::Polygon(parts)) = &ds.features[1].geometry else { panic!() }; + assert_eq!(parts.len(), 3, "the bowtie's two lobes and the square"); + assert_eq!(area(ds.features[2].geometry.as_ref().unwrap()), 200.0 - 12.0); + assert_eq!( + r.messages[..3], + [ + "Repair Geometry: repaired 3 of 3 features (self_intersection: 2, parts_overlap: 1, no_area: 3)", + "Repair Geometry: 3 holes or parts narrower than a step of the overlay's grid, which would leave them no area, were dropped from 3 features, with no_area in repairs", + "Repair Geometry: the parts that overlapped or touched along a line were merged in 1 feature", + ] + ); + let said = "Repair Geometry: 1 hole or part narrower than a step of the overlay's grid, which would leave it no area, was dropped from 1 feature, with no_area in repairs"; + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(multi(&[&[SQUARE], &[OVERLAPPING], &[&tiny]])), "one"); + assert_eq!(run(&l).messages[1], said); + + // Long slivers at an angle, many steps across and up, but three quarters of a step wide; and a + // strip at 45° a step wide, which rounding can put on one line, as it moves a point by up to + // half a step on each axis: across the diagonal, by up to √2 / 2 steps. + let step = (-20f64).exp2(); + let slivers = [strip(1000., 1000., 10., 45., 0.75 * step), strip(1000., 1000., 10., 30., 0.75 * step), diagonal(1.0)]; + for sliver in &slivers { + for (g, want, parts_out) in [ + (multi(&[&[SQUARE], &[OVERLAPPING], &[sliver]]), "parts_overlap, no_area", 1), + (multi(&[&[BOWTIE], &[&rect(5., 5., 2., 2.)], &[sliver]]), "self_intersection, no_area", 3), + ] { + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(g.clone()), "x"); + let r = run(&l); + let ds = dataset(&r); + valid(ds); + assert_eq!(text(ds, 0, "repairs"), want, "{sliver:?}"); + let Some(Geometry::Polygon(parts)) = &ds.features[0].geometry else { panic!() }; + assert_eq!(parts.len(), parts_out, "{sliver:?}"); + assert!(r.messages.iter().any(|m| m == said), "{:?}", r.messages); + } + } + // A hole at 30°, a unit long and three quarters of a step of the difference's grid wide. + let hole = strip(17., 12., 1., 30., 0.75 * (-26f64).exp2()); + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(poly(&[BIG_BOWTIE, &rect(1., 8., 3., 4.), &hole])), "x"); + let r = run(&l); + let ds = dataset(&r); + valid(ds); + assert_eq!(text(ds, 0, "repairs"), "self_intersection, no_area"); + let Some(Geometry::Polygon(parts)) = &ds.features[0].geometry else { panic!() }; + assert_eq!(parts.iter().map(Vec::len).sum::(), 3, "the lobes and the hole left of them"); + assert_eq!(r.messages[1], said); +} + +/// The same features without the narrow part or hole, or with one exactly a grid step wide, or at +/// an angle as wide as rounding can move a point across it (√2 steps at 45°, cos 30° + sin 30° at +/// 30°) or a little more, which the overlay keeps: nothing is dropped, and nothing said of it. +#[test] +fn parts_and_holes_a_grid_step_wide_are_kept_unsaid() { + let step = (-20f64).exp2(); + let sliver = rect(1000., 1000., step, 10.); + let hole_sliver = rect(18., 9., (-26f64).exp2(), 2.); + let (at_45, at_30) = (1.45 * step, 1.4 * step); + let hole_at_45 = strip(17., 12., 2., 45., 1.45 * (-26f64).exp2()); + // Rounded, a side of a strip at an angle moves by up to √2 / 2 steps across it, so its area + // changes by up to √2 steps times its length (10 at most) on the union's grid, and its ends + // move too: twice that is allowed. + let (exact, drift) = (1e-7, 2.0 * 10.0 * std::f64::consts::SQRT_2 * step); + let squares = |far: &[(f64, f64)]| multi(&[&[SQUARE], &[OVERLAPPING], &[far]]); + let apart = |far: &[(f64, f64)]| multi(&[&[BOWTIE], &[&rect(5., 5., 2., 2.)], &[far]]); + let holes = |far: &[(f64, f64)]| poly(&[BIG_BOWTIE, &rect(1., 8., 3., 4.), far]); + for (g, want, parts_out, rings_out, want_area, tol) in [ + (multi(&[&[SQUARE], &[OVERLAPPING]]), "parts_overlap", 1, 1, 175.0, exact), + (squares(&sliver), "parts_overlap", 2, 2, 175.0 + 10.0 * step, exact), + (squares(&diagonal(1.45)), "parts_overlap", 2, 2, 175.0 + 50f64.sqrt() * at_45, drift), + (squares(&strip(1000., 1000., 10., 45., at_45)), "parts_overlap", 2, 2, 175.0 + 10.0 * at_45, drift), + (squares(&strip(1000., 1000., 10., 30., at_30)), "parts_overlap", 2, 2, 175.0 + 10.0 * at_30, drift), + (multi(&[&[BOWTIE], &[&rect(5., 5., 2., 2.)]]), "self_intersection", 3, 3, 6.0, exact), + (apart(&strip(1000., 1000., 10., 30., at_30)), "self_intersection", 4, 4, 6.0 + 10.0 * at_30, drift), + (poly(&[BIG_BOWTIE, &rect(1., 8., 3., 4.)]), "self_intersection", 2, 3, 188.0, exact), + (holes(&hole_sliver), "self_intersection", 2, 4, 188.0 - 2.0 * (-26f64).exp2(), exact), + (holes(&hole_at_45), "self_intersection", 2, 4, 188.0 - 2.0 * 1.45 * (-26f64).exp2(), drift), + ] { + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(g.clone()), "x"); + let r = run(&l); + let ds = dataset(&r); + valid(ds); + assert_eq!(text(ds, 0, "repairs"), want, "{g:?}"); + let got = ds.features[0].geometry.as_ref().unwrap(); + let Geometry::Polygon(parts) = got else { panic!() }; + assert_eq!(parts.len(), parts_out, "{g:?}"); + assert_eq!(parts.iter().map(Vec::len).sum::(), rings_out, "{g:?}"); + assert!((area(got) - want_area).abs() < tol, "{g:?}: area {}", area(got)); + assert!(!r.messages.iter().any(|m| m.contains("no_area") || m.contains("narrower")), "{:?}", r.messages); + } +} + +/// A feature too busy to check, or whose overlay would be past the work bound, is written as it +/// was, with too_complex, never dropped. +#[test] +fn a_feature_too_complex_to_repair_is_written_unchanged() { + // A zigzag ring whose 12,000 segments all overlap each other's x and y ranges: too busy to check. + let n = 12_000; + let mut zigzag: Vec = (0..n).map(|i| Coord::new(f64::from(i % 2), f64::from(i) * 1e-3 + f64::from(i % 2) * 100.0)).collect(); + zigzag.push(Coord::new(-1., 50.)); + zigzag.push(Coord::new(0., 0.)); + // A star of 2001 points, each joined to the one 1000 on: about 2 million crossings, which the + // checks find within the bound, but which the overlay would turn into 4 million vertices. + let (k, m) = (2001u32, 1000u32); + let star: Vec = (0..=k) + .map(|i| { + let a = std::f64::consts::TAU * f64::from((i * m) % k) / f64::from(k); + Coord::new(a.cos(), a.sin()) + }) + .collect(); + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(Geometry::Polygon(vec![vec![zigzag]])), "busy"); + add(&mut l, Some(Geometry::Polygon(vec![vec![star]])), "star"); + add(&mut l, Some(poly(&[BOWTIE])), "bowtie"); + let started = std::time::Instant::now(); + let r = run(&l); + assert!(started.elapsed().as_secs() < 60, "bounded"); + let ds = dataset(&r); + for i in 0..2 { + assert_eq!(text(ds, i, "repairs"), "too_complex"); + assert_eq!(ds.features[i].geometry, l.ds.features[i].geometry, "written as stored"); + } + assert_eq!(text(ds, 2, "repairs"), "self_intersection"); + assert_eq!( + r.messages, + [ + "Repair Geometry: repaired 1 of 3 features (self_intersection: 1)", + "Repair Geometry: 2 features were too complex to repair within the work limit and are written unchanged, with too_complex in repairs", + "Repair Geometry: 1 feature was rebuilt by geo's boolean overlay, which rounds coordinates to a fine grid: they may have moved by up to 9.4e-9 degrees of longitude or latitude, and vertices in the middle of straight edges were left out", + ] + ); +} + +#[test] +fn selection_is_honoured_once_per_feature_in_table_order() { + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(poly(&[BOWTIE])), "unselected bowtie"); + add(&mut l, Some(poly(&[SQUARE])), "selected square"); + add(&mut l, Some(poly(&[SQUARE, OUTSIDE])), "selected hole outside"); + add(&mut l, None, "selected null"); + add(&mut l, Some(poly(&[BOWTIE])), "another unselected bowtie"); + l.selected = vec![3, 1, 2, 3, usize::MAX]; + let r = run(&l); + let ds = dataset(&r); + let names: Vec = (0..ds.features.len()).map(|i| text(ds, i, "name")).collect(); + assert_eq!(names, ["selected square", "selected hole outside", "selected null"]); + assert_eq!( + r.messages[..2], + [ + "Repair Geometry: repaired 1 of 3 features (hole_outside_shell: 1)", + "Repair Geometry: 1 feature couldn't be repaired and is written without geometry (null_geometry: 1)" + ] + ); +} + +#[test] +fn attributes_crs_ids_and_names_are_kept() { + let mut l = Layer { + ds: Dataset::new( + "Parcels", + GeometryType::Polygon, + vec![Field::new("name", FieldType::Text), Field::new("area", FieldType::Double), Field::new("n", FieldType::Integer)], + ), + selected: vec![], + }; + l.ds.crs = Some(rustgis_data::raster::Crs::Epsg(2180)); + l.ds.crs_incomplete = true; + l.ds.zm_dropped = true; + l.ds.features.push(Feature { + id: 41, + geometry: Some(poly(&[SQUARE])), + attrs: vec![Value::Text("ok".into()), Value::Float(1.5), Value::Int(1)], + }); + // A short row: its missing value is null. + l.ds.features.push(Feature { id: 42, geometry: Some(poly(&[BOWTIE])), attrs: vec![Value::Text("bad".into()), Value::Float(2.5)] }); + // A long row: its extra value is left out, and said so. + l.ds.features.push(Feature { + id: 7, + geometry: None, + attrs: vec![Value::Text("none".into()), Value::Null, Value::Int(3), Value::Int(9)], + }); + let named = |output: Option<&str>| { + let mut p = json!({"input": "input"}); + if let Some(o) = output { + p["output"] = json!(o); + } + rustgis_analysis::run("repairGeometry", &l, &p).unwrap() + }; + let r = named(None); + let ds = dataset(&r); + assert_eq!(ds.name, "Parcels_repaired"); + assert_eq!(ds.geometry_type, GeometryType::Polygon); + assert_eq!(ds.crs, l.ds.crs); + assert!(ds.crs_incomplete && ds.zm_dropped); + assert_eq!(ds.fields.len(), 4); + assert_eq!(ds.fields[..3], l.ds.fields[..]); + assert_eq!((ds.fields[3].name.as_str(), ds.fields[3].ty), ("repairs", FieldType::Text)); + let ids: Vec = ds.features.iter().map(|f| f.id).collect(); + assert_eq!(ids, [41, 42, 7], "the input's ids lead back to its features"); + assert_eq!(ds.features[0].attrs, [Value::Text("ok".into()), Value::Float(1.5), Value::Int(1), Value::Text(String::new())]); + assert_eq!(ds.features[1].attrs, [Value::Text("bad".into()), Value::Float(2.5), Value::Null, Value::Text("self_intersection".into())]); + assert_eq!(ds.features[2].attrs, [Value::Text("none".into()), Value::Null, Value::Int(3), Value::Text("null_geometry".into())]); + assert_eq!( + r.messages.last().map(String::as_str), + Some("Repair Geometry: 1 feature had more attribute values than the layer has fields; the extra values aren't in the output") + ); + assert_eq!(dataset(&named(Some(" Fixed parcels "))).name, "Fixed parcels"); +} + +#[test] +fn an_existing_repairs_field_gets_another_name() { + let mut l = Layer { + ds: Dataset::new( + "input", + GeometryType::Polygon, + vec![Field::new("Repairs", FieldType::Integer), Field::new("repairs_2", FieldType::Text)], + ), + selected: vec![], + }; + l.ds.push_next(Some(poly(&[BOWTIE])), vec![Value::Int(7), Value::Text("mine".into())]); + let r = run(&l); + let ds = dataset(&r); + assert_eq!(ds.fields.iter().map(|f| f.name.as_str()).collect::>(), ["Repairs", "repairs_2", "repairs_3"]); + assert_eq!(ds.features[0].attrs[..2], l.ds.features[0].attrs[..]); + assert_eq!(text(ds, 0, "repairs_3"), "self_intersection"); + assert_eq!(r.messages[0], "Repair Geometry: repaired 1 of 1 feature (self_intersection: 1)"); + assert_eq!( + r.messages.last().map(String::as_str), + Some("Repair Geometry: 'input' already has a field named repairs, so the output's repairs field is repairs_3") + ); +} + +#[test] +fn empty_layers_tables_and_unknown_layers() { + let r = run(&layer(GeometryType::Polygon)); + assert_eq!(r.messages, ["Repair Geometry: no features to repair"]); + assert!(dataset(&r).features.is_empty()); + let table = Layer { ds: Dataset::new("input", GeometryType::None, vec![]), selected: vec![] }; + let e = rustgis_analysis::run("repairGeometry", &table, &json!({"input": "input"})).unwrap_err(); + assert_eq!(e, "Repair Geometry needs a layer with geometry: 'input' is a table"); + assert_eq!(rustgis_analysis::run("repairGeometry", &table, &json!({"input": "nope"})).unwrap_err(), "no layer named 'nope'"); +} + +/// The distance from `c` to the segment `a`–`b`. +fn to_segment(c: Coord, a: Coord, b: Coord) -> f64 { + let (dx, dy) = (b.x - a.x, b.y - a.y); + let len = dx * dx + dy * dy; + let t = if len == 0.0 { 0.0 } else { (((c.x - a.x) * dx + (c.y - a.y) * dy) / len).clamp(0.0, 1.0) }; + (c.x - (a.x + t * dx)).hypot(c.y - (a.y + t * dy)) +} +/// How far the farthest vertex of `out` is from the rings of `input`. +fn farthest(out: &Geometry, input: &Geometry) -> f64 { + let Geometry::Polygon(parts) = input else { panic!() }; + let segs: Vec<(Coord, Coord)> = + parts.iter().flatten().flat_map(|r| r.windows(2).map(|w| (w[0], w[1])).chain(std::iter::once((r[r.len() - 1], r[0])))).collect(); + coords(out).into_iter().map(|c| segs.iter().map(|&(a, b)| to_segment(c, a, b)).fold(f64::INFINITY, f64::min)).fold(0.0, f64::max) +} + +/// The overlay rounds coordinates to a grid of 2^29 steps across half the feature's extent: what +/// it writes stays within a step of the input's rings, well inside the bound the notice gives. +#[test] +fn the_overlay_moves_coordinates_by_less_than_a_grid_step() { + let shift = |g: &Geometry, dx: f64, dy: f64, k: f64| { + let Geometry::Polygon(parts) = g else { panic!() }; + Geometry::Polygon( + parts + .iter() + .map(|rings| rings.iter().map(|r| r.iter().map(|c| Coord::new(dx + c.x * k, dy + c.y * k)).collect()).collect()) + .collect(), + ) + }; + let mut worst_steps = 0f64; + for (dx, dy, k) in + [(500_000.123_456, 5_600_000.987_654, 1000.37), (-179.123_456_789, -89.987_654_321, 0.017_3), (0.1, 0.7, 1.000_000_3)] + { + for (name, g, want_area) in rebuilt() { + let g = shift(&g, dx, dy, k); + let mut l = layer(GeometryType::Polygon); + add(&mut l, Some(g.clone()), name); + let r = run(&l); + valid(dataset(&r)); + let out = dataset(&r).features[0].geometry.clone().unwrap(); + let Geometry::Polygon(parts) = &g else { panic!() }; + let (mut lo, mut hi) = ((f64::INFINITY, f64::INFINITY), (f64::NEG_INFINITY, f64::NEG_INFINITY)); + for c in parts.iter().flatten().flatten() { + (lo, hi) = ((lo.0.min(c.x), lo.1.min(c.y)), (hi.0.max(c.x), hi.1.max(c.y))); + } + let step = (((hi.0 - lo.0).max(hi.1 - lo.1) / 2.0).log2().round() - 29.0).exp2(); + let moved = farthest(&out, &g); + worst_steps = worst_steps.max(moved / step); + assert!(moved <= step, "{name} at {dx}: moved {moved}, a step is {step}"); + let bound: f64 = r.messages.last().unwrap().split("up to ").nth(1).unwrap().split(' ').next().unwrap().parse().unwrap(); + assert!(moved <= bound && bound >= 5.0 * step, "{name}: {moved} past {bound}"); + let a = area(&out); + assert!((a - want_area * k * k).abs() <= 1e-6 * want_area * k * k, "{name}: area {a}"); + } + } + println!("the farthest a coordinate moved: {worst_steps:.3} grid steps"); +} + +/// GeoJSON keeps rings as written, wound either way, and the repairs see what the file holds. +#[test] +fn geojson_read_by_the_real_reader() { + let geojson = r#"{"type":"FeatureCollection","features":[ + {"type":"Feature","properties":{"name":"bowtie"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[2,2],[2,0],[0,2],[0,0]]]}}, + {"type":"Feature","properties":{"name":"open"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[4,0],[4,4],[0,4]]]}}, + {"type":"Feature","properties":{"name":"hole outside"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[4,0],[4,4],[0,4],[0,0]],[[10,10],[11,10],[11,11],[10,10]]]}}, + {"type":"Feature","properties":{"name":"empty"},"geometry":{"type":"Polygon","coordinates":[]}}, + {"type":"Feature","properties":{"name":"split"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[10,0],[10,10],[0,10],[0,0]],[[0,0],[6,4],[10,10],[4,6],[0,0]]]}}, + {"type":"Feature","properties":{"name":"overlapping, wound clockwise"},"geometry":{"type":"MultiPolygon","coordinates":[[[[0,0],[0,10],[10,10],[10,0],[0,0]]],[[[5,5],[15,5],[15,15],[5,15],[5,5]]]]}}, + {"type":"Feature","properties":{"name":"null"},"geometry":null}, + {"type":"Feature","properties":{"name":"fine"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[4,0],[4,4],[0,4],[0,0]]]}} + ]}"#; + let (ds, report) = rustgis_data::io::read_geojson(geojson, "input").unwrap(); + assert!(report.messages.is_empty() && report.skipped == 0); + let l = Layer { ds, selected: vec![] }; + let r = run(&l); + let out = dataset(&r); + valid(out); + let got: Vec<(String, String, Option)> = (0..out.features.len()) + .map(|i| (text(out, i, "name"), text(out, i, "repairs"), out.features[i].geometry.as_ref().map(area))) + .collect(); + let row = |n: &str, c: &str, a: Option| (n.to_string(), c.to_string(), a); + assert_eq!( + got, + [ + row("bowtie", "self_intersection", Some(2.0)), + row("open", "ring_not_closed", Some(16.0)), + row("hole outside", "hole_outside_shell", Some(16.0)), + row("empty", "null_geometry", None), + row("split", "disconnected_interior", Some(80.0)), + row("overlapping, wound clockwise", "parts_overlap", Some(175.0)), + row("null", "null_geometry", None), + row("fine", "", Some(16.0)), + ] + ); + let Some(Geometry::Polygon(split)) = &out.features[4].geometry else { panic!() }; + assert_eq!(split.len(), 2, "the split interior as two parts"); + assert_eq!(out.features[7].geometry, l.ds.features[7].geometry); +} + +/// The Shapefile writer winds exterior rings clockwise, against GeoJSON: read back, the rebuilt +/// features come out the same, coordinate for coordinate. +#[test] +fn a_shapefile_round_trip_gives_the_same_repairs() { + let mut l = layer(GeometryType::Polygon); + for (name, g, _) in rebuilt() { + add(&mut l, Some(g), name); + } + let direct = run(&l); + let (parts, left_out) = rustgis_data::io::shapefile_parts(&l.ds).unwrap(); + assert_eq!(left_out, 0); + let dir = std::env::temp_dir().join(format!("rustgis-repair-geometry-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for (ext, bytes) in &parts { + std::fs::write(dir.join(format!("input.{ext}")), bytes).unwrap(); + } + let read = rustgis_data::io::read_shapefile_in(&dir.join("input.shp"), "input", None); + let _ = std::fs::remove_dir_all(&dir); + let (ds, _) = read.unwrap(); + let shp = Layer { ds, selected: vec![] }; + let via_shapefile = run(&shp); + let a = dataset(&direct); + let b = dataset(&via_shapefile); + assert_eq!(a.features.len(), b.features.len()); + for i in 0..a.features.len() { + let name = text(a, i, "name"); + // The Shapefile reader assigns holes to the shell holding them: what it keeps of a hole + // outside its shell may change what the checks find, never what the rebuild makes. + assert_eq!(b.features[i].geometry.as_ref().map(norm), a.features[i].geometry.as_ref().map(norm), "{name}"); + assert_eq!(b.features[i].geometry, a.features[i].geometry, "{name}"); + } +} diff --git a/crates/assistant/skills/geoprocessing/SKILL.md b/crates/assistant/skills/geoprocessing/SKILL.md index d04e048012..f9d1987b6a 100644 --- a/crates/assistant/skills/geoprocessing/SKILL.md +++ b/crates/assistant/skills/geoprocessing/SKILL.md @@ -26,7 +26,8 @@ Based on the RustGIS user guide, Analysis › Geoprocessing tools (`input`, `x`, `y`, `crs`), `calculateGeometry`; `polygonToLine`, `splitLineAtVertices`, `featureEnvelopeToPolygon`, `unsplitLine` (`input`, optional `output`; see the user guide for envelope axes and endpoint-joining rules); `checkGeometry` (`input`: a layer of the - features with invalid geometry, saying what is wrong with each) + features with invalid geometry, saying what is wrong with each); `repairGeometry` (`input`: + every feature with its geometry made valid, and a field saying what was repaired) - selection: `selectByLocation` (`input`, `select`, `distance`, `invert`), `selectByAttributes` (`input`, `where`) - statistics: `countPoints` (Summarize Within: `polygons`, `points`, `statistics`: summary diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index a541b1b26e..e9e89284de 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -152,6 +152,7 @@ Use these ids in `gp.run`'s `tool` parameter (`gp.tools` lists the available too | Feature Envelope To Polygon | `featureEnvelopeToPolygon` | Data Management › Features | | Unsplit Line | `unsplitLine` | Data Management › Features | | Check Geometry | `checkGeometry` | Data Management › Features | +| Repair Geometry | `repairGeometry` | Data Management › Features | | XY Table To Point | `xyTableToPoint` | Data Management › Features | | Calculate Geometry Attributes | `calculateGeometry` | Data Management › Features | | Simplify | `simplify` | Cartography › Generalization | @@ -888,6 +889,83 @@ of their rings or parts already have problems of their own. Rings and parts that gp.run={"tool":"checkGeometry","params":{"input":"Parcels"}} ``` +### Repairing geometry + +**Repair Geometry** (`repairGeometry`) writes every feature to a new layer, `_repaired` +unless you name it, with its geometry made valid by the rules Check Geometry checks. Only the +selected features are processed when some are selected. Each output row keeps its feature id, +fields, attributes and CRS, and adds a text field `repairs`: the codes of what was repaired, +separated by `, ` and in the order of the table above, or empty for a feature that needed +nothing. Features with nothing wrong are copied exactly as stored. If the input already has a +`repairs` field, the new one gets the next free name (`repairs_2`) and the tool says so. + +How each problem is repaired: + +- `ring_not_closed`: the ring is closed with its first point. `repeated_vertex`: the repeats are + dropped. `too_few_points`: the ring or line part is dropped; an exterior ring takes its part's + holes with it, and a polygon part without rings is dropped too. These fixes move no coordinate: + a feature with only these problems keeps every other vertex exactly. +- `self_intersection`, `hole_outside_shell`, `rings_intersect`, `disconnected_interior` and + `parts_overlap`: the polygon is rebuilt from its area, as the "structure" method of make-valid + tools does. Each ring stands for the area it winds around, so a bowtie keeps both of its + triangles and the direction rings are wound in doesn't matter: GeoJSON and Shapefiles, which + wind them in opposite directions, give the same result. Each part is its exterior ring's area + less the union of its holes' areas, and the feature is the union of its parts. So overlapping + parts merge into one, overlapping holes merge, a hole outside its exterior ring disappears (what + of a hole lies inside it stays a hole), and an interior cut into pieces by rings touching at + points becomes one part per piece. A stretch of ring that runs back over itself, with no area, + is left out. Rebuilt exterior rings wind counter-clockwise, holes clockwise. +- Lines may cross themselves, so only their repeated vertices and their parts with fewer than 2 + distinct points are repaired. Points have nothing to repair but non-finite coordinates. + +When a ring of a part crosses itself, Check Geometry doesn't check that part against its other +rings or the other parts; Repair Geometry checks the feature again after the fixes that move no +coordinate, and adds `hole_outside_shell`, `rings_intersect` and `parts_overlap` when the rebuild +finds a hole outside, merges holes or merges parts. + +The rebuild uses the [geo](https://github.com/georust/geo) library's boolean overlay, which works on +an integer grid: half the larger side of the feature's extent spans about 229 steps. +Coordinates are stored in degrees of longitude and latitude, so a step is about +0.000 000 000 12° (0.013 mm) for a feature 10 km across, and about 0.000 000 24° (2.6 cm at the +equator) for one spanning the world. Rebuilt coordinates may move by up to five steps from the +input's rings; in the tests they moved at most two thirds of a step. The overlay also leaves out +vertices in the middle of straight edges. The tool says how many features were rebuilt and the +farthest, in degrees, their coordinates may have moved. Every rebuilt geometry is checked again and +written only when it is valid. + +A hole or part narrower than one step of that grid in any direction can be rounded onto a single +line of it and lose its area, depending on where it lies between the grid's lines. Rounding moves a +point by up to half a step on each axis, so across a diagonal a piece needs to be wider, up to √2 +steps at 45°. The grid follows the extent of all the rings of a part, or of all the parts of a +feature, so this happens to a tiny or thin hole or part far from the others, such as a long sliver +at an angle. So before the overlay takes out a part's holes or merges the parts, a hole or part +that narrow in any direction is dropped: `no_area` is added after the other codes in `repairs`, and +the tool says how many holes and parts were dropped this way, from how many features. + +A feature that can't be repaired stays in the output without geometry, with its attributes, and +`repairs` says why. One the tool couldn't repair is written unchanged, never dropped: + +| Code | Written | Why | +| --- | --- | --- | +| `null_geometry` | without geometry | No geometry, or one without coordinates | +| `non_finite_coordinate` | without geometry | A coordinate is not a number or infinite | +| `too_few_points` | without geometry | Every part has too few distinct points | +| `no_area` | without geometry | Nothing with an area is left once repaired: a ring whose points all lie on one line, or holes covering their whole part | +| `too_complex` | unchanged | Checking or rebuilding it needs more work than the tool spends on one feature (for the rebuild, about 2 million vertices, counting those it adds where rings cross), or there wasn't enough memory for it | +| `repair_failed` | unchanged | The geo library failed on it: its relate operation while checking, the overlay, coordinates too large or too small for the overlay's grid, or a rebuilt geometry that isn't valid | + +The messages say how many features were repaired, with each code; how many were written without +geometry and why; how many were left unchanged for each reason; how many rings or parts were +dropped, and how many holes or parts narrower than the overlay's grid; from how many features holes +were taken out; in how many parts were merged or pieces written as parts of their own; and how far +rebuilt coordinates may have moved. On the desktop and the command line a failure of the overlay is +caught and the feature written unchanged; in the browser it can't be caught, so the rebuild runs only on finite coordinates the grid can hold, with +its work bounded and its memory asked of the browser first, as in Clip and Dissolve. + +```text +gp.run={"tool":"repairGeometry","params":{"input":"Parcels"}} +``` + ## GeoLibre tools The desktop app and the command line can also run the