From 04812873609ab4e44a4e3a23b184f89e6de977a6 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:52:35 -0600 Subject: [PATCH 01/15] fix(analysis): Minimum Bounding Geometry counts what it hulls; Kernel Density counts the multipoint members it leaves out Minimum Bounding Geometry, all features together, set its count field to every feature in the layer, even with a selection or without geometry; it's now the features hulled. Its notice of hulls with no area no longer counts a hull of points in one place (a single point), which has none as a matter of course. Kernel Density left out a multipoint member missing a coordinate (or with an infinite one), in a feature with others, silently. It now says how many in a notice of its own, and in the error when nothing is left to place. A multipoint wholly past what the system can hold counts its points as beyond, as a point does, and no longer as skipped too. Tests: count with and without a selection, a feature without geometry not counted; single points give no notice; members left out are counted, clean multipoints say nothing, and the error names both. --- crates/analysis/src/tools.rs | 110 ++++++++++++++++++++++++++++------- docs/user-guide/analysis.md | 3 +- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index aa708ccc6e..c4cd8c218f 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2184,8 +2184,10 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { let features: Vec<(&Feature, &Geometry)> = rows(inp, p, "input", ds).into_iter().filter_map(|r| ds.features.get(r)).filter_map(|f| Some((f, f.geometry.as_ref()?))).collect(); let mut pts: Vec> = Vec::new(); - // Hulls of at least one point that came out with no area. + // Hulls of points in more than one place that came out with no area: one point's hull has none + // as a matter of course, so it isn't counted. let mut flat = 0usize; + let spread = |these: &[geo::Point]| these.first().is_some_and(|first| these.iter().any(|q| q != first)); if each { // Each feature's hull made on several cores for many (#1252), placed in their order. crate::control::begin(features.len()); @@ -2195,32 +2197,37 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { || (), |_, _, (_, g)| { let these = points(g); - let had = !these.is_empty(); + let spread = spread(&these); let hull = geo::MultiPoint::new(these).convex_hull(); - Ok::<_, String>((had && geo::Area::unsigned_area(&hull) == 0.0, hull)) + Ok::<_, String>((spread && geo::Area::unsigned_area(&hull) == 0.0, hull)) }, )?; for ((f, _), (is_flat, hull)) in features.iter().zip(hulls) { flat += usize::from(is_flat); out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), f.attrs.clone()); } - } else { + } + // The features hulled together: those picked (the selection, or all) with a point to give. + let mut hulled = 0usize; + if !each { for (_, g) in &features { - pts.extend(points(g)); + let these = points(g); + hulled += usize::from(!these.is_empty()); + pts.extend(these); } } if !each { - let n = ds.features.len() as i64; - let had = !pts.is_empty(); + let n = i64::try_from(hulled).unwrap_or(i64::MAX); + let spread = spread(&pts); let hull = geo::MultiPoint::new(pts).convex_hull(); - flat += usize::from(had && geo::Area::unsigned_area(&hull) == 0.0); + flat += usize::from(spread && geo::Area::unsigned_area(&hull) == 0.0); out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(n)]); } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; if flat > 0 { r.messages.push(format!( - "Minimum Bounding Geometry: {flat} of {n} hulls have no area: their points lie on one straight line on the map (a single point, or points on one meridian or parallel; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" + "Minimum Bounding Geometry: {flat} of {n} hulls have no area: their points lie on one straight line on the map (points on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" )); } Ok(r) @@ -3581,6 +3588,8 @@ fn kernel_density_tool(inp: &dyn Inputs, p: &Json) -> Result let mut pts: Vec<(Coord, f64)> = Vec::new(); let mut skipped: usize = 0; + // Members of multipoints with another finite member, missing a coordinate or with an infinite one. + let mut missing_members: usize = 0; // A point the system can't hold is left out, and counted, as one past 85.05° in Web Mercator, // which would land on the map's edge rather than where it is (#1033): its latitude doesn't // come back. @@ -3621,18 +3630,20 @@ fn kernel_density_tool(inp: &dyn Inputs, p: &Json) -> Result } } Geometry::MultiPoint(cs) => { - let mut any = false; - for c in cs { - if c.is_finite() - && let Some(p_proj) = project(*c, &mut beyond) - { + // As for a point: a feature with no finite member is skipped, and a member the + // system can't hold counts as beyond; a member missing a coordinate in a feature + // with a finite one is left out and counted on its own. + let finite = cs.iter().filter(|c| c.is_finite()).count(); + if finite == 0 { + skipped += 1; + continue; + } + missing_members += cs.len() - finite; + for c in cs.iter().filter(|c| c.is_finite()) { + if let Some(p_proj) = project(*c, &mut beyond) { pts.push((p_proj, w)); - any = true; } } - if !any { - skipped += 1; - } } _ => { skipped += 1; @@ -3643,9 +3654,12 @@ fn kernel_density_tool(inp: &dyn Inputs, p: &Json) -> Result // Only Web Mercator's limit is known to be a latitude. let why = if crs == Crs::WebMercator { " (past 85.05° of latitude)" } else { "" }; let beyond_note = (beyond > 0).then(|| format!("{beyond} point(s) beyond what {} can hold{why} were left out", crs.label())); + let missing_note = + (missing_members > 0).then(|| format!("{missing_members} multipoint member(s) were left out: a coordinate missing or infinite")); if pts.is_empty() { let none = "input layer has no valid points with positive weights to analyze"; - return Err(beyond_note.map_or_else(|| none.to_string(), |note| format!("{none}: {note}"))); + let notes: Vec<&str> = beyond_note.iter().chain(missing_note.iter()).map(String::as_str).collect(); + return Err(if notes.is_empty() { none.to_string() } else { format!("{none}: {}", notes.join("; ")) }); } let pt_extent = crate::density::points_extent(&pts).ok_or("input layer has no valid points with positive weights to analyze")?; @@ -3708,7 +3722,7 @@ fn kernel_density_tool(inp: &dyn Inputs, p: &Json) -> Result }; Ok(ToolResult { output: Output::Layers(vec![OutputLayer::Raster { file_name: format!("{name}.tif"), name, bytes }]), - messages: std::iter::once(msg).chain(beyond_note).collect(), + messages: std::iter::once(msg).chain(beyond_note).chain(missing_note).collect(), }) } @@ -8294,7 +8308,8 @@ mod tests { /// Minimum Bounding Geometry says how many hulls have no area: points around a pole on one /// parallel (one line in the analysis plane), all together or one feature each, and points - /// spread over an area say nothing of the kind. + /// spread over an area say nothing of the kind; nor do single points, one feature each, whose + /// hulls have no area as a matter of course. #[test] fn convex_hull_counts_hulls_with_no_area() { let layer = |coords: &[(f64, f64)], multi: bool| { @@ -8323,6 +8338,26 @@ mod tests { let spread = [(0.0, 60.0), (10.0, 60.0), (5.0, 65.0), (0.0, 89.0), (90.0, 88.0)]; assert_eq!(note(&layer(&spread, false), "All"), None); assert_eq!(note(&layer(&spread, true), "None"), None); + assert_eq!(note(&layer(&spread, false), "None"), None, "single points"); + } + + /// Minimum Bounding Geometry's `count`, all features together, is the features it hulled: with a + /// selection, the selected ones, and never a feature without geometry. + #[test] + fn convex_hull_counts_the_features_it_hulls() { + let mut pts = Dataset::new("pts", GeometryType::Point, vec![]); + for c in [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (5.0, 5.0)] { + pts.push(Some(Geometry::Point(Coord::new(c.0, c.1))), vec![]); + } + pts.push(None, vec![]); + let count = |sel: Vec| { + let m = Selected(Map(HashMap::from([("pts".to_string(), pts.clone())])), HashMap::from([("pts".to_string(), sel)])); + let out = layer(crate::run("convexHull", &m, &serde_json::json!({"input": "pts"})).unwrap_or_else(|e| panic!("{e}"))); + out.value(0, 0).as_f64() + }; + // No selection: every feature with a geometry (the one without isn't counted). + assert_eq!(count(vec![]), Some(4.0)); + assert_eq!(count(vec![0, 1, 2]), Some(3.0)); } /// Buffer says how many features' buffers kept edges coarser than their outline allows: a line @@ -8594,6 +8629,39 @@ mod tests { assert!(note.starts_with("1 point(s) beyond what") && !note.contains("85.05°"), "{:?}", res.messages); } + /// Kernel Density counts the multipoint members it leaves out for a missing or infinite + /// coordinate, in a feature with others it places; a feature with none it can place is + /// skipped, and one wholly past 85.05° counts its points as beyond, not as skipped too, as a + /// point does. Whole multipoints say nothing of the kind. + #[test] + fn kernel_density_counts_multipoint_members_left_out() { + let run = |members: Vec>| { + let mut pts = Dataset::new("pts", GeometryType::Point, vec![]); + pts.push(Some(Geometry::MultiPoint(vec![Coord::new(0.0, 60.0), Coord::new(0.01, 60.0)])), vec![]); + for m in members { + pts.push(Some(Geometry::MultiPoint(m)), vec![]); + } + let m = Map(HashMap::from([("pts".to_string(), pts)])); + let params = serde_json::json!({"input": "pts", "crs": "EPSG:3857", "radius": 5000.0, "pixelSize": 1000.0}); + crate::run("kernelDensity", &m, ¶ms).unwrap_or_else(|e| panic!("{e}")).messages + }; + let (gap, inf) = (Coord::new(f64::NAN, 60.0), Coord::new(0.0, f64::INFINITY)); + let messages = run(vec![vec![Coord::new(0.02, 60.0), gap, inf], vec![gap], vec![Coord::new(0.0, 87.0), Coord::new(1.0, 88.0)]]); + let all = messages.join(" | "); + assert!(all.contains("2 multipoint member(s) were left out: a coordinate missing or infinite"), "{all}"); + assert!(all.contains("from 3 points (1 skipped"), "{all}"); + assert!(all.contains("2 point(s) beyond what"), "{all}"); + let clean = run(vec![vec![Coord::new(0.02, 60.0), Coord::new(0.03, 60.0)]]).join(" | "); + assert!(!clean.contains("left out") && !clean.contains("skipped"), "{clean}"); + // With nothing left to place, the error says both why. + let mut only = Dataset::new("only", GeometryType::Point, vec![]); + only.push(Some(Geometry::MultiPoint(vec![gap, Coord::new(0.0, 87.0)])), vec![]); + let m = Map(HashMap::from([("only".to_string(), only)])); + let params = serde_json::json!({"input": "only", "crs": "EPSG:3857", "radius": 5000.0, "pixelSize": 1000.0}); + let err = crate::run("kernelDensity", &m, ¶ms).err().unwrap_or_default(); + assert!(err.contains("1 point(s) beyond what") && err.contains("1 multipoint member(s) were left out"), "{err}"); + } + #[test] fn kernel_density_analysis_tool_tests() { // Create point dataset with UTM projection diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 956307d91e..5d353cd9b2 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -267,7 +267,8 @@ surface from point features using a Quartic (biweight, default) or Gaussian kern renormalised to integrate to 1). Points can optionally be weighted by a numeric population field (invalid or non-positive weights are excluded and reported in the tool messages; for multipoint features, each constituent point part -receives the feature's weight). Calculations are performed in the linear units +receives the feature's weight, a part with a coordinate missing or infinite is left out and +counted, and a feature with no other part is skipped). Calculations are performed in the linear units of a projected coordinate system (defaults to the layer's projected system; if the input layer lacks a coordinate system or is in degrees, or to project into a specific system, supply a projected system via `crs`, e.g. `EPSG:32618`). From 508f3af1223a33f679a5f6bc8c8d88ae14b5d6c3 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:16:34 -0600 Subject: [PATCH 02/15] fix(analysis): Minimum Bounding Geometry passes over a missing coordinate instead of failing (#1556) A vertex with a coordinate missing or infinite made the hull panic (its ordering can't compare a NaN), so the tool failed for the whole layer. It's now passed over and counted. A feature without geometry, left out silently one feature each, is counted too, and with nothing at all to enclose, all together, the tool says so as an error rather than writing an empty hull. The no-area notice counts any hull of two points or more again, stacked ones included; only a single point's is left out. Kernel Density's error with nothing to place also says how many features it skipped. Tests: missing coordinates and a feature without geometry are counted, all together and one feature each, and clean points say nothing; the error with nothing to enclose; stacked points give the no-area notice. --- crates/analysis/src/tools.rs | 108 +++++++++++++++++++++++++---------- docs/user-guide/analysis.md | 2 +- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index c4cd8c218f..cf85f92c33 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2172,22 +2172,28 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } else { Dataset::new(out_name(p, format!("{}_Hull", ds.name)), GeometryType::Polygon, vec![Field::new("count", FieldType::Integer)]) }; + // A feature's points in the analysis plane, and how many of its vertices have a coordinate + // missing or infinite: those are passed over, since the hull can't order them. let points = |g: &Geometry| { - let mut these = Vec::new(); + let (mut these, mut gaps) = (Vec::new(), 0usize); g.for_each_coord(|c| { - let m = rustgis_geom::lonlat_to_plane(c); - these.push(geo::Point::new(m.x, m.y)); + if c.is_finite() { + let m = rustgis_geom::lonlat_to_plane(c); + these.push(geo::Point::new(m.x, m.y)); + } else { + gaps += 1; + } }); - these + (these, gaps) }; - // The features with a geometry, each with it. + let picked = rows(inp, p, "input", ds); + // The features with a geometry, each with it; the rest are left out, and counted. let features: Vec<(&Feature, &Geometry)> = - rows(inp, p, "input", ds).into_iter().filter_map(|r| ds.features.get(r)).filter_map(|f| Some((f, f.geometry.as_ref()?))).collect(); - let mut pts: Vec> = Vec::new(); - // Hulls of points in more than one place that came out with no area: one point's hull has none - // as a matter of course, so it isn't counted. - let mut flat = 0usize; - let spread = |these: &[geo::Point]| these.first().is_some_and(|first| these.iter().any(|q| q != first)); + picked.iter().filter_map(|r| ds.features.get(*r)).filter_map(|f| Some((f, f.geometry.as_ref()?))).collect(); + let without = picked.len().saturating_sub(features.len()); + // A hull of two points or more with no area: one point's hull has none as a matter of course. + let is_flat = |points: usize, hull: &geo::Polygon| points >= 2 && geo::Area::unsigned_area(hull) == 0.0; + let (mut flat, mut gaps) = (0usize, 0usize); if each { // Each feature's hull made on several cores for many (#1252), placed in their order. crate::control::begin(features.len()); @@ -2196,38 +2202,45 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { PARALLEL_FEATURES, || (), |_, _, (_, g)| { - let these = points(g); - let spread = spread(&these); + let (these, passed) = points(g); + let n = these.len(); let hull = geo::MultiPoint::new(these).convex_hull(); - Ok::<_, String>((spread && geo::Area::unsigned_area(&hull) == 0.0, hull)) + Ok::<_, String>((is_flat(n, &hull), passed, hull)) }, )?; - for ((f, _), (is_flat, hull)) in features.iter().zip(hulls) { - flat += usize::from(is_flat); + for ((f, _), (flat_one, passed, hull)) in features.iter().zip(hulls) { + flat += usize::from(flat_one); + gaps += passed; out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), f.attrs.clone()); } - } - // The features hulled together: those picked (the selection, or all) with a point to give. - let mut hulled = 0usize; - if !each { + } else { + // All together; `count` is the features hulled, those with a point to give. + let (mut pts, mut hulled) = (Vec::new(), 0usize); for (_, g) in &features { - let these = points(g); + let (these, passed) = points(g); hulled += usize::from(!these.is_empty()); + gaps += passed; pts.extend(these); } - } - if !each { - let n = i64::try_from(hulled).unwrap_or(i64::MAX); - let spread = spread(&pts); + if hulled == 0 { + return Err("no feature has a point to enclose (no geometry, or only coordinates missing or infinite)".into()); + } + let n = pts.len(); let hull = geo::MultiPoint::new(pts).convex_hull(); - flat += usize::from(spread && geo::Area::unsigned_area(&hull) == 0.0); - out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(n)]); + flat += usize::from(is_flat(n, &hull)); + out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; + if without > 0 { + r.messages.push(format!("Minimum Bounding Geometry: {without} feature(s) without geometry skipped")); + } + if gaps > 0 { + r.messages.push(format!("Minimum Bounding Geometry: {gaps} vertex(es) with a coordinate missing or infinite passed over")); + } if flat > 0 { r.messages.push(format!( - "Minimum Bounding Geometry: {flat} of {n} hulls have no area: their points lie on one straight line on the map (points on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" + "Minimum Bounding Geometry: {flat} of {n} hulls have no area: their points lie in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" )); } Ok(r) @@ -3658,7 +3671,9 @@ fn kernel_density_tool(inp: &dyn Inputs, p: &Json) -> Result (missing_members > 0).then(|| format!("{missing_members} multipoint member(s) were left out: a coordinate missing or infinite")); if pts.is_empty() { let none = "input layer has no valid points with positive weights to analyze"; - let notes: Vec<&str> = beyond_note.iter().chain(missing_note.iter()).map(String::as_str).collect(); + let skipped_note = (skipped > 0) + .then(|| format!("{skipped} feature(s) skipped (no point geometry, a weight that isn't positive, or no finite coordinate)")); + let notes: Vec<&str> = skipped_note.iter().chain(beyond_note.iter()).chain(missing_note.iter()).map(String::as_str).collect(); return Err(if notes.is_empty() { none.to_string() } else { format!("{none}: {}", notes.join("; ")) }); } @@ -8339,6 +8354,41 @@ mod tests { assert_eq!(note(&layer(&spread, false), "All"), None); assert_eq!(note(&layer(&spread, true), "None"), None); assert_eq!(note(&layer(&spread, false), "None"), None, "single points"); + // Features stacked in one place, together, have a hull with no area, and say so. + let stacked = [(10.0, 20.0), (10.0, 20.0), (10.0, 20.0)]; + let note_stacked = note(&layer(&stacked, false), "All").unwrap_or_default(); + assert!(note_stacked.contains(": 1 of 1 hulls have no area"), "{note_stacked}"); + } + + /// Minimum Bounding Geometry passes over a vertex with a coordinate missing or infinite, which + /// the hull can't order (it failed on one before), and skips a feature without geometry, saying + /// how many of each, all together or one feature each; with nothing left to enclose, all + /// together, it's an error. Clean points say nothing of the kind. + #[test] + fn convex_hull_passes_over_missing_coordinates() { + let run = |geoms: Vec>, group: &str| { + let mut pts = Dataset::new("pts", GeometryType::Point, vec![]); + for g in geoms { + pts.push(g, vec![]); + } + let m = Map(HashMap::from([("pts".to_string(), pts)])); + crate::run("convexHull", &m, &serde_json::json!({"input": "pts", "group": group})) + }; + let gap = Coord::new(f64::NAN, 40.0); + let stations = Geometry::MultiPoint(vec![Coord::new(-75.0, 40.0), Coord::new(-74.99, 40.0), Coord::new(-75.0, 40.01), gap]); + for group in ["All", "None"] { + let res = run(vec![Some(stations.clone()), Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); + let all = res.messages.join(" | "); + assert!(all.contains("2 vertex(es) with a coordinate missing or infinite passed over"), "{group}: {all}"); + assert!(all.contains("1 feature(s) without geometry skipped"), "{group}: {all}"); + let clean = run(vec![Some(Geometry::Point(Coord::new(0.0, 0.0))), Some(Geometry::Point(Coord::new(1.0, 1.0)))], group) + .unwrap_or_else(|e| panic!("{group}: {e}")) + .messages + .join(" | "); + assert!(!clean.contains("passed over") && !clean.contains("skipped"), "{group}: {clean}"); + } + let err = run(vec![Some(Geometry::Point(gap)), None], "All").err().unwrap_or_default(); + assert!(err.contains("no feature has a point to enclose"), "{err}"); } /// Minimum Bounding Geometry's `count`, all features together, is the features it hulled: with a diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 5d353cd9b2..312b2dc9d0 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -268,7 +268,7 @@ renormalised to integrate to 1). Points can optionally be weighted by a numeric population field (invalid or non-positive weights are excluded and reported in the tool messages; for multipoint features, each constituent point part receives the feature's weight, a part with a coordinate missing or infinite is left out and -counted, and a feature with no other part is skipped). Calculations are performed in the linear units +counted, and a feature with no finite part is skipped). Calculations are performed in the linear units of a projected coordinate system (defaults to the layer's projected system; if the input layer lacks a coordinate system or is in degrees, or to project into a specific system, supply a projected system via `crs`, e.g. `EPSG:32618`). From 542cbf26bf84891b0f16948f3babb2c8a93df478 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:34:08 -0600 Subject: [PATCH 03/15] fix(analysis): Minimum Bounding Geometry passes over far-out vertices and writes no hull for a feature with no usable point A vertex so far out (a longitude in the millions of degrees) that the hull's products of differences overflow could still make it panic; it's passed over and counted like a missing coordinate. One feature each, a feature with no point the hull can use got an empty polygon; it's now skipped and counted with those without geometry. The no-area notice counts a single point's hull again and names it, and count holds each row once however often a selection lists it. Tests: a far-out vertex is passed over (the hull fails without the limit), no hull is written for a feature with no usable point, single points give the notice, and a row listed twice counts once. --- crates/analysis/src/tools.rs | 107 +++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index cf85f92c33..f1c5a6424b 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2164,6 +2164,11 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { Ok(ToolResult { output: Output::Layer(out), messages }) } +/// The farthest (meters, either way) Minimum Bounding Geometry takes a point from the analysis +/// plane's origin: about 9 million degrees of longitude, far past any real data and far short of +/// where the hull's products of coordinate differences overflow. +const HULL_LIMIT_M: f64 = 1e12; + fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { let ds = input(inp, p, "input")?; let each = s(p, "group") == "None"; @@ -2172,27 +2177,28 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } else { Dataset::new(out_name(p, format!("{}_Hull", ds.name)), GeometryType::Polygon, vec![Field::new("count", FieldType::Integer)]) }; - // A feature's points in the analysis plane, and how many of its vertices have a coordinate - // missing or infinite: those are passed over, since the hull can't order them. + // A feature's points in the analysis plane, and how many of its vertices are passed over: a + // coordinate missing or infinite, which the hull can't order, or one so far out (a longitude + // in the millions of degrees) that the hull's arithmetic would overflow. let points = |g: &Geometry| { let (mut these, mut gaps) = (Vec::new(), 0usize); g.for_each_coord(|c| { - if c.is_finite() { - let m = rustgis_geom::lonlat_to_plane(c); - these.push(geo::Point::new(m.x, m.y)); - } else { - gaps += 1; + let m = if c.is_finite() { Some(rustgis_geom::lonlat_to_plane(c)) } else { None }; + match m { + Some(m) if m.x.abs() <= HULL_LIMIT_M && m.y.abs() <= HULL_LIMIT_M => these.push(geo::Point::new(m.x, m.y)), + _ => gaps += 1, } }); (these, gaps) }; let picked = rows(inp, p, "input", ds); - // The features with a geometry, each with it; the rest are left out, and counted. - let features: Vec<(&Feature, &Geometry)> = - picked.iter().filter_map(|r| ds.features.get(*r)).filter_map(|f| Some((f, f.geometry.as_ref()?))).collect(); - let without = picked.len().saturating_sub(features.len()); - // A hull of two points or more with no area: one point's hull has none as a matter of course. - let is_flat = |points: usize, hull: &geo::Polygon| points >= 2 && geo::Area::unsigned_area(hull) == 0.0; + // The features with a geometry, by row, each with it; the rest are left out, and counted. + let features: Vec<(usize, &Feature, &Geometry)> = + picked.iter().filter_map(|&r| ds.features.get(r).and_then(|f| Some((r, f, f.geometry.as_ref()?)))).collect(); + // Features skipped: no geometry, or no point the hull can use. + let mut skipped = picked.len().saturating_sub(features.len()); + // A hull with no area: a single point's, or points in one place or along one line. + let is_flat = |hull: &geo::Polygon| geo::Area::unsigned_area(hull) == 0.0; let (mut flat, mut gaps) = (0usize, 0usize); if each { // Each feature's hull made on several cores for many (#1252), placed in their order. @@ -2201,46 +2207,57 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { &features, PARALLEL_FEATURES, || (), - |_, _, (_, g)| { + |_, _, (_, _, g)| { let (these, passed) = points(g); - let n = these.len(); - let hull = geo::MultiPoint::new(these).convex_hull(); - Ok::<_, String>((is_flat(n, &hull), passed, hull)) + // A feature with no point to give has no hull: none is written for it. + let hull = (!these.is_empty()).then(|| geo::MultiPoint::new(these).convex_hull()); + Ok::<_, String>((passed, hull)) }, )?; - for ((f, _), (flat_one, passed, hull)) in features.iter().zip(hulls) { - flat += usize::from(flat_one); + for ((_, f, _), (passed, hull)) in features.iter().zip(hulls) { gaps += passed; + let Some(hull) = hull else { + skipped += 1; + continue; + }; + flat += usize::from(is_flat(&hull)); out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), f.attrs.clone()); } } else { - // All together; `count` is the features hulled, those with a point to give. - let (mut pts, mut hulled) = (Vec::new(), 0usize); - for (_, g) in &features { + // All together; `count` is the features hulled, those with a point to give, each once + // however often a selection lists it. + let (mut pts, mut hulled) = (Vec::new(), Vec::new()); + for (r, _, g) in &features { let (these, passed) = points(g); - hulled += usize::from(!these.is_empty()); gaps += passed; + if these.is_empty() { + skipped += 1; + } else { + hulled.push(*r); + } pts.extend(these); } - if hulled == 0 { - return Err("no feature has a point to enclose (no geometry, or only coordinates missing or infinite)".into()); + hulled.sort_unstable(); + hulled.dedup(); + if hulled.is_empty() { + return Err("no feature has a point to enclose (no geometry, or only coordinates missing, infinite or out of range)".into()); } - let n = pts.len(); let hull = geo::MultiPoint::new(pts).convex_hull(); - flat += usize::from(is_flat(n, &hull)); - out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); + flat += usize::from(is_flat(&hull)); + out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled.len()).unwrap_or(i64::MAX))]); } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; - if without > 0 { - r.messages.push(format!("Minimum Bounding Geometry: {without} feature(s) without geometry skipped")); + if skipped > 0 { + r.messages.push(format!("Minimum Bounding Geometry: {skipped} feature(s) skipped: no geometry, or no point it can use")); } if gaps > 0 { - r.messages.push(format!("Minimum Bounding Geometry: {gaps} vertex(es) with a coordinate missing or infinite passed over")); + r.messages + .push(format!("Minimum Bounding Geometry: {gaps} vertex(es) with a coordinate missing, infinite or out of range passed over")); } if flat > 0 { r.messages.push(format!( - "Minimum Bounding Geometry: {flat} of {n} hulls have no area: their points lie in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" + "Minimum Bounding Geometry: {flat} of {n} hulls have no area: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" )); } Ok(r) @@ -8323,8 +8340,8 @@ mod tests { /// Minimum Bounding Geometry says how many hulls have no area: points around a pole on one /// parallel (one line in the analysis plane), all together or one feature each, and points - /// spread over an area say nothing of the kind; nor do single points, one feature each, whose - /// hulls have no area as a matter of course. + /// spread over an area say nothing of the kind. Single points, one feature each, and points + /// stacked in one place have hulls with no area too, and say so. #[test] fn convex_hull_counts_hulls_with_no_area() { let layer = |coords: &[(f64, f64)], multi: bool| { @@ -8353,7 +8370,8 @@ mod tests { let spread = [(0.0, 60.0), (10.0, 60.0), (5.0, 65.0), (0.0, 89.0), (90.0, 88.0)]; assert_eq!(note(&layer(&spread, false), "All"), None); assert_eq!(note(&layer(&spread, true), "None"), None); - assert_eq!(note(&layer(&spread, false), "None"), None, "single points"); + let note_single = note(&layer(&spread, false), "None").unwrap_or_default(); + assert!(note_single.contains(": 5 of 5 hulls have no area: a single point"), "{note_single}"); // Features stacked in one place, together, have a hull with no area, and say so. let stacked = [(10.0, 20.0), (10.0, 20.0), (10.0, 20.0)]; let note_stacked = note(&layer(&stacked, false), "All").unwrap_or_default(); @@ -8361,9 +8379,10 @@ mod tests { } /// Minimum Bounding Geometry passes over a vertex with a coordinate missing or infinite, which - /// the hull can't order (it failed on one before), and skips a feature without geometry, saying - /// how many of each, all together or one feature each; with nothing left to enclose, all - /// together, it's an error. Clean points say nothing of the kind. + /// the hull can't order (it failed on one before), or so far out that its arithmetic would + /// overflow, and skips a feature without geometry or with no point it can use (writing no hull + /// for it), saying how many of each, all together or one feature each; with nothing left to + /// enclose, all together, it's an error. Clean points say nothing of the kind. #[test] fn convex_hull_passes_over_missing_coordinates() { let run = |geoms: Vec>, group: &str| { @@ -8374,13 +8393,15 @@ mod tests { let m = Map(HashMap::from([("pts".to_string(), pts)])); crate::run("convexHull", &m, &serde_json::json!({"input": "pts", "group": group})) }; - let gap = Coord::new(f64::NAN, 40.0); - let stations = Geometry::MultiPoint(vec![Coord::new(-75.0, 40.0), Coord::new(-74.99, 40.0), Coord::new(-75.0, 40.01), gap]); + let (gap, far) = (Coord::new(f64::NAN, 40.0), Coord::new(1e308, 0.0)); + let stations = Geometry::MultiPoint(vec![Coord::new(-75.0, 40.0), Coord::new(-74.99, 40.0), Coord::new(-75.0, 40.01), gap, far]); for group in ["All", "None"] { let res = run(vec![Some(stations.clone()), Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); let all = res.messages.join(" | "); - assert!(all.contains("2 vertex(es) with a coordinate missing or infinite passed over"), "{group}: {all}"); - assert!(all.contains("1 feature(s) without geometry skipped"), "{group}: {all}"); + assert!(all.contains("3 vertex(es) with a coordinate missing, infinite or out of range passed over"), "{group}: {all}"); + assert!(all.contains("2 feature(s) skipped: no geometry, or no point it can use"), "{group}: {all}"); + let Output::Layer(out) = res.output else { panic!("{group}: not a layer") }; + assert_eq!(out.features.len(), 1, "{group}: a hull for the stations only"); let clean = run(vec![Some(Geometry::Point(Coord::new(0.0, 0.0))), Some(Geometry::Point(Coord::new(1.0, 1.0)))], group) .unwrap_or_else(|e| panic!("{group}: {e}")) .messages @@ -8408,6 +8429,8 @@ mod tests { // No selection: every feature with a geometry (the one without isn't counted). assert_eq!(count(vec![]), Some(4.0)); assert_eq!(count(vec![0, 1, 2]), Some(3.0)); + // A row a selection lists twice is one feature. + assert_eq!(count(vec![0, 0, 1]), Some(2.0)); } /// Buffer says how many features' buffers kept edges coarser than their outline allows: a line From 441be1c12c34f823d4f836248bd658edfd0ac6ba Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:47:26 -0600 Subject: [PATCH 04/15] fix(analysis): Minimum Bounding Geometry takes each picked row once and writes no hull with nothing to enclose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A row a selection lists twice was counted twice in the skipped and passed-over notices, and hulled twice one feature each; the rows are taken once. A closed ring's repeated last point no longer counts a bad vertex twice. All together with nothing to enclose now writes no hull and says why, as one feature each does, rather than failing the run. The passed-over notice says what it catches (a longitude far out of range; a latitude past ±90° is the pole, as the analysis plane takes it), and the user guide describes the tool. Tests: nothing to enclose gives no hull either way, with the skipped count; a far-out vertex in a closed ring counts once. --- crates/analysis/src/tools.rs | 79 +++++++++++++++++++++++------------- docs/user-guide/analysis.md | 7 ++++ 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index f1c5a6424b..0e8773b088 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2166,7 +2166,8 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { /// The farthest (meters, either way) Minimum Bounding Geometry takes a point from the analysis /// plane's origin: about 9 million degrees of longitude, far past any real data and far short of -/// where the hull's products of coordinate differences overflow. +/// where the hull's products of coordinate differences overflow. (The plane's y is bounded: a +/// latitude past ±90° is taken as the pole.) const HULL_LIMIT_M: f64 = 1e12; fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { @@ -2177,12 +2178,13 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } else { Dataset::new(out_name(p, format!("{}_Hull", ds.name)), GeometryType::Polygon, vec![Field::new("count", FieldType::Integer)]) }; - // A feature's points in the analysis plane, and how many of its vertices are passed over: a - // coordinate missing or infinite, which the hull can't order, or one so far out (a longitude - // in the millions of degrees) that the hull's arithmetic would overflow. + // A feature's points in the analysis plane, and how many of its vertices (a closed ring's + // repeated last point left out) are passed over: a coordinate missing or infinite, which the + // hull can't order, or a longitude so far out (millions of degrees) that the hull's arithmetic + // would overflow. A latitude past ±90° is taken as the pole, as the analysis plane takes it. let points = |g: &Geometry| { let (mut these, mut gaps) = (Vec::new(), 0usize); - g.for_each_coord(|c| { + g.for_each_vertex(|_, c| { let m = if c.is_finite() { Some(rustgis_geom::lonlat_to_plane(c)) } else { None }; match m { Some(m) if m.x.abs() <= HULL_LIMIT_M && m.y.abs() <= HULL_LIMIT_M => these.push(geo::Point::new(m.x, m.y)), @@ -2191,10 +2193,13 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { }); (these, gaps) }; - let picked = rows(inp, p, "input", ds); - // The features with a geometry, by row, each with it; the rest are left out, and counted. - let features: Vec<(usize, &Feature, &Geometry)> = - picked.iter().filter_map(|&r| ds.features.get(r).and_then(|f| Some((r, f, f.geometry.as_ref()?)))).collect(); + // Each row once, however often a selection lists it. + let mut picked = rows(inp, p, "input", ds); + picked.sort_unstable(); + picked.dedup(); + // The features with a geometry, each with it; the rest are left out, and counted. + let features: Vec<(&Feature, &Geometry)> = + picked.iter().filter_map(|&r| ds.features.get(r).and_then(|f| Some((f, f.geometry.as_ref()?)))).collect(); // Features skipped: no geometry, or no point the hull can use. let mut skipped = picked.len().saturating_sub(features.len()); // A hull with no area: a single point's, or points in one place or along one line. @@ -2207,14 +2212,14 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { &features, PARALLEL_FEATURES, || (), - |_, _, (_, _, g)| { + |_, _, (_, g)| { let (these, passed) = points(g); // A feature with no point to give has no hull: none is written for it. let hull = (!these.is_empty()).then(|| geo::MultiPoint::new(these).convex_hull()); Ok::<_, String>((passed, hull)) }, )?; - for ((_, f, _), (passed, hull)) in features.iter().zip(hulls) { + for ((f, _), (passed, hull)) in features.iter().zip(hulls) { gaps += passed; let Some(hull) = hull else { skipped += 1; @@ -2224,27 +2229,24 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), f.attrs.clone()); } } else { - // All together; `count` is the features hulled, those with a point to give, each once - // however often a selection lists it. - let (mut pts, mut hulled) = (Vec::new(), Vec::new()); - for (r, _, g) in &features { + // All together; `count` is the features hulled, those with a point to give. With none, + // no hull is written, as one feature each. + let (mut pts, mut hulled) = (Vec::new(), 0usize); + for (_, g) in &features { let (these, passed) = points(g); gaps += passed; if these.is_empty() { skipped += 1; } else { - hulled.push(*r); + hulled += 1; } pts.extend(these); } - hulled.sort_unstable(); - hulled.dedup(); - if hulled.is_empty() { - return Err("no feature has a point to enclose (no geometry, or only coordinates missing, infinite or out of range)".into()); + if hulled > 0 { + let hull = geo::MultiPoint::new(pts).convex_hull(); + flat += usize::from(is_flat(&hull)); + out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); } - let hull = geo::MultiPoint::new(pts).convex_hull(); - flat += usize::from(is_flat(&hull)); - out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled.len()).unwrap_or(i64::MAX))]); } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; @@ -2252,8 +2254,9 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { r.messages.push(format!("Minimum Bounding Geometry: {skipped} feature(s) skipped: no geometry, or no point it can use")); } if gaps > 0 { - r.messages - .push(format!("Minimum Bounding Geometry: {gaps} vertex(es) with a coordinate missing, infinite or out of range passed over")); + r.messages.push(format!( + "Minimum Bounding Geometry: {gaps} vertex(es) passed over: a coordinate missing or infinite, or a longitude far out of range" + )); } if flat > 0 { r.messages.push(format!( @@ -8382,7 +8385,8 @@ mod tests { /// the hull can't order (it failed on one before), or so far out that its arithmetic would /// overflow, and skips a feature without geometry or with no point it can use (writing no hull /// for it), saying how many of each, all together or one feature each; with nothing left to - /// enclose, all together, it's an error. Clean points say nothing of the kind. + /// enclose, it writes no hull either way and says why. A bad vertex in a closed ring counts + /// once. Clean points say nothing of the kind. #[test] fn convex_hull_passes_over_missing_coordinates() { let run = |geoms: Vec>, group: &str| { @@ -8398,7 +8402,10 @@ mod tests { for group in ["All", "None"] { let res = run(vec![Some(stations.clone()), Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); let all = res.messages.join(" | "); - assert!(all.contains("3 vertex(es) with a coordinate missing, infinite or out of range passed over"), "{group}: {all}"); + assert!( + all.contains("3 vertex(es) passed over: a coordinate missing or infinite, or a longitude far out of range"), + "{group}: {all}" + ); assert!(all.contains("2 feature(s) skipped: no geometry, or no point it can use"), "{group}: {all}"); let Output::Layer(out) = res.output else { panic!("{group}: not a layer") }; assert_eq!(out.features.len(), 1, "{group}: a hull for the stations only"); @@ -8408,8 +8415,22 @@ mod tests { .join(" | "); assert!(!clean.contains("passed over") && !clean.contains("skipped"), "{group}: {clean}"); } - let err = run(vec![Some(Geometry::Point(gap)), None], "All").err().unwrap_or_default(); - assert!(err.contains("no feature has a point to enclose"), "{err}"); + for group in ["All", "None"] { + let res = run(vec![Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); + let Output::Layer(out) = &res.output else { panic!("{group}: not a layer") }; + assert!(out.features.is_empty(), "{group}: no hull with nothing to enclose"); + assert!(res.messages.iter().any(|m| m.contains("2 feature(s) skipped")), "{group}: {:?}", res.messages); + } + // A ring with one vertex too far out, its closing point the same: one vertex passed over. + let far_ring = Geometry::Polygon(vec![vec![vec![ + Coord::new(1e15, 0.0), + Coord::new(1.0, 0.0), + Coord::new(1.0, 1.0), + Coord::new(0.0, 1.0), + Coord::new(1e15, 0.0), + ]]]); + let all = run(vec![Some(far_ring)], "All").unwrap_or_else(|e| panic!("{e}")).messages.join(" | "); + assert!(all.contains(": 1 vertex(es) passed over"), "{all}"); } /// Minimum Bounding Geometry's `count`, all features together, is the features it hulled: with a diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 312b2dc9d0..910ad09b36 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -686,6 +686,13 @@ notice. A Double field's values are decimals already, rounded as decimal arithme get no notice. Without `statistics`, `field` gives that field's `COUNT`, `SUM`, `MEAN`, `MIN`, `MAX` and `STD`, reading its values as numbers. +**Minimum Bounding Geometry** (`convexHull`) encloses each feature in its convex hull (Group +Option `None`), or all of them in one (`All`, with `count`, the features it enclosed). It passes +over a vertex with a coordinate missing or infinite, or a longitude far out of range, and skips a +feature without geometry or with no point left to enclose, writing no hull for it; it says how +many of each, and how many hulls have no area (a single point, or points in one place or on one +line on the map). + **Dissolve** (`dissolve`) merges the polygons sharing a value of `field` into one feature for each value, or all of them into one without a field. Each feature has the value and `count`, the polygons merged; records with no value form a feature of their own. **Summarize Within** From bdcf18b28912d4e905b939b6244db48fe7898552 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:26:55 -0600 Subject: [PATCH 05/15] fix(analysis): Minimum Bounding Geometry keeps the selection's order while taking each row once Dropping repeated rows by sorting them put the hulls, one feature each, in the layer's order instead of the selection's. Repeats are now dropped as they come. Test: a selection of rows 2, 0, 2 gives the hulls of 2 and 0, in that order. --- crates/analysis/src/tools.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 0e8773b088..4cf522a0bd 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2193,10 +2193,9 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { }); (these, gaps) }; - // Each row once, however often a selection lists it. - let mut picked = rows(inp, p, "input", ds); - picked.sort_unstable(); - picked.dedup(); + // Each row once, however often a selection lists it, in the selection's order. + let mut seen = std::collections::HashSet::new(); + let picked: Vec = rows(inp, p, "input", ds).into_iter().filter(|r| seen.insert(*r)).collect(); // The features with a geometry, each with it; the rest are left out, and counted. let features: Vec<(&Feature, &Geometry)> = picked.iter().filter_map(|&r| ds.features.get(r).and_then(|f| Some((f, f.geometry.as_ref()?)))).collect(); @@ -8452,6 +8451,16 @@ mod tests { assert_eq!(count(vec![0, 1, 2]), Some(3.0)); // A row a selection lists twice is one feature. assert_eq!(count(vec![0, 0, 1]), Some(2.0)); + // One feature each, the hulls follow the selection's order, each row once. + let mut named = Dataset::new("pts", GeometryType::Point, vec![Field::new("name", FieldType::Text)]); + for (name, x) in [("a", 0.0), ("b", 1.0), ("c", 2.0)] { + named.push(Some(Geometry::Point(Coord::new(x, 0.0))), vec![Value::Text(name.into())]); + } + let m = Selected(Map(HashMap::from([("pts".to_string(), named)])), HashMap::from([("pts".to_string(), vec![2, 0, 2])])); + let out = + layer(crate::run("convexHull", &m, &serde_json::json!({"input": "pts", "group": "None"})).unwrap_or_else(|e| panic!("{e}"))); + let names: Vec = (0..out.features.len()).map(|r| out.value(r, 0).to_string()).collect(); + assert_eq!(names, ["c", "a"]); } /// Buffer says how many features' buffers kept edges coarser than their outline allows: a line From 59ead33b7d0a2f8268379d2b284491624ca2c718 Mon Sep 17 00:00:00 2001 From: giswqs <5016453+giswqs@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:16:27 +0000 Subject: [PATCH 06/15] fix(analysis): Minimum Bounding Geometry errors when nothing hulls With every feature skipped, it returned an empty layer; it now returns an error saying how many features had no point to hull, as Kernel Density does for an unusable result. A latitude past a pole is taken as that pole, as Points Along Lines takes it, and the notice now says how many such vertices were found. The passed-over notice names only what its limit catches: a coordinate missing or infinite, or far beyond about 9 million degrees. A ring that repeats a NaN start point as its end counts that vertex once: the closing point is matched by its bits, as NaN equals nothing as a number. Co-Authored-By: Claude Haiku 5.5 Applied by the claude-autofix workflow. --- crates/analysis/src/tools.rs | 66 ++++++++++++++++++++++++++---------- crates/data/src/geometry.rs | 8 ++++- docs/user-guide/analysis.md | 9 ++--- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 4cf522a0bd..88080a6f2c 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2181,17 +2181,21 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { // A feature's points in the analysis plane, and how many of its vertices (a closed ring's // repeated last point left out) are passed over: a coordinate missing or infinite, which the // hull can't order, or a longitude so far out (millions of degrees) that the hull's arithmetic - // would overflow. A latitude past ±90° is taken as the pole, as the analysis plane takes it. + // would overflow. A latitude past ±90° is taken as the pole, as the analysis plane takes it, and + // counted too. let points = |g: &Geometry| { - let (mut these, mut gaps) = (Vec::new(), 0usize); + let (mut these, mut gaps, mut poles) = (Vec::new(), 0usize, 0usize); g.for_each_vertex(|_, c| { let m = if c.is_finite() { Some(rustgis_geom::lonlat_to_plane(c)) } else { None }; match m { - Some(m) if m.x.abs() <= HULL_LIMIT_M && m.y.abs() <= HULL_LIMIT_M => these.push(geo::Point::new(m.x, m.y)), + Some(m) if m.x.abs() <= HULL_LIMIT_M && m.y.abs() <= HULL_LIMIT_M => { + poles += usize::from(c.y.abs() > 90.0); + these.push(geo::Point::new(m.x, m.y)); + } _ => gaps += 1, } }); - (these, gaps) + (these, gaps, poles) }; // Each row once, however often a selection lists it, in the selection's order. let mut seen = std::collections::HashSet::new(); @@ -2203,7 +2207,7 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { let mut skipped = picked.len().saturating_sub(features.len()); // A hull with no area: a single point's, or points in one place or along one line. let is_flat = |hull: &geo::Polygon| geo::Area::unsigned_area(hull) == 0.0; - let (mut flat, mut gaps) = (0usize, 0usize); + let (mut flat, mut gaps, mut poles) = (0usize, 0usize, 0usize); if each { // Each feature's hull made on several cores for many (#1252), placed in their order. crate::control::begin(features.len()); @@ -2212,14 +2216,15 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { PARALLEL_FEATURES, || (), |_, _, (_, g)| { - let (these, passed) = points(g); + let (these, passed, past) = points(g); // A feature with no point to give has no hull: none is written for it. let hull = (!these.is_empty()).then(|| geo::MultiPoint::new(these).convex_hull()); - Ok::<_, String>((passed, hull)) + Ok::<_, String>((passed, past, hull)) }, )?; - for ((f, _), (passed, hull)) in features.iter().zip(hulls) { + for ((f, _), (passed, past, hull)) in features.iter().zip(hulls) { gaps += passed; + poles += past; let Some(hull) = hull else { skipped += 1; continue; @@ -2232,8 +2237,9 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { // no hull is written, as one feature each. let (mut pts, mut hulled) = (Vec::new(), 0usize); for (_, g) in &features { - let (these, passed) = points(g); + let (these, passed, past) = points(g); gaps += passed; + poles += past; if these.is_empty() { skipped += 1; } else { @@ -2247,6 +2253,10 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); } } + // Nothing hulled, every feature skipped: an unusable result, so an error saying why. + if out.features.is_empty() && skipped > 0 { + return Err(format!("Minimum Bounding Geometry: none of the {} feature(s) has a point it can hull", picked.len())); + } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; if skipped > 0 { @@ -2254,9 +2264,12 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } if gaps > 0 { r.messages.push(format!( - "Minimum Bounding Geometry: {gaps} vertex(es) passed over: a coordinate missing or infinite, or a longitude far out of range" + "Minimum Bounding Geometry: {gaps} vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)" )); } + if poles > 0 { + r.messages.push(format!("Minimum Bounding Geometry: {poles} vertex(es) with a latitude past a pole found (taken as that pole)")); + } if flat > 0 { r.messages.push(format!( "Minimum Bounding Geometry: {flat} of {n} hulls have no area: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" @@ -8384,8 +8397,9 @@ mod tests { /// the hull can't order (it failed on one before), or so far out that its arithmetic would /// overflow, and skips a feature without geometry or with no point it can use (writing no hull /// for it), saying how many of each, all together or one feature each; with nothing left to - /// enclose, it writes no hull either way and says why. A bad vertex in a closed ring counts - /// once. Clean points say nothing of the kind. + /// enclose, it is an error saying why. A latitude past a pole is taken as that pole, and says + /// so. A bad vertex in a closed ring counts once, a repeated NaN point included. Clean points + /// say nothing of the kind. #[test] fn convex_hull_passes_over_missing_coordinates() { let run = |geoms: Vec>, group: &str| { @@ -8402,7 +8416,9 @@ mod tests { let res = run(vec![Some(stations.clone()), Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); let all = res.messages.join(" | "); assert!( - all.contains("3 vertex(es) passed over: a coordinate missing or infinite, or a longitude far out of range"), + all.contains( + "3 vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)" + ), "{group}: {all}" ); assert!(all.contains("2 feature(s) skipped: no geometry, or no point it can use"), "{group}: {all}"); @@ -8415,10 +8431,8 @@ mod tests { assert!(!clean.contains("passed over") && !clean.contains("skipped"), "{group}: {clean}"); } for group in ["All", "None"] { - let res = run(vec![Some(Geometry::Point(gap)), None], group).unwrap_or_else(|e| panic!("{group}: {e}")); - let Output::Layer(out) = &res.output else { panic!("{group}: not a layer") }; - assert!(out.features.is_empty(), "{group}: no hull with nothing to enclose"); - assert!(res.messages.iter().any(|m| m.contains("2 feature(s) skipped")), "{group}: {:?}", res.messages); + let err = run(vec![Some(Geometry::Point(gap)), None], group).err().unwrap_or_default(); + assert!(err.contains("none of the 2 feature(s) has a point it can hull"), "{group}: {err}"); } // A ring with one vertex too far out, its closing point the same: one vertex passed over. let far_ring = Geometry::Polygon(vec![vec![vec![ @@ -8430,6 +8444,24 @@ mod tests { ]]]); let all = run(vec![Some(far_ring)], "All").unwrap_or_else(|e| panic!("{e}")).messages.join(" | "); assert!(all.contains(": 1 vertex(es) passed over"), "{all}"); + // A ring that repeats its NaN start point as its end: that vertex is passed over once. + let nan_ring = Geometry::Polygon(vec![vec![vec![ + gap, + Coord::new(1.0, 0.0), + Coord::new(1.0, 1.0), + Coord::new(0.0, 1.0), + gap, + ]]]); + let all = run(vec![Some(nan_ring)], "All").unwrap_or_else(|e| panic!("{e}")).messages.join(" | "); + assert!(all.contains(": 1 vertex(es) passed over"), "{all}"); + // A latitude past a pole is taken as that pole, and says so; a pole's own latitude says nothing. + let past = run(vec![Some(Geometry::Point(Coord::new(10.0, 95.0)))], "All") + .unwrap_or_else(|e| panic!("{e}")) + .messages + .join(" | "); + assert!(past.contains("1 vertex(es) with a latitude past a pole found (taken as that pole)"), "{past}"); + let at_pole = run(vec![Some(Geometry::Point(Coord::new(10.0, 90.0)))], "All").unwrap_or_else(|e| panic!("{e}")); + assert!(!at_pole.messages.join(" | ").contains("past a pole"), "{:?}", at_pole.messages); } /// Minimum Bounding Geometry's `count`, all features together, is the features it hulled: with a diff --git a/crates/data/src/geometry.rs b/crates/data/src/geometry.rs index 2e980d58f4..b5cffb0ee7 100644 --- a/crates/data/src/geometry.rs +++ b/crates/data/src/geometry.rs @@ -332,11 +332,17 @@ impl VertexId { /// How many vertices a path has: a closed ring's repeated last point doesn't count. fn open_len(path: &[Coord], ring: bool) -> usize { match (path.first(), path.last()) { - (Some(a), Some(b)) if ring && path.len() > 1 && a == b => path.len() - 1, + (Some(a), Some(b)) if ring && path.len() > 1 && same_point(a, b) => path.len() - 1, _ => path.len(), } } +/// Whether two points are the same place. A NaN coordinate equals nothing as a number, so NaN is +/// matched by its bits: a ring that repeats a NaN point closes on it. +fn same_point(a: &Coord, b: &Coord) -> bool { + a == b || (a.x.to_bits() == b.x.to_bits() && a.y.to_bits() == b.y.to_bits()) +} + impl Geometry { /// Calls `f` with each path that vertices lie on: its part and ring, and whether it is a polygon /// ring. Nothing is allocated, so a geometry of many parts costs no memory to walk. diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 910ad09b36..ee063f8625 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -688,10 +688,11 @@ get no notice. Without `statistics`, `field` gives that field's `COUNT`, `SUM`, **Minimum Bounding Geometry** (`convexHull`) encloses each feature in its convex hull (Group Option `None`), or all of them in one (`All`, with `count`, the features it enclosed). It passes -over a vertex with a coordinate missing or infinite, or a longitude far out of range, and skips a -feature without geometry or with no point left to enclose, writing no hull for it; it says how -many of each, and how many hulls have no area (a single point, or points in one place or on one -line on the map). +over a vertex with a coordinate missing or infinite, or so far out the hull can't take it (past +about 9 million degrees), and skips a feature without geometry or with no point left to enclose, +writing no hull for it; if every feature is skipped, it is an error. A latitude past a pole is +taken as that pole. It says how many of each, and how many hulls have no area (a single point, or +points in one place or on one line on the map). **Dissolve** (`dissolve`) merges the polygons sharing a value of `field` into one feature for each value, or all of them into one without a field. Each feature has the value and `count`, the From 7820369921ac0812fd64a1365aff7809f80b2e3e Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:23:54 -0600 Subject: [PATCH 07/15] fix(analysis): Minimum Bounding Geometry writes a hull with no area as the point or line it is A hull with no area was written as a polygon whose ring has fewer than the four positions one needs: GeoJSON wrote it invalid and a Shapefile refused the layer. Points in one place now give that point, and points on one straight line the segment between the two farthest apart (found in two passes, the farthest from any point and then the farthest from that). A Shapefile export of a layer with both leaves the points and lines out and says how many. The error with nothing to hull also gives the vertices passed over and the latitudes past a pole. The ring closed on a NaN point (59ead33b) gets a test in the data crate: four vertices and edges, and deleting the first keeps it closed. Merged main, which counts a multipoint's non-finite members in Kernel Density (#1607); kept that, with a multipoint wholly beyond the system counted as beyond only, and the error naming every cause. Tests: single points and stacked points hull to a point, polar points on one parallel to the line between the farthest two, and a hull with area keeps a closed ring; the error's counts; the NaN-closed ring. --- crates/analysis/src/tools.rs | 83 ++++++++++++++++++++++++++++-------- crates/data/src/geometry.rs | 18 ++++++++ docs/user-guide/analysis.md | 5 ++- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 76496c6f41..5dc45fd80a 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2242,8 +2242,29 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { picked.iter().filter_map(|&r| ds.features.get(r).and_then(|f| Some((f, f.geometry.as_ref()?)))).collect(); // Features skipped: no geometry, or no point the hull can use. let mut skipped = picked.len().saturating_sub(features.len()); - // A hull with no area: a single point's, or points in one place or along one line. + // A hull with no area isn't a polygon (its ring would have fewer than the four positions one + // needs): points in one place have a point for their hull, and points on one line the segment + // between the two farthest apart, so it's written as that point or line. Those two are found by + // taking the point farthest from any one of them, then the point farthest from that. let is_flat = |hull: &geo::Polygon| geo::Area::unsigned_area(hull) == 0.0; + let shape = |hull: geo::Polygon, flat: &mut usize| { + if !is_flat(&hull) { + return from_multipolygon(&MultiPolygon::new(vec![hull])); + } + *flat += 1; + let ring = &hull.exterior().0; + let farthest = |from: geo::Coord| { + ring.iter().copied().fold((from, 0.0), |(best, d), c| { + let dc = (c.x - from.x).hypot(c.y - from.y); + if dc > d { (c, dc) } else { (best, d) } + }) + }; + let first = *ring.first()?; + let (a, _) = farthest(first); + let (b, span) = farthest(a); + let back = |c: geo::Coord| rustgis_geom::plane_to_lonlat(Coord::new(c.x, c.y)); + Some(if span > 0.0 { Geometry::Line(vec![vec![back(a), back(b)]]) } else { Geometry::Point(back(a)) }) + }; let (mut flat, mut gaps, mut poles) = (0usize, 0usize, 0usize); if each { // Each feature's hull made on several cores for many (#1252), placed in their order. @@ -2266,8 +2287,7 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { skipped += 1; continue; }; - flat += usize::from(is_flat(&hull)); - out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), f.attrs.clone()); + out.push(shape(hull, &mut flat), f.attrs.clone()); } } else { // All together; `count` is the features hulled, those with a point to give. With none, @@ -2286,30 +2306,28 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } if hulled > 0 { let hull = geo::MultiPoint::new(pts).convex_hull(); - flat += usize::from(is_flat(&hull)); - out.push(from_multipolygon(&MultiPolygon::new(vec![hull])), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); + out.push(shape(hull, &mut flat), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); } } - // Nothing hulled, every feature skipped: an unusable result, so an error saying why. + let gaps_note = (gaps > 0).then(|| { + format!("{gaps} vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)") + }); + let poles_note = (poles > 0).then(|| format!("{poles} vertex(es) with a latitude past a pole found (taken as that pole)")); + // Nothing hulled, every feature skipped: an unusable result, so an error saying why, with the + // vertices it passed over. if out.features.is_empty() && skipped > 0 { - return Err(format!("Minimum Bounding Geometry: none of the {} feature(s) has a point it can hull", picked.len())); + let why = gaps_note.iter().chain(poles_note.iter()).map(|n| format!("; {n}")).collect::(); + return Err(format!("Minimum Bounding Geometry: none of the {} feature(s) has a point it can hull{why}", picked.len())); } let n = out.features.len(); let mut r = result(out, format!("Minimum Bounding Geometry: {n} hulls"))?; if skipped > 0 { r.messages.push(format!("Minimum Bounding Geometry: {skipped} feature(s) skipped: no geometry, or no point it can use")); } - if gaps > 0 { - r.messages.push(format!( - "Minimum Bounding Geometry: {gaps} vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)" - )); - } - if poles > 0 { - r.messages.push(format!("Minimum Bounding Geometry: {poles} vertex(es) with a latitude past a pole found (taken as that pole)")); - } + r.messages.extend(gaps_note.into_iter().chain(poles_note).map(|n| format!("Minimum Bounding Geometry: {n}"))); if flat > 0 { r.messages.push(format!( - "Minimum Bounding Geometry: {flat} of {n} hulls have no area: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" + "Minimum Bounding Geometry: {flat} of {n} hulls have no area, so they are written as a point or a line: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" )); } Ok(r) @@ -8542,11 +8560,40 @@ mod tests { assert_eq!(note(&layer(&spread, false), "All"), None); assert_eq!(note(&layer(&spread, true), "None"), None); let note_single = note(&layer(&spread, false), "None").unwrap_or_default(); - assert!(note_single.contains(": 5 of 5 hulls have no area: a single point"), "{note_single}"); + assert!( + note_single.contains(": 5 of 5 hulls have no area, so they are written as a point or a line: a single point"), + "{note_single}" + ); // Features stacked in one place, together, have a hull with no area, and say so. let stacked = [(10.0, 20.0), (10.0, 20.0), (10.0, 20.0)]; let note_stacked = note(&layer(&stacked, false), "All").unwrap_or_default(); assert!(note_stacked.contains(": 1 of 1 hulls have no area"), "{note_stacked}"); + // A hull with no area is written as what it is (its ring would have fewer than the four + // positions a polygon needs): a point for points in one place, the segment between the + // farthest two for points on one line. A hull with area keeps a closed ring. + let geoms = |m: &Map, group: &str| { + let res = crate::run("convexHull", m, &serde_json::json!({"input": "pts", "group": group})).unwrap_or_else(|e| panic!("{e}")); + let Output::Layer(out) = res.output else { panic!("not a layer") }; + out.features.into_iter().map(|f| f.geometry).collect::>() + }; + let singles = geoms(&layer(&spread, false), "None"); + assert!(singles.len() == 5 && singles.iter().all(|g| matches!(g, Some(Geometry::Point(_)))), "{singles:?}"); + let near = |a: Coord, b: (f64, f64)| (a.x - b.0).abs() < 1e-9 && (a.y - b.1).abs() < 1e-9; + assert!(matches!(geoms(&layer(&stacked, false), "All").first(), Some(Some(Geometry::Point(c))) if near(*c, (10.0, 20.0)))); + // The polar points on 89°N, as the analysis plane draws them: from 90°W to 180°. + let each = geoms(&layer(&polar, true), "None"); + let ends = match each.first() { + Some(Some(Geometry::Line(parts))) => parts.first().cloned().unwrap_or_default(), + _ => vec![], + }; + let mut lons: Vec = ends.iter().map(|c| c.x).collect(); + lons.sort_by(f64::total_cmp); + assert!(ends.len() == 2 && near(Coord::new(lons[0], ends[0].y), (-90.0, 89.0)) && (lons[1] - 180.0).abs() < 1e-9, "{each:?}"); + let ring_len = |g: &Option| match g { + Some(Geometry::Polygon(p)) => p.first().and_then(|rings| rings.first()).map_or(0, Vec::len), + _ => 0, + }; + assert!(each.get(1).map_or(0, ring_len) >= 4, "{each:?}"); } /// Minimum Bounding Geometry passes over a vertex with a coordinate missing or infinite, which @@ -8588,7 +8635,7 @@ mod tests { } for group in ["All", "None"] { let err = run(vec![Some(Geometry::Point(gap)), None], group).err().unwrap_or_default(); - assert!(err.contains("none of the 2 feature(s) has a point it can hull"), "{group}: {err}"); + assert!(err.contains("none of the 2 feature(s) has a point it can hull; 1 vertex(es) passed over"), "{group}: {err}"); } // A ring with one vertex too far out, its closing point the same: one vertex passed over. let far_ring = Geometry::Polygon(vec![vec![vec![ diff --git a/crates/data/src/geometry.rs b/crates/data/src/geometry.rs index df810c0c10..0b8ffe206e 100644 --- a/crates/data/src/geometry.rs +++ b/crates/data/src/geometry.rs @@ -768,6 +768,24 @@ mod tests { assert!(polygon_contains(&[square(0.0, 0.0, 10.0), square(4.0, 4.0, 2.0)], lp)); } + /// A polygon ring that starts and ends on the same NaN point is closed on it (NaN equals + /// nothing as a number, so it's matched by its bits): its last point isn't a vertex of its own, + /// so every ring reader sees four vertices and four edges, and deleting the first vertex keeps + /// the ring closed on the next one. A line with the same ends isn't a ring: all three count. + #[test] + fn a_ring_closed_on_a_nan_point_counts_it_once() { + let gap = Coord::new(f64::NAN, 0.0); + let ring = vec![gap, Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(0.0, 1.0), gap]; + let g = Geometry::Polygon(vec![vec![ring]]); + assert_eq!((g.vertices().len(), g.edges().len()), (4, 4)); + let Ok(Geometry::Polygon(after)) = g.delete_vertex(VertexId { part: 0, ring: 0, index: 0 }) else { panic!("not deleted") }; + let ring = after.first().and_then(|rings| rings.first()).cloned().unwrap_or_default(); + assert_eq!(ring.len(), 4); + assert_eq!((ring.first(), ring.last()), (Some(&Coord::new(1.0, 0.0)), Some(&Coord::new(1.0, 0.0)))); + let line = Geometry::Line(vec![vec![gap, Coord::new(1.0, 0.0), gap]]); + assert_eq!(line.vertices().len(), 3); + } + #[test] fn line_hit_and_box() { let g = Geometry::Line(vec![vec![Coord::new(0.0, 0.0), Coord::new(10.0, 0.0)]]); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 9c179983ae..defbaa0d9c 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -760,8 +760,9 @@ Option `None`), or all of them in one (`All`, with `count`, the features it encl over a vertex with a coordinate missing or infinite, or so far out the hull can't take it (past about 9 million degrees), and skips a feature without geometry or with no point left to enclose, writing no hull for it; if every feature is skipped, it is an error. A latitude past a pole is -taken as that pole. It says how many of each, and how many hulls have no area (a single point, or -points in one place or on one line on the map). +taken as that pole. A hull with no area is written as what it is: a point for points in one place, +or the line between the two farthest apart for points on one straight line on the map. It says how +many of each. **Dissolve** (`dissolve`) merges the polygons sharing a value of `field` into one feature for each value, or all of them into one without a field. Each feature has the value and `count`, the From 9618e6fb427354d301e8f30e4c9e132d397dbe87 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:53:27 -0600 Subject: [PATCH 08/15] fix(analysis): Minimum Bounding Geometry's layer type follows its hulls; a ring's NaN closing point matches with a signed zero With hulls with no area written as points or lines, a layer of only points was still declared a polygon layer, so point tools such as Kernel Density refused it. The layer is now a point layer when every hull is a point and a line layer when every hull is a line; a mix stays a polygon layer. A hull that somehow has no ring is counted as skipped rather than written without a geometry. The comment on the far-out limit no longer says it guards against overflow, which only starts near 1e154 m; it's a conservative cut past any real data. A ring's closing point is matched coordinate by coordinate, equal as numbers or a NaN by its bits, so a NaN endpoint written once with 0 and once with -0 still closes the ring. Tests: the layer's type for points, lines and a mix; a feature whose only point is missing a coordinate isn't counted; the signed-zero ring. --- crates/analysis/src/tools.rs | 51 +++++++++++++++++++++++++++--------- crates/data/src/geometry.rs | 7 ++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 5dc45fd80a..49a7528904 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2217,9 +2217,9 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { }; // A feature's points in the analysis plane, and how many of its vertices (a closed ring's // repeated last point left out) are passed over: a coordinate missing or infinite, which the - // hull can't order, or a longitude so far out (millions of degrees) that the hull's arithmetic - // would overflow. A latitude past ±90° is taken as the pole, as the analysis plane takes it, and - // counted too. + // hull can't order, or a longitude past [`HULL_LIMIT_M`] (about 9 million degrees), which no + // real data reaches, well before the hull's arithmetic could overflow (near 1e154 m). A + // latitude past ±90° is taken as the pole, as the analysis plane takes it, and counted too. let points = |g: &Geometry| { let (mut these, mut gaps, mut poles) = (Vec::new(), 0usize, 0usize); g.for_each_vertex(|_, c| { @@ -2251,8 +2251,10 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { if !is_flat(&hull) { return from_multipolygon(&MultiPolygon::new(vec![hull])); } - *flat += 1; let ring = &hull.exterior().0; + // (A hull of a point or more always has one; if not, the caller counts the feature skipped.) + ring.first()?; + *flat += 1; let farthest = |from: geo::Coord| { ring.iter().copied().fold((from, 0.0), |(best, d), c| { let dc = (c.x - from.x).hypot(c.y - from.y); @@ -2283,11 +2285,10 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { for ((f, _), (passed, past, hull)) in features.iter().zip(hulls) { gaps += passed; poles += past; - let Some(hull) = hull else { - skipped += 1; - continue; - }; - out.push(shape(hull, &mut flat), f.attrs.clone()); + match hull.and_then(|hull| shape(hull, &mut flat)) { + Some(g) => out.push(Some(g), f.attrs.clone()), + None => skipped += 1, + } } } else { // All together; `count` is the features hulled, those with a point to give. With none, @@ -2305,10 +2306,21 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { pts.extend(these); } if hulled > 0 { - let hull = geo::MultiPoint::new(pts).convex_hull(); - out.push(shape(hull, &mut flat), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]); + match shape(geo::MultiPoint::new(pts).convex_hull(), &mut flat) { + Some(g) => out.push(Some(g), vec![Value::Int(i64::try_from(hulled).unwrap_or(i64::MAX))]), + None => skipped += hulled, + } } } + // The layer's type follows its hulls: points or lines when every hull is one (so a tool + // taking points takes them), polygons otherwise, holding any points and lines among them. + let kinds: Vec = out.features.iter().filter_map(|f| f.geometry.as_ref().map(Geometry::kind)).collect(); + if let Some(&first) = kinds.first() + && first != GeometryType::Polygon + && kinds.iter().all(|k| *k == first) + { + out.geometry_type = first; + } let gaps_note = (gaps > 0).then(|| { format!("{gaps} vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)") }); @@ -8578,6 +8590,17 @@ mod tests { }; let singles = geoms(&layer(&spread, false), "None"); assert!(singles.len() == 5 && singles.iter().all(|g| matches!(g, Some(Geometry::Point(_)))), "{singles:?}"); + // The layer's type follows its hulls: all points, a point layer; all lines, a line layer; + // a mix, a polygon layer. + let kind = |m: &Map, group: &str| { + let res = crate::run("convexHull", m, &serde_json::json!({"input": "pts", "group": group})).unwrap_or_else(|e| panic!("{e}")); + let Output::Layer(out) = res.output else { panic!("not a layer") }; + out.geometry_type + }; + assert_eq!(kind(&layer(&spread, false), "None"), GeometryType::Point); + assert_eq!(kind(&layer(&polar, false), "All"), GeometryType::Polyline); + assert_eq!(kind(&layer(&polar, true), "None"), GeometryType::Polygon); + assert_eq!(kind(&layer(&spread, false), "All"), GeometryType::Polygon); let near = |a: Coord, b: (f64, f64)| (a.x - b.0).abs() < 1e-9 && (a.y - b.1).abs() < 1e-9; assert!(matches!(geoms(&layer(&stacked, false), "All").first(), Some(Some(Geometry::Point(c))) if near(*c, (10.0, 20.0)))); // The polar points on 89°N, as the analysis plane draws them: from 90°W to 180°. @@ -8667,13 +8690,17 @@ mod tests { pts.push(Some(Geometry::Point(Coord::new(c.0, c.1))), vec![]); } pts.push(None, vec![]); + // A feature whose only point is missing a coordinate gives the hull nothing either. + pts.push(Some(Geometry::Point(Coord::new(f64::NAN, 0.0))), vec![]); let count = |sel: Vec| { let m = Selected(Map(HashMap::from([("pts".to_string(), pts.clone())])), HashMap::from([("pts".to_string(), sel)])); let out = layer(crate::run("convexHull", &m, &serde_json::json!({"input": "pts"})).unwrap_or_else(|e| panic!("{e}"))); out.value(0, 0).as_f64() }; - // No selection: every feature with a geometry (the one without isn't counted). + // No selection: every feature with a usable point (not the one without geometry, nor the + // one whose point is missing a coordinate). assert_eq!(count(vec![]), Some(4.0)); + assert_eq!(count(vec![0, 5]), Some(1.0)); assert_eq!(count(vec![0, 1, 2]), Some(3.0)); // A row a selection lists twice is one feature. assert_eq!(count(vec![0, 0, 1]), Some(2.0)); diff --git a/crates/data/src/geometry.rs b/crates/data/src/geometry.rs index 0b8ffe206e..7a5a0429e5 100644 --- a/crates/data/src/geometry.rs +++ b/crates/data/src/geometry.rs @@ -342,7 +342,9 @@ fn open_len(path: &[Coord], ring: bool) -> usize { /// Whether two points are the same place. A NaN coordinate equals nothing as a number, so NaN is /// matched by its bits: a ring that repeats a NaN point closes on it. fn same_point(a: &Coord, b: &Coord) -> bool { - a == b || (a.x.to_bits() == b.x.to_bits() && a.y.to_bits() == b.y.to_bits()) + // Each coordinate on its own: equal as numbers (so 0 and −0 match), or a NaN by its bits. + let same = |p: f64, q: f64| p == q || p.to_bits() == q.to_bits(); + same(a.x, b.x) && same(a.y, b.y) } impl Geometry { @@ -784,6 +786,9 @@ mod tests { assert_eq!((ring.first(), ring.last()), (Some(&Coord::new(1.0, 0.0)), Some(&Coord::new(1.0, 0.0)))); let line = Geometry::Line(vec![vec![gap, Coord::new(1.0, 0.0), gap]]); assert_eq!(line.vertices().len(), 3); + // The same NaN point written once with 0 and once with −0 still closes the ring. + let signed = vec![Coord::new(f64::NAN, 0.0), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(f64::NAN, -0.0)]; + assert_eq!(Geometry::Polygon(vec![vec![signed]]).vertices().len(), 3); } #[test] From ea55e4b510695bf291becc1deb28ef77189d509e Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:57:35 -0600 Subject: [PATCH 09/15] fix(data): a GeoPackage export leaves out geometry of another kind than the layer's, and says how many A layer can hold geometry of more than one kind: a GeoJSON of points and polygons, or Minimum Bounding Geometry's hulls, where a hull with no area is a point or line among polygons. A GeoPackage export declared the column as the layer's kind (MULTIPOLYGON) and wrote the points into it, breaking the column's contract, silently. A feature of another kind is now left out and counted in the export's notice, as a Shapefile leaves it out; a feature without geometry still goes in, as a table holds it. The user guide says what a mixed hull layer holds and how it exports. Tests: a GeoPackage of a polygon, a point and a feature without geometry holds the polygon and the feature without geometry (checked with SQL), the point counted as left out; a layer of one kind loses nothing. A ring closed on a NaN point with the signed zero in x. --- crates/data/src/geometry.rs | 3 +++ crates/data/src/io.rs | 53 ++++++++++++++++++++++++++++++++++++- docs/user-guide/analysis.md | 6 +++-- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/crates/data/src/geometry.rs b/crates/data/src/geometry.rs index 7a5a0429e5..77b399638f 100644 --- a/crates/data/src/geometry.rs +++ b/crates/data/src/geometry.rs @@ -789,6 +789,9 @@ mod tests { // The same NaN point written once with 0 and once with −0 still closes the ring. let signed = vec![Coord::new(f64::NAN, 0.0), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(f64::NAN, -0.0)]; assert_eq!(Geometry::Polygon(vec![vec![signed]]).vertices().len(), 3); + // And the other way round: a signed zero in x, the NaN in y. + let mirrored = vec![Coord::new(0.0, f64::NAN), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(-0.0, f64::NAN)]; + assert_eq!(Geometry::Polygon(vec![vec![mirrored]]).vertices().len(), 3); } #[test] diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 685c329a83..109288e3b3 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -545,9 +545,11 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R "gpkg" => { #[cfg(not(target_arch = "wasm32"))] { + let (kept, left_out) = of_its_kind(ds); + let ds = kept.as_ref().unwrap_or(ds); crate::progress::begin(ds.features.len()); let table = crate::source::gpkg::write_table_in(ds, path, crs, gpkg)?; - Ok((0, Some(table))) + Ok((left_out, Some(table))) } #[cfg(target_arch = "wasm32")] { @@ -2516,6 +2518,25 @@ fn dbf_boolean(value: &Value) -> Option { } } +/// A dataset as a GeoPackage table holds it: geometry of one kind, the layer's (its column's +/// declared type). A feature of another kind, as a mixed layer can hold (a GeoJSON of points and +/// polygons, say), is left out and counted, as a Shapefile leaves it out; one without geometry +/// stays, as a table holds it. `None` when every feature already fits, so nothing is copied. +#[cfg(not(target_arch = "wasm32"))] +fn of_its_kind(ds: &Dataset) -> (Option, usize) { + if ds.geometry_type == GeometryType::None { + return (None, 0); + } + let off = |f: &crate::Feature| f.geometry.as_ref().is_some_and(|g| g.kind() != ds.geometry_type); + let n = ds.features.iter().filter(|f| off(f)).count(); + if n == 0 { + return (None, 0); + } + let mut kept = ds.clone(); + kept.features.retain(|f| !off(f)); + (Some(kept), n) +} + /// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one /// of the layer's kind with something drawable (a finite point, a line part of two finite points, /// a polygon whose outer ring has three). @@ -2682,6 +2703,36 @@ mod tests { use super::*; use std::io::Write; + /// A GeoPackage column holds the layer's one kind of geometry: a layer read from a GeoJSON of + /// a polygon and a point (declared a polygon layer) writes the polygon and leaves the point out, + /// counted, as a Shapefile does; a feature without geometry stays, as a table holds it. A layer + /// of one kind loses nothing. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_geopackage_leaves_out_geometry_of_another_kind() -> std::result::Result<(), Box> { + let geojson = r#"{"type":"FeatureCollection","features":[ + {"type":"Feature","properties":{"name":"area"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}}, + {"type":"Feature","properties":{"name":"spot"},"geometry":{"type":"Point","coordinates":[5,5]}}, + {"type":"Feature","properties":{"name":"none"},"geometry":null}]}"#; + let (mixed, _) = read_geojson(geojson, "mixed")?; + assert_eq!(mixed.geometry_type, GeometryType::Polygon); + let dir = tempfile::tempdir()?; + let path = dir.path().join("mixed.gpkg"); + assert_eq!(write_path(&mixed, &path)?, 1); + // The rows written, read with SQL: the polygon, and the feature without geometry. + let conn = rusqlite::Connection::open(&path)?; + let mut rows = conn.prepare(r#"SELECT "name", "geom" IS NULL FROM "mixed" ORDER BY "fid""#)?; + let written: Vec<(String, bool)> = rows.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; + assert_eq!(written, [("area".to_string(), false), ("none".to_string(), true)]); + let (back, _) = read_path(&path)?; + assert!(back.features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); + // One kind: nothing left out. + let mut areas = mixed.clone(); + areas.features.retain(|f| !matches!(f.geometry, Some(Geometry::Point(_)))); + assert_eq!(write_path(&areas, &dir.path().join("areas.gpkg"))?, 0); + Ok(()) + } + /// Exercises caller cleanup through the public Shapefile writer, one fresh process per collision. #[cfg(not(target_arch = "wasm32"))] #[test] diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index defbaa0d9c..94afba6fe0 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -761,8 +761,10 @@ over a vertex with a coordinate missing or infinite, or so far out the hull can' about 9 million degrees), and skips a feature without geometry or with no point left to enclose, writing no hull for it; if every feature is skipped, it is an error. A latitude past a pole is taken as that pole. A hull with no area is written as what it is: a point for points in one place, -or the line between the two farthest apart for points on one straight line on the map. It says how -many of each. +or the line between the two farthest apart for points on one straight line on the map. The layer is +a point or line layer when every hull is one; otherwise it's a polygon layer holding any points and +lines among its polygons, which a Shapefile or GeoPackage export leaves out, saying how many. It +says how many of each. **Dissolve** (`dissolve`) merges the polygons sharing a value of `field` into one feature for each value, or all of them into one without a field. Each feature has the value and `count`, the From 8a3a418235c1ce3894c423c3248492243d9d847d Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:35:57 -0600 Subject: [PATCH 10/15] fix(data): a streamed GeoPackage export leaves out geometry of another kind too, and none is copied The kind filter ran only on a layer held whole: a streamed export (a large GeoParquet, say) wrote a point into a polygon column, silently. Each chunk now goes through the same filter, and its count is reported. The filter hands the writer the runs of features the table can hold, borrowed from the layer, so a large layer with one point among its polygons isn't copied; only an export to be projected copies it, as projecting already did, and it filters first, so a feature the system can't place no longer fails the export when it would be left out anyway. The export notice now says why features were left out: "no drawable geometry, or a geometry of another kind than the layer's". Tests: a streamed export of squares with points among them, a few features a chunk, matches the whole export and counts the points; a projected export leaves out a point UTM zone 18N can't place (it failed before) and writes the polygon. --- crates/data/src/export.rs | 15 +++++- crates/data/src/export/tests.rs | 29 ++++++++++++ crates/data/src/io.rs | 81 +++++++++++++++++++++++++++------ crates/engine/src/cmd/share.rs | 6 ++- crates/engine/src/tests.rs | 7 ++- crates/ui-egui/src/lib.rs | 2 +- 6 files changed, 120 insertions(+), 20 deletions(-) diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index a4f8428e4e..5063b5e05d 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -116,8 +116,19 @@ fn write_chunks_to( let left_out = match ext.as_str() { #[cfg(not(target_arch = "wasm32"))] "gpkg" => { - table = Some(crate::source::gpkg::write_features_in(&layer.name, geometry_type, &layer.fields, path, crs, gpkg, &mut feed)?); - 0 + // A GeoPackage table holds one kind of geometry: each chunk's features of another + // kind are left out, and counted, as the rest is handed over. + let mut left_out = 0usize; + let mut kinded = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { + // An export that starts again feeds the features again, and counts them again. + left_out = 0; + feed(&mut |chunk: &[Feature]| { + left_out = left_out.saturating_add(crate::io::feed_of_kind(chunk, geometry_type, add)?); + Ok(()) + }) + }; + table = Some(crate::source::gpkg::write_features_in(&layer.name, geometry_type, &layer.fields, path, crs, gpkg, &mut kinded)?); + left_out } #[cfg(target_arch = "wasm32")] "gpkg" => { diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index 3fc019f4b2..99ea793c2d 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -370,3 +370,32 @@ fn geoparquet_counts_values_written_as_null() { assert_eq!(note, format!("{want}, '{}…'", "x".repeat(40))); let _ = std::fs::remove_dir_all(d); } + +/// A streamed GeoPackage export leaves out geometry of another kind than the layer's, chunk by +/// chunk, and counts it, as the export of the whole layer does: squares with points among them, +/// a few features a chunk, give the same table and the same count. +#[test] +fn a_streamed_geopackage_leaves_out_geometry_of_another_kind() { + let d = folder("mixed-kinds"); + let mut mixed = squares(); + mixed.name = "mixed".into(); + for i in [0usize, 5, 6, 299] { + if let Some(f) = mixed.features.get_mut(i) { + f.geometry = Some(Geometry::Point(Coord::new(i as f64, 1.0))); + } + } + let layer = Dataset::new(mixed.name.clone(), mixed.geometry_type, mixed.fields.clone()); + let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); + let streamed = write_chunks_in(&Runs::new(&mixed, 4), &layer, &a, Crs::LonLat, 1).unwrap(); + assert_eq!(streamed.left_out, 4); + assert_eq!(crate::io::write_path(&mixed, &b).unwrap(), 4); + let blobs = |p: &Path| { + let c = rusqlite::Connection::open(p).unwrap(); + c.query_row("SELECT count(*) FROM mixed WHERE geom IS NOT NULL AND substr(hex(geom), 1, 4) = '4750'", [], |r| r.get::<_, i64>(0)) + .unwrap() + }; + // Every geometry row is a GeoPackage blob of the table's kind: 296 squares, none of the points. + assert_eq!((blobs(&a), blobs(&b)), (296, 296)); + assert_eq!(read_back(&a).features, read_back(&b).features); + assert!(read_back(&a).features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); +} diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 109288e3b3..04e745f318 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -533,8 +533,15 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result { /// also returns the table written. pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> Result<(usize, Option)> { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + let reprojects = crs != Crs::LonLat && PROJECTED_FORMATS.contains(&ext.as_str()); + // A GeoPackage export to be projected leaves out what its table can't hold first, so a + // feature left out can't fail the projection (which copies the layer anyway). + #[cfg(not(target_arch = "wasm32"))] + let (kept, mut left_out) = if ext == "gpkg" && reprojects { of_its_kind(ds) } else { (None, 0) }; + #[cfg(not(target_arch = "wasm32"))] + let ds = kept.as_ref().unwrap_or(ds); let projected; - let ds = if crs == Crs::LonLat || !PROJECTED_FORMATS.contains(&ext.as_str()) { + let ds = if !reprojects { ds } else { check_projected(&ext, crs)?; @@ -545,10 +552,19 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R "gpkg" => { #[cfg(not(target_arch = "wasm32"))] { - let (kept, left_out) = of_its_kind(ds); - let ds = kept.as_ref().unwrap_or(ds); crate::progress::begin(ds.features.len()); - let table = crate::source::gpkg::write_table_in(ds, path, crs, gpkg)?; + let table = if ds.features.iter().any(|f| off_kind(f, ds.geometry_type)) { + // What the table can't hold is left out as the rest is handed over, in runs + // borrowed from the layer, so a large one isn't copied. + let mut feed = |add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>| -> Result<()> { + // An export that starts again hands the features over again. + left_out = feed_of_kind(&ds.features, ds.geometry_type, add)?; + Ok(()) + }; + crate::source::gpkg::write_features_in(&ds.name, ds.geometry_type, &ds.fields, path, crs, gpkg, &mut feed)? + } else { + crate::source::gpkg::write_table_in(ds, path, crs, gpkg)? + }; Ok((left_out, Some(table))) } #[cfg(target_arch = "wasm32")] @@ -2518,22 +2534,50 @@ fn dbf_boolean(value: &Value) -> Option { } } -/// A dataset as a GeoPackage table holds it: geometry of one kind, the layer's (its column's -/// declared type). A feature of another kind, as a mixed layer can hold (a GeoJSON of points and -/// polygons, say), is left out and counted, as a Shapefile leaves it out; one without geometry -/// stays, as a table holds it. `None` when every feature already fits, so nothing is copied. +/// Whether a GeoPackage table of geometry type `kind` can't hold `f`'s geometry: a column holds +/// one kind of geometry, the layer's. A feature of another kind, as a mixed layer can hold (a +/// GeoJSON of points and polygons, say), is left out and counted, as a Shapefile leaves it out; +/// one without geometry fits, as a table holds it. #[cfg(not(target_arch = "wasm32"))] -fn of_its_kind(ds: &Dataset) -> (Option, usize) { - if ds.geometry_type == GeometryType::None { - return (None, 0); +fn off_kind(f: &crate::Feature, kind: GeometryType) -> bool { + kind != GeometryType::None && f.geometry.as_ref().is_some_and(|g| g.kind() != kind) +} + +/// Hands `add` the runs of `features` a GeoPackage table of `kind` can hold, borrowed (nothing is +/// copied), and returns how many it left out ([`off_kind`]). +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn feed_of_kind( + features: &[crate::Feature], + kind: GeometryType, + add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>, +) -> Result { + let (mut start, mut left_out) = (0usize, 0usize); + for (i, f) in features.iter().enumerate() { + if off_kind(f, kind) { + if start < i { + add(features.get(start..i).unwrap_or_default())?; + } + left_out += 1; + start = i + 1; + } } - let off = |f: &crate::Feature| f.geometry.as_ref().is_some_and(|g| g.kind() != ds.geometry_type); - let n = ds.features.iter().filter(|f| off(f)).count(); + if start < features.len() { + add(features.get(start..).unwrap_or_default())?; + } + Ok(left_out) +} + +/// `ds` with the features a GeoPackage table of its type can't hold left out, and how many: for +/// an export about to be projected, which copies the layer anyway, so a feature left out can't +/// fail the projection. `None` when every feature fits. +#[cfg(not(target_arch = "wasm32"))] +fn of_its_kind(ds: &Dataset) -> (Option, usize) { + let n = ds.features.iter().filter(|f| off_kind(f, ds.geometry_type)).count(); if n == 0 { return (None, 0); } let mut kept = ds.clone(); - kept.features.retain(|f| !off(f)); + kept.features.retain(|f| !off_kind(f, ds.geometry_type)); (Some(kept), n) } @@ -2726,6 +2770,15 @@ mod tests { assert_eq!(written, [("area".to_string(), false), ("none".to_string(), true)]); let (back, _) = read_path(&path)?; assert!(back.features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); + // Projected, the point is left out before the projection: one UTM zone 18N can't hold (a + // quarter of the way round the equator from its central meridian, 75°W) doesn't fail the + // export of the polygon. + let mut far = mixed.clone(); + if let Some(f) = far.features.get_mut(1) { + f.geometry = Some(Geometry::Point(Coord::new(15.0, 0.0))); + } + let utm = Crs::from_epsg(32618).ok_or("no UTM zone 18N")?; + assert_eq!(write_path_in(&far, &dir.path().join("far.gpkg"), utm)?, 1); // One kind: nothing left out. let mut areas = mixed.clone(); areas.features.retain(|f| !matches!(f.geometry, Some(Geometry::Point(_)))); diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 63b8cb88d4..55324a16bc 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -54,7 +54,7 @@ pub fn commands() -> Vec { rustgis_data::io::write_path_to(&t.dataset, Path::new(path), Crs::LonLat, &gpkg).map_err(|e| e.to_string())?; s.check_continuation(&owner).map_err(|why| super::exports::completion_refused(Some(path), why))?; if left_out > 0 { - s.notify(format!("{path}: {left_out} features without drawable geometry were left out"), false); + s.notify(format!("{path}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's"), false); } for note in notes.into_iter().chain(dbf_notes) { s.notify(note, false); @@ -638,7 +638,9 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk // geometry it can draw). let mut said = Vec::new(); if left_out > 0 { - said.push(format!("{path}: {left_out} features without drawable geometry were left out")); + said.push(format!( + "{path}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's" + )); } if skipped > 0 { said.push(format!("{path}: {skipped} rows couldn't be read and were left out")); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d6b2476e85..85b79d4d83 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -30838,7 +30838,12 @@ fn shapefile_value_losses_reach_export_notices() { notices.iter().any(|n| n.contains("3 values") && n.contains("id") && n.contains("number") && n.contains("left empty")), "{command}: {notices:?}" ); - assert!(notices.iter().any(|n| n.contains("1 features without drawable geometry were left out")), "{command}: {notices:?}"); + assert!( + notices + .iter() + .any(|n| n.contains("1 features were left out: no drawable geometry, or a geometry of another kind than the layer's")), + "{command}: {notices:?}" + ); } } // A clean table gives neither loss notice; other formats keep the lossy attributes. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 57c8a75526..2667db62d5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -3935,7 +3935,7 @@ impl RustGisApp { dbf_notes.extend(rustgis_engine::data::source::parquet::values_note(ds, &name)); } if left_out > 0 { - self.session.notify(format!("{name}: {left_out} features without drawable geometry were left out"), false); + self.session.notify(format!("{name}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's"), false); } if ext == "zip" && rustgis_engine::data::raster::wkt::to_wkt(crs).is_none() { self.session.notify(format!("{name}: no .prj written, as {} has no WKT form", crs.label()), false); From 6b29dc79d8c6275d233b159d4b5524f731cb962e Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:54:28 -0600 Subject: [PATCH 11/15] fix(data): a GeoPackage export of a layer with no declared type keeps its geometry A layer that declares no geometry type but carries geometry was written, whole, as a table with no geometry column, dropping every geometry with no notice, while a streamed export took its first geometry's kind. The whole export now takes that kind too, and leaves out and counts the geometry of other kinds, borrowing the features as before. Tests: such a layer exports a MULTIPOLYGON table with its point left out and counted, and its streamed and whole exports give the same table. --- crates/data/src/export/tests.rs | 23 +++++++++++++++++++++++ crates/data/src/io.rs | 33 ++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index 99ea793c2d..ef7083c8a8 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -399,3 +399,26 @@ fn a_streamed_geopackage_leaves_out_geometry_of_another_kind() { assert_eq!(read_back(&a).features, read_back(&b).features); assert!(read_back(&a).features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); } + +/// A layer that declares no geometry type but carries geometry exports to the same GeoPackage +/// streamed as whole: a table of its first geometry's kind, with the others left out and counted. +#[test] +fn a_typeless_layer_exports_the_same_geopackage_streamed_or_whole() { + let d = folder("typeless"); + let mut ds = squares(); + ds.name = "typeless".into(); + ds.geometry_type = GeometryType::None; + if let Some(f) = ds.features.get_mut(3) { + f.geometry = Some(Geometry::Point(Coord::new(3.0, 1.0))); + } + let layer = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone()); + let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); + assert_eq!(write_chunks_in(&Runs::new(&ds, 4), &layer, &a, Crs::LonLat, 1).unwrap().left_out, 1); + assert_eq!(crate::io::write_path(&ds, &b).unwrap(), 1); + let kind = |p: &Path| { + let c = rusqlite::Connection::open(p).unwrap(); + c.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get::<_, String>(0)).unwrap() + }; + assert_eq!((kind(&a), kind(&b)), ("MULTIPOLYGON".to_string(), "MULTIPOLYGON".to_string())); + assert_eq!(read_back(&a).features, read_back(&b).features); +} diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 04e745f318..d9997ad6e1 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -553,15 +553,16 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R #[cfg(not(target_arch = "wasm32"))] { crate::progress::begin(ds.features.len()); - let table = if ds.features.iter().any(|f| off_kind(f, ds.geometry_type)) { + let kind = gpkg_kind(ds); + let table = if kind != ds.geometry_type || ds.features.iter().any(|f| off_kind(f, kind)) { // What the table can't hold is left out as the rest is handed over, in runs // borrowed from the layer, so a large one isn't copied. let mut feed = |add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>| -> Result<()> { // An export that starts again hands the features over again. - left_out = feed_of_kind(&ds.features, ds.geometry_type, add)?; + left_out = feed_of_kind(&ds.features, kind, add)?; Ok(()) }; - crate::source::gpkg::write_features_in(&ds.name, ds.geometry_type, &ds.fields, path, crs, gpkg, &mut feed)? + crate::source::gpkg::write_features_in(&ds.name, kind, &ds.fields, path, crs, gpkg, &mut feed)? } else { crate::source::gpkg::write_table_in(ds, path, crs, gpkg)? }; @@ -2572,15 +2573,27 @@ pub(crate) fn feed_of_kind( /// fail the projection. `None` when every feature fits. #[cfg(not(target_arch = "wasm32"))] fn of_its_kind(ds: &Dataset) -> (Option, usize) { - let n = ds.features.iter().filter(|f| off_kind(f, ds.geometry_type)).count(); + let kind = gpkg_kind(ds); + let n = ds.features.iter().filter(|f| off_kind(f, kind)).count(); if n == 0 { return (None, 0); } let mut kept = ds.clone(); - kept.features.retain(|f| !off_kind(f, ds.geometry_type)); + kept.features.retain(|f| !off_kind(f, kind)); (Some(kept), n) } +/// The kind of geometry a GeoPackage table of `ds` holds: its type, or, for a layer that +/// declares none but carries geometry, its first geometry's (as a streamed export takes it), so +/// that geometry isn't dropped for want of a column. +#[cfg(not(target_arch = "wasm32"))] +fn gpkg_kind(ds: &Dataset) -> GeometryType { + match ds.geometry_type { + GeometryType::None => ds.features.iter().find_map(|f| f.geometry.as_ref()).map_or(GeometryType::None, Geometry::kind), + t => t, + } +} + /// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one /// of the layer's kind with something drawable (a finite point, a line part of two finite points, /// a polygon whose outer ring has three). @@ -2779,6 +2792,16 @@ mod tests { } let utm = Crs::from_epsg(32618).ok_or("no UTM zone 18N")?; assert_eq!(write_path_in(&far, &dir.path().join("far.gpkg"), utm)?, 1); + // A layer that declares no geometry type but carries geometry takes its first geometry's + // (as a streamed export does): a polygon table, the point left out, nothing dropped silently. + let mut typeless = mixed.clone(); + typeless.geometry_type = GeometryType::None; + let at = dir.path().join("typeless.gpkg"); + assert_eq!(write_path(&typeless, &at)?, 1); + let conn = rusqlite::Connection::open(&at)?; + let kind: String = conn.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get(0))?; + let rows: i64 = conn.query_row(r#"SELECT count(*) FROM "mixed" WHERE "geom" IS NOT NULL"#, [], |r| r.get(0))?; + assert_eq!((kind.as_str(), rows), ("MULTIPOLYGON", 1)); // One kind: nothing left out. let mut areas = mixed.clone(); areas.features.retain(|f| !matches!(f.geometry, Some(Geometry::Point(_)))); From d1af9f3c65ff0b854fc3aa3f0fc05970c62967d5 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:35:32 -0600 Subject: [PATCH 12/15] fix(data): a projected GeoPackage export leaves out off-kind features before projecting, on both paths, with one copy A streamed GeoPackage export projected each chunk before leaving out the features of another kind, so one the system couldn't place failed the export where the whole export wrote the rest. Each chunk is now trimmed first; its features are its own, so nothing more is copied. A whole projected export copies the layer once (it copied it twice: once to leave features out, again to project), and a layer with no declared type keeps the count of what it left out before the projection. The scan for off-kind features stops on a cancel. Tests: a streamed export to UTM zone 31N with points on the other side of the world writes the squares and counts the points, as the whole export does; a projected export of a typeless layer keeps its count. --- crates/data/src/export.rs | 30 +++++++++++--------- crates/data/src/export/tests.rs | 22 +++++++++++++++ crates/data/src/io.rs | 50 ++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 5063b5e05d..48abf8dae1 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -95,12 +95,27 @@ fn write_chunks_to( let t = if crs == Crs::LonLat { None } else { Some(crs.transformer().map_err(DataError::Format)?) }; let mut total = 0usize; let mut skipped = 0usize; + // A GeoPackage table holds one kind of geometry: each chunk's features of another kind are + // left out, and counted, before it's projected, so one left out can't fail the projection. + #[cfg(not(target_arch = "wasm32"))] + let gpkg_kinds = ext == "gpkg"; + #[cfg(target_arch = "wasm32")] + let gpkg_kinds = false; + let mut off_kind = 0usize; let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { // A GeoPackage export that starts again feeds the features again, and counts them again. total = 0; + off_kind = 0; // The writers count the features; a source that doesn't know how many leaves the total at 0. crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX))); let batch = src.visit_chunks(None, bytes, &mut |mut chunk| { + #[cfg(not(target_arch = "wasm32"))] + if gpkg_kinds { + let before = chunk.len(); + chunk.retain(|f| !crate::io::off_kind(f, geometry_type)); + off_kind = off_kind.saturating_add(before - chunk.len()); + total = total.saturating_add(before - chunk.len()); + } if let Some(t) = &t { // Counted as the writer writes them, so only stopped on a cancel here. project_each(&mut chunk, t, crs, false)?; @@ -116,19 +131,8 @@ fn write_chunks_to( let left_out = match ext.as_str() { #[cfg(not(target_arch = "wasm32"))] "gpkg" => { - // A GeoPackage table holds one kind of geometry: each chunk's features of another - // kind are left out, and counted, as the rest is handed over. - let mut left_out = 0usize; - let mut kinded = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { - // An export that starts again feeds the features again, and counts them again. - left_out = 0; - feed(&mut |chunk: &[Feature]| { - left_out = left_out.saturating_add(crate::io::feed_of_kind(chunk, geometry_type, add)?); - Ok(()) - }) - }; - table = Some(crate::source::gpkg::write_features_in(&layer.name, geometry_type, &layer.fields, path, crs, gpkg, &mut kinded)?); - left_out + table = Some(crate::source::gpkg::write_features_in(&layer.name, geometry_type, &layer.fields, path, crs, gpkg, &mut feed)?); + off_kind } #[cfg(target_arch = "wasm32")] "gpkg" => { diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index ef7083c8a8..dffe703d3d 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -422,3 +422,25 @@ fn a_typeless_layer_exports_the_same_geopackage_streamed_or_whole() { assert_eq!((kind(&a), kind(&b)), ("MULTIPOLYGON".to_string(), "MULTIPOLYGON".to_string())); assert_eq!(read_back(&a).features, read_back(&b).features); } + +/// A projected streamed GeoPackage export leaves out geometry of another kind before projecting +/// each chunk, as the whole export does: points UTM zone 31N can't place (on the other side of +/// the world from its central meridian, 3°E) don't fail the export of the squares. +#[test] +fn a_projected_streamed_geopackage_leaves_out_before_projecting() { + let d = folder("mixed-projected"); + let mut mixed = squares(); + mixed.name = "mixed".into(); + for i in [2usize, 7] { + if let Some(f) = mixed.features.get_mut(i) { + f.geometry = Some(Geometry::Point(Coord::new(-177.0, 0.0))); + } + } + let utm = Crs::from_epsg(32631).unwrap(); + let layer = Dataset::new(mixed.name.clone(), mixed.geometry_type, mixed.fields.clone()); + let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); + let streamed = write_chunks_in(&Runs::new(&mixed, 4), &layer, &a, utm, 1).unwrap(); + assert_eq!((streamed.left_out, streamed.written), (2, 299)); + assert_eq!(crate::io::write_path_in(&mixed, &b, utm).unwrap(), 2); + assert_eq!(read_back(&a).features.len(), read_back(&b).features.len()); +} diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index d9997ad6e1..b7ca39762d 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -534,18 +534,27 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result { pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> Result<(usize, Option)> { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); let reprojects = crs != Crs::LonLat && PROJECTED_FORMATS.contains(&ext.as_str()); - // A GeoPackage export to be projected leaves out what its table can't hold first, so a - // feature left out can't fail the projection (which copies the layer anyway). + // Features a GeoPackage table can't hold, left out before projecting. #[cfg(not(target_arch = "wasm32"))] - let (kept, mut left_out) = if ext == "gpkg" && reprojects { of_its_kind(ds) } else { (None, 0) }; - #[cfg(not(target_arch = "wasm32"))] - let ds = kept.as_ref().unwrap_or(ds); + let mut left_out = 0usize; let projected; let ds = if !reprojects { ds } else { check_projected(&ext, crs)?; - projected = project(ds, crs)?; + // Projected into one copy of the layer. A GeoPackage export leaves out what its table + // can't hold first, so a feature left out can't fail the projection. + let mut out = ds.clone(); + #[cfg(not(target_arch = "wasm32"))] + if ext == "gpkg" { + let kind = gpkg_kind(ds); + let before = out.features.len(); + out.features.retain(|f| !off_kind(f, kind)); + left_out = before - out.features.len(); + } + let t = crs.transformer().map_err(DataError::Format)?; + project_features(&mut out.features, &t, crs)?; + projected = out; &projected }; match ext.as_str() { @@ -557,9 +566,11 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R let table = if kind != ds.geometry_type || ds.features.iter().any(|f| off_kind(f, kind)) { // What the table can't hold is left out as the rest is handed over, in runs // borrowed from the layer, so a large one isn't copied. + // Added to any left out before a projection. + let before = left_out; let mut feed = |add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>| -> Result<()> { // An export that starts again hands the features over again. - left_out = feed_of_kind(&ds.features, kind, add)?; + left_out = before + feed_of_kind(&ds.features, kind, add)?; Ok(()) }; crate::source::gpkg::write_features_in(&ds.name, kind, &ds.fields, path, crs, gpkg, &mut feed)? @@ -2540,7 +2551,7 @@ fn dbf_boolean(value: &Value) -> Option { /// GeoJSON of points and polygons, say), is left out and counted, as a Shapefile leaves it out; /// one without geometry fits, as a table holds it. #[cfg(not(target_arch = "wasm32"))] -fn off_kind(f: &crate::Feature, kind: GeometryType) -> bool { +pub(crate) fn off_kind(f: &crate::Feature, kind: GeometryType) -> bool { kind != GeometryType::None && f.geometry.as_ref().is_some_and(|g| g.kind() != kind) } @@ -2554,6 +2565,10 @@ pub(crate) fn feed_of_kind( ) -> Result { let (mut start, mut left_out) = (0usize, 0usize); for (i, f) in features.iter().enumerate() { + // A large layer's scan stops on a cancel. + if i % 65_536 == 0 { + crate::progress::check()?; + } if off_kind(f, kind) { if start < i { add(features.get(start..i).unwrap_or_default())?; @@ -2568,21 +2583,6 @@ pub(crate) fn feed_of_kind( Ok(left_out) } -/// `ds` with the features a GeoPackage table of its type can't hold left out, and how many: for -/// an export about to be projected, which copies the layer anyway, so a feature left out can't -/// fail the projection. `None` when every feature fits. -#[cfg(not(target_arch = "wasm32"))] -fn of_its_kind(ds: &Dataset) -> (Option, usize) { - let kind = gpkg_kind(ds); - let n = ds.features.iter().filter(|f| off_kind(f, kind)).count(); - if n == 0 { - return (None, 0); - } - let mut kept = ds.clone(); - kept.features.retain(|f| !off_kind(f, kind)); - (Some(kept), n) -} - /// The kind of geometry a GeoPackage table of `ds` holds: its type, or, for a layer that /// declares none but carries geometry, its first geometry's (as a streamed export takes it), so /// that geometry isn't dropped for want of a column. @@ -2802,6 +2802,10 @@ mod tests { let kind: String = conn.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get(0))?; let rows: i64 = conn.query_row(r#"SELECT count(*) FROM "mixed" WHERE "geom" IS NOT NULL"#, [], |r| r.get(0))?; assert_eq!((kind.as_str(), rows), ("MULTIPOLYGON", 1)); + // Projected too, the count of what was left out before the projection holds. + let mut typeless_far = far.clone(); + typeless_far.geometry_type = GeometryType::None; + assert_eq!(write_path_in(&typeless_far, &dir.path().join("typeless-far.gpkg"), utm)?, 1); // One kind: nothing left out. let mut areas = mixed.clone(); areas.features.retain(|f| !matches!(f.geometry, Some(Geometry::Point(_)))); From 14428b8e22377aa38ef5e5a0b17dd40af40b711b Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:49:07 -0600 Subject: [PATCH 13/15] fix(data): no unused variable in the browser build of the streamed export The GeoPackage kind check is desktop-only; its flag was also defined, unused, for wasm32, which CI builds with warnings as errors. --- crates/data/src/export.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 48abf8dae1..25ac7cd8cb 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -99,8 +99,6 @@ fn write_chunks_to( // left out, and counted, before it's projected, so one left out can't fail the projection. #[cfg(not(target_arch = "wasm32"))] let gpkg_kinds = ext == "gpkg"; - #[cfg(target_arch = "wasm32")] - let gpkg_kinds = false; let mut off_kind = 0usize; let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { // A GeoPackage export that starts again feeds the features again, and counts them again. From 3b485c7f834b0ee6236b0d73cda983391eae1b84 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:28 -0600 Subject: [PATCH 14/15] fix(data): a GeoPackage export of a mixed layer keeps every feature in a GEOMETRY column The export had started leaving out geometry of another kind than the layer's declared type. Readers declare a layer by its most common kind (or, streamed, its first geometry's), so an ordinary GeoJSON or KML of points and polygons lost its minority kind on GeoPackage export, where it used to be written. GeoPackage allows a GEOMETRY column, so a mixed layer now gets one: the whole export decides up front, and a streamed or added table starts again with it when it meets another kind (Redo::Mixed), as it does for multipoints. The left-out notice is worded per format (io::left_out_note): a Shapefile names both reasons it drops a feature; FlatGeobuf, KML and GPX only lack geometry. Minimum Bounding Geometry says when its output is a point or line layer, or where flat hulls go in a polygon layer. Dissolve counts the features it left out for having no polygon (it skipped them silently), and errors when none has one. --- crates/analysis/src/tools.rs | 81 ++++++++++++- crates/data/src/export.rs | 16 +-- crates/data/src/export/tests.rs | 107 +++++++++++----- crates/data/src/io.rs | 188 ++++++++++++----------------- crates/data/src/source/gpkg.rs | 90 ++++++++++---- crates/data/src/source/gpkg/add.rs | 11 +- crates/engine/src/cmd/share.rs | 12 +- crates/engine/src/tests.rs | 7 +- crates/ui-egui/src/lib.rs | 4 +- docs/user-guide/analysis.md | 12 +- docs/user-guide/data-formats.md | 2 +- 11 files changed, 322 insertions(+), 208 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 49a7528904..b1e608ee2f 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2156,8 +2156,14 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { // Each group's key, polygons and records, in the order first met; found by its key's hash. let mut groups: Vec<(Value, Vec>, Vec)> = Vec::new(); let mut index: HashMap, usize> = HashMap::new(); + // Features with no polygon to dissolve (no geometry, or a point or line in the polygon layer, + // as Minimum Bounding Geometry's flat hulls are): left out of the groups, and counted. + let mut not_polygons = 0usize; for r in each_once(rows(inp, p, "input", ds)) { - let Some(mp) = ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon) else { continue }; + let Some(mp) = ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon) else { + not_polygons += 1; + continue; + }; let value = fi.map_or(&Value::Null, |fi| ds.value(r, fi)); // A NaN equals no value, so each one stays a group of its own; −0 is 0, as it compares. let part = crate::stats::equal_key(value); @@ -2196,7 +2202,16 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { out.push(from_multipolygon(&u), attrs); } let n = out.features.len(); + // Features read, none with a polygon: an unusable result, so an error saying why. + if n == 0 && not_polygons > 0 { + return Err(format!("Dissolve: none of the {not_polygons} feature(s) has a polygon: no geometry, or a point or line")); + } let mut messages = vec![format!("Dissolve: {n} features")]; + if not_polygons > 0 { + messages.push(format!( + "Dissolve: {not_polygons} feature(s) left out, with their values: no geometry, or a point or line rather than a polygon" + )); + } messages.extend(rounded_notice(&rounded, out.fields.get(first_added..).unwrap_or_default())); Ok(ToolResult { output: Output::Layer(out), messages }) } @@ -2321,6 +2336,7 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { { out.geometry_type = first; } + let out_kind = out.geometry_type; let gaps_note = (gaps > 0).then(|| { format!("{gaps} vertex(es) passed over: a coordinate missing or infinite, or so far out that the hull can't take it (past about 9 million degrees)") }); @@ -2338,8 +2354,20 @@ fn convex_hull(inp: &dyn Inputs, p: &Json) -> Result { } r.messages.extend(gaps_note.into_iter().chain(poles_note).map(|n| format!("Minimum Bounding Geometry: {n}"))); if flat > 0 { + // Where they go: a layer of their kind when every hull is flat, else the polygon layer, + // where tools taking polygons pass them over. + let layer = match out_kind { + GeometryType::Point => "the output is a point layer".to_string(), + GeometryType::Polyline => "the output is a line layer".to_string(), + // Points and lines, every hull flat: a polygon layer, there being no one kind. + _ if flat >= n => "they are kept in the output's polygon layer, points and lines together, where Dissolve leaves them out and as Clip masks they cover nothing".to_string(), + _ => format!( + "they are kept in the output's polygon layer, among its {} polygon(s), where Dissolve leaves them out and as Clip masks they cover nothing", + n.saturating_sub(flat) + ), + }; r.messages.push(format!( - "Minimum Bounding Geometry: {flat} of {n} hulls have no area, so they are written as a point or a line: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground)" + "Minimum Bounding Geometry: {flat} of {n} hulls have no area, so they are written as a point or a line: a single point, or points in one place or on one straight line on the map (on one meridian or parallel, say; beyond 85.05° that includes points around a pole, which enclose a cap on the ground); {layer}" )); } Ok(r) @@ -6677,6 +6705,41 @@ mod tests { } } + /// Dissolve says how many features it left out for having no polygon: a point and a line in a + /// polygon layer (as Minimum Bounding Geometry's flat hulls are) and a feature without + /// geometry. A layer of polygons says nothing of the kind. + #[test] + fn dissolve_counts_features_with_no_polygon() { + let notice = |extra: bool| { + let mut polys = Dataset::new("polys", GeometryType::Polygon, vec![Field::new("k", FieldType::Integer)]); + polys.push(Some(sq(0.0, 0.0, 1.0)), vec![Value::Int(1)]); + polys.push(Some(sq(3.0, 0.0, 1.0)), vec![Value::Int(1)]); + if extra { + polys.push(Some(Geometry::Point(Coord::new(5.0, 5.0))), vec![Value::Int(1)]); + polys.push(Some(Geometry::Line(vec![vec![Coord::new(0.0, 5.0), Coord::new(1.0, 5.0)]])), vec![Value::Int(2)]); + polys.push(None, vec![Value::Int(3)]); + } + let m = Map(HashMap::from([("polys".to_string(), polys)])); + let res = crate::run("dissolve", &m, &serde_json::json!({"input": "polys", "field": "k"})).unwrap(); + let note = res.messages.iter().find(|m| m.contains("left out")).cloned(); + (layer(res).features.len(), note) + }; + let (groups, note) = notice(true); + assert_eq!(groups, 1); + assert_eq!( + note.as_deref(), + Some("Dissolve: 3 feature(s) left out, with their values: no geometry, or a point or line rather than a polygon") + ); + assert_eq!(notice(false), (1, None)); + // None with a polygon: an error saying why, not an empty layer. + let mut flat = Dataset::new("flat", GeometryType::Polygon, vec![]); + flat.push(Some(Geometry::Point(Coord::new(5.0, 5.0))), vec![]); + flat.push(None, vec![]); + let m = Map(HashMap::from([("flat".to_string(), flat)])); + let e = crate::run("dissolve", &m, &serde_json::json!({"input": "flat"})).err().unwrap_or_default(); + assert!(e.contains("none of the 2 feature(s) has a polygon"), "{e}"); + } + /// Dissolve's groups, found by their keys' hash: in the order first met, −0 with 0, each NaN /// on its own (it equals no value), and a number apart from the same text and a whole number /// apart from the same fraction. @@ -8565,9 +8628,20 @@ mod tests { let polar = [(0.0, 89.0), (90.0, 89.0), (180.0, 89.0), (-90.0, 89.0)]; let note_all = note(&layer(&polar, false), "All").unwrap_or_default(); assert!(note_all.contains(": 1 of 1 hulls have no area"), "{note_all}"); - // One feature each: the polar multipoint's hull is flat, the other's isn't. + // Every hull flat, and a line: the notice says the output is a line layer. + assert!(note_all.ends_with("; the output is a line layer"), "{note_all}"); + // One feature each: the polar multipoint's hull is flat, the other's isn't, so the flat one + // stays in the polygon layer, and the notice says where and what polygon tools do with it. let note_each = note(&layer(&polar, true), "None").unwrap_or_default(); assert!(note_each.contains(": 1 of 2 hulls have no area"), "{note_each}"); + assert!(note_each.contains("kept in the output's polygon layer, among its 1 polygon(s)"), "{note_each}"); + // Every hull flat, a point and a line: no one kind, so a polygon layer, said as such. + let mut both = Dataset::new("pts", GeometryType::Point, vec![]); + both.push(Some(Geometry::Point(Coord::new(10.0, 20.0))), vec![]); + both.push(Some(Geometry::MultiPoint(vec![Coord::new(0.0, 0.0), Coord::new(5.0, 0.0)])), vec![]); + let note_both = note(&Map(HashMap::from([("pts".to_string(), both)])), "None").unwrap_or_default(); + assert!(note_both.contains(": 2 of 2 hulls have no area"), "{note_both}"); + assert!(note_both.contains("polygon layer, points and lines together"), "{note_both}"); let spread = [(0.0, 60.0), (10.0, 60.0), (5.0, 65.0), (0.0, 89.0), (90.0, 88.0)]; assert_eq!(note(&layer(&spread, false), "All"), None); assert_eq!(note(&layer(&spread, true), "None"), None); @@ -8576,6 +8650,7 @@ mod tests { note_single.contains(": 5 of 5 hulls have no area, so they are written as a point or a line: a single point"), "{note_single}" ); + assert!(note_single.ends_with("; the output is a point layer"), "{note_single}"); // Features stacked in one place, together, have a hull with no area, and say so. let stacked = [(10.0, 20.0), (10.0, 20.0), (10.0, 20.0)]; let note_stacked = note(&layer(&stacked, false), "All").unwrap_or_default(); diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 25ac7cd8cb..40fad08759 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -95,25 +95,12 @@ fn write_chunks_to( let t = if crs == Crs::LonLat { None } else { Some(crs.transformer().map_err(DataError::Format)?) }; let mut total = 0usize; let mut skipped = 0usize; - // A GeoPackage table holds one kind of geometry: each chunk's features of another kind are - // left out, and counted, before it's projected, so one left out can't fail the projection. - #[cfg(not(target_arch = "wasm32"))] - let gpkg_kinds = ext == "gpkg"; - let mut off_kind = 0usize; let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { // A GeoPackage export that starts again feeds the features again, and counts them again. total = 0; - off_kind = 0; // The writers count the features; a source that doesn't know how many leaves the total at 0. crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX))); let batch = src.visit_chunks(None, bytes, &mut |mut chunk| { - #[cfg(not(target_arch = "wasm32"))] - if gpkg_kinds { - let before = chunk.len(); - chunk.retain(|f| !crate::io::off_kind(f, geometry_type)); - off_kind = off_kind.saturating_add(before - chunk.len()); - total = total.saturating_add(before - chunk.len()); - } if let Some(t) = &t { // Counted as the writer writes them, so only stopped on a cancel here. project_each(&mut chunk, t, crs, false)?; @@ -129,8 +116,9 @@ fn write_chunks_to( let left_out = match ext.as_str() { #[cfg(not(target_arch = "wasm32"))] "gpkg" => { + // Every feature is written: one of another kind makes the column GEOMETRY. table = Some(crate::source::gpkg::write_features_in(&layer.name, geometry_type, &layer.fields, path, crs, gpkg, &mut feed)?); - off_kind + 0 } #[cfg(target_arch = "wasm32")] "gpkg" => { diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index dffe703d3d..5dfc7389cd 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -371,11 +371,12 @@ fn geoparquet_counts_values_written_as_null() { let _ = std::fs::remove_dir_all(d); } -/// A streamed GeoPackage export leaves out geometry of another kind than the layer's, chunk by -/// chunk, and counts it, as the export of the whole layer does: squares with points among them, -/// a few features a chunk, give the same table and the same count. +/// A streamed GeoPackage export of a layer holding another kind of geometry than its own writes +/// every feature in a GEOMETRY column, as the export of the whole layer does, even when the +/// other kind comes only in a late chunk (the export starts again for it): squares with points +/// among them, a few features a chunk, give the same table. #[test] -fn a_streamed_geopackage_leaves_out_geometry_of_another_kind() { +fn a_streamed_geopackage_of_a_mixed_layer_keeps_every_kind() { let d = folder("mixed-kinds"); let mut mixed = squares(); mixed.name = "mixed".into(); @@ -387,60 +388,106 @@ fn a_streamed_geopackage_leaves_out_geometry_of_another_kind() { let layer = Dataset::new(mixed.name.clone(), mixed.geometry_type, mixed.fields.clone()); let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); let streamed = write_chunks_in(&Runs::new(&mixed, 4), &layer, &a, Crs::LonLat, 1).unwrap(); - assert_eq!(streamed.left_out, 4); - assert_eq!(crate::io::write_path(&mixed, &b).unwrap(), 4); - let blobs = |p: &Path| { + assert_eq!((streamed.left_out, streamed.written), (0, 301)); + assert_eq!(crate::io::write_path(&mixed, &b).unwrap(), 0); + let table = |p: &Path| { let c = rusqlite::Connection::open(p).unwrap(); - c.query_row("SELECT count(*) FROM mixed WHERE geom IS NOT NULL AND substr(hex(geom), 1, 4) = '4750'", [], |r| r.get::<_, i64>(0)) - .unwrap() + let kind = c.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get::<_, String>(0)).unwrap(); + let blobs = c + .query_row("SELECT count(*) FROM mixed WHERE geom IS NOT NULL AND substr(hex(geom), 1, 4) = '4750'", [], |r| r.get::<_, i64>(0)) + .unwrap(); + (kind, blobs) }; - // Every geometry row is a GeoPackage blob of the table's kind: 296 squares, none of the points. - assert_eq!((blobs(&a), blobs(&b)), (296, 296)); + // Every geometry row is a GeoPackage blob: the 296 squares and the 4 points. + assert_eq!(table(&a), ("GEOMETRY".to_string(), 300)); + assert_eq!(table(&b), ("GEOMETRY".to_string(), 300)); assert_eq!(read_back(&a).features, read_back(&b).features); - assert!(read_back(&a).features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); + let points = read_back(&a).features.iter().filter(|f| matches!(f.geometry, Some(Geometry::Point(_)))).count(); + assert_eq!(points, 4); } /// A layer that declares no geometry type but carries geometry exports to the same GeoPackage -/// streamed as whole: a table of its first geometry's kind, with the others left out and counted. +/// streamed as whole: a table of its geometry's kind when it holds one, a GEOMETRY table when it +/// holds several, nothing left out either way. #[test] fn a_typeless_layer_exports_the_same_geopackage_streamed_or_whole() { let d = folder("typeless"); let mut ds = squares(); ds.name = "typeless".into(); ds.geometry_type = GeometryType::None; - if let Some(f) = ds.features.get_mut(3) { - f.geometry = Some(Geometry::Point(Coord::new(3.0, 1.0))); - } - let layer = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone()); - let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); - assert_eq!(write_chunks_in(&Runs::new(&ds, 4), &layer, &a, Crs::LonLat, 1).unwrap().left_out, 1); - assert_eq!(crate::io::write_path(&ds, &b).unwrap(), 1); let kind = |p: &Path| { let c = rusqlite::Connection::open(p).unwrap(); c.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get::<_, String>(0)).unwrap() }; - assert_eq!((kind(&a), kind(&b)), ("MULTIPOLYGON".to_string(), "MULTIPOLYGON".to_string())); - assert_eq!(read_back(&a).features, read_back(&b).features); + let layer = Dataset::new(ds.name.clone(), ds.geometry_type, ds.fields.clone()); + for (with_point, want) in [(false, "MULTIPOLYGON"), (true, "GEOMETRY")] { + if with_point && let Some(f) = ds.features.get_mut(3) { + f.geometry = Some(Geometry::Point(Coord::new(3.0, 1.0))); + } + let (a, b) = (d.join(format!("streamed-{with_point}.gpkg")), d.join(format!("whole-{with_point}.gpkg"))); + assert_eq!(write_chunks_in(&Runs::new(&ds, 4), &layer, &a, Crs::LonLat, 1).unwrap().left_out, 0); + assert_eq!(crate::io::write_path(&ds, &b).unwrap(), 0); + assert_eq!((kind(&a), kind(&b)), (want.to_string(), want.to_string())); + assert_eq!(read_back(&a).features, read_back(&b).features); + assert_eq!(read_back(&a).features.len(), 300); + } } -/// A projected streamed GeoPackage export leaves out geometry of another kind before projecting -/// each chunk, as the whole export does: points UTM zone 31N can't place (on the other side of -/// the world from its central meridian, 3°E) don't fail the export of the squares. +/// A projected streamed GeoPackage export of a mixed layer writes every feature, projected, in a +/// GEOMETRY column, as the whole export does. #[test] -fn a_projected_streamed_geopackage_leaves_out_before_projecting() { +fn a_projected_streamed_geopackage_of_a_mixed_layer_keeps_every_kind() { let d = folder("mixed-projected"); let mut mixed = squares(); mixed.name = "mixed".into(); for i in [2usize, 7] { if let Some(f) = mixed.features.get_mut(i) { - f.geometry = Some(Geometry::Point(Coord::new(-177.0, 0.0))); + f.geometry = Some(Geometry::Point(Coord::new(3.0, 1.0))); } } let utm = Crs::from_epsg(32631).unwrap(); let layer = Dataset::new(mixed.name.clone(), mixed.geometry_type, mixed.fields.clone()); let (a, b) = (d.join("streamed.gpkg"), d.join("whole.gpkg")); let streamed = write_chunks_in(&Runs::new(&mixed, 4), &layer, &a, utm, 1).unwrap(); - assert_eq!((streamed.left_out, streamed.written), (2, 299)); - assert_eq!(crate::io::write_path_in(&mixed, &b, utm).unwrap(), 2); - assert_eq!(read_back(&a).features.len(), read_back(&b).features.len()); + assert_eq!((streamed.left_out, streamed.written), (0, 301)); + assert_eq!(crate::io::write_path_in(&mixed, &b, utm).unwrap(), 0); + assert_eq!(read_back(&a).features, read_back(&b).features); + assert_eq!(read_back(&a).features.len(), 300); +} + +/// A mixed layer added as a table to an existing GeoPackage, streamed or whole, gets a GEOMETRY +/// column holding every feature, the file's other table left as it was. +#[test] +fn a_mixed_layer_added_to_a_geopackage_keeps_every_kind() { + let d = folder("mixed-added"); + let mut mixed = squares(); + mixed.name = "mixed".into(); + for i in [1usize, 298] { + if let Some(f) = mixed.features.get_mut(i) { + f.geometry = Some(Geometry::Point(Coord::new(i as f64, 1.0))); + } + } + let layer = Dataset::new(mixed.name.clone(), mixed.geometry_type, mixed.fields.clone()); + let add = GpkgTable { mode: crate::io::GpkgMode::AddTable, name: None }; + let mut other = squares(); + other.name = "other".into(); + other.features.truncate(3); + for (i, streamed) in [false, true].into_iter().enumerate() { + let path = d.join(format!("county-{i}.gpkg")); + crate::io::write_path(&other, &path).unwrap(); + if streamed { + let (e, _) = write_chunks_to(&Runs::new(&mixed, 4), &layer, &path, Crs::LonLat, &add, 1).unwrap(); + assert_eq!((e.left_out, e.written), (0, 301)); + } else { + assert_eq!(crate::io::write_path_to(&mixed, &path, Crs::LonLat, &add).unwrap().0, 0); + } + let c = rusqlite::Connection::open(&path).unwrap(); + let kind = |t: &str| { + c.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?1", [t], |r| r.get::<_, String>(0)) + .unwrap() + }; + assert_eq!((kind("mixed"), kind("other")), ("GEOMETRY".to_string(), "MULTIPOLYGON".to_string())); + let rows: i64 = c.query_row("SELECT count(*) FROM mixed WHERE geom IS NOT NULL", [], |r| r.get(0)).unwrap(); + assert_eq!(rows, 300); + } } diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index b7ca39762d..775da8387f 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -533,28 +533,12 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result { /// also returns the table written. pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> Result<(usize, Option)> { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - let reprojects = crs != Crs::LonLat && PROJECTED_FORMATS.contains(&ext.as_str()); - // Features a GeoPackage table can't hold, left out before projecting. - #[cfg(not(target_arch = "wasm32"))] - let mut left_out = 0usize; let projected; - let ds = if !reprojects { + let ds = if crs == Crs::LonLat || !PROJECTED_FORMATS.contains(&ext.as_str()) { ds } else { check_projected(&ext, crs)?; - // Projected into one copy of the layer. A GeoPackage export leaves out what its table - // can't hold first, so a feature left out can't fail the projection. - let mut out = ds.clone(); - #[cfg(not(target_arch = "wasm32"))] - if ext == "gpkg" { - let kind = gpkg_kind(ds); - let before = out.features.len(); - out.features.retain(|f| !off_kind(f, kind)); - left_out = before - out.features.len(); - } - let t = crs.transformer().map_err(DataError::Format)?; - project_features(&mut out.features, &t, crs)?; - projected = out; + projected = project(ds, crs)?; &projected }; match ext.as_str() { @@ -562,22 +546,10 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R #[cfg(not(target_arch = "wasm32"))] { crate::progress::begin(ds.features.len()); - let kind = gpkg_kind(ds); - let table = if kind != ds.geometry_type || ds.features.iter().any(|f| off_kind(f, kind)) { - // What the table can't hold is left out as the rest is handed over, in runs - // borrowed from the layer, so a large one isn't copied. - // Added to any left out before a projection. - let before = left_out; - let mut feed = |add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>| -> Result<()> { - // An export that starts again hands the features over again. - left_out = before + feed_of_kind(&ds.features, kind, add)?; - Ok(()) - }; - crate::source::gpkg::write_features_in(&ds.name, kind, &ds.fields, path, crs, gpkg, &mut feed)? - } else { - crate::source::gpkg::write_table_in(ds, path, crs, gpkg)? - }; - Ok((left_out, Some(table))) + // Every feature is written: a layer of several kinds of geometry gets a GEOMETRY + // column, and one that declares no type takes its geometry's. + let table = crate::source::gpkg::write_table_in(ds, path, crs, gpkg)?; + Ok((0, Some(table))) } #[cfg(target_arch = "wasm32")] { @@ -2546,52 +2518,17 @@ fn dbf_boolean(value: &Value) -> Option { } } -/// Whether a GeoPackage table of geometry type `kind` can't hold `f`'s geometry: a column holds -/// one kind of geometry, the layer's. A feature of another kind, as a mixed layer can hold (a -/// GeoJSON of points and polygons, say), is left out and counted, as a Shapefile leaves it out; -/// one without geometry fits, as a table holds it. -#[cfg(not(target_arch = "wasm32"))] -pub(crate) fn off_kind(f: &crate::Feature, kind: GeometryType) -> bool { - kind != GeometryType::None && f.geometry.as_ref().is_some_and(|g| g.kind() != kind) -} - -/// Hands `add` the runs of `features` a GeoPackage table of `kind` can hold, borrowed (nothing is -/// copied), and returns how many it left out ([`off_kind`]). -#[cfg(not(target_arch = "wasm32"))] -pub(crate) fn feed_of_kind( - features: &[crate::Feature], - kind: GeometryType, - add: &mut dyn FnMut(&[crate::Feature]) -> Result<()>, -) -> Result { - let (mut start, mut left_out) = (0usize, 0usize); - for (i, f) in features.iter().enumerate() { - // A large layer's scan stops on a cancel. - if i % 65_536 == 0 { - crate::progress::check()?; - } - if off_kind(f, kind) { - if start < i { - add(features.get(start..i).unwrap_or_default())?; - } - left_out += 1; - start = i + 1; - } - } - if start < features.len() { - add(features.get(start..).unwrap_or_default())?; - } - Ok(left_out) -} - -/// The kind of geometry a GeoPackage table of `ds` holds: its type, or, for a layer that -/// declares none but carries geometry, its first geometry's (as a streamed export takes it), so -/// that geometry isn't dropped for want of a column. -#[cfg(not(target_arch = "wasm32"))] -fn gpkg_kind(ds: &Dataset) -> GeometryType { - match ds.geometry_type { - GeometryType::None => ds.features.iter().find_map(|f| f.geometry.as_ref()).map_or(GeometryType::None, Geometry::kind), - t => t, - } +/// The notice for the `left_out` features an export to `name` (a file of extension `ext`) left +/// out, as [`write_path_in`] and [`write_bytes_in`] count them; `None` when none were. A +/// Shapefile (`.shp`, or zipped) holds only drawable geometry of the layer's one kind; FlatGeobuf, +/// KML and GPX hold only features with geometry; the other formats leave nothing out. +pub fn left_out_note(name: &str, ext: &str, left_out: usize) -> Option { + (left_out > 0).then(|| match ext { + "shp" | "zip" => format!( + "{name}: {left_out} features a Shapefile can't hold were left out: without drawable geometry, or with geometry of another kind than the layer's" + ), + _ => format!("{name}: {left_out} features without drawable geometry were left out"), + }) } /// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one @@ -2760,13 +2697,33 @@ mod tests { use super::*; use std::io::Write; - /// A GeoPackage column holds the layer's one kind of geometry: a layer read from a GeoJSON of - /// a polygon and a point (declared a polygon layer) writes the polygon and leaves the point out, - /// counted, as a Shapefile does; a feature without geometry stays, as a table holds it. A layer - /// of one kind loses nothing. + /// The left-out notice names what each format can't hold: a Shapefile, features without + /// drawable geometry or of another kind; the others, features without drawable geometry. + /// Nothing left out, no notice. + #[test] + fn left_out_notes_say_what_the_format_cant_hold() { + for ext in ["shp", "zip"] { + assert_eq!( + left_out_note("out", ext, 2).as_deref(), + Some( + "out: 2 features a Shapefile can't hold were left out: without drawable geometry, or with geometry of another kind than the layer's" + ) + ); + } + for ext in ["fgb", "kml", "gpx"] { + assert_eq!(left_out_note("out", ext, 1).as_deref(), Some("out: 1 features without drawable geometry were left out")); + } + assert_eq!(left_out_note("out", "shp", 0), None); + assert_eq!(left_out_note("out", "gpkg", 0), None); + } + + /// A GeoPackage export loses no geometry: a layer read from a GeoJSON of a polygon and a point + /// (declared a polygon layer, by its most common kind) gets a GEOMETRY column holding both, and + /// a feature without geometry stays, as a table holds it. A layer of one kind keeps its kind's + /// column, whether it declares its type or not. #[cfg(not(target_arch = "wasm32"))] #[test] - fn a_geopackage_leaves_out_geometry_of_another_kind() -> std::result::Result<(), Box> { + fn a_geopackage_of_a_mixed_layer_keeps_every_kind() -> std::result::Result<(), Box> { let geojson = r#"{"type":"FeatureCollection","features":[ {"type":"Feature","properties":{"name":"area"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}}, {"type":"Feature","properties":{"name":"spot"},"geometry":{"type":"Point","coordinates":[5,5]}}, @@ -2774,42 +2731,45 @@ mod tests { let (mixed, _) = read_geojson(geojson, "mixed")?; assert_eq!(mixed.geometry_type, GeometryType::Polygon); let dir = tempfile::tempdir()?; + // The column's type, and the rows written with whether each has geometry, read with SQL. + let table = |path: &Path| -> std::result::Result<(String, Vec<(String, bool)>), Box> { + let conn = rusqlite::Connection::open(path)?; + let kind: String = conn.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get(0))?; + let mut rows = conn.prepare(r#"SELECT "name", "geom" IS NOT NULL FROM "mixed" ORDER BY "fid""#)?; + let written = rows.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; + Ok((kind, written)) + }; + let all = vec![("area".to_string(), true), ("spot".to_string(), true), ("none".to_string(), false)]; let path = dir.path().join("mixed.gpkg"); - assert_eq!(write_path(&mixed, &path)?, 1); - // The rows written, read with SQL: the polygon, and the feature without geometry. - let conn = rusqlite::Connection::open(&path)?; - let mut rows = conn.prepare(r#"SELECT "name", "geom" IS NULL FROM "mixed" ORDER BY "fid""#)?; - let written: Vec<(String, bool)> = rows.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; - assert_eq!(written, [("area".to_string(), false), ("none".to_string(), true)]); + assert_eq!(write_path(&mixed, &path)?, 0); + assert_eq!(table(&path)?, ("GEOMETRY".to_string(), all.clone())); + // Read back, the point and the polygon are both there. let (back, _) = read_path(&path)?; - assert!(back.features.iter().all(|f| f.geometry.as_ref().is_none_or(|g| g.kind() == GeometryType::Polygon))); - // Projected, the point is left out before the projection: one UTM zone 18N can't hold (a - // quarter of the way round the equator from its central meridian, 75°W) doesn't fail the - // export of the polygon. - let mut far = mixed.clone(); - if let Some(f) = far.features.get_mut(1) { - f.geometry = Some(Geometry::Point(Coord::new(15.0, 0.0))); - } - let utm = Crs::from_epsg(32618).ok_or("no UTM zone 18N")?; - assert_eq!(write_path_in(&far, &dir.path().join("far.gpkg"), utm)?, 1); - // A layer that declares no geometry type but carries geometry takes its first geometry's - // (as a streamed export does): a polygon table, the point left out, nothing dropped silently. + let mut kinds: Vec = back.features.iter().filter_map(|f| f.geometry.as_ref().map(Geometry::kind)).collect(); + kinds.sort_by_key(|k| format!("{k:?}")); + assert_eq!(kinds, [GeometryType::Point, GeometryType::Polygon]); + // A layer that declares no geometry type does the same. let mut typeless = mixed.clone(); typeless.geometry_type = GeometryType::None; let at = dir.path().join("typeless.gpkg"); - assert_eq!(write_path(&typeless, &at)?, 1); - let conn = rusqlite::Connection::open(&at)?; - let kind: String = conn.query_row("SELECT geometry_type_name FROM gpkg_geometry_columns", [], |r| r.get(0))?; - let rows: i64 = conn.query_row(r#"SELECT count(*) FROM "mixed" WHERE "geom" IS NOT NULL"#, [], |r| r.get(0))?; - assert_eq!((kind.as_str(), rows), ("MULTIPOLYGON", 1)); - // Projected too, the count of what was left out before the projection holds. - let mut typeless_far = far.clone(); - typeless_far.geometry_type = GeometryType::None; - assert_eq!(write_path_in(&typeless_far, &dir.path().join("typeless-far.gpkg"), utm)?, 1); - // One kind: nothing left out. + assert_eq!(write_path(&typeless, &at)?, 0); + assert_eq!(table(&at)?, ("GEOMETRY".to_string(), all)); + // Projected, every feature is written too. + let utm = Crs::from_epsg(32631).ok_or("no UTM zone 31N")?; + let projected = dir.path().join("projected.gpkg"); + assert_eq!(write_path_in(&mixed, &projected, utm)?, 0); + assert_eq!(table(&projected)?.0, "GEOMETRY"); + // One kind, declared or not: that kind's column, nothing left out. let mut areas = mixed.clone(); areas.features.retain(|f| !matches!(f.geometry, Some(Geometry::Point(_)))); - assert_eq!(write_path(&areas, &dir.path().join("areas.gpkg"))?, 0); + let one = vec![("area".to_string(), true), ("none".to_string(), false)]; + let at = dir.path().join("areas.gpkg"); + assert_eq!(write_path(&areas, &at)?, 0); + assert_eq!(table(&at)?, ("MULTIPOLYGON".to_string(), one.clone())); + areas.geometry_type = GeometryType::None; + let at = dir.path().join("typeless-areas.gpkg"); + assert_eq!(write_path(&areas, &at)?, 0); + assert_eq!(table(&at)?, ("MULTIPOLYGON".to_string(), one)); Ok(()) } diff --git a/crates/data/src/source/gpkg.rs b/crates/data/src/source/gpkg.rs index f30c8a09c8..94b0d96871 100644 --- a/crates/data/src/source/gpkg.rs +++ b/crates/data/src/source/gpkg.rs @@ -547,7 +547,7 @@ pub fn write_in(ds: &crate::Dataset, path: &Path, crs: Crs) -> Result<()> { pub fn write_table_in(ds: &crate::Dataset, path: &Path, crs: Crs, target: &GpkgTable) -> Result { let name = target.name.as_deref().unwrap_or(&ds.name); if let Some(replace) = into_existing(path, target)? { - return add::write_features(path, name, ds.geometry_type, &ds.fields, crs, replace, &mut |put| put(&ds.features)); + return add::write_features(path, name, table_kind(ds), &ds.fields, crs, replace, &mut |put| put(&ds.features)); } check_new_destination(path)?; let tmp = temp_database_beside(path)?; @@ -557,6 +557,24 @@ pub fn write_table_in(ds: &crate::Dataset, path: &Path, crs: Crs, target: &GpkgT Ok(GpkgWritten { table, replaced: false, dependents: Vec::new() }) } +/// The kind of geometry a GeoPackage table of `ds` is made for: its type or, for a layer that +/// declares none but carries geometry, its first geometry's (as a streamed export takes it), so +/// that geometry isn't dropped for want of a column. A layer holding other kinds as well gets a +/// GEOMETRY column, which holds any kind ([`Redo::Mixed`]). +fn table_kind(ds: &crate::Dataset) -> GeometryType { + match ds.geometry_type { + GeometryType::None => ds.features.iter().find_map(|f| f.geometry.as_ref()).map_or(GeometryType::None, crate::Geometry::kind), + t => t, + } +} + +/// Whether a table made for geometry of `kind` can't hold `f`'s in a column of that type, so it +/// needs a GEOMETRY column: a feature of another kind (a mixed layer's, as a GeoJSON of points +/// and polygons is). One without geometry fits any table. +fn other_kind(f: &crate::Feature, kind: GeometryType) -> bool { + kind != GeometryType::None && f.geometry.as_ref().is_some_and(|g| g.kind() != kind) +} + /// Whether an export goes into the existing GeoPackage at `path` (anything there, which opening /// it as one then checks) and, if so, whether it replaces a table: `None` for a new file. A /// table is added to a new file when there is none; one can't be replaced in a file not there. @@ -572,9 +590,10 @@ fn into_existing(path: &Path, target: &GpkgTable) -> Result> { /// As [`write_table_in`], for a layer `name` of `geometry_type` with `fields`: the features `feed` /// hands to the function it is given, a run at a time, already in `crs`'s coordinates, so a -/// whole layer needn't be held. A point layer's table type and whether ids are kept are chosen -/// from the features so far; should a later one contradict that choice, the export starts -/// again with it settled (feeding the features again), giving the file [`write_in`] would. +/// whole layer needn't be held. A point layer's table type, whether the column holds any kind +/// (a feature of another kind than `geometry_type`) and whether ids are kept are chosen from the +/// features so far; should a later one contradict that choice, the export starts again with it +/// settled (feeding the features again), giving the file [`write_in`] would. pub(crate) fn write_features_in( name: &str, geometry_type: GeometryType, @@ -589,14 +608,15 @@ pub(crate) fn write_features_in( return add::write_features(path, name, geometry_type, fields, crs, replace, feed); } check_new_destination(path)?; - let (mut multi_points, mut keep_ids) = (None, true); + let (mut multi_points, mut mixed, mut keep_ids) = (None, false, true); // Each choice changes at most once. - for _ in 0..3 { + for _ in 0..4 { let tmp = temp_database_beside(path)?; let conn = Connection::open(&tmp).map_err(err)?; - let mut make = |multi| create(&conn, name, geometry_type, fields, crs, multi); - match fill(&conn, multi_points, keep_ids, &mut make, feed)? { + let mut make = |multi, mixed| create(&conn, name, geometry_type, fields, crs, multi, mixed); + match fill(&conn, multi_points, mixed, keep_ids, &mut make, feed)? { Err(Redo::MultiPoints) => multi_points = Some(true), + Err(Redo::Mixed) => mixed = true, Err(Redo::Renumber) => keep_ids = false, Ok((layout, extent)) => { finish(&conn, &layout, extent)?; @@ -609,21 +629,22 @@ pub(crate) fn write_features_in( } /// Feeds the features into the table `make` creates on `conn`, made once the first run shows -/// whether a point layer holds multipoints (unless `multi_points` already says): the table and -/// the extent of what went in, or what the export must start again for. With `keep_ids` each -/// feature keeps its id. +/// whether a point layer holds multipoints (unless `multi_points` already says), its column one +/// for any kind of geometry when `mixed`: the table and the extent of what went in, or what the +/// export must start again for. With `keep_ids` each feature keeps its id. fn fill( conn: &Connection, multi_points: Option, + mixed: bool, keep_ids: bool, - make: &mut dyn FnMut(bool) -> Result, + make: &mut dyn FnMut(bool, bool) -> Result, feed: &mut dyn FnMut(&mut dyn FnMut(&[crate::Feature]) -> Result<()>) -> Result<()>, ) -> Result> { let (mut layout, mut extent, mut redo) = (None, Extent::EMPTY, None); let fed = feed(&mut |features| { if layout.is_none() { let multi = multi_points.unwrap_or_else(|| features.iter().any(|f| matches!(f.geometry, Some(crate::Geometry::MultiPoint(_))))); - layout = Some(make(multi)?); + layout = Some(make(multi, mixed)?); } let Some(l) = layout.as_ref() else { return Err(DataError::Format("GeoPackage: no table to write".into())) }; for g in features.iter().filter_map(|f| f.geometry.as_ref()) { @@ -642,7 +663,7 @@ fn fill( (Ok(()), None) => { let layout = match layout { Some(l) => l, - None => make(multi_points.unwrap_or(false))?, + None => make(multi_points.unwrap_or(false), mixed)?, }; Ok(Ok((layout, extent))) } @@ -712,7 +733,9 @@ fn destination_error(path: &Path, error: std::io::Error) -> DataError { /// returns the table's name as written. fn build(conn: &Connection, ds: &crate::Dataset, name: &str, crs: Crs) -> Result { let multi_points = ds.features.iter().any(|f| matches!(f.geometry, Some(crate::Geometry::MultiPoint(_)))); - let layout = create(conn, name, ds.geometry_type, &ds.fields, crs, multi_points)?; + let kind = table_kind(ds); + let mixed = ds.features.iter().any(|f| other_kind(f, kind)); + let layout = create(conn, name, kind, &ds.fields, crs, multi_points, mixed)?; // Feature ids are kept when they are unique and fit; otherwise SQLite numbers the rows. let mut seen = std::collections::HashSet::new(); let keep_ids = ds.features.iter().all(|f| i64::try_from(f.id).is_ok() && seen.insert(f.id)); @@ -733,6 +756,9 @@ struct Layout { fields: Vec, has_geom: bool, multi_points: bool, + /// The kind of geometry the table is made for, and whether its column holds any kind. + kind: GeometryType, + mixed: bool, geom_type: &'static str, srs_id: i32, /// The table, geometry and id column, and spatial index, quoted for SQL. @@ -763,14 +789,23 @@ const FEATURE_METADATA: &str = "CREATE TABLE IF NOT EXISTS gpkg_geometry_columns /// Creates the GeoPackage's tables and the empty feature table (and its spatial index) for a /// layer, in a transaction [`finish`] commits. A point layer holding any multipoint is a -/// MULTIPOINT table, whose single points [`insert`] writes as one-point multipoints. -fn create(conn: &Connection, name: &str, geometry_type: GeometryType, fields: &[Field], crs: Crs, multi_points: bool) -> Result { +/// MULTIPOINT table, whose single points [`insert`] writes as one-point multipoints; a `mixed` +/// layer's is a GEOMETRY table. +fn create( + conn: &Connection, + name: &str, + geometry_type: GeometryType, + fields: &[Field], + crs: Crs, + multi_points: bool, + mixed: bool, +) -> Result { // An EPSG system by its code; a custom one numbered as GDAL numbers them. let srs_id = match crs.epsg() { 0 => 100_000, c => i32::from(c), }; - let layout = plan(name, geometry_type, fields, multi_points, srs_id); + let layout = plan(name, geometry_type, fields, multi_points, mixed, srs_id); conn.execute_batch(&format!( "PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA application_id = 1196444487; PRAGMA user_version = 10400; @@ -802,8 +837,9 @@ fn create(conn: &Connection, name: &str, geometry_type: GeometryType, fields: &[ /// The names a layer's feature table is written with, in spatial reference system `srs_id`: the /// table named for the layer (not a reserved name, never empty), its id and geometry columns -/// clear of its fields, and its fields' columns unique without regard to case. -fn plan(name: &str, geometry_type: GeometryType, fields: &[Field], multi_points: bool, srs_id: i32) -> Layout { +/// clear of its fields, and its fields' columns unique without regard to case. A `mixed` layer's +/// geometry column is GEOMETRY, which holds every kind, its features written as they are. +fn plan(name: &str, geometry_type: GeometryType, fields: &[Field], multi_points: bool, mixed: bool, srs_id: i32) -> Layout { let mut table: String = name.chars().filter(|c| !c.is_control()).collect::().trim().to_string(); if table.is_empty() { table = "layer".into(); @@ -829,7 +865,11 @@ fn plan(name: &str, geometry_type: GeometryType, fields: &[Field], multi_points: .unwrap_or_else(|| f.name.clone()); columns.push(name); } + let mixed = mixed && has_geom; + // A GEOMETRY column holds single points as they are. + let multi_points = multi_points && !mixed; let geom_type = match geometry_type { + _ if mixed => "GEOMETRY", GeometryType::Point if multi_points => "MULTIPOINT", GeometryType::Point => "POINT", GeometryType::Polyline => "MULTILINESTRING", @@ -845,6 +885,8 @@ fn plan(name: &str, geometry_type: GeometryType, fields: &[Field], multi_points: fields: fields.to_vec(), has_geom, multi_points, + kind: geometry_type, + mixed, geom_type, srs_id, table_q: t, @@ -882,6 +924,8 @@ fn create_table(conn: &Connection, layout: &Layout) -> Result<()> { enum Redo { /// A multipoint in a POINT table: the table is MULTIPOINT. MultiPoints, + /// Geometry of another kind than the table's: its column is GEOMETRY, holding every kind. + Mixed, /// An id that doesn't fit or is taken: SQLite numbers every row. Renumber, } @@ -889,7 +933,8 @@ enum Redo { /// Inserts features into the table [`create`] made, and their boxes into its spatial index. /// With `keep_ids` each keeps its id; otherwise SQLite numbers it. Stops at the first feature /// the table as made can't hold as the whole layer would have it (a multipoint in a POINT -/// table, an id that doesn't fit or is taken), saying what to [`Redo`]. +/// table, geometry of another kind, an id that doesn't fit or is taken), saying what to +/// [`Redo`]. fn insert(conn: &Connection, layout: &Layout, features: &[crate::Feature], keep_ids: bool) -> Result> { use rusqlite::types::Value as Sql; let Layout { columns, fields, has_geom, multi_points, srs_id, table_q: t, geom_q: g, fid_q: i, rtree_q: rt, .. } = layout; @@ -904,6 +949,9 @@ fn insert(conn: &Connection, layout: &Layout, features: &[crate::Feature], keep_ if has_geom { Some(conn.prepare_cached(&format!("INSERT INTO {rt} VALUES (?1, ?2, ?3, ?4, ?5)")).map_err(err)?) } else { None }; for f in features { crate::progress::step(1)?; + if has_geom && !layout.mixed && other_kind(f, layout.kind) { + return Ok(Some(Redo::Mixed)); + } if has_geom && !multi_points && matches!(f.geometry, Some(crate::Geometry::MultiPoint(_))) && layout.geom_type == "POINT" { return Ok(Some(Redo::MultiPoints)); } diff --git a/crates/data/src/source/gpkg/add.rs b/crates/data/src/source/gpkg/add.rs index b324783a5f..6e42965930 100644 --- a/crates/data/src/source/gpkg/add.rs +++ b/crates/data/src/source/gpkg/add.rs @@ -37,17 +37,17 @@ pub(super) fn write_features( // records) into deletes elsewhere; a table referring to the replaced one is refused instead. conn.execute_batch("PRAGMA foreign_keys = OFF").map_err(err)?; conn.busy_timeout(Duration::from_secs(5)).map_err(err)?; - let (mut multi_points, mut keep_ids) = (None, true); + let (mut multi_points, mut mixed, mut keep_ids) = (None, false, true); // Each choice changes at most once; an export starting again rolls back what it wrote. - for _ in 0..3 { + for _ in 0..4 { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate).map_err(err)?; check_geopackage(&tx, &file)?; // An attribute table has no spatial reference system. let srs_id = if geometry_type == GeometryType::None { 0 } else { srs_id(&tx, crs)? }; // The views and triggers naming a replaced table, when one is. let replaced = Cell::new(None); - let mut make = |multi| { - let layout = plan(name, geometry_type, fields, multi, srs_id); + let mut make = |multi, mixed| { + let layout = plan(name, geometry_type, fields, multi, mixed, srs_id); replaced.set(make_room(&tx, &file, &layout.table, replace)?); if layout.has_geom { tx.execute_batch(FEATURE_METADATA).map_err(err)?; @@ -62,8 +62,9 @@ pub(super) fn write_features( create_table(&tx, &layout)?; Ok(layout) }; - match fill(&tx, multi_points, keep_ids, &mut make, feed)? { + match fill(&tx, multi_points, mixed, keep_ids, &mut make, feed)? { Err(Redo::MultiPoints) => multi_points = Some(true), + Err(Redo::Mixed) => mixed = true, Err(Redo::Renumber) => keep_ids = false, Ok((layout, extent)) => { register(&tx, &layout, extent)?; diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 55324a16bc..d077807e17 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -53,8 +53,8 @@ pub fn commands() -> Vec { let (left_out, table) = rustgis_data::io::write_path_to(&t.dataset, Path::new(path), Crs::LonLat, &gpkg).map_err(|e| e.to_string())?; s.check_continuation(&owner).map_err(|why| super::exports::completion_refused(Some(path), why))?; - if left_out > 0 { - s.notify(format!("{path}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's"), false); + if let Some(note) = rustgis_data::io::left_out_note(path, &ext, left_out) { + s.notify(note, false); } for note in notes.into_iter().chain(dbf_notes) { s.notify(note, false); @@ -635,13 +635,9 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk dbf_notes.extend(rustgis_data::io::shapefile_values_notes(ds, path)); } // FlatGeobuf, KML, GPX and Shapefile hold features with geometry only (a Shapefile, only - // geometry it can draw). + // geometry it can draw, of the layer's kind). let mut said = Vec::new(); - if left_out > 0 { - said.push(format!( - "{path}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's" - )); - } + said.extend(rustgis_data::io::left_out_note(path, &ext, left_out)); if skipped > 0 { said.push(format!("{path}: {skipped} rows couldn't be read and were left out")); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 85b79d4d83..fc3d375d35 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -30838,12 +30838,7 @@ fn shapefile_value_losses_reach_export_notices() { notices.iter().any(|n| n.contains("3 values") && n.contains("id") && n.contains("number") && n.contains("left empty")), "{command}: {notices:?}" ); - assert!( - notices - .iter() - .any(|n| n.contains("1 features were left out: no drawable geometry, or a geometry of another kind than the layer's")), - "{command}: {notices:?}" - ); + assert!(notices.iter().any(|n| n.contains("1 features a Shapefile can't hold were left out")), "{command}: {notices:?}"); } } // A clean table gives neither loss notice; other formats keep the lossy attributes. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2667db62d5..33bb1a3ffe 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -3934,8 +3934,8 @@ impl RustGisApp { if matches!(ext.as_str(), "parquet" | "geoparquet") { dbf_notes.extend(rustgis_engine::data::source::parquet::values_note(ds, &name)); } - if left_out > 0 { - self.session.notify(format!("{name}: {left_out} features were left out: no drawable geometry, or a geometry of another kind than the layer's"), false); + if let Some(note) = io::left_out_note(&name, &ext, left_out) { + self.session.notify(note, false); } if ext == "zip" && rustgis_engine::data::raster::wkt::to_wkt(crs).is_none() { self.session.notify(format!("{name}: no .prj written, as {} has no WKT form", crs.label()), false); diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 94afba6fe0..41c0f1557b 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -762,13 +762,17 @@ about 9 million degrees), and skips a feature without geometry or with no point writing no hull for it; if every feature is skipped, it is an error. A latitude past a pole is taken as that pole. A hull with no area is written as what it is: a point for points in one place, or the line between the two farthest apart for points on one straight line on the map. The layer is -a point or line layer when every hull is one; otherwise it's a polygon layer holding any points and -lines among its polygons, which a Shapefile or GeoPackage export leaves out, saying how many. It -says how many of each. +a point or line layer when every hull is one, and the tool says so; otherwise it's a polygon layer +holding any points and lines among its polygons. Dissolve leaves those out, saying how many; as +Clip masks they cover nothing; a Shapefile export leaves them out, saying how many; and a +GeoPackage export keeps them in a column that holds any kind of geometry. It says how many of +each. **Dissolve** (`dissolve`) merges the polygons sharing a value of `field` into one feature for each value, or all of them into one without a field. Each feature has the value and `count`, the -polygons merged; records with no value form a feature of their own. **Summarize Within** +polygons merged; records with no value form a feature of their own. A feature with no polygon (no +geometry, or a point or line in the polygon layer) is left out, with its values, and the tool says +how many; if none has a polygon, it is an error. **Summarize Within** (`countPoints`) copies the polygons and adds `point_count`, the points inside each polygon or on its edge; a point where polygons overlap counts in each of them. diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 6b76bfa7b9..50abc03d70 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -574,7 +574,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | Format | How to write it | Notes | | --- | --- | --- | | GeoJSON (`.geojson`, `.json`), CSV | Data › Export (`share.exportLayer`) | The Export Features dialog picks the format; the chosen extension is appended when needed. In the web app it downloads any vector format below except GeoPackage and FlatGeobuf (a Shapefile as one zip). A streamed layer exports its whole source: on the desktop, to GeoJSON, CSV, FlatGeobuf, GeoPackage or GeoParquet a chunk at a time with no limit on its size (a GeoParquet source's row groups are read whole, so on the desktop each holds at most 16,777,216 rows; a layer read in place from the GeoPackage it goes into is read whole first), otherwise loaded whole (see [Streamed layers](data.md)); remote sources on the web must be downloaded first. | -| GeoPackage | Data › Export to `.gpkg` (desktop) | A new file with one feature table (an attributes table for a layer without geometry) in the layer's coordinate system (see below), with an R-tree and GeoPackage 1.4 triggers to maintain it. An existing destination is refused unless the export adds a table to it or replaces one of its feature or attribute tables (`mode`, see [Exporting](sharing.md)), leaving its other tables as they were. Unique feature ids that fit signed 64-bit integers are kept; otherwise SQLite assigns new ids. | +| GeoPackage | Data › Export to `.gpkg` (desktop) | A new file with one feature table (an attributes table for a layer without geometry) in the layer's coordinate system (see below), with an R-tree and GeoPackage 1.4 triggers to maintain it. An existing destination is refused unless the export adds a table to it or replaces one of its feature or attribute tables (`mode`, see [Exporting](sharing.md)), leaving its other tables as they were. A layer holding more than one kind of geometry (a GeoJSON of points and polygons, say) is written whole, in a `GEOMETRY` column, which holds any kind; a layer of one kind gets that kind's column. Unique feature ids that fit signed 64-bit integers are kept; otherwise SQLite assigns new ids. | | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index; features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | From 61bd34e60f487d712ef3e6e6cde29b595dbbf9b1 Mon Sep 17 00:00:00 2001 From: Cory Douglas Martin <4108748+flight1973@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:21:12 -0600 Subject: [PATCH 15/15] fix(analysis): Dissolve leaves out empty polygons too; the Shapefile notice says what the export left out An empty polygon in a Dissolve group added nothing to the union but counted in the group's count and statistics, and escaped the notice. It is now left out and counted with the features that have no polygon. The Shapefile left-out notice described the format ("can't hold"), though a Shapefile can hold a null shape; it now says what the export left out: "N features left out of the Shapefile: without drawable geometry, or with geometry of another kind than the layer's". Tests: a group's count and SUM leave out a point and an empty polygon among its polygons; a Shapefile export of a polygon layer with a point and a feature without geometry counts both and writes the polygon; a multipoint in a mixed GeoPackage export comes back with both members, streamed and whole. --- crates/analysis/src/tools.rs | 50 ++++++++++++++++++++------------- crates/data/src/export/tests.rs | 14 +++++++++ crates/data/src/io.rs | 29 ++++++++++++++++--- crates/engine/src/tests.rs | 2 +- 4 files changed, 71 insertions(+), 24 deletions(-) diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 5ead35350d..24b71ec969 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2200,15 +2200,17 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { // Each group's key, polygons and records, in the order first met; found by its key's hash. let mut groups: Vec<(Value, Vec>, Vec)> = Vec::new(); let mut index: HashMap, usize> = HashMap::new(); - // Features with no polygon to dissolve (no geometry, or a point or line in the polygon layer, - // as Minimum Bounding Geometry's flat hulls are): left out of the groups, and counted. + // Features with no polygon to dissolve (no geometry, an empty polygon, or a point or line in + // the polygon layer, as Minimum Bounding Geometry's flat hulls are): left out of the groups, + // their counts and statistics, and counted. let mut not_polygons = 0usize; let input_rows = each_once(rows(inp, p, "input", ds)); crate::control::begin(input_rows.len()); for (done, r) in input_rows.into_iter().enumerate() { crate::control::check()?; crate::control::reached(done); - let Some(mp) = ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon) else { + let polygon = ds.features.get(r).and_then(|f| f.geometry.as_ref()).and_then(to_multipolygon); + let Some(mp) = polygon.filter(|mp| mp.0.iter().any(|p| !p.exterior().0.is_empty())) else { not_polygons += 1; continue; }; @@ -2256,12 +2258,14 @@ fn dissolve(inp: &dyn Inputs, p: &Json) -> Result { let n = out.features.len(); // Features read, none with a polygon: an unusable result, so an error saying why. if n == 0 && not_polygons > 0 { - return Err(format!("Dissolve: none of the {not_polygons} feature(s) has a polygon: no geometry, or a point or line")); + return Err(format!( + "Dissolve: none of the {not_polygons} feature(s) has a polygon: no geometry, an empty polygon, or a point or line" + )); } let mut messages = vec![format!("Dissolve: {n} features")]; if not_polygons > 0 { messages.push(format!( - "Dissolve: {not_polygons} feature(s) left out, with their values: no geometry, or a point or line rather than a polygon" + "Dissolve: {not_polygons} feature(s) left out, with their values: no geometry, an empty polygon, or a point or line rather than a polygon" )); } messages.extend(rounded_notice(&rounded, out.fields.get(first_added..).unwrap_or_default())); @@ -6782,31 +6786,39 @@ mod tests { } /// Dissolve says how many features it left out for having no polygon: a point and a line in a - /// polygon layer (as Minimum Bounding Geometry's flat hulls are) and a feature without - /// geometry. A layer of polygons says nothing of the kind. + /// polygon layer (as Minimum Bounding Geometry's flat hulls are), an empty polygon and a + /// feature without geometry. Those in a group add nothing to its count or statistics. A layer + /// of polygons says nothing of the kind. #[test] fn dissolve_counts_features_with_no_polygon() { + let fields = vec![Field::new("k", FieldType::Integer), Field::new("v", FieldType::Double)]; let notice = |extra: bool| { - let mut polys = Dataset::new("polys", GeometryType::Polygon, vec![Field::new("k", FieldType::Integer)]); - polys.push(Some(sq(0.0, 0.0, 1.0)), vec![Value::Int(1)]); - polys.push(Some(sq(3.0, 0.0, 1.0)), vec![Value::Int(1)]); + let mut polys = Dataset::new("polys", GeometryType::Polygon, fields.clone()); + polys.push(Some(sq(0.0, 0.0, 1.0)), vec![Value::Int(1), Value::Float(1.0)]); + polys.push(Some(sq(3.0, 0.0, 1.0)), vec![Value::Int(1), Value::Float(2.0)]); if extra { - polys.push(Some(Geometry::Point(Coord::new(5.0, 5.0))), vec![Value::Int(1)]); - polys.push(Some(Geometry::Line(vec![vec![Coord::new(0.0, 5.0), Coord::new(1.0, 5.0)]])), vec![Value::Int(2)]); - polys.push(None, vec![Value::Int(3)]); + // In group 1 with the polygons, with values that would show in its sum. + polys.push(Some(Geometry::Point(Coord::new(5.0, 5.0))), vec![Value::Int(1), Value::Float(100.0)]); + polys.push(Some(Geometry::Polygon(vec![])), vec![Value::Int(1), Value::Float(1000.0)]); + polys.push(Some(Geometry::Line(vec![vec![Coord::new(0.0, 5.0), Coord::new(1.0, 5.0)]])), vec![Value::Int(2), Value::Null]); + polys.push(None, vec![Value::Int(3), Value::Null]); } let m = Map(HashMap::from([("polys".to_string(), polys)])); - let res = crate::run("dissolve", &m, &serde_json::json!({"input": "polys", "field": "k"})).unwrap(); + let params = serde_json::json!({"input": "polys", "field": "k", "statistics": [{"field": "v", "statistic": "SUM"}]}); + let res = crate::run("dissolve", &m, ¶ms).unwrap(); let note = res.messages.iter().find(|m| m.contains("left out")).cloned(); - (layer(res).features.len(), note) + let out = layer(res); + (out.features.len(), out.value(0, 1).as_f64(), out.value(0, 2).as_f64(), note) }; - let (groups, note) = notice(true); - assert_eq!(groups, 1); + let (groups, count, sum, note) = notice(true); + assert_eq!((groups, count, sum), (1, Some(2.0), Some(3.0))); assert_eq!( note.as_deref(), - Some("Dissolve: 3 feature(s) left out, with their values: no geometry, or a point or line rather than a polygon") + Some( + "Dissolve: 4 feature(s) left out, with their values: no geometry, an empty polygon, or a point or line rather than a polygon" + ) ); - assert_eq!(notice(false), (1, None)); + assert_eq!(notice(false), (1, Some(2.0), Some(3.0), None)); // None with a polygon: an error saying why, not an empty layer. let mut flat = Dataset::new("flat", GeometryType::Polygon, vec![]); flat.push(Some(Geometry::Point(Coord::new(5.0, 5.0))), vec![]); diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index 5dfc7389cd..b7e05cd1a6 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -404,6 +404,20 @@ fn a_streamed_geopackage_of_a_mixed_layer_keeps_every_kind() { assert_eq!(read_back(&a).features, read_back(&b).features); let points = read_back(&a).features.iter().filter(|f| matches!(f.geometry, Some(Geometry::Point(_)))).count(); assert_eq!(points, 4); + // A multipoint among them comes back a multipoint, with its members, streamed and whole. + let pair = Geometry::MultiPoint(vec![Coord::new(1.5, 2.5), Coord::new(3.5, 4.5)]); + if let Some(f) = mixed.features.get_mut(10) { + f.geometry = Some(pair.clone()); + } + let (c, e) = (d.join("streamed-multi.gpkg"), d.join("whole-multi.gpkg")); + write_chunks_in(&Runs::new(&mixed, 4), &layer, &c, Crs::LonLat, 1).unwrap(); + crate::io::write_path(&mixed, &e).unwrap(); + for p in [&c, &e] { + let back = read_back(p); + let multis: Vec<_> = + back.features.iter().filter_map(|f| f.geometry.as_ref()).filter(|g| matches!(g, Geometry::MultiPoint(_))).collect(); + assert_eq!(multis, [&pair], "{}", p.display()); + } } /// A layer that declares no geometry type but carries geometry exports to the same GeoPackage diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 775da8387f..c0797ba5ec 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -2520,12 +2520,13 @@ fn dbf_boolean(value: &Value) -> Option { /// The notice for the `left_out` features an export to `name` (a file of extension `ext`) left /// out, as [`write_path_in`] and [`write_bytes_in`] count them; `None` when none were. A -/// Shapefile (`.shp`, or zipped) holds only drawable geometry of the layer's one kind; FlatGeobuf, -/// KML and GPX hold only features with geometry; the other formats leave nothing out. +/// Shapefile export (`.shp`, or zipped) writes only drawable geometry of the layer's one kind (it +/// writes no null shapes, though the format has them); FlatGeobuf, KML and GPX exports write only +/// features with geometry; the other formats leave nothing out. pub fn left_out_note(name: &str, ext: &str, left_out: usize) -> Option { (left_out > 0).then(|| match ext { "shp" | "zip" => format!( - "{name}: {left_out} features a Shapefile can't hold were left out: without drawable geometry, or with geometry of another kind than the layer's" + "{name}: {left_out} features left out of the Shapefile: without drawable geometry, or with geometry of another kind than the layer's" ), _ => format!("{name}: {left_out} features without drawable geometry were left out"), }) @@ -2697,6 +2698,26 @@ mod tests { use super::*; use std::io::Write; + /// A Shapefile export of a polygon layer holding a point and a feature without geometry writes + /// the polygon and counts both left out, the point for its kind; a layer of polygons loses none. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_shapefile_counts_geometry_of_another_kind_left_out() -> std::result::Result<(), Box> { + let geojson = r#"{"type":"FeatureCollection","features":[ + {"type":"Feature","properties":{"name":"area"},"geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}}, + {"type":"Feature","properties":{"name":"spot"},"geometry":{"type":"Point","coordinates":[5,5]}}, + {"type":"Feature","properties":{"name":"none"},"geometry":null}]}"#; + let (mixed, _) = read_geojson(geojson, "mixed")?; + let dir = tempfile::tempdir()?; + assert_eq!(write_path(&mixed, &dir.path().join("mixed.shp"))?, 2); + let (back, _) = read_path(&dir.path().join("mixed.shp"))?; + assert_eq!(back.features.len(), 1); + let mut areas = mixed.clone(); + areas.features.truncate(1); + assert_eq!(write_path(&areas, &dir.path().join("areas.shp"))?, 0); + Ok(()) + } + /// The left-out notice names what each format can't hold: a Shapefile, features without /// drawable geometry or of another kind; the others, features without drawable geometry. /// Nothing left out, no notice. @@ -2706,7 +2727,7 @@ mod tests { assert_eq!( left_out_note("out", ext, 2).as_deref(), Some( - "out: 2 features a Shapefile can't hold were left out: without drawable geometry, or with geometry of another kind than the layer's" + "out: 2 features left out of the Shapefile: without drawable geometry, or with geometry of another kind than the layer's" ) ); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 53ccd14e27..e4bce1231b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -32296,7 +32296,7 @@ fn shapefile_value_losses_reach_export_notices() { notices.iter().any(|n| n.contains("3 values") && n.contains("id") && n.contains("number") && n.contains("left empty")), "{command}: {notices:?}" ); - assert!(notices.iter().any(|n| n.contains("1 features a Shapefile can't hold were left out")), "{command}: {notices:?}"); + assert!(notices.iter().any(|n| n.contains("1 features left out of the Shapefile")), "{command}: {notices:?}"); } } // A clean table gives neither loss notice; other formats keep the lossy attributes.