From 69184878c0f40e6fad5acc3028905e4a07c9a852 Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 02:02:19 -0600 Subject: [PATCH 1/4] fix(data): CSV and GPX exports say what they leave out CSV wrote no geometry for a line or polygon layer and one point of a multipoint; GPX dropped every field but the name, wrote polygons as tracks without their holes and multipoints as a waypoint per point. None of it was said. CsvLosses (counted chunk by chunk when streamed) and gpx_notes give one notice per kind with a count, in share.exportLayer and data.convert; a layer the format holds whole gets none. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/export.rs | 9 ++- crates/data/src/export/tests.rs | 2 + crates/data/src/io.rs | 101 ++++++++++++++++++++++++++++- crates/engine/src/cmd/share.rs | 14 +++- crates/engine/src/cmd/workflows.rs | 13 +++- crates/engine/src/tests.rs | 68 +++++++++++++++++++ 6 files changed, 199 insertions(+), 8 deletions(-) diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 3488373cf0..8afd18ede5 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -41,6 +41,8 @@ pub struct Exported { /// Vertices beyond 85.05° north or south, written at 85.05° by an export in Web Mercator, /// where it ends (0 in another system). See [`crate::web_mercator_moved_note`]. pub beyond_web_mercator: usize, + /// Geometry a CSV can't hold, counted over all chunks (none for the other formats). + pub csv: crate::io::CsvLosses, } /// Writes every feature of `src` to `path` in the format of its extension, in `crs` (see @@ -102,11 +104,13 @@ fn write_chunks_to( let mut skipped = 0usize; let mut beyond_web_mercator = 0usize; let mut null_numbers = 0usize; + let mut csv = crate::io::CsvLosses::default(); let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { // A GeoPackage export that starts again feeds the features again, and counts them again. total = 0; beyond_web_mercator = 0; null_numbers = 0; + csv = crate::io::CsvLosses::default(); // The writers count the features; a source that doesn't know how many leaves the total at 0. crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX))); let batch = src.visit_chunks(None, bytes, &mut |mut chunk| { @@ -124,6 +128,9 @@ fn write_chunks_to( } total = total.saturating_add(chunk.len()); null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext)); + if ext == "csv" { + csv += crate::io::CsvLosses::of(&chunk, geometry_type); + } add(&chunk) })?; skipped = batch.skipped; @@ -164,7 +171,7 @@ fn write_chunks_to( Ok::<_, DataError>(0) })?, }; - Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit, beyond_web_mercator }, table)) + Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit, beyond_web_mercator, csv }, table)) } /// Counts attributes a GeoJSON or GeoPackage writer replaces with null. Other formats have diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index a815b8f3f6..d2a7c8b216 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -171,6 +171,8 @@ fn a_streamed_export_writes_what_a_whole_export_writes() { null_numbers: 0, unfit, beyond_web_mercator: 0, + // Counted over the chunks as the whole layer counts it. + csv: if ext == "csv" { crate::io::CsvLosses::of(&whole.features, whole.geometry_type) } else { Default::default() }, }; assert_eq!(streamed, expected, "{what}"); let (a, b) = (read_back(&a), read_back(&b)); diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index c6c1e78514..31327ca61e 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -372,7 +372,7 @@ fn check_size(path: &Path) -> Result<()> { } /// The field KML and GPX name placemarks by: one called `name`, if any. -fn name_field(ds: &Dataset) -> Option { +pub fn name_field(ds: &Dataset) -> Option { ds.fields.iter().position(|f| f.name.eq_ignore_ascii_case("name")) } @@ -2667,6 +2667,105 @@ pub fn left_out_note(name: &str, ext: &str, left_out: usize) -> Option { }) } +/// What CSV can't hold of a layer's geometry: it has a point a row (a point layer's longitude +/// and latitude, see [`CsvWriter`]), and no geometry for a line or polygon layer. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CsvLosses { + /// Features of a point layer with more than one point (a multipoint), or another shape, + /// written as one point. + pub as_one_point: usize, + /// Features of a line or polygon layer whose geometry isn't written. + pub unwritten: usize, +} + +impl CsvLosses { + /// The losses of `features` in a layer of type `gt`; a streamed export adds up its chunks'. + pub fn of(features: &[crate::Feature], gt: GeometryType) -> Self { + let mut out = Self::default(); + for g in features.iter().filter_map(|f| f.geometry.as_ref()) { + match (gt, g) { + (GeometryType::Point, Geometry::Point(_)) => {} + (GeometryType::Point, Geometry::MultiPoint(v)) if v.len() <= 1 => {} + (GeometryType::Point, _) => out.as_one_point += 1, + (GeometryType::None, _) => {} + _ => out.unwritten += 1, + } + } + out + } + + /// One notice for each loss of a layer of type `gt` written to `target`; none without loss. + pub fn notes(self, gt: GeometryType, target: &str) -> Vec { + let mut notes = Vec::new(); + if self.as_one_point > 0 { + let what = if self.as_one_point == 1 { "1 feature has".to_string() } else { format!("{} features have", self.as_one_point) }; + notes.push(format!( + "{target}: {what} more than one point (a multipoint, line or polygon) and was written as one, as CSV holds a point a row" + )); + } + if self.unwritten > 0 { + let kind = if gt == GeometryType::Polygon { "polygon" } else { "line" }; + let what = if self.unwritten == 1 { format!("1 {kind}") } else { format!("{} {kind}s", self.unwritten) }; + notes.push(format!( + "{target}: CSV holds points, so the geometry of {what} wasn't written, only the attributes; CSV with WKT (Data Interop, data.convert \"format\": \"wkt\") keeps it" + )); + } + notes + } +} + +impl std::ops::AddAssign for CsvLosses { + /// Adds a chunk's losses. + fn add_assign(&mut self, other: Self) { + self.as_one_point += other.as_one_point; + self.unwritten += other.unwritten; + } +} + +/// What a GPX file of `ds` can't hold, each feature named from field `name_field`: other +/// fields' values, polygons (written as tracks along their outer rings, holes left out) and +/// multipoints (a waypoint per point). One notice per kind; none for a layer GPX holds whole. +pub fn gpx_notes(ds: &Dataset, name_field: Option, target: &str) -> Vec { + let drawn = || ds.features.iter().filter(|f| f.geometry.is_some()); + let fields: Vec<&str> = (ds.fields.iter().enumerate()) + .filter(|(k, _)| Some(*k) != name_field) + .filter(|(k, _)| drawn().any(|f| f.attrs.get(*k).is_some_and(|v| !v.is_null()))) + .map(|(_, f)| f.name.as_str()) + .collect(); + let (mut polygons, mut holes, mut multipoints) = (0, 0, 0); + for g in drawn().filter_map(|f| f.geometry.as_ref()) { + match g { + Geometry::Polygon(polys) => { + polygons += 1; + holes += polys.iter().map(|rings| rings.len().saturating_sub(1)).sum::(); + } + Geometry::MultiPoint(v) if v.len() > 1 => multipoints += 1, + _ => {} + } + } + let mut notes = Vec::new(); + if !fields.is_empty() { + let names: Vec<_> = fields.iter().take(5).map(|s| format!("'{}'", dbf_notice_name(s))).collect(); + let more = if fields.len() > 5 { format!(" and {} more", fields.len() - 5) } else { String::new() }; + let what = if fields.len() == 1 { "1 field wasn't" } else { "fields weren't" }; + notes.push(format!("{target}: GPX holds a name for each feature, so {what} written: {}{more}", names.join(", "))); + } + if polygons > 0 { + let what = if polygons == 1 { "1 polygon was".to_string() } else { format!("{polygons} polygons were") }; + let left = match holes { + 0 => String::new(), + 1 => ", and 1 hole left out".to_string(), + n => format!(", and {n} holes left out"), + }; + notes.push(format!("{target}: GPX has no areas, so {what} written as tracks along their outer edges{left}")); + } + if multipoints > 0 { + let what = if multipoints == 1 { "1 multipoint was".to_string() } else { format!("{multipoints} multipoints were") }; + notes.push(format!("{target}: {what} written as a waypoint for each of its points")); + } + notes +} + /// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one /// of the layer's kind with something drawable (a finite point, a line part of two finite points, /// a polygon whose outer ring has three). diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index c201be79db..5b19f4389f 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -628,20 +628,21 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk return Err(format!("{path}: {e}")); } let parquet = matches!(ext.as_str(), "parquet" | "geoparquet"); - let (written, left_out, skipped, table, unfit, null_numbers, moved) = match &chunks { + let (written, left_out, skipped, table, unfit, null_numbers, moved, csv) = match &chunks { Some((src, layer)) => { let (e, table) = rustgis_data::export::write_source_to(src.as_ref(), layer, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; // Counted as the chunks were written, as the layer holds none of them. let unfit = rustgis_data::source::parquet::unfit_note(&layer.fields, &e.unfit, path); let moved = rustgis_data::web_mercator_moved_note(path, &layer.name, e.beyond_web_mercator); - (e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved) + (e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved, e.csv) } None => { let (left_out, table) = rustgis_data::io::write_path_to(ds, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; let unfit = if parquet { rustgis_data::source::parquet::values_note(ds, path) } else { None }; let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, &ext); - (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None) + let csv = rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type); + (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None, csv) } }; // The features written give their own extent. A streamed layer written a chunk at a time has @@ -663,6 +664,13 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk // geometry it can draw, of the layer's kind). let mut said = Vec::new(); said.extend(rustgis_data::io::left_out_note(path, &ext, left_out)); + // CSV holds a point a row, GPX a name for each feature and no areas. + if ext == "csv" { + said.extend(csv.notes(ds.geometry_type, path)); + } + if ext == "gpx" { + said.extend(rustgis_data::io::gpx_notes(ds, rustgis_data::io::name_field(ds), path)); + } if skipped > 0 { said.push(format!("{path}: {skipped} rows couldn't be read and were left out")); } diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 234bfc3d65..ee3f6c2c91 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1404,8 +1404,8 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult { } /// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when -/// the layer's Z or M values were dropped on import, or its positions are approximate or aren't -/// longitude/latitude. +/// the layer's Z or M values were dropped on import, its positions are approximate or aren't +/// longitude/latitude, or CSV or GPX couldn't hold some of it. fn convert(s: &mut Session, p: &Value) -> CmdResult { let ds = feature_ds(s, p)?; let name_field = @@ -1419,7 +1419,14 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { f => return Err(format!("unknown format '{f}'")), }; // Every format here is longitude/latitude. - let notes = rustgis_data::export_notes([(ds, ds.extent())], ps(p, "path").unwrap_or(&ds.name), rustgis_data::raster::Crs::LonLat); + let target = ps(p, "path").unwrap_or(&ds.name); + let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat); + // CSV holds a point a row, GPX a name for each feature and no areas. + match ps(p, "format") { + Some("csv") => notes.extend(rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type).notes(ds.geometry_type, target)), + Some("gpx") => notes.extend(rustgis_data::io::gpx_notes(ds, name_field, target)), + _ => {} + } let out = match ps(p, "path") { Some(path) => { rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ed85d93a58..97c3419fa9 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -18190,6 +18190,74 @@ fn blank_project_starts_in_a_projected_crs() { assert!(crate::cmd::share::map_png(&mut s, Some(400.0), Some(300.0), None).unwrap().len() > 1000); } +/// CSV and GPX exports say what the format couldn't hold (a line layer's geometry and a +/// multipoint's other points in CSV; other fields, polygons' holes and multipoints in GPX); a +/// layer each holds whole gets no such notice. +#[test] +fn csv_and_gpx_exports_say_what_they_leave_out() { + use rustgis_data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value as V}; + let dir = std::env::temp_dir().join(format!("rustgis-engine-csv-gpx-losses-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let square = |x: f64, hole: bool| { + let ring = |a: f64, b: f64| vec![Coord::new(a, a), Coord::new(b, a), Coord::new(b, b), Coord::new(a, b), Coord::new(a, a)]; + let mut rings = vec![ring(x, x + 4.0)]; + if hole { + rings.push(ring(x + 1.0, x + 2.0)); + } + Geometry::Polygon(vec![rings]) + }; + let layer = |name: &str, kind, rows: Vec<(Geometry, Vec)>| { + let mut ds = Dataset::new(name, kind, vec![Field::new("name", FieldType::Text), Field::new("height", FieldType::Double)]); + for (g, a) in rows { + ds.push(Some(g), a); + } + ds + }; + let mut s = Session::new(Project::default()); + s.map_mut().add_dataset(layer("Areas", GeometryType::Polygon, vec![(square(0.0, true), vec![V::Text("A".into()), V::Float(2.5)])])); + let pts = vec![ + (Geometry::Point(Coord::new(1.0, 2.0)), vec![V::Text("P".into()), V::Null]), + (Geometry::MultiPoint(vec![Coord::new(3.0, 4.0), Coord::new(5.0, 6.0)]), vec![V::Text("M".into()), V::Null]), + ]; + s.map_mut().add_dataset(layer("Sites", GeometryType::Point, pts)); + let clean = vec![(Geometry::Point(Coord::new(1.0, 2.0)), vec![V::Text("P".into()), V::Null])]; + s.map_mut().add_dataset(layer("Clean", GeometryType::Point, clean)); + let said = |s: &mut Session, id: &str, p: Value| { + let mark = s.notices.len(); + s.run(id, &p).unwrap(); + s.notices[mark..].iter().map(|n| n.text.clone()).filter(|t| !t.starts_with("Exported ")).collect::>() + }; + let out = |name: &str| dir.join(name).display().to_string(); + let n = said(&mut s, "share.exportLayer", json!({"layer": "Areas", "path": out("areas.csv")})); + assert!(n.len() == 1 && n[0].contains("the geometry of 1 polygon wasn't written"), "{n:?}"); + let n = said(&mut s, "share.exportLayer", json!({"layer": "Sites", "path": out("sites.csv")})); + assert!(n.len() == 1 && n[0].contains("1 feature has more than one point"), "{n:?}"); + let n = said(&mut s, "share.exportLayer", json!({"layer": "Areas", "path": out("areas.gpx")})); + assert_eq!(n.len(), 2, "{n:?}"); + assert!( + n[0].contains("1 field wasn't written: 'height'") + && n[1].contains("1 polygon was written as tracks") + && n[1].contains("1 hole left out"), + "{n:?}" + ); + let n = said(&mut s, "share.exportLayer", json!({"layer": "Sites", "path": out("sites.gpx")})); + assert!(n.len() == 1 && n[0].contains("1 multipoint was written as a waypoint for each"), "{n:?}"); + let n = said(&mut s, "data.convert", json!({"layer": "Areas", "format": "gpx"})); + assert!(n.iter().any(|t| t.contains("'height'")) && n.iter().any(|t| t.contains("tracks")), "{n:?}"); + let n = said(&mut s, "data.convert", json!({"layer": "Areas", "format": "csv"})); + assert!(n.iter().any(|t| t.contains("CSV holds points")), "{n:?}"); + // Points alone, their other field empty: CSV and GPX hold them whole. + for ext in ["csv", "gpx"] { + assert_eq!( + said(&mut s, "share.exportLayer", json!({"layer": "Clean", "path": out(&format!("clean.{ext}"))})), + Vec::::new() + ); + assert_eq!(said(&mut s, "data.convert", json!({"layer": "Clean", "format": ext})), Vec::::new()); + } + std::fs::remove_dir_all(dir).unwrap(); +} + #[test] fn exporting_features_refuses_to_replace_an_existing_geopackage() { let dir = std::env::temp_dir().join(format!("rustgis-engine-gpkg-preserve-{}", std::process::id())); From 25406f18b51036ce70f4415295ff5deb10e079bd Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 05:29:23 -0600 Subject: [PATCH 2/4] fix(data): CSV counts an untyped layer's geometry; Data Interop says what KML and GPX leave out Review of #1780: a layer that declares no geometry type gets no coordinate columns in CSV, so its geometry is now counted as not written; Data Interop's KML and GPX now count rows without geometry, as Export Features does; the CSV loss scan runs for CSV exports only. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 28 +++++++++++++++++++++++++--- crates/engine/src/cmd/share.rs | 2 +- crates/engine/src/cmd/workflows.rs | 7 ++++++- crates/engine/src/tests.rs | 13 +++++++++++-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 31327ca61e..2cf10ddd6f 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -2674,7 +2674,8 @@ pub struct CsvLosses { /// Features of a point layer with more than one point (a multipoint), or another shape, /// written as one point. pub as_one_point: usize, - /// Features of a line or polygon layer whose geometry isn't written. + /// Features of a line or polygon layer (or one that declares no type) whose geometry isn't + /// written. pub unwritten: usize, } @@ -2687,7 +2688,7 @@ impl CsvLosses { (GeometryType::Point, Geometry::Point(_)) => {} (GeometryType::Point, Geometry::MultiPoint(v)) if v.len() <= 1 => {} (GeometryType::Point, _) => out.as_one_point += 1, - (GeometryType::None, _) => {} + // A layer of lines or polygons, or one that declares no type: no coordinates. _ => out.unwritten += 1, } } @@ -2704,7 +2705,11 @@ impl CsvLosses { )); } if self.unwritten > 0 { - let kind = if gt == GeometryType::Polygon { "polygon" } else { "line" }; + let kind = match gt { + GeometryType::Polygon => "polygon", + GeometryType::Polyline => "line", + _ => "feature", + }; let what = if self.unwritten == 1 { format!("1 {kind}") } else { format!("{} {kind}s", self.unwritten) }; notes.push(format!( "{target}: CSV holds points, so the geometry of {what} wasn't written, only the attributes; CSV with WKT (Data Interop, data.convert \"format\": \"wkt\") keeps it" @@ -4040,6 +4045,23 @@ mod tests { assert!(to_csv(&ds).unwrap().contains("longitude")); } + /// A layer that declares no geometry type gets no coordinate columns, so geometry it holds + /// is counted as not written, as for a line or polygon layer; a point layer's points aren't. + #[test] + fn csv_losses_count_geometry_a_layer_without_a_type_holds() { + let square = + Geometry::Polygon(vec![vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(0.0, 0.0)]]]); + let mut ds = Dataset::new("t", GeometryType::None, vec![]); + ds.push(Some(square), vec![]); + ds.push(None, vec![]); + let losses = CsvLosses::of(&ds.features, ds.geometry_type); + assert_eq!(losses, CsvLosses { as_one_point: 0, unwritten: 1 }); + assert!(losses.notes(ds.geometry_type, "t.csv")[0].contains("the geometry of 1 feature wasn't written")); + assert!(!to_csv(&ds).unwrap().contains("longitude")); + let points = [crate::Feature { id: 0, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }]; + assert_eq!(CsvLosses::of(&points, GeometryType::Point), CsvLosses::default()); + } + #[test] fn csv_leading_zeros_and_guard_roundtrip() { let (ds, _) = read_csv("zip,n,note\n02134,0,-abc\n10001,0.5,=x\n", "t").unwrap(); diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 5b19f4389f..840f1e74ef 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -641,7 +641,7 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk let (left_out, table) = rustgis_data::io::write_path_to(ds, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; let unfit = if parquet { rustgis_data::source::parquet::values_note(ds, path) } else { None }; let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, &ext); - let csv = rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type); + let csv = if ext == "csv" { rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type) } else { Default::default() }; (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None, csv) } }; diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index ee3f6c2c91..990550ae35 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1405,7 +1405,7 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult { /// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when /// the layer's Z or M values were dropped on import, its positions are approximate or aren't -/// longitude/latitude, or CSV or GPX couldn't hold some of it. +/// longitude/latitude, or CSV, KML or GPX couldn't hold some of it. fn convert(s: &mut Session, p: &Value) -> CmdResult { let ds = feature_ds(s, p)?; let name_field = @@ -1427,6 +1427,11 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { Some("gpx") => notes.extend(rustgis_data::io::gpx_notes(ds, name_field, target)), _ => {} } + // KML and GPX hold features with geometry only. + if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") { + let without = ds.features.iter().filter(|f| f.geometry.is_none()).count(); + notes.extend(rustgis_data::io::left_out_note(target, ext, without)); + } let out = match ps(p, "path") { Some(path) => { rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 97c3419fa9..2864b81c10 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -18191,8 +18191,9 @@ fn blank_project_starts_in_a_projected_crs() { } /// CSV and GPX exports say what the format couldn't hold (a line layer's geometry and a -/// multipoint's other points in CSV; other fields, polygons' holes and multipoints in GPX); a -/// layer each holds whole gets no such notice. +/// multipoint's other points in CSV; other fields, polygons' holes and multipoints in GPX), and +/// Data Interop's KML and GPX the rows without geometry; a layer each holds whole gets no such +/// notice. #[test] fn csv_and_gpx_exports_say_what_they_leave_out() { use rustgis_data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value as V}; @@ -18247,6 +18248,14 @@ fn csv_and_gpx_exports_say_what_they_leave_out() { assert!(n.iter().any(|t| t.contains("'height'")) && n.iter().any(|t| t.contains("tracks")), "{n:?}"); let n = said(&mut s, "data.convert", json!({"layer": "Areas", "format": "csv"})); assert!(n.iter().any(|t| t.contains("CSV holds points")), "{n:?}"); + // A row without geometry, which KML and GPX can't hold, is counted. + let mut rows = layer("Rows", GeometryType::Point, vec![(Geometry::Point(Coord::new(1.0, 2.0)), vec![V::Text("P".into()), V::Null])]); + rows.push(None, vec![V::Text("no place".into()), V::Null]); + s.map_mut().add_dataset(rows); + for format in ["kml", "gpx"] { + let n = said(&mut s, "data.convert", json!({"layer": "Rows", "format": format})); + assert!(n.len() == 1 && n[0].contains("1 features without drawable geometry were left out"), "{format}: {n:?}"); + } // Points alone, their other field empty: CSV and GPX hold them whole. for ext in ["csv", "gpx"] { assert_eq!( From 9b90e2e85cf3971eb158493789d55f76af5b30df Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 10:23:58 -0600 Subject: [PATCH 3/4] fix(data): CSV notice counts features; GPX counts empty multipoints left out Review of #1780: geometry CSV doesn't write is counted by feature, as a layer of one type can hold others; the notice no longer points to Data Interop's CSV with WKT, which converts only a streamed layer's loaded features; a multipoint without a point gives no GPX waypoint, so it is counted as left out (io::gpx_left_out) in export and Data Interop. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 43 +++++++++++++++++++++--------- crates/engine/src/cmd/share.rs | 2 +- crates/engine/src/cmd/workflows.rs | 5 ++-- crates/engine/src/tests.rs | 2 +- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 2cf10ddd6f..fe1f23e11c 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -516,7 +516,7 @@ fn projected_bytes(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec, usize) let kml = crate::interop::to_kml(ds, name_field(ds))?.into_bytes(); (crate::zip::write(&[("doc.kml".to_string(), kml)]).map_err(DataError::Format)?, without_geometry(ds)) } - "gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), without_geometry(ds)), + "gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), gpx_left_out(ds)), "shp" | "zip" => { let (parts, left_out) = shapefile_parts_in(ds, crs)?; let base: String = ds.name.chars().map(|c| if c.is_alphanumeric() || matches!(c, '-' | '_' | ' ') { c } else { '_' }).collect(); @@ -2695,8 +2695,9 @@ impl CsvLosses { out } - /// One notice for each loss of a layer of type `gt` written to `target`; none without loss. - pub fn notes(self, gt: GeometryType, target: &str) -> Vec { + /// One notice for each loss of a layer written to `target`; none without loss. Geometry + /// not written is counted by feature, whatever its kind: a layer of one type can hold others. + pub fn notes(self, target: &str) -> Vec { let mut notes = Vec::new(); if self.as_one_point > 0 { let what = if self.as_one_point == 1 { "1 feature has".to_string() } else { format!("{} features have", self.as_one_point) }; @@ -2705,15 +2706,8 @@ impl CsvLosses { )); } if self.unwritten > 0 { - let kind = match gt { - GeometryType::Polygon => "polygon", - GeometryType::Polyline => "line", - _ => "feature", - }; - let what = if self.unwritten == 1 { format!("1 {kind}") } else { format!("{} {kind}s", self.unwritten) }; - notes.push(format!( - "{target}: CSV holds points, so the geometry of {what} wasn't written, only the attributes; CSV with WKT (Data Interop, data.convert \"format\": \"wkt\") keeps it" - )); + let what = if self.unwritten == 1 { "1 feature".to_string() } else { format!("{} features", self.unwritten) }; + notes.push(format!("{target}: CSV holds points, so the geometry of {what} wasn't written, only the attributes")); } notes } @@ -2727,6 +2721,17 @@ impl std::ops::AddAssign for CsvLosses { } } +/// Features a GPX file of `ds` leaves out: those without geometry, and multipoints without a +/// point, which give no waypoint. +pub fn gpx_left_out(ds: &Dataset) -> usize { + let empty = |g: &Option| match g { + None => true, + Some(Geometry::MultiPoint(v)) => v.is_empty(), + Some(_) => false, + }; + ds.features.iter().filter(|f| empty(&f.geometry)).count() +} + /// What a GPX file of `ds` can't hold, each feature named from field `name_field`: other /// fields' values, polygons (written as tracks along their outer rings, holes left out) and /// multipoints (a waypoint per point). One notice per kind; none for a layer GPX holds whole. @@ -4056,12 +4061,24 @@ mod tests { ds.push(None, vec![]); let losses = CsvLosses::of(&ds.features, ds.geometry_type); assert_eq!(losses, CsvLosses { as_one_point: 0, unwritten: 1 }); - assert!(losses.notes(ds.geometry_type, "t.csv")[0].contains("the geometry of 1 feature wasn't written")); + assert!(losses.notes("t.csv")[0].contains("the geometry of 1 feature wasn't written")); assert!(!to_csv(&ds).unwrap().contains("longitude")); let points = [crate::Feature { id: 0, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }]; assert_eq!(CsvLosses::of(&points, GeometryType::Point), CsvLosses::default()); } + /// GPX leaves out a feature without geometry and a multipoint without a point (no waypoint + /// for it), and counts both; a point is written. + #[test] + fn gpx_counts_a_multipoint_without_points_as_left_out() { + let mut ds = Dataset::new("t", GeometryType::Point, vec![]); + ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![]); + ds.push(Some(Geometry::MultiPoint(vec![])), vec![]); + ds.push(None, vec![]); + assert_eq!(gpx_left_out(&ds), 2); + assert_eq!(write_bytes(&ds, "gpx").unwrap().1, 2); + } + #[test] fn csv_leading_zeros_and_guard_roundtrip() { let (ds, _) = read_csv("zip,n,note\n02134,0,-abc\n10001,0.5,=x\n", "t").unwrap(); diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 840f1e74ef..e73e536723 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -666,7 +666,7 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk said.extend(rustgis_data::io::left_out_note(path, &ext, left_out)); // CSV holds a point a row, GPX a name for each feature and no areas. if ext == "csv" { - said.extend(csv.notes(ds.geometry_type, path)); + said.extend(csv.notes(path)); } if ext == "gpx" { said.extend(rustgis_data::io::gpx_notes(ds, rustgis_data::io::name_field(ds), path)); diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 990550ae35..5787e79484 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1423,13 +1423,14 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat); // CSV holds a point a row, GPX a name for each feature and no areas. match ps(p, "format") { - Some("csv") => notes.extend(rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type).notes(ds.geometry_type, target)), + Some("csv") => notes.extend(rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type).notes(target)), Some("gpx") => notes.extend(rustgis_data::io::gpx_notes(ds, name_field, target)), _ => {} } // KML and GPX hold features with geometry only. if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") { - let without = ds.features.iter().filter(|f| f.geometry.is_none()).count(); + let without = + if ext == "gpx" { rustgis_data::io::gpx_left_out(ds) } else { ds.features.iter().filter(|f| f.geometry.is_none()).count() }; notes.extend(rustgis_data::io::left_out_note(target, ext, without)); } let out = match ps(p, "path") { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 2864b81c10..b7b1692130 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -18231,7 +18231,7 @@ fn csv_and_gpx_exports_say_what_they_leave_out() { }; let out = |name: &str| dir.join(name).display().to_string(); let n = said(&mut s, "share.exportLayer", json!({"layer": "Areas", "path": out("areas.csv")})); - assert!(n.len() == 1 && n[0].contains("the geometry of 1 polygon wasn't written"), "{n:?}"); + assert!(n.len() == 1 && n[0].contains("the geometry of 1 feature wasn't written"), "{n:?}"); let n = said(&mut s, "share.exportLayer", json!({"layer": "Sites", "path": out("sites.csv")})); assert!(n.len() == 1 && n[0].contains("1 feature has more than one point"), "{n:?}"); let n = said(&mut s, "share.exportLayer", json!({"layer": "Areas", "path": out("areas.gpx")})); From 2ed0b33a43740cf81c82a99cd4d4265406e505ca Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 14:06:13 -0600 Subject: [PATCH 4/4] fix(data): Export Table and the web export say what CSV and GPX leave out Review of #1780: Export Table and the web app's Export Features wrote CSV and GPX without the new notices. io::export_losses_notes gives them for a whole dataset and is used there and by Data Interop. In a point layer, a geometry without a point to write (an empty multipoint) is now counted as not written in CSV, rather than as written as one point. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 22 +++++++++++++++++++--- crates/engine/src/cmd/share.rs | 3 +++ crates/engine/src/cmd/workflows.rs | 6 +----- crates/engine/src/tests.rs | 20 +++++++++++++++++--- crates/ui-egui/src/lib.rs | 2 ++ 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index fe1f23e11c..ad9ca9fe6c 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -2674,8 +2674,8 @@ pub struct CsvLosses { /// Features of a point layer with more than one point (a multipoint), or another shape, /// written as one point. pub as_one_point: usize, - /// Features of a line or polygon layer (or one that declares no type) whose geometry isn't - /// written. + /// Features whose geometry isn't written: in a line or polygon layer (or one that declares + /// no type), and in a point layer those with no point to write (an empty multipoint). pub unwritten: usize, } @@ -2686,6 +2686,8 @@ impl CsvLosses { for g in features.iter().filter_map(|f| f.geometry.as_ref()) { match (gt, g) { (GeometryType::Point, Geometry::Point(_)) => {} + // No point to write: its coordinate cells are left empty. + (GeometryType::Point, g) if g.label_point().is_none() => out.unwritten += 1, (GeometryType::Point, Geometry::MultiPoint(v)) if v.len() <= 1 => {} (GeometryType::Point, _) => out.as_one_point += 1, // A layer of lines or polygons, or one that declares no type: no coordinates. @@ -2707,7 +2709,7 @@ impl CsvLosses { } if self.unwritten > 0 { let what = if self.unwritten == 1 { "1 feature".to_string() } else { format!("{} features", self.unwritten) }; - notes.push(format!("{target}: CSV holds points, so the geometry of {what} wasn't written, only the attributes")); + notes.push(format!("{target}: CSV holds a point a row, so the geometry of {what} wasn't written, only the attributes")); } notes } @@ -2721,6 +2723,17 @@ impl std::ops::AddAssign for CsvLosses { } } +/// What a CSV or GPX file (extension `ext`) of the whole dataset `ds` can't hold, written to +/// `target`: [`CsvLosses`]' notices, or [`gpx_notes`] with each feature named from `name_field`; +/// none for another format. A streamed export counts CSV's a chunk at a time instead. +pub fn export_losses_notes(ds: &Dataset, ext: &str, name_field: Option, target: &str) -> Vec { + match ext { + "csv" => CsvLosses::of(&ds.features, ds.geometry_type).notes(target), + "gpx" => gpx_notes(ds, name_field, target), + _ => Vec::new(), + } +} + /// Features a GPX file of `ds` leaves out: those without geometry, and multipoints without a /// point, which give no waypoint. pub fn gpx_left_out(ds: &Dataset) -> usize { @@ -4065,6 +4078,9 @@ mod tests { assert!(!to_csv(&ds).unwrap().contains("longitude")); let points = [crate::Feature { id: 0, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }]; assert_eq!(CsvLosses::of(&points, GeometryType::Point), CsvLosses::default()); + // A multipoint without a point in a point layer gets empty coordinate cells. + let empty = [crate::Feature { id: 1, geometry: Some(Geometry::MultiPoint(vec![])), attrs: vec![] }]; + assert_eq!(CsvLosses::of(&empty, GeometryType::Point), CsvLosses { as_one_point: 0, unwritten: 1 }); } /// GPX leaves out a feature without geometry and a multipoint without a point (no waypoint diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index e73e536723..23c3cfdf90 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -52,6 +52,9 @@ pub fn commands() -> Vec { if matches!(ext.as_str(), "parquet" | "geoparquet") { dbf_notes.extend(rustgis_data::source::parquet::values_note(&t.dataset, path)); } + // CSV holds a point a row, GPX a name for each feature and no areas. + let name_field = rustgis_data::io::name_field(&t.dataset); + dbf_notes.extend(rustgis_data::io::export_losses_notes(&t.dataset, &ext, name_field, path)); let (left_out, table) = rustgis_data::io::write_path_to(&t.dataset, Path::new(path), Crs::LonLat, &gpkg).map_err(|e| e.to_string())?; s.check_continuation(&owner).map_err(|why| super::exports::completion_refused(Some(path), why))?; diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 5787e79484..f0351d08d8 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1422,11 +1422,7 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { let target = ps(p, "path").unwrap_or(&ds.name); let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat); // CSV holds a point a row, GPX a name for each feature and no areas. - match ps(p, "format") { - Some("csv") => notes.extend(rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type).notes(target)), - Some("gpx") => notes.extend(rustgis_data::io::gpx_notes(ds, name_field, target)), - _ => {} - } + notes.extend(rustgis_data::io::export_losses_notes(ds, ps(p, "format").unwrap_or(""), name_field, target)); // KML and GPX hold features with geometry only. if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") { let without = diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index b7b1692130..c42bb5783a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -18192,8 +18192,8 @@ fn blank_project_starts_in_a_projected_crs() { /// CSV and GPX exports say what the format couldn't hold (a line layer's geometry and a /// multipoint's other points in CSV; other fields, polygons' holes and multipoints in GPX), and -/// Data Interop's KML and GPX the rows without geometry; a layer each holds whole gets no such -/// notice. +/// Data Interop's KML and GPX the rows without geometry, and Export Table the same for a table +/// that keeps geometry; a layer or table each holds whole gets no such notice. #[test] fn csv_and_gpx_exports_say_what_they_leave_out() { use rustgis_data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value as V}; @@ -18247,7 +18247,7 @@ fn csv_and_gpx_exports_say_what_they_leave_out() { let n = said(&mut s, "data.convert", json!({"layer": "Areas", "format": "gpx"})); assert!(n.iter().any(|t| t.contains("'height'")) && n.iter().any(|t| t.contains("tracks")), "{n:?}"); let n = said(&mut s, "data.convert", json!({"layer": "Areas", "format": "csv"})); - assert!(n.iter().any(|t| t.contains("CSV holds points")), "{n:?}"); + assert!(n.iter().any(|t| t.contains("wasn't written, only the attributes")), "{n:?}"); // A row without geometry, which KML and GPX can't hold, is counted. let mut rows = layer("Rows", GeometryType::Point, vec![(Geometry::Point(Coord::new(1.0, 2.0)), vec![V::Text("P".into()), V::Null])]); rows.push(None, vec![V::Text("no place".into()), V::Null]); @@ -18256,6 +18256,20 @@ fn csv_and_gpx_exports_say_what_they_leave_out() { let n = said(&mut s, "data.convert", json!({"layer": "Rows", "format": format})); assert!(n.len() == 1 && n[0].contains("1 features without drawable geometry were left out"), "{format}: {n:?}"); } + // A standalone table keeps geometry too: Export Table says the same, and nothing for a clean one. + let areas = layer("Area table", GeometryType::Polygon, vec![(square(0.0, true), vec![V::Text("A".into()), V::Float(2.5)])]); + let points = + layer("Point table", GeometryType::Point, vec![(Geometry::Point(Coord::new(1.0, 2.0)), vec![V::Text("P".into()), V::Null])]); + s.project.tables.push(crate::project::Table { id: 9001, dataset: areas }); + s.project.tables.push(crate::project::Table { id: 9002, dataset: points }); + let n = said(&mut s, "share.exportTable", json!({"table": 9001, "path": out("table.csv")})); + assert!(n.len() == 1 && n[0].contains("the geometry of 1 feature wasn't written"), "{n:?}"); + let n = said(&mut s, "share.exportTable", json!({"table": 9001, "path": out("table.gpx")})); + assert!(n.iter().any(|t| t.contains("'height'")) && n.iter().any(|t| t.contains("1 hole left out")), "{n:?}"); + for ext in ["csv", "gpx"] { + let n = said(&mut s, "share.exportTable", json!({"table": 9002, "path": out(&format!("clean-table.{ext}"))})); + assert_eq!(n, Vec::::new(), "{ext}"); + } // Points alone, their other field empty: CSV and GPX hold them whole. for ext in ["csv", "gpx"] { assert_eq!( diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 60a22e175f..a51db1250b 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4122,6 +4122,8 @@ impl RustGisApp { if matches!(ext.as_str(), "parquet" | "geoparquet") { dbf_notes.extend(rustgis_engine::data::source::parquet::values_note(ds, &name)); } + // CSV holds a point a row, GPX a name for each feature and no areas. + dbf_notes.extend(io::export_losses_notes(ds, &ext, io::name_field(ds), &name)); if let Some(note) = io::left_out_note(&name, &ext, left_out) { self.session.notify(note, false); }