Repository navigation
fix(data): CSV and GPX exports say what they leave out #1780
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
6918487
25406f1
9b90e2e
2ed0b33
34ecf2e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -372,7 +372,7 @@ fn check_size(path: &Path) -> Result<()> { | |||||
| } | ||||||
|
|
||||||
| /// The field KML and GPX name placemarks by: one called `name`, if any. | ||||||
| fn name_field(ds: &Dataset) -> Option<usize> { | ||||||
| pub fn name_field(ds: &Dataset) -> Option<usize> { | ||||||
| ds.fields.iter().position(|f| f.name.eq_ignore_ascii_case("name")) | ||||||
| } | ||||||
|
|
||||||
|
|
@@ -516,7 +516,7 @@ fn projected_bytes(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec<u8>, usize) | |||||
| let kml = crate::interop::to_kml(ds, name_field(ds))?.into_bytes(); | ||||||
| (crate::zip::write(&[("doc.kml".to_string(), kml)]).map_err(DataError::Format)?, without_geometry(ds)) | ||||||
| } | ||||||
| "gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), without_geometry(ds)), | ||||||
| "gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), gpx_left_out(ds)), | ||||||
| "shp" | "zip" => { | ||||||
| let (parts, left_out) = shapefile_parts_in(ds, crs)?; | ||||||
| let base: String = ds.name.chars().map(|c| if c.is_alphanumeric() || matches!(c, '-' | '_' | ' ') { c } else { '_' }).collect(); | ||||||
|
|
@@ -2686,6 +2686,128 @@ pub fn left_out_note(name: &str, ext: &str, left_out: usize) -> Option<String> { | |||||
| }) | ||||||
| } | ||||||
|
|
||||||
| /// What CSV can't hold of a layer's geometry: it has a point a row (a point layer's longitude | ||||||
| /// and latitude, see [`CsvWriter`]), and no geometry for a line or polygon layer. | ||||||
| #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] | ||||||
| pub struct CsvLosses { | ||||||
| /// Features of a point layer with more than one point (a multipoint), or another shape, | ||||||
| /// written as one point. | ||||||
| pub as_one_point: usize, | ||||||
| /// Features whose geometry isn't written: in a line or polygon layer (or one that declares | ||||||
| /// no type), and in a point layer those with no point to write (an empty multipoint). | ||||||
| pub unwritten: usize, | ||||||
| } | ||||||
|
|
||||||
| impl CsvLosses { | ||||||
| /// The losses of `features` in a layer of type `gt`; a streamed export adds up its chunks'. | ||||||
| pub fn of(features: &[crate::Feature], gt: GeometryType) -> Self { | ||||||
| let mut out = Self::default(); | ||||||
| for g in features.iter().filter_map(|f| f.geometry.as_ref()) { | ||||||
| match (gt, g) { | ||||||
| (GeometryType::Point, Geometry::Point(_)) => {} | ||||||
| // No point to write: its coordinate cells are left empty. | ||||||
| (GeometryType::Point, g) if g.label_point().is_none() => out.unwritten += 1, | ||||||
| (GeometryType::Point, Geometry::MultiPoint(v)) if v.len() <= 1 => {} | ||||||
| (GeometryType::Point, _) => out.as_one_point += 1, | ||||||
| // A layer of lines or polygons, or one that declares no type: no coordinates. | ||||||
| _ => out.unwritten += 1, | ||||||
| } | ||||||
| } | ||||||
| out | ||||||
| } | ||||||
|
|
||||||
| /// One notice for each loss of a layer written to `target`; none without loss. Geometry | ||||||
| /// not written is counted by feature, whatever its kind: a layer of one type can hold others. | ||||||
| pub fn notes(self, target: &str) -> Vec<String> { | ||||||
| let mut notes = Vec::new(); | ||||||
| if self.as_one_point > 0 { | ||||||
| let what = if self.as_one_point == 1 { "1 feature has".to_string() } else { format!("{} features have", self.as_one_point) }; | ||||||
| notes.push(format!( | ||||||
| "{target}: {what} more than one point (a multipoint, line or polygon) and was written as one, as CSV holds a point a row" | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Grammar nit in the new notice: with two or more features the text reads "2 features have more than one point … and was written as one". The verb should be "were", and the singular and plural forms can't share one phrase. Rewording to a participle covers both cases:
Suggested change
Wording point, so it can wait for the release quality pass if you prefer.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right: the plural should read "were written as one each". It's a wording fix, left for the release quality pass. |
||||||
| )); | ||||||
| } | ||||||
| if self.unwritten > 0 { | ||||||
| let what = if self.unwritten == 1 { "1 feature".to_string() } else { format!("{} features", self.unwritten) }; | ||||||
| notes.push(format!("{target}: CSV holds a point a row, so the geometry of {what} wasn't written, only the attributes")); | ||||||
|
Mavrick-F marked this conversation as resolved.
Resolved
|
||||||
| } | ||||||
| notes | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| impl std::ops::AddAssign for CsvLosses { | ||||||
| /// Adds a chunk's losses. | ||||||
| fn add_assign(&mut self, other: Self) { | ||||||
| self.as_one_point += other.as_one_point; | ||||||
| self.unwritten += other.unwritten; | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /// What a CSV or GPX file (extension `ext`) of the whole dataset `ds` can't hold, written to | ||||||
| /// `target`: [`CsvLosses`]' notices, or [`gpx_notes`] with each feature named from `name_field`; | ||||||
| /// none for another format. A streamed export counts CSV's a chunk at a time instead. | ||||||
| pub fn export_losses_notes(ds: &Dataset, ext: &str, name_field: Option<usize>, target: &str) -> Vec<String> { | ||||||
| match ext { | ||||||
| "csv" => CsvLosses::of(&ds.features, ds.geometry_type).notes(target), | ||||||
| "gpx" => gpx_notes(ds, name_field, target), | ||||||
| _ => Vec::new(), | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /// Features a GPX file of `ds` leaves out: those without geometry, and multipoints without a | ||||||
| /// point, which give no waypoint. | ||||||
| pub fn gpx_left_out(ds: &Dataset) -> usize { | ||||||
| let empty = |g: &Option<Geometry>| match g { | ||||||
| None => true, | ||||||
| Some(Geometry::MultiPoint(v)) => v.is_empty(), | ||||||
| Some(_) => false, | ||||||
| }; | ||||||
| ds.features.iter().filter(|f| empty(&f.geometry)).count() | ||||||
| } | ||||||
|
|
||||||
| /// What a GPX file of `ds` can't hold, each feature named from field `name_field`: other | ||||||
| /// fields' values, polygons (written as tracks along their outer rings, holes left out) and | ||||||
| /// multipoints (a waypoint per point). One notice per kind; none for a layer GPX holds whole. | ||||||
| pub fn gpx_notes(ds: &Dataset, name_field: Option<usize>, target: &str) -> Vec<String> { | ||||||
| let drawn = || ds.features.iter().filter(|f| f.geometry.is_some()); | ||||||
| let fields: Vec<&str> = (ds.fields.iter().enumerate()) | ||||||
| .filter(|(k, _)| Some(*k) != name_field) | ||||||
| .filter(|(k, _)| drawn().any(|f| f.attrs.get(*k).is_some_and(|v| !v.is_null()))) | ||||||
| .map(|(_, f)| f.name.as_str()) | ||||||
| .collect(); | ||||||
| let (mut polygons, mut holes, mut multipoints) = (0, 0, 0); | ||||||
| for g in drawn().filter_map(|f| f.geometry.as_ref()) { | ||||||
| match g { | ||||||
| Geometry::Polygon(polys) => { | ||||||
| polygons += 1; | ||||||
| holes += polys.iter().map(|rings| rings.len().saturating_sub(1)).sum::<usize>(); | ||||||
| } | ||||||
| Geometry::MultiPoint(v) if v.len() > 1 => multipoints += 1, | ||||||
| _ => {} | ||||||
| } | ||||||
| } | ||||||
| let mut notes = Vec::new(); | ||||||
| if !fields.is_empty() { | ||||||
| let names: Vec<_> = fields.iter().take(5).map(|s| format!("'{}'", dbf_notice_name(s))).collect(); | ||||||
| let more = if fields.len() > 5 { format!(" and {} more", fields.len() - 5) } else { String::new() }; | ||||||
| let what = if fields.len() == 1 { "1 field wasn't" } else { "fields weren't" }; | ||||||
| notes.push(format!("{target}: GPX holds a name for each feature, so {what} written: {}{more}", names.join(", "))); | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Quality nits, grouped (left for the release quality pass):
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Both are left for the release quality pass: the wording, and a GPX-specific name helper. The 40-character cap ends a cut name with |
||||||
| } | ||||||
| if polygons > 0 { | ||||||
| let what = if polygons == 1 { "1 polygon was".to_string() } else { format!("{polygons} polygons were") }; | ||||||
| let left = match holes { | ||||||
| 0 => String::new(), | ||||||
| 1 => ", and 1 hole left out".to_string(), | ||||||
| n => format!(", and {n} holes left out"), | ||||||
| }; | ||||||
| notes.push(format!("{target}: GPX has no areas, so {what} written as tracks along their outer edges{left}")); | ||||||
| } | ||||||
| if multipoints > 0 { | ||||||
| let what = if multipoints == 1 { "1 multipoint was".to_string() } else { format!("{multipoints} multipoints were") }; | ||||||
| notes.push(format!("{target}: {what} written as a waypoint for each of its points")); | ||||||
| } | ||||||
| notes | ||||||
| } | ||||||
|
|
||||||
| /// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one | ||||||
| /// of the layer's kind with something drawable (a finite point, a line part of two finite points, | ||||||
| /// a polygon whose outer ring has three). | ||||||
|
|
@@ -3975,6 +4097,38 @@ mod tests { | |||||
| assert!(to_csv(&ds).unwrap().contains("longitude")); | ||||||
| } | ||||||
|
|
||||||
| /// A layer that declares no geometry type gets no coordinate columns, so geometry it holds | ||||||
| /// is counted as not written, as for a line or polygon layer; a point layer's points aren't. | ||||||
| #[test] | ||||||
| fn csv_losses_count_geometry_a_layer_without_a_type_holds() { | ||||||
| let square = | ||||||
| Geometry::Polygon(vec![vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(0.0, 0.0)]]]); | ||||||
| let mut ds = Dataset::new("t", GeometryType::None, vec![]); | ||||||
| ds.push(Some(square), vec![]); | ||||||
| ds.push(None, vec![]); | ||||||
| let losses = CsvLosses::of(&ds.features, ds.geometry_type); | ||||||
| assert_eq!(losses, CsvLosses { as_one_point: 0, unwritten: 1 }); | ||||||
| assert!(losses.notes("t.csv")[0].contains("the geometry of 1 feature wasn't written")); | ||||||
| assert!(!to_csv(&ds).unwrap().contains("longitude")); | ||||||
| let points = [crate::Feature { id: 0, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }]; | ||||||
| assert_eq!(CsvLosses::of(&points, GeometryType::Point), CsvLosses::default()); | ||||||
| // A multipoint without a point in a point layer gets empty coordinate cells. | ||||||
| let empty = [crate::Feature { id: 1, geometry: Some(Geometry::MultiPoint(vec![])), attrs: vec![] }]; | ||||||
| assert_eq!(CsvLosses::of(&empty, GeometryType::Point), CsvLosses { as_one_point: 0, unwritten: 1 }); | ||||||
| } | ||||||
|
|
||||||
| /// GPX leaves out a feature without geometry and a multipoint without a point (no waypoint | ||||||
| /// for it), and counts both; a point is written. | ||||||
| #[test] | ||||||
| fn gpx_counts_a_multipoint_without_points_as_left_out() { | ||||||
| let mut ds = Dataset::new("t", GeometryType::Point, vec![]); | ||||||
| ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![]); | ||||||
| ds.push(Some(Geometry::MultiPoint(vec![])), vec![]); | ||||||
| ds.push(None, vec![]); | ||||||
| assert_eq!(gpx_left_out(&ds), 2); | ||||||
| assert_eq!(write_bytes(&ds, "gpx").unwrap().1, 2); | ||||||
| } | ||||||
|
|
||||||
| #[test] | ||||||
| fn csv_leading_zeros_and_guard_roundtrip() { | ||||||
| let (ds, _) = read_csv("zip,n,note\n02134,0,-abc\n10001,0.5,=x\n", "t").unwrap(); | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -53,6 +53,9 @@ pub fn commands() -> Vec<C> { | |
| dbf_notes.extend(rustgis_data::source::parquet::values_note(&t.dataset, path)); | ||
| } | ||
| dbf_notes.extend(rustgis_data::io::dates_as_text_note(&t.dataset.fields, path)); | ||
| // CSV holds a point a row, GPX a name for each feature and no areas. | ||
| let name_field = rustgis_data::io::name_field(&t.dataset); | ||
| dbf_notes.extend(rustgis_data::io::export_losses_notes(&t.dataset, &ext, name_field, path)); | ||
| let (left_out, table) = | ||
| rustgis_data::io::write_path_to(&t.dataset, Path::new(path), Crs::LonLat, &gpkg).map_err(|e| e.to_string())?; | ||
| s.check_continuation(&owner).map_err(|why| super::exports::completion_refused(Some(path), why))?; | ||
|
|
@@ -666,20 +669,21 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option<Crs>, gpkg: Gpk | |
| return Err(format!("{path}: {e}")); | ||
| } | ||
| let parquet = matches!(ext.as_str(), "parquet" | "geoparquet"); | ||
| let (written, left_out, skipped, table, unfit, null_numbers, moved) = match &chunks { | ||
| let (written, left_out, skipped, table, unfit, null_numbers, moved, csv) = match &chunks { | ||
| Some((src, layer)) => { | ||
| let (e, table) = | ||
| rustgis_data::export::write_source_to(src.as_ref(), layer, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; | ||
| // Counted as the chunks were written, as the layer holds none of them. | ||
| let unfit = rustgis_data::source::parquet::unfit_note(&layer.fields, &e.unfit, path); | ||
| let moved = rustgis_data::web_mercator_moved_note(path, &layer.name, e.beyond_web_mercator); | ||
| (e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved) | ||
| (e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved, e.csv) | ||
| } | ||
| None => { | ||
| let (left_out, table) = rustgis_data::io::write_path_to(ds, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; | ||
| let unfit = if parquet { rustgis_data::source::parquet::values_note(ds, path) } else { None }; | ||
| let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, &ext); | ||
| (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None) | ||
| let csv = if ext == "csv" { rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type) } else { Default::default() }; | ||
| (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None, csv) | ||
| } | ||
| }; | ||
| // The features written give their own extent. A streamed layer written a chunk at a time has | ||
|
|
@@ -702,6 +706,13 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option<Crs>, gpkg: Gpk | |
| // geometry it can draw, of the layer's kind). | ||
| let mut said = Vec::new(); | ||
| said.extend(rustgis_data::io::left_out_note(path, &ext, left_out)); | ||
| // CSV holds a point a row, GPX a name for each feature and no areas. | ||
| if ext == "csv" { | ||
| said.extend(csv.notes(path)); | ||
| } | ||
| if ext == "gpx" { | ||
| said.extend(rustgis_data::io::gpx_notes(ds, rustgis_data::io::name_field(ds), path)); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Gap: Export Table to CSV or GPX still drops silently (medium-high confidence). The same omission is in the browser's The dispatch is now in three places (
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in 2ed0b33. The new |
||
| } | ||
| if skipped > 0 { | ||
| said.push(format!("{path}: {skipped} rows couldn't be read and were left out")); | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1590,8 +1590,8 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult { | |
| } | ||
|
|
||
| /// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AGENTS.md, "Never hold the window" (medium confidence, partly pre-existing).
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This predates the PR. On |
||
| /// the layer's Z or M values were dropped on import, or its positions are approximate or aren't | ||
| /// longitude/latitude. | ||
| /// the layer's Z or M values were dropped on import, its positions are approximate or aren't | ||
| /// longitude/latitude, or CSV, KML or GPX couldn't hold some of it. | ||
| fn convert(s: &mut Session, p: &Value) -> CmdResult { | ||
| let ds = feature_ds(s, p)?; | ||
| let name_field = | ||
|
|
@@ -1605,7 +1605,16 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { | |
| f => return Err(format!("unknown format '{f}'")), | ||
| }; | ||
| // Every format here is longitude/latitude. | ||
| let notes = rustgis_data::export_notes([(ds, ds.extent())], ps(p, "path").unwrap_or(&ds.name), rustgis_data::raster::Crs::LonLat); | ||
| let target = ps(p, "path").unwrap_or(&ds.name); | ||
| let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat); | ||
| // CSV holds a point a row, GPX a name for each feature and no areas. | ||
| notes.extend(rustgis_data::io::export_losses_notes(ds, ps(p, "format").unwrap_or(""), name_field, target)); | ||
| // KML and GPX hold features with geometry only. | ||
| if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") { | ||
| let without = | ||
| if ext == "gpx" { rustgis_data::io::gpx_left_out(ds) } else { ds.features.iter().filter(|f| f.geometry.is_none()).count() }; | ||
| notes.extend(rustgis_data::io::left_out_note(target, ext, without)); | ||
| } | ||
| let out = match ps(p, "path") { | ||
| Some(path) => { | ||
| rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Quality nit, left for the release quality pass: "it has a point a row" reads as a typo for "it holds a point a row". Suggested wording:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Wording point, left for the release quality pass.