Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ pub struct Exported {
/// Vertices beyond 85.05° north or south, written at 85.05° by an export in Web Mercator,
/// where it ends (0 in another system). See [`crate::web_mercator_moved_note`].
pub beyond_web_mercator: usize,
/// Geometry a CSV can't hold, counted over all chunks (none for the other formats).
pub csv: crate::io::CsvLosses,
}

/// Writes every feature of `src` to `path` in the format of its extension, in `crs` (see
Expand Down Expand Up @@ -102,11 +104,13 @@ fn write_chunks_to(
let mut skipped = 0usize;
let mut beyond_web_mercator = 0usize;
let mut null_numbers = 0usize;
let mut csv = crate::io::CsvLosses::default();
let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> {
// A GeoPackage export that starts again feeds the features again, and counts them again.
total = 0;
beyond_web_mercator = 0;
null_numbers = 0;
csv = crate::io::CsvLosses::default();
// The writers count the features; a source that doesn't know how many leaves the total at 0.
crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX)));
let batch = src.visit_chunks(None, bytes, &mut |mut chunk| {
Expand All @@ -124,6 +128,9 @@ fn write_chunks_to(
}
total = total.saturating_add(chunk.len());
null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext));
if ext == "csv" {
csv += crate::io::CsvLosses::of(&chunk, geometry_type);
}
add(&chunk)
})?;
skipped = batch.skipped;
Expand Down Expand Up @@ -164,7 +171,7 @@ fn write_chunks_to(
Ok::<_, DataError>(0)
})?,
};
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit, beyond_web_mercator }, table))
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit, beyond_web_mercator, csv }, table))
}

/// Counts attributes a GeoJSON or GeoPackage writer replaces with null. Other formats have
Expand Down
2 changes: 2 additions & 0 deletions crates/data/src/export/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ fn a_streamed_export_writes_what_a_whole_export_writes() {
null_numbers: 0,
unfit,
beyond_web_mercator: 0,
// Counted over the chunks as the whole layer counts it.
csv: if ext == "csv" { crate::io::CsvLosses::of(&whole.features, whole.geometry_type) } else { Default::default() },
};
assert_eq!(streamed, expected, "{what}");
let (a, b) = (read_back(&a), read_back(&b));
Expand Down
158 changes: 156 additions & 2 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ fn check_size(path: &Path) -> Result<()> {
}

/// The field KML and GPX name placemarks by: one called `name`, if any.
fn name_field(ds: &Dataset) -> Option<usize> {
pub fn name_field(ds: &Dataset) -> Option<usize> {
ds.fields.iter().position(|f| f.name.eq_ignore_ascii_case("name"))
}

Expand Down Expand Up @@ -516,7 +516,7 @@ fn projected_bytes(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec<u8>, usize)
let kml = crate::interop::to_kml(ds, name_field(ds))?.into_bytes();
(crate::zip::write(&[("doc.kml".to_string(), kml)]).map_err(DataError::Format)?, without_geometry(ds))
}
"gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), without_geometry(ds)),
"gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), gpx_left_out(ds)),
"shp" | "zip" => {
let (parts, left_out) = shapefile_parts_in(ds, crs)?;
let base: String = ds.name.chars().map(|c| if c.is_alphanumeric() || matches!(c, '-' | '_' | ' ') { c } else { '_' }).collect();
Expand Down Expand Up @@ -2686,6 +2686,128 @@ pub fn left_out_note(name: &str, ext: &str, left_out: usize) -> Option<String> {
})
}

/// What CSV can't hold of a layer's geometry: it has a point a row (a point layer's longitude

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quality nit, left for the release quality pass: "it has a point a row" reads as a typo for "it holds a point a row". Suggested wording:

Suggested change
/// What CSV can't hold of a layer's geometry: it has a point a row (a point layer's longitude
/// What CSV can't hold of a layer's geometry: it holds a point a row (a point layer's longitude

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wording point, left for the release quality pass.

/// and latitude, see [`CsvWriter`]), and no geometry for a line or polygon layer.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct CsvLosses {
/// Features of a point layer with more than one point (a multipoint), or another shape,
/// written as one point.
pub as_one_point: usize,
/// Features whose geometry isn't written: in a line or polygon layer (or one that declares
/// no type), and in a point layer those with no point to write (an empty multipoint).
pub unwritten: usize,
}

impl CsvLosses {
/// The losses of `features` in a layer of type `gt`; a streamed export adds up its chunks'.
pub fn of(features: &[crate::Feature], gt: GeometryType) -> Self {
let mut out = Self::default();
for g in features.iter().filter_map(|f| f.geometry.as_ref()) {
match (gt, g) {
(GeometryType::Point, Geometry::Point(_)) => {}
// No point to write: its coordinate cells are left empty.
(GeometryType::Point, g) if g.label_point().is_none() => out.unwritten += 1,
(GeometryType::Point, Geometry::MultiPoint(v)) if v.len() <= 1 => {}
(GeometryType::Point, _) => out.as_one_point += 1,
// A layer of lines or polygons, or one that declares no type: no coordinates.
_ => out.unwritten += 1,
}
}
out
}

/// One notice for each loss of a layer written to `target`; none without loss. Geometry
/// not written is counted by feature, whatever its kind: a layer of one type can hold others.
pub fn notes(self, target: &str) -> Vec<String> {
let mut notes = Vec::new();
if self.as_one_point > 0 {
let what = if self.as_one_point == 1 { "1 feature has".to_string() } else { format!("{} features have", self.as_one_point) };
notes.push(format!(
"{target}: {what} more than one point (a multipoint, line or polygon) and was written as one, as CSV holds a point a row"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Grammar nit in the new notice: with two or more features the text reads "2 features have more than one point … and was written as one". The verb should be "were", and the singular and plural forms can't share one phrase. Rewording to a participle covers both cases:

Suggested change
"{target}: {what} more than one point (a multipoint, line or polygon) and was written as one, as CSV holds a point a row"
"{target}: {what} more than one point (a multipoint, line or polygon), written as one because CSV holds a point a row"

Wording point, so it can wait for the release quality pass if you prefer.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right: the plural should read "were written as one each". It's a wording fix, left for the release quality pass.

));
}
if self.unwritten > 0 {
let what = if self.unwritten == 1 { "1 feature".to_string() } else { format!("{} features", self.unwritten) };
notes.push(format!("{target}: CSV holds a point a row, so the geometry of {what} wasn't written, only the attributes"));
Comment thread
Mavrick-F marked this conversation as resolved.
Resolved
}
notes
}
}

impl std::ops::AddAssign for CsvLosses {
/// Adds a chunk's losses.
fn add_assign(&mut self, other: Self) {
self.as_one_point += other.as_one_point;
self.unwritten += other.unwritten;
}
}

/// What a CSV or GPX file (extension `ext`) of the whole dataset `ds` can't hold, written to
/// `target`: [`CsvLosses`]' notices, or [`gpx_notes`] with each feature named from `name_field`;
/// none for another format. A streamed export counts CSV's a chunk at a time instead.
pub fn export_losses_notes(ds: &Dataset, ext: &str, name_field: Option<usize>, target: &str) -> Vec<String> {
match ext {
"csv" => CsvLosses::of(&ds.features, ds.geometry_type).notes(target),
"gpx" => gpx_notes(ds, name_field, target),
_ => Vec::new(),
}
}

/// Features a GPX file of `ds` leaves out: those without geometry, and multipoints without a
/// point, which give no waypoint.
pub fn gpx_left_out(ds: &Dataset) -> usize {
let empty = |g: &Option<Geometry>| match g {
None => true,
Some(Geometry::MultiPoint(v)) => v.is_empty(),
Some(_) => false,
};
ds.features.iter().filter(|f| empty(&f.geometry)).count()
}

/// What a GPX file of `ds` can't hold, each feature named from field `name_field`: other
/// fields' values, polygons (written as tracks along their outer rings, holes left out) and
/// multipoints (a waypoint per point). One notice per kind; none for a layer GPX holds whole.
pub fn gpx_notes(ds: &Dataset, name_field: Option<usize>, target: &str) -> Vec<String> {
let drawn = || ds.features.iter().filter(|f| f.geometry.is_some());
let fields: Vec<&str> = (ds.fields.iter().enumerate())
.filter(|(k, _)| Some(*k) != name_field)
.filter(|(k, _)| drawn().any(|f| f.attrs.get(*k).is_some_and(|v| !v.is_null())))
.map(|(_, f)| f.name.as_str())
.collect();
let (mut polygons, mut holes, mut multipoints) = (0, 0, 0);
for g in drawn().filter_map(|f| f.geometry.as_ref()) {
match g {
Geometry::Polygon(polys) => {
polygons += 1;
holes += polys.iter().map(|rings| rings.len().saturating_sub(1)).sum::<usize>();
}
Geometry::MultiPoint(v) if v.len() > 1 => multipoints += 1,
_ => {}
}
}
let mut notes = Vec::new();
if !fields.is_empty() {
let names: Vec<_> = fields.iter().take(5).map(|s| format!("'{}'", dbf_notice_name(s))).collect();
let more = if fields.len() > 5 { format!(" and {} more", fields.len() - 5) } else { String::new() };
let what = if fields.len() == 1 { "1 field wasn't" } else { "fields weren't" };
notes.push(format!("{target}: GPX holds a name for each feature, so {what} written: {}{more}", names.join(", ")));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quality nits, grouped (left for the release quality pass):

  • "GPX holds a name for each feature, so 1 field wasn't written" reads as if the name is the reason the field was dropped. Clearer: "GPX keeps only a name for each feature, so 1 field wasn't written".
  • dbf_notice_name is a dBase helper, and it also truncates at 40 characters. GPX has no such limit, so a GPX notice can show a field name cut short without saying so. Either give GPX its own name helper or drop the truncation for this notice.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both are left for the release quality pass: the wording, and a GPX-specific name helper. The 40-character cap ends a cut name with …, so the cut is visible in the notice.

}
if polygons > 0 {
let what = if polygons == 1 { "1 polygon was".to_string() } else { format!("{polygons} polygons were") };
let left = match holes {
0 => String::new(),
1 => ", and 1 hole left out".to_string(),
n => format!(", and {n} holes left out"),
};
notes.push(format!("{target}: GPX has no areas, so {what} written as tracks along their outer edges{left}"));
}
if multipoints > 0 {
let what = if multipoints == 1 { "1 multipoint was".to_string() } else { format!("{multipoints} multipoints were") };
notes.push(format!("{target}: {what} written as a waypoint for each of its points"));
}
notes
}

/// Whether [`shapefile_parts_in`] writes a feature with geometry `g` to a layer of type `gt`: one
/// of the layer's kind with something drawable (a finite point, a line part of two finite points,
/// a polygon whose outer ring has three).
Expand Down Expand Up @@ -3975,6 +4097,38 @@ mod tests {
assert!(to_csv(&ds).unwrap().contains("longitude"));
}

/// A layer that declares no geometry type gets no coordinate columns, so geometry it holds
/// is counted as not written, as for a line or polygon layer; a point layer's points aren't.
#[test]
fn csv_losses_count_geometry_a_layer_without_a_type_holds() {
let square =
Geometry::Polygon(vec![vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 0.0), Coord::new(1.0, 1.0), Coord::new(0.0, 0.0)]]]);
let mut ds = Dataset::new("t", GeometryType::None, vec![]);
ds.push(Some(square), vec![]);
ds.push(None, vec![]);
let losses = CsvLosses::of(&ds.features, ds.geometry_type);
assert_eq!(losses, CsvLosses { as_one_point: 0, unwritten: 1 });
assert!(losses.notes("t.csv")[0].contains("the geometry of 1 feature wasn't written"));
assert!(!to_csv(&ds).unwrap().contains("longitude"));
let points = [crate::Feature { id: 0, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }];
assert_eq!(CsvLosses::of(&points, GeometryType::Point), CsvLosses::default());
// A multipoint without a point in a point layer gets empty coordinate cells.
let empty = [crate::Feature { id: 1, geometry: Some(Geometry::MultiPoint(vec![])), attrs: vec![] }];
assert_eq!(CsvLosses::of(&empty, GeometryType::Point), CsvLosses { as_one_point: 0, unwritten: 1 });
}

/// GPX leaves out a feature without geometry and a multipoint without a point (no waypoint
/// for it), and counts both; a point is written.
#[test]
fn gpx_counts_a_multipoint_without_points_as_left_out() {
let mut ds = Dataset::new("t", GeometryType::Point, vec![]);
ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![]);
ds.push(Some(Geometry::MultiPoint(vec![])), vec![]);
ds.push(None, vec![]);
assert_eq!(gpx_left_out(&ds), 2);
assert_eq!(write_bytes(&ds, "gpx").unwrap().1, 2);
}

#[test]
fn csv_leading_zeros_and_guard_roundtrip() {
let (ds, _) = read_csv("zip,n,note\n02134,0,-abc\n10001,0.5,=x\n", "t").unwrap();
Expand Down
17 changes: 14 additions & 3 deletions crates/engine/src/cmd/share.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ pub fn commands() -> Vec<C> {
dbf_notes.extend(rustgis_data::source::parquet::values_note(&t.dataset, path));
}
dbf_notes.extend(rustgis_data::io::dates_as_text_note(&t.dataset.fields, path));
// CSV holds a point a row, GPX a name for each feature and no areas.
let name_field = rustgis_data::io::name_field(&t.dataset);
dbf_notes.extend(rustgis_data::io::export_losses_notes(&t.dataset, &ext, name_field, path));
let (left_out, table) =
rustgis_data::io::write_path_to(&t.dataset, Path::new(path), Crs::LonLat, &gpkg).map_err(|e| e.to_string())?;
s.check_continuation(&owner).map_err(|why| super::exports::completion_refused(Some(path), why))?;
Expand Down Expand Up @@ -666,20 +669,21 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option<Crs>, gpkg: Gpk
return Err(format!("{path}: {e}"));
}
let parquet = matches!(ext.as_str(), "parquet" | "geoparquet");
let (written, left_out, skipped, table, unfit, null_numbers, moved) = match &chunks {
let (written, left_out, skipped, table, unfit, null_numbers, moved, csv) = match &chunks {
Some((src, layer)) => {
let (e, table) =
rustgis_data::export::write_source_to(src.as_ref(), layer, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?;
// Counted as the chunks were written, as the layer holds none of them.
let unfit = rustgis_data::source::parquet::unfit_note(&layer.fields, &e.unfit, path);
let moved = rustgis_data::web_mercator_moved_note(path, &layer.name, e.beyond_web_mercator);
(e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved)
(e.written, e.left_out, e.skipped, table, unfit, e.null_numbers, moved, e.csv)
}
None => {
let (left_out, table) = rustgis_data::io::write_path_to(ds, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?;
let unfit = if parquet { rustgis_data::source::parquet::values_note(ds, path) } else { None };
let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, &ext);
(ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None)
let csv = if ext == "csv" { rustgis_data::io::CsvLosses::of(&ds.features, ds.geometry_type) } else { Default::default() };
(ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers, None, csv)
}
};
// The features written give their own extent. A streamed layer written a chunk at a time has
Expand All @@ -702,6 +706,13 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option<Crs>, gpkg: Gpk
// geometry it can draw, of the layer's kind).
let mut said = Vec::new();
said.extend(rustgis_data::io::left_out_note(path, &ext, left_out));
// CSV holds a point a row, GPX a name for each feature and no areas.
if ext == "csv" {
said.extend(csv.notes(path));
}
if ext == "gpx" {
said.extend(rustgis_data::io::gpx_notes(ds, rustgis_data::io::name_field(ds), path));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gap: Export Table to CSV or GPX still drops silently (medium-high confidence). share.exportTable (lines 34-68) writes a table with write_path_to and notifies only left_out_note, export_notes and the dBase notes. It never calls CsvLosses::of(..).notes or gpx_notes. The comment at line 44 says tables can keep geometry, so a table with geometry exported to CSV loses it with no notice, and a GPX export loses fields and polygon holes. Add the same two dispatches there.

The same omission is in the browser's ui.exportLayer branch (crates/ui-egui/src/lib.rs around line 4125), which has no CSV or GPX notices either.

The dispatch is now in three places (write_layer, convert, and the two missing ones). A shared helper in io, such as export_losses_notes(ds, ext, name_field, target) -> Vec<String>, would cover all of them. There are no tests for either path yet. Please add one that exports a table with geometry to CSV and GPX, and a clean table that gets no notice.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 2ed0b33. The new io::export_losses_notes(ds, ext, name_field, target) gives the CSV and GPX notices for a whole dataset. Export Table and the web app's ui.exportLayer download now call it, and so does data.convert. share.exportLayer keeps its chunk-by-chunk CSV count for streamed exports. csv_and_gpx_exports_say_what_they_leave_out now exports a table holding a polygon to CSV and GPX through Export Table, and checks that a table of points gets no notice. cargo xtask ci passed.

}
if skipped > 0 {
said.push(format!("{path}: {skipped} rows couldn't be read and were left out"));
}
Expand Down
15 changes: 12 additions & 3 deletions crates/engine/src/cmd/workflows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1590,8 +1590,8 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult {
}

/// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AGENTS.md, "Never hold the window" (medium confidence, partly pre-existing). data.convert is not in runs_in_background, and the Data Interop pane calls it with app.session.run(...) on the UI thread (crates/ui-egui/src/analysispane.rs:883). Its work grows with the layer, and this doc comment doesn't say why it must stay on the UI thread. This PR edits this doc and adds three whole-layer passes to the command: CsvLosses::of, gpx_left_out and gpx_notes. None of them calls progress::step, so none looks at Cancel. These passes are cheap next to the text serialization, so the practical cost is small, but the rule still applies. Either route the command through a background path, or state the UI-thread reason here and note that the new passes don't check Cancel.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This predates the PR. On main, data.convert already serializes the whole layer synchronously on the UI thread, with no Cancel check. The three counting passes this PR adds are cheap next to that serialization. Moving Data Interop onto a background path (workflows::runs_in_background, with progress and Cancel) is its own change to a different command path, so I'm leaving it out of this PR and noting it as a follow-up rather than writing a UI-thread justification that isn't true.

/// the layer's Z or M values were dropped on import, or its positions are approximate or aren't
/// longitude/latitude.
/// the layer's Z or M values were dropped on import, its positions are approximate or aren't
/// longitude/latitude, or CSV, KML or GPX couldn't hold some of it.
fn convert(s: &mut Session, p: &Value) -> CmdResult {
let ds = feature_ds(s, p)?;
let name_field =
Expand All @@ -1605,7 +1605,16 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult {
f => return Err(format!("unknown format '{f}'")),
};
// Every format here is longitude/latitude.
let notes = rustgis_data::export_notes([(ds, ds.extent())], ps(p, "path").unwrap_or(&ds.name), rustgis_data::raster::Crs::LonLat);
let target = ps(p, "path").unwrap_or(&ds.name);
let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat);
// CSV holds a point a row, GPX a name for each feature and no areas.
notes.extend(rustgis_data::io::export_losses_notes(ds, ps(p, "format").unwrap_or(""), name_field, target));
// KML and GPX hold features with geometry only.
if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") {
let without =
if ext == "gpx" { rustgis_data::io::gpx_left_out(ds) } else { ds.features.iter().filter(|f| f.geometry.is_none()).count() };
notes.extend(rustgis_data::io::left_out_note(target, ext, without));
}
let out = match ps(p, "path") {
Some(path) => {
rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?;
Expand Down
Loading
Loading