From dc209014a933242b9c1bb96a0b8268757ed07f32 Mon Sep 17 00:00:00 2001 From: Li-Sheng Chou <142213184+choupeanut@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:04:49 +0800 Subject: [PATCH 1/4] fix(data): preserve descriptions across KML round trips --- crates/assistant/src/docs.rs | 1 + crates/data/src/interop.rs | 11 +++++- crates/data/src/kml.rs | 62 +++++++++++++++++++++++++++++++++ docs/user-guide/data-formats.md | 2 +- 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/crates/assistant/src/docs.rs b/crates/assistant/src/docs.rs index b05e0b8f4e..5e4b55fd1a 100644 --- a/crates/assistant/src/docs.rs +++ b/crates/assistant/src/docs.rs @@ -41,6 +41,7 @@ pub const PAGES: &[Page] = pages![ "user-guide/assistant.md", "comparison/index.md", "comparison/geolibre.md", + "geolibre-parity.md", "comparison/qgis.md", "cli.md", "python.md", diff --git a/crates/data/src/interop.rs b/crates/data/src/interop.rs index f25f8bd124..d2f84e7291 100644 --- a/crates/data/src/interop.rs +++ b/crates/data/src/interop.rs @@ -62,10 +62,19 @@ pub fn to_kml(ds: &Dataset, name_field: Option) -> crate::Result "\n{}\n", xml(&ds.name) ); + // KML's description is text. Other types and differently-cased columns stay in + // ExtendedData, where the reader preserves their values and reports name collisions. + let description = ds.fields.iter().position(|f| f.name == "description" && f.ty == crate::FieldType::Text); for (r, f) in ds.features.iter().enumerate() { crate::progress::step(1)?; let Some(g) = &f.geometry else { continue }; let name = name_field.map(|fi| ds.value(r, fi).to_file_text()).unwrap_or_else(|| f.id.to_string()); + let description = description + .and_then(|fi| match ds.value(r, fi) { + crate::Value::Text(text) => Some(format!("{}", xml(text))), + _ => None, + }) + .unwrap_or_default(); let data: String = ds .fields .iter() @@ -74,7 +83,7 @@ pub fn to_kml(ds: &Dataset, name_field: Option) -> crate::Result .map(|(fd, v)| format!("{}", xml(&fd.name), xml(&v.to_file_text()))) .collect(); out.push_str(&format!( - "{}{data}{}\n", + "{}{description}{data}{}\n", xml(&name), kml_geometry(g) )); diff --git a/crates/data/src/kml.rs b/crates/data/src/kml.rs index d791b263f8..c59faddc90 100644 --- a/crates/data/src/kml.rs +++ b/crates/data/src/kml.rs @@ -751,6 +751,68 @@ mod tests { assert_eq!(polys.styles[0].1.fill, Some([0, 170, 0, 255])); } + #[test] + fn exported_text_descriptions_keep_their_column_on_repeated_kml_and_kmz_round_trips() { + let descriptions = [Some("Café & \"river\"\nSecond line"), Some("]]> & text"), None]; + for ext in ["kml", "kmz"] { + let mut ds = Dataset::new( + "places", + GeometryType::Point, + vec![Field::new("name", FieldType::Text), Field::new("description", FieldType::Text)], + ); + for (row, description) in descriptions.iter().enumerate() { + ds.push( + Some(Geometry::Point(Coord::new(row as f64, 2.0))), + vec![Value::Text(format!("Place {row}")), description.map(|v| Value::Text(v.into())).unwrap_or_default()], + ); + } + for _ in 0..3 { + let xml = crate::interop::to_kml(&ds, Some(0)).unwrap(); + let doc = roxmltree::Document::parse(&xml).unwrap(); + let placemarks: Vec<_> = + doc.descendants().filter(|n| n.has_tag_name(("http://www.opengis.net/kml/2.2", "Placemark"))).collect(); + for (pm, expected) in placemarks.iter().zip(descriptions) { + assert_eq!(child_text(pm, "description").as_deref(), expected); + } + let (bytes, skipped) = crate::io::write_bytes(&ds, ext).unwrap(); + assert_eq!(skipped, 0); + let layers = read(&format!("places.{ext}"), &bytes).unwrap(); + assert_eq!(layers.len(), 1); + let layer = &layers[0]; + assert!(layer.report.messages.is_empty(), "{ext}: {:?}", layer.report.messages); + let back = &layer.dataset; + assert!(back.field_index("description_1").is_none()); + let fi = back.field_index("description").unwrap(); + for (row, expected) in descriptions.iter().enumerate() { + let expected = expected.map(|v| Value::Text(v.into())).unwrap_or_default(); + assert_eq!(back.value(row, fi), &expected, "{ext}, row {row}"); + } + ds = back.clone(); + } + } + } + + #[test] + fn exported_other_reserved_columns_still_report_their_collisions() { + for (name, ty, value) in [ + ("description", FieldType::Double, Value::Float(2.5)), + ("Description", FieldType::Text, Value::Text("Keep case".into())), + ("folder", FieldType::Text, Value::Text("My folder".into())), + ("style", FieldType::Text, Value::Text("My style".into())), + ] { + let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new(name, ty)]); + ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![value.clone()]); + let xml = crate::interop::to_kml(&ds, None).unwrap(); + assert!(!xml.contains(""), "{xml}"); + let layers = read("places.kml", xml.as_bytes()).unwrap(); + let layer = &layers[0]; + let fi = layer.dataset.field_index(&format!("{name}_1")).unwrap(); + assert_eq!(layer.dataset.value(0, fi), &value); + assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); + assert!(layer.report.messages[0].contains("renamed"), "{:?}", layer.report.messages); + } + } + #[test] fn extended_data_named_like_a_fixed_column_is_kept_under_a_free_name() { // `name` becomes the placemark's name and, like every field, extended data too; `NAME` diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 016008ae9f..0c88a94a22 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -622,7 +622,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index. Features are written in spatial-index order, so record order can change; keep an attribute key to match records after export. Features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | -| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; features without geometry are left out. | +| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its column name on re-import. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | | GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. | From 31320a51242de2bba98c8dfab4a309e9d96e1a0a Mon Sep 17 00:00:00 2001 From: Li-Sheng Chou <142213184+choupeanut@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:36:02 +0800 Subject: [PATCH 2/4] fix(data): retain KML description edge cases --- crates/data/src/interop.rs | 1 + crates/data/src/kml.rs | 40 +++++++++++++++++++++++++++++---- docs/user-guide/data-formats.md | 4 ++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/data/src/interop.rs b/crates/data/src/interop.rs index d2f84e7291..7c2edb2a66 100644 --- a/crates/data/src/interop.rs +++ b/crates/data/src/interop.rs @@ -15,6 +15,7 @@ fn xml(s: &str) -> String { .replace('<', "<") .replace('>', ">") .replace('"', """) + .replace('\r', " ") } fn coords(pts: &[Coord]) -> String { diff --git a/crates/data/src/kml.rs b/crates/data/src/kml.rs index c59faddc90..cce0a2fa41 100644 --- a/crates/data/src/kml.rs +++ b/crates/data/src/kml.rs @@ -463,7 +463,7 @@ impl Reader { let style = self.style_for(pm); let folder = folders.iter().filter(|f| !f.is_empty()).cloned().collect::>().join(" / "); let name = child_text(pm, "name"); - let description = child_text(pm, "description"); + let description = child(pm, "description").map(|n| text_of(&n)); // A placemark mixing kinds (a MultiGeometry of a point and a line) gives one feature per // kind, each in its layer. let mut any = false; @@ -529,7 +529,7 @@ impl Reader { }; r.data.iter().find(|(dk, _)| dk == k).is_none_or(|(_, v)| v == fixed.unwrap_or_default()) }; - if self.rows.iter().all(repeats) { + if (i != 1 || k == "description") && self.rows.iter().all(repeats) { continue; } let column = crate::io::free_column_name(&taken, k); @@ -753,7 +753,8 @@ mod tests { #[test] fn exported_text_descriptions_keep_their_column_on_repeated_kml_and_kmz_round_trips() { - let descriptions = [Some("Café & \"river\"\nSecond line"), Some("]]> & text"), None]; + let descriptions = + [Some("Café & \"river\"\nSecond line"), Some("]]> & text"), Some("a\rb"), Some("a\r\nb"), Some(""), None]; for ext in ["kml", "kmz"] { let mut ds = Dataset::new( "places", @@ -772,7 +773,7 @@ mod tests { let placemarks: Vec<_> = doc.descendants().filter(|n| n.has_tag_name(("http://www.opengis.net/kml/2.2", "Placemark"))).collect(); for (pm, expected) in placemarks.iter().zip(descriptions) { - assert_eq!(child_text(pm, "description").as_deref(), expected); + assert_eq!(child(pm, "description").map(|n| text_of(&n)).as_deref(), expected); } let (bytes, skipped) = crate::io::write_bytes(&ds, ext).unwrap(); assert_eq!(skipped, 0); @@ -792,6 +793,37 @@ mod tests { } } + #[test] + fn exported_case_variant_descriptions_stay_distinct_from_the_canonical_column() { + for variant in ["Description", "DESCRIPTION"] { + for other in ["Keep case", "Another value"] { + for ext in ["kml", "kmz"] { + let mut ds = Dataset::new( + "places", + GeometryType::Point, + vec![ + Field::new("name", FieldType::Text), + Field::new("description", FieldType::Text), + Field::new(variant, FieldType::Text), + ], + ); + ds.push( + Some(Geometry::Point(Coord::new(1.0, 2.0))), + vec![Value::Text("Place".into()), Value::Text("Keep case".into()), Value::Text(other.into())], + ); + let (bytes, _) = crate::io::write_bytes(&ds, ext).unwrap(); + let layers = read(&format!("places.{ext}"), &bytes).unwrap(); + let layer = &layers[0]; + let back = &layer.dataset; + assert_eq!(back.value(0, back.field_index("description").unwrap()), &Value::Text("Keep case".into())); + assert_eq!(back.value(0, back.field_index(&format!("{variant}_1")).unwrap()), &Value::Text(other.into())); + assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); + assert!(layer.report.messages[0].contains(&format!("'{variant}' → '{variant}_1'")), "{:?}", layer.report.messages); + } + } + } + } + #[test] fn exported_other_reserved_columns_still_report_their_collisions() { for (name, ty, value) in [ diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 0c88a94a22..22bbe43afb 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -21,7 +21,7 @@ export names its feature. | GeoJSON | `.geojson`, `.json` | Home › Add › Add Data, or drop the file | Features with typed attributes; a single-member `GeometryCollection` is unwrapped (including nested single-member collections). Empty or multi-member collections are unsupported: the whole feature, including its attributes, is skipped and counted in an unreadable-geometry notice. A FeatureCollection's legacy `crs` member (`{"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::28992"}}`) is honoured for EPSG codes (`EPSG:28992`, `urn:ogc:def:crs:EPSG::28992`, `urn:ogc:def:crs:EPSG:6.6:28992`) and for `CRS84`: a projected or other non-WGS 84 system converts the coordinates to longitude/latitude and the layer keeps it as its coordinate system (Layer properties, export in it, Define Projection); `CRS84` and `EPSG:4326` are longitude/latitude already, so the coordinates stay as they are and the layer has no other coordinate system. Coordinates are x/y (easting/northing, or longitude/latitude) whatever the EPSG axis order. A `crs` that isn't understood (a `link`, WKT or PROJ text, any other name, an unknown code) is named in a notice and the coordinates are read as longitude/latitude. A feature whose coordinates can't be converted, or convert to beyond ±180.5° longitude or ±90.5° latitude (the declared system doesn't fit the numbers), is left out and counted in a notice, and Define Projection then asks for the file to be added again. A feature read as longitude/latitude (no `crs`, `CRS84` or `EPSG:4326`, or a `crs` that isn't understood) with a position whose longitude is within ±180.5° but whose latitude is past ±90.5° is kept, and one notice counts them: longitude and latitude may be swapped. A `crs` on a feature, a geometry or a lone Feature is ignored. GeoJSON is always written in WGS 84 without a `crs`. Desktop FeatureCollections larger than 32 MiB use an in-place reader (which honours `crs` too, reading the file once more to check the features convert); layers with more than 100,000 features stream a view at a time. Other GeoJSON files load whole, up to 256 MiB. | | Delimited text | `.csv`, `.tsv`, `.txt`, recognised `.dat`; UTF-8 `.csv.gz`, `.tsv.gz`, `.txt.gz`, `.dat.gz` | Add Data, or Home › Add › Add Data › Delimited Text… to choose the columns from the file's header | Comma, semicolon, tab or pipe separated (found from the header, or `"delimiter"`); `info.fileColumns` lists the columns and the ones found. Points from x/y columns (named like longitude/latitude, GTFS `stop_lon`/`stop_lat` or easting/northing, or chosen with `"x"` and `"y"`), or any geometry from a WKT/EWKT column (named like `wkt` or `geometry`, or `"wkt"`). When an inferred pair exceeds degree ranges, a recognised longitude/latitude pair is preferred only if its in-range, non-0,0 points cover at least as many rows as the first pair can place, unless a projected `"crs"` is given (see [Delimited Text](data.md) for aliases and sampling). Coordinates in another system are converted from `"crs"` (`EPSG:2263`, or WKT) or an EWKT `SRID=`; rows still read outside longitude/latitude are counted in a notice. Points placed at exactly 0,0 are kept and counted in a notice, as they may represent missing coordinates. Uncompressed text is UTF-8, or UTF-16 with its byte-order mark; other uncompressed files are read as Windows-1252 (Latin-1, as many public sources still publish CSV), with a notice saying so. Gzip input is UTF-8 only, streamed within 256 MiB of expanded text; decompress other encodings and add as `.txt`. Retained CSV rows and cells have a separate estimated memory budget (256 MiB in the browser; a quarter of physical memory between 256 MiB and 4 GiB on desktop). A file without coordinates opens as a standalone table. | | Shapefile | `.shp` with `.dbf` (and `.shx`), or zipped (`.zip`) | Add Data or drop the file; in the web app, pick or drop the `.shp` and `.dbf` together, with the `.shx` and `.prj` if there are (they open as one layer), or the zip | A zip may hold several Shapefiles, in folders, and other vector files (GeoJSON, CSV, GeoParquet, FlatGeobuf); each becomes a layer, at most 32 for the whole archive (Shapefiles first, so other files are left out when Shapefiles fill it). A `.dbf` table without a `.shp` of the same name in its folder (such as a flood map's `L_` tables) isn't read, and a notice names it. Delimited `.txt` members (including GTFS tables) are read; unrecognised text members and single-column `README.txt` documentation are skipped with a notice and macOS `__MACOSX` entries are passed over, and a member that can't be read is skipped and named in a notice. Converted to longitude/latitude from the coordinate system in its `.prj` (EPSG, Esri or OGC WKT, or WKT2 as PROJ and QGIS write it; Web Mercator, UTM, State Plane and other common projections); without a `.prj` it must be in longitude/latitude. On the desktop, one with more than 100,000 shapes (counted from its `.shx`) is read in place, a view at a time. Numeric (`N`) and floating point (`F`) fields are read at full double precision. Date (`D`) fields are read as dates; a blank or zero date is empty, and one that isn't a date (`20260230`) is empty and counted in a notice. | -| KML / KMZ | `.kml`, `.kmz` | Add Data, drop the file, or upload it in the web app | A layer for each kind of geometry (points, lines, polygons), with each placemark's name, description, folder path and extended data (a key named like the `name`, `description`, `folder` or `style` column, ignoring case, gets a column of its own, such as `NAME_1`, unless it only repeats that column). Styles are kept: the layer is drawn by unique values of its `style` field, with each KML style's line and fill colours, widths and icon colours (up to 64 styles). Multi-geometries and `gx:Track`s are read. Ground overlays (pictures on a `LatLonBox`, turned by its `rotation`) are raster layers kept in the project: the picture inside a KMZ, or a file beside a KML (not a web address, nor outside its folder); a `gx:LatLonQuad` isn't drawn. | +| KML / KMZ | `.kml`, `.kmz` | Add Data, drop the file, or upload it in the web app | A layer for each kind of geometry (points, lines, polygons), with each placemark's name, description, folder path and extended data (a key named like the `name`, `description`, `folder` or `style` column, ignoring case, gets a column of its own, such as `NAME_1`, unless it only repeats that column). For descriptions, only the exact `description` key is treated as a repeated copy; case variants keep a separate column and report the rename. Styles are kept: the layer is drawn by unique values of its `style` field, with each KML style's line and fill colours, widths and icon colours (up to 64 styles). Multi-geometries and `gx:Track`s are read. Ground overlays (pictures on a `LatLonBox`, turned by its `rotation`) are raster layers kept in the project: the picture inside a KMZ, or a file beside a KML (not a web address, nor outside its folder); a `gx:LatLonQuad` isn't drawn. | | GPX | `.gpx` | Add Data, drop the file, or upload it in the web app | Waypoints, routes, tracks (one line per track, its segments as parts) and track points (with elevation and time) as separate layers. | | Spreadsheets | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.ods` | Add Data (or Delimited Text… to choose the columns), drop the file, or upload it in the web app | One sheet at a time, read like delimited text: the first non-empty row names the columns, and x/y or WKT columns (found, or given with `"x"`, `"y"`, `"wkt"` and `"crs"`) make the geometry; a sheet without them is a table. The first sheet is added and the others named in a notice; `"layer"` picks one, and `info.fileLayers` lists them. Dates read as ISO 8601. | | GeoRSS | `.rss`, `.atom`, `.georss`, or `.xml` | Add Data, drop the file, upload, or From URL… | RSS 2.0, RSS 1.0 and Atom feeds with GeoRSS Simple, GeoRSS GML or W3C Basic Geo places: points, lines and polygons layers with each item's title, link, description, date, id and category. | @@ -622,7 +622,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index. Features are written in spatial-index order, so record order can change; keep an attribute key to match records after export. Features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | -| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its column name on re-import. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | +| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its column name on re-import. An empty description stays empty text; an absent one is null. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | | GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. | From ba36cd31e543d7eea33ac689080431826203b259 Mon Sep 17 00:00:00 2001 From: Li-Sheng Chou <142213184+choupeanut@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:10:58 +0800 Subject: [PATCH 3/4] fix(data): reject unrepresentable KML descriptions --- crates/data/src/interop.rs | 54 +++++++++++++++++++++++---------- crates/data/src/kml.rs | 15 +++++++++ docs/user-guide/data-formats.md | 2 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/crates/data/src/interop.rs b/crates/data/src/interop.rs index 7c2edb2a66..a2a97b6803 100644 --- a/crates/data/src/interop.rs +++ b/crates/data/src/interop.rs @@ -8,14 +8,30 @@ use crate::{Dataset, Geometry}; /// Text for XML: special characters escaped, control characters (invalid in XML 1.0) dropped. fn xml(s: &str) -> String { - s.chars() - .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')) - .collect::() - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) - .replace('\r', " ") + let clean: String = s.chars().filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')).collect(); + escaped_xml(&clean) +} + +fn escaped_xml(s: &str) -> String { + s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\r', " ") +} + +/// XML 1.0 Char production: https://www.w3.org/TR/xml/#charsets. +/// A description is kept exactly or refused, never cleaned without reporting its loss. +fn description_xml(text: &str, layer: &str, feature: u64) -> crate::Result { + let mut invalid = text.chars().filter(|c| { + !matches!(*c, + '\t' | '\n' | '\r' | '\u{20}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' | '\u{10000}'..='\u{10ffff}' + ) + }); + if let Some(first) = invalid.next() { + let count = invalid.count().saturating_add(1); + return Err(crate::DataError::Format(format!( + "KML: {layer}, feature id {feature}, description: {count} character(s) cannot be represented in XML 1.0 (first U+{:04X})", + first as u32 + ))); + } + Ok(escaped_xml(text)) } fn coords(pts: &[Coord]) -> String { @@ -70,19 +86,25 @@ pub fn to_kml(ds: &Dataset, name_field: Option) -> crate::Result crate::progress::step(1)?; let Some(g) = &f.geometry else { continue }; let name = name_field.map(|fi| ds.value(r, fi).to_file_text()).unwrap_or_else(|| f.id.to_string()); - let description = description - .and_then(|fi| match ds.value(r, fi) { - crate::Value::Text(text) => Some(format!("{}", xml(text))), - _ => None, - }) - .unwrap_or_default(); + let description = match description.map(|fi| ds.value(r, fi)) { + Some(crate::Value::Text(text)) => format!("{}", description_xml(text, &ds.name, f.id)?), + _ => String::new(), + }; let data: String = ds .fields .iter() .zip(&f.attrs) .filter(|(_, v)| !v.is_null()) - .map(|(fd, v)| format!("{}", xml(&fd.name), xml(&v.to_file_text()))) - .collect(); + .map(|(fd, v)| { + let value = match v { + crate::Value::Text(text) if fd.name == "description" && fd.ty == crate::FieldType::Text => { + description_xml(text, &ds.name, f.id)? + } + _ => xml(&v.to_file_text()), + }; + Ok(format!("{value}", xml(&fd.name))) + }) + .collect::>()?; out.push_str(&format!( "{}{description}{data}{}\n", xml(&name), diff --git a/crates/data/src/kml.rs b/crates/data/src/kml.rs index bda895ab43..cf0af379e7 100644 --- a/crates/data/src/kml.rs +++ b/crates/data/src/kml.rs @@ -769,6 +769,7 @@ mod tests { Some("]]> & text"), Some("a\rb"), Some("a\r\nb"), + Some("a\u{7f}b\u{85}c\u{9f}"), Some(" padded \n"), Some("\t \r\n "), Some(""), @@ -812,6 +813,20 @@ mod tests { } } + #[test] + fn unrepresentable_description_characters_are_refused_with_their_count_and_location() { + for bad in ['\0', '\u{b}', '\u{1b}', '\u{1f}', '\u{fffe}', '\u{ffff}'] { + let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("description", FieldType::Text)]); + ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text(format!("a{bad}b{bad}c"))]); + let error = crate::interop::to_kml(&ds, None).unwrap_err().to_string(); + assert!(error.contains("places") && error.contains(&format!("feature id {}", ds.features[0].id)), "{error}"); + assert!(error.contains("description: 2 character(s)") && error.contains(&format!("U+{:04X}", bad as u32)), "{error}"); + for ext in ["kml", "kmz"] { + assert_eq!(crate::io::write_bytes(&ds, ext).unwrap_err().to_string(), error); + } + } + } + #[test] fn description_copies_keep_whitespace_in_data_and_simple_data() { for copy in [ diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 976a725d3c..0bc22efcf5 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -626,7 +626,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index. Features are written in spatial-index order, so record order can change; keep an attribute key to match records after export. Features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | -| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its text and column name on re-import. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | +| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its text and column name on re-import. A description containing characters XML 1.0 cannot represent is refused with a count and feature location. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | | GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. | From f7aee9d8e57edc069147608f233601b78266aaed Mon Sep 17 00:00:00 2001 From: choupeanut Date: Tue, 17 Nov 2026 20:31:25 +0800 Subject: [PATCH 4/4] docs(data): qualify KML extended-data and GPX attribute limits --- docs/user-guide/data-formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index cc20ada593..e2209c926b 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -637,7 +637,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index. Features are written in spatial-index order, so record order can change; keep an attribute key to match records after export. Features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | -| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its text and column name on re-import. A description containing characters XML 1.0 cannot represent is refused with a count and feature location. All attributes also go into extended data; reserved-name collisions are reported when read. Features without geometry are left out. | +| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its text and column name on re-import. A description containing characters XML 1.0 cannot represent is refused with a count and feature location. KML/KMZ also write all attributes to extended data (a null value is not written, so a field that is null for every feature is not recreated on re-import), and reserved-name collisions are reported when read. GPX writes only the name and geometry, so other attributes are not kept. Features without geometry are left out. | | GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. |