diff --git a/crates/data/src/interop.rs b/crates/data/src/interop.rs index f25f8bd12..a2a97b680 100644 --- a/crates/data/src/interop.rs +++ b/crates/data/src/interop.rs @@ -8,13 +8,30 @@ use crate::{Dataset, Geometry}; /// Text for XML: special characters escaped, control characters (invalid in XML 1.0) dropped. fn xml(s: &str) -> String { - s.chars() - .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')) - .collect::() - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) + let clean: String = s.chars().filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')).collect(); + escaped_xml(&clean) +} + +fn escaped_xml(s: &str) -> String { + s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\r', " ") +} + +/// XML 1.0 Char production: https://www.w3.org/TR/xml/#charsets. +/// A description is kept exactly or refused, never cleaned without reporting its loss. +fn description_xml(text: &str, layer: &str, feature: u64) -> crate::Result { + let mut invalid = text.chars().filter(|c| { + !matches!(*c, + '\t' | '\n' | '\r' | '\u{20}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' | '\u{10000}'..='\u{10ffff}' + ) + }); + if let Some(first) = invalid.next() { + let count = invalid.count().saturating_add(1); + return Err(crate::DataError::Format(format!( + "KML: {layer}, feature id {feature}, description: {count} character(s) cannot be represented in XML 1.0 (first U+{:04X})", + first as u32 + ))); + } + Ok(escaped_xml(text)) } fn coords(pts: &[Coord]) -> String { @@ -62,19 +79,34 @@ pub fn to_kml(ds: &Dataset, name_field: Option) -> crate::Result "\n{}\n", xml(&ds.name) ); + // KML's description is text. Other types and differently-cased columns stay in + // ExtendedData, where the reader preserves their values and reports name collisions. + let description = ds.fields.iter().position(|f| f.name == "description" && f.ty == crate::FieldType::Text); for (r, f) in ds.features.iter().enumerate() { crate::progress::step(1)?; let Some(g) = &f.geometry else { continue }; let name = name_field.map(|fi| ds.value(r, fi).to_file_text()).unwrap_or_else(|| f.id.to_string()); + let description = match description.map(|fi| ds.value(r, fi)) { + Some(crate::Value::Text(text)) => format!("{}", description_xml(text, &ds.name, f.id)?), + _ => String::new(), + }; let data: String = ds .fields .iter() .zip(&f.attrs) .filter(|(_, v)| !v.is_null()) - .map(|(fd, v)| format!("{}", xml(&fd.name), xml(&v.to_file_text()))) - .collect(); + .map(|(fd, v)| { + let value = match v { + crate::Value::Text(text) if fd.name == "description" && fd.ty == crate::FieldType::Text => { + description_xml(text, &ds.name, f.id)? + } + _ => xml(&v.to_file_text()), + }; + Ok(format!("{value}", xml(&fd.name))) + }) + .collect::>()?; out.push_str(&format!( - "{}{data}{}\n", + "{}{description}{data}{}\n", xml(&name), kml_geometry(g) )); diff --git a/crates/data/src/kml.rs b/crates/data/src/kml.rs index d791b263f..cf0af379e 100644 --- a/crates/data/src/kml.rs +++ b/crates/data/src/kml.rs @@ -201,9 +201,18 @@ fn child<'a, 'i>(n: &Node<'a, 'i>, name: &str) -> Option> { n.children().find(|c| c.is_element() && local(c) == name) } +/// The text and CDATA children joined, keeping description whitespace as data. +fn raw_text_of(n: &Node) -> String { + n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect() +} + /// The text of an element (its text and CDATA children joined), trimmed. fn text_of(n: &Node) -> String { - n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect::().trim().to_string() + raw_text_of(n).trim().to_string() +} + +fn data_text(n: &Node, key: &str) -> String { + if key == "description" { raw_text_of(n) } else { text_of(n) } } fn child_text(n: &Node, name: &str) -> Option { @@ -436,8 +445,10 @@ impl Reader { if let Some(ext) = child(pm, "ExtendedData") { for d in ext.descendants().filter(|d| d.is_element()) { let pair = match local(&d) { - "Data" => d.attribute("name").map(|k| (k.to_string(), child_text(&d, "value").unwrap_or_default())), - "SimpleData" => d.attribute("name").map(|k| (k.to_string(), text_of(&d))), + "Data" => { + d.attribute("name").map(|k| (k.to_string(), child(&d, "value").map(|v| data_text(&v, k)).unwrap_or_default())) + } + "SimpleData" => d.attribute("name").map(|k| (k.to_string(), data_text(&d, k))), _ => None, }; if let Some((k, v)) = pair.filter(|(k, _)| !k.trim().is_empty()) { @@ -463,7 +474,7 @@ impl Reader { let style = self.style_for(pm); let folder = folders.iter().filter(|f| !f.is_empty()).cloned().collect::>().join(" / "); let name = child_text(pm, "name"); - let description = child_text(pm, "description"); + let description = child(pm, "description").map(|n| raw_text_of(&n)); // A placemark mixing kinds (a MultiGeometry of a point and a line) gives one feature per // kind, each in its layer. let mut any = false; @@ -529,7 +540,7 @@ impl Reader { }; r.data.iter().find(|(dk, _)| dk == k).is_none_or(|(_, v)| v == fixed.unwrap_or_default()) }; - if self.rows.iter().all(repeats) { + if (i != 1 || k == "description") && self.rows.iter().all(repeats) { continue; } let column = crate::io::free_column_name(&taken, k); @@ -751,6 +762,141 @@ mod tests { assert_eq!(polys.styles[0].1.fill, Some([0, 170, 0, 255])); } + #[test] + fn exported_text_descriptions_keep_their_column_on_repeated_kml_and_kmz_round_trips() { + let descriptions = [ + Some("Café & \"river\"\nSecond line"), + Some("]]> & text"), + Some("a\rb"), + Some("a\r\nb"), + Some("a\u{7f}b\u{85}c\u{9f}"), + Some(" padded \n"), + Some("\t \r\n "), + Some(""), + None, + ]; + for ext in ["kml", "kmz"] { + let mut ds = Dataset::new( + "places", + GeometryType::Point, + vec![Field::new("name", FieldType::Text), Field::new("description", FieldType::Text)], + ); + for (row, description) in descriptions.iter().enumerate() { + ds.push( + Some(Geometry::Point(Coord::new(row as f64, 2.0))), + vec![Value::Text(format!("Place {row}")), description.map(|v| Value::Text(v.into())).unwrap_or_default()], + ); + } + for _ in 0..3 { + let xml = crate::interop::to_kml(&ds, Some(0)).unwrap(); + let doc = roxmltree::Document::parse(&xml).unwrap(); + let placemarks: Vec<_> = + doc.descendants().filter(|n| n.has_tag_name(("http://www.opengis.net/kml/2.2", "Placemark"))).collect(); + for (pm, expected) in placemarks.iter().zip(descriptions) { + assert_eq!(child(pm, "description").map(|n| raw_text_of(&n)).as_deref(), expected); + } + let (bytes, skipped) = crate::io::write_bytes(&ds, ext).unwrap(); + assert_eq!(skipped, 0); + let layers = read(&format!("places.{ext}"), &bytes).unwrap(); + assert_eq!(layers.len(), 1); + let layer = &layers[0]; + assert!(layer.report.messages.is_empty(), "{ext}: {:?}", layer.report.messages); + let back = &layer.dataset; + assert!(back.field_index("description_1").is_none()); + let fi = back.field_index("description").unwrap(); + for (row, expected) in descriptions.iter().enumerate() { + let expected = expected.map(|v| Value::Text(v.into())).unwrap_or_default(); + assert_eq!(back.value(row, fi), &expected, "{ext}, row {row}"); + } + ds = back.clone(); + } + } + } + + #[test] + fn unrepresentable_description_characters_are_refused_with_their_count_and_location() { + for bad in ['\0', '\u{b}', '\u{1b}', '\u{1f}', '\u{fffe}', '\u{ffff}'] { + let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("description", FieldType::Text)]); + ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text(format!("a{bad}b{bad}c"))]); + let error = crate::interop::to_kml(&ds, None).unwrap_err().to_string(); + assert!(error.contains("places") && error.contains(&format!("feature id {}", ds.features[0].id)), "{error}"); + assert!(error.contains("description: 2 character(s)") && error.contains(&format!("U+{:04X}", bad as u32)), "{error}"); + for ext in ["kml", "kmz"] { + assert_eq!(crate::io::write_bytes(&ds, ext).unwrap_err().to_string(), error); + } + } + } + + #[test] + fn description_copies_keep_whitespace_in_data_and_simple_data() { + for copy in [ + " padded \n", + " padded \n", + ] { + let xml = format!( + " padded \n{copy}1,2" + ); + let layers = read("spaces.kml", xml.as_bytes()).unwrap(); + let layer = &layers[0]; + assert!(layer.report.messages.is_empty(), "{:?}", layer.report.messages); + let ds = &layer.dataset; + assert_eq!(ds.value(0, ds.field_index("description").unwrap()), &Value::Text(" padded \r\n".into())); + assert!(ds.field_index("description_1").is_none()); + } + } + + #[test] + fn exported_case_variant_descriptions_stay_distinct_from_the_canonical_column() { + for variant in ["Description", "DESCRIPTION"] { + for other in ["Keep case", "Another value"] { + for ext in ["kml", "kmz"] { + let mut ds = Dataset::new( + "places", + GeometryType::Point, + vec![ + Field::new("name", FieldType::Text), + Field::new("description", FieldType::Text), + Field::new(variant, FieldType::Text), + ], + ); + ds.push( + Some(Geometry::Point(Coord::new(1.0, 2.0))), + vec![Value::Text("Place".into()), Value::Text("Keep case".into()), Value::Text(other.into())], + ); + let (bytes, _) = crate::io::write_bytes(&ds, ext).unwrap(); + let layers = read(&format!("places.{ext}"), &bytes).unwrap(); + let layer = &layers[0]; + let back = &layer.dataset; + assert_eq!(back.value(0, back.field_index("description").unwrap()), &Value::Text("Keep case".into())); + assert_eq!(back.value(0, back.field_index(&format!("{variant}_1")).unwrap()), &Value::Text(other.into())); + assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); + assert!(layer.report.messages[0].contains(&format!("'{variant}' → '{variant}_1'")), "{:?}", layer.report.messages); + } + } + } + } + + #[test] + fn exported_other_reserved_columns_still_report_their_collisions() { + for (name, ty, value) in [ + ("description", FieldType::Double, Value::Float(2.5)), + ("Description", FieldType::Text, Value::Text("Keep case".into())), + ("folder", FieldType::Text, Value::Text("My folder".into())), + ("style", FieldType::Text, Value::Text("My style".into())), + ] { + let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new(name, ty)]); + ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![value.clone()]); + let xml = crate::interop::to_kml(&ds, None).unwrap(); + assert!(!xml.contains(""), "{xml}"); + let layers = read("places.kml", xml.as_bytes()).unwrap(); + let layer = &layers[0]; + let fi = layer.dataset.field_index(&format!("{name}_1")).unwrap(); + assert_eq!(layer.dataset.value(0, fi), &value); + assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); + assert!(layer.report.messages[0].contains("renamed"), "{:?}", layer.report.messages); + } + } + #[test] fn extended_data_named_like_a_fixed_column_is_kept_under_a_free_name() { // `name` becomes the placemark's name and, like every field, extended data too; `NAME` diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 1d0136dc1..e2209c926 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -22,7 +22,7 @@ export names its feature. | Delimited text | `.csv`, `.tsv`, `.txt`, recognised `.dat`; UTF-8 `.csv.gz`, `.tsv.gz`, `.txt.gz`, `.dat.gz` | Add Data, or Home › Add › Add Data › Delimited Text… to choose the columns from the file's header | Comma, semicolon, tab or pipe separated (found from the header, or `"delimiter"`); `info.fileColumns` lists the columns and the ones found. Points from x/y columns (named like longitude/latitude, GTFS `stop_lon`/`stop_lat` or easting/northing, or chosen with `"x"` and `"y"`), or any geometry from a WKT/EWKT column (named like `wkt` or `geometry`, or `"wkt"`). When an inferred pair exceeds degree ranges, a recognised longitude/latitude pair is preferred only if its in-range, non-0,0 points cover at least as many rows as the first pair can place, unless a projected `"crs"` is given (see [Delimited Text](data.md) for aliases and sampling). Coordinates in another system are converted from `"crs"` (`EPSG:2263`, or WKT) or an EWKT `SRID=`; rows still read outside longitude/latitude are counted in a notice. Points placed at exactly 0,0 are kept and counted in a notice, as they may represent missing coordinates. Uncompressed text is UTF-8, or UTF-16 with its byte-order mark; other uncompressed files are read as Windows-1252 (Latin-1, as many public sources still publish CSV), with a notice saying so. Gzip input is UTF-8 only, streamed within 256 MiB of expanded text; decompress other encodings and add as `.txt`. Retained CSV rows and cells have a separate estimated memory budget (256 MiB in the browser; a quarter of physical memory between 256 MiB and 4 GiB on desktop). A file without coordinates opens as a standalone table. | | Shapefile | `.shp` with `.dbf` (and `.shx`), or zipped (`.zip`) | Add Data or drop the file; in the web app, pick or drop the `.shp` and `.dbf` together, with the `.shx` and `.prj` if there are (they open as one layer), or the zip | A zip may hold several Shapefiles, in folders, and other vector files (GeoJSON, CSV, GeoParquet, FlatGeobuf); each becomes a layer or standalone table, at most 32 layers and tables together for the whole archive (Shapefiles first, so other files are left out when Shapefiles fill it). A `.dbf` without a `.shp` of the same name in its folder (such as a flood map's `L_` tables) opens as a standalone table, with its `.cpg` when present. An orphan MapInfo `.mid` without its `.mif` is skipped and named in the archive's notice. Delimited `.txt` members (including GTFS tables) are read; unrecognised text members and single-column `README.txt` documentation are skipped with a notice and macOS `__MACOSX` entries are passed over, and a member that can't be read is skipped and named in a notice. Converted to longitude/latitude from the coordinate system in its `.prj` (EPSG, Esri or OGC WKT, or WKT2 as PROJ and QGIS write it; Web Mercator, UTM, State Plane and other common projections); without a `.prj` it must be in longitude/latitude. On the desktop, one with more than 100,000 shapes (counted from its `.shx`) is read in place, a view at a time. Numeric (`N`) and floating point (`F`) fields are read at full double precision; on whole-file reads, `nan` or `inf` becomes empty and is counted in a notice. Date (`D`) fields are read as dates; a blank or zero date is empty, and one that isn't a date (`20260230`) is empty and counted in a notice. | | dBASE table | `.dbf`, with `.cpg` when present | Add Data or drop the file; in the browser, pick or drop the `.dbf` and `.cpg` together, or zip them | Without a `.shp` beside it, opens as an editable standalone table. Uses the same code page, field types and loss notices as Shapefile attributes (including integer, double, Boolean and date fields). On the desktop, picking a `.dbf` with a `.shp` beside it opens that Shapefile instead; in the browser pick its parts together. ZIP tables share the archive's 32-layer/table budget. | -| KML / KMZ | `.kml`, `.kmz` | Add Data, drop the file, or upload it in the web app | A layer for each kind of geometry (points, lines, polygons), with each placemark's name, description, folder path and extended data (a key named like the `name`, `description`, `folder` or `style` column, ignoring case, gets a column of its own, such as `NAME_1`, unless it only repeats that column). Styles are kept: the layer is drawn by unique values of its `style` field, with each KML style's line and fill colours, widths and icon colours (up to 64 styles). Multi-geometries and `gx:Track`s are read. Ground overlays (pictures on a `LatLonBox`, turned by its `rotation`) are raster layers kept in the project: the picture inside a KMZ, or a file beside a KML (not a web address, nor outside its folder); a `gx:LatLonQuad` isn't drawn. | +| KML / KMZ | `.kml`, `.kmz` | Add Data, drop the file, or upload it in the web app | A layer for each kind of geometry (points, lines, polygons), with each placemark's name, description, folder path and extended data (a key named like the `name`, `description`, `folder` or `style` column, ignoring case, gets a column of its own, such as `NAME_1`, unless it only repeats that column). For descriptions, only the exact `description` key is treated as a repeated copy; case variants keep a separate column and report the rename. Description text, including surrounding whitespace, is kept as written; an empty description is empty text and an absent one is null. Styles are kept: the layer is drawn by unique values of its `style` field, with each KML style's line and fill colours, widths and icon colours (up to 64 styles). Multi-geometries and `gx:Track`s are read. Ground overlays (pictures on a `LatLonBox`, turned by its `rotation`) are raster layers kept in the project: the picture inside a KMZ, or a file beside a KML (not a web address, nor outside its folder); a `gx:LatLonQuad` isn't drawn. | | GPX | `.gpx` | Add Data, drop the file, or upload it in the web app | Waypoints, routes, tracks (one line per track, its segments as parts) and track points (with elevation and time) as separate layers. | | Spreadsheets | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.ods` | Add Data (or Delimited Text… to choose the columns), drop the file, or upload it in the web app | One sheet at a time, read like delimited text: the first non-empty row names the columns, and x/y or WKT columns (found, or given with `"x"`, `"y"`, `"wkt"` and `"crs"`) make the geometry; a sheet without them is a table. The first sheet is added and the others named in a notice; `"layer"` picks one, and `info.fileLayers` lists them. Dates read as ISO 8601. | | GeoRSS | `.rss`, `.atom`, `.georss`, or `.xml` | Add Data, drop the file, upload, or From URL… | RSS 2.0, RSS 1.0 and Atom feeds with GeoRSS Simple, GeoRSS GML or W3C Basic Geo places: points, lines and polygons layers with each item's title, link, description, date, id and category. | @@ -637,7 +637,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | GeoParquet (`.parquet`, `.geoparquet`) | Data › Export | GeoParquet 1.1: WKB geometry in WGS 84, Zstd level 9 compression, row groups of up to 65,536 features and a `bbox` covering column with byte-stream-split encoding. The covering keeps spatial range reads available; compression favours smaller files over export speed. Reading the file needs Zstd support, and reading its `bbox` column needs byte-stream-split support, which current GDAL, PyArrow and DuckDB have and some older readers lack (some of those refuse the file). A standalone table without geometry writes plain Parquet. | | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index. Features are written in spatial-index order, so record order can change; keep an attribute key to match records after export. Features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | -| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; features without geometry are left out. | +| KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; KML/KMZ writes a text field named exactly `description` as the placemark description, keeping its text and column name on re-import. A description containing characters XML 1.0 cannot represent is refused with a count and feature location. KML/KMZ also write all attributes to extended data (a null value is not written, so a field that is null for every feature is not recreated on re-import), and reserved-name collisions are reported when read. GPX writes only the name and geometry, so other attributes are not kept. Features without geometry are left out. | | GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. |