Repository navigation
fix(data): preserve descriptions across KML round trips #2174
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
dc20901
31320a5
6e94d8c
ba36cd3
de0f089
f7aee9d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,13 +8,30 @@ use crate::{Dataset, Geometry}; | |
|
|
||
| /// Text for XML: special characters escaped, control characters (invalid in XML 1.0) dropped. | ||
| fn xml(s: &str) -> String { | ||
| s.chars() | ||
| .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')) | ||
| .collect::<String>() | ||
| .replace('&', "&") | ||
| .replace('<', "<") | ||
| .replace('>', ">") | ||
| .replace('"', """) | ||
| let clean: String = s.chars().filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')).collect(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Re-raising the earlier silent-loss concern (still present; this PR rewrites this function). What is new since the earlier thread: the function was edited in this PR and left with the silent filter. AGENTS.md ("never silent", "existing silent losses are bugs") says this belongs in the PR. Suggested direction: make the name, field-name and value paths use the same validating check (returning the layer, feature, field, count and first code point through the existing Confidence: high that the output is malformed; medium-high that this PR should own it. |
||
| escaped_xml(&clean) | ||
| } | ||
|
|
||
| fn escaped_xml(s: &str) -> String { | ||
| s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\r', " ") | ||
| } | ||
|
|
||
| /// XML 1.0 Char production: https://www.w3.org/TR/xml/#charsets. | ||
| /// A description is kept exactly or refused, never cleaned without reporting its loss. | ||
| fn description_xml(text: &str, layer: &str, feature: u64) -> crate::Result<String> { | ||
| let mut invalid = text.chars().filter(|c| { | ||
| !matches!(*c, | ||
| '\t' | '\n' | '\r' | '\u{20}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' | '\u{10000}'..='\u{10ffff}' | ||
| ) | ||
| }); | ||
| if let Some(first) = invalid.next() { | ||
| let count = invalid.count().saturating_add(1); | ||
| return Err(crate::DataError::Format(format!( | ||
| "KML: {layer}, feature id {feature}, description: {count} character(s) cannot be represented in XML 1.0 (first U+{:04X})", | ||
| first as u32 | ||
| ))); | ||
| } | ||
| Ok(escaped_xml(text)) | ||
| } | ||
|
|
||
| fn coords(pts: &[Coord]) -> String { | ||
|
|
@@ -62,19 +79,34 @@ pub fn to_kml(ds: &Dataset, name_field: Option<usize>) -> crate::Result<String> | |
| "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document><name>{}</name>\n", | ||
| xml(&ds.name) | ||
| ); | ||
| // KML's description is text. Other types and differently-cased columns stay in | ||
| // ExtendedData, where the reader preserves their values and reports name collisions. | ||
| let description = ds.fields.iter().position(|f| f.name == "description" && f.ty == crate::FieldType::Text); | ||
| for (r, f) in ds.features.iter().enumerate() { | ||
| crate::progress::step(1)?; | ||
| let Some(g) = &f.geometry else { continue }; | ||
| let name = name_field.map(|fi| ds.value(r, fi).to_file_text()).unwrap_or_else(|| f.id.to_string()); | ||
| let description = match description.map(|fi| ds.value(r, fi)) { | ||
| Some(crate::Value::Text(text)) => format!("<description>{}</description>", description_xml(text, &ds.name, f.id)?), | ||
| _ => String::new(), | ||
| }; | ||
| let data: String = ds | ||
| .fields | ||
| .iter() | ||
| .zip(&f.attrs) | ||
| .filter(|(_, v)| !v.is_null()) | ||
| .map(|(fd, v)| format!("<Data name=\"{}\"><value>{}</value></Data>", xml(&fd.name), xml(&v.to_file_text()))) | ||
| .collect(); | ||
| .map(|(fd, v)| { | ||
| let value = match v { | ||
| crate::Value::Text(text) if fd.name == "description" && fd.ty == crate::FieldType::Text => { | ||
| description_xml(text, &ds.name, f.id)? | ||
| } | ||
| _ => xml(&v.to_file_text()), | ||
| }; | ||
| Ok(format!("<Data name=\"{}\"><value>{value}</value></Data>", xml(&fd.name))) | ||
| }) | ||
| .collect::<crate::Result<String>>()?; | ||
| out.push_str(&format!( | ||
| "<Placemark><name>{}</name><ExtendedData>{data}</ExtendedData>{}</Placemark>\n", | ||
| "<Placemark><name>{}</name>{description}<ExtendedData>{data}</ExtendedData>{}</Placemark>\n", | ||
| xml(&name), | ||
| kml_geometry(g) | ||
| )); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -201,9 +201,18 @@ fn child<'a, 'i>(n: &Node<'a, 'i>, name: &str) -> Option<Node<'a, 'i>> { | |
| n.children().find(|c| c.is_element() && local(c) == name) | ||
| } | ||
|
|
||
| /// The text and CDATA children joined, keeping description whitespace as data. | ||
| fn raw_text_of(n: &Node) -> String { | ||
| n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect() | ||
| } | ||
|
|
||
| /// The text of an element (its text and CDATA children joined), trimmed. | ||
| fn text_of(n: &Node) -> String { | ||
| n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect::<String>().trim().to_string() | ||
| raw_text_of(n).trim().to_string() | ||
| } | ||
|
|
||
| fn data_text(n: &Node, key: &str) -> String { | ||
| if key == "description" { raw_text_of(n) } else { text_of(n) } | ||
| } | ||
|
|
||
| fn child_text(n: &Node, name: &str) -> Option<String> { | ||
|
|
@@ -436,8 +445,10 @@ impl Reader { | |
| if let Some(ext) = child(pm, "ExtendedData") { | ||
| for d in ext.descendants().filter(|d| d.is_element()) { | ||
| let pair = match local(&d) { | ||
| "Data" => d.attribute("name").map(|k| (k.to_string(), child_text(&d, "value").unwrap_or_default())), | ||
| "SimpleData" => d.attribute("name").map(|k| (k.to_string(), text_of(&d))), | ||
| "Data" => { | ||
| d.attribute("name").map(|k| (k.to_string(), child(&d, "value").map(|v| data_text(&v, k)).unwrap_or_default())) | ||
| } | ||
| "SimpleData" => d.attribute("name").map(|k| (k.to_string(), data_text(&d, k))), | ||
| _ => None, | ||
| }; | ||
| if let Some((k, v)) = pair.filter(|(k, _)| !k.trim().is_empty()) { | ||
|
|
@@ -463,7 +474,7 @@ impl Reader { | |
| let style = self.style_for(pm); | ||
| let folder = folders.iter().filter(|f| !f.is_empty()).cloned().collect::<Vec<_>>().join(" / "); | ||
| let name = child_text(pm, "name"); | ||
| let description = child_text(pm, "description"); | ||
| let description = child(pm, "description").map(|n| raw_text_of(&n)); | ||
|
choupeanut marked this conversation as resolved.
Resolved
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Re-raising the third-party whitespace concern with a case the earlier reply does not cover. The earlier reply confirmed that empty and whitespace-only descriptions are intentionally text. The case it leaves open is non-empty content in pretty-printed third-party files, such as: <description>
<![CDATA[Hello]]>
</description>
Suggested: decide explicitly how formatting whitespace from third-party KML should be treated, and add a fixture for a pretty-printed third-party placemark that pins the chosen result. The docs sentence "kept as written" should match that choice. Confidence: medium (about 60%) that this is a regression for existing third-party files; I have not run it.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Re-raising the third-party whitespace concern with a case the earlier reply does not cover (medium confidence, ~60%). The reply confirmed that empty and whitespace-only descriptions are intentional. It does not address non-empty text with formatting indentation. Raw reading now keeps that indentation as data: <Placemark>
<description>
Hello
</description>
<ExtendedData><Data name="description"><value>Hello</value></Data></ExtendedData>
</Placemark>
Existing imports of such files also change value without a notice. Either the formatting whitespace should be dropped for third-party text (for example, trim only when the text is pure indentation around one run), or the change needs a notice and a fixture pinning the chosen result. The docs sentence "kept as written" should match whichever is chosen. |
||
| // A placemark mixing kinds (a MultiGeometry of a point and a line) gives one feature per | ||
| // kind, each in its layer. | ||
| let mut any = false; | ||
|
|
@@ -529,7 +540,7 @@ impl Reader { | |
| }; | ||
| r.data.iter().find(|(dk, _)| dk == k).is_none_or(|(_, v)| v == fixed.unwrap_or_default()) | ||
| }; | ||
| if self.rows.iter().all(repeats) { | ||
| if (i != 1 || k == "description") && self.rows.iter().all(repeats) { | ||
| continue; | ||
| } | ||
| let column = crate::io::free_column_name(&taken, k); | ||
|
|
@@ -751,6 +762,141 @@ mod tests { | |
| assert_eq!(polys.styles[0].1.fill, Some([0, 170, 0, 255])); | ||
| } | ||
|
|
||
| #[test] | ||
| fn exported_text_descriptions_keep_their_column_on_repeated_kml_and_kmz_round_trips() { | ||
| let descriptions = [ | ||
| Some("<b>Café & \"river\"</b>\nSecond line"), | ||
| Some("]]> <plain> & text"), | ||
| Some("a\rb"), | ||
| Some("a\r\nb"), | ||
| Some("a\u{7f}b\u{85}c\u{9f}"), | ||
| Some(" padded \n"), | ||
| Some("\t \r\n "), | ||
| Some(""), | ||
| None, | ||
| ]; | ||
| for ext in ["kml", "kmz"] { | ||
| let mut ds = Dataset::new( | ||
| "places", | ||
| GeometryType::Point, | ||
| vec![Field::new("name", FieldType::Text), Field::new("description", FieldType::Text)], | ||
| ); | ||
| for (row, description) in descriptions.iter().enumerate() { | ||
| ds.push( | ||
| Some(Geometry::Point(Coord::new(row as f64, 2.0))), | ||
| vec![Value::Text(format!("Place {row}")), description.map(|v| Value::Text(v.into())).unwrap_or_default()], | ||
| ); | ||
| } | ||
| for _ in 0..3 { | ||
| let xml = crate::interop::to_kml(&ds, Some(0)).unwrap(); | ||
| let doc = roxmltree::Document::parse(&xml).unwrap(); | ||
| let placemarks: Vec<_> = | ||
| doc.descendants().filter(|n| n.has_tag_name(("http://www.opengis.net/kml/2.2", "Placemark"))).collect(); | ||
| for (pm, expected) in placemarks.iter().zip(descriptions) { | ||
| assert_eq!(child(pm, "description").map(|n| raw_text_of(&n)).as_deref(), expected); | ||
| } | ||
| let (bytes, skipped) = crate::io::write_bytes(&ds, ext).unwrap(); | ||
| assert_eq!(skipped, 0); | ||
| let layers = read(&format!("places.{ext}"), &bytes).unwrap(); | ||
| assert_eq!(layers.len(), 1); | ||
| let layer = &layers[0]; | ||
| assert!(layer.report.messages.is_empty(), "{ext}: {:?}", layer.report.messages); | ||
| let back = &layer.dataset; | ||
| assert!(back.field_index("description_1").is_none()); | ||
| let fi = back.field_index("description").unwrap(); | ||
| for (row, expected) in descriptions.iter().enumerate() { | ||
| let expected = expected.map(|v| Value::Text(v.into())).unwrap_or_default(); | ||
| assert_eq!(back.value(row, fi), &expected, "{ext}, row {row}"); | ||
| } | ||
| ds = back.clone(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn unrepresentable_description_characters_are_refused_with_their_count_and_location() { | ||
| for bad in ['\0', '\u{b}', '\u{1b}', '\u{1f}', '\u{fffe}', '\u{ffff}'] { | ||
| let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("description", FieldType::Text)]); | ||
| ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text(format!("a{bad}b{bad}c"))]); | ||
| let error = crate::interop::to_kml(&ds, None).unwrap_err().to_string(); | ||
| assert!(error.contains("places") && error.contains(&format!("feature id {}", ds.features[0].id)), "{error}"); | ||
| assert!(error.contains("description: 2 character(s)") && error.contains(&format!("U+{:04X}", bad as u32)), "{error}"); | ||
| for ext in ["kml", "kmz"] { | ||
| assert_eq!(crate::io::write_bytes(&ds, ext).unwrap_err().to_string(), error); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn description_copies_keep_whitespace_in_data_and_simple_data() { | ||
| for copy in [ | ||
| "<Data name=\"description\"><value> padded \n</value></Data>", | ||
| "<SchemaData><SimpleData name=\"description\"> padded \n</SimpleData></SchemaData>", | ||
| ] { | ||
| let xml = format!( | ||
| "<kml><Placemark><description> padded \n</description><ExtendedData>{copy}</ExtendedData><Point><coordinates>1,2</coordinates></Point></Placemark></kml>" | ||
| ); | ||
| let layers = read("spaces.kml", xml.as_bytes()).unwrap(); | ||
| let layer = &layers[0]; | ||
| assert!(layer.report.messages.is_empty(), "{:?}", layer.report.messages); | ||
| let ds = &layer.dataset; | ||
| assert_eq!(ds.value(0, ds.field_index("description").unwrap()), &Value::Text(" padded \r\n".into())); | ||
| assert!(ds.field_index("description_1").is_none()); | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn exported_case_variant_descriptions_stay_distinct_from_the_canonical_column() { | ||
| for variant in ["Description", "DESCRIPTION"] { | ||
| for other in ["Keep case", "Another value"] { | ||
| for ext in ["kml", "kmz"] { | ||
| let mut ds = Dataset::new( | ||
| "places", | ||
| GeometryType::Point, | ||
| vec![ | ||
| Field::new("name", FieldType::Text), | ||
| Field::new("description", FieldType::Text), | ||
| Field::new(variant, FieldType::Text), | ||
| ], | ||
| ); | ||
| ds.push( | ||
| Some(Geometry::Point(Coord::new(1.0, 2.0))), | ||
| vec![Value::Text("Place".into()), Value::Text("Keep case".into()), Value::Text(other.into())], | ||
| ); | ||
| let (bytes, _) = crate::io::write_bytes(&ds, ext).unwrap(); | ||
| let layers = read(&format!("places.{ext}"), &bytes).unwrap(); | ||
| let layer = &layers[0]; | ||
| let back = &layer.dataset; | ||
| assert_eq!(back.value(0, back.field_index("description").unwrap()), &Value::Text("Keep case".into())); | ||
| assert_eq!(back.value(0, back.field_index(&format!("{variant}_1")).unwrap()), &Value::Text(other.into())); | ||
| assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); | ||
| assert!(layer.report.messages[0].contains(&format!("'{variant}' → '{variant}_1'")), "{:?}", layer.report.messages); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn exported_other_reserved_columns_still_report_their_collisions() { | ||
| for (name, ty, value) in [ | ||
| ("description", FieldType::Double, Value::Float(2.5)), | ||
| ("Description", FieldType::Text, Value::Text("Keep case".into())), | ||
| ("folder", FieldType::Text, Value::Text("My folder".into())), | ||
| ("style", FieldType::Text, Value::Text("My style".into())), | ||
| ] { | ||
| let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new(name, ty)]); | ||
| ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![value.clone()]); | ||
| let xml = crate::interop::to_kml(&ds, None).unwrap(); | ||
| assert!(!xml.contains("<description>"), "{xml}"); | ||
| let layers = read("places.kml", xml.as_bytes()).unwrap(); | ||
| let layer = &layers[0]; | ||
| let fi = layer.dataset.field_index(&format!("{name}_1")).unwrap(); | ||
| assert_eq!(layer.dataset.value(0, fi), &value); | ||
| assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages); | ||
| assert!(layer.report.messages[0].contains("renamed"), "{:?}", layer.report.messages); | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn extended_data_named_like_a_fixed_column_is_kept_under_a_free_name() { | ||
| // `name` becomes the placemark's name and, like every field, extended data too; `NAME` | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Silent loss and malformed output for non-description text (pre-existing, but in the function this PR rewrites).
xml()still drops everyis_control()character silently, and it does not check the XML 1.0Charproduction, so U+FFFE and U+FFFF pass through unchanged. An attribute value, placemark name or field name containing U+FFFE or U+FFFF is written into a KML file that is not well-formed XML, and re-import fails in the parser. The newdescription_xmlrefuses exactly these characters, so the same input is refused in the description but silently mangled everywhere else. The new tests only cover the description field.Per AGENTS.md ("never silent", "existing silent losses are bugs"), the non-description path should use the same validating check (refuse with count and first code point through the existing error path) and get a test for a U+FFFF attribute value. Confidence: high on the malformed-output path, medium on whether this PR should own the fix or a follow-up should.