Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 42 additions & 10 deletions crates/data/src/interop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,30 @@ use crate::{Dataset, Geometry};

/// Text for XML: special characters escaped, control characters (invalid in XML 1.0) dropped.
fn xml(s: &str) -> String {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Silent loss and malformed output for non-description text (pre-existing, but in the function this PR rewrites). xml() still drops every is_control() character silently, and it does not check the XML 1.0 Char production, so U+FFFE and U+FFFF pass through unchanged. An attribute value, placemark name or field name containing U+FFFE or U+FFFF is written into a KML file that is not well-formed XML, and re-import fails in the parser. The new description_xml refuses exactly these characters, so the same input is refused in the description but silently mangled everywhere else. The new tests only cover the description field.

Per AGENTS.md ("never silent", "existing silent losses are bugs"), the non-description path should use the same validating check (refuse with count and first code point through the existing error path) and get a test for a U+FFFF attribute value. Confidence: high on the malformed-output path, medium on whether this PR should own the fix or a follow-up should.

s.chars()
.filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r'))
.collect::<String>()
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
let clean: String = s.chars().filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')).collect();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re-raising the earlier silent-loss concern (still present; this PR rewrites this function). xml() still drops every is_control() character (C0, DEL and C1) without a notice, and it does not check the XML 1.0 Char production. U+FFFE and U+FFFF pass through into names, field names and non-description values, so the file is not well-formed XML and re-import fails. The new description_xml refuses exactly these characters, so the same input is refused in description but silently mangled everywhere else. The new tests only cover the description path.

What is new since the earlier thread: the function was edited in this PR and left with the silent filter. AGENTS.md ("never silent", "existing silent losses are bugs") says this belongs in the PR. Suggested direction: make the name, field-name and value paths use the same validating check (returning the layer, feature, field, count and first code point through the existing Result path), and add a test that a U+FFFF in a non-description value is refused.

Confidence: high that the output is malformed; medium-high that this PR should own it.

escaped_xml(&clean)
}

fn escaped_xml(s: &str) -> String {
s.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;").replace('"', "&quot;").replace('\r', "&#13;")
}

/// XML 1.0 Char production: https://www.w3.org/TR/xml/#charsets.
/// A description is kept exactly or refused, never cleaned without reporting its loss.
fn description_xml(text: &str, layer: &str, feature: u64) -> crate::Result<String> {
let mut invalid = text.chars().filter(|c| {
!matches!(*c,
'\t' | '\n' | '\r' | '\u{20}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' | '\u{10000}'..='\u{10ffff}'
)
});
if let Some(first) = invalid.next() {
let count = invalid.count().saturating_add(1);
return Err(crate::DataError::Format(format!(
"KML: {layer}, feature id {feature}, description: {count} character(s) cannot be represented in XML 1.0 (first U+{:04X})",
first as u32
)));
}
Ok(escaped_xml(text))
}

fn coords(pts: &[Coord]) -> String {
Expand Down Expand Up @@ -62,19 +79,34 @@ pub fn to_kml(ds: &Dataset, name_field: Option<usize>) -> crate::Result<String>
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document><name>{}</name>\n",
xml(&ds.name)
);
// KML's description is text. Other types and differently-cased columns stay in
// ExtendedData, where the reader preserves their values and reports name collisions.
let description = ds.fields.iter().position(|f| f.name == "description" && f.ty == crate::FieldType::Text);
for (r, f) in ds.features.iter().enumerate() {
crate::progress::step(1)?;
let Some(g) = &f.geometry else { continue };
let name = name_field.map(|fi| ds.value(r, fi).to_file_text()).unwrap_or_else(|| f.id.to_string());
let description = match description.map(|fi| ds.value(r, fi)) {
Some(crate::Value::Text(text)) => format!("<description>{}</description>", description_xml(text, &ds.name, f.id)?),
_ => String::new(),
};
let data: String = ds
.fields
.iter()
.zip(&f.attrs)
.filter(|(_, v)| !v.is_null())
.map(|(fd, v)| format!("<Data name=\"{}\"><value>{}</value></Data>", xml(&fd.name), xml(&v.to_file_text())))
.collect();
.map(|(fd, v)| {
let value = match v {
crate::Value::Text(text) if fd.name == "description" && fd.ty == crate::FieldType::Text => {
description_xml(text, &ds.name, f.id)?
}
_ => xml(&v.to_file_text()),
};
Ok(format!("<Data name=\"{}\"><value>{value}</value></Data>", xml(&fd.name)))
})
.collect::<crate::Result<String>>()?;
out.push_str(&format!(
"<Placemark><name>{}</name><ExtendedData>{data}</ExtendedData>{}</Placemark>\n",
"<Placemark><name>{}</name>{description}<ExtendedData>{data}</ExtendedData>{}</Placemark>\n",
xml(&name),
kml_geometry(g)
));
Expand Down
156 changes: 151 additions & 5 deletions crates/data/src/kml.rs
Original file line number Diff line number Diff line change
Expand Up @@ -201,9 +201,18 @@ fn child<'a, 'i>(n: &Node<'a, 'i>, name: &str) -> Option<Node<'a, 'i>> {
n.children().find(|c| c.is_element() && local(c) == name)
}

/// The text and CDATA children joined, keeping description whitespace as data.
fn raw_text_of(n: &Node) -> String {
n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect()
}

/// The text of an element (its text and CDATA children joined), trimmed.
fn text_of(n: &Node) -> String {
n.children().filter(|c| c.is_text()).filter_map(|c| c.text()).collect::<String>().trim().to_string()
raw_text_of(n).trim().to_string()
}

fn data_text(n: &Node, key: &str) -> String {
if key == "description" { raw_text_of(n) } else { text_of(n) }
}

fn child_text(n: &Node, name: &str) -> Option<String> {
Expand Down Expand Up @@ -436,8 +445,10 @@ impl Reader {
if let Some(ext) = child(pm, "ExtendedData") {
for d in ext.descendants().filter(|d| d.is_element()) {
let pair = match local(&d) {
"Data" => d.attribute("name").map(|k| (k.to_string(), child_text(&d, "value").unwrap_or_default())),
"SimpleData" => d.attribute("name").map(|k| (k.to_string(), text_of(&d))),
"Data" => {
d.attribute("name").map(|k| (k.to_string(), child(&d, "value").map(|v| data_text(&v, k)).unwrap_or_default()))
}
"SimpleData" => d.attribute("name").map(|k| (k.to_string(), data_text(&d, k))),
_ => None,
};
if let Some((k, v)) = pair.filter(|(k, _)| !k.trim().is_empty()) {
Expand All @@ -463,7 +474,7 @@ impl Reader {
let style = self.style_for(pm);
let folder = folders.iter().filter(|f| !f.is_empty()).cloned().collect::<Vec<_>>().join(" / ");
let name = child_text(pm, "name");
let description = child_text(pm, "description");
let description = child(pm, "description").map(|n| raw_text_of(&n));
Comment thread
choupeanut marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re-raising the third-party whitespace concern with a case the earlier reply does not cover. The earlier reply confirmed that empty and whitespace-only descriptions are intentionally text. The case it leaves open is non-empty content in pretty-printed third-party files, such as:

<description>
  <![CDATA[Hello]]>
</description>

raw_text_of now returns "\n Hello\n" where the old text_of returned "Hello". The indentation is XML formatting, not content, so existing imports change without a notice. The same change can also break the repeat check. If a file has a pretty-printed <description> and a trimmed ExtendedData description copy, the two no longer match, so the copy becomes description_1 and a collision notice appears.

Suggested: decide explicitly how formatting whitespace from third-party KML should be treated, and add a fixture for a pretty-printed third-party placemark that pins the chosen result. The docs sentence "kept as written" should match that choice.

Confidence: medium (about 60%) that this is a regression for existing third-party files; I have not run it.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re-raising the third-party whitespace concern with a case the earlier reply does not cover (medium confidence, ~60%).

The reply confirmed that empty and whitespace-only descriptions are intentional. It does not address non-empty text with formatting indentation. Raw reading now keeps that indentation as data:

<Placemark>
  <description>
    Hello
  </description>
  <ExtendedData><Data name="description"><value>Hello</value></Data></ExtendedData>
</Placemark>
  • Before: description reads as "Hello", and the trimmed Data copy matches, so no notice.
  • Now: description reads as "\n Hello\n ". The Data copy is "Hello", so repeats fails, the copy becomes description_1, and a rename notice appears for a file that imported cleanly before.

Existing imports of such files also change value without a notice. Either the formatting whitespace should be dropped for third-party text (for example, trim only when the text is pure indentation around one run), or the change needs a notice and a fixture pinning the chosen result. The docs sentence "kept as written" should match whichever is chosen.

// A placemark mixing kinds (a MultiGeometry of a point and a line) gives one feature per
// kind, each in its layer.
let mut any = false;
Expand Down Expand Up @@ -529,7 +540,7 @@ impl Reader {
};
r.data.iter().find(|(dk, _)| dk == k).is_none_or(|(_, v)| v == fixed.unwrap_or_default())
};
if self.rows.iter().all(repeats) {
if (i != 1 || k == "description") && self.rows.iter().all(repeats) {
continue;
}
let column = crate::io::free_column_name(&taken, k);
Expand Down Expand Up @@ -751,6 +762,141 @@ mod tests {
assert_eq!(polys.styles[0].1.fill, Some([0, 170, 0, 255]));
}

#[test]
fn exported_text_descriptions_keep_their_column_on_repeated_kml_and_kmz_round_trips() {
let descriptions = [
Some("<b>Café & \"river\"</b>\nSecond line"),
Some("]]> <plain> & text"),
Some("a\rb"),
Some("a\r\nb"),
Some("a\u{7f}b\u{85}c\u{9f}"),
Some(" padded \n"),
Some("\t \r\n "),
Some(""),
None,
];
for ext in ["kml", "kmz"] {
let mut ds = Dataset::new(
"places",
GeometryType::Point,
vec![Field::new("name", FieldType::Text), Field::new("description", FieldType::Text)],
);
for (row, description) in descriptions.iter().enumerate() {
ds.push(
Some(Geometry::Point(Coord::new(row as f64, 2.0))),
vec![Value::Text(format!("Place {row}")), description.map(|v| Value::Text(v.into())).unwrap_or_default()],
);
}
for _ in 0..3 {
let xml = crate::interop::to_kml(&ds, Some(0)).unwrap();
let doc = roxmltree::Document::parse(&xml).unwrap();
let placemarks: Vec<_> =
doc.descendants().filter(|n| n.has_tag_name(("http://www.opengis.net/kml/2.2", "Placemark"))).collect();
for (pm, expected) in placemarks.iter().zip(descriptions) {
assert_eq!(child(pm, "description").map(|n| raw_text_of(&n)).as_deref(), expected);
}
let (bytes, skipped) = crate::io::write_bytes(&ds, ext).unwrap();
assert_eq!(skipped, 0);
let layers = read(&format!("places.{ext}"), &bytes).unwrap();
assert_eq!(layers.len(), 1);
let layer = &layers[0];
assert!(layer.report.messages.is_empty(), "{ext}: {:?}", layer.report.messages);
let back = &layer.dataset;
assert!(back.field_index("description_1").is_none());
let fi = back.field_index("description").unwrap();
for (row, expected) in descriptions.iter().enumerate() {
let expected = expected.map(|v| Value::Text(v.into())).unwrap_or_default();
assert_eq!(back.value(row, fi), &expected, "{ext}, row {row}");
}
ds = back.clone();
}
}
}

#[test]
fn unrepresentable_description_characters_are_refused_with_their_count_and_location() {
for bad in ['\0', '\u{b}', '\u{1b}', '\u{1f}', '\u{fffe}', '\u{ffff}'] {
let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("description", FieldType::Text)]);
ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text(format!("a{bad}b{bad}c"))]);
let error = crate::interop::to_kml(&ds, None).unwrap_err().to_string();
assert!(error.contains("places") && error.contains(&format!("feature id {}", ds.features[0].id)), "{error}");
assert!(error.contains("description: 2 character(s)") && error.contains(&format!("U+{:04X}", bad as u32)), "{error}");
for ext in ["kml", "kmz"] {
assert_eq!(crate::io::write_bytes(&ds, ext).unwrap_err().to_string(), error);
}
}
}

#[test]
fn description_copies_keep_whitespace_in_data_and_simple_data() {
for copy in [
"<Data name=\"description\"><value> padded &#13;\n</value></Data>",
"<SchemaData><SimpleData name=\"description\"> padded &#13;\n</SimpleData></SchemaData>",
] {
let xml = format!(
"<kml><Placemark><description> padded &#13;\n</description><ExtendedData>{copy}</ExtendedData><Point><coordinates>1,2</coordinates></Point></Placemark></kml>"
);
let layers = read("spaces.kml", xml.as_bytes()).unwrap();
let layer = &layers[0];
assert!(layer.report.messages.is_empty(), "{:?}", layer.report.messages);
let ds = &layer.dataset;
assert_eq!(ds.value(0, ds.field_index("description").unwrap()), &Value::Text(" padded \r\n".into()));
assert!(ds.field_index("description_1").is_none());
}
}

#[test]
fn exported_case_variant_descriptions_stay_distinct_from_the_canonical_column() {
for variant in ["Description", "DESCRIPTION"] {
for other in ["Keep case", "Another value"] {
for ext in ["kml", "kmz"] {
let mut ds = Dataset::new(
"places",
GeometryType::Point,
vec![
Field::new("name", FieldType::Text),
Field::new("description", FieldType::Text),
Field::new(variant, FieldType::Text),
],
);
ds.push(
Some(Geometry::Point(Coord::new(1.0, 2.0))),
vec![Value::Text("Place".into()), Value::Text("Keep case".into()), Value::Text(other.into())],
);
let (bytes, _) = crate::io::write_bytes(&ds, ext).unwrap();
let layers = read(&format!("places.{ext}"), &bytes).unwrap();
let layer = &layers[0];
let back = &layer.dataset;
assert_eq!(back.value(0, back.field_index("description").unwrap()), &Value::Text("Keep case".into()));
assert_eq!(back.value(0, back.field_index(&format!("{variant}_1")).unwrap()), &Value::Text(other.into()));
assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages);
assert!(layer.report.messages[0].contains(&format!("'{variant}' → '{variant}_1'")), "{:?}", layer.report.messages);
}
}
}
}

#[test]
fn exported_other_reserved_columns_still_report_their_collisions() {
for (name, ty, value) in [
("description", FieldType::Double, Value::Float(2.5)),
("Description", FieldType::Text, Value::Text("Keep case".into())),
("folder", FieldType::Text, Value::Text("My folder".into())),
("style", FieldType::Text, Value::Text("My style".into())),
] {
let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new(name, ty)]);
ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![value.clone()]);
let xml = crate::interop::to_kml(&ds, None).unwrap();
assert!(!xml.contains("<description>"), "{xml}");
let layers = read("places.kml", xml.as_bytes()).unwrap();
let layer = &layers[0];
let fi = layer.dataset.field_index(&format!("{name}_1")).unwrap();
assert_eq!(layer.dataset.value(0, fi), &value);
assert_eq!(layer.report.messages.len(), 1, "{:?}", layer.report.messages);
assert!(layer.report.messages[0].contains("renamed"), "{:?}", layer.report.messages);
}
}

#[test]
fn extended_data_named_like_a_fixed_column_is_kept_under_a_free_name() {
// `name` becomes the placemark's name and, like every field, extended data too; `NAME`
Expand Down
Loading
Loading