From 69cbbb1286c4fdd08594cbd5760a0d2455bed654 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 00:58:32 -0600 Subject: [PATCH 1/6] fix(data): long digit strings in CSV, KML and GPX stay text instead of being rounded A column of text values was typed Double whenever its values parsed as finite f64s, even when the f64 couldn't hold all their digits. A 20-digit ID read as 12345678901234567000, a 21-significant-digit decimal lost its tail, and a whole number past 2^53 rounded once another value made its column decimal, all without a notice. text_type now keeps a value as a Double only when its significant digits match those of the f64's shortest round-trip text. Otherwise the column stays Text, as leading-zero codes already do. This covers CSV, KML and GPX (type_text_columns) and GML. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 39 ++++++++++++++++++++++++++++++++++++--- crates/data/src/kml.rs | 10 ++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 699c389c41..8954866e70 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1804,16 +1804,20 @@ fn out_of_lonlat(g: &Geometry) -> bool { } /// The type of a column of text values: Integer or Double when every non-empty value is one, -/// otherwise (or with no values) Text. +/// otherwise (or with no values) Text. A number with more digits than a Double holds, such as a +/// 20-digit identifier, keeps its column Text, so no digit is rounded away. pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { let mut ty: Option = None; + // A whole number a Double can't hold: fine in an Integer column, not in a Double one. + let mut whole_past_double = false; for v in vals.map(str::trim).filter(|s| !s.is_empty()) { let t = if has_leading_zero(v) { // Identifiers such as ZIP codes keep their leading zeros. FieldType::Text } else if v.parse::().is_ok() { + whole_past_double |= !double_holds(v); FieldType::Integer - } else if v.parse::().is_ok_and(f64::is_finite) { + } else if double_holds(v) { FieldType::Double } else { FieldType::Text @@ -1823,7 +1827,24 @@ pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { Some(prev) => widen(prev, Some(t)), }); } - ty.unwrap_or(FieldType::Text) + match ty { + Some(FieldType::Double) if whole_past_double => FieldType::Text, + ty => ty.unwrap_or(FieldType::Text), + } +} + +/// Whether a number's text reads as a finite Double with the same significant digits, so the +/// Double is the number written: `1.50`, `0.1` and `1e300` are; `12345678901234567891` and +/// `0.12345678901234567891` aren't. +fn double_holds(v: &str) -> bool { + /// The digits of a number's mantissa without leading or trailing zeros. + fn digits(s: &str) -> String { + let mantissa = s.split(['e', 'E']).next().unwrap_or_default(); + let all: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + all.trim_start_matches('0').trim_end_matches('0').to_string() + } + // `{:e}` writes the shortest digits that read back as the same Double. + v.parse::().is_ok_and(|x| x.is_finite() && digits(v) == digits(&format!("{x:e}"))) } /// Types a dataset's Text columns from their values (see [`text_type`]), for readers whose @@ -4425,6 +4446,18 @@ mod tests { assert_eq!(to_csv(&back).unwrap(), once, "export/import is stable"); } + #[test] + fn csv_digits_a_number_cannot_hold_stay_text() { + let text = "id,share,mixed,n,x\n12345678901234567891,0.12345678901234567891,9007199254740993,7,1.50\n2,0.5,1.5,8,1e300\n"; + let (ds, _) = read_csv(text, "t").unwrap(); + let types: Vec = ds.fields.iter().map(|f| f.ty).collect(); + assert_eq!(types, [FieldType::Text, FieldType::Text, FieldType::Text, FieldType::Integer, FieldType::Double]); + assert_eq!(ds.value(0, 0), &Value::Text("12345678901234567891".into())); + assert_eq!(ds.value(0, 1), &Value::Text("0.12345678901234567891".into())); + assert_eq!(ds.value(0, 2), &Value::Text("9007199254740993".into()), "a whole number past 2^53 in a decimal column"); + assert_eq!(ds.value(1, 4), &Value::Float(1e300)); + } + /// CSV export writes numbers as stored (#513): through `Display` a 0.4 m² area in km² was 0. #[test] fn csv_floats_keep_full_precision() { diff --git a/crates/data/src/kml.rs b/crates/data/src/kml.rs index 58c175ddad..285d4ca9de 100644 --- a/crates/data/src/kml.rs +++ b/crates/data/src/kml.rs @@ -616,6 +616,16 @@ mod tests { map.png "##; + #[test] + fn digits_a_number_cannot_hold_stay_text() { + let doc = r#"12345678901234567891 + 1,2"#; + let layers = read("ids.kml", doc.as_bytes()).unwrap(); + let ds = &layers[0].dataset; + let id = ds.field_index("id").unwrap(); + assert_eq!(ds.value(0, id), &Value::Text("12345678901234567891".into())); + } + #[test] fn reads_placemarks_folders_data_and_styles() { let layers = read("trip.kml", DOC.as_bytes()).unwrap(); From 76abc9e0ffd6c3a72caefbfccfa7774fefa2ce70 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 01:40:04 -0600 Subject: [PATCH 2/6] fix(data): a number written to 17 digits still reads as a number A value counts as a Double when, written with as many significant digits, the Double gives the same digits back; comparing with the shortest form made 0.10000000000000001 (0.1 as %.17g writes it) text. Whole numbers up to 2^53 skip the check, as every one is a Double. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 8954866e70..e325313385 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1804,8 +1804,8 @@ fn out_of_lonlat(g: &Geometry) -> bool { } /// The type of a column of text values: Integer or Double when every non-empty value is one, -/// otherwise (or with no values) Text. A number with more digits than a Double holds, such as a -/// 20-digit identifier, keeps its column Text, so no digit is rounded away. +/// otherwise (or with no values) Text. A number whose digits a Double doesn't keep, such as a +/// 20-digit identifier, keeps its column Text, so no digit is rounded away ([`double_holds`]). pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { let mut ty: Option = None; // A whole number a Double can't hold: fine in an Integer column, not in a Double one. @@ -1814,8 +1814,9 @@ pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { let t = if has_leading_zero(v) { // Identifiers such as ZIP codes keep their leading zeros. FieldType::Text - } else if v.parse::().is_ok() { - whole_past_double |= !double_holds(v); + } else if let Ok(i) = v.parse::() { + // Every whole number up to 2^53 is a Double exactly. + whole_past_double |= i.unsigned_abs() > 1 << 53 && !double_holds(v); FieldType::Integer } else if double_holds(v) { FieldType::Double @@ -1833,9 +1834,10 @@ pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { } } -/// Whether a number's text reads as a finite Double with the same significant digits, so the -/// Double is the number written: `1.50`, `0.1` and `1e300` are; `12345678901234567891` and -/// `0.12345678901234567891` aren't. +/// Whether a number's text reads as a finite Double that, written with as many significant digits, +/// gives the same digits back, so no digit is lost: `1.50`, `0.1`, `1e300` and +/// `0.10000000000000001` (0.1 to 17 digits) are; `12345678901234567891` and +/// `0.12345678901234567891`, past the 17 digits a Double holds, aren't. fn double_holds(v: &str) -> bool { /// The digits of a number's mantissa without leading or trailing zeros. fn digits(s: &str) -> String { @@ -1843,8 +1845,9 @@ fn double_holds(v: &str) -> bool { let all: String = mantissa.chars().filter(char::is_ascii_digit).collect(); all.trim_start_matches('0').trim_end_matches('0').to_string() } - // `{:e}` writes the shortest digits that read back as the same Double. - v.parse::().is_ok_and(|x| x.is_finite() && digits(v) == digits(&format!("{x:e}"))) + let want = digits(v); + v.parse::() + .is_ok_and(|x| x.is_finite() && (want.is_empty() || want.len() <= 17 && digits(&format!("{x:.*e}", want.len() - 1)) == want)) } /// Types a dataset's Text columns from their values (see [`text_type`]), for readers whose @@ -4456,6 +4459,9 @@ mod tests { assert_eq!(ds.value(0, 1), &Value::Text("0.12345678901234567891".into())); assert_eq!(ds.value(0, 2), &Value::Text("9007199254740993".into()), "a whole number past 2^53 in a decimal column"); assert_eq!(ds.value(1, 4), &Value::Float(1e300)); + // A number written to 17 digits, as `%.17g` writes 0.1, is a Double. + let (ds, _) = read_csv("v\n0.10000000000000001\n2.5\n", "t").unwrap(); + assert_eq!((ds.fields[0].ty, ds.value(0, 0)), (FieldType::Double, &Value::Float(0.1))); } /// CSV export writes numbers as stored (#513): through `Display` a 0.4 m² area in km² was 0. From 4986f36798a26356a1d41c53a0b40ab580174eba Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 01:43:48 -0600 Subject: [PATCH 3/6] fix(data): a whole number beside decimals is checked by its value Its Double is compared with it in i128, so 2^60 stays a number and 10^18 + 100, which a Double rounds, keeps its column text. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index e325313385..f73c72288f 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1815,8 +1815,8 @@ pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { // Identifiers such as ZIP codes keep their leading zeros. FieldType::Text } else if let Ok(i) = v.parse::() { - // Every whole number up to 2^53 is a Double exactly. - whole_past_double |= i.unsigned_abs() > 1 << 53 && !double_holds(v); + // A whole number is held when its Double is the same number (in i128: no saturation). + whole_past_double |= (i as f64) as i128 != i128::from(i); FieldType::Integer } else if double_holds(v) { FieldType::Double @@ -4462,6 +4462,9 @@ mod tests { // A number written to 17 digits, as `%.17g` writes 0.1, is a Double. let (ds, _) = read_csv("v\n0.10000000000000001\n2.5\n", "t").unwrap(); assert_eq!((ds.fields[0].ty, ds.value(0, 0)), (FieldType::Double, &Value::Float(0.1))); + // A whole number beside a decimal: 2^60 is a Double exactly, 10^18 + 100 isn't. + let (ds, _) = read_csv("a,b\n1152921504606846976,1000000000000000100\n0.5,0.5\n", "t").unwrap(); + assert_eq!((ds.fields[0].ty, ds.fields[1].ty), (FieldType::Double, FieldType::Text)); } /// CSV export writes numbers as stored (#513): through `Display` a 0.4 m² area in km² was 0. From 0970ef338673c426faab9b7ed0f4a4c82a38ecb6 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 02:22:30 -0600 Subject: [PATCH 4/6] fix(data): a decimal a Double holds exactly reads as a number however long Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index f73c72288f..5f5f0f8d36 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1836,8 +1836,9 @@ pub(crate) fn text_type<'a>(vals: impl Iterator) -> FieldType { /// Whether a number's text reads as a finite Double that, written with as many significant digits, /// gives the same digits back, so no digit is lost: `1.50`, `0.1`, `1e300` and -/// `0.10000000000000001` (0.1 to 17 digits) are; `12345678901234567891` and -/// `0.12345678901234567891`, past the 17 digits a Double holds, aren't. +/// `0.10000000000000001` (0.1 to 17 digits) and `0.00000000023283064365386962890625` (2^-32 +/// exactly) are; `12345678901234567891` and `0.12345678901234567891` aren't. No Double's exact +/// value has more than 767 significant digits, so longer text isn't one, and isn't formatted. fn double_holds(v: &str) -> bool { /// The digits of a number's mantissa without leading or trailing zeros. fn digits(s: &str) -> String { @@ -1847,7 +1848,7 @@ fn double_holds(v: &str) -> bool { } let want = digits(v); v.parse::() - .is_ok_and(|x| x.is_finite() && (want.is_empty() || want.len() <= 17 && digits(&format!("{x:.*e}", want.len() - 1)) == want)) + .is_ok_and(|x| x.is_finite() && (want.is_empty() || want.len() <= 767 && digits(&format!("{x:.*e}", want.len() - 1)) == want)) } /// Types a dataset's Text columns from their values (see [`text_type`]), for readers whose @@ -4462,6 +4463,8 @@ mod tests { // A number written to 17 digits, as `%.17g` writes 0.1, is a Double. let (ds, _) = read_csv("v\n0.10000000000000001\n2.5\n", "t").unwrap(); assert_eq!((ds.fields[0].ty, ds.value(0, 0)), (FieldType::Double, &Value::Float(0.1))); + // So is one written out exactly, however many digits that takes. + assert!(double_holds("0.00000000023283064365386962890625") && !double_holds(&"1".repeat(800))); // A whole number beside a decimal: 2^60 is a Double exactly, 10^18 + 100 isn't. let (ds, _) = read_csv("a,b\n1152921504606846976,1000000000000000100\n0.5,0.5\n", "t").unwrap(); assert_eq!((ds.fields[0].ty, ds.fields[1].ty), (FieldType::Double, FieldType::Text)); From ad3cf820352b008f1303a15ff840e7e8d97ba6a9 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 03:03:30 -0600 Subject: [PATCH 5/6] fix(data): a whole number past i64 is a Double only if it is one exactly 12345678901234567000 matched a Double's 17 digits with zeros after, but that Double is 12345678901234567168: such an ID stays text now. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 5f5f0f8d36..414d45eccb 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1846,6 +1846,12 @@ fn double_holds(v: &str) -> bool { let all: String = mantissa.chars().filter(char::is_ascii_digit).collect(); all.trim_start_matches('0').trim_end_matches('0').to_string() } + // A whole number written as one is held only exactly: `12345678901234567000` is a Double's + // 17 digits and zeros, but that Double is 12345678901234567168. + let whole = v.strip_prefix(['-', '+']).unwrap_or(v); + if !whole.is_empty() && whole.bytes().all(|b| b.is_ascii_digit()) { + return v.parse::().is_ok_and(|i| v.parse::().is_ok_and(|x| x.is_finite() && x as i128 == i)); + } let want = digits(v); v.parse::() .is_ok_and(|x| x.is_finite() && (want.is_empty() || want.len() <= 767 && digits(&format!("{x:.*e}", want.len() - 1)) == want)) @@ -4465,6 +4471,8 @@ mod tests { assert_eq!((ds.fields[0].ty, ds.value(0, 0)), (FieldType::Double, &Value::Float(0.1))); // So is one written out exactly, however many digits that takes. assert!(double_holds("0.00000000023283064365386962890625") && !double_holds(&"1".repeat(800))); + // A whole number past i64 with trailing zeros is held only if exactly. + assert!(!double_holds("12345678901234567000") && double_holds("18446744073709551616")); // A whole number beside a decimal: 2^60 is a Double exactly, 10^18 + 100 isn't. let (ds, _) = read_csv("a,b\n1152921504606846976,1000000000000000100\n0.5,0.5\n", "t").unwrap(); assert_eq!((ds.fields[0].ty, ds.fields[1].ty), (FieldType::Double, FieldType::Text)); From a6ea9ca3c7c5e6f56e7c5e8c23fa0aa68ffd6296 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 03:46:11 -0600 Subject: [PATCH 6/6] fix(data): a whole number is checked against its Double's exact digits, at any size Comparing in i128 let i128::MAX pass by saturating and refused whole Doubles past it. The Double's exact digits are compared instead. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 414d45eccb..e25bf3888d 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1850,7 +1850,10 @@ fn double_holds(v: &str) -> bool { // 17 digits and zeros, but that Double is 12345678901234567168. let whole = v.strip_prefix(['-', '+']).unwrap_or(v); if !whole.is_empty() && whole.bytes().all(|b| b.is_ascii_digit()) { - return v.parse::().is_ok_and(|i| v.parse::().is_ok_and(|x| x.is_finite() && x as i128 == i)); + // A whole Double's exact digits, written out (no finite Double has more than 309). + let digits = whole.trim_start_matches('0'); + let digits = if digits.is_empty() { "0" } else { digits }; + return digits.len() <= 309 && v.parse::().is_ok_and(|x| x.is_finite() && format!("{:.0}", x.abs()) == digits); } let want = digits(v); v.parse::() @@ -4473,6 +4476,9 @@ mod tests { assert!(double_holds("0.00000000023283064365386962890625") && !double_holds(&"1".repeat(800))); // A whole number past i64 with trailing zeros is held only if exactly. assert!(!double_holds("12345678901234567000") && double_holds("18446744073709551616")); + // At any size, with no saturation: i128::MAX isn't one, 2^200 is. + assert!(!double_holds("170141183460469231731687303715884105727")); + assert!(double_holds("1606938044258990275541962092341162602522202993782792835301376")); // A whole number beside a decimal: 2^60 is a Double exactly, 10^18 + 100 isn't. let (ds, _) = read_csv("a,b\n1152921504606846976,1000000000000000100\n0.5,0.5\n", "t").unwrap(); assert_eq!((ds.fields[0].ty, ds.fields[1].ty), (FieldType::Double, FieldType::Text));