From 8ca684501a775a0e60af386c0c82a486c41eb1cf Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 05:27:42 -0600 Subject: [PATCH 1/4] fix(data): keep non-finite attributes from poisoning saved projects Use the existing Null encoding for human-readable non-finite values and report counted losses from rows, saved design and archive snapshots. Reject non-finite interactive edits and retain source values until output. --- apps/rustgis-web/src/web.rs | 4 +- crates/data/src/gml.rs | 2 +- crates/data/src/io.rs | 4 +- crates/data/src/lib.rs | 12 +++ crates/data/src/mapinfo.rs | 2 +- crates/data/src/source/gpkg.rs | 4 +- crates/data/src/source/gpkg_mem.rs | 2 +- crates/data/src/value.rs | 67 ++++++++++++++++- crates/engine/src/cmd/edit.rs | 40 +++++++--- crates/engine/src/durable.rs | 72 +++++++++++++++++- crates/engine/src/notice_text.rs | 2 +- crates/engine/src/project.rs | 1 + crates/engine/tests/project_save.rs | 111 ++++++++++++++++++++++++++++ crates/map/src/design.rs | 26 +++++++ crates/ui-egui/src/lib.rs | 4 +- 15 files changed, 323 insertions(+), 30 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index f9d3812a2a..afc93d6756 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -414,9 +414,7 @@ impl Embed { (true, _) => match app.session.peer_text(rustgis_engine::Peer::Embedder) { Ok((text, report)) => { self.answer(&crate::embed::project(&text)); - for notice in report.notices() { - app.session.notify(notice, false); - } + report.notify(&mut app.session); } Err(e) => app.session.notify(e, true), }, diff --git a/crates/data/src/gml.rs b/crates/data/src/gml.rs index b9d2de93e2..ed776366e9 100644 --- a/crates/data/src/gml.rs +++ b/crates/data/src/gml.rs @@ -567,7 +567,7 @@ pub fn read(file_name: &str, bytes: &[u8]) -> Result> let e = g.extent(); outside |= e.min_x < -180.0 || e.max_x > 180.0 || e.min_y < -90.0 || e.max_y > 90.0; } - let row = ds.fields.iter().map(|f| attrs.get(&f.name).map(|v| Value::parse(v, f.ty)).unwrap_or_default()).collect(); + let row = ds.fields.iter().map(|f| attrs.get(&f.name).map(|v| Value::parse_source(v, f.ty)).unwrap_or_default()).collect(); ds.push_next(g, row); } ds.geometry_type = crate::io::dominant_type(&ds); diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 876ab041aa..02a846c1d9 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1655,7 +1655,7 @@ pub(crate) fn read_rows( let attrs = columns .iter() .zip(&ds.fields) - .map(|(&i, f)| Value::parse(&unguard(r.get(i).map(|s| s.as_str()).unwrap_or("")), f.ty)) + .map(|(&i, f)| Value::parse_source(&unguard(r.get(i).map(|s| s.as_str()).unwrap_or("")), f.ty)) .collect(); ds.push_next(geom, attrs); } @@ -1794,7 +1794,7 @@ pub(crate) fn type_text_columns(ds: &mut Dataset, keep: &[&str]) { if let Some(v) = f.attrs.get_mut(i) && let Value::Text(t) = v { - *v = if t.trim().is_empty() { Value::Null } else { Value::parse(t, ty) }; + *v = if t.trim().is_empty() { Value::Null } else { Value::parse_source(t, ty) }; } } } diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 28564991df..6d65e74ad7 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -329,6 +329,18 @@ pub struct Dataset { pub windowed: bool, } +impl Dataset { + /// How many of its values a saved project holds as null, not as they are: numbers that aren't + /// finite (NaN, infinity), which JSON can't write ([`Value`]'s `Serialize`). None for a window + /// onto a source, whose rows aren't saved. + pub fn unsaved_numbers(&self) -> usize { + if self.windowed { + return 0; + } + self.features.iter().flat_map(|f| &f.attrs).filter(|v| v.is_non_finite()).count() + } +} + impl Serialize for Dataset { /// Saves CRS recovery and dropped Z/M state with the dataset, but omits disposable streamed rows. fn serialize(&self, s: S) -> std::result::Result { diff --git a/crates/data/src/mapinfo.rs b/crates/data/src/mapinfo.rs index 71f63a4f0e..540bba7e5e 100644 --- a/crates/data/src/mapinfo.rs +++ b/crates/data/src/mapinfo.rs @@ -403,7 +403,7 @@ pub fn read(mif: &str, mid: Option<&str>, name: &str) -> Result<(Dataset, ReadRe .fields .iter() .enumerate() - .map(|(c, f)| Value::parse(row.and_then(|r| r.get(c)).map(String::as_str).unwrap_or(""), f.ty)) + .map(|(c, f)| Value::parse_source(row.and_then(|r| r.get(c)).map(String::as_str).unwrap_or(""), f.ty)) .collect(); ds.push_next(g, attrs); } diff --git a/crates/data/src/source/gpkg.rs b/crates/data/src/source/gpkg.rs index 0fb46b88f6..2fd818acf5 100644 --- a/crates/data/src/source/gpkg.rs +++ b/crates/data/src/source/gpkg.rs @@ -1149,7 +1149,7 @@ fn sql_value(v: &Value, ty: FieldType) -> rusqlite::types::Value { (Value::Float(_), _) => Sql::Null, (Value::Bool(b), _) => Sql::Integer(i64::from(*b)), (Value::Text(s), FieldType::Text) => Sql::Text(s.clone()), - (v, ty) => match Value::parse(&v.to_string(), ty) { + (v, ty) => match Value::parse_source(&v.to_string(), ty) { Value::Int(x) => Sql::Integer(x), Value::Float(x) if x.is_finite() => Sql::Real(x), Value::Bool(b) => Sql::Integer(i64::from(b)), @@ -1167,7 +1167,7 @@ fn value(v: ValueRef<'_>, ty: FieldType) -> Value { (ValueRef::Real(f), FieldType::Integer) if f.fract() == 0.0 && f.abs() < 9e15 => Value::Int(f as i64), (ValueRef::Real(f), _) => Value::Float(f), (ValueRef::Text(t), FieldType::Text) => Value::Text(String::from_utf8_lossy(t).into_owned()), - (ValueRef::Text(t), ty) => Value::parse(&String::from_utf8_lossy(t), ty), + (ValueRef::Text(t), ty) => Value::parse_source(&String::from_utf8_lossy(t), ty), (ValueRef::Blob(_), _) => Value::Null, } } diff --git a/crates/data/src/source/gpkg_mem.rs b/crates/data/src/source/gpkg_mem.rs index 1121def400..eda8a51750 100644 --- a/crates/data/src/source/gpkg_mem.rs +++ b/crates/data/src/source/gpkg_mem.rs @@ -320,7 +320,7 @@ fn value(v: &Val, ty: FieldType) -> Value { (Val::Real(f), FieldType::Integer) if f.fract() == 0.0 && f.abs() < 9e15 => Value::Int(*f as i64), (Val::Real(f), _) => Value::Float(*f), (Val::Text(t), FieldType::Text) => Value::Text(t.clone()), - (Val::Text(t), ty) => Value::parse(t, ty), + (Val::Text(t), ty) => Value::parse_source(t, ty), } } diff --git a/crates/data/src/value.rs b/crates/data/src/value.rs index 9d062d3fa8..f14d373316 100644 --- a/crates/data/src/value.rs +++ b/crates/data/src/value.rs @@ -42,7 +42,7 @@ impl Field { } } -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] pub enum Value { #[default] Null, @@ -52,7 +52,39 @@ pub enum Value { Bool(bool), } +impl Serialize for Value { + /// As its variant, except that a number that isn't finite (NaN, infinity), which JSON can't + /// write, is written as null: JSON writes it as `null` in its place, which a number can't be + /// read from, so the file holding it wouldn't open. Saving says how many + /// ([`crate::Dataset::unsaved_numbers`]). + fn serialize(&self, s: S) -> Result { + match self { + Value::Float(f) if !f.is_finite() && s.is_human_readable() => s.serialize_unit_variant("Value", 0, "Null"), + Value::Null => s.serialize_unit_variant("Value", 0, "Null"), + Value::Int(i) => s.serialize_newtype_variant("Value", 1, "Int", i), + Value::Float(f) => s.serialize_newtype_variant("Value", 2, "Float", f), + Value::Text(t) => s.serialize_newtype_variant("Value", 3, "Text", t), + Value::Bool(b) => s.serialize_newtype_variant("Value", 4, "Bool", b), + } + } +} + impl Value { + /// Whether human-readable output must substitute Null for this numeric value. + pub fn is_non_finite(&self) -> bool { + matches!(self, Value::Float(value) if !value.is_finite()) + } + + /// Parses a typed source field without discarding a non-finite Double. Its original number + /// stays available until output, where the counted save notice explains conversion to null. + /// Interactive edits use [`Self::parse`] and refuse non-finite input at command ingress. + pub fn parse_source(text: &str, ty: FieldType) -> Value { + if ty == FieldType::Double { + return text.trim().parse::().map(Value::Float).unwrap_or(Value::Null); + } + Self::parse(text, ty) + } + pub fn as_f64(&self) -> Option { match self { Value::Int(i) => Some(*i as f64), @@ -87,7 +119,8 @@ impl Value { Value::Bool(b) => (*b).into(), } } - /// Parses user text into a value of `ty` (empty → Null). + /// Parses user text into a value of `ty` (empty → Null). Text a number field can't hold, + /// including a non-finite Double (`NaN`, `inf`), is Null. pub fn parse(text: &str, ty: FieldType) -> Value { let t = text.trim(); if t.is_empty() || t.eq_ignore_ascii_case("") { @@ -97,7 +130,7 @@ impl Value { FieldType::Integer => { t.parse::().map(Value::Int).or_else(|_| t.parse::().map(|f| Value::Int(f as i64))).unwrap_or(Value::Null) } - FieldType::Double => t.parse::().map(Value::Float).unwrap_or(Value::Null), + FieldType::Double => t.parse::().ok().filter(|f| f.is_finite()).map_or(Value::Null, Value::Float), FieldType::Boolean => Value::Bool(matches!(t.to_ascii_lowercase().as_str(), "true" | "yes" | "1" | "y")), FieldType::Text => Value::Text(t.to_string()), } @@ -386,6 +419,34 @@ pub fn sort_class_values(values: &mut Vec) { mod tests { use super::*; + /// Interactive Double text rejects non-finite numbers, while source parsing preserves them + /// for the counted output notice. Finite numbers keep their established parsing. + #[test] + fn a_number_field_reads_text_that_is_not_a_finite_number_as_null() { + for text in ["NaN", "inf", "-inf", "infinity", "1e400"] { + assert_eq!(Value::parse(text, FieldType::Double), Value::Null, "{text}"); + assert!(matches!(Value::parse_source(text, FieldType::Double), Value::Float(f) if !f.is_finite())); + } + assert_eq!(Value::parse("2.5", FieldType::Double), Value::Float(2.5)); + assert_eq!(Value::parse("2.5", FieldType::Integer), Value::Int(2)); + assert_eq!(Value::parse("1e3", FieldType::Integer), Value::Int(1000)); + } + + /// JSON cannot represent non-finite numbers; output uses a null value with the save report. + /// Finite values keep their established encoding. + #[test] + fn a_value_that_is_not_finite_is_written_and_read_as_null() { + for f in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert_eq!(serde_json::to_string(&Value::Float(f)).ok().as_deref(), Some(r#""Null""#)); + } + for v in [Value::Null, Value::Int(-3), Value::Float(0.1), Value::Text("NaN".into()), Value::Bool(true)] { + let text = serde_json::to_string(&v).unwrap_or_default(); + assert_eq!(serde_json::from_str::(&text).ok(), Some(v.clone()), "{text}"); + } + assert_eq!(serde_json::to_string(&Value::Float(0.1)).ok().as_deref(), Some(r#"{"Float":0.1}"#), "the format is unchanged"); + assert!(serde_json::from_str::(r#"{"Date":"2026-01-01"}"#).is_err_and(|e| e.to_string().contains("unknown variant"))); + } + fn sorted(values: &[&str]) -> Vec { let mut v: Vec = values.iter().map(|s| s.to_string()).collect(); sort_class_values(&mut v); diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index da977dcdc8..3bbfe632f0 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -47,20 +47,22 @@ pub(crate) fn add_feature(s: &mut Session, p: &Json) -> CmdResult { } None => None, }; - let attrs: Vec = l + let attrs: Result, String> = l .dataset .fields .iter() .map(|f| { - let given = p.get("attributes").and_then(|a| a.get(&f.name)).map(|v| coerce(Value::from_json(v), f.ty)); - let default = template - .as_ref() - .and_then(|t| t.defaults.iter().find(|(k, _)| k.eq_ignore_ascii_case(&f.name))) - .map(|(_, v)| coerce(v.clone(), f.ty)); - given.or(default).unwrap_or(Value::Null) + let given = p.get("attributes").and_then(|a| a.get(&f.name)).map(Value::from_json); + let default = + template.as_ref().and_then(|t| t.defaults.iter().find(|(k, _)| k.eq_ignore_ascii_case(&f.name))).map(|(_, v)| v.clone()); + let value = given.or(default).unwrap_or(Value::Null); + if non_finite_input(&value, f.ty) { + return Err(format!("field '{}' requires a finite number", f.name)); + } + Ok(coerce(value, f.ty)) }) .collect(); - l.dataset.push(Some(g), attrs); + l.dataset.push(Some(g), attrs?); l.touch_geometry(); let row = l.dataset.features.len() - 1; l.selection = vec![row]; @@ -170,10 +172,24 @@ impl Cuts { /// [`coerce`], with a number past the 64-bit range null, and what it took from a number to put it /// in a whole-number field. pub(crate) fn coerce_cut(v: Value, ty: FieldType) -> (Value, Cut) { + if ty == FieldType::Double && non_finite_input(&v, ty) { + return (Value::Null, Cut::NotANumber); + } let (v, cut, _) = convert(v, ty); (v, cut) } +/// Text and native numbers that cannot be retained as numeric field values. +/// Ordinary invalid words keep the existing parser's behavior; non-finite numbers are explicit. +fn non_finite_input(value: &Value, ty: FieldType) -> bool { + ty.is_numeric() + && match value { + Value::Float(f) => !f.is_finite(), + Value::Text(text) => text.trim().parse::().is_ok_and(|f| !f.is_finite()), + _ => false, + } +} + /// [`coerce_cut`], and for a number past the range the decimal it was read as. fn convert(v: Value, ty: FieldType) -> (Value, Cut, Option) { match (v, ty) { @@ -195,7 +211,8 @@ fn convert(v: Value, ty: FieldType) -> (Value, Cut, Option) { _ => (Value::parse(&t, ty), Cut::None, None), }, }, - (Value::Text(t), _) => (Value::parse(&t, ty), Cut::None, None), + // Uncounted coercion retains source numbers until output can explain any loss. + (Value::Text(t), _) => (Value::parse_source(&t, ty), Cut::None, None), (other, _) => (Value::parse(&other.to_string(), ty), Cut::None, None), } } @@ -273,6 +290,11 @@ fn set_value(s: &mut Session, p: &Json) -> CmdResult { let ty = l.dataset.fields.get(fi).map(|f| f.ty).ok_or("no field")?; let raw = p.get("value").cloned().unwrap_or(Json::Null); let v = match raw { + // A number that isn't finite (NaN, infinity) is no value a number field holds, nor one a + // project can save: refused, as the attribute table refuses it. + Json::String(t) if ty.is_numeric() && t.trim().parse::().is_ok_and(|f| !f.is_finite()) => { + return Err(format!("'{}' isn't a finite number, which a {} field holds", t.trim(), ty.label())); + } Json::String(t) => Value::parse(&t, ty), other => coerce(Value::from_json(&other), ty), }; diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs index 7f298e0433..ef201d7b6b 100644 --- a/crates/engine/src/durable.rs +++ b/crates/engine/src/durable.rs @@ -23,6 +23,10 @@ pub struct Report { pub(crate) not_retained: u32, pub(crate) executable_changes: u32, pub(crate) reconnect: u32, + /// Attribute values written as null: numbers that aren't finite, which JSON can't hold. + pub(crate) numbers: u32, + /// First affected dataset names; the total count still covers every dataset. + pub(crate) number_places: Vec, } impl Report { @@ -40,6 +44,20 @@ impl Report { /// One notice for each kind of loss, and none for a clean output. pub fn notices(&self) -> Vec { + self.notice_texts().into_iter().map(|text| text.english()).collect() + } + + /// Publishes counted output losses, retaining new templates for translation in the interface. + /// Stored recovery diagnostics and non-interactive callers keep [`Self::notices`] in English. + pub fn notify(&self, session: &mut crate::Session) { + for text in self.notice_texts() { + session.notify_text(text, false); + } + } + + /// Keeps typed count/name arguments separate from the trusted text of each new notice. + fn notice_texts(&self) -> Vec { + use crate::notice_text::NoticeText; let mut out = Vec::new(); for (n, text) in [ ( @@ -60,11 +78,21 @@ impl Report { ), ] { if n > 0 { - out.push(format!("{n} {text}")); + out.push(format!("{n} {text}").into()); } } + if self.numbers > 0 { + out.push(NoticeText::format( + "{count} attribute value(s) that aren't finite numbers (NaN or infinity) were written as null; affected datasets include {places}", + [("count", self.numbers.to_string().into()), ("places", NoticeText::list(self.number_places.clone()))], + )); + } if self.not_remembered { - out.push("This output's credential restoration exceeds the memory limit; reopening it will not bring its keys back".into()); + out.push( + "This output's credential restoration exceeds the memory limit; reopening it will not bring its keys back" + .to_string() + .into(), + ); } out } @@ -90,6 +118,12 @@ impl Report { self.not_retained = self.not_retained.saturating_add(other.not_retained); self.executable_changes = self.executable_changes.saturating_add(other.executable_changes); self.reconnect = self.reconnect.saturating_add(other.reconnect); + self.numbers = self.numbers.saturating_add(other.numbers); + for place in &other.number_places { + if self.number_places.len() < 3 && !self.number_places.contains(place) { + self.number_places.push(place.clone()); + } + } } } @@ -244,6 +278,30 @@ impl<'a> Prepare<'a> { self.owners.pop(); } + /// Counts the values `dataset` is written without ([`rustgis_data::Dataset::unsaved_numbers`]). + pub(crate) fn count_numbers(&mut self, dataset: &rustgis_data::Dataset) { + self.record_numbers(dataset.unsaved_numbers(), format!("'{}'", crate::project::short_name(&dataset.name)).into()); + } + + /// Counts saved design values separately from current rows so archived snapshots and + /// feature defaults cannot be silently changed by Value's human-readable serialization. + fn count_design_numbers(&mut self, layer: &rustgis_map::Layer) { + let place = crate::notice_text::NoticeText::format( + "{name} (design or archived values)", + [("name", format!("'{}'", crate::project::short_name(&layer.name)).into())], + ); + self.record_numbers(layer.design.unsaved_numbers(), place); + } + + /// Aggregates every value while retaining only a bounded list of affected locations. + fn record_numbers(&mut self, count: usize, place: crate::notice_text::NoticeText) { + let n = u32::try_from(count).unwrap_or(u32::MAX); + self.report.numbers = self.report.numbers.saturating_add(n); + if n > 0 && self.report.number_places.len() < 3 && !self.report.number_places.contains(&place) { + self.report.number_places.push(place); + } + } + /// Records a single replacement of a whole owning field, never a subfield of an edit. fn edit(&mut self, leaf: Leaf, held: Payload, kind: Kind) { self.edits.push(Edit { at: At { owners: self.owners.clone(), leaf }, held, kind }); @@ -686,8 +744,8 @@ fn split_look(address: &str) -> (&str, &str) { /// Emits preparation losses after the durable projection's journal has restored live fields. /// This does not confirm delivery or commit the enclosing command's notices. pub(crate) fn notify(s: &mut crate::Session, report: &Report) { - for text in report.notices() { - s.notify(format!("Prepared output: {text}"), false); + for text in report.notice_texts() { + s.notify_text(crate::notice_text::NoticeText::format("Prepared output: {message}", [("message", text)]), false); } } @@ -746,6 +804,10 @@ impl Durable for rustgis_map::Map { for s in self.slots() { p.string(Leaf::Map(s.at), s.value, s.address, s.local_path); } + for l in &self.layers { + p.count_numbers(&l.dataset); + p.count_design_numbers(l); + } } /// Resolves a prepared location through owned fields, without allocation or policy. @@ -764,6 +826,8 @@ impl Durable for rustgis_map::Layer { for (role, value, address, path) in self.output_slots() { p.string(Leaf::Layer(role), value, address, path); } + p.count_numbers(&self.dataset); + p.count_design_numbers(self); } /// Resolves a prepared location through owned fields, without allocation or policy. diff --git a/crates/engine/src/notice_text.rs b/crates/engine/src/notice_text.rs index 1e08950946..ddb5362ab4 100644 --- a/crates/engine/src/notice_text.rs +++ b/crates/engine/src/notice_text.rs @@ -1,7 +1,7 @@ //! Deferred interface text. The engine keeps English diagnostics; a front end translates //! trusted templates while treating project names and other file contents as literal data. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum NoticeText { Plain(String), Format { key: &'static str, args: Vec<(&'static str, NoticeText)> }, diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 795aa6c7fd..5167ca1884 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -1888,6 +1888,7 @@ impl Durable for Table { if let Some(v) = provenance { p.string(Leaf::Table, v, true, rustgis_map::paths::is_local(v)); } + p.count_numbers(dataset); } /// Resolves a prepared location through owned fields, without allocation or policy. diff --git a/crates/engine/tests/project_save.rs b/crates/engine/tests/project_save.rs index a4487987fa..8bde5bf404 100644 --- a/crates/engine/tests/project_save.rs +++ b/crates/engine/tests/project_save.rs @@ -221,3 +221,114 @@ fn file_compatibility_resaves_declare_the_current_writer() -> Result<(), String> } Ok(()) } + +/// A number that isn't finite (NaN, infinity) can't make a project unopenable: Set Value refuses +/// one, and one a reader put in a layer is saved as null with a count and dataset location. +/// A project of finite numbers saves and opens without a loss notice. +#[test] +fn numbers_that_are_not_finite_never_make_a_project_unopenable() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + let mut s = Session::new(Project::default()); + let id = s.map_mut().add_dataset(Dataset::new("Readings", GeometryType::None, vec![Field::new("level", FieldType::Double)])); + let ds = &mut s.map_mut().layer_mut(id).ok_or("no layer")?.dataset; + ds.push(None, vec![V::Float(1.5)]); + ds.push(None, vec![V::Float(2.5)]); + for typed in ["NaN", " inf ", "-infinity", "1e400"] { + let e = s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "level", "value": typed})).err().ok_or(typed)?; + assert!(e.contains("isn't a finite number"), "{typed}: {e}"); + } + assert_eq!(s.map().layer(id).ok_or("no layer")?.dataset.value(0, 0), &V::Float(1.5), "a refused value changes nothing"); + let losses = |notices: Vec| notices.into_iter().filter(|n| n.contains("finite numbers")).collect::>(); + + // Clean: no notice, and the numbers open as they were. + let (text, report) = s.project.saved_text(None)?; + assert!(losses(report.notices()).is_empty(), "{:?}", report.notices()); + assert_eq!(Project::from_json(&text)?.maps[0].layers[0].dataset.value(1, 0), &V::Float(2.5)); + + // As a reader leaves them (a NaN REAL in a GeoPackage): saved as null, and counted. + let ds = &mut s.map_mut().layer_mut(id).ok_or("no layer")?.dataset; + ds.features[0].attrs[0] = V::Float(f64::NAN); + ds.features[1].attrs[0] = V::Float(f64::NEG_INFINITY); + let (text, report) = s.project.saved_text(None)?; + assert_eq!( + losses(report.notices()), + ["2 attribute value(s) that aren't finite numbers (NaN or infinity) were written as null; affected datasets include 'Readings'"] + ); + let mark = s.notice_mark(); + report.notify(&mut s); + let notice = s.notices_since(mark).last().ok_or("no counted output notice")?; + assert_eq!( + notice + .translated(|key, args| { + assert!(key.starts_with("{count} attribute value(s)")); + assert!(args.contains(&("places", "'Readings'"))); + "translated loss notice".to_string() + }) + .as_deref(), + Some("translated loss notice") + ); + let opened = Project::from_json(&text)?; + assert_eq!(opened.maps[0].layers[0].dataset.value(0, 0), &V::Null); + assert_eq!(opened.maps[0].layers[0].dataset.value(1, 0), &V::Null); + + // Opened clean: no notice. + let mark = s.notice_mark(); + s.run("project.openText", &json!({"text": Project::from_json(&text).and_then(|mut p| p.saved_text(None))?.0}))?; + assert!(losses(s.notices_since(mark).iter().map(|n| n.text.clone()).collect()).is_empty()); + // Other interactive ingress either rejects a non-finite value or reports its conversion. + let before = s.map().layers[0].dataset.features.len(); + assert!( + s.run( + "edit.addFeature", + &json!({"layer": "Readings", "geometry": {"type": "Point", "coordinates": [0, 0]}, "attributes": {"level": "NaN"}}) + ) + .is_err() + ); + assert_eq!(s.map().layers[0].dataset.features.len(), before); + let mark = s.notice_mark(); + s.run("edit.calculateField", &json!({"layer": "Readings", "field": "level", "expression": "'NaN'"}))?; + assert_eq!(s.notices_since(mark).iter().filter(|n| n.text.contains("'level' is null for 2 results that aren't numbers")).count(), 1); + + // A declared Double in MIF/MID retains its non-finite source number until counted output. + // CSV inference deliberately keeps a mixed NaN/numeric column as lossless text. + let mif = "Version 300\nCoordSys Earth Projection 1, 104\nColumns 1\n level Float\nData\nPoint 0 0\nPoint 1 1\n"; + let (dataset, _) = rustgis_engine::data::mapinfo::read(mif, Some("NaN\n1.5\n"), "Imported").map_err(|e| e.to_string())?; + assert!(matches!(dataset.value(0, 0), V::Float(f) if f.is_nan())); + s.map_mut().add_dataset(dataset); + let (_, report) = s.project.saved_text(None)?; + assert_eq!( + losses(report.notices()), + ["1 attribute value(s) that aren't finite numbers (NaN or infinity) were written as null; affected datasets include 'Imported'"] + ); + Ok(()) +} + +/// Finite current rows must not hide non-finite values kept in saved design and edit history. +#[test] +fn nonfinite_design_and_archived_values_are_counted_when_output_is_prepared() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + let mut s = Session::new(Project::default()); + let mut dataset = Dataset::new("Design", GeometryType::None, vec![Field::new("level", FieldType::Double)]); + dataset.push(None, vec![V::Float(f64::NAN)]); + s.map_mut().add_dataset(dataset); + s.run("data.archive", &json!({"layer": "Design"}))?; + s.run("edit.setValue", &json!({"layer": "Design", "row": 0, "field": "level", "value": 1}))?; + s.run("edit.template", &json!({"layer": "Design", "name": "T", "defaults": {"level": "NaN"}}))?; + assert_eq!(s.map().layers[0].dataset.unsaved_numbers(), 0); + assert_eq!(s.map().layers[0].design.unsaved_numbers(), 2); + for (text, report) in [s.project.saved_text(None)?, s.project.clone().recovery_text_with_report()?] { + assert_eq!( + report + .notices() + .iter() + .filter(|notice| notice.contains("2 attribute value(s)") && notice.contains("'Design' (design or archived values)")) + .count(), + 1 + ); + let opened = Project::from_json(&text)?; + let design = &opened.maps[0].layers[0].design; + assert_eq!(design.templates[0].defaults[0].1, V::Null); + assert_eq!(design.archive.as_ref().ok_or("no archive")?[0].before.as_ref().ok_or("no archived feature")?.attrs[0], V::Null); + } + Ok(()) +} diff --git a/crates/map/src/design.rs b/crates/map/src/design.rs index 0d5000953a..6f795dd96a 100644 --- a/crates/map/src/design.rs +++ b/crates/map/src/design.rs @@ -121,6 +121,32 @@ pub struct Design { } impl Design { + /// Counts typed values written as Null in the saved design, including archived feature + /// snapshots, template/subtype defaults, domain codes and contingent combinations. + pub fn unsaved_numbers(&self) -> usize { + let mut count = 0; + for domain in &self.domains { + if let DomainKind::Coded(codes) = &domain.kind { + count += codes.iter().filter(|(value, _)| value.is_non_finite()).count(); + } + } + if let Some(subtypes) = &self.subtypes { + count += subtypes.subtypes.iter().flat_map(|subtype| &subtype.defaults).filter(|(_, value)| value.is_non_finite()).count(); + } + count += self.contingent.iter().flat_map(|set| &set.combos).flatten().filter(|value| value.is_non_finite()).count(); + count += self.templates.iter().flat_map(|template| &template.defaults).filter(|(_, value)| value.is_non_finite()).count(); + count += self + .archive + .iter() + .flatten() + .flat_map(|entry| [&entry.before, &entry.after]) + .flatten() + .flat_map(|feature| &feature.attrs) + .filter(|value| value.is_non_finite()) + .count(); + count + } + pub fn is_empty(&self) -> bool { *self == Design::default() } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f6c119d1f6..da1fbd4172 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1784,9 +1784,7 @@ impl RustGisApp { crate::i18n::fmt("Download offered: {name}. The browser doesn't say whether it saved the file.", &[("name", &file)]), false, ); - for notice in report.notices() { - self.session.notify(notice, false); - } + report.notify(&mut self.session); self.ui.web_file = Some(file); Ok(()) } From 5f024a58e5241183b72ff752ce00b52fe6644f22 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 07:15:42 -0600 Subject: [PATCH 2/4] fix(engine): reject non-finite numeric template defaults Keep refused templates intact and stop integer parsing from inventing values for NaN or infinity. Retain source numbers until counted output. --- crates/data/src/value.rs | 19 +++++++++++++------ crates/engine/src/cmd/design.rs | 8 ++++++-- crates/engine/src/cmd/edit.rs | 2 +- crates/engine/tests/project_save.rs | 8 +++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/crates/data/src/value.rs b/crates/data/src/value.rs index f14d373316..d25ba5f28d 100644 --- a/crates/data/src/value.rs +++ b/crates/data/src/value.rs @@ -75,12 +75,15 @@ impl Value { matches!(self, Value::Float(value) if !value.is_finite()) } - /// Parses a typed source field without discarding a non-finite Double. Its original number + /// Parses a typed numeric source field without discarding a non-finite value. Its original number /// stays available until output, where the counted save notice explains conversion to null. /// Interactive edits use [`Self::parse`] and refuse non-finite input at command ingress. pub fn parse_source(text: &str, ty: FieldType) -> Value { - if ty == FieldType::Double { - return text.trim().parse::().map(Value::Float).unwrap_or(Value::Null); + if ty.is_numeric() + && let Ok(number) = text.trim().parse::() + && (ty == FieldType::Double || !number.is_finite()) + { + return Value::Float(number); } Self::parse(text, ty) } @@ -127,9 +130,11 @@ impl Value { return Value::Null; } match ty { - FieldType::Integer => { - t.parse::().map(Value::Int).or_else(|_| t.parse::().map(|f| Value::Int(f as i64))).unwrap_or(Value::Null) - } + FieldType::Integer => t + .parse::() + .ok() + .or_else(|| t.parse::().ok().filter(|f| f.is_finite()).map(|f| f as i64)) + .map_or(Value::Null, Value::Int), FieldType::Double => t.parse::().ok().filter(|f| f.is_finite()).map_or(Value::Null, Value::Float), FieldType::Boolean => Value::Bool(matches!(t.to_ascii_lowercase().as_str(), "true" | "yes" | "1" | "y")), FieldType::Text => Value::Text(t.to_string()), @@ -425,6 +430,8 @@ mod tests { fn a_number_field_reads_text_that_is_not_a_finite_number_as_null() { for text in ["NaN", "inf", "-inf", "infinity", "1e400"] { assert_eq!(Value::parse(text, FieldType::Double), Value::Null, "{text}"); + assert_eq!(Value::parse(text, FieldType::Integer), Value::Null, "{text}"); + assert!(matches!(Value::parse_source(text, FieldType::Integer), Value::Float(f) if !f.is_finite())); assert!(matches!(Value::parse_source(text, FieldType::Double), Value::Float(f) if !f.is_finite())); } assert_eq!(Value::parse("2.5", FieldType::Double), Value::Float(2.5)); diff --git a/crates/engine/src/cmd/design.rs b/crates/engine/src/cmd/design.rs index 0504c37c74..2155ddce07 100644 --- a/crates/engine/src/cmd/design.rs +++ b/crates/engine/src/cmd/design.rs @@ -1853,6 +1853,7 @@ fn related(s: &mut Session, p: &Json) -> CmdResult { Ok(json!({"related": total, "layers": names})) } +/// Validates defaults before replacing a template; a refused numeric value leaves it intact. fn template(s: &mut Session, p: &Json) -> CmdResult { let id = s.layer_id(p)?; let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or("name is required")?.to_string(); @@ -1867,8 +1868,11 @@ fn template(s: &mut Session, p: &Json) -> CmdResult { for (k, v) in o { let fi = l.dataset.field_index(k).ok_or_else(|| format!("no field named '{k}'"))?; let ty = l.dataset.fields.get(fi).map(|f| f.ty).unwrap_or(FieldType::Text); - defaults - .push((l.dataset.fields.get(fi).map(|f| f.name.clone()).unwrap_or_default(), super::edit::coerce(Value::from_json(v), ty))); + let value = Value::from_json(v); + if super::edit::non_finite_input(&value, ty) { + return Err(format!("field '{k}' requires a finite number")); + } + defaults.push((l.dataset.fields.get(fi).map(|f| f.name.clone()).unwrap_or_default(), super::edit::coerce(value, ty))); } } l.design.templates.retain(|t| !t.name.eq_ignore_ascii_case(&name)); diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 3bbfe632f0..8ec5322053 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -181,7 +181,7 @@ pub(crate) fn coerce_cut(v: Value, ty: FieldType) -> (Value, Cut) { /// Text and native numbers that cannot be retained as numeric field values. /// Ordinary invalid words keep the existing parser's behavior; non-finite numbers are explicit. -fn non_finite_input(value: &Value, ty: FieldType) -> bool { +pub(crate) fn non_finite_input(value: &Value, ty: FieldType) -> bool { ty.is_numeric() && match value { Value::Float(f) => !f.is_finite(), diff --git a/crates/engine/tests/project_save.rs b/crates/engine/tests/project_save.rs index 8bde5bf404..9ba5684122 100644 --- a/crates/engine/tests/project_save.rs +++ b/crates/engine/tests/project_save.rs @@ -313,7 +313,13 @@ fn nonfinite_design_and_archived_values_are_counted_when_output_is_prepared() -> s.map_mut().add_dataset(dataset); s.run("data.archive", &json!({"layer": "Design"}))?; s.run("edit.setValue", &json!({"layer": "Design", "row": 0, "field": "level", "value": 1}))?; - s.run("edit.template", &json!({"layer": "Design", "name": "T", "defaults": {"level": "NaN"}}))?; + s.run("edit.template", &json!({"layer": "Design", "name": "T", "defaults": {"level": 2}}))?; + for text in ["NaN", "inf", "1e400"] { + assert!(s.run("edit.template", &json!({"layer": "Design", "name": "T", "defaults": {"level": text}})).is_err()); + assert_eq!(s.map().layers[0].design.templates[0].defaults[0].1, V::Float(2.0)); + } + // Native callers and imported designs can still hold hostile defaults: output must count them. + s.map_mut().layers[0].design.templates[0].defaults[0].1 = V::Float(f64::NAN); assert_eq!(s.map().layers[0].dataset.unsaved_numbers(), 0); assert_eq!(s.map().layers[0].design.unsaved_numbers(), 2); for (text, report) in [s.project.saved_text(None)?, s.project.clone().recovery_text_with_report()?] { From 69448690c120e63c38dd12deaa7e4cb451ef7c2c Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 10:43:59 -0600 Subject: [PATCH 3/4] fix(data): retain numeric source edits and report export losses --- crates/data/src/export.rs | 27 ++++++- crates/data/src/export/tests.rs | 33 ++++++++- crates/data/src/source/gpkg/edit.rs | 30 +++++++- crates/engine/src/cmd/design.rs | 12 ++- crates/engine/src/cmd/edit.rs | 28 +++++++ crates/engine/src/cmd/share.rs | 38 ++++++++-- crates/engine/tests/project_save.rs | 111 ++++++++++++++++++++++++++++ crates/map/src/source.rs | 35 ++++++++- 8 files changed, 299 insertions(+), 15 deletions(-) diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 40fad08759..fae0679866 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -9,7 +9,7 @@ use std::path::Path; use crate::io::{CsvWriter, GeoJsonWriter, GpkgTable, GpkgWritten, PROJECTED_FORMATS, check_projected, project_each, write_atomic_with}; use crate::raster::Crs; use crate::source::FeatureSource; -use crate::{DataError, Dataset, Feature, GeometryType, Result}; +use crate::{DataError, Dataset, Feature, Field, GeometryType, Result, Value}; /// The feature storage one chunk of an export holds. const CHUNK_BYTES: usize = 32 << 20; @@ -32,6 +32,8 @@ pub struct Exported { pub left_out: usize, /// Rows of the source that couldn't be read, so aren't in the export. pub skipped: usize, + /// Non-finite attributes written as null in GeoJSON or GeoPackage, counted over all chunks. + pub null_numbers: usize, /// Per field of the layer, values written as null because the field's type couldn't hold /// them (GeoParquet; empty for the other formats). See /// [`crate::source::parquet::unfit_note`]. @@ -95,9 +97,11 @@ fn write_chunks_to( let t = if crs == Crs::LonLat { None } else { Some(crs.transformer().map_err(DataError::Format)?) }; let mut total = 0usize; let mut skipped = 0usize; + let mut null_numbers = 0usize; let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> { // A GeoPackage export that starts again feeds the features again, and counts them again. total = 0; + null_numbers = 0; // The writers count the features; a source that doesn't know how many leaves the total at 0. crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX))); let batch = src.visit_chunks(None, bytes, &mut |mut chunk| { @@ -106,6 +110,7 @@ fn write_chunks_to( project_each(&mut chunk, t, crs, false)?; } total = total.saturating_add(chunk.len()); + null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext)); add(&chunk) })?; skipped = batch.skipped; @@ -146,7 +151,25 @@ fn write_chunks_to( Ok::<_, DataError>(0) })?, }; - Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, unfit }, table)) + Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit }, table)) +} + +/// Counts attributes a GeoJSON or GeoPackage writer replaces with null. Other formats have +/// their own loss reports or keep the number as text. Only fields the writer exports count. +pub fn numbers_written_as_null(features: &[Feature], fields: &[Field], ext: &str) -> usize { + if !matches!(ext, "geojson" | "json" | "gpkg") { + return 0; + } + features + .iter() + .flat_map(|f| f.attrs.iter().zip(fields)) + .filter(|(v, field)| { + v.is_non_finite() + || (ext == "gpkg" + && field.ty.is_numeric() + && matches!(v, Value::Text(t) if Value::parse_source(t, field.ty).is_non_finite())) + }) + .count() } /// The kind of a source's first geometry, read up to it; `None` for a source without any (read diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index b7e05cd1a6..ea85801562 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -130,7 +130,11 @@ fn a_streamed_export_writes_what_a_whole_export_writes() { let what = format!("{format} to .{ext} in {}", crs.label()); let unfit = streamed.unfit.clone(); assert!(unfit.iter().all(|n| *n == 0), "{what}: {unfit:?}"); - assert_eq!(streamed, Exported { written: whole.features.len() - left_out, left_out, skipped: 0, unfit }, "{what}"); + assert_eq!( + streamed, + Exported { written: whole.features.len() - left_out, left_out, skipped: 0, null_numbers: 0, unfit }, + "{what}" + ); let (a, b) = (read_back(&a), read_back(&b)); assert_eq!((&a.fields, a.geometry_type, a.crs), (&b.fields, b.geometry_type, b.crs), "{what}"); assert_eq!(a.features, b.features, "{what}"); @@ -140,6 +144,33 @@ fn a_streamed_export_writes_what_a_whole_export_writes() { } } +/// Every chunk contributes to the loss count; a finite export remains quiet. +#[test] +fn nonfinite_numbers_are_counted_across_export_chunks() { + let d = folder("nonfinite"); + for ext in ["geojson", "gpkg"] { + for lossy in [false, true] { + let mut ds = squares(); + if lossy { + ds.features[0].attrs[2] = Value::Float(f64::NAN); + ds.features[1].attrs[2] = Value::Float(f64::INFINITY); + } + let src = Runs::new(&ds, 1); + let out = d.join(format!("numbers-{lossy}.{ext}")); + let exported = write_chunks_in(&src, &ds, &out, Crs::LonLat, 2048).unwrap(); + assert_eq!(exported.null_numbers, if lossy { 2 } else { 0 }); + let read = read_back(&out); + let share = read.field_index("share").unwrap(); + if lossy { + assert_eq!(read.features[0].attrs[share], Value::Null); + assert_eq!(read.features[1].attrs[share], Value::Null); + } else { + assert_eq!(read.features[0].attrs[share], ds.features[0].attrs[2]); + } + } + } +} + /// A GeoPackage export decides its table from the features so far and starts again when a later /// one contradicts it, giving the file a whole export gives. #[test] diff --git a/crates/data/src/source/gpkg/edit.rs b/crates/data/src/source/gpkg/edit.rs index dab45fd6b6..f0f5724a69 100644 --- a/crates/data/src/source/gpkg/edit.rs +++ b/crates/data/src/source/gpkg/edit.rs @@ -267,7 +267,9 @@ impl EditTable { let field = fields.get(i).ok_or_else(|| self.shape_error())?; let old = row.get(c).ok_or_else(|| self.shape_error())?; let attr = f.attrs.get(i).unwrap_or(&Value::Null); - if value(ValueRef::from(old), field.ty) != *attr { + let previous = value(ValueRef::from(old), field.ty); + let same_nan = matches!((&previous, attr), (Value::Float(a), Value::Float(b)) if a.is_nan() && b.is_nan()); + if previous != *attr && !same_nan { sets.push(format!("{} = ?", quote(columns.get(c).ok_or_else(|| self.shape_error())?))); args.push(sql_value(attr, field.ty)); } @@ -794,6 +796,32 @@ mod tests { const REVISION: &str = "2026-01-01T00:00:00.000Z"; + #[test] + fn gpkg_editing_another_field_preserves_an_unchanged_nan() { + let (_dir, path) = fixture(); + let conn = Connection::open(&path).unwrap(); + conn.execute("UPDATE roads SET lanes = 'NaN'", []).unwrap(); + let table = EditTable::open(&path, "roads").unwrap(); + let read = table.read_editable(MAX_EDITABLE).unwrap(); + let lane = table.info().fields.iter().position(|f| f.name == "lanes").unwrap(); + let name = table.info().fields.iter().position(|f| f.name == "name").unwrap(); + assert!(matches!(read.features[0].attrs[lane], Value::Float(v) if v.is_nan())); + let mut changed = read.features[0].clone(); + changed.attrs[name] = Value::Text("edited".into()); + let edits = Edits { updates: vec![(changed, read.versions[0].1)], ..Default::default() }; + let saved = table.save(&edits, &table.info().fields, std::iter::empty(), &read.revision).unwrap().unwrap(); + assert_eq!(saved.changes.updated, 1); + let stored: (String, String) = conn.query_row("SELECT name, lanes FROM roads", [], |r| Ok((r.get(0)?, r.get(1)?))).unwrap(); + assert_eq!(stored, ("edited".into(), "NaN".into())); + // An explicit clear still writes SQL NULL. + let read = table.read_editable(MAX_EDITABLE).unwrap(); + let mut cleared = read.features[0].clone(); + cleared.attrs[lane] = Value::Null; + let edits = Edits { updates: vec![(cleared, read.versions[0].1)], ..Default::default() }; + table.save(&edits, &table.info().fields, std::iter::empty(), &saved.revision).unwrap().unwrap(); + assert!(conn.query_row("SELECT lanes IS NULL FROM roads", [], |r| r.get::<_, bool>(0)).unwrap()); + } + #[test] fn gpkg_insert_revision_is_durable_and_strictly_advances() { let (_dir, path) = fixture(); diff --git a/crates/engine/src/cmd/design.rs b/crates/engine/src/cmd/design.rs index 2155ddce07..a1b6f3cc8d 100644 --- a/crates/engine/src/cmd/design.rs +++ b/crates/engine/src/cmd/design.rs @@ -162,6 +162,9 @@ pub(crate) fn apply_with(s: &mut Session, id: u32, changes: Vec, new_def && let Some(cell) = l.dataset.features.get_mut(*r).and_then(|f| f.attrs.get_mut(fi)) && cell.is_null() { + if super::edit::non_finite_input(v, ty) { + return Err(format!("field '{field}' requires a finite number")); + } *cell = super::edit::coerce(v.clone(), ty); } } @@ -1572,11 +1575,18 @@ fn subtypes(s: &mut Session, p: &Json) -> CmdResult { for v in list.iter().take(1000) { let code = v.get("code").and_then(Json::as_i64).ok_or("each subtype has an integer code")?; let name = v.get("name").and_then(Json::as_str).ok_or("each subtype has a name")?.chars().take(200).collect(); - let defaults = v + let defaults: Vec<(String, Value)> = v .get("defaults") .and_then(Json::as_object) .map(|o| o.iter().map(|(k, v)| (k.clone(), Value::from_json(v))).collect()) .unwrap_or_default(); + for (field, value) in &defaults { + if let Some(ty) = l.dataset.field_index(field).and_then(|i| l.dataset.fields.get(i)).map(|f| f.ty) + && super::edit::non_finite_input(value, ty) + { + return Err(format!("field '{field}' requires a finite number")); + } + } if subs.iter().any(|x: &Subtype| x.code == code) { return Err(format!("subtype code {code} is used twice")); } diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 8ec5322053..28f9289535 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -289,6 +289,9 @@ fn set_value(s: &mut Session, p: &Json) -> CmdResult { let l = s.map_mut().layer_mut(id).ok_or("no layer")?; let ty = l.dataset.fields.get(fi).map(|f| f.ty).ok_or("no field")?; let raw = p.get("value").cloned().unwrap_or(Json::Null); + let input = Value::from_json(&raw); + let field = l.dataset.fields.get(fi).map(|f| f.name.clone()).ok_or("no field")?; + let layer = l.dataset.name.clone(); let v = match raw { // A number that isn't finite (NaN, infinity) is no value a number field holds, nor one a // project can save: refused, as the attribute table refuses it. @@ -298,11 +301,36 @@ fn set_value(s: &mut Session, p: &Json) -> CmdResult { Json::String(t) => Value::parse(&t, ty), other => coerce(Value::from_json(&other), ty), }; + // Keep the existing finite coercion, but say when it discards information. + let intentional_null = + input.is_null() || matches!(&input, Value::Text(t) if t.trim().is_empty() || t.trim().eq_ignore_ascii_case("")); + let loss = if ty.is_numeric() && v.is_null() && !intentional_null { + Some("{layer}, field '{field}', row {row}: 1 attribute value was written as null because it isn't a number") + } else if ty == FieldType::Integer { + let (exact, cut) = coerce_cut(input, ty); + match cut { + Cut::Fraction => Some("{layer}, field '{field}', row {row}: 1 attribute value lost its fraction"), + Cut::Range => Some("{layer}, field '{field}', row {row}: 1 attribute value was clamped to the 64-bit integer range"), + Cut::None if exact != v => Some("{layer}, field '{field}', row {row}: 1 attribute value lost integer precision"), + _ => None, + } + } else { + None + }; let f = l.dataset.features.get_mut(row).ok_or("no such row")?; let before = f.clone(); let cell = f.attrs.get_mut(fi).ok_or("no such row")?; *cell = v; super::design::apply(s, id, vec![super::design::Change { before: Some(before), row: Some(row) }])?; + if let Some(key) = loss { + s.notify_text( + crate::notice_text::NoticeText::format( + key, + [("layer", layer.into()), ("field", field.into()), ("row", row.saturating_add(1).to_string().into())], + ), + false, + ); + } Ok(Json::Null) } diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index e8a5afb121..2fb47a48ff 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -40,6 +40,7 @@ pub fn commands() -> Vec { let gpkg = gpkg_table(p, &ext)?; let t = s.project.tables.get(s.table_index(p)?).ok_or("no such table")?; let default = t.dataset.name.clone(); + let null_numbers = rustgis_data::export::numbers_written_as_null(&t.dataset.features, &t.dataset.fields, &ext); // Tables may retain geometry and its import losses, just as feature layers do. let notes = rustgis_data::export_notes([(&t.dataset, t.dataset.extent())], path, Crs::LonLat); let mut dbf_notes = Vec::new(); @@ -59,6 +60,7 @@ pub fn commands() -> Vec { for note in notes.into_iter().chain(dbf_notes) { s.notify(note, false); } + notify_null_numbers(s, path, null_numbers); let mut out = json!({"path": path}); report_table(s, path, &gpkg, &default, table, &mut out); Ok(out) @@ -619,18 +621,19 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk return Err(format!("{path}: {e}")); } let parquet = matches!(ext.as_str(), "parquet" | "geoparquet"); - let (written, left_out, skipped, table, unfit) = match &chunks { + let (written, left_out, skipped, table, unfit, null_numbers) = match &chunks { Some((src, layer)) => { let (e, table) = rustgis_data::export::write_source_to(src.as_ref(), layer, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; // Counted as the chunks were written, as the layer holds none of them. let unfit = rustgis_data::source::parquet::unfit_note(&layer.fields, &e.unfit, path); - (e.written, e.left_out, e.skipped, table, unfit) + (e.written, e.left_out, e.skipped, table, unfit, e.null_numbers) } None => { let (left_out, table) = rustgis_data::io::write_path_to(ds, Path::new(&path), crs, &gpkg).map_err(|e| e.to_string())?; let unfit = if parquet { rustgis_data::source::parquet::values_note(ds, path) } else { None }; - (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit) + let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, &ext); + (ds.features.len().saturating_sub(left_out), left_out, unread, table, unfit, null_numbers) } }; // A streamed layer's extent is its whole source's. @@ -662,12 +665,17 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk let written_path = Some(path.to_string()); let path = path.to_string(); let default = ds.name.clone(); - let then: Option = table.map(|t| { - Box::new(move |s: &mut Session, out: &mut Value| { - report_table(s, &path, &gpkg, &default, Some(t), out); + let then: Option = if table.is_some() || null_numbers > 0 { + Some(Box::new(move |s: &mut Session, out: &mut Value| { + notify_null_numbers(s, &path, null_numbers); + if let Some(t) = table { + report_table(s, &path, &gpkg, &default, Some(t), out); + } Ok(()) - }) as super::exports::Then - }); + }) as super::exports::Then) + } else { + None + }; Ok(Wrote { answer, notes: said, @@ -678,6 +686,20 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk }) } +/// Reports numeric loss after the file has been written, keeping the output path literal in +/// translated interface text. A clean export emits no loss notice. +fn notify_null_numbers(s: &mut Session, path: &str, count: usize) { + if count > 0 { + s.notify_text( + crate::notice_text::NoticeText::format( + "{path}: {count} attribute value(s) that aren't finite numbers (NaN or infinity) were written as null", + [("path", path.to_string().into()), ("count", count.to_string().into())], + ), + false, + ); + } +} + /// Whether `a` and `b` are the same existing file. fn same_file(a: &Path, b: &Path) -> bool { matches!((std::fs::canonicalize(a), std::fs::canonicalize(b)), (Ok(a), Ok(b)) if a == b) diff --git a/crates/engine/tests/project_save.rs b/crates/engine/tests/project_save.rs index 9ba5684122..fb9d050408 100644 --- a/crates/engine/tests/project_save.rs +++ b/crates/engine/tests/project_save.rs @@ -338,3 +338,114 @@ fn nonfinite_design_and_archived_values_are_counted_when_output_is_prepared() -> } Ok(()) } + +/// Both resident and streamed exports report their numeric replacements once, with the path. +#[test] +fn nonfinite_layer_exports_report_nulls() -> Result<(), String> { + use rustgis_engine::data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value as V}; + let root = std::env::temp_dir().join(format!("rustgis-nonfinite-export-{}", std::process::id())); + std::fs::create_dir_all(&root).map_err(|e| e.to_string())?; + let source = root.join("source.gpkg"); + let mut ds = Dataset::new("Readings", GeometryType::Point, vec![Field::new("level", FieldType::Double)]); + ds.push(Some(Geometry::Point(Coord::new(0.0, 0.0))), vec![V::Float(1.5)]); + rustgis_engine::data::io::write_path(&ds, &source).map_err(|e| e.to_string())?; + for stream in [false, true] { + for lossy in [false, true] { + let conn = rusqlite::Connection::open(&source).map_err(|e| e.to_string())?; + // Attribute UPDATEs also compile the R-tree triggers. This fixture's geometry is + // always the non-empty point (0, 0), so all five spatial functions return zero. + let flags = rusqlite::functions::FunctionFlags::SQLITE_UTF8 + | rusqlite::functions::FunctionFlags::SQLITE_DETERMINISTIC + | rusqlite::functions::FunctionFlags::SQLITE_INNOCUOUS; + for name in ["ST_IsEmpty", "ST_MinX", "ST_MaxX", "ST_MinY", "ST_MaxY"] { + conn.create_scalar_function(name, 1, flags, |_| Ok(0i64)).map_err(|e| e.to_string())?; + } + conn.execute("UPDATE Readings SET level = ?1", [if lossy { "NaN" } else { "1.5" }]).map_err(|e| e.to_string())?; + drop(conn); + let mut s = Session::new(Project::default()); + let id = s.run("layer.addData", &json!({"path": source, "stream": stream}))?["layer"].as_u64().ok_or("no id")?; + assert_eq!(s.map().layer(id as u32).ok_or("no layer")?.is_streamed(), stream); + for ext in ["geojson", "gpkg"] { + let out = root.join(format!("out-{stream}-{lossy}.{ext}")); + if out.exists() { + std::fs::remove_file(&out).map_err(|e| e.to_string())?; + } + let mark = s.notice_mark(); + s.run("share.exportLayer", &json!({"layer": id, "path": out}))?; + let losses: Vec<_> = s.notices_since(mark).iter().filter(|n| n.text.contains("finite numbers")).collect(); + assert_eq!(losses.len(), usize::from(lossy), "stream {stream}: {losses:?}"); + if lossy { + assert!(losses[0].text.contains("1 attribute value(s)")); + assert!(losses[0].text.contains(out.to_str().ok_or("path")?)); + assert!(losses[0].translated(|key, _| key.to_string()).is_some()); + } + let read = rustgis_engine::data::io::read_path(&out).map_err(|e| e.to_string())?.0; + assert_eq!(read.value(0, 0), if lossy { &V::Null } else { &V::Float(1.5) }); + } + } + } + std::fs::remove_dir_all(root).map_err(|e| e.to_string())?; + Ok(()) +} + +/// Interactive and previously imported subtype defaults cannot introduce a non-finite cell. +#[test] +fn nonfinite_subtype_defaults_are_refused_before_mutation() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + let mut s = Session::new(Project::default()); + let id = s.map_mut().add_dataset(Dataset::new( + "Defaults", + GeometryType::Point, + vec![Field::new("kind", FieldType::Integer), Field::new("level", FieldType::Double)], + )); + s.run("data.subtypes", &json!({"layer": id, "field": "kind", "subtypes": [{"code": 1, "name": "One", "defaults": {"level": 2.5}}]}))?; + for text in ["NaN", "inf", "1e400"] { + assert!( + s.run( + "data.subtypes", + &json!({"layer": id, "field": "kind", "subtypes": [{"code": 1, "name": "One", "defaults": {"level": text}}]}) + ) + .is_err() + ); + let sub = s.map().layer(id).ok_or("no layer")?.design.subtypes.as_ref().ok_or("no subtypes")?; + assert_eq!(sub.subtypes[0].defaults[0].1, V::Float(2.5)); + } + let add = json!({"layer": id, "geometry": {"type": "Point", "coordinates": [0,0]}, "attributes": {"kind": 1}}); + s.run("edit.addFeature", &add)?; + assert_eq!(s.map().layer(id).ok_or("no layer")?.dataset.value(0, 1), &V::Float(2.5)); + s.map_mut().layer_mut(id).ok_or("no layer")?.design.subtypes.as_mut().ok_or("no subtypes")?.subtypes[0].defaults[0].1 = + V::Text("NaN".into()); + let count = s.map().layer(id).ok_or("no layer")?.dataset.features.len(); + assert!(s.run("edit.addFeature", &add).is_err()); + assert_eq!(s.map().layer(id).ok_or("no layer")?.dataset.features.len(), count); + Ok(()) +} + +/// Set Value keeps its finite coercions, while reporting every lossy one and staying quiet for exact input. +#[test] +fn set_value_reports_numeric_approximations() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + let mut s = Session::new(Project::default()); + let mut ds = Dataset::new("Numbers", GeometryType::None, vec![Field::new("n", FieldType::Integer)]); + ds.push(None, vec![V::Int(1)]); + let id = s.map_mut().add_dataset(ds); + for (text, expected, fragment) in [ + ("2.7", V::Int(2), "lost its fraction"), + ("1e30", V::Int(i64::MAX), "clamped"), + ("unknown", V::Null, "written as null"), + ("9007199254740993.0", V::Int(9_007_199_254_740_992), "lost integer precision"), + ("", V::Null, ""), + ("3", V::Int(3), ""), + ] { + let mark = s.notice_mark(); + s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "n", "value": text}))?; + assert_eq!(s.map().layer(id).ok_or("no layer")?.dataset.value(0, 0), &expected); + let losses: Vec<_> = s.notices_since(mark).iter().filter(|n| n.text.contains("1 attribute value")).collect(); + assert_eq!(losses.len(), usize::from(!fragment.is_empty())); + if !fragment.is_empty() { + assert!(losses[0].text.contains(fragment)); + assert!(losses[0].text.contains("Numbers, field 'n', row 1")); + } + } + Ok(()) +} diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index e014050d3c..3206f63f2a 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -239,13 +239,20 @@ pub struct GpkgEdit { } /// How a loaded row is told apart from the same row edited: a hash of its geometry and -/// attributes (FNV-1a over their JSON, the same in every build). +/// attributes (FNV-1a over their JSON, the same in every build). Non-finite attributes also +/// contribute their position and native bits, since output JSON substitutes Null for them. pub fn row_hash(f: &rustgis_data::Feature) -> u64 { /// FNV-1a over the bytes written to it, kept as they come (no buffer per row). struct Fnv(u64); + impl Fnv { + /// Feeds bytes without buffering; also used for values JSON cannot distinguish. + fn bytes(&mut self, buf: &[u8]) { + self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + } + } impl std::io::Write for Fnv { fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + self.bytes(buf); Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { @@ -257,6 +264,15 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { if serde_json::to_writer(&mut h, &(&f.geometry, &f.attrs)).is_err() { return 0xcbf2_9ce4_8422_2325; } + for (index, value) in f.attrs.iter().enumerate() { + if let rustgis_data::Value::Float(number) = value + && !number.is_finite() + { + h.bytes(b"\xffnon-finite:"); + h.bytes(&(index as u64).to_le_bytes()); + h.bytes(&number.to_bits().to_le_bytes()); + } + } h.0 } @@ -1686,6 +1702,21 @@ mod tests { assert_eq!(super::row_hash(&f), 8_547_063_913_914_077_080); } + /// Clearing a non-finite source cell is a write-back update even though output JSON uses Null. + #[test] + fn clearing_nonfinite_source_values_is_a_row_update() { + use rustgis_data::{Feature, Value}; + for number in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let loaded = Feature { id: 1, geometry: None, attrs: vec![Value::Float(number)] }; + let versions: super::Versions = [(1, super::RowVersion { version: 7, hash: super::row_hash(&loaded) })].into_iter().collect(); + assert!(versions.changes(std::slice::from_ref(&loaded)).is_empty()); + let cleared = Feature { attrs: vec![Value::Null], ..loaded }; + let changes = versions.changes(std::slice::from_ref(&cleared)); + assert_eq!(changes.updates.iter().map(|(feature, version)| (feature.id, *version)).collect::>(), [(1, 7)]); + assert_eq!(changes.updates[0].0.attrs, [Value::Null]); + } + } + #[test] fn changes_are_what_differs_from_the_rows_as_read() { use rustgis_data::{Coord, Feature, Geometry, Value}; From 51d0a22c76f24ce7d46f349ede6a21c4ba380a96 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 15:03:18 -0600 Subject: [PATCH 4/4] fix(engine): report numeric coercion and protect editable source output Keep fractional and oversized source numbers, report lossy feature creation, and refuse project output that would manufacture edits from non-finite rows. --- crates/data/src/mapinfo.rs | 11 ++++ crates/data/src/value.rs | 12 +++-- crates/engine/src/cmd/edit.rs | 52 ++++++++++++------ crates/engine/src/durable.rs | 26 ++++++--- crates/engine/tests/project_save.rs | 81 +++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 28 deletions(-) diff --git a/crates/data/src/mapinfo.rs b/crates/data/src/mapinfo.rs index 540bba7e5e..7bce3b2ea0 100644 --- a/crates/data/src/mapinfo.rs +++ b/crates/data/src/mapinfo.rs @@ -421,6 +421,17 @@ mod tests { const MIF: &str = "Version 300\nCharset \"WindowsLatin1\"\nDelimiter \",\"\nCoordSys Earth Projection 1, 104\nColumns 3\n name Char(20)\n pop Integer\n area Decimal(10,2)\nData\n\nPoint 10 20\n Symbol (35,0,12)\nPline 3\n10 20\n11 21\n12 20\n Pen (1,2,0)\nRegion 2\n 5\n0 0\n10 0\n10 10\n0 10\n0 0\n 4\n2 2\n4 2\n3 4\n2 2\n Pen (1,2,0)\n Brush (2,16777215,16777215)\n Center 5 5\nPline Multiple 2\n 2\n0 0\n1 1\n 2\n2 2\n3 3\nText \"Hello world\"\n 1 1 2 2\nNone\nRect 0 0 1 1\n"; const MID: &str = "\"A\",1,1.5\n\"B, c\",2,2.5\n\"C\",3,3.5\n\"D\",4,4.5\n\"E\",5,5.5\n\"F\",6,6.5\n\"G\",7,7.5\n"; + #[test] + fn integer_source_numbers_are_not_truncated_or_clamped() { + let mif = "Version 300\nCoordSys Earth Projection 1, 104\nColumns 1\n n Integer\nData\nPoint 0 0\nPoint 1 1\nPoint 2 2\n"; + let (ds, report) = read(mif, Some("2.7\n1e30\n3\n"), "numbers").unwrap(); + assert_eq!(ds.fields[0].ty, FieldType::Integer); + assert_eq!(ds.value(0, 0), &Value::Float(2.7)); + assert_eq!(ds.value(1, 0), &Value::Float(1e30)); + assert_eq!(ds.value(2, 0), &Value::Int(3)); + assert!(report.messages.is_empty(), "{:?}", report.messages); + } + #[test] fn widths_with_spaces_and_bad_mid_rows() { // `Decimal (10, 2)` is one column; a .mid record the reader rejects keeps its place. diff --git a/crates/data/src/value.rs b/crates/data/src/value.rs index d25ba5f28d..5ae55b0cf9 100644 --- a/crates/data/src/value.rs +++ b/crates/data/src/value.rs @@ -75,13 +75,14 @@ impl Value { matches!(self, Value::Float(value) if !value.is_finite()) } - /// Parses a typed numeric source field without discarding a non-finite value. Its original number - /// stays available until output, where the counted save notice explains conversion to null. + /// Parses a typed numeric source field without truncating or clamping its numbers. Integer + /// text is kept as an integer when exact; other numeric text is kept as a float, as native + /// real source cells are. Non-finite values stay available until counted output conversion. /// Interactive edits use [`Self::parse`] and refuse non-finite input at command ingress. pub fn parse_source(text: &str, ty: FieldType) -> Value { if ty.is_numeric() && let Ok(number) = text.trim().parse::() - && (ty == FieldType::Double || !number.is_finite()) + && (ty == FieldType::Double || text.trim().parse::().is_err()) { return Value::Float(number); } @@ -425,7 +426,7 @@ mod tests { use super::*; /// Interactive Double text rejects non-finite numbers, while source parsing preserves them - /// for the counted output notice. Finite numbers keep their established parsing. + /// for the counted output notice. Exact integer text stays integer; source decimals stay real. #[test] fn a_number_field_reads_text_that_is_not_a_finite_number_as_null() { for text in ["NaN", "inf", "-inf", "infinity", "1e400"] { @@ -437,6 +438,9 @@ mod tests { assert_eq!(Value::parse("2.5", FieldType::Double), Value::Float(2.5)); assert_eq!(Value::parse("2.5", FieldType::Integer), Value::Int(2)); assert_eq!(Value::parse("1e3", FieldType::Integer), Value::Int(1000)); + assert_eq!(Value::parse_source("3", FieldType::Integer), Value::Int(3)); + assert_eq!(Value::parse_source("2.7", FieldType::Integer), Value::Float(2.7)); + assert_eq!(Value::parse_source("1e30", FieldType::Integer), Value::Float(1e30)); } /// JSON cannot represent non-finite numbers; output uses a null value with the save report. diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 28f9289535..ad5cea3a2d 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -47,6 +47,8 @@ pub(crate) fn add_feature(s: &mut Session, p: &Json) -> CmdResult { } None => None, }; + let mut losses = Vec::new(); + let layer = l.dataset.name.clone(); let attrs: Result, String> = l .dataset .fields @@ -59,7 +61,11 @@ pub(crate) fn add_feature(s: &mut Session, p: &Json) -> CmdResult { if non_finite_input(&value, f.ty) { return Err(format!("field '{}' requires a finite number", f.name)); } - Ok(coerce(value, f.ty)) + let coerced = coerce(value.clone(), f.ty); + if let Some(key) = numeric_loss(&value, &coerced, f.ty) { + losses.push((f.name.clone(), key)); + } + Ok(coerced) }) .collect(); l.dataset.push(Some(g), attrs?); @@ -67,6 +73,15 @@ pub(crate) fn add_feature(s: &mut Session, p: &Json) -> CmdResult { let row = l.dataset.features.len() - 1; l.selection = vec![row]; super::design::apply(s, id, vec![super::design::Change { before: None, row: Some(row) }])?; + for (field, key) in losses { + s.notify_text( + crate::notice_text::NoticeText::format( + key, + [("layer", layer.clone().into()), ("field", field.into()), ("row", row.saturating_add(1).to_string().into())], + ), + false, + ); + } Ok(json!({"row": row})) } @@ -282,6 +297,25 @@ fn field_index(s: &Session, id: u32, name: &str) -> Result { s.map().layer(id).and_then(|l| l.dataset.field_index(name)).ok_or_else(|| format!("no field named '{name}'")) } +/// A numeric conversion's loss, shared by edits and feature creation. Null entered explicitly is quiet. +fn numeric_loss(input: &Value, v: &Value, ty: FieldType) -> Option<&'static str> { + let intentional_null = + input.is_null() || matches!(&input, Value::Text(t) if t.trim().is_empty() || t.trim().eq_ignore_ascii_case("")); + if ty.is_numeric() && v.is_null() && !intentional_null { + Some("{layer}, field '{field}', row {row}: 1 attribute value was written as null because it isn't a number") + } else if ty == FieldType::Integer { + let (exact, cut) = coerce_cut(input.clone(), ty); + match cut { + Cut::Fraction => Some("{layer}, field '{field}', row {row}: 1 attribute value lost its fraction"), + Cut::Range => Some("{layer}, field '{field}', row {row}: 1 attribute value was clamped to the 64-bit integer range"), + Cut::None if &exact != v => Some("{layer}, field '{field}', row {row}: 1 attribute value lost integer precision"), + _ => None, + } + } else { + None + } +} + fn set_value(s: &mut Session, p: &Json) -> CmdResult { let id = s.layer_id(p)?; let row = p.get("row").and_then(Json::as_u64).ok_or("row is required")? as usize; @@ -302,21 +336,7 @@ fn set_value(s: &mut Session, p: &Json) -> CmdResult { other => coerce(Value::from_json(&other), ty), }; // Keep the existing finite coercion, but say when it discards information. - let intentional_null = - input.is_null() || matches!(&input, Value::Text(t) if t.trim().is_empty() || t.trim().eq_ignore_ascii_case("")); - let loss = if ty.is_numeric() && v.is_null() && !intentional_null { - Some("{layer}, field '{field}', row {row}: 1 attribute value was written as null because it isn't a number") - } else if ty == FieldType::Integer { - let (exact, cut) = coerce_cut(input, ty); - match cut { - Cut::Fraction => Some("{layer}, field '{field}', row {row}: 1 attribute value lost its fraction"), - Cut::Range => Some("{layer}, field '{field}', row {row}: 1 attribute value was clamped to the 64-bit integer range"), - Cut::None if exact != v => Some("{layer}, field '{field}', row {row}: 1 attribute value lost integer precision"), - _ => None, - } - } else { - None - }; + let loss = numeric_loss(&input, &v, ty); let f = l.dataset.features.get_mut(row).ok_or("no such row")?; let before = f.clone(); let cell = f.attrs.get_mut(fi).ok_or("no such row")?; diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs index ef201d7b6b..873c9696a7 100644 --- a/crates/engine/src/durable.rs +++ b/crates/engine/src/durable.rs @@ -263,12 +263,13 @@ pub(crate) struct Prepare<'a> { pub(crate) report: Report, relative_to: Option<&'a str>, pub(crate) depth: usize, + refusal: Option, } impl<'a> Prepare<'a> { /// Makes every replacement and journal allocation before the live borrow is mutated. fn new(relative_to: Option<&'a str>, depth: usize) -> Self { - Self { owners: Vec::new(), edits: Vec::new(), report: Report::default(), relative_to, depth } + Self { owners: Vec::new(), edits: Vec::new(), report: Report::default(), relative_to, depth, refusal: None } } /// Prepares one child under a fixed, typed owning location. @@ -283,9 +284,17 @@ impl<'a> Prepare<'a> { self.record_numbers(dataset.unsaved_numbers(), format!("'{}'", crate::project::short_name(&dataset.name)).into()); } - /// Counts saved design values separately from current rows so archived snapshots and - /// feature defaults cannot be silently changed by Value's human-readable serialization. - fn count_design_numbers(&mut self, layer: &rustgis_map::Layer) { + /// Counts saved row and design values. Editable source rows cannot be nulled in output: + /// their retained baseline hashes would turn the substitution into a source edit on reopen. + fn count_layer_numbers(&mut self, layer: &rustgis_map::Layer) { + let count = layer.dataset.unsaved_numbers(); + if count > 0 && layer.edit_versions().is_some() && self.refusal.is_none() { + self.refusal = Some(format!( + "project output refused: {count} non-finite attribute value(s) in editable layer '{}' cannot be saved without creating source edits; explicitly clear or replace them first", + crate::project::short_name(&layer.name) + )); + } + self.count_numbers(&layer.dataset); let place = crate::notice_text::NoticeText::format( "{name} (design or archived values)", [("name", format!("'{}'", crate::project::short_name(&layer.name)).into())], @@ -458,6 +467,9 @@ impl Drop for Transaction<'_, T> { fn prepared<'a>(value: &impl Durable, relative_to: Option<&'a str>, depth: usize) -> Result, String> { let mut p = Prepare::new(relative_to, depth); value.prepare(&mut p); + if let Some(reason) = p.refusal.take() { + return Err(reason); + } let mut seen = HashSet::new(); if p.edits.iter().any(|e| !seen.insert(&e.at)) { return Err("internal duplicate output location".into()); @@ -805,8 +817,7 @@ impl Durable for rustgis_map::Map { p.string(Leaf::Map(s.at), s.value, s.address, s.local_path); } for l in &self.layers { - p.count_numbers(&l.dataset); - p.count_design_numbers(l); + p.count_layer_numbers(l); } } @@ -826,8 +837,7 @@ impl Durable for rustgis_map::Layer { for (role, value, address, path) in self.output_slots() { p.string(Leaf::Layer(role), value, address, path); } - p.count_numbers(&self.dataset); - p.count_design_numbers(self); + p.count_layer_numbers(self); } /// Resolves a prepared location through owned fields, without allocation or policy. diff --git a/crates/engine/tests/project_save.rs b/crates/engine/tests/project_save.rs index fb9d050408..be1821d67d 100644 --- a/crates/engine/tests/project_save.rs +++ b/crates/engine/tests/project_save.rs @@ -449,3 +449,84 @@ fn set_value_reports_numeric_approximations() -> Result<(), String> { } Ok(()) } + +#[test] +fn add_feature_reports_numeric_approximations() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + let mut s = Session::new(Project::default()); + let ds = Dataset::new("Numbers", GeometryType::Point, vec![Field::new("n", FieldType::Integer)]); + let id = s.map_mut().add_dataset(ds); + for (row, (text, expected, fragment)) in [ + ("2.7", V::Int(2), "lost its fraction"), + ("1e30", V::Int(i64::MAX), "clamped"), + ("unknown", V::Null, "written as null"), + ("", V::Null, ""), + ("3", V::Int(3), ""), + ] + .into_iter() + .enumerate() + { + let mark = s.notice_mark(); + s.run("edit.addFeature", &json!({"layer": id, "geometry": {"type":"Point", "coordinates":[0,0]}, "attributes": {"n": text}}))?; + assert_eq!(s.map().layer(id).ok_or("no layer")?.dataset.value(row, 0), &expected); + let losses: Vec<_> = s.notices_since(mark).iter().filter(|n| n.text.contains("1 attribute value")).collect(); + assert_eq!(losses.len(), usize::from(!fragment.is_empty())); + if !fragment.is_empty() { + assert!(losses[0].text.contains(fragment)); + assert!(losses[0].text.contains(&format!("Numbers, field 'n', row {}", row + 1))); + } + } + Ok(()) +} + +/// Null substitution cannot be persisted with an editable source's non-finite row baseline. +#[test] +fn project_output_refuses_to_manufacture_source_edits() -> Result<(), String> { + use rustgis_engine::data::{Dataset, Field, FieldType, GeometryType, Value as V}; + use rustgis_engine::map::{ + Layer, + source::{PostgisEdit, PostgisOrigin, RowVersion, row_hash}, + }; + let nonce = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos(); + let path = std::env::temp_dir().join(format!("rustgis-editable-nonfinite-{}-{nonce}.rgis", std::process::id())); + std::fs::write(&path, b"keep destination").map_err(|e| e.to_string())?; + for number in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut s = Session::new(Project::default()); + let mut ds = Dataset::new("Editable", GeometryType::None, vec![Field::new("n", FieldType::Double)]); + ds.push(None, vec![V::Float(number)]); + let versions = ds.features.iter().map(|f| (f.id, RowVersion { version: 7, hash: row_hash(f) })).collect(); + let id = s.map_mut().add_dataset(ds); + let layer = s.map_mut().layer_mut(id).ok_or("no layer")?; + layer.postgis = Some(PostgisEdit { + origin: PostgisOrigin { database: "postgres:dbname=example".into(), table: "numbers".into(), family: None, crs: None }, + key: "id".into(), + srid: 4326, + writable: true, + condition: None, + limit: 0, + versions, + }); + // Demonstrate why serializing the value independently is unsafe for editable snapshots. + let raw = serde_json::to_string(layer).map_err(|e| e.to_string())?; + let back: Layer = serde_json::from_str(&raw).map_err(|e| e.to_string())?; + assert_eq!(back.edit_versions().ok_or("no baseline")?.changes(&back.dataset.features).updates.len(), 1); + for error in [s.project.saved_text(None).unwrap_err(), s.project.clone().recovery_text_with_report().unwrap_err()] { + assert!(error.contains("1 non-finite attribute value(s)"), "{error}"); + assert!(error.contains("editable layer 'Editable'"), "{error}"); + } + assert!(s.project.save(&path).is_err()); + assert_eq!(std::fs::read(&path).map_err(|e| e.to_string())?, b"keep destination"); + let layer = s.map().layer(id).ok_or("no layer")?; + assert!(layer.dataset.value(0, 0).is_non_finite()); + assert!(layer.edit_versions().ok_or("no baseline")?.changes(&layer.dataset.features).is_empty()); + // Clearing it intentionally remains a real pending source edit after save and reopen. + s.run("edit.setValue", &json!({"layer":id,"row":0,"field":"n","value":null}))?; + let (text, report) = s.project.saved_text(None)?; + assert!(report.notices().is_empty()); + let reopened = Project::from_json(&text)?; + let layer = reopened.maps[0].layer(id).ok_or("no reopened layer")?; + assert_eq!(layer.edit_versions().ok_or("no baseline")?.changes(&layer.dataset.features).updates.len(), 1); + } + std::fs::remove_file(path).map_err(|e| e.to_string())?; + Ok(()) +}