Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions apps/rustgis-web/src/web.rs
Original file line number Diff line number Diff line change
Expand Up @@ -414,9 +414,7 @@ impl Embed {
(true, _) => match app.session.peer_text(rustgis_engine::Peer::Embedder) {
Ok((text, report)) => {
self.answer(&crate::embed::project(&text));
for notice in report.notices() {
app.session.notify(notice, false);
}
report.notify(&mut app.session);
}
Err(e) => app.session.notify(e, true),
},
Expand Down
27 changes: 25 additions & 2 deletions crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::path::Path;
use crate::io::{CsvWriter, GeoJsonWriter, GpkgTable, GpkgWritten, PROJECTED_FORMATS, check_projected, project_each, write_atomic_with};
use crate::raster::Crs;
use crate::source::FeatureSource;
use crate::{DataError, Dataset, Feature, GeometryType, Result};
use crate::{DataError, Dataset, Feature, Field, GeometryType, Result, Value};

/// The feature storage one chunk of an export holds.
const CHUNK_BYTES: usize = 32 << 20;
Expand All @@ -32,6 +32,8 @@ pub struct Exported {
pub left_out: usize,
/// Rows of the source that couldn't be read, so aren't in the export.
pub skipped: usize,
/// Non-finite attributes written as null in GeoJSON or GeoPackage, counted over all chunks.
pub null_numbers: usize,
/// Per field of the layer, values written as null because the field's type couldn't hold
/// them (GeoParquet; empty for the other formats). See
/// [`crate::source::parquet::unfit_note`].
Expand Down Expand Up @@ -95,9 +97,11 @@ fn write_chunks_to(
let t = if crs == Crs::LonLat { None } else { Some(crs.transformer().map_err(DataError::Format)?) };
let mut total = 0usize;
let mut skipped = 0usize;
let mut null_numbers = 0usize;
let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> {
// A GeoPackage export that starts again feeds the features again, and counts them again.
total = 0;
null_numbers = 0;
// The writers count the features; a source that doesn't know how many leaves the total at 0.
crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX)));
let batch = src.visit_chunks(None, bytes, &mut |mut chunk| {
Expand All @@ -106,6 +110,7 @@ fn write_chunks_to(
project_each(&mut chunk, t, crs, false)?;
}
total = total.saturating_add(chunk.len());
null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext));
add(&chunk)
})?;
skipped = batch.skipped;
Expand Down Expand Up @@ -146,7 +151,25 @@ fn write_chunks_to(
Ok::<_, DataError>(0)
})?,
};
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, unfit }, table))
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit }, table))
}

/// Counts attributes a GeoJSON or GeoPackage writer replaces with null. Other formats have
/// their own loss reports or keep the number as text. Only fields the writer exports count.
pub fn numbers_written_as_null(features: &[Feature], fields: &[Field], ext: &str) -> usize {
if !matches!(ext, "geojson" | "json" | "gpkg") {
return 0;
}
features
.iter()
.flat_map(|f| f.attrs.iter().zip(fields))
.filter(|(v, field)| {
v.is_non_finite()
|| (ext == "gpkg"
&& field.ty.is_numeric()
&& matches!(v, Value::Text(t) if Value::parse_source(t, field.ty).is_non_finite()))
})
.count()
}

/// The kind of a source's first geometry, read up to it; `None` for a source without any (read
Expand Down
33 changes: 32 additions & 1 deletion crates/data/src/export/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,11 @@ fn a_streamed_export_writes_what_a_whole_export_writes() {
let what = format!("{format} to .{ext} in {}", crs.label());
let unfit = streamed.unfit.clone();
assert!(unfit.iter().all(|n| *n == 0), "{what}: {unfit:?}");
assert_eq!(streamed, Exported { written: whole.features.len() - left_out, left_out, skipped: 0, unfit }, "{what}");
assert_eq!(
streamed,
Exported { written: whole.features.len() - left_out, left_out, skipped: 0, null_numbers: 0, unfit },
"{what}"
);
let (a, b) = (read_back(&a), read_back(&b));
assert_eq!((&a.fields, a.geometry_type, a.crs), (&b.fields, b.geometry_type, b.crs), "{what}");
assert_eq!(a.features, b.features, "{what}");
Expand All @@ -140,6 +144,33 @@ fn a_streamed_export_writes_what_a_whole_export_writes() {
}
}

/// Every chunk contributes to the loss count; a finite export remains quiet.
#[test]
fn nonfinite_numbers_are_counted_across_export_chunks() {
let d = folder("nonfinite");
for ext in ["geojson", "gpkg"] {
for lossy in [false, true] {
let mut ds = squares();
if lossy {
ds.features[0].attrs[2] = Value::Float(f64::NAN);
ds.features[1].attrs[2] = Value::Float(f64::INFINITY);
}
let src = Runs::new(&ds, 1);
let out = d.join(format!("numbers-{lossy}.{ext}"));
let exported = write_chunks_in(&src, &ds, &out, Crs::LonLat, 2048).unwrap();
assert_eq!(exported.null_numbers, if lossy { 2 } else { 0 });
let read = read_back(&out);
let share = read.field_index("share").unwrap();
if lossy {
assert_eq!(read.features[0].attrs[share], Value::Null);
assert_eq!(read.features[1].attrs[share], Value::Null);
} else {
assert_eq!(read.features[0].attrs[share], ds.features[0].attrs[2]);
}
}
}
}

/// A GeoPackage export decides its table from the features so far and starts again when a later
/// one contradicts it, giving the file a whole export gives.
#[test]
Expand Down
2 changes: 1 addition & 1 deletion crates/data/src/gml.rs
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,7 @@ pub fn read(file_name: &str, bytes: &[u8]) -> Result<Vec<(Dataset, ReadReport)>>
let e = g.extent();
outside |= e.min_x < -180.0 || e.max_x > 180.0 || e.min_y < -90.0 || e.max_y > 90.0;
}
let row = ds.fields.iter().map(|f| attrs.get(&f.name).map(|v| Value::parse(v, f.ty)).unwrap_or_default()).collect();
let row = ds.fields.iter().map(|f| attrs.get(&f.name).map(|v| Value::parse_source(v, f.ty)).unwrap_or_default()).collect();
ds.push_next(g, row);
}
ds.geometry_type = crate::io::dominant_type(&ds);
Expand Down
4 changes: 2 additions & 2 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1655,7 +1655,7 @@ pub(crate) fn read_rows(
let attrs = columns
.iter()
.zip(&ds.fields)
.map(|(&i, f)| Value::parse(&unguard(r.get(i).map(|s| s.as_str()).unwrap_or("")), f.ty))
.map(|(&i, f)| Value::parse_source(&unguard(r.get(i).map(|s| s.as_str()).unwrap_or("")), f.ty))
.collect();
ds.push_next(geom, attrs);
}
Expand Down Expand Up @@ -1794,7 +1794,7 @@ pub(crate) fn type_text_columns(ds: &mut Dataset, keep: &[&str]) {
if let Some(v) = f.attrs.get_mut(i)
&& let Value::Text(t) = v
{
*v = if t.trim().is_empty() { Value::Null } else { Value::parse(t, ty) };
*v = if t.trim().is_empty() { Value::Null } else { Value::parse_source(t, ty) };
}
}
}
Expand Down
12 changes: 12 additions & 0 deletions crates/data/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,18 @@ pub struct Dataset {
pub windowed: bool,
}

impl Dataset {
/// How many of its values a saved project holds as null, not as they are: numbers that aren't
/// finite (NaN, infinity), which JSON can't write ([`Value`]'s `Serialize`). None for a window
/// onto a source, whose rows aren't saved.
pub fn unsaved_numbers(&self) -> usize {
if self.windowed {
return 0;
}
self.features.iter().flat_map(|f| &f.attrs).filter(|v| v.is_non_finite()).count()
}
}

impl Serialize for Dataset {
/// Saves CRS recovery and dropped Z/M state with the dataset, but omits disposable streamed rows.
fn serialize<S: serde::Serializer>(&self, s: S) -> std::result::Result<S::Ok, S::Error> {
Expand Down
13 changes: 12 additions & 1 deletion crates/data/src/mapinfo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ pub fn read(mif: &str, mid: Option<&str>, name: &str) -> Result<(Dataset, ReadRe
.fields
.iter()
.enumerate()
.map(|(c, f)| Value::parse(row.and_then(|r| r.get(c)).map(String::as_str).unwrap_or(""), f.ty))
.map(|(c, f)| Value::parse_source(row.and_then(|r| r.get(c)).map(String::as_str).unwrap_or(""), f.ty))
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
.collect();
ds.push_next(g, attrs);
}
Expand All @@ -421,6 +421,17 @@ mod tests {
const MIF: &str = "Version 300\nCharset \"WindowsLatin1\"\nDelimiter \",\"\nCoordSys Earth Projection 1, 104\nColumns 3\n name Char(20)\n pop Integer\n area Decimal(10,2)\nData\n\nPoint 10 20\n Symbol (35,0,12)\nPline 3\n10 20\n11 21\n12 20\n Pen (1,2,0)\nRegion 2\n 5\n0 0\n10 0\n10 10\n0 10\n0 0\n 4\n2 2\n4 2\n3 4\n2 2\n Pen (1,2,0)\n Brush (2,16777215,16777215)\n Center 5 5\nPline Multiple 2\n 2\n0 0\n1 1\n 2\n2 2\n3 3\nText \"Hello world\"\n 1 1 2 2\nNone\nRect 0 0 1 1\n";
const MID: &str = "\"A\",1,1.5\n\"B, c\",2,2.5\n\"C\",3,3.5\n\"D\",4,4.5\n\"E\",5,5.5\n\"F\",6,6.5\n\"G\",7,7.5\n";

#[test]
fn integer_source_numbers_are_not_truncated_or_clamped() {
let mif = "Version 300\nCoordSys Earth Projection 1, 104\nColumns 1\n n Integer\nData\nPoint 0 0\nPoint 1 1\nPoint 2 2\n";
let (ds, report) = read(mif, Some("2.7\n1e30\n3\n"), "numbers").unwrap();
assert_eq!(ds.fields[0].ty, FieldType::Integer);
assert_eq!(ds.value(0, 0), &Value::Float(2.7));
assert_eq!(ds.value(1, 0), &Value::Float(1e30));
assert_eq!(ds.value(2, 0), &Value::Int(3));
assert!(report.messages.is_empty(), "{:?}", report.messages);
}

#[test]
fn widths_with_spaces_and_bad_mid_rows() {
// `Decimal (10, 2)` is one column; a .mid record the reader rejects keeps its place.
Expand Down
4 changes: 2 additions & 2 deletions crates/data/src/source/gpkg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1149,7 +1149,7 @@ fn sql_value(v: &Value, ty: FieldType) -> rusqlite::types::Value {
(Value::Float(_), _) => Sql::Null,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GeoPackage export still nulls non-finite numbers silently (pre-existing, but on a line this PR edits).

(Value::Float(_), _) => Sql::Null drops NaN/infinity with no notice. The project save path now counts these, but this writer doesn't. That breaks the "never silent" rule for writers. This PR widens the set of values that can reach this arm: an Integer field can now hold a non-finite Float after a source read (see parse_source in value.rs).

Suggest counting the nulled values and returning a notice through the export's report, following the same pattern as Dataset::unsaved_numbers. It can be a follow-up, but the PR description says output losses are counted.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 6944869. GeoPackage and GeoJSON exports now count values written as null and publish a translated notice with the count and output path. The regression covers clean and lossy resident and streamed exports.

(Value::Bool(b), _) => Sql::Integer(i64::from(*b)),
(Value::Text(s), FieldType::Text) => Sql::Text(s.clone()),
(v, ty) => match Value::parse(&v.to_string(), ty) {
(v, ty) => match Value::parse_source(&v.to_string(), ty) {
Value::Int(x) => Sql::Integer(x),
Value::Float(x) if x.is_finite() => Sql::Real(x),
Value::Bool(b) => Sql::Integer(i64::from(b)),
Expand All @@ -1167,7 +1167,7 @@ fn value(v: ValueRef<'_>, ty: FieldType) -> Value {
(ValueRef::Real(f), FieldType::Integer) if f.fract() == 0.0 && f.abs() < 9e15 => Value::Int(f as i64),
(ValueRef::Real(f), _) => Value::Float(f),
(ValueRef::Text(t), FieldType::Text) => Value::Text(String::from_utf8_lossy(t).into_owned()),
(ValueRef::Text(t), ty) => Value::parse(&String::from_utf8_lossy(t), ty),
(ValueRef::Text(t), ty) => Value::parse_source(&String::from_utf8_lossy(t), ty),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
(ValueRef::Blob(_), _) => Value::Null,
}
}
Expand Down
30 changes: 29 additions & 1 deletion crates/data/src/source/gpkg/edit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,9 @@ impl EditTable {
let field = fields.get(i).ok_or_else(|| self.shape_error())?;
let old = row.get(c).ok_or_else(|| self.shape_error())?;
let attr = f.attrs.get(i).unwrap_or(&Value::Null);
if value(ValueRef::from(old), field.ty) != *attr {
let previous = value(ValueRef::from(old), field.ty);
let same_nan = matches!((&previous, attr), (Value::Float(a), Value::Float(b)) if a.is_nan() && b.is_nan());
if previous != *attr && !same_nan {
sets.push(format!("{} = ?", quote(columns.get(c).ok_or_else(|| self.shape_error())?)));
args.push(sql_value(attr, field.ty));
}
Expand Down Expand Up @@ -794,6 +796,32 @@ mod tests {

const REVISION: &str = "2026-01-01T00:00:00.000Z";

#[test]
fn gpkg_editing_another_field_preserves_an_unchanged_nan() {
let (_dir, path) = fixture();
let conn = Connection::open(&path).unwrap();
conn.execute("UPDATE roads SET lanes = 'NaN'", []).unwrap();
let table = EditTable::open(&path, "roads").unwrap();
let read = table.read_editable(MAX_EDITABLE).unwrap();
let lane = table.info().fields.iter().position(|f| f.name == "lanes").unwrap();
let name = table.info().fields.iter().position(|f| f.name == "name").unwrap();
assert!(matches!(read.features[0].attrs[lane], Value::Float(v) if v.is_nan()));
let mut changed = read.features[0].clone();
changed.attrs[name] = Value::Text("edited".into());
let edits = Edits { updates: vec![(changed, read.versions[0].1)], ..Default::default() };
let saved = table.save(&edits, &table.info().fields, std::iter::empty(), &read.revision).unwrap().unwrap();
assert_eq!(saved.changes.updated, 1);
let stored: (String, String) = conn.query_row("SELECT name, lanes FROM roads", [], |r| Ok((r.get(0)?, r.get(1)?))).unwrap();
assert_eq!(stored, ("edited".into(), "NaN".into()));
// An explicit clear still writes SQL NULL.
let read = table.read_editable(MAX_EDITABLE).unwrap();
let mut cleared = read.features[0].clone();
cleared.attrs[lane] = Value::Null;
let edits = Edits { updates: vec![(cleared, read.versions[0].1)], ..Default::default() };
table.save(&edits, &table.info().fields, std::iter::empty(), &saved.revision).unwrap().unwrap();
assert!(conn.query_row("SELECT lanes IS NULL FROM roads", [], |r| r.get::<_, bool>(0)).unwrap());
}

#[test]
fn gpkg_insert_revision_is_durable_and_strictly_advances() {
let (_dir, path) = fixture();
Expand Down
2 changes: 1 addition & 1 deletion crates/data/src/source/gpkg_mem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ fn value(v: &Val, ty: FieldType) -> Value {
(Val::Real(f), FieldType::Integer) if f.fract() == 0.0 && f.abs() < 9e15 => Value::Int(*f as i64),
(Val::Real(f), _) => Value::Float(*f),
(Val::Text(t), FieldType::Text) => Value::Text(t.clone()),
(Val::Text(t), ty) => Value::parse(t, ty),
(Val::Text(t), ty) => Value::parse_source(t, ty),
}
}

Expand Down
Loading
Loading