From 8911ef796b86e3a3eff4d41be345862d02dbd5f4 Mon Sep 17 00:00:00 2001 From: mavrick Date: Mon, 9 Nov 2026 08:22:22 -0600 Subject: [PATCH 1/4] fix(data): keep GeoPackage rows without geometry A GeoPackage row whose geometry is NULL or an empty geometry is a valid row, and RustGIS's own GeoPackage export writes a feature without geometry that way. Both readers (SQLite and the browser's pure-Rust reader) dropped such rows from whole reads and chunked visits, without a count or a notice, so a round trip or a conversion lost them and their attributes. They are now kept as features without geometry in whole reads and visits, as the GeoParquet reader keeps null-geometry rows; a read of an extent still leaves them out. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/source/gpkg.rs | 20 +++++----- crates/data/src/source/gpkg_mem.rs | 62 ++++++++++++++++-------------- crates/data/src/source/tests.rs | 50 +++++++++++++++++++++--- docs/user-guide/data-formats.md | 2 +- 4 files changed, 90 insertions(+), 44 deletions(-) diff --git a/crates/data/src/source/gpkg.rs b/crates/data/src/source/gpkg.rs index f30c8a09c8..2736f04fe9 100644 --- a/crates/data/src/source/gpkg.rs +++ b/crates/data/src/source/gpkg.rs @@ -307,14 +307,18 @@ impl GpkgSource { fn feature(&self, r: &rusqlite::Row<'_>, native: Option<&Extent>, req: &Request) -> rusqlite::Result { // Integer keys, negative ones too, map to distinct ids. let Some(id) = r.get::<_, Option>(0)?.map(|i| i as u64) else { return Ok(Row::Bad) }; + let attrs = + || self.info.fields.iter().enumerate().map(|(i, f)| r.get_ref(i + 2).map(|v| value(v, f.ty)).unwrap_or(Value::Null)).collect(); let blob = match r.get_ref(1)? { - ValueRef::Blob(b) => b, - ValueRef::Null => return Ok(Row::Skip), + ValueRef::Blob(b) if !is_empty_blob(b) => b, + // A row without geometry (NULL, or an empty one) is kept in a whole read; it has no + // place to match an extent. + ValueRef::Null | ValueRef::Blob(_) if req.extent.is_none() => { + return Ok(Row::Feature(Feature { id, geometry: None, attrs: attrs() })); + } + ValueRef::Null | ValueRef::Blob(_) => return Ok(Row::Skip), _ => return Ok(Row::Bad), }; - if is_empty_blob(blob) { - return Ok(Row::Skip); - } let Some((envelope, body)) = split_blob(blob) else { return Ok(Row::Bad) }; if let (Some(q), Some(e)) = (native, envelope) && !e.intersects(q) @@ -325,15 +329,13 @@ impl GpkgSource { if !in_request(&geometry, req) { return Ok(Row::Skip); } - let attrs = - self.info.fields.iter().enumerate().map(|(i, f)| r.get_ref(i + 2).map(|v| value(v, f.ty)).unwrap_or(Value::Null)).collect(); - Ok(Row::Feature(Feature { id, geometry: Some(geometry), attrs })) + Ok(Row::Feature(Feature { id, geometry: Some(geometry), attrs: attrs() })) } } enum Row { Feature(Feature), - /// Outside the request, or without geometry. + /// Outside the request (a row without geometry is, when an extent is asked for). Skip, /// Unreadable. Bad, diff --git a/crates/data/src/source/gpkg_mem.rs b/crates/data/src/source/gpkg_mem.rs index 2329513f4c..4bb9a1fca0 100644 --- a/crates/data/src/source/gpkg_mem.rs +++ b/crates/data/src/source/gpkg_mem.rs @@ -270,6 +270,7 @@ impl FeatureSource for GpkgMemSource { &self.info } + /// Keeps rows without geometry in whole reads; they have no location to match an extent. fn read(&self, req: &Request) -> Result { let limit = clamp_limit(req.limit); let native = req.extent.map(|e| self.reproject.extent_from_lonlat(&e)); @@ -278,16 +279,20 @@ impl FeatureSource for GpkgMemSource { self.db .rows(&self.table, |rowid, r| { let blob = match r.get(self.geom_col) { - Some(Val::Blob(b)) => b, - Some(Val::Null) | None => return ControlFlow::Continue(()), + Some(Val::Blob(b)) if !super::gpkg_blob::is_empty_blob(b) => b, + // A row without geometry (NULL, or an empty one) is kept in a whole read; it + // has no place to match an extent. + Some(Val::Null | Val::Blob(_)) | None => { + if native.is_none() && sample.wants() { + sample.keep(Feature { id: rowid as u64, geometry: None, attrs: self.attrs(r) }); + } + return ControlFlow::Continue(()); + } Some(_) => { skipped += 1; return ControlFlow::Continue(()); } }; - if super::gpkg_blob::is_empty_blob(blob) { - return ControlFlow::Continue(()); - } let Some((envelope, body)) = super::gpkg_blob::split_blob(blob) else { skipped += 1; return ControlFlow::Continue(()); @@ -333,35 +338,36 @@ impl FeatureSource for GpkgMemSource { let mut failed = None; self.db .rows(&self.table, |rowid, r| { - let blob = match r.get(self.geom_col) { - Some(Val::Blob(b)) => b, - Some(Val::Null) | None => return ControlFlow::Continue(()), + let geometry = match r.get(self.geom_col) { + Some(Val::Blob(b)) if !super::gpkg_blob::is_empty_blob(b) => { + let Some((envelope, body)) = super::gpkg_blob::split_blob(b) else { + out.skipped += 1; + return ControlFlow::Continue(()); + }; + if let (Some(q), Some(e)) = (&native, envelope) + && !e.intersects(q) + { + return ControlFlow::Continue(()); + } + let Some(geometry) = wkb::decode(body).and_then(|g| self.reproject.geometry(g)) else { + out.skipped += 1; + return ControlFlow::Continue(()); + }; + if !in_request(&geometry, &req) { + return ControlFlow::Continue(()); + } + Some(geometry) + } + // Without geometry (NULL, or an empty one): kept unless an extent is asked for. + Some(Val::Null | Val::Blob(_)) | None if native.is_none() => None, + Some(Val::Null | Val::Blob(_)) | None => return ControlFlow::Continue(()), Some(_) => { out.skipped += 1; return ControlFlow::Continue(()); } }; - if super::gpkg_blob::is_empty_blob(blob) { - return ControlFlow::Continue(()); - } - let Some((envelope, body)) = super::gpkg_blob::split_blob(blob) else { - out.skipped += 1; - return ControlFlow::Continue(()); - }; - if let (Some(q), Some(e)) = (&native, envelope) - && !e.intersects(q) - { - return ControlFlow::Continue(()); - } - let Some(geometry) = wkb::decode(body).and_then(|g| self.reproject.geometry(g)) else { - out.skipped += 1; - return ControlFlow::Continue(()); - }; - if !in_request(&geometry, &req) { - return ControlFlow::Continue(()); - } out.matched += 1; - match chunks.push(Feature { id: rowid as u64, geometry: Some(geometry), attrs: self.attrs(r) }) { + match chunks.push(Feature { id: rowid as u64, geometry, attrs: self.attrs(r) }) { Ok(()) => ControlFlow::Continue(()), Err(e) => { failed = Some(e); diff --git a/crates/data/src/source/tests.rs b/crates/data/src/source/tests.rs index b72df946b1..c283790cdd 100644 --- a/crates/data/src/source/tests.rs +++ b/crates/data/src/source/tests.rs @@ -1001,8 +1001,8 @@ fn writes_geopackage_geoparquet_and_shapefile() { let holds_all = matches!(ext, "gpkg" | "parquet"); assert_eq!(left_out, if holds_all { 0 } else { 1 }, "{ext}"); let (back, _) = crate::io::read_path(&p).unwrap(); - // GeoParquet retains nullable geometry; the other readers omit rows without a shape. - assert_eq!(back.features.len(), if ext == "parquet" { 3 } else { 2 }, "{ext}"); + // GeoParquet and GeoPackage keep a row without a shape; the other formats can't hold it. + assert_eq!(back.features.len(), if holds_all { 3 } else { 2 }, "{ext}"); assert_eq!(back.geometry_type, GeometryType::Polygon, "{ext}"); let e = back.extent(); assert_eq!((e.min_x, e.min_y, e.max_x, e.max_y), (-30.0, -40.0, 21.0, 21.0), "{ext}"); @@ -2383,6 +2383,44 @@ fn shapefile_complete_chunks_keep_all_rows_order_attributes_and_budget() { } /// Every feature of a complete traversal, in order, checking each chunk against the budget. +/// GeoPackage rows without geometry (NULL, or an empty geometry) keep their row and attributes in +/// whole reads and visits, with and without the R-tree and in the browser's reader, and are left +/// out of a read of an extent. +#[test] +fn geopackage_keeps_rows_without_geometry() { + let d = dir().join("geometryless"); + std::fs::create_dir_all(&d).unwrap(); + let (indexed, scan) = (d.join("indexed.gpkg"), d.join("scan.gpkg")); + for (p, rtree) in [(&indexed, true), (&scan, false)] { + write_gpkg(p, rtree); + let c = rusqlite::Connection::open(p).unwrap(); + let empty = b"GP\x00\x11\x11\x0f\x00\x00".to_vec(); + c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (601, NULL, 'no place', 7, 1)", []).unwrap(); + c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (602, ?1, 'empty', 8, 0)", [empty]).unwrap(); + } + let sources: Vec> = vec![ + open_path(&indexed, None).unwrap(), + open_path(&scan, None).unwrap(), + Box::new(gpkg_mem::GpkgMemSource::open_shared(Shared(Arc::new(std::fs::read(&scan).unwrap())), None).unwrap()), + ]; + for src in &sources { + let format = src.info().format; + assert_eq!(src.info().count, Some(602), "{format}"); + let whole = src.read(&Request::all()).unwrap(); + assert_eq!((whole.features.len(), whole.skipped), (602, 0), "{format}"); + let bare: Vec<_> = whole.features.iter().filter(|f| f.geometry.is_none()).map(|f| (f.id, f.attrs[0].clone())).collect(); + assert_eq!(bare, vec![(601, Value::Text("no place".into())), (602, Value::Text("empty".into()))], "{format}"); + let (rows, summary, _) = visit_all(src.as_ref(), None, 1024); + assert_eq!((rows, summary.matched), (whole.features, 602), "{format}"); + let world = Extent::new(-180.0, -85.0, 180.0, 85.0); + let part = src.read(&Request { extent: Some(world), limit: 10_000 }).unwrap(); + assert_eq!(part.features.len(), 600, "{format}"); + assert_eq!(visit_all(src.as_ref(), Some(world), 1024).0.len(), 600, "{format}"); + } +} + +/// Every chunk a visit of `extent` gives, joined, checking each fits `budget` (or is one row too +/// big for it): the rows, the visit's summary and how many chunks came. fn visit_all(src: &dyn FeatureSource, extent: Option, budget: usize) -> (Vec, Batch, usize) { let (mut rows, mut calls) = (Vec::new(), 0); let summary = src @@ -2491,20 +2529,20 @@ fn geopackage_flatgeobuf_and_geoparquet_visit_what_a_whole_read_gives() { let error = src.visit_chunks(None, 4096, &mut |_| Ok(())).unwrap_err(); assert!(matches!(&error, crate::DataError::Io(e) if e.kind() == std::io::ErrorKind::NotFound), "{error}"); } - // An unreadable geometry is skipped and counted, and a row without one left out, as a read does. + // An unreadable geometry is skipped and counted, and a row without one kept, as a read does. drop(sources); let c = rusqlite::Connection::open(&gpkg_scan).unwrap(); c.execute_batch("INSERT INTO \"grid \"\"q\"\"\" (geom, name) VALUES (X'4750000300000000DEAD', 'bad'), (NULL, 'none');").unwrap(); drop(c); let src = open_path(&gpkg_scan, None).unwrap(); let whole = src.read(&Request::all()).unwrap(); - assert_eq!((whole.features.len(), whole.skipped), (600, 1)); + assert_eq!((whole.features.len(), whole.skipped), (601, 1)); let (rows, summary, _) = visit_all(src.as_ref(), None, 4096); assert_eq!((rows, summary.skipped), (whole.features, 1)); - // The browser's reader of the same file skips and leaves out the same rows. + // The browser's reader of the same file skips and keeps the same rows. let mem = gpkg_mem::GpkgMemSource::open_shared(Shared(Arc::new(std::fs::read(&gpkg_scan).unwrap())), None).unwrap(); let whole = mem.read(&Request::all()).unwrap(); - assert_eq!((whole.features.len(), whole.skipped), (600, 1)); + assert_eq!((whole.features.len(), whole.skipped), (601, 1)); let (rows, summary, _) = visit_all(&mem, None, 4096); assert_eq!((rows, summary.skipped), (whole.features, 1)); } diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index e74829a9c6..602882350c 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -33,7 +33,7 @@ not every EPSG code; azimuthal equidistant systems such as Equi7 need the deskto | AutoCAD DXF | `.dxf` (ASCII) | Add Data, drop the file, or upload | Points, lines and polygons layers from the drawing's entities: POINT, LINE, LWPOLYLINE and POLYLINE (bulges drawn as arcs; closed ones stay lines; polyface meshes become polygons), CIRCLE, ARC, ELLIPSE, SPLINE, HELIX, LEADER, MLINE (its centre line), TEXT, MTEXT, ATTRIB and constant ATTDEF (points with their text, at the alignment point of justified text), HATCH boundaries (holes found by nesting), SOLID, TRACE and 3DFACE (polygons). Blocks (named in any case) are drawn where INSERTs place them (scaled, rotated, in arrays, nested up to 8 deep), and so are DIMENSIONs' blocks (a linear dimension without one draws its lines and measured text). Entities in an object's own coordinate system are turned into the drawing's. Fields: `layer`, `entity`, `color` (ACI, BYLAYER and BYBLOCK resolved), `linetype`, `text`, `height`, `rotation`, `block`, `elevation` and `paperspace`. The drawing's layouts (paper space: title blocks, viewport frames) are left out, with a notice, unless `"paperSpace": true`. A drawing never says its coordinate system: give `"crs"` (such as `EPSG:2263`), or its coordinates are kept as they are, with a notice. `"layer"` picks one of the drawing's own layers, and `info.fileLayers` lists them. Up to 2 million features (500,000 in the web app). Not read: binary DXF, MULTILEADER, WIPEOUT, IMAGE and ACIS solids (3DSOLID, REGION), named in a notice. | | Geotagged photos | `.jpg`, `.jpeg` | Add Data (several at once in the web app), drop them, upload, or `layer.addPhotos` with `"paths"` or a `"folder"` | JPEGs with a GPS position in their EXIF become points in the **Photos** layer (photos added one at a time join it): each with its `name`, `file`, `time` (when it was taken), `altitude`, `direction` (where the camera faced), `camera` and a `thumbnail` of at most 160 pixels, kept with the feature so the project needs no other file. Identify shows the thumbnail. Photos without a position are named in a notice and left out; a layer holds up to 2,000 photos. A JPEG is always read as a photo unless `layer.addData` or `layer.addDataBytes` is given `"as": "scan"` (a phone photo of a paper map, say: see [Scans with no coordinates](#rasters)). | | LiDAR point clouds | `.las`, `.laz` (COPC too) | Add Data, drop the file, upload, or From URL… | LAS 1.0–1.4 (point formats 0–10) and LAZ, on desktop and web: points with `z`, `intensity`, `classification`, `return` and colour (`red`, `green`, `blue`), coloured by ASPRS class or elevation. An evenly spread sample is read once, at most 2 million points (500,000 in the web app); views use that sample, not the COPC spatial index. LAZ decompression reads at most the first 400 million points (40 million on the web), with a notice when capped. Converted from the file's WKT or GeoTIFF-key record; without one, coordinates are kept with a notice. A local project keeps the path; uploads keep the file bytes. Remote files download whole. | -| GeoPackage | `.gpkg` | Add Data, or upload in the web app | Local files and browser uploads add every feature table, up to 32 (with the rest counted in `skippedCount` and up to 100 named in `skipped`); `"layer"` picks one, and `info.fileLayers` lists them. Desktop byte uploads and downloads add the first table unless `"layer"` names another. Large local tables use their R-tree. A table is read in its coordinate system: by its EPSG code, or, for a system without one (or with a code RustGIS doesn't know), by the WKT definition the file gives it, on the desktop and in the web app alike (an azimuthal equidistant one opens on the desktop only, and the web app says so). Browser tables are scanned, testing each geometry header's envelope first; UTF-16 databases aren't read there. Raster tile tables aren't imported. | +| GeoPackage | `.gpkg` | Add Data, or upload in the web app | Local files and browser uploads add every feature table, up to 32 (with the rest counted in `skippedCount` and up to 100 named in `skipped`); `"layer"` picks one, and `info.fileLayers` lists them. Desktop byte uploads and downloads add the first table unless `"layer"` names another. Large local tables use their R-tree. Rows with no geometry (NULL or empty) are kept, with their attributes, in whole reads and exports; a read of an extent leaves them out, as they have no location. A table is read in its coordinate system: by its EPSG code, or, for a system without one (or with a code RustGIS doesn't know), by the WKT definition the file gives it, on the desktop and in the web app alike (an azimuthal equidistant one opens on the desktop only, and the web app says so). Browser tables are scanned, testing each geometry header's envelope first; UTF-16 databases aren't read there. Raster tile tables aren't imported. | | GeoParquet | `.parquet`, `.geoparquet` | Add Data, or upload in the web app | WKB geometry, with the CRS from the file's `geo` metadata. A binary geometry column is required; plain non-spatial Parquet tables and GeoArrow-native geometry encodings aren't read by this importer. Row groups are pruned by the bbox covering. Whole reads and exports retain rows with null geometry and their attributes; extent-filtered reads exclude them because they have no location. Snappy, gzip, LZ4 and zstd compression are read, in the browser too; Brotli isn't. Text, whole-number, floating-point and true/false columns become fields, as do decimals stored as 32- or 64-bit integers (up to 18 digits), read as floating-point numbers. Dates and times stored as 32- or 64-bit integers are read as those whole numbers (such as milliseconds since 1970). Other columns are left out: decimals stored as bytes (how many writers store every decimal), binary, lists, maps, structs, INT96 timestamps and further geometry columns. A notice gives how many were left out and names the first 12, each with its kind; another names the date and time columns with their unit. | | FlatGeobuf | `.fgb` | Add Data, or upload | Read in place by its packed R-tree. Binary columns are left out, with a notice counting them and naming the first 12. | | Esri File Geodatabase | `.gdb` folder (desktop) | In Add Data pick a `.gdbtable` file inside it; or double-click the `.gdb` in the Browser, or drop the folder | The native reader loads up to 512 feature classes and tables, up to 2 GiB per table. Feature classes become layers and non-spatial tables standalone tables; dates are text. Coordinates use the EPSG/Esri code, else WKT. `"layer"` picks one; `info.fileLayers` lists them. Multipatches become footprints with `z_min`, `z_max` and `height`; large or wall-only ones may use a convex hull. Curves become straight segments between their ends. Unreadable tables (compressed storage, raster fields or unsupported row indexes) are tried through GDAL when installed; otherwise they are named in a notice. | From 2334e4818e2d2b81bf472f16bb3e7c177481d7e0 Mon Sep 17 00:00:00 2001 From: mavrick Date: Mon, 9 Nov 2026 09:53:46 -0600 Subject: [PATCH 2/4] fix(data): count a cut-off empty GeoPackage geometry as unreadable A blob flagged empty is only a valid empty geometry when its header is whole (8 bytes and the envelope its flags give). A shorter one was kept as a row without geometry; it is now skipped and counted, as other unreadable geometry is. Also puts visit_all's doc comment back on it. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/source/gpkg_blob.rs | 24 +++++++++++++++--------- crates/data/src/source/tests.rs | 14 +++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/data/src/source/gpkg_blob.rs b/crates/data/src/source/gpkg_blob.rs index 54a909afc0..544bd26c6c 100644 --- a/crates/data/src/source/gpkg_blob.rs +++ b/crates/data/src/source/gpkg_blob.rs @@ -4,9 +4,21 @@ use crate::FieldType; use rustgis_geom::Extent; -/// A GeoPackage geometry blob flagged empty: valid, with nothing to draw. +/// A GeoPackage geometry blob flagged empty: valid, with nothing to draw. Its header must be +/// whole (8 bytes and the envelope its flags give); a cut-off one is unreadable, not empty. pub(crate) fn is_empty_blob(b: &[u8]) -> bool { - b.starts_with(b"GP") && b.get(3).is_some_and(|f| f & 0b1_0000 != 0) + b.starts_with(b"GP") && b.get(3).is_some_and(|&f| f & 0b1_0000 != 0 && envelope_len(f).is_some_and(|n| b.len() >= 8 + n)) +} + +/// The length of the envelope a header's flags give; `None` for a code the standard doesn't define. +fn envelope_len(flags: u8) -> Option { + match (flags >> 1) & 0b111 { + 0 => Some(0), + 1 => Some(32), + 2 | 3 => Some(48), + 4 => Some(64), + _ => None, + } } /// A GeoPackage geometry blob: the header's envelope (if any) and the WKB after it. @@ -18,13 +30,7 @@ pub(crate) fn split_blob(b: &[u8]) -> Option<(Option, &[u8])> { } let &flags = rest.get(1)?; let little = flags & 1 == 1; - let env_len = match (flags >> 1) & 0b111 { - 0 => 0, - 1 => 32, - 2 | 3 => 48, - 4 => 64, - _ => return None, - }; + let env_len = envelope_len(flags)?; if flags & 0b1_0000 != 0 { // The empty flag. return None; diff --git a/crates/data/src/source/tests.rs b/crates/data/src/source/tests.rs index c283790cdd..2f24093764 100644 --- a/crates/data/src/source/tests.rs +++ b/crates/data/src/source/tests.rs @@ -2382,10 +2382,9 @@ fn shapefile_complete_chunks_keep_all_rows_order_attributes_and_budget() { assert_eq!(rows, expected.features); } -/// Every feature of a complete traversal, in order, checking each chunk against the budget. /// GeoPackage rows without geometry (NULL, or an empty geometry) keep their row and attributes in /// whole reads and visits, with and without the R-tree and in the browser's reader, and are left -/// out of a read of an extent. +/// out of a read of an extent; one flagged empty but cut off is skipped and counted. #[test] fn geopackage_keeps_rows_without_geometry() { let d = dir().join("geometryless"); @@ -2397,6 +2396,8 @@ fn geopackage_keeps_rows_without_geometry() { let empty = b"GP\x00\x11\x11\x0f\x00\x00".to_vec(); c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (601, NULL, 'no place', 7, 1)", []).unwrap(); c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (602, ?1, 'empty', 8, 0)", [empty]).unwrap(); + // Flagged empty but cut off before its header ends: unreadable, skipped and counted. + c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (603, X'47500011', 'cut', 9, 1)", []).unwrap(); } let sources: Vec> = vec![ open_path(&indexed, None).unwrap(), @@ -2405,13 +2406,13 @@ fn geopackage_keeps_rows_without_geometry() { ]; for src in &sources { let format = src.info().format; - assert_eq!(src.info().count, Some(602), "{format}"); + assert_eq!(src.info().count, Some(603), "{format}"); let whole = src.read(&Request::all()).unwrap(); - assert_eq!((whole.features.len(), whole.skipped), (602, 0), "{format}"); + assert_eq!((whole.features.len(), whole.skipped), (602, 1), "{format}"); let bare: Vec<_> = whole.features.iter().filter(|f| f.geometry.is_none()).map(|f| (f.id, f.attrs[0].clone())).collect(); assert_eq!(bare, vec![(601, Value::Text("no place".into())), (602, Value::Text("empty".into()))], "{format}"); let (rows, summary, _) = visit_all(src.as_ref(), None, 1024); - assert_eq!((rows, summary.matched), (whole.features, 602), "{format}"); + assert_eq!((rows, summary.matched, summary.skipped), (whole.features, 602, 1), "{format}"); let world = Extent::new(-180.0, -85.0, 180.0, 85.0); let part = src.read(&Request { extent: Some(world), limit: 10_000 }).unwrap(); assert_eq!(part.features.len(), 600, "{format}"); @@ -2419,8 +2420,7 @@ fn geopackage_keeps_rows_without_geometry() { } } -/// Every chunk a visit of `extent` gives, joined, checking each fits `budget` (or is one row too -/// big for it): the rows, the visit's summary and how many chunks came. +/// Every feature of a complete traversal, in order, checking each chunk against the budget. fn visit_all(src: &dyn FeatureSource, extent: Option, budget: usize) -> (Vec, Batch, usize) { let (mut rows, mut calls) = (Vec::new(), 0); let summary = src From d7eb5ab6deb2d1e11156b79cad564a4bf5c000f2 Mon Sep 17 00:00:00 2001 From: mavrick Date: Mon, 9 Nov 2026 18:24:28 -0600 Subject: [PATCH 3/4] fix(data): keep the editor's empty-geometry test; test more cut-off headers The whole-header check for blobs flagged empty applies to the readers only (is_whole_empty_blob), so the GeoPackage editor and its R-tree triggers keep treating a flagged-empty blob as before. The readers' test adds an undefined envelope code and a declared envelope that is missing, both skipped and counted. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/source/gpkg.rs | 4 ++-- crates/data/src/source/gpkg_blob.rs | 11 ++++++++--- crates/data/src/source/gpkg_mem.rs | 4 ++-- crates/data/src/source/tests.rs | 15 +++++++++------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/crates/data/src/source/gpkg.rs b/crates/data/src/source/gpkg.rs index 2736f04fe9..e7a4dffdd8 100644 --- a/crates/data/src/source/gpkg.rs +++ b/crates/data/src/source/gpkg.rs @@ -3,7 +3,7 @@ //! Extent queries use the table's R-tree spatial index (`rtree__`) when it has one, //! and otherwise scan, testing the envelope in each geometry's header before decoding it. -use super::gpkg_blob::{field_type, is_empty_blob, split_blob}; +use super::gpkg_blob::{field_type, is_empty_blob, is_whole_empty_blob, split_blob}; use super::{Batch, FeatureSource, Reproject, Request, Sampler, SourceInfo, clamp_limit, in_request, stride}; use crate::raster::Crs; use crate::{DataError, Feature, Field, FieldType, GeometryType, Result, Value, wkb}; @@ -310,7 +310,7 @@ impl GpkgSource { let attrs = || self.info.fields.iter().enumerate().map(|(i, f)| r.get_ref(i + 2).map(|v| value(v, f.ty)).unwrap_or(Value::Null)).collect(); let blob = match r.get_ref(1)? { - ValueRef::Blob(b) if !is_empty_blob(b) => b, + ValueRef::Blob(b) if !is_whole_empty_blob(b) => b, // A row without geometry (NULL, or an empty one) is kept in a whole read; it has no // place to match an extent. ValueRef::Null | ValueRef::Blob(_) if req.extent.is_none() => { diff --git a/crates/data/src/source/gpkg_blob.rs b/crates/data/src/source/gpkg_blob.rs index 544bd26c6c..9ffa89254f 100644 --- a/crates/data/src/source/gpkg_blob.rs +++ b/crates/data/src/source/gpkg_blob.rs @@ -4,10 +4,15 @@ use crate::FieldType; use rustgis_geom::Extent; -/// A GeoPackage geometry blob flagged empty: valid, with nothing to draw. Its header must be -/// whole (8 bytes and the envelope its flags give); a cut-off one is unreadable, not empty. +/// A GeoPackage geometry blob flagged empty: valid, with nothing to draw. pub(crate) fn is_empty_blob(b: &[u8]) -> bool { - b.starts_with(b"GP") && b.get(3).is_some_and(|&f| f & 0b1_0000 != 0 && envelope_len(f).is_some_and(|n| b.len() >= 8 + n)) + b.starts_with(b"GP") && b.get(3).is_some_and(|f| f & 0b1_0000 != 0) +} + +/// A blob flagged empty whose header is whole (8 bytes and the envelope its flags give): a row +/// the readers keep without geometry. A cut-off one is unreadable, not empty. +pub(crate) fn is_whole_empty_blob(b: &[u8]) -> bool { + is_empty_blob(b) && b.get(3).and_then(|&f| envelope_len(f)).is_some_and(|n| b.len() >= 8 + n) } /// The length of the envelope a header's flags give; `None` for a code the standard doesn't define. diff --git a/crates/data/src/source/gpkg_mem.rs b/crates/data/src/source/gpkg_mem.rs index 4bb9a1fca0..cc930393ea 100644 --- a/crates/data/src/source/gpkg_mem.rs +++ b/crates/data/src/source/gpkg_mem.rs @@ -279,7 +279,7 @@ impl FeatureSource for GpkgMemSource { self.db .rows(&self.table, |rowid, r| { let blob = match r.get(self.geom_col) { - Some(Val::Blob(b)) if !super::gpkg_blob::is_empty_blob(b) => b, + Some(Val::Blob(b)) if !super::gpkg_blob::is_whole_empty_blob(b) => b, // A row without geometry (NULL, or an empty one) is kept in a whole read; it // has no place to match an extent. Some(Val::Null | Val::Blob(_)) | None => { @@ -339,7 +339,7 @@ impl FeatureSource for GpkgMemSource { self.db .rows(&self.table, |rowid, r| { let geometry = match r.get(self.geom_col) { - Some(Val::Blob(b)) if !super::gpkg_blob::is_empty_blob(b) => { + Some(Val::Blob(b)) if !super::gpkg_blob::is_whole_empty_blob(b) => { let Some((envelope, body)) = super::gpkg_blob::split_blob(b) else { out.skipped += 1; return ControlFlow::Continue(()); diff --git a/crates/data/src/source/tests.rs b/crates/data/src/source/tests.rs index 2f24093764..eca1b7b9c3 100644 --- a/crates/data/src/source/tests.rs +++ b/crates/data/src/source/tests.rs @@ -2384,7 +2384,7 @@ fn shapefile_complete_chunks_keep_all_rows_order_attributes_and_budget() { /// GeoPackage rows without geometry (NULL, or an empty geometry) keep their row and attributes in /// whole reads and visits, with and without the R-tree and in the browser's reader, and are left -/// out of a read of an extent; one flagged empty but cut off is skipped and counted. +/// out of a read of an extent; one flagged empty with an unreadable header is skipped and counted. #[test] fn geopackage_keeps_rows_without_geometry() { let d = dir().join("geometryless"); @@ -2396,8 +2396,11 @@ fn geopackage_keeps_rows_without_geometry() { let empty = b"GP\x00\x11\x11\x0f\x00\x00".to_vec(); c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (601, NULL, 'no place', 7, 1)", []).unwrap(); c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (602, ?1, 'empty', 8, 0)", [empty]).unwrap(); - // Flagged empty but cut off before its header ends: unreadable, skipped and counted. - c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (603, X'47500011', 'cut', 9, 1)", []).unwrap(); + // Flagged empty but unreadable, each skipped and counted: cut off before its header ends, + // with an envelope code the standard doesn't define, and an envelope declared but missing. + for (fid, blob) in [(603, "47500011"), (604, "4750001b0f000000"), (605, "475000130f000000")] { + c.execute(&format!("INSERT INTO \"grid \"\"q\"\"\" VALUES ({fid}, X'{blob}', 'cut', 9, 1)"), []).unwrap(); + } } let sources: Vec> = vec![ open_path(&indexed, None).unwrap(), @@ -2406,13 +2409,13 @@ fn geopackage_keeps_rows_without_geometry() { ]; for src in &sources { let format = src.info().format; - assert_eq!(src.info().count, Some(603), "{format}"); + assert_eq!(src.info().count, Some(605), "{format}"); let whole = src.read(&Request::all()).unwrap(); - assert_eq!((whole.features.len(), whole.skipped), (602, 1), "{format}"); + assert_eq!((whole.features.len(), whole.skipped), (602, 3), "{format}"); let bare: Vec<_> = whole.features.iter().filter(|f| f.geometry.is_none()).map(|f| (f.id, f.attrs[0].clone())).collect(); assert_eq!(bare, vec![(601, Value::Text("no place".into())), (602, Value::Text("empty".into()))], "{format}"); let (rows, summary, _) = visit_all(src.as_ref(), None, 1024); - assert_eq!((rows, summary.matched, summary.skipped), (whole.features, 602, 1), "{format}"); + assert_eq!((rows, summary.matched, summary.skipped), (whole.features, 602, 3), "{format}"); let world = Extent::new(-180.0, -85.0, 180.0, 85.0); let part = src.read(&Request { extent: Some(world), limit: 10_000 }).unwrap(); assert_eq!(part.features.len(), 600, "{format}"); From c5876206dc75033880a8471c51bcb9e053061c01 Mon Sep 17 00:00:00 2001 From: mavrick Date: Mon, 9 Nov 2026 23:27:34 -0600 Subject: [PATCH 4/4] fix(data): a GeoPackage blob of an undefined header version isn't an empty row The readers keep a blob flagged empty as a row without geometry only when its header is version 0, the one GeoPackage defines; another version is unreadable, skipped and counted. The test adds such a row. Two export tests from main counted 300 features read back from a GeoPackage export of 301; the 301st has no geometry and is now kept, so they expect 301. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/export/tests.rs | 6 ++++-- crates/data/src/source/gpkg_blob.rs | 7 ++++--- crates/data/src/source/tests.rs | 11 ++++++----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index b7e05cd1a6..455d2bcd73 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -443,7 +443,8 @@ fn a_typeless_layer_exports_the_same_geopackage_streamed_or_whole() { assert_eq!(crate::io::write_path(&ds, &b).unwrap(), 0); assert_eq!((kind(&a), kind(&b)), (want.to_string(), want.to_string())); assert_eq!(read_back(&a).features, read_back(&b).features); - assert_eq!(read_back(&a).features.len(), 300); + // Every square and the feature without geometry, which a GeoPackage keeps. + assert_eq!(read_back(&a).features.len(), 301); } } @@ -466,7 +467,8 @@ fn a_projected_streamed_geopackage_of_a_mixed_layer_keeps_every_kind() { assert_eq!((streamed.left_out, streamed.written), (0, 301)); assert_eq!(crate::io::write_path_in(&mixed, &b, utm).unwrap(), 0); assert_eq!(read_back(&a).features, read_back(&b).features); - assert_eq!(read_back(&a).features.len(), 300); + // Every feature, the one without geometry included. + assert_eq!(read_back(&a).features.len(), 301); } /// A mixed layer added as a table to an existing GeoPackage, streamed or whole, gets a GEOMETRY diff --git a/crates/data/src/source/gpkg_blob.rs b/crates/data/src/source/gpkg_blob.rs index 9ffa89254f..af43818ae2 100644 --- a/crates/data/src/source/gpkg_blob.rs +++ b/crates/data/src/source/gpkg_blob.rs @@ -9,10 +9,11 @@ pub(crate) fn is_empty_blob(b: &[u8]) -> bool { b.starts_with(b"GP") && b.get(3).is_some_and(|f| f & 0b1_0000 != 0) } -/// A blob flagged empty whose header is whole (8 bytes and the envelope its flags give): a row -/// the readers keep without geometry. A cut-off one is unreadable, not empty. +/// A blob flagged empty whose header is whole (version 0, the one GeoPackage defines; 8 bytes and +/// the envelope its flags give): a row the readers keep without geometry. Any other is unreadable, +/// not empty. pub(crate) fn is_whole_empty_blob(b: &[u8]) -> bool { - is_empty_blob(b) && b.get(3).and_then(|&f| envelope_len(f)).is_some_and(|n| b.len() >= 8 + n) + is_empty_blob(b) && b.get(2) == Some(&0) && b.get(3).and_then(|&f| envelope_len(f)).is_some_and(|n| b.len() >= 8 + n) } /// The length of the envelope a header's flags give; `None` for a code the standard doesn't define. diff --git a/crates/data/src/source/tests.rs b/crates/data/src/source/tests.rs index d7305b30e1..91c26d0781 100644 --- a/crates/data/src/source/tests.rs +++ b/crates/data/src/source/tests.rs @@ -2547,8 +2547,9 @@ fn geopackage_keeps_rows_without_geometry() { c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (601, NULL, 'no place', 7, 1)", []).unwrap(); c.execute("INSERT INTO \"grid \"\"q\"\"\" VALUES (602, ?1, 'empty', 8, 0)", [empty]).unwrap(); // Flagged empty but unreadable, each skipped and counted: cut off before its header ends, - // with an envelope code the standard doesn't define, and an envelope declared but missing. - for (fid, blob) in [(603, "47500011"), (604, "4750001b0f000000"), (605, "475000130f000000")] { + // with an envelope code the standard doesn't define, an envelope declared but missing, and a + // header version the standard doesn't define. + for (fid, blob) in [(603, "47500011"), (604, "4750001b0f000000"), (605, "475000130f000000"), (606, "475001110f000000")] { c.execute(&format!("INSERT INTO \"grid \"\"q\"\"\" VALUES ({fid}, X'{blob}', 'cut', 9, 1)"), []).unwrap(); } } @@ -2559,13 +2560,13 @@ fn geopackage_keeps_rows_without_geometry() { ]; for src in &sources { let format = src.info().format; - assert_eq!(src.info().count, Some(605), "{format}"); + assert_eq!(src.info().count, Some(606), "{format}"); let whole = src.read(&Request::all()).unwrap(); - assert_eq!((whole.features.len(), whole.skipped), (602, 3), "{format}"); + assert_eq!((whole.features.len(), whole.skipped), (602, 4), "{format}"); let bare: Vec<_> = whole.features.iter().filter(|f| f.geometry.is_none()).map(|f| (f.id, f.attrs[0].clone())).collect(); assert_eq!(bare, vec![(601, Value::Text("no place".into())), (602, Value::Text("empty".into()))], "{format}"); let (rows, summary, _) = visit_all(src.as_ref(), None, 1024); - assert_eq!((rows, summary.matched, summary.skipped), (whole.features, 602, 3), "{format}"); + assert_eq!((rows, summary.matched, summary.skipped), (whole.features, 602, 4), "{format}"); let world = Extent::new(-180.0, -85.0, 180.0, 85.0); let part = src.read(&Request { extent: Some(world), limit: 10_000 }).unwrap(); assert_eq!(part.features.len(), 600, "{format}");