Skip to content
15 changes: 14 additions & 1 deletion crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ pub struct Exported {
/// them (GeoParquet; empty for the other formats). See
/// [`crate::source::parquet::unfit_note`].
pub unfit: Vec<usize>,
/// Vertices beyond 85.05° north or south, written at 85.05° by an export in Web Mercator,
/// where it ends (0 in another system). See [`crate::web_mercator_moved_note`].
pub beyond_web_mercator: usize,
}

/// Writes every feature of `src` to `path` in the format of its extension, in `crs` (see
Expand Down Expand Up @@ -97,14 +100,24 @@ fn write_chunks_to(
let t = if crs == Crs::LonLat { None } else { Some(crs.transformer().map_err(DataError::Format)?) };
let mut total = 0usize;
let mut skipped = 0usize;
let mut beyond_web_mercator = 0usize;
let mut null_numbers = 0usize;
let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> {
// A GeoPackage export that starts again feeds the features again, and counts them again.
total = 0;
beyond_web_mercator = 0;
null_numbers = 0;
// The writers count the features; a source that doesn't know how many leaves the total at 0.
crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX)));
let batch = src.visit_chunks(None, bytes, &mut |mut chunk| {
// Counted from the longitude/latitude, before projecting. Every vertex is written:
// the formats written in Web Mercator a chunk at a time (GeoPackage, FlatGeobuf)
// leave out only features without geometry.
if crs == Crs::WebMercator {
for g in chunk.iter().filter_map(|f| f.geometry.as_ref()) {
g.for_each_coord(|c| beyond_web_mercator += usize::from(c.x.is_finite() && c.y.abs() > rustgis_geom::MAX_LAT));
}
}
if let Some(t) = &t {
// Counted as the writer writes them, so only stopped on a cancel here.
project_each(&mut chunk, t, crs, false)?;
Expand Down Expand Up @@ -151,7 +164,7 @@ fn write_chunks_to(
Ok::<_, DataError>(0)
})?,
};
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit }, table))
Ok((Exported { written: total.saturating_sub(left_out), left_out, skipped, null_numbers, unfit, beyond_web_mercator }, table))
}

/// Counts attributes a GeoJSON or GeoPackage writer replaces with null. Other formats have
Expand Down
48 changes: 43 additions & 5 deletions crates/data/src/export/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,40 @@ impl FeatureSource for Runs {
}
}

/// An export in Web Mercator written a chunk at a time counts the vertices beyond 85.05° it
/// writes at 85.05°, as it writes them: from a source that declares no extent, whose features
/// beyond aren't in its first chunk. In longitude/latitude, or within 85.05°, it counts none.
#[test]
fn a_streamed_web_mercator_export_counts_what_it_writes_at_85_degrees() {
let d = folder("merc");
let mut ds = Dataset::new("polar", GeometryType::Point, vec![]);
for (x, y) in [(0.0, 0.0), (1.0, 10.0), (2.0, 89.0), (3.0, -90.0), (4.0, 60.0), (5.0, 86.0)] {
ds.push_next(Some(Geometry::Point(Coord::new(x, y))), vec![]);
}
ds.push_next(None, vec![]);
let mut src = Runs::new(&ds, 2);
src.info.extent = rustgis_geom::Extent::EMPTY;
for ext in ["gpkg", "fgb"] {
for (crs, n) in [(Crs::WebMercator, 3), (Crs::LonLat, 0)] {
let path = d.join(format!("polar-{}.{ext}", crs.epsg()));
let e = write_chunks_in(&src, &ds, &path, crs, 1).unwrap();
assert_eq!(e.beyond_web_mercator, n, ".{ext} in {}", crs.label());
// As many as the file holds at 85.05°.
let mut at_edge = 0;
for g in read_back(&path).features.iter().filter_map(|f| f.geometry.as_ref()) {
g.for_each_coord(|c| at_edge += usize::from((c.y.abs() - rustgis_geom::MAX_LAT).abs() < 1e-6));
}
assert_eq!(at_edge, n, ".{ext} in {}", crs.label());
}
}
assert_eq!(
crate::web_mercator_moved_note("polar.gpkg", "polar", 3).as_deref(),
Some("polar.gpkg: 3 vertices of polar are beyond 85.05° north or south, and written at 85.05°, where Web Mercator ends")
);
assert_eq!(crate::web_mercator_moved_note("polar.gpkg", "polar", 0), None);
let _ = std::fs::remove_dir_all(d);
}

/// Every streamed format, from a source read in place a few features a chunk, holds what the
/// same export of the whole layer holds, in longitude/latitude and in a projected system.
#[test]
Expand Down Expand Up @@ -130,11 +164,15 @@ fn a_streamed_export_writes_what_a_whole_export_writes() {
let what = format!("{format} to .{ext} in {}", crs.label());
let unfit = streamed.unfit.clone();
assert!(unfit.iter().all(|n| *n == 0), "{what}: {unfit:?}");
assert_eq!(
streamed,
Exported { written: whole.features.len() - left_out, left_out, skipped: 0, null_numbers: 0, unfit },
"{what}"
);
let expected = Exported {
written: whole.features.len() - left_out,
left_out,
skipped: 0,
null_numbers: 0,
unfit,
beyond_web_mercator: 0,
};
assert_eq!(streamed, expected, "{what}");
let (a, b) = (read_back(&a), read_back(&b));
assert_eq!((&a.fields, a.geometry_type, a.crs), (&b.fields, b.geometry_type, b.crs), "{what}");
assert_eq!(a.features, b.features, "{what}");
Expand Down
146 changes: 146 additions & 0 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2686,6 +2686,26 @@ fn shapefile_keeps(g: Option<&Geometry>, gt: GeometryType) -> bool {
}
}

/// Calls `f` with each vertex of `g` that [`shapefile_parts_in`] writes, in a layer of type `gt`,
/// once a projected export has made every coordinate finite (`project` refuses one it can't):
/// none of another kind than the layer's, and of a line or a polygon only the parts it can draw,
/// a line part of two vertices, a ring of three (a polygon's holes go with its outer ring).
pub(crate) fn shapefile_vertices(g: &Geometry, gt: GeometryType, f: impl FnMut(Coord)) {
match (g, gt) {
(Geometry::Point(_) | Geometry::MultiPoint(_), GeometryType::Point) => g.for_each_coord(f),
(Geometry::Line(parts), GeometryType::Polyline) => parts.iter().filter(|p| p.len() >= 2).flatten().copied().for_each(f),
(Geometry::Polygon(polys), GeometryType::Polygon) => polys
.iter()
.filter(|p| p.first().is_some_and(|outer| outer.len() >= 3))
.flatten()
.filter(|r| r.len() >= 3)
.flatten()
.copied()
.for_each(f),
_ => {}
}
}

/// What [`shapefile_parts_in`] leaves out of geometry `g` it writes to a layer of type `gt`, as
/// (vertices, parts), `drawable` telling the vertices it writes: the vertices that aren't, and the
/// line parts and polygon rings with too few drawable vertices (a part needs two, a ring three),
Expand Down Expand Up @@ -3694,6 +3714,132 @@ mod tests {
);
}

/// An export in Web Mercator writes a latitude beyond 85.05°, an infinite one too, at 85.05°,
/// where Web Mercator ends, and says how many vertices it moved; for a streamed layer, its
/// source's extent tells. Within 85.05°, or in another system, nothing is said of it.
#[test]
fn a_web_mercator_export_says_what_it_writes_at_85_degrees() {
use rustgis_geom::{Extent, MAX_LAT, lonlat_to_merc, merc_to_lonlat};
let c = Coord::new;
let ext = |ds: &Dataset| ds.extent();
let layer = |name: &str, points: &[Coord]| {
let mut ds = Dataset::new(name, GeometryType::Point, vec![]);
for p in points {
ds.push(Some(Geometry::Point(*p)), vec![]);
}
ds
};
let polar = layer("polar", &[c(10.0, 89.0), c(20.0, -90.0), c(30.0, 60.0), c(40.0, f64::INFINITY)]);
// What the export writes: each latitude beyond at 85.05°.
let merc = project(&polar, Crs::WebMercator).unwrap();
let written: Vec<Coord> = merc
.features
.iter()
.filter_map(|f| match f.geometry {
Some(Geometry::Point(p)) => Some(merc_to_lonlat(p)),
_ => None,
})
.collect();
assert_eq!(written.len(), 4);
for (w, lat) in written.iter().zip([MAX_LAT, -MAX_LAT, 60.0, MAX_LAT]) {
assert!((w.y - lat).abs() < 1e-9, "{w:?}");
}
assert_eq!(merc.features[0].geometry, Some(Geometry::Point(lonlat_to_merc(c(10.0, MAX_LAT)))));
assert_eq!(
crate::export_notes([(&polar, ext(&polar))], "polar.shp", Crs::WebMercator),
["polar.shp: 3 vertices of polar are beyond 85.05° north or south, and written at 85.05°, where Web Mercator ends"]
);
// In another system, or within 85.05°: nothing.
for crs in [Crs::LonLat, Crs::Epsg(32631)] {
assert!(crate::export_notes([(&polar, ext(&polar))], "polar.shp", crs).is_empty(), "{crs:?}");
}
let within = layer("within", &[c(30.0, 60.0), c(0.0, -MAX_LAT)]);
assert!(crate::export_notes([(&within, ext(&within))], "within.shp", Crs::WebMercator).is_empty());
// One vertex; and several layers in one notice, counting the vertices of all.
let one = layer("one", &[c(0.0, 86.0)]);
assert_eq!(
crate::export_notes([(&one, ext(&one))], "one.gpkg", Crs::WebMercator),
["one.gpkg: 1 vertex of one is beyond 85.05° north or south, and written at 85.05°, where Web Mercator ends"]
);
assert_eq!(
crate::export_notes([(&polar, ext(&polar)), (&within, ext(&within)), (&one, ext(&one))], "map.gpkg", Crs::WebMercator),
[
"map.gpkg: 4 vertices of 2 layers (polar, one) are beyond 85.05° north or south, and written at 85.05°, where Web Mercator ends"
]
);
// A streamed layer's features are only those in view: its source's extent tells, without
// a count, and only that it reaches beyond, as a source can declare a wider one than its
// data's.
let mut window = layer("tiles", &[c(0.0, 0.0)]);
window.windowed = true;
assert_eq!(
crate::export_notes([(&window, Extent::new(-180.0, -90.0, 180.0, 84.0))], "tiles.gpkg", Crs::WebMercator),
[
"tiles.gpkg: the extent of tiles reaches beyond 85.05° north or south, and what lies beyond is written at 85.05°, where Web Mercator ends"
]
);
assert!(crate::export_notes([(&window, Extent::new(-180.0, -80.0, 180.0, 84.0))], "tiles.gpkg", Crs::WebMercator).is_empty());
// A layer counted and a window told by its extent: one notice, saying both.
assert_eq!(
crate::export_notes([(&polar, ext(&polar)), (&window, Extent::new(-180.0, -90.0, 180.0, 84.0))], "map.gpkg", Crs::WebMercator),
[
"map.gpkg: 3 vertices of polar are beyond 85.05° north or south, and written at 85.05°; the extent of tiles reaches beyond 85.05° north or south, and what lies beyond is written at 85.05°, where Web Mercator ends"
]
);
// A layer without features that isn't a window (a raster layer's, or a streamed layer's
// written a chunk at a time, whose export counts as it writes) has no extent tell.
let empty = Dataset::new("empty", GeometryType::Point, vec![]);
assert!(crate::export_notes([(&empty, Extent::new(-180.0, -90.0, 180.0, 90.0))], "empty.gpkg", Crs::WebMercator).is_empty());
// A Shapefile holds only geometry of the layer's kind, and of it only parts it can draw:
// the vertices it leaves out aren't counted, though a GeoPackage holds them. Of the lines,
// the 2 vertices of the first are written, not the one-vertex part or the point. Of the
// polygons, the 4 of the first one's three-vertex hole, not its two-vertex one, nor the
// second polygon, whose outer ring has 2 vertices, with its hole.
let mut lines = Dataset::new("lines", GeometryType::Polyline, vec![]);
lines.push(Some(Geometry::Line(vec![vec![c(0.0, 88.0), c(10.0, 88.0)]])), vec![]);
lines.push(Some(Geometry::Line(vec![vec![c(0.0, 89.0)], vec![c(0.0, 0.0), c(1.0, 0.0)]])), vec![]);
lines.push(Some(Geometry::Point(c(5.0, 89.0))), vec![]);
let mut polys = Dataset::new("polys", GeometryType::Polygon, vec![]);
let ring = |r: &[(f64, f64)]| r.iter().map(|&(x, y)| c(x, y)).collect::<Vec<_>>();
polys.push(
Some(Geometry::Polygon(vec![vec![
ring(&[(0.0, 0.0), (0.0, 10.0), (10.0, 10.0), (0.0, 0.0)]),
ring(&[(1.0, 86.0), (2.0, 86.0), (1.5, 87.0), (1.0, 86.0)]),
ring(&[(3.0, 89.0), (3.0, 89.0)]),
]])),
vec![],
);
polys.push(
Some(Geometry::Polygon(vec![vec![
ring(&[(0.0, 89.0), (1.0, 89.0)]),
ring(&[(0.0, 86.0), (1.0, 86.0), (1.0, 87.0), (0.0, 86.0)]),
]])),
vec![],
);
for (ds, in_shapefile, in_geopackage) in [(&lines, 2, 4), (&polys, 4, 12)] {
let name = ds.name.as_str();
for (target, n) in
[(format!("{name}.shp"), in_shapefile), (format!("{name}.zip"), in_shapefile), (format!("{name}.gpkg"), in_geopackage)]
{
assert_eq!(
crate::export_notes([(ds, ext(ds))], &target, Crs::WebMercator),
[format!(
"{target}: {n} vertices of {name} are beyond 85.05° north or south, and written at 85.05°, where Web Mercator ends"
)]
);
}
// As many as the Shapefile written holds at 85.05°.
let (zip, _) = write_bytes_in(ds, "shp", Crs::WebMercator).unwrap();
let mut at_edge = 0;
for (back, _) in read_zip(&format!("{name}.zip"), &zip).unwrap() {
for g in back.features.iter().filter_map(|f| f.geometry.as_ref()) {
g.for_each_coord(|c| at_edge += usize::from((c.y.abs() - MAX_LAT).abs() < 1e-6));
}
}
assert_eq!(at_edge, in_shapefile, "{name}");
}
}

#[test]
fn wkt_geometries() {
use crate::interop::parse_wkt;
Expand Down
83 changes: 83 additions & 0 deletions crates/data/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,8 @@ fn layer_list(names: &[&str]) -> String {
/// body's name ([`raster::Crs::is_planetary`]).
/// - Coordinates that aren't longitude/latitude (beyond ±180.5° or ±90.5°, as the readers check),
/// which an export writes as if they were.
/// - Latitudes beyond 85.05° north or south, which an export in Web Mercator writes at 85.05°,
/// where it ends ([`rustgis_geom::lonlat_to_merc`]).
pub fn export_notes<'a>(
layers: impl IntoIterator<Item = (&'a Dataset, rustgis_geom::Extent)>,
target: &str,
Expand Down Expand Up @@ -484,9 +486,90 @@ pub fn export_notes<'a>(
layer_list(&names)
)),
}
if crs == raster::Crs::WebMercator {
notes.extend(web_mercator_note(&layers, target));
}
notes
}

/// What an export in Web Mercator says of the layers it writes past 85.05° north or south, where
/// Web Mercator ends: [`rustgis_geom::lonlat_to_merc`] writes a latitude beyond (an infinite one
/// too) at 85.05°. The vertices moved are counted from a layer's features, those the file holds:
/// a Shapefile target (`.shp`, or zipped to `.zip`, as the exports name it) holds only geometry of
/// the layer's kind, and of it only parts it can draw ([`io::shapefile_vertices`]). A missing
/// coordinate, or an infinite longitude, isn't counted: it can't be projected, so the export
/// refuses it. A layer whose features are only a window onto its source (`windowed`) has its
/// extent tell instead, without a count; as a source can declare a wider extent than its data's,
/// that notice says only that the extent reaches beyond. A streamed layer written a chunk at a
/// time comes with no features, and its export counts them as it writes them
/// ([`export::Exported::beyond_web_mercator`], said by [`web_mercator_moved_note`]). The count is
/// one pass over the coordinates, without a Cancel point, as the other notices' passes are.
fn web_mercator_note(layers: &[(&Dataset, rustgis_geom::Extent)], target: &str) -> Option<String> {
use rustgis_geom::MAX_LAT;
let shapefile = std::path::Path::new(target)
.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| e.eq_ignore_ascii_case("shp") || e.eq_ignore_ascii_case("zip"));
let beyond: Vec<(&str, Option<usize>)> = layers
.iter()
.filter_map(|(ds, e)| {
let mut moved = 0usize;
if !ds.windowed {
let mut count = |c: Coord| moved += usize::from(c.x.is_finite() && c.y.abs() > MAX_LAT);
for g in ds.features.iter().filter_map(|f| f.geometry.as_ref()) {
if shapefile {
crate::io::shapefile_vertices(g, ds.geometry_type, &mut count);
} else {
g.for_each_coord(&mut count);
}
}
}
let past = ds.windowed && !e.is_empty() && (e.min_y < -MAX_LAT || e.max_y > MAX_LAT);
if moved > 0 { Some((ds.name.as_str(), Some(moved))) } else { past.then_some((ds.name.as_str(), None)) }
})
.collect();
web_mercator_said(&beyond, target)
}

/// What an export in Web Mercator written a chunk at a time says of the `moved` vertices of
/// `layer` it wrote at 85.05° north or south, where Web Mercator ends, as it counted them
/// ([`export::Exported::beyond_web_mercator`]): as [`export_notes`] says it of a layer it has
/// whole. `None` for none.
pub fn web_mercator_moved_note(target: &str, layer: &str, moved: usize) -> Option<String> {
if moved == 0 {
return None;
}
web_mercator_said(&[(layer, Some(moved))], target)
}

/// The 85.05° notice for the layers `beyond` (each with its count of vertices moved, or `None`
/// when only its extent tells), or `None` for none. Layers counted and layers told by their
/// extent are said in one notice, each in its own clause, so neither hides the other.
fn web_mercator_said(beyond: &[(&str, Option<usize>)], target: &str) -> Option<String> {
let counted: Vec<&str> = beyond.iter().filter(|(_, n)| n.is_some()).map(|(name, _)| *name).collect();
let total: usize = beyond.iter().filter_map(|(_, n)| *n).sum();
let extents: Vec<&str> = beyond.iter().filter(|(_, n)| n.is_none()).map(|(name, _)| *name).collect();
let mut said = Vec::new();
match (counted.as_slice(), total) {
([], _) => {}
(_, 1) => said.push(format!("1 vertex of {} is beyond 85.05° north or south, and written at 85.05°", counted.join(", "))),
([one], n) => said.push(format!("{n} vertices of {one} are beyond 85.05° north or south, and written at 85.05°")),
(_, n) => said.push(format!("{n} vertices of {} are beyond 85.05° north or south, and written at 85.05°", layer_list(&counted))),
}
match extents.as_slice() {
[] => {}
[one] => said.push(format!("the extent of {one} reaches beyond 85.05° north or south, and what lies beyond is written at 85.05°")),
_ => said.push(format!(
"the extents of {} reach beyond 85.05° north or south, and what lies beyond is written at 85.05°",
layer_list(&extents)
)),
}
if said.is_empty() {
return None;
}
Some(format!("{target}: {}, where Web Mercator ends", said.join("; ")))
}

/// Why exporting `ds` in `crs` would write other data than the layer's: a layer on another body
/// projected onto another ellipsoid ([`raster::planetary::export_error`]). `None` when the export
/// writes its numbers, which [`export_notes`] says when they go under another system's name.
Expand Down
Loading
Loading