Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 108 additions & 4 deletions crates/data/src/source/fgb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,11 @@ fn writable(browser: bool) -> Result<()> {
/// temporary file and only its index entry in memory, so a whole layer needn't be held. The
/// writer needs that file whether or not it writes an index, and the web app has no files, so
/// there a FlatGeobuf is refused rather than written.
///
/// A point layer's file takes the type of its first point. Should a later one be of the other
/// kind (a multipoint after points, or a point after multipoints), the export starts again
/// (feeding the features again) with each feature's type in the file, which a file of one type
/// can't hold. Nothing reaches `out` before the last feature, so the first run leaves it empty.
pub(crate) fn write_features(
name: &str,
geometry_type: GeometryType,
Expand All @@ -711,6 +716,29 @@ pub(crate) fn write_features(
out: &mut dyn std::io::Write,
feed: &mut dyn FnMut(&mut dyn FnMut(&[Feature]) -> Result<()>) -> Result<()>,
) -> Result<usize> {
for mixed_points in [false, true] {
if let Some(left_out) = write_run(name, geometry_type, fields, crs, mixed_points, out, feed)? {
return Ok(left_out);
}
}
Err(DataError::Format("FlatGeobuf: the export didn't settle on a geometry type".into()))
}

/// One run of [`write_features`], each feature's geometry type in the file when `mixed_points`:
/// how many were left out, or `None` when a point layer turned out to mix points and
/// multipoints and the export must start again so. With `mixed_points`, a point layer's
/// geometry of another kind (a line, a polygon) is refused: a file of mixed types is read as
/// the layer of its first feature's kind.
#[allow(clippy::too_many_arguments)]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit (high confidence): write_run takes 7 parameters, and clippy's too_many_arguments only fires above 7, so this allow isn't needed. Removing it:

Suggested change
#[allow(clippy::too_many_arguments)]

Also a readability nit, grouped here: the match guard if *first_multi.get_or_insert(m) != m mutates first_multi inside the guard. Setting it in a separate statement before the match would be easier to follow. These are quality points, so they can wait for the release quality pass.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Left for the release quality pass: both are style points, and the allow is harmless.

fn write_run(
name: &str,
geometry_type: GeometryType,
fields: &[Field],
crs: Crs,
mixed_points: bool,
out: &mut dyn std::io::Write,
feed: &mut dyn FnMut(&mut dyn FnMut(&[Feature]) -> Result<()>) -> Result<()>,
) -> Result<Option<usize>> {
use flatgeobuf::geozero::wkb::Wkb;
use flatgeobuf::{FgbCrs, FgbWriter, FgbWriterOptions, GeometryType as G};
writable(cfg!(target_arch = "wasm32"))?;
Expand All @@ -724,6 +752,7 @@ pub(crate) fn write_features(
let code = i32::from(crs.epsg());
let options = FgbWriterOptions {
crs: FgbCrs { org: (code > 0).then_some("EPSG"), code, wkt: wkt.as_deref(), ..Default::default() },
detect_type: !mixed_points,
..Default::default()
};
let mut w = FgbWriter::create_with_options(name, ty, options).map_err(err)?;
Expand All @@ -736,14 +765,35 @@ pub(crate) fn write_features(
};
w.add_column(&f.name, t, |_, c| c.nullable = true);
}
let mut left_out = 0;
feed(&mut |features| {
let (mut left_out, mut first_multi, mut again) = (0, None, false);
let fed = feed(&mut |features| {
for feat in features {
crate::progress::step(1)?;
let Some(g) = &feat.geometry else {
left_out += 1;
continue;
};
if geometry_type == GeometryType::Point {
let multi = match g {
crate::Geometry::Point(_) => Some(false),
crate::Geometry::MultiPoint(_) => Some(true),
_ => None,
};
match (multi, mixed_points) {
(Some(m), false) if *first_multi.get_or_insert(m) != m => {
again = true;
// Stops the traversal; the export starts again.
return Err(DataError::Format("FlatGeobuf: starting again".into()));
}
(None, true) => {
return Err(DataError::Format(format!(
"FlatGeobuf: feature {} of a point layer isn't a point or multipoint",
feat.id
)));
}
_ => {}
}
}
let mut failed = None;
w.add_feature_geom(Wkb(wkb::encode(g)), |out| {
use flatgeobuf::geozero::PropertyProcessor;
Expand All @@ -766,9 +816,13 @@ pub(crate) fn write_features(
}
}
Ok(())
})?;
});
if again {
return Ok(None);
}
fed?;
w.write(out).map_err(err)?;
Ok(left_out)
Ok(Some(left_out))
}

#[cfg(test)]
Expand All @@ -782,4 +836,54 @@ mod write_tests {
assert!(matches!(writable(true), Err(DataError::Unsupported(m)) if m.contains("browser")));
assert!(writable(false).is_ok());
}

/// A point layer with points and multipoints (as a GeoJSON file gives) is written whole,
/// each feature its own kind; a layer of points alone keeps its one geometry type.
#[test]
fn a_point_layer_mixing_points_and_multipoints_is_written() {
use crate::{Coord, Dataset, Geometry};
let mut ds = Dataset::new("m", GeometryType::Point, vec![Field::new("n", FieldType::Integer)]);
ds.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Int(1)]);
ds.push(Some(Geometry::MultiPoint(vec![Coord::new(3.0, 4.0), Coord::new(5.0, 6.0)])), vec![Value::Int(2)]);
let bytes = write_in(&ds, Crs::LonLat).unwrap();
let (back, _) = crate::io::read_bytes("m.fgb", &bytes).unwrap();
let mut got: Vec<(Value, Option<Geometry>)> = back.features.iter().map(|f| (f.attrs[0].clone(), f.geometry.clone())).collect();
got.sort_by_key(|(n, _)| n.as_f64().map(|n| n as i64));
let want: Vec<(Value, Option<Geometry>)> = ds.features.iter().map(|f| (f.attrs[0].clone(), f.geometry.clone())).collect();
assert_eq!(got, want);
let mut points = Dataset::new("p", GeometryType::Point, vec![]);
points.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![]);
let header = flatgeobuf::FgbReader::open(std::io::Cursor::new(write_in(&points, Crs::LonLat).unwrap())).unwrap();
assert_eq!(header.header().geometry_type(), flatgeobuf::GeometryType::Point);
}

/// Fed a chunk at a time (a layer read in place), the multipoint arrives after the point
/// has set the file's type: the export starts again and writes both. A line in a point layer
/// isn't written into a file of mixed types, which would read back as a line layer.
#[test]
fn a_streamed_point_layer_mixing_points_and_multipoints_starts_again() {
use crate::{Coord, Geometry};
let chunks = [
vec![Feature { id: 1, geometry: Some(Geometry::Point(Coord::new(1.0, 2.0))), attrs: vec![] }],
vec![Feature { id: 2, geometry: Some(Geometry::MultiPoint(vec![Coord::new(3.0, 4.0), Coord::new(5.0, 6.0)])), attrs: vec![] }],
];
let mut runs = 0;
let mut out = Vec::new();
write_features("m", GeometryType::Point, &[], Crs::LonLat, &mut out, &mut |add| {
runs += 1;
chunks.iter().try_for_each(|c| add(c))
})
.unwrap();
assert_eq!(runs, 2);
let (back, _) = crate::io::read_bytes("m.fgb", &out).unwrap();
let mut got: Vec<_> = back.features.iter().filter_map(|f| f.geometry.clone()).collect();
got.sort_by_key(|g| matches!(g, Geometry::MultiPoint(_)));
assert_eq!(got, chunks.iter().flatten().filter_map(|f| f.geometry.clone()).collect::<Vec<_>>());
let line = Feature { id: 3, geometry: Some(Geometry::Line(vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 1.0)]])), attrs: vec![] };
let with_line = [chunks[0].clone(), chunks[1].clone(), vec![line]];
let e = write_features("m", GeometryType::Point, &[], Crs::LonLat, &mut Vec::new(), &mut |add| {
with_line.iter().try_for_each(|c| add(c))
});
assert!(matches!(&e, Err(DataError::Format(m)) if m.contains("feature 3 of a point layer isn't a point")), "{e:?}");
}
}
Loading