Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/rustgis/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ impl eframe::App for App {
/// Every way eframe closes the app comes here, Cmd+Q on macOS included, which may not
/// return from `run_native`: the run ended cleanly.
fn on_exit(&mut self) {
self.0.end_exports();
self.0.end_autosave();
running::close(&self.1);
}
Expand Down
15 changes: 15 additions & 0 deletions crates/analysis/src/control.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@ impl Control {
}
}

/// The same flag and count reach the data crate's writers (an export run under the control):
/// their steps count here, and they stop when the run is cancelled.
impl rustgis_data::progress::Probe for Control {
fn cancelled(&self) -> bool {
self.is_cancelled()
}
fn begin(&self, total: u64) {
self.done.store(0, Ordering::Relaxed);
self.total.store(total, Ordering::Relaxed);
}
fn advance(&self, n: u64) {
self.done.fetch_add(n, Ordering::Relaxed);
}
}

/// A tool stopped because it was asked to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Cancelled;
Expand Down
2 changes: 2 additions & 0 deletions crates/analysis/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,8 @@ pub fn run_under<T>(name: &str, control: &std::sync::Arc<control::Control>, work
return Err(format!("{name} was cancelled before it started"));
}
let _current = control::install(Some(control.clone()));
// A file the work writes or reads counts and stops with it too.
let _data = rustgis_data::progress::install(Some(control.clone()));
// No count until the work starts one: a control used before doesn't show that run's.
control::begin(0);
let res = work();
Expand Down
9 changes: 6 additions & 3 deletions crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

use std::path::Path;

use crate::io::{CsvWriter, GeoJsonWriter, PROJECTED_FORMATS, check_projected, project_features, write_atomic_with};
use crate::io::{CsvWriter, GeoJsonWriter, PROJECTED_FORMATS, check_projected, project_each, write_atomic_with};
use crate::raster::Crs;
use crate::source::FeatureSource;
use crate::{DataError, Dataset, Feature, GeometryType, Result};
Expand Down Expand Up @@ -68,11 +68,14 @@ fn write_chunks_in(src: &dyn FeatureSource, layer: &Dataset, path: &Path, crs: C
let mut total = 0usize;
let mut skipped = 0usize;
let mut feed = |add: &mut dyn FnMut(&[Feature]) -> Result<()>| -> Result<()> {
// A GeoPackage export that starts again feeds the features again.
// A GeoPackage export that starts again feeds the features again, and counts them again.
total = 0;
// The writers count the features; a source that doesn't know how many leaves the total at 0.
crate::progress::begin(src.info().count.map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX)));
let batch = src.visit_chunks(None, bytes, &mut |mut chunk| {
if let Some(t) = &t {
project_features(&mut chunk, t, crs)?;
// Counted as the writer writes them, so only stopped on a cancel here.
project_each(&mut chunk, t, crs, false)?;
}
total = total.saturating_add(chunk.len());
add(&chunk)
Expand Down
16 changes: 10 additions & 6 deletions crates/data/src/interop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,14 @@ fn kml_geometry(g: &Geometry) -> String {
}

/// A KML document, one placemark per feature, named by `name_field` and with every attribute.
pub fn to_kml(ds: &Dataset, name_field: Option<usize>) -> String {
/// A cancelled run ([`crate::progress`]) stops it with [`crate::DataError::Cancelled`].
pub fn to_kml(ds: &Dataset, name_field: Option<usize>) -> crate::Result<String> {
let mut out = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document><name>{}</name>\n",
xml(&ds.name)
);
for (r, f) in ds.features.iter().enumerate() {
crate::progress::step(1)?;
let Some(g) = &f.geometry else { continue };
let name = name_field.map(|fi| ds.value(r, fi).to_string()).unwrap_or_else(|| f.id.to_string());
let data: String = ds
Expand All @@ -78,17 +80,19 @@ pub fn to_kml(ds: &Dataset, name_field: Option<usize>) -> String {
));
}
out.push_str("</Document></kml>\n");
out
Ok(out)
}

/// A GPX file: points become waypoints, lines tracks (polygon outlines too).
pub fn to_gpx(ds: &Dataset, name_field: Option<usize>) -> String {
/// A cancelled run ([`crate::progress`]) stops it with [`crate::DataError::Cancelled`].
pub fn to_gpx(ds: &Dataset, name_field: Option<usize>) -> crate::Result<String> {
let mut out = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<gpx version=\"1.1\" creator=\"RustGIS\" xmlns=\"http://www.topografix.com/GPX/1/1\">\n",
);
let pt = |c: &Coord| format!("lat=\"{}\" lon=\"{}\"", c.y, c.x);
let mut tracks = String::new();
for (r, f) in ds.features.iter().enumerate() {
crate::progress::step(1)?;
let Some(g) = &f.geometry else { continue };
let name = xml(&name_field.map(|fi| ds.value(r, fi).to_string()).unwrap_or_else(|| f.id.to_string()));
match g {
Expand All @@ -113,7 +117,7 @@ pub fn to_gpx(ds: &Dataset, name_field: Option<usize>) -> String {
}
out.push_str(&tracks);
out.push_str("</gpx>\n");
out
Ok(out)
}

/// Well-known text for a geometry.
Expand Down Expand Up @@ -515,9 +519,9 @@ mod tests {
fn writes_kml_gpx_wkt() {
let mut ds = Dataset::new("t", GeometryType::Point, vec![Field::new("name", FieldType::Text)]);
ds.push(Some(Geometry::Point(Coord::new(2.35, 48.85))), vec![Value::Text("Paris & co".into())]);
let k = to_kml(&ds, Some(0));
let k = to_kml(&ds, Some(0)).unwrap();
assert!(k.contains("<name>Paris &amp; co</name>") && k.contains("<coordinates>2.35,48.85</coordinates>"));
let g = to_gpx(&ds, Some(0));
let g = to_gpx(&ds, Some(0)).unwrap();
assert!(g.contains("<wpt lat=\"48.85\" lon=\"2.35\">"));
assert_eq!(wkt(&Geometry::Line(vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 1.0)]])), "LINESTRING (0 0, 1 1)");
assert!(to_wkt_csv(&ds).starts_with("wkt,name\nPOINT (2.35 48.85),Paris & co"));
Expand Down
101 changes: 95 additions & 6 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -373,9 +373,22 @@ pub fn project(ds: &Dataset, crs: Crs) -> Result<Dataset> {
}

/// Converts features' longitude/latitude geometry in place to the coordinates of `crs`, through
/// its transformer `t`. Fails on a coordinate the system can't hold, naming its feature.
/// its transformer `t`, as a step of the progress. Fails on a coordinate the system can't hold,
/// naming its feature.
pub(crate) fn project_features(features: &mut [crate::Feature], t: &crate::raster::Transformer, crs: Crs) -> Result<()> {
crate::progress::begin(features.len());
project_each(features, t, crs, true)
}

/// [`project_features`] without a step of its own: it counts each feature in the step under way
/// when `count` is set, and otherwise only stops once the run is cancelled.
pub(crate) fn project_each(features: &mut [crate::Feature], t: &crate::raster::Transformer, crs: Crs, count: bool) -> Result<()> {
for f in features {
if count {
crate::progress::step(1)?;
} else {
crate::progress::check()?;
}
if let Some(g) = f.geometry.take() {
let mut ok = true;
let p = g.map_coords(&|c| t.from_lonlat(c).unwrap_or(Coord::new(f64::NAN, f64::NAN)));
Expand Down Expand Up @@ -445,6 +458,7 @@ pub fn write_bytes_in(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec<u8>, usi

/// [`write_bytes_in`] for a dataset already in `crs`'s coordinates.
fn projected_bytes(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec<u8>, usize)> {
crate::progress::begin(ds.features.len());
Ok(match ext {
"geojson" | "json" => {
let mut w = GeoJsonWriter::new(Vec::new(), &ds.name, &ds.fields)?;
Expand All @@ -454,12 +468,12 @@ fn projected_bytes(ds: &Dataset, ext: &str, crs: Crs) -> Result<(Vec<u8>, usize)
"csv" => (to_csv(ds)?.into_bytes(), 0),
"fgb" => (crate::source::fgb::write_in(ds, crs)?, without_geometry(ds)),
"parquet" | "geoparquet" => (crate::source::parquet::write(ds)?, 0),
"kml" => (crate::interop::to_kml(ds, name_field(ds)).into_bytes(), without_geometry(ds)),
"kml" => (crate::interop::to_kml(ds, name_field(ds))?.into_bytes(), without_geometry(ds)),
"kmz" => {
let kml = crate::interop::to_kml(ds, name_field(ds)).into_bytes();
let kml = crate::interop::to_kml(ds, name_field(ds))?.into_bytes();
(crate::zip::write(&[("doc.kml".to_string(), kml)]).map_err(DataError::Format)?, without_geometry(ds))
}
"gpx" => (crate::interop::to_gpx(ds, name_field(ds)).into_bytes(), without_geometry(ds)),
"gpx" => (crate::interop::to_gpx(ds, name_field(ds))?.into_bytes(), without_geometry(ds)),
"shp" | "zip" => {
let (parts, left_out) = shapefile_parts_in(ds, crs)?;
let base: String = ds.name.chars().map(|c| if c.is_alphanumeric() || matches!(c, '-' | '_' | ' ') { c } else { '_' }).collect();
Expand Down Expand Up @@ -496,13 +510,15 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result<usize> {
"gpkg" => {
#[cfg(not(target_arch = "wasm32"))]
{
crate::progress::begin(ds.features.len());
Comment thread
pick2510 marked this conversation as resolved.
Resolved
crate::source::gpkg::write_in(ds, path, crs)?;
Ok(0)
}
#[cfg(target_arch = "wasm32")]
Err(DataError::Unsupported("GeoPackage in the browser".into()))
}
"shp" => {
crate::progress::begin(ds.features.len());
let (parts, left_out) = shapefile_parts_in(ds, crs)?;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
// Every part is written to a temporary file first and renamed into place only once all
// are written, the .shp last, so a failed write leaves the old Shapefile whole.
Expand All @@ -517,11 +533,12 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result<usize> {
}
result
});
if let Err(e) = staged {
// A cancel before the parts land leaves the old Shapefile as it was.
if let Err(e) = staged.map_err(DataError::from).and_then(|()| crate::progress::check()) {
Comment thread
pick2510 marked this conversation as resolved.
Resolved
for (tmp, _) in &written {
let _ = std::fs::remove_file(tmp);
}
return Err(e.into());
return Err(e);
}
written.sort_by_key(|(_, target)| target == path);
// The old parts are moved aside first and put back if any new part fails to land, so
Expand All @@ -530,6 +547,11 @@ pub fn write_path_in(ds: &Dataset, path: &Path, crs: Crs) -> Result<usize> {
let mut placed: Vec<std::path::PathBuf> = Vec::new();
let mut result = Ok(());
for (tmp, target) in &written {
// A cancel before the last part lands rolls back like a failed rename.
if crate::progress::check().is_err() {
result = Err(std::io::Error::other(DataError::Cancelled));
break;
}
if target.exists() {
// Any failure here rolls back like a failed rename.
let bak = match temp_beside(target) {
Expand Down Expand Up @@ -648,6 +670,11 @@ pub fn write_atomic_with<T, E: From<std::io::Error>>(
out.get_ref().sync_all()?;
// Closed before the rename, including on Windows.
drop(out);
// A cancel that came while it was written leaves the file there before as it was. This is
// the last look: the rename commits the file, and a Cancel after it finds it written.
if crate::progress::check().is_err() {
return Err(std::io::Error::other(DataError::Cancelled).into());
}
std::fs::rename(&tmp, path)?;
std::mem::forget(staged);
Ok(value)
Expand Down Expand Up @@ -1198,6 +1225,7 @@ impl<W: std::io::Write> GeoJsonWriter<W> {
/// Writes the next features.
pub(crate) fn write(&mut self, features: &[crate::Feature]) -> Result<()> {
for f in features {
crate::progress::step(1)?;
if !std::mem::take(&mut self.first) {
self.out.write_all(b",")?;
}
Expand Down Expand Up @@ -1668,6 +1696,7 @@ impl<W: std::io::Write> CsvWriter<W> {
/// Writes the next features, a row each.
pub(crate) fn write(&mut self, features: &[crate::Feature]) -> Result<()> {
for f in features {
crate::progress::step(1)?;
let mut row: Vec<String> = f.attrs.iter().map(csv_cell).collect();
if self.point {
let c = f.geometry.as_ref().and_then(|g| g.label_point());
Expand Down Expand Up @@ -2472,6 +2501,7 @@ pub fn shapefile_parts_in(ds: &Dataset, crs: Crs) -> Result<(Vec<(&'static str,
let mut w = shapefile::Writer::new(shapes, table.build_with_dest(&mut dbf));
let multi_points = ds.features.iter().any(|f| matches!(f.geometry, Some(Geometry::MultiPoint(_))));
for f in &ds.features {
crate::progress::step(1)?;
let ok = match (&f.geometry, ds.geometry_type) {
(Some(Geometry::Point(c)), GeometryType::Point) if !multi_points && finite(&c) => {
w.write_shape_and_record(&pt(c), &record(f)).map_err(fmt)?;
Expand Down Expand Up @@ -3627,4 +3657,63 @@ mod tests {
let note = shapefile_names_note(&layer(&[&huge]).fields, "h.shp").unwrap();
assert!(note.len() < 200 && note.contains(&format!("'{}…' → 'xxxxxxxxxx'", "x".repeat(40))), "{note}");
}

/// A cancelled export stops with [`DataError::Cancelled`] and leaves nothing at (or beside)
/// its destination, in every format; without a probe the same exports are written.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn a_cancelled_export_leaves_no_file() {
use crate::progress::tests::Counted;
let fields = vec![Field::new("name", FieldType::Text), Field::new("n", FieldType::Integer)];
let mut ds = Dataset::new("points", GeometryType::Point, fields);
for i in 0..3000u64 {
let c = Coord::new(i as f64 / 100.0, (i % 80) as f64);
ds.features.push(crate::Feature {
id: i + 1,
geometry: Some(Geometry::Point(c)),
attrs: vec![Value::Text(format!("p{i}")), Value::Int(i as i64)],
});
}
let dir = tempfile::tempdir().unwrap();
let exts = ["geojson", "csv", "fgb", "gpkg", "parquet", "shp", "kml", "gpx"];
for ext in exts {
let path = dir.path().join(format!("out.{ext}"));
let probe = Counted::cancelling_after(100);
let res = {
let _g = crate::progress::install(Some(probe.clone()));
write_path_in(&ds, &path, Crs::LonLat)
};
assert!(matches!(res, Err(DataError::Cancelled)), "{ext}: {res:?}");
let left: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name()).collect();
assert!(left.is_empty(), "{ext} left {left:?}");
}
// Cancelled once every feature is written, before the file is moved into place: a file
// there before stays as it was (a GeoPackage export never replaces one).
for ext in exts {
let path = dir.path().join(format!("old.{ext}"));
if ext != "gpkg" {
std::fs::write(&path, b"old").unwrap();
}
let probe = Counted::cancelling_after(ds.features.len() as u64);
let res = {
let _g = crate::progress::install(Some(probe.clone()));
write_path_in(&ds, &path, Crs::LonLat)
};
assert!(matches!(res, Err(DataError::Cancelled)), "{ext}: {res:?}");
if ext != "gpkg" {
assert_eq!(std::fs::read(&path).unwrap(), b"old", "{ext}");
std::fs::remove_file(&path).unwrap();
}
let left: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name()).collect();
assert!(left.is_empty(), "{ext} left {left:?}");
}
for ext in exts {
let path = dir.path().join(format!("out.{ext}"));
write_path_in(&ds, &path, Crs::LonLat).unwrap();
assert!(path.exists(), "{ext}");
}
for e in ["shp", "shx", "dbf"] {
assert!(dir.path().join(format!("out.{e}")).exists());
}
}
}
18 changes: 17 additions & 1 deletion crates/data/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ pub mod memory;
pub mod ogc;
pub mod osm;
pub mod photo;
pub mod progress;
pub mod query;
pub mod raster;
pub mod s3_address;
Expand All @@ -55,7 +56,7 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, thiserror::Error)]
pub enum DataError {
#[error("i/o: {0}")]
Io(#[from] std::io::Error),
Io(std::io::Error),
#[error("{0}")]
Format(String),
/// The server refused the login: no password, or a wrong one.
Expand All @@ -67,10 +68,24 @@ pub enum DataError {
/// which Zarr reads as its fill value, told apart from a slice that can't be read.
#[error("{0}: none of the slice's chunks could be read")]
NoChunks(String),
/// Stopped because its run was cancelled ([`progress`]).
#[error("cancelled")]
Cancelled,
}

pub type Result<T> = std::result::Result<T, DataError>;

/// An I/O error, or [`DataError::Cancelled`] when it carries one: a write that only returns I/O
/// errors stops with it for a cancelled run ([`progress`]).
impl From<std::io::Error> for DataError {
fn from(e: std::io::Error) -> Self {
if e.get_ref().and_then(|inner| inner.downcast_ref::<DataError>()).is_some_and(|d| matches!(d, DataError::Cancelled)) {
return DataError::Cancelled;
}
DataError::Io(e)
}
}

/// Replaces `secret` in `message` with asterisks; the text itself when it holds none.
fn without_secret(message: String, secret: &str) -> String {
if message.contains(secret) { message.replace(secret, "********") } else { message }
Expand Down Expand Up @@ -102,6 +117,7 @@ impl DataError {
DataError::Io(std::io::Error::new(e.kind(), without_secret(e.to_string(), secret)))
}
DataError::Io(e) => DataError::Io(e),
DataError::Cancelled => DataError::Cancelled,
}
}
}
Expand Down
Loading
Loading