diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index b93e7d4a95..c427b07b15 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -88,17 +88,37 @@ fn write_chunks_to( bytes: usize, ) -> Result<(Exported, Option)> { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); - if !streams(&ext) { + write_chunks_as(src, layer, path, crs, gpkg, bytes, &ext) +} + +/// Writes a source in the explicitly chosen format (ignoring case), regardless of the destination's +/// extension. Used by Data Interop, whose format parameter is independent of its path. +pub fn write_source_as_in(src: &dyn FeatureSource, layer: &Dataset, path: &Path, crs: Crs, format: &str) -> Result { + let format = format.to_ascii_lowercase(); + write_chunks_as(src, layer, path, crs, &GpkgTable::default(), CHUNK_BYTES, &format).map(|(e, _)| e) +} + +/// Shared chunk writer for extension-based exports and explicit-format conversion. +fn write_chunks_as( + src: &dyn FeatureSource, + layer: &Dataset, + path: &Path, + crs: Crs, + gpkg: &GpkgTable, + bytes: usize, + ext: &str, +) -> Result<(Exported, Option)> { + if !streams(ext) { return Err(DataError::Unsupported(format!(".{ext} written a chunk at a time"))); } if !src.visits_chunks() { return Err(DataError::Unsupported("a source that can't be read a chunk at a time".into())); } if crs != Crs::LonLat { - if !PROJECTED_FORMATS.contains(&ext.as_str()) { + if !PROJECTED_FORMATS.contains(&ext) { return Err(DataError::Unsupported(format!(".{ext} in {} (it holds longitude/latitude only)", crs.label()))); } - check_projected(&ext, crs)?; + check_projected(ext, crs)?; } // A layer that declares no geometry type takes its first geometry's, as a whole read does. let geometry_type = match layer.geometry_type { @@ -137,7 +157,7 @@ fn write_chunks_to( project_each(&mut chunk, t, crs, false)?; } total = total.saturating_add(chunk.len()); - null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext)); + null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, ext)); if ext == "csv" { csv += crate::io::CsvLosses::of(&chunk, geometry_type); } @@ -152,7 +172,7 @@ fn write_chunks_to( }; let mut table = None; let mut unfit = Vec::new(); - let left_out = match ext.as_str() { + let left_out = match ext { #[cfg(not(target_arch = "wasm32"))] "gpkg" => { // Every feature is written: one of another kind makes the column GEOMETRY. diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index b1b287321e..d1af83ea91 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -109,6 +109,27 @@ impl FeatureSource for Runs { } } +/// An explicit mixed-case format selects its text writer even when the path says otherwise. +#[test] +fn an_explicit_export_format_ignores_case_and_the_filename() { + let d = folder("explicit-format"); + let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("name", FieldType::Text)]); + ds.push_next(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text("place".into())]); + let src = Runs::new(&ds, 1); + for format in ["CSV", "GeOjSoN"] { + let path = d.join(format!("{format}.bin")); + assert_eq!(write_source_as_in(&src, &ds, &path, Crs::LonLat, format).unwrap().written, 1); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.contains("place"), "{format}: {text}"); + if format == "GeOjSoN" { + assert_eq!(serde_json::from_str::(&text).unwrap()["features"].as_array().unwrap().len(), 1); + } else { + assert_eq!(text, crate::io::to_csv(&ds).unwrap()); + } + } + std::fs::remove_dir_all(d).unwrap(); +} + /// An export in Web Mercator written a chunk at a time counts the vertices beyond 85.05° it /// writes at 85.05°, as it writes them: from a source that declares no extent, whose features /// beyond aren't in its first chunk. In longitude/latitude, or within 85.05°, it counts none. diff --git a/crates/data/src/interop.rs b/crates/data/src/interop.rs index 86cbcce8d3..f25f8bd124 100644 --- a/crates/data/src/interop.rs +++ b/crates/data/src/interop.rs @@ -465,8 +465,8 @@ impl WktParser { } } -/// CSV with a `wkt` geometry column and every attribute. -pub fn to_wkt_csv(ds: &Dataset) -> String { +/// CSV with a `wkt` geometry column and every attribute; cancellable between features. +pub fn to_wkt_csv(ds: &Dataset) -> crate::Result { // Text starting like a formula is prefixed with ' so spreadsheets show it rather than run it. let q = |s: &str| { let s = if s.starts_with(['=', '+', '-', '@']) && s.parse::().is_err() { format!("'{s}") } else { s.to_string() }; @@ -479,6 +479,7 @@ pub fn to_wkt_csv(ds: &Dataset) -> String { } out.push('\n'); for f in &ds.features { + crate::progress::step(1)?; out.push_str(&q(&f.geometry.as_ref().map(wkt).unwrap_or_default())); for v in &f.attrs { out.push(','); @@ -486,7 +487,7 @@ pub fn to_wkt_csv(ds: &Dataset) -> String { } out.push('\n'); } - out + Ok(out) } #[cfg(test)] @@ -524,7 +525,7 @@ mod tests { let g = to_gpx(&ds, Some(0)).unwrap(); assert!(g.contains("")); assert_eq!(wkt(&Geometry::Line(vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 1.0)]])), "LINESTRING (0 0, 1 1)"); - assert!(to_wkt_csv(&ds).starts_with("wkt,name\nPOINT (2.35 48.85),Paris & co")); + assert!(to_wkt_csv(&ds).unwrap().starts_with("wkt,name\nPOINT (2.35 48.85),Paris & co")); } /// Numbers are written in full and a null name as nothing, not as they're shown (rounded to @@ -545,7 +546,7 @@ mod tests { numbered.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Float(0.1234567891)]); assert!(to_kml(&numbered, Some(0)).unwrap().contains("0.1234567891")); assert!(to_gpx(&numbered, Some(0)).unwrap().contains("0.1234567891")); - let c = to_wkt_csv(&ds); + let c = to_wkt_csv(&ds).unwrap(); assert!(c.contains("POINT (1 2),,0.1234567891\n") && c.contains(",B,0.000000001\n"), "{c}"); } } diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 6a993945a2..7277ac6715 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -1389,6 +1389,13 @@ pub fn to_geojson(ds: &Dataset) -> Json { json!({"type": "FeatureCollection", "name": ds.name, "features": features}) } +/// A GeoJSON text conversion with per-feature progress and cancellation. +pub fn to_geojson_text(ds: &Dataset) -> Result { + let mut writer = GeoJsonWriter::new(Vec::new(), &ds.name, &ds.fields)?; + writer.write(&ds.features)?; + String::from_utf8(writer.finish()?).map_err(|e| DataError::Format(e.to_string())) +} + /// A feature as a GeoJSON Feature, its attributes named by `fields`. fn feature_to_geojson(fields: &[Field], f: &crate::Feature) -> Json { let mut props = Map::new(); diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index c8a08d1047..4485de10d4 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -38,6 +38,7 @@ type Start = fn(&Session, &Value) -> Result; /// The exports the app writes on another thread ([`export_start`]), each with what it is called /// (in its progress, and a cancelled run's error) and what it reads of the session. pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ + ("data.convert", "Data Interop", super::workflows::convert_work), ("share.exportLayer", "Export Features", super::share::layer_work), ("share.exportRaster", "Export Raster", super::raster::export_work), ("share.exportMap", "Export Map", super::share::map_work), diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 4212b9b5d0..79425c55a1 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -778,7 +778,7 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk /// Reports numeric loss after the file has been written, keeping the output path literal in /// translated interface text. A clean export emits no loss notice. -fn notify_null_numbers(s: &mut Session, path: &str, count: usize) { +pub(crate) fn notify_null_numbers(s: &mut Session, path: &str, count: usize) { if count > 0 { s.notify_text( crate::notice_text::NoticeText::format( diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 6b19b655cb..6edc8d5f35 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1596,43 +1596,130 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult { Ok(json!({"similar": rows})) } -/// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when -/// the layer's Z or M values were dropped on import, its positions are approximate or aren't -/// longitude/latitude, or CSV, KML or GPX couldn't hold some of it. +/// Converts text on the caller's thread for scripts and browser downloads. A streamed layer +/// needs a file path so its full source can be read off the desktop frame, rather than returning +/// only its loaded view. File conversions use the shared export lifecycle. fn convert(s: &mut Session, p: &Value) -> CmdResult { + if ps(p, "path").is_some() { + return super::exports::write_here(s, p, "data.convert"); + } + let id = s.layer_id(p)?; + if s.map().layer(id).is_some_and(|l| l.is_streamed()) { + return Err("a streamed layer needs a path to convert its whole source; text conversion would contain only the loaded view".into()); + } let ds = feature_ds(s, p)?; + let (text, notes) = converted(ds, p)?; + let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, conversion_format(p)?); + let target = ds.name.clone(); + for note in notes { + s.notify(note, false); + } + super::share::notify_null_numbers(s, &target, null_numbers); + Ok(json!({"text": text})) +} + +/// Captures a conversion's layer and parameters without reading its source on the UI thread. +/// GeoJSON and CSV use the chunk writer; KML, GPX and WKT read whole within the existing memory +/// limits, then check Cancel between features (one geometry's serialization is indivisible). +pub(crate) fn convert_work(s: &Session, p: &Value) -> Result { + feature_ds(s, p)?; + let id = s.layer_id(p)?; + let layer = s.map().layer(id).ok_or("no such layer")?.clone(); + let path = ps(p, "path").ok_or("path is required")?.to_string(); + conversion_format(p)?; + let params = p.clone(); + Ok(Box::new(move || { + let format = conversion_format(¶ms)?; + let chunks = if layer.is_streamed() && matches!(format, "geojson" | "csv") { layer.chunk_export()? } else { None }; + let (mut notes, null_numbers) = if let Some((source, ds)) = chunks { + let result = rustgis_data::export::write_source_as_in( + source.as_ref(), + &ds, + std::path::Path::new(&path), + rustgis_data::raster::Crs::LonLat, + format, + ) + .map_err(|e| e.to_string())?; + let mut notes = rustgis_data::export_notes([(&ds, layer.extent())], &path, rustgis_data::raster::Crs::LonLat); + if format == "csv" { + notes.extend(result.csv.notes(&path)); + } + if result.skipped > 0 { + notes.push(format!("{path}: {} rows couldn't be read and were left out", result.skipped)); + } + notes.extend( + rustgis_data::source::unread_dates_note(&ds.fields, &result.unread_dates) + .map(|n| format!("{path}: {}: {n}, so they're empty in the export", ds.name)), + ); + (notes, result.null_numbers) + } else { + let (whole, unreadable) = if layer.is_streamed() { + layer.load_all_counted().map(|(ds, n)| (Some(ds), n))? + } else { + (None, rustgis_map::Unreadable::default()) + }; + let ds = whole.as_ref().unwrap_or(&layer.dataset); + rustgis_data::progress::begin(ds.features.len()); + let (text, mut notes) = converted(ds, ¶ms)?; + rustgis_data::io::write_atomic(std::path::Path::new(&path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?; + if unreadable.rows > 0 { + notes.push(format!("{path}: {} rows couldn't be read and were left out", unreadable.rows)); + } + notes.extend(unreadable.dates.iter().map(|n| format!("{path}: {}: {n}, so they're empty in the export", ds.name))); + (notes, rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, format)) + }; + let bytes = std::fs::metadata(&path).map_err(|e| format!("{path}: {e}"))?.len(); + notes.push(format!("Exported {path}")); + let target = path.clone(); + let then = (null_numbers > 0).then(|| { + Box::new(move |s: &mut Session, _: &mut Value| { + super::share::notify_null_numbers(s, &target, null_numbers); + Ok(()) + }) as super::exports::Then + }); + Ok(super::exports::Wrote { + answer: json!({"path": path, "bytes": bytes}), + notes, + written_path: Some(path), + then, + #[cfg(not(target_arch = "wasm32"))] + owner: None, + }) + })) +} + +/// Validates the existing case-sensitive format parameter before starting background work. +fn conversion_format(p: &Value) -> Result<&str, String> { + match ps(p, "format").ok_or("format is required")? { + f @ ("geojson" | "csv" | "kml" | "gpx" | "wkt") => Ok(f), + f => Err(format!("unknown format '{f}'")), + } +} + +/// Serializes a complete dataset and collects its loss notices for delivery after success. +fn converted(ds: &Dataset, p: &Value) -> Result<(String, Vec), String> { let name_field = ps(p, "nameField").and_then(|n| ds.field_index(n)).or_else(|| ds.fields.iter().position(|f| f.name.eq_ignore_ascii_case("name"))); - let text = match ps(p, "format").ok_or("format is required")? { - "geojson" => rustgis_data::io::to_geojson(ds).to_string(), + let format = conversion_format(p)?; + let text = match format { + "geojson" => rustgis_data::io::to_geojson_text(ds).map_err(|e| e.to_string())?, "csv" => rustgis_data::io::to_csv(ds).map_err(|e| e.to_string())?, "kml" => rustgis_data::interop::to_kml(ds, name_field).map_err(|e| e.to_string())?, "gpx" => rustgis_data::interop::to_gpx(ds, name_field).map_err(|e| e.to_string())?, - "wkt" => rustgis_data::interop::to_wkt_csv(ds), - f => return Err(format!("unknown format '{f}'")), + _ => rustgis_data::interop::to_wkt_csv(ds).map_err(|e| e.to_string())?, }; // Every format here is longitude/latitude. let target = ps(p, "path").unwrap_or(&ds.name); let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat); // CSV holds a point a row, GPX a name for each feature and no areas. - notes.extend(rustgis_data::io::export_losses_notes(ds, ps(p, "format").unwrap_or(""), name_field, target)); + notes.extend(rustgis_data::io::export_losses_notes(ds, format, name_field, target)); // KML and GPX hold features with geometry only. - if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") { + if let ext @ ("kml" | "gpx") = format { let without = if ext == "gpx" { rustgis_data::io::gpx_left_out(ds) } else { ds.features.iter().filter(|f| f.geometry.is_none()).count() }; notes.extend(rustgis_data::io::left_out_note(target, ext, without)); } - let out = match ps(p, "path") { - Some(path) => { - rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?; - json!({"path": path, "bytes": text.len()}) - } - None => json!({"text": text}), - }; - for note in notes { - s.notify(note, false); - } - Ok(out) + Ok((text, notes)) } /// Binds the extent layer, or all full-extent layers, without computing a view or loading pixels. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index e1ce774e7d..979f27b12c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -20272,7 +20272,7 @@ fn dropped_z_values_are_said_on_import_and_export() { assert!(said(&s, mark), "stream {stream}: {:?}", s.notices_since(mark)); assert!(std::fs::read_to_string(&out).unwrap().contains("[1.0,2.0]")); let mark = s.notice_mark(); - s.run("data.convert", &json!({"layer": id, "format": "wkt"})).unwrap(); + s.run("data.convert", &json!({"layer": id, "format": "wkt", "path": dir.join(format!("converted-{stream}.csv"))})).unwrap(); assert!(said(&s, mark), "stream {stream}"); } let mut s = world(); @@ -34624,6 +34624,133 @@ impl Drop for GpkgRegressionFixture { } } +/// Conversion reads the source even when the current view contains no features; the explicit +/// format wins over the filename. Text-only conversion must not return a partial view. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn data_interop_reads_the_whole_source() { + let fixture = GpkgRegressionFixture::new("interop"); + let mut s = Session::new(Project::default()); + let loaded = s.run("layer.addData", &json!({"path": fixture.path(), "stream": true})).unwrap(); + let id = loaded["layer"].as_u64().unwrap() as u32; + assert!(s.map().layer(id).unwrap().is_streamed()); + s.map_mut().layer_mut(id).unwrap().dataset.features.clear(); + for format in ["geojson", "csv", "kml", "gpx", "wkt"] { + let path = fixture.dir.join(format!("{format}.txt")); + let mark = s.notice_mark(); + let answer = s.run("data.convert", &json!({"layer": id, "format": format, "path": path, "nameField": "lanes"})).unwrap(); + assert_eq!(s.notices_since(mark).iter().filter(|n| n.text == format!("Exported {}", path.display())).count(), 1, "{format}"); + let text = std::fs::read_to_string(&path).unwrap(); + assert_eq!(answer["bytes"].as_u64(), Some(text.len() as u64)); + let count = match format { + "geojson" => serde_json::from_str::(&text).unwrap()["features"].as_array().unwrap().len(), + "kml" => { + assert!(text.contains("2")); + text.matches("").count() + } + "gpx" => { + assert!(text.contains("2")); + text.matches(" text.lines().count() - 1, + }; + assert_eq!(count, 3, "{format}: conversion lost source features"); + } + let error = s.run("data.convert", &json!({"layer": id, "format": "geojson"})).unwrap_err(); + assert!(error.contains("path"), "{error}"); +} + +/// A non-finite REAL survives source reading but becomes JSON null: both whole and chunked +/// conversion report the count after success, and a clean conversion gives no such notice. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn data_interop_reports_non_finite_numbers() { + let fixture = GpkgRegressionFixture::new("interop-numbers"); + let db = rusqlite::Connection::open(fixture.path()).unwrap(); + db.execute_batch("ALTER TABLE roads ADD COLUMN reading REAL; UPDATE roads SET reading = 1e999 WHERE fid IN (1, 2);").unwrap(); + for stream in [false, true] { + let mut s = Session::new(Project::default()); + let id = s.run("layer.addData", &json!({"path": fixture.path(), "stream": stream})).unwrap()["layer"].clone(); + let mark = s.notice_mark(); + let path = fixture.dir.join(format!("numbers-{stream}.json")); + s.run("data.convert", &json!({"layer": id, "format": "geojson", "path": path})).unwrap(); + let text = std::fs::read_to_string(path).unwrap(); + assert_eq!(text.matches("\"reading\":null").count(), 3); + let notes: Vec<_> = s.notices_since(mark).iter().filter(|n| n.text.contains("aren't finite numbers")).collect(); + assert_eq!(notes.len(), 1, "stream {stream}: {:?}", s.notices_since(mark)); + assert!(notes[0].text.contains("2 attribute value(s)"), "{}", notes[0].text); + if !stream { + let mark = s.notice_mark(); + s.run("data.convert", &json!({"layer": id, "format": "geojson"})).unwrap(); + assert!(s.notices_since(mark).iter().any(|n| n.text.contains("2 attribute value(s)"))); + } + } + db.execute_batch("UPDATE roads SET reading = 1.5;").unwrap(); + let mut s = Session::new(Project::default()); + let id = s.run("layer.addData", &json!({"path": fixture.path(), "stream": true})).unwrap()["layer"].clone(); + let mark = s.notice_mark(); + s.run("data.convert", &json!({"layer": id, "format": "geojson", "path": fixture.dir.join("clean.json")})).unwrap(); + assert_eq!( + s.notices_since(mark).iter().map(|n| n.text.as_str()).collect::>(), + [format!("Exported {}", fixture.dir.join("clean.json").display())], + "a clean conversion reports completion without loss notices" + ); +} + +/// Date parsing can empty a value outside the loaded view. Both source-read paths report +/// that loss once, while a valid date produces no unread-date notice. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn data_interop_reports_unread_dates_from_the_whole_source() { + let fixture = GpkgRegressionFixture::new("interop-dates"); + let db = rusqlite::Connection::open(fixture.path()).unwrap(); + db.execute_batch("ALTER TABLE roads ADD COLUMN issued DATE; UPDATE roads SET issued = '2026-01-01' WHERE fid = 1;").unwrap(); + for valid in [false, true] { + db.execute("UPDATE roads SET issued = ?1 WHERE fid = 2", [if valid { "2026-02-02" } else { "not-a-date" }]).unwrap(); + for format in ["geojson", "kml"] { + let mut s = Session::new(Project::default()); + let added = s.run("layer.addData", &json!({"path": fixture.path(), "stream": true})).unwrap(); + let id = added["layer"].as_u64().unwrap() as u32; + assert!(s.map().layer(id).unwrap().is_streamed()); + s.map_mut().layer_mut(id).unwrap().dataset.features.clear(); + let path = fixture.dir.join(format!("dates-{valid}.{format}")); + let mark = s.notice_mark(); + s.run("data.convert", &json!({"layer": id, "format": format, "path": path})).unwrap(); + let notes: Vec<_> = + s.notices_since(mark).iter().filter(|n| n.text.contains("read as empty") && n.text.contains("issued")).collect(); + assert_eq!(notes.len(), usize::from(!valid), "{format} valid {valid}: {:?}", s.notices_since(mark)); + if !valid { + assert!(notes[0].text.contains("1 value isn't a date"), "{}", notes[0].text); + } + if format == "geojson" { + let out: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let row = out["features"].as_array().unwrap().iter().find(|f| f["id"] == 2).unwrap(); + assert_eq!(row["properties"]["issued"].is_null(), !valid); + } + } + } +} + +/// Each serializer observes Cancel during its work and preserves the previous destination. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn data_interop_cancels_every_format_before_replacing_the_file() { + use crate::cmd::exports::export_start; + use rustgis_analysis::control::Control; + let fixture = GpkgRegressionFixture::new("interop-cancel"); + let mut s = world(); + for format in ["geojson", "csv", "kml", "gpx", "wkt"] { + let path = fixture.dir.join(format!("{format}.txt")); + std::fs::write(&path, "previous file").unwrap(); + let run = export_start(&mut s, "data.convert", &json!({"layer": "Cities", "format": format, "path": path})).unwrap(); + let control = Control::cancelled_once_started(); + let error = run.run(&control).err().unwrap(); + assert!(control.looked() > 0, "{format}: serializer did not check Cancel"); + assert!(error.contains("cancelled"), "{format}: {error}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "previous file"); + } +} + /// A streamed layer's export says how many of its source's rows couldn't be read, written a /// chunk at a time (GeoJSON) or from a whole read (KML) alike. #[cfg(not(target_arch = "wasm32"))] @@ -39947,6 +40074,8 @@ fn exports_run_on_another_thread() { let at = |name: &str| dir.join(name).display().to_string(); s.run("layout.new", &json!({"page": "a4", "orientation": "landscape", "template": "map"})).unwrap(); for (command, file, p) in [ + ("data.convert", "converted.geojson", json!({"layer": "Cities", "format": "geojson"})), + ("data.convert", "converted.wkt.csv", json!({"layer": "Cities", "format": "wkt"})), ("share.exportMap", "map.png", json!({"width": 64, "height": 48})), ("layout.export", "layout.png", json!({"dpi": 48})), ("layout.export", "layout.pdf", json!({"dpi": 48})), @@ -43230,6 +43359,7 @@ fn long_commands_stop_when_cancelled_while_they_run() { } let exports = [ + ("data.convert", json!({"layer": "Cities", "format": "geojson", "path": out("converted.geojson")})), ("share.exportLayer", json!({"layer": "Cities", "path": out("cities.geojson")})), ("share.exportRaster", json!({"layer": dem, "path": out("dem.tif")})), ("share.exportMap", json!({"path": out("map.png"), "width": 320, "height": 200})), diff --git a/crates/ui-egui/src/analysispane.rs b/crates/ui-egui/src/analysispane.rs index f48f897710..bc02320c39 100644 --- a/crates/ui-egui/src/analysispane.rs +++ b/crates/ui-egui/src/analysispane.rs @@ -186,6 +186,30 @@ fn combo(ui: &mut Ui, id: &str, name: &str, value: &mut String, options: &[Strin .on_hover_text(tl!(name)); } +/// Asks where a desktop conversion goes before starting its export worker. Browser downloads +/// retain the text-returning command; that command refuses a streamed layer's partial view. +fn convert_layer(app: &mut RustGisApp) -> Result<(), String> { + let format = app.workflow.format.clone(); + let ext = if format == "wkt" { "csv" } else { &format }; + let name = format!("{}.{ext}", app.workflow.layer); + let mut params = json!({"layer": app.workflow.layer, "format": format}); + if app.services.download.is_some() { + let answer = app.session.run("data.convert", ¶ms)?; + app.download_bytes(&name, answer["text"].as_str().unwrap_or("").as_bytes().to_vec()); + } else if let Some(path) = crate::dialogs::pick_save(&name, &[ext], tl!("Converted data")) { + params["path"] = json!(path); + #[cfg(not(target_arch = "wasm32"))] + { + crate::exportjob::start(app, "data.convert", params)?; + } + #[cfg(target_arch = "wasm32")] + { + app.session.run("data.convert", ¶ms)?; + } + } + Ok(()) +} + /// Runs a workflow's command: on the desktop the slow ones on a thread of their own /// ([`background`]), the rest here. fn run(app: &mut RustGisApp, id: &str, p: Value) { @@ -886,20 +910,7 @@ pub fn workflows(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { } combo(ui, "wf_xf", "Format", &mut app.workflow.format, &formats); if ui.button(tl!("Convert…")).clicked() { - match app.session.run("data.convert", &json!({"layer": app.workflow.layer, "format": app.workflow.format})) { - Ok(v) => { - let ext = if app.workflow.format == "wkt" { "csv".to_string() } else { app.workflow.format.clone() }; - let name = format!("{}.{ext}", app.workflow.layer); - let bytes = v["text"].as_str().unwrap_or("").as_bytes().to_vec(); - if app.services.download.is_some() { - app.download_bytes(&name, bytes); - } else if let Some(path) = crate::dialogs::pick_save(&name, &[ext.as_str()], tl!("Converted data")) { - let r = rustgis_engine::data::io::write_atomic(std::path::Path::new(&path), &bytes).map_err(|e| e.to_string()); - app.workflow.error = r.err(); - } - } - Err(e) => app.workflow.error = Some(e), - } + app.workflow.error = convert_layer(app).err(); } } "environments" => { @@ -2631,6 +2642,57 @@ mod tests { } } + /// The real Convert button asks for a destination first, starts the export job and lands + /// its file; dismissing the picker never starts a conversion. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn interop_button_starts_an_export_after_picking_a_file() { + let mut app = RustGisApp::with_sample(); + app.ui.right = Some(crate::RightPane::Workflows); + app.workflow.section = "interop"; + app.workflow.layer = "Cities".into(); + app.workflow.format = "wkt".into(); + let dir = std::env::temp_dir().join(format!("rustgis-interop-button-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("cities.csv"); + let mut fonts = false; + let mut h = egui_kittest::Harness::builder().with_size(egui::vec2(1480.0, 900.0)).with_pixels_per_point(1.3).build_ui_state( + move |ui, app: &mut RustGisApp| { + if !fonts { + crate::theme::install_fonts(ui.ctx(), &[], None); + fonts = true; + return; + } + app.ui(ui); + }, + app, + ); + h.run(); + let logged = h.state().session.log.entries().len(); + crate::dialogs::PICKED.with(|c| *c.borrow_mut() = None); + h.get_by_role_and_label(egui::accesskit::Role::Button, "Convert…").click(); + h.run(); + assert!(h.state().export_job.is_none()); + assert_eq!(h.state().session.log.entries().len(), logged, "dismissing the picker does no conversion"); + assert!(h.state().workflow.error.is_none()); + crate::dialogs::PICKED.with(|c| *c.borrow_mut() = Some(path.display().to_string())); + h.get_by_role_and_label(egui::accesskit::Role::Button, "Convert…").click(); + h.step(); + assert!(h.state().export_job.is_some(), "{:?}", h.state().workflow.error); + let end = std::time::Instant::now() + std::time::Duration::from_secs(60); + while h.state().export_job.is_some() { + assert!(std::time::Instant::now() < end, "conversion never landed"); + h.step(); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + h.run(); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.starts_with("wkt,")); + assert_eq!(text.lines().count() - 1, h.state().session.map().layer_by_name("Cities").unwrap().dataset.features.len()); + assert_eq!(h.state().session.log.entries().last().unwrap().command, "data.convert"); + std::fs::remove_dir_all(dir).unwrap(); + } + /// Every workflow has named inputs, including both weights sharing a row (#790). #[test] fn every_workflow_names_its_inputs() { diff --git a/crates/ui-egui/src/diskwatch.rs b/crates/ui-egui/src/diskwatch.rs index bddb9ea20c..b1f7511d18 100644 --- a/crates/ui-egui/src/diskwatch.rs +++ b/crates/ui-egui/src/diskwatch.rs @@ -587,6 +587,8 @@ pub(crate) fn plan_write(app: &RustGisApp, command: &str, params: &Value) -> Opt let edit = l.gpkg.as_ref()?; (edit.path.clone(), edit.table.clone(), Some(Target::Layer { map: s.map().instance_id(), layer: l.instance_id() })) } + // Conversion replaces a whole file; it writes no container table and refreshes no reader. + "data.convert" => (params.get("path").and_then(Value::as_str)?.to_string(), String::new(), None), // An existing GeoPackage written into; a new file isn't watched yet. "share.exportLayer" | "share.exportTable" => { let path = params.get("path").and_then(Value::as_str)?.to_string(); @@ -610,7 +612,12 @@ pub(crate) fn plan_write(app: &RustGisApp, command: &str, params: &Value) -> Opt if !app.diskwatch.monitor.watches(&key) { return None; } - let stale = stale_for(&app.diskwatch.found, &key, &path, &table, writer.as_ref()); + let stale = if command == "data.convert" { + // The explicit format can differ from the suffix: even a .gpkg is replaced whole. + app.diskwatch.found.iter().filter(|f| f.key == key).map(|f| f.target.clone()).collect() + } else { + stale_for(&app.diskwatch.found, &key, &path, &table, writer.as_ref()) + }; Some(Write { before: None, result: None, key, path, stale, writer, table }) } @@ -665,7 +672,7 @@ pub(crate) fn ran(app: &mut RustGisApp, command: &str, params: &Value, answer: & } // A save that wrote nothing (no edits) isn't a write. "gpkg.saveEdits" if answer.get("saved").and_then(Value::as_bool) == Some(false) => {} - "gpkg.saveEdits" | "share.exportLayer" | "share.exportTable" => { + "gpkg.saveEdits" | "share.exportLayer" | "share.exportTable" | "data.convert" => { if let Some(write) = before { wrote(app, write, answer); } @@ -1259,6 +1266,14 @@ mod tests { let stale = |app: &RustGisApp, command, params: Value| before_write(app, command, ¶ms).map(|w| w.stale.len()); // Over a single-dataset file: the layer reading it now shows what was there before. assert_eq!(stale(&app, "share.exportLayer", json!({"path": geojson})), Some(1)); + // Interop replaces a whole file in its explicit format, even with a .gpkg suffix: + // it refreshes no layer or table, so every reader keeps its old rows. + for path in [&geojson, &package] { + let write = before_write(&app, "data.convert", &json!({"path": path, "format": "csv"})).unwrap(); + assert_eq!(write.stale.len(), 1); + assert!(write.writer.is_none()); + assert!(write.table.is_empty()); + } // Into the GeoPackage table a layer reads, by table id or by name; another table, none. let by_id = before_write(&app, "share.exportTable", &json!({"table": 7, "path": package, "mode": "replaceTable"})).unwrap(); assert!(by_id.stale.contains(&table)); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index cbb7efb516..3160233f4f 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -600,7 +600,7 @@ were rounded, and Export Raster, which writes the rounded values, says so again. | Shapefile | Data › Export to `.shp` (or `.zip` for the files zipped) | Writes `.shp`, `.shx`, `.dbf`, `.prj` (the coordinate system, see below) and `.cpg` (UTF-8). Field names are cut to 10 bytes (made unique ignoring case, and reported), text to at most 254 bytes, and numbers that do not fit their dBase field are left empty. Date fields are written as dBase dates; date-time fields as ISO 8601 text, since dBase has no date-time field other programs read (said in a notice). Export notices count shortened text, non-null values left empty (including incompatible types), and numbers rounded to the field's precision, and name the affected source fields; features without geometry, and lines of a single point, are left out (and reported). An export in longitude/latitude leaves out vertices with a missing (NaN) or infinite coordinate; an export in a projected system refuses any coordinate that doesn't project to a finite one, naming its feature. In either, line parts and polygon rings too short to draw (fewer than 2 or 3 drawable vertices, whether written so or left so) and the holes of a polygon whose outer ring goes are left out. The export counts the vertices, and the parts or rings, in the features it writes; a feature with nothing drawable left is counted with the features left out. | | FlatGeobuf | Data › Export to `.fgb` | In the layer's coordinate system (see below), with a spatial index; features without geometry are left out (and reported). Desktop only: the writer needs a temporary file, which the browser doesn't have. | | KML, KMZ, GPX | Data › Export to `.kml`, `.kmz` or `.gpx` | Placemarks named by a `name` field, if there is one; features without geometry are left out. | -| GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; writes a file on desktop when it is given. | +| GeoJSON, KML, GPX, CSV, CSV with WKT | Analyze › Workflows › Data Interop (`data.convert`; `"format": "geojson"`, `"kml"`, `"gpx"`, `"csv"` or `"wkt"`) | Returns text when `"path"` is omitted; a streamed layer requires a path to avoid returning only its loaded view. Desktop conversion asks for a file first, then reads the whole source in the background with progress and Cancel. GeoJSON and CSV read a chunk at a time where supported; KML, GPX and WKT need the whole layer within the memory limit. | | GeoTIFF / Cloud-optimised GeoTIFF | Data › Export Raster (`share.exportRaster`) | Untiled GeoTIFF by default; `"cog": true` writes 512-pixel tiles, Deflate compression and overviews. Writes the loaded raster's resolution (an overview when that is what was opened), the selected band or its colour bands. A WMS, WMTS or XYZ tile layer (`"tiles"`) is written as its tiles at one zoom, RGBA in Web Mercator (see [Sharing](sharing.md#exporting)). The web app's Export Raster downloads an ordinary GeoTIFF. | | PNG | Publish › Export Map (`share.exportMap`) | Configurable DPI, with labels, scale bar and tile credits. | | GeoTIFF of the map view | Publish › Export Map to `.tif` (`share.exportMap`) | The view as an RGB GeoTIFF in the map's projection, with labels but no scale bar; tile credits in the Copyright tag; `"cog": true` for a cloud-optimised one. Flat, untilted maps of the Earth only (see [Exporting](sharing.md#exporting)). |