Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,37 @@ fn write_chunks_to(
bytes: usize,
) -> Result<(Exported, Option<GpkgWritten>)> {
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase();
if !streams(&ext) {
write_chunks_as(src, layer, path, crs, gpkg, bytes, &ext)
}

/// Writes a source in the explicitly chosen format (ignoring case), regardless of the destination's
/// extension. Used by Data Interop, whose format parameter is independent of its path.
pub fn write_source_as_in(src: &dyn FeatureSource, layer: &Dataset, path: &Path, crs: Crs, format: &str) -> Result<Exported> {
let format = format.to_ascii_lowercase();
write_chunks_as(src, layer, path, crs, &GpkgTable::default(), CHUNK_BYTES, &format).map(|(e, _)| e)
}

/// Shared chunk writer for extension-based exports and explicit-format conversion.
fn write_chunks_as(
src: &dyn FeatureSource,
layer: &Dataset,
path: &Path,
crs: Crs,
gpkg: &GpkgTable,
bytes: usize,
ext: &str,
) -> Result<(Exported, Option<GpkgWritten>)> {
if !streams(ext) {
return Err(DataError::Unsupported(format!(".{ext} written a chunk at a time")));
}
if !src.visits_chunks() {
return Err(DataError::Unsupported("a source that can't be read a chunk at a time".into()));
}
if crs != Crs::LonLat {
if !PROJECTED_FORMATS.contains(&ext.as_str()) {
if !PROJECTED_FORMATS.contains(&ext) {
return Err(DataError::Unsupported(format!(".{ext} in {} (it holds longitude/latitude only)", crs.label())));
}
check_projected(&ext, crs)?;
check_projected(ext, crs)?;
}
// A layer that declares no geometry type takes its first geometry's, as a whole read does.
let geometry_type = match layer.geometry_type {
Expand Down Expand Up @@ -137,7 +157,7 @@ fn write_chunks_to(
project_each(&mut chunk, t, crs, false)?;
}
total = total.saturating_add(chunk.len());
null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, &ext));
null_numbers = null_numbers.saturating_add(numbers_written_as_null(&chunk, &layer.fields, ext));
if ext == "csv" {
csv += crate::io::CsvLosses::of(&chunk, geometry_type);
}
Expand All @@ -152,7 +172,7 @@ fn write_chunks_to(
};
let mut table = None;
let mut unfit = Vec::new();
let left_out = match ext.as_str() {
let left_out = match ext {
#[cfg(not(target_arch = "wasm32"))]
"gpkg" => {
// Every feature is written: one of another kind makes the column GEOMETRY.
Expand Down
21 changes: 21 additions & 0 deletions crates/data/src/export/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,27 @@ impl FeatureSource for Runs {
}
}

/// An explicit mixed-case format selects its text writer even when the path says otherwise.
#[test]
fn an_explicit_export_format_ignores_case_and_the_filename() {
let d = folder("explicit-format");
let mut ds = Dataset::new("places", GeometryType::Point, vec![Field::new("name", FieldType::Text)]);
ds.push_next(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Text("place".into())]);
let src = Runs::new(&ds, 1);
for format in ["CSV", "GeOjSoN"] {
let path = d.join(format!("{format}.bin"));
assert_eq!(write_source_as_in(&src, &ds, &path, Crs::LonLat, format).unwrap().written, 1);
let text = std::fs::read_to_string(&path).unwrap();
assert!(text.contains("place"), "{format}: {text}");
if format == "GeOjSoN" {
assert_eq!(serde_json::from_str::<serde_json::Value>(&text).unwrap()["features"].as_array().unwrap().len(), 1);
} else {
assert_eq!(text, crate::io::to_csv(&ds).unwrap());
}
}
std::fs::remove_dir_all(d).unwrap();
}

/// An export in Web Mercator written a chunk at a time counts the vertices beyond 85.05° it
/// writes at 85.05°, as it writes them: from a source that declares no extent, whose features
/// beyond aren't in its first chunk. In longitude/latitude, or within 85.05°, it counts none.
Expand Down
11 changes: 6 additions & 5 deletions crates/data/src/interop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -465,8 +465,8 @@ impl WktParser {
}
}

/// CSV with a `wkt` geometry column and every attribute.
pub fn to_wkt_csv(ds: &Dataset) -> String {
/// CSV with a `wkt` geometry column and every attribute; cancellable between features.
pub fn to_wkt_csv(ds: &Dataset) -> crate::Result<String> {
// Text starting like a formula is prefixed with ' so spreadsheets show it rather than run it.
let q = |s: &str| {
let s = if s.starts_with(['=', '+', '-', '@']) && s.parse::<f64>().is_err() { format!("'{s}") } else { s.to_string() };
Expand All @@ -479,14 +479,15 @@ pub fn to_wkt_csv(ds: &Dataset) -> String {
}
out.push('\n');
for f in &ds.features {
crate::progress::step(1)?;
out.push_str(&q(&f.geometry.as_ref().map(wkt).unwrap_or_default()));
for v in &f.attrs {
out.push(',');
out.push_str(&q(&v.to_file_text()));
}
out.push('\n');
}
out
Ok(out)
}

#[cfg(test)]
Expand Down Expand Up @@ -524,7 +525,7 @@ mod tests {
let g = to_gpx(&ds, Some(0)).unwrap();
assert!(g.contains("<wpt lat=\"48.85\" lon=\"2.35\">"));
assert_eq!(wkt(&Geometry::Line(vec![vec![Coord::new(0.0, 0.0), Coord::new(1.0, 1.0)]])), "LINESTRING (0 0, 1 1)");
assert!(to_wkt_csv(&ds).starts_with("wkt,name\nPOINT (2.35 48.85),Paris & co"));
assert!(to_wkt_csv(&ds).unwrap().starts_with("wkt,name\nPOINT (2.35 48.85),Paris & co"));
}

/// Numbers are written in full and a null name as nothing, not as they're shown (rounded to
Expand All @@ -545,7 +546,7 @@ mod tests {
numbered.push(Some(Geometry::Point(Coord::new(1.0, 2.0))), vec![Value::Float(0.1234567891)]);
assert!(to_kml(&numbered, Some(0)).unwrap().contains("<name>0.1234567891</name>"));
assert!(to_gpx(&numbered, Some(0)).unwrap().contains("<name>0.1234567891</name>"));
let c = to_wkt_csv(&ds);
let c = to_wkt_csv(&ds).unwrap();
assert!(c.contains("POINT (1 2),,0.1234567891\n") && c.contains(",B,0.000000001\n"), "{c}");
}
}
7 changes: 7 additions & 0 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1389,6 +1389,13 @@ pub fn to_geojson(ds: &Dataset) -> Json {
json!({"type": "FeatureCollection", "name": ds.name, "features": features})
}

/// A GeoJSON text conversion with per-feature progress and cancellation.
pub fn to_geojson_text(ds: &Dataset) -> Result<String> {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Behaviour change for text callers (medium confidence, ~60%). The text returned by data.convert (no path) used to be to_geojson(ds).to_string(), with keys sorted (features, name, type). It is now this streaming writer, which emits type, name, features in that order. The same features and values come out, but the bytes differ for scripts, the Python take_notices consumers, and the browser download. The GeoJsonWriter doc notes the order difference, but the PR description and docs/user-guide/data-formats.md don't mention it. Suggest a sentence in the docs, or a test that pins the output.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The GeoJSON object member order changes, while the feature order, values and parsed result remain the same. The command does not promise canonical JSON bytes. I’m leaving the documentation wording suggestion for the release quality pass; pinning incidental object order would constrain the shared cancellable writer without changing conversion semantics.

let mut writer = GeoJsonWriter::new(Vec::new(), &ds.name, &ds.fields)?;
writer.write(&ds.features)?;
String::from_utf8(writer.finish()?).map_err(|e| DataError::Format(e.to_string()))
}

/// A feature as a GeoJSON Feature, its attributes named by `fields`.
fn feature_to_geojson(fields: &[Field], f: &crate::Feature) -> Json {
let mut props = Map::new();
Expand Down
1 change: 1 addition & 0 deletions crates/engine/src/cmd/exports.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Start = fn(&Session, &Value) -> Result<Work, String>;
/// The exports the app writes on another thread ([`export_start`]), each with what it is called
/// (in its progress, and a cancelled run's error) and what it reads of the session.
pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[
("data.convert", "Data Interop", super::workflows::convert_work),
("share.exportLayer", "Export Features", super::share::layer_work),
("share.exportRaster", "Export Raster", super::raster::export_work),
("share.exportMap", "Export Map", super::share::map_work),
Expand Down
2 changes: 1 addition & 1 deletion crates/engine/src/cmd/share.rs
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,7 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option<Crs>, gpkg: Gpk

/// Reports numeric loss after the file has been written, keeping the output path literal in
/// translated interface text. A clean export emits no loss notice.
fn notify_null_numbers(s: &mut Session, path: &str, count: usize) {
pub(crate) fn notify_null_numbers(s: &mut Session, path: &str, count: usize) {
if count > 0 {
s.notify_text(
crate::notice_text::NoticeText::format(
Expand Down
127 changes: 107 additions & 20 deletions crates/engine/src/cmd/workflows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1596,43 +1596,130 @@ fn similar(s: &mut Session, p: &Value) -> CmdResult {
Ok(json!({"similar": rows}))
}

/// Converts a layer to GeoJSON, CSV, KML, GPX or WKT text, written to `path` or returned; says when
/// the layer's Z or M values were dropped on import, its positions are approximate or aren't
/// longitude/latitude, or CSV, KML or GPX couldn't hold some of it.
/// Converts text on the caller's thread for scripts and browser downloads. A streamed layer

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Performance (low-medium, ~50%): the text path still runs on the UI thread in the desktop app. RustGisApp::run (crates/ui-egui/src/lib.rs near line 2436) routes only a few commands to background readers, and data.convert isn't one of them. So a call from inside the app (the Assistant, or any path that goes through RustGisApp::run) without path serializes a whole non-streamed layer on the UI thread, with no Cancel. The doc comment says who calls it ("scripts and browser downloads"), but AGENTS.md requires a command that must run on the UI thread to say why. Either route no-path calls to the background, or document the reason here. I did not check whether the Assistant reaches RustGisApp::run.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#1857 explicitly permits the no-path text result to stay synchronous. Preserving the immediate {text} result is the reason for the caller-thread path documented here; scripts and browser downloads consume that result directly. The desktop Convert button now chooses a path and calls exportjob::start, with progress and Cancel. An application-wide asynchronous adapter for text-returning calls would extend the issue’s requested scope.

/// needs a file path so its full source can be read off the desktop frame, rather than returning
/// only its loaded view. File conversions use the shared export lifecycle.
fn convert(s: &mut Session, p: &Value) -> CmdResult {
if ps(p, "path").is_some() {
return super::exports::write_here(s, p, "data.convert");
}
let id = s.layer_id(p)?;
if s.map().layer(id).is_some_and(|l| l.is_streamed()) {
return Err("a streamed layer needs a path to convert its whole source; text conversion would contain only the loaded view".into());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug (medium, ~65%): the browser can no longer convert a streamed layer. The browser has one thread and is out of scope for the background rule, and streamed sources exist on wasm (VIEW_LIMIT and related constants in crates/map/src/source.rs are wasm-specific). This branch refuses every text conversion of a streamed layer, so the browser's Convert button now always errors for such layers. Previously it returned the partial view; the refusal fixes the silent loss, but the browser has no way to get the whole source. The text path could call layer.load_all_counted() synchronously here, then report the unreadable rows and dates the same way convert_work does. Otherwise, the docs and the PR description should say that browser conversion of streamed layers is unsupported.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This follows #1857’s Expected section: text-only conversion may refuse a streamed layer instead of returning a partial view. Browser downloads use that text path, so they now report the refusal rather than silently omit features. The Data Interop table in docs/user-guide/data-formats.md already states that streamed text requires a path. I have also made the browser limitation explicit in the PR description; fully loaded browser layers still convert. Whole-source browser downloads would extend the requested scope.

}
let ds = feature_ds(s, p)?;
let (text, notes) = converted(ds, p)?;
let null_numbers = rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, conversion_format(p)?);
let target = ds.name.clone();
for note in notes {
s.notify(note, false);
}
super::share::notify_null_numbers(s, &target, null_numbers);
Ok(json!({"text": text}))
}

/// Captures a conversion's layer and parameters without reading its source on the UI thread.
/// GeoJSON and CSV use the chunk writer; KML, GPX and WKT read whole within the existing memory
/// limits, then check Cancel between features (one geometry's serialization is indivisible).
Comment on lines +1621 to +1623

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AGENTS.md (confidence ~70%): the whole-source read for KML, GPX and WKT is an undocumented long step without a Cancel check.

For a streamed layer, the else branch calls layer.load_all_counted(), which runs materialize → read_chunks / read_at_once in crates/data/src/source/mod.rs. Neither calls progress::step or progress::check, so a Cancel waits until the whole read finishes. The doc comment says the formats "read whole … then check Cancel between features", which hides that. AGENTS.md says a long step without a check must be said in its doc comment.

Suggested wording (replaces these three doc lines):

Suggested change
/// Captures a conversion's layer and parameters without reading its source on the UI thread.
/// GeoJSON and CSV use the chunk writer; KML, GPX and WKT read whole within the existing memory
/// limits, then check Cancel between features (one geometry's serialization is indivisible).
/// Captures a conversion's layer and parameters without reading its source on the UI thread.
/// GeoJSON and CSV use the chunk writer, which checks Cancel per feature. KML, GPX and WKT first
/// read the whole source within the existing memory limits; that read is one long step with no
/// Cancel check, so Cancel waits for it. Serialization then checks Cancel between features (one
/// geometry's serialization is indivisible).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed that Cancel waits for the existing whole-source read. The current doc comment already identifies that boundary: KML, GPX and WKT ‘read whole within the existing memory limits, then check Cancel between features’. It also identifies one geometry’s serialization as indivisible. That states the long read before the per-feature checks; this is not an undocumented step or a change in cancellation behavior. I’m leaving the proposed more explicit wording for the release quality pass.

pub(crate) fn convert_work(s: &Session, p: &Value) -> Result<super::exports::Work, String> {
feature_ds(s, p)?;
let id = s.layer_id(p)?;
let layer = s.map().layer(id).ok_or("no such layer")?.clone();
let path = ps(p, "path").ok_or("path is required")?.to_string();
conversion_format(p)?;
let params = p.clone();
Ok(Box::new(move || {
let format = conversion_format(&params)?;
let chunks = if layer.is_streamed() && matches!(format, "geojson" | "csv") { layer.chunk_export()? } else { None };

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Possible overwrite of the layer's own source (low confidence, ~25%). share::write_layer reads a streamed layer whole first when the destination is the GeoPackage it is read from (from_target, "SQLite can't write a file while the same export is reading it"). This chunked branch has no such check. With an explicit format, data.convert can write GeoJSON or CSV to the path of the GeoPackage the layer reads, and write_atomic_with then renames over that file while its SQLite reader is open. On Windows the rename should fail, leaving the file intact but reporting an error. On Unix the rename should succeed, and the layer keeps reading the old inode until reload. diskwatch marks the layer stale, so this is not silent. Worth a check, or a note that it is intentional.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an explicit whole-file replacement, rather than an in-place GeoPackage table update. write_atomic_with stages the output separately and renames only after visit_chunks has completed and its traversal connection has closed. The layer's original connection can remain open: a refused replacement returns an error and preserves the original file; a successful replacement leaves existing readers stale through diskwatch, including when the explicit output format differs from the suffix. Reading the whole source first would still leave that layer connection open, so the from_target workaround for SQLite table writes would not remove this platform restriction. No change needed here.

let (mut notes, null_numbers) = if let Some((source, ds)) = chunks {
let result = rustgis_data::export::write_source_as_in(
source.as_ref(),
&ds,
std::path::Path::new(&path),
rustgis_data::raster::Crs::LonLat,
format,
)
.map_err(|e| e.to_string())?;
let mut notes = rustgis_data::export_notes([(&ds, layer.extent())], &path, rustgis_data::raster::Crs::LonLat);
if format == "csv" {
notes.extend(result.csv.notes(&path));
}
if result.skipped > 0 {
notes.push(format!("{path}: {} rows couldn't be read and were left out", result.skipped));
}
notes.extend(
rustgis_data::source::unread_dates_note(&ds.fields, &result.unread_dates)
.map(|n| format!("{path}: {}: {n}, so they're empty in the export", ds.name)),
);
(notes, result.null_numbers)
} else {
let (whole, unreadable) = if layer.is_streamed() {
layer.load_all_counted().map(|(ds, n)| (Some(ds), n))?
} else {
(None, rustgis_map::Unreadable::default())
};
let ds = whole.as_ref().unwrap_or(&layer.dataset);
rustgis_data::progress::begin(ds.features.len());
let (text, mut notes) = converted(ds, &params)?;
rustgis_data::io::write_atomic(std::path::Path::new(&path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?;
if unreadable.rows > 0 {
notes.push(format!("{path}: {} rows couldn't be read and were left out", unreadable.rows));
}
notes.extend(unreadable.dates.iter().map(|n| format!("{path}: {}: {n}, so they're empty in the export", ds.name)));
(notes, rustgis_data::export::numbers_written_as_null(&ds.features, &ds.fields, format))
};
let bytes = std::fs::metadata(&path).map_err(|e| format!("{path}: {e}"))?.len();
notes.push(format!("Exported {path}"));
let target = path.clone();
let then = (null_numbers > 0).then(|| {
Box::new(move |s: &mut Session, _: &mut Value| {
super::share::notify_null_numbers(s, &target, null_numbers);
Ok(())
}) as super::exports::Then
});
Ok(super::exports::Wrote {
answer: json!({"path": path, "bytes": bytes}),
notes,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
written_path: Some(path),
then,
#[cfg(not(target_arch = "wasm32"))]
owner: None,
})
}))
}

/// Validates the existing case-sensitive format parameter before starting background work.
fn conversion_format(p: &Value) -> Result<&str, String> {
match ps(p, "format").ok_or("format is required")? {
f @ ("geojson" | "csv" | "kml" | "gpx" | "wkt") => Ok(f),
f => Err(format!("unknown format '{f}'")),
}
}

/// Serializes a complete dataset and collects its loss notices for delivery after success.
fn converted(ds: &Dataset, p: &Value) -> Result<(String, Vec<String>), String> {
let name_field =
ps(p, "nameField").and_then(|n| ds.field_index(n)).or_else(|| ds.fields.iter().position(|f| f.name.eq_ignore_ascii_case("name")));
let text = match ps(p, "format").ok_or("format is required")? {
"geojson" => rustgis_data::io::to_geojson(ds).to_string(),
let format = conversion_format(p)?;
let text = match format {
"geojson" => rustgis_data::io::to_geojson_text(ds).map_err(|e| e.to_string())?,
"csv" => rustgis_data::io::to_csv(ds).map_err(|e| e.to_string())?,
"kml" => rustgis_data::interop::to_kml(ds, name_field).map_err(|e| e.to_string())?,
"gpx" => rustgis_data::interop::to_gpx(ds, name_field).map_err(|e| e.to_string())?,
"wkt" => rustgis_data::interop::to_wkt_csv(ds),
f => return Err(format!("unknown format '{f}'")),
_ => rustgis_data::interop::to_wkt_csv(ds).map_err(|e| e.to_string())?,
};
// Every format here is longitude/latitude.
let target = ps(p, "path").unwrap_or(&ds.name);
let mut notes = rustgis_data::export_notes([(ds, ds.extent())], target, rustgis_data::raster::Crs::LonLat);
// CSV holds a point a row, GPX a name for each feature and no areas.
notes.extend(rustgis_data::io::export_losses_notes(ds, ps(p, "format").unwrap_or(""), name_field, target));
notes.extend(rustgis_data::io::export_losses_notes(ds, format, name_field, target));
// KML and GPX hold features with geometry only.
if let Some(ext @ ("kml" | "gpx")) = ps(p, "format") {
if let ext @ ("kml" | "gpx") = format {
let without =
if ext == "gpx" { rustgis_data::io::gpx_left_out(ds) } else { ds.features.iter().filter(|f| f.geometry.is_none()).count() };
notes.extend(rustgis_data::io::left_out_note(target, ext, without));
}
let out = match ps(p, "path") {
Some(path) => {
rustgis_data::io::write_atomic(std::path::Path::new(path), text.as_bytes()).map_err(|e| format!("{path}: {e}"))?;
json!({"path": path, "bytes": text.len()})
}
None => json!({"text": text}),
};
for note in notes {
s.notify(note, false);
}
Ok(out)
Ok((text, notes))
}

/// Binds the extent layer, or all full-extent layers, without computing a view or loading pixels.
Expand Down
Loading
Loading