diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index ed45a24e3d..c8a08d1047 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -42,6 +42,7 @@ pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ ("share.exportRaster", "Export Raster", super::raster::export_work), ("share.exportMap", "Export Map", super::share::map_work), ("layout.export", "Export Layout", super::layout::export_work), + ("report.export", "Export Report", super::report::export_work), ]; /// `command` and what it is called, when the app writes its file on another thread (see @@ -125,14 +126,7 @@ pub(crate) fn drawn( let written_path = written.is_ok().then(|| path.clone()); let answer = written.as_ref().map_or(Value::Null, Value::clone); let then: Then = Box::new(move |s: &mut Session, _: &mut Value| { - use rustgis_data::diagnostics::Severity; - for (severity, text) in notices { - match severity { - Severity::Error => s.notify(text, true), - Severity::Warning => s.warn(text), - Severity::Info => s.notify(text, false), - } - } + say(s, notices); let mut missing = Vec::new(); for (instance, name, error, copied) in failed { let live = s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).find(|l| l.instance_id() == instance); @@ -167,6 +161,35 @@ pub(crate) fn drawn( }) } +/// What drawing an export on `copy` (a copy of the session) said, each notice with its severity, +/// to say again where the export lands; past the notices a session keeps, how many earlier ones +/// there were. `path` names the file in that count. +pub(crate) fn copy_notices(copy: &mut Session, path: &str) -> Vec<(rustgis_data::diagnostics::Severity, String)> { + let kept = std::mem::take(&mut copy.notices); + let earlier = copy.notices_given().saturating_sub(kept.len() as u64); + let mut said = Vec::new(); + if earlier > 0 { + said.push(( + rustgis_data::diagnostics::Severity::Warning, + format!("{path}: {earlier} earlier notices from drawing it aren't shown"), + )); + } + said.extend(kept.into_iter().map(|n| (n.severity, n.text))); + said +} + +/// Says `notices` (what an export said on a copy of the session) in `s`, each as it was said. +pub(crate) fn say(s: &mut Session, notices: Vec<(rustgis_data::diagnostics::Severity, String)>) { + use rustgis_data::diagnostics::Severity; + for (severity, text) in notices { + match severity { + Severity::Error => s.notify(text, true), + Severity::Warning => s.warn(text), + Severity::Info => s.notify(text, false), + } + } +} + #[cfg(not(target_arch = "wasm32"))] /// An export ([`IN_BACKGROUND`]) started to write its file on another thread ([`ExportRun::run`]) /// while the session goes on: its inputs are taken now (cheaply: the features and pixels are diff --git a/crates/engine/src/cmd/labels.rs b/crates/engine/src/cmd/labels.rs index f2298aabb0..c6a810c5c4 100644 --- a/crates/engine/src/cmd/labels.rs +++ b/crates/engine/src/cmd/labels.rs @@ -642,15 +642,46 @@ fn distance_range(s: &mut Session, p: &Value) -> CmdResult { }) } +/// Whether layer `l` has text to convert into annotations at `scale`: a class's label field, or a +/// text expression. Decided on its fields, so before a layer read in place is read whole. +pub(crate) fn has_label_text(l: &rustgis_map::Layer, scale: f64) -> bool { + l.labels.in_order().any(|c| { + l.dataset.field_index(&c.field).is_some() + || rustgis_map::label_by::LabelExprs::new(&c.by, &l.dataset, Some(scale)).is_some_and(|e| e.has_text()) + }) +} + /// Converts a layer's labels into an annotation layer: one point per label, at the label point, /// carrying the label text and drawn with the same text symbol. Each class labelling a feature /// gives it a point (a hidden class none); with classes besides Default, a CLASS field keeps /// which, and the annotation has the same classes, each picking its points by it. The layer's -/// own labels turn off. +/// own labels turn off. A layer read in place converts all of its source's features, not just +/// those of the view, and says the rows it couldn't read; it stays read in place. The app reads +/// them ahead on a thread of its own, as for Load All Features, whatever asks for the conversion +/// there; a script or the command line reads them where the command runs. The conversion itself +/// runs on the app's thread: it adds the annotation layer and turns the layer's labels off in one +/// undoable step, a pass over features already in memory. fn convert(s: &mut Session, p: &Value) -> CmdResult { let id = s.layer_id(p)?; let scale = s.map().scale(); - let l = s.map().layer(id).ok_or("no such layer")?; + // Refused before a layer read in place is read whole for nothing. + if !has_label_text(s.map().layer(id).ok_or("no such layer")?, scale) { + return Err("the layer has no label field".into()); + } + // Its features, all of them: a copy of a layer read in place holds those of its whole source. + let whole = if s.map().layer(id).ok_or("no such layer")?.is_streamed() { + let (ds, unread) = super::whole::take_or_read(s, id)?; + let mut copy = s.map().layer(id).ok_or("no such layer")?.clone(); + copy.dataset = ds; + copy.source = None; + Some((copy, unread)) + } else { + None + }; + let l = match &whole { + Some((copy, _)) => copy, + None => s.map().layer(id).ok_or("no such layer")?, + }; let fields: Vec> = l.labels.in_order().map(|c| l.dataset.field_index(&c.field)).collect(); // Its classes' expressions, as the map draws them now (at its scale). let exprs: Vec> = @@ -793,5 +824,8 @@ fn convert(s: &mut Session, p: &Value) -> CmdResult { if let Some(note) = cut_note { s.notify(note, false); } + if let Some((_, unread)) = &whole { + unread.say(s); + } Ok(json!({"layer": new_id, "annotations": n})) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 9703fe0769..746debdea0 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2985,23 +2985,12 @@ fn define_projection(s: &mut Session, p: &Value) -> CmdResult { /// Loads a streamed layer's features into memory, so it can be edited like any other layer. fn load_all(s: &mut Session, p: &Value) -> CmdResult { let id = s.layer_id(p)?; - let key = load_all_key(s.map().layer(id).ok_or("no such layer")?); - let ahead = super::reads::take::(s, &key); + let (mut ds, unread) = super::whole::take_or_read(s, id)?; let l = s.map().layer(id).ok_or("no such layer")?; - let (mut ds, unread) = match ahead { - // Read (or failed) from the source the layer still reads, listing the fields it lists - // now: one opened afresh since, or with other fields, is read again. - Some((copied, fields, read)) if l.source.as_ref().is_some_and(|src| src.same_opening(&copied)) && fields == l.dataset.fields => { - read? - } - _ => l.load_all_counted()?, - }; // Named as the layer is now: it may have been renamed while the read ran. ds.name = l.dataset.name.clone(); - let (n, name) = (ds.features.len(), l.name.clone()); - if unread > 0 { - s.warn(super::whole::unread_notice(&name, unread)); - } + let n = ds.features.len(); + unread.say(s); let l = s.map_mut().layer_mut(id).ok_or("no such layer")?; l.dataset = ds; l.dataset.windowed = false; @@ -3011,25 +3000,10 @@ fn load_all(s: &mut Session, p: &Value) -> CmdResult { Ok(json!({"layer": id, "features": n})) } -/// Load All's read made before it runs: the source it read, the fields the layer listed then, and -/// what it read with the number of rows that couldn't be read. -type LoadedAll = (rustgis_map::source::SourceLayer, Vec, Result<(rustgis_data::Dataset, usize), String>); - -/// The key Load All's read of `l` is taken by (see [`super::reads`]). -fn load_all_key(l: &rustgis_map::Layer) -> String { - format!("loadAll:{}", l.instance_id()) -} - /// Load All's read of the streamed layer `p` names, made before it runs ([`super::reads`]), on a /// copy of the layer. pub(crate) fn load_all_ahead(s: &Session, p: &Value) -> Option { - let l = s.map().layer(s.layer_id(p).ok()?)?; - let mut copy = l.clone(); - let source = copy.source.as_mut()?; - // Its own read: the live layer keeps the one under way. - source.detach_read(); - let (copied, fields) = (source.clone(), l.dataset.fields.clone()); - Some(super::reads::ahead("Load All Features", load_all_key(l), move || (copied, fields, copy.load_all_counted()))) + super::whole::ahead(s, p, "Load All Features") } /// Adds a layer file: a layer with its data, symbology and labels. diff --git a/crates/engine/src/cmd/layout.rs b/crates/engine/src/cmd/layout.rs index 54227953c2..e6c9b80541 100644 --- a/crates/engine/src/cmd/layout.rs +++ b/crates/engine/src/cmd/layout.rs @@ -3769,17 +3769,8 @@ pub(crate) fn export_work(s: &Session, p: &Value) -> Result = Vec::new(); - if earlier > 0 { - said.push(( - rustgis_data::diagnostics::Severity::Warning, - format!("{path}: {earlier} earlier notices from drawing it aren't shown"), - )); - } - said.extend(kept.into_iter().map(|n| (n.severity, n.text))); + // here. + let said = super::exports::copy_notices(&mut copy, &path); super::exports::drawn(©.project.maps, said, written, path) })) } diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index 91d06ff15e..935028a3d1 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -1,7 +1,8 @@ //! Reads the desktop app makes on a thread of its own before a command adds what they read //! (#1291 step 3): Add Data's whole-file reads (a raster, a raster GDAL converts, a GeoJSON, -//! delimited text or spreadsheet read whole, a CityJSON or CityGML model's layers), Reload Data, Load All Features, native models or scripts and -//! the Error Inspector's topology check (#1291). [`read_ahead`] +//! delimited text or spreadsheet read whole, a CityJSON or CityGML model's layers), Reload Data, Load All +//! Features, Convert To Annotation's read of a layer read in place, native models or scripts and the +//! Error Inspector's topology check (#1291). [`read_ahead`] //! takes what the read needs of the session, [`ReadAhead::run`] reads on the thread, and [`read_land`] //! runs the admitted command under its initiating owner: the command takes //! it instead of reading again, so what it adds, says, logs and puts in Undo is the same as when @@ -98,6 +99,14 @@ pub(crate) fn prepare(s: &Session, op: Authorized, owner: Continuatio let layer = s.map().layer(s.layer_id(params)?).ok_or("no such layer")?; ("Load All Features", super::layer::load_all_ahead(s, params), layer.source.clone()) } + "labels.convert" => { + let layer = s.map().layer(s.layer_id(params)?).ok_or("no such layer")?; + // Nothing to convert: the command says so at once, with no read. + let work = super::labels::has_label_text(layer, s.map().scale()) + .then(|| super::whole::ahead(s, params, "Convert To Annotation")) + .flatten(); + ("Convert To Annotation", work, layer.source.clone()) + } "layer.reloadData" => { #[cfg(not(target_arch = "wasm32"))] let work = super::repair::reload_ahead(s, params); @@ -130,7 +139,7 @@ pub(crate) fn prepare(s: &Session, op: Authorized, owner: Continuatio (name, work, None) } "topology.validate" => ("Error Inspector", super::design::topology_ahead(s, params), None), - _ => return Err("only Add Data, Reload Data, Load All, tile archives, models, scripts and topology prepare background work".into()), + _ => return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()), }; let name = work.as_ref().map_or(name, |work| work.name); Ok(ReadAhead { name, op, owner, source, work }) diff --git a/crates/engine/src/cmd/report.rs b/crates/engine/src/cmd/report.rs index 19ba4f8084..0c4834d09a 100644 --- a/crates/engine/src/cmd/report.rs +++ b/crates/engine/src/cmd/report.rs @@ -208,19 +208,7 @@ pub fn commands() -> Vec { .when(need_report) .params(r#"{"report"?, "category": field|null (no chart), "value"?: field to sum (default: count records)}"#), // Output. - C::new("report.export", "Export Report", "Publish › Output", |s, p| { - let path = ps(p, "path").ok_or("path is required")?.to_string(); - let i = index(s, p)?; - let dpi = f(p, "dpi").unwrap_or(150.0).clamp(72.0, 600.0); - let lower = path.to_ascii_lowercase(); - let pdf = lower.ends_with(".pdf"); - if !pdf && !lower.ends_with(".png") { - return Err("export to a .pdf or .png file".into()); - } - let (bytes, pages) = report_bytes(s, i, dpi, pdf)?; - rustgis_data::io::write_atomic(Path::new(&path), &bytes).map_err(|e| format!("{path}: {e}"))?; - Ok(json!({"path": path, "pages": pages, "bytes": bytes.len()})) - }).inputs(export_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) + C::new("report.export", "Export Report", "Publish › Output", |s, p| super::exports::write_here(s, p, "report.export")).inputs(export_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) .view() .when(need_report) .params(r#"{"report"?, "path": "*.pdf|*.png" (PNG: the first page), "dpi"?: 72..600}"#), @@ -774,6 +762,19 @@ fn blocks(s: &Session, i: usize, out: &mut Vec, seen: &mut Vec, de if truncated { about.push_str(&format!(" — the first {MAX_RECORDS} are listed")); } + // Only a preview draws a layer read in place (an export reads it whole first), here or + // under a record through a relationship. + let related_in_view = r.sections.iter().any(|sec| { + let Section::Subreport { relationship, .. } = sec else { return false }; + relationships(s) + .find(|x| x.name.eq_ignore_ascii_case(relationship)) + .and_then(|rel| opposite(rel, &layer.name)) + .and_then(|other| find_layer(s, other)) + .is_some_and(Layer::is_streamed) + }); + if layer.is_streamed() || related_in_view { + about.push_str(" — in this preview, only the records in the map's view; the export lists them all"); + } out.push(Block::Text(about)); let fi: Vec> = r.fields.iter().map(|n| ds.field_index(n)).collect(); let columns = Block::Columns(r.fields.clone()); @@ -947,15 +948,139 @@ fn blocks(s: &Session, i: usize, out: &mut Vec, seen: &mut Vec, de Ok(()) } -/// The report's pages, laid out. +/// The report's pages, laid out: for the preview, from the features the layers hold now (a layer +/// read in place holds those of the view, and the report says so); an export reads them whole +/// ([`report_bytes`]). pub fn pages(s: &Session, i: usize) -> Result<(Doc, Vec), String> { let doc = build(s, i)?; let pages = rustgis_render::report::paginate(&doc); Ok((doc, pages)) } -/// The report as a PDF (every page) or a PNG (the first page), and its page count. +/// The report as a PDF (every page) or a PNG (the first page), and its page count: every record of +/// a layer read in place, read whole on a copy of the project ([`whole_copy`]); what reading and +/// drawing it said is said in `s`. The browser's export, where the whole read runs here; the +/// app's runs on a thread of its own ([`export_work`]). pub fn report_bytes(s: &mut Session, i: usize, dpi: f64, pdf: bool) -> Result<(Vec, usize), String> { + let name = s.project.reports.get(i).map(|r| r.name.clone()).unwrap_or_default(); + let (bytes, mut copy) = whole_bytes(detached(&s.project), s.path.clone(), i, dpi, pdf); + super::exports::say(s, super::exports::copy_notices(&mut copy, &name)); + bytes +} + +/// What Export Report reads of the session (a copy of the project, its layers' reads left with +/// the live ones), and the work that reads the report's layers whole, draws it and writes it. +pub(crate) fn export_work(s: &Session, p: &Value) -> Result { + let path = ps(p, "path").ok_or("path is required")?.to_string(); + let i = index(s, p)?; + let dpi = f(p, "dpi").unwrap_or(150.0).clamp(72.0, 600.0); + let lower = path.to_ascii_lowercase(); + let pdf = lower.ends_with(".pdf"); + if !pdf && !lower.ends_with(".png") { + return Err("export to a .pdf or .png file".into()); + } + let (project, folder) = (detached(&s.project), s.path.clone()); + Ok(Box::new(move || { + let (bytes, mut copy) = whole_bytes(project, folder, i, dpi, pdf); + let written = bytes.and_then(|(bytes, pages)| { + rustgis_data::progress::check().map_err(|e| e.to_string())?; + rustgis_data::io::write_atomic(Path::new(&path), &bytes).map_err(|e| format!("{path}: {e}"))?; + Ok(json!({"path": path, "pages": pages, "bytes": bytes.len()})) + }); + // The layers it lists are read whole; a layout drawn in it draws the others' views, and + // one whose source fails there is said, as Export Layout says it. + let said = super::exports::copy_notices(&mut copy, &path); + super::exports::drawn(©.project.maps, said, written, path) + })) +} + +/// `project`, its streamed layers' reads under way left with the live layers. +fn detached(project: &crate::project::Project) -> crate::project::Project { + let mut project = project.clone(); + for m in &mut project.maps { + m.detach_reads(); + } + project +} + +/// Report `i` drawn ([`drawn_bytes`]) from a copy of `project` saved at `path` (pictures are found +/// beside it), its layers read whole ([`whole_copy`]), with the copy, which holds what reading and +/// drawing it said. +fn whole_bytes( + project: crate::project::Project, + path: Option, + i: usize, + dpi: f64, + pdf: bool, +) -> (Result<(Vec, usize), String>, Session) { + let mut copy = Session::new(project); + copy.path = path; + let bytes = whole_copy(&mut copy, i).and_then(|()| drawn_bytes(&mut copy, i, dpi, pdf)); + (bytes, copy) +} + +/// Reads whole, in `copy` (a copy of the project to draw report `i` from), the layers read in +/// place that the report lists records of ([`reported_layers`]), so the report lists every +/// record of their sources and not only those of the view, their selections kept; the rows a +/// source couldn't read are said on the copy. Each read looks at Cancel as Load All Features' +/// does: between the chunks of a source read so, not during one read at once. +pub(crate) fn whole_copy(copy: &mut Session, i: usize) -> Result<(), String> { + for (m, id) in reported_layers(copy, i) { + // Cancelled after one layer's read: the next isn't started. + rustgis_data::progress::check().map_err(|e| e.to_string())?; + let Some(l) = copy.project.maps.get_mut(m).and_then(|map| map.layer_mut(id)) else { continue }; + let unread = super::whole::load_into(l)?; + unread.say(copy); + } + rustgis_data::progress::check().map_err(|e| e.to_string()) +} + +/// The layers report `i` lists records of, by map and id: its own, its subreports' related +/// layers, and those of the reports it appends, as far as [`blocks`] draws them (the same +/// reports, in the same order, to the same depth), so an export reads no layer it doesn't list. +pub(crate) fn reported_layers(input: &impl super::InputSource, i: usize) -> Vec<(usize, u32)> { + /// Report `i`'s layers at `depth`, then those of the reports it appends, as `blocks` walks + /// them (sharing its `seen`). + fn walk(input: &impl super::InputSource, i: usize, depth: usize, seen: &mut Vec, out: &mut Vec<(usize, u32)>) { + let context = input.input_context(); + let s = &context; + let place = |name: &str| { + let (map, l) = find_layer_map(input, name)?; + Some((s.project.maps.iter().position(|m| std::ptr::eq(m, map))?, l.id, l.name.clone())) + }; + let Some(r) = s.project.reports.get(i) else { return }; + // A report whose layer is gone draws nothing (`blocks` fails), nor the reports it appends. + let Some((m, id, name)) = place(&r.layer) else { return }; + out.push((m, id)); + for sec in &r.sections { + if let Section::Subreport { relationship, .. } = sec { + let other = relationships(input) + .find(|x| x.name.eq_ignore_ascii_case(relationship)) + .and_then(|rel| opposite(rel, &name).map(str::to_string)); + out.extend(other.and_then(|other| place(&other)).map(|(m, id, _)| (m, id))); + } + } + for sec in &r.sections { + if let Section::Related { report } = sec + && depth < MAX_DEPTH + && let Some(j) = s.project.reports.iter().position(|x| x.name.eq_ignore_ascii_case(report)) + && !seen.contains(&j) + { + seen.push(j); + walk(input, j, depth + 1, seen, out); + } + } + } + let mut out = Vec::new(); + walk(input, i, 0, &mut vec![i], &mut out); + out.sort_unstable(); + out.dedup(); + out +} + +/// The report as a PDF (every page) or a PNG (the first page), and its page count, drawn from +/// the features `s`'s layers hold. +fn drawn_bytes(s: &mut Session, i: usize, dpi: f64, pdf: bool) -> Result<(Vec, usize), String> { let (doc, pages) = pages(s, i)?; let revision = |s: &Session| { pages @@ -984,7 +1109,10 @@ fn report_pages_bytes( let mut out = Vec::new(); let mut held = 0usize; let (mut frames_pictured, mut pages_pictured) = (Vec::new(), Vec::new()); + // Counted, and stopped on Cancel, a page at a time. + rustgis_data::progress::begin(n); for k in 0..n { + rustgis_data::progress::step(1).map_err(|e| e.to_string())?; if !pdf { let (img, ..) = rustgis_render::report::draw_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; return Ok((img.encode_png().map_err(|e| e.to_string())?, pages.len())); diff --git a/crates/engine/src/cmd/whole.rs b/crates/engine/src/cmd/whole.rs index 82ab5cb48b..27556219cf 100644 --- a/crates/engine/src/cmd/whole.rs +++ b/crates/engine/src/cmd/whole.rs @@ -80,6 +80,63 @@ impl Unread { } } +/// A whole read made before its command runs ([`super::reads`]): the source it read, the fields +/// the layer listed then, and what it read with the number of rows that couldn't be read. +type ReadAhead = (rustgis_map::source::SourceLayer, Vec, Result<(Dataset, usize), String>); + +/// The key the whole read of `l` made ahead is taken by. +fn ahead_key(l: &Layer) -> String { + format!("whole:{}", l.instance_id()) +} + +/// The whole read of the layer read in place that `p` names, made before `name` (the command that +/// takes it with [`take_or_read`]) runs, on a copy of the layer: `None` for a layer in memory. +pub(crate) fn ahead(s: &Session, p: &serde_json::Value, name: &'static str) -> Option { + let l = s.map().layer(s.layer_id(p).ok()?)?; + let mut copy = l.clone(); + let source = copy.source.as_mut()?; + // Its own read: the live layer keeps the one under way. + source.detach_read(); + let (copied, fields) = (source.clone(), l.dataset.fields.clone()); + Some(super::reads::ahead(name, ahead_key(l), move || (copied, fields, copy.load_all_counted()))) +} + +/// Every feature of layer `id` read in place, with the rows of its source that couldn't be read: +/// the read made ahead ([`ahead`]) when there is one from the source the layer still reads, +/// listing the fields it lists now; read here otherwise (one opened afresh since, or with other +/// fields, is read again). +pub(crate) fn take_or_read(s: &mut Session, id: u32) -> Result<(Dataset, Unread), String> { + let key = ahead_key(s.map().layer(id).ok_or("no such layer")?); + let ahead = super::reads::take::(s, &key); + let l = s.map().layer(id).ok_or("no such layer")?; + let (ds, rows) = match ahead { + Some((copied, fields, read)) if l.source.as_ref().is_some_and(|src| src.same_opening(&copied)) && fields == l.dataset.fields => { + read? + } + _ => l.load_all_counted()?, + }; + Ok((ds, Unread { layer: l.name.clone(), rows })) +} + +/// Loads a layer read in place whole, in place of its view, as an export's copy of the project +/// needs it: its features are then all of its source's, its selection is kept (by feature id), +/// and it no longer reads its source. A layer in memory is left as it is. The read looks at +/// Cancel as [`Whole::read`] does. +pub(crate) fn load_into(l: &mut Layer) -> Result { + let name = l.name.clone(); + if !l.is_streamed() { + return Ok(Unread { layer: name, rows: 0 }); + } + let selected = l.selected_ids(); + let (ds, rows) = l.load_all_counted().map_err(|e| format!("{name}: {e}"))?; + l.dataset = ds; + l.dataset.windowed = false; + l.source = None; + l.selection = l.dataset.features.iter().enumerate().filter(|(_, f)| selected.contains(&f.id)).map(|(i, _)| i).collect(); + l.touch_geometry(); + Ok(Unread { layer: name, rows }) +} + /// The notice for `unread` rows of `layer`'s source that couldn't be read. pub(crate) fn unread_notice(layer: &str, unread: usize) -> String { match unread { @@ -91,9 +148,18 @@ pub(crate) fn unread_notice(layer: &str, unread: usize) -> String { /// The layers read in place that `command` with `params` reads whole from their sources (for its /// permission effects, which then reach those sources as Load All Features does). pub(crate) fn layers_read_whole<'a>(s: &super::InputContext<'a>, command: &str, params: &serde_json::Value) -> Vec<&'a Layer> { - let names: Vec<&str> = match command { - "dl.train" | "dl.classify" | "raster.classifySupervised" => super::ps(params, "training").into_iter().collect(), + let layers: Vec<&'a Layer> = match command { + "dl.train" | "dl.classify" | "raster.classifySupervised" => { + super::ps(params, "training").and_then(|n| s.map().layer_by_name(n)).into_iter().collect() + } + "labels.convert" => s.layer_id(params).ok().and_then(|id| s.map().layer(id)).into_iter().collect(), + "report.export" => super::report::index(s, params) + .map(|i| super::report::reported_layers(s, i)) + .unwrap_or_default() + .into_iter() + .filter_map(|(m, id)| s.project.maps.get(m)?.layer(id)) + .collect(), _ => Vec::new(), }; - names.into_iter().filter_map(|n| s.map().layer_by_name(n)).filter(|l| l.is_streamed()).collect() + layers.into_iter().filter(|l| l.is_streamed()).collect() } diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 518cf02c60..8d21782204 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -145,16 +145,17 @@ impl Session { "layer.addData" | "layer.loadAll" | "layer.reloadData" + | "labels.convert" | "model.run" | "script.run" | "map.reloadTiles" | "map.repairTiles" | "topology.validate" ) { - return Err("only Add Data, Load All, Reload Data, models, scripts, tile archives and topology prepare background work".into()); + return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()); } let mut params = params.as_object().cloned().ok_or("read parameters must be an object")?; - if matches!(command, "layer.loadAll" | "layer.reloadData") { + if matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") { let id = self.layer_id(&Value::Object(params.clone()))?; params.insert("layer".into(), json!(id)); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c9374d7d48..203442dbe6 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -11952,6 +11952,122 @@ fn two_class_b64() -> String { base64::engine::general_purpose::STANDARD.encode(geotiff::write(&r, 0).unwrap()) } +/// The layers a report export reads whole are those its pages list: its own, and those of the +/// reports it appends as far as they are drawn, not one appended past the depth drawn. +#[test] +fn a_report_export_reads_only_the_layers_it_draws() { + let mut s = world(); + let point = json!({"type": "FeatureCollection", "features": [{"type": "Feature", "properties": {"name": "a"}, "geometry": {"type": "Point", "coordinates": [0, 0]}}]}); + s.run("layer.addDataText", &json!({"fileName": "deep.geojson", "text": point.to_string()})).unwrap(); + let layers = ["Cities", "Countries", "Rivers", "Lakes", "deep"]; + for (k, layer) in layers.iter().enumerate() { + s.run("report.new", &json!({"layer": layer, "name": format!("r{k}")})).unwrap(); + } + // Each appends the next: r0 draws r1 to r3, and r3 is too deep to draw r4. + for k in 0..4 { + s.run("report.relatedReport", &json!({"report": format!("r{k}"), "related": format!("r{}", k + 1)})).unwrap(); + } + let mut read: Vec<_> = + crate::cmd::report::reported_layers(&s, 0).into_iter().map(|(m, id)| s.project.maps[m].layer(id).unwrap().name.clone()).collect(); + read.sort(); + assert_eq!(read, ["Cities", "Countries", "Lakes", "Rivers"]); +} + +/// A layer read in place a view at a time, its view holding one of its three points: its labels +/// convert to an annotation for every point, read here or ahead as the app reads it, and a +/// report's export lists every record (its selection kept by feature) while its preview says it +/// shows those of the view. Rows the source can't give are said once; clean sources say nothing. +#[test] +fn labels_and_reports_of_a_layer_read_in_place_use_its_whole_source() { + use rustgis_analysis::control::Control; + use rustgis_map::report::Block; + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + // Its own each run (the process id alone can repeat). + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); + let dir = Dir(std::env::temp_dir().join(format!("rustgis-whole-labels-{}-{nanos}", std::process::id()))); + std::fs::create_dir_all(&dir.0).unwrap(); + let dir = &dir.0; + let point = |name: &str, x: f64, y: f64| json!({"type": "Feature", "properties": {"name": name}, "geometry": {"type": "Point", "coordinates": [x, y]}}); + let bad = json!({"type": "Feature", "properties": {"name": "Bad"}, "geometry": {"type": "Point", "coordinates": "x"}}); + // The notice for the one row of the layer its source can't give. + let one_unread = crate::cmd::whole::unread_notice("places", 1); + let unread = |s: &Session| s.notices.iter().filter(|n| n.text == one_unread).count(); + let view = rustgis_geom::Extent::new(113.0, 22.0, 115.0, 24.0); + let records = |doc: &rustgis_map::report::Doc| { + doc.blocks + .iter() + .find_map(|b| match b { + Block::Text(t) if t.contains(" records from ") => Some(t.clone()), + _ => None, + }) + .unwrap() + }; + for (file, extra) in [("clean.geojson", None), ("bad.geojson", Some(bad))] { + let mut features = vec![point("Lima", -77.0, -12.0), point("Oslo", 10.7, 59.9), point("Hong Kong", 114.17, 22.3)]; + features.extend(extra.clone()); + let path = dir.join(file); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": features}).to_string()).unwrap(); + let said = usize::from(extra.is_some()); + for ahead in [false, true] { + let mut s = Session::new(Project::default()); + let id = + s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "places"})).unwrap()["layer"] + .as_u64() + .unwrap() as u32; + s.map_mut().sync_sources(&view, 1000.0, true); + assert_eq!(s.map().layer(id).unwrap().dataset.features.len(), 1, "{file}: the view holds Hong Kong only"); + let p = json!({"layer": id}); + let r = if ahead { + let read = crate::cmd::reads::read_ahead(&mut s, "labels.convert", &p).unwrap(); + assert!(read.reads(), "{file}: read ahead"); + crate::cmd::reads::read_land(&mut s, read.run(&Control::new()).unwrap()) + } else { + s.run("labels.convert", &p) + } + .unwrap_or_else(|e| panic!("{file}: {e}")); + assert_eq!(r["annotations"], 3, "{file}, ahead {ahead}: {r}"); + assert_eq!(unread(&s), said, "{file}, ahead {ahead}: labels.convert"); + let l = s.map().layer(id).unwrap(); + assert!(l.is_streamed() && !l.labels.enabled, "{file}: still read in place, its own labels off"); + } + + // A report of it, then of only its selection (Hong Kong, the view's first row but not the + // source's first). + let mut s = Session::new(Project::default()); + let id = s.run("layer.addData", &json!({"path": path.display().to_string(), "stream": true, "name": "places"})).unwrap()["layer"] + .as_u64() + .unwrap() as u32; + s.map_mut().sync_sources(&view, 1000.0, true); + s.run("report.new", &json!({"layer": "places"})).unwrap(); + let preview = records(&crate::cmd::report::build(&s, 0).unwrap()); + assert!(preview.starts_with("1 records") && preview.contains("only the records in the map's view"), "{preview}"); + let mut copy = Session::new(s.project.clone()); + crate::cmd::report::whole_copy(&mut copy, 0).unwrap(); + let whole = records(&crate::cmd::report::build(©, 0).unwrap()); + assert!(whole.starts_with("3 records") && !whole.contains("view"), "{whole}"); + let before = unread(&s); + let r = s.run("report.export", &json!({"path": dir.join("report.pdf").display().to_string(), "dpi": 72})).unwrap(); + assert!(r["pages"].as_u64().unwrap() >= 1, "{r}"); + assert_eq!(unread(&s), before + said, "{file}: report.export"); + s.run("select.byAttributes", &json!({"layer": id, "where": "name = 'Hong Kong'"})).unwrap(); + s.run("report.update", &json!({"selectedOnly": true})).unwrap(); + let mut copy = Session::new(s.project.clone()); + crate::cmd::report::whole_copy(&mut copy, 0).unwrap(); + let doc = crate::cmd::report::build(©, 0).unwrap(); + assert!(records(&doc).starts_with("1 records"), "{}", records(&doc)); + let rows: Vec<_> = doc.blocks.iter().filter_map(|b| if let Block::Row(r) = b { Some(r.clone()) } else { None }).collect(); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!(rows[0].iter().any(|c| c == "Hong Kong"), "{rows:?}"); + assert!(s.map().layer(id).unwrap().is_streamed()); + } +} + /// Training read in place a view at a time: the view holds none of the training polygons, yet /// a model and a supervised classification learn from all of them, read from the source, and the /// rows the source can't give are said once. @@ -38629,7 +38745,9 @@ fn long_commands_stop_when_cancelled_while_they_run() { ("share.exportRaster", json!({"layer": dem, "path": out("dem.tif")})), ("share.exportMap", json!({"path": out("map.png"), "width": 320, "height": 200})), ("layout.export", json!({"path": out("layout.pdf"), "dpi": 72})), + ("report.export", json!({"path": out("report.pdf"), "dpi": 72})), ]; + s.run("report.new", &json!({"layer": "Cities"})).unwrap(); let registered = crate::cmd::exports::IN_BACKGROUND.iter().map(|(c, ..)| *c).collect::>(); assert_eq!(listed(&exports), registered, "add a new background export here"); for (command, p) in &exports { diff --git a/crates/engine/src/tests/permission_policy.rs b/crates/engine/src/tests/permission_policy.rs index 0b70be50d8..feecdb9d3f 100644 --- a/crates/engine/src/tests/permission_policy.rs +++ b/crates/engine/src/tests/permission_policy.rs @@ -1712,6 +1712,49 @@ fn permission_policy_training_read_whole_reaches_its_source() { } } +/// Converting the labels of, or exporting a report of, a layer read in place reads it whole, so +/// the call reaches its source; the same layer in memory reaches no network. +#[test] +fn permission_policy_labels_and_reports_read_whole_reach_their_source() { + let mut s = world(); + s.run("report.new", &json!({"layer":"Cities"})).unwrap(); + let (scope, _) = assistant(3603); + let convert = Call::command("labels.convert", json!({"layer":"Cities"})); + let export = Call::command("report.export", json!({"path":"/tmp/report.pdf"})); + let internet = |s: &Session, call: &Call| match s.review(&scope, call) { + Outcome::NeedsApproval(p) => p.review().why.contains("internet"), + _ => false, + }; + assert!(!internet(&s, &convert) && !internet(&s, &export), "in memory: no network"); + let id = s.map().layer_by_name("Cities").unwrap().id; + let mut source = rustgis_map::source::SourceLayer::default(); + source.path = Some("https://example.invalid/cities.fgb".into()); + s.map_mut().layer_mut(id).unwrap().source = Some(source); + assert!(internet(&s, &convert), "labels.convert"); + assert!(internet(&s, &export), "report.export"); + + // The report's own layer in memory, the web source only under its records (a subreport) or + // in a report it appends: the export reads those whole too. + for under in ["subreport", "appended"] { + let mut s = world(); + s.run("report.new", &json!({"layer":"Countries","name":"Main"})).unwrap(); + if under == "subreport" { + s.run("data.relate", &json!({"name":"Country of city","origin":"Countries","originKey":"name","destination":"Cities","destinationKey":"adm0name"})).unwrap(); + s.run("report.subreport", &json!({"report":"Main","relationship":"Country of city"})).unwrap(); + } else { + s.run("report.new", &json!({"layer":"Cities","name":"Appended"})).unwrap(); + s.run("report.relatedReport", &json!({"report":"Main","related":"Appended"})).unwrap(); + } + let export = Call::command("report.export", json!({"report":"Main","path":"/tmp/report.pdf"})); + assert!(!internet(&s, &export), "{under}: in memory"); + let id = s.map().layer_by_name("Cities").unwrap().id; + let mut source = rustgis_map::source::SourceLayer::default(); + source.path = Some("https://example.invalid/cities.fgb".into()); + s.map_mut().layer_mut(id).unwrap().source = Some(source); + assert!(internet(&s, &export), "{under}: its layer read whole from the web"); + } +} + /// In-memory range reads need Network, while local paths and external drivers retain Files. #[test] fn permission_policy_remote_range_reads_need_only_network() { diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 9fb195f787..98e7b958fb 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -579,6 +579,15 @@ impl Layer { self.source.as_ref().is_some_and(|s| s.is_pending()) } + /// The ids of the selected features: those of the selected rows, and for a layer read in place + /// those parked for its next read (a project just opened, its view not yet read). + pub fn selected_ids(&self) -> std::collections::HashSet { + let mut ids: std::collections::HashSet = + self.selection.iter().filter_map(|r| self.dataset.features.get(*r)).map(|f| f.id).collect(); + ids.extend(self.source.iter().flat_map(|src| src.saved_selection.iter().copied())); + ids + } + /// Replaces a streamed layer's features, keeping the selection (by feature id). fn install(&mut self, features: Vec) { let areas = self.source.as_ref().and_then(|source| source.ready_cache.as_ref()).map(|cache| cache.areas).unwrap_or_default(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bca74bc326..9b2035f078 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2263,6 +2263,13 @@ impl RustGisApp { let params = if params.is_null() { json!({}) } else { params }; return readjob::run(self, "topology.validate", params); } + // Converting the labels of a layer read in place reads all of its source first, on a + // thread of its own, whoever asks (a click, the assistant). + if id == "labels.convert" + && self.session.layer_id(¶ms).ok().and_then(|l| self.session.map().layer(l)).is_some_and(|l| l.is_streamed()) + { + return readjob::run(self, "labels.convert", params); + } } self.run_with_prepared(id, params, AppDispatch::Direct) } @@ -3512,7 +3519,13 @@ impl RustGisApp { .0; self.download(&format!("{name}.pdf"), bytes); } else if let Some(path) = dialogs::pick_save(&format!("{name}.pdf"), &["pdf", "png"], tl!("PDF or PNG")) { - return self.run("report.export", json!({"report": i, "path": path, "dpi": 200})); + // Drawn on a thread of its own, with progress and Cancel in the status bar: a + // layer read in place is read whole for it. + let params = json!({"report": i, "path": path, "dpi": 200}); + #[cfg(not(target_arch = "wasm32"))] + return exportjob::start(self, "report.export", params); + #[cfg(target_arch = "wasm32")] + return self.run("report.export", params); } } "ui.saveLayout" => { diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 76e0d7f37c..da9220edc9 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -2260,7 +2260,7 @@ fn layer_menu(app: &mut RustGisApp, ui: &mut Ui, l: &Layer, reorder: bool) { if !l.is_raster() && ui.button(tl!("Convert To Annotation")).on_hover_text(tl!("Convert this layer's labels into an annotation layer.")).clicked() { - if let Err(e) = app.run("labels.convert", json!({"layer": l.id})) { + if let Err(e) = app.run_by_user("labels.convert", json!({"layer": l.id})) { app.session.notify(e, true); } ui.close(); diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 6aa729e8f2..a4ee355a4f 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -1,5 +1,6 @@ -//! Add Data's whole-file reads, layer reload, Load All Features, local archive validation and -//! model/script tools on the desktop run on a thread of their own (#1291 step 3), so large reads +//! Add Data's whole-file reads, layer reload, Load All Features, Convert To Annotation's read of a +//! layer read in place, local archive validation and model/script tools on the desktop run on a +//! thread of their own (#1291 step 3), so large reads //! and tools don't hold the window: the status bar shows the run under way, //! how far it has come and Cancel, and [`land`] then runs the command, which takes what was made //! ([`reads`]) and adds it as it does when it runs here. They go one at a time, in the order @@ -62,9 +63,9 @@ struct ReadJob { slot: crate::Slot>, } -/// Runs Add Data, Load All, Reload Data, archive reload/repair, model/script tools or the topology -/// check with `p`: its read on a thread of its own when the desktop app makes it so -/// ([`reads::read_ahead`]), after any under way; here and now otherwise. While one is under way or +/// Runs Add Data, Load All, Reload Data, Convert To Annotation, archive reload/repair, model/script +/// tools or the topology check with `p`: its read on a thread of its own when the desktop app makes it +/// so ([`reads::read_ahead`]), after any under way; here and now otherwise. While one is under way or /// waiting it answers that, not what the command installs, which comes as it lands. pub fn run(app: &mut RustGisApp, command: &'static str, p: Value) -> Result { run_then(app, command, p, None) @@ -77,7 +78,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: let r = app.run(command, p); return answered(app, then, r); } - let layer = matches!(command, "layer.loadAll" | "layer.reloadData") + let layer = matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") .then(|| app.session.layer_id(&p).ok().and_then(|id| app.session.map().layer(id)).map(|l| (l.instance_id(), l.name.clone()))) .flatten(); let notice_mark = app.session.notice_mark(); @@ -1074,6 +1075,28 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + /// Convert To Annotation, clicked for a layer read in place (whose view holds none of its + /// features), reads all of its source on a thread first: every feature gets its annotation. + #[test] + fn labels_of_a_layer_read_in_place_convert_after_a_read_on_a_thread() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("streamed.geojson"); + let point = |name: &str, x: f64| json!({"type": "Feature", "properties": {"name": name}, "geometry": {"type": "Point", "coordinates": [x, 47.0]}}); + std::fs::write(&path, json!({"type": "FeatureCollection", "features": [point("A", 8.0), point("B", 9.0)]}).to_string()).unwrap(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.reads_off_frame = true; + let id = + app.run("layer.addData", json!({"path": path, "stream": true, "name": "places"})).unwrap()["layer"].as_u64().unwrap() as u32; + assert!(app.session.map().layer(id).unwrap().dataset.features.is_empty(), "nothing read for a view yet"); + app.session.active_layer = Some(id); + assert_eq!(app.run_by_user("labels.convert", Value::Null).unwrap()["started"], "Convert To Annotation"); + landed(&mut app, 60); + let annotation = app.session.map().layer_by_name("places Annotation").unwrap(); + assert_eq!(annotation.dataset.features.len(), 2); + let l = app.session.map().layer(id).unwrap(); + assert!(l.is_streamed() && !l.labels.enabled); + } + fn reload_fixture() -> (tempfile::TempDir, RustGisApp, u32, std::path::PathBuf) { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("streamed.geojson");