diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index 3e626f844f..014b95becb 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -244,6 +244,27 @@ mod native { } Ok(changed) } + + /// The names of the files that differ from this record by what metadata alone shows: + /// added, removed, or another length. Reads no file, so a change that keeps every length + /// isn't seen; [`FileSync::changed`] hashes them. An unreadable file counts as differing. + pub fn resized(&self, path: &Path) -> Vec { + let now = parts(path); + let mut differ: Vec = + now.iter().filter(|(name, p)| !self.files.iter().any(|(n, _)| n == name) && p.exists()).map(|(n, _)| n.clone()).collect(); + for (name, was) in &self.files { + let is = now.iter().find(|(n, _)| n == name).map(|(_, p)| len(p)); + let same = match (was, is) { + (None, None | Some(Ok(None))) => true, + (Some(was), Some(Ok(Some(len)))) => len == was.len, + _ => false, + }; + if !same { + differ.push(name.clone()); + } + } + differ + } } } @@ -251,6 +272,21 @@ mod native { mod tests { use super::*; + #[test] + fn resized_sees_lengths_and_presence_without_reading() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a.geojson"); + std::fs::write(&path, "{}").unwrap(); + let record = FileSync::read(&path).unwrap(); + assert!(record.resized(&path).is_empty()); + std::fs::write(&path, "[]").unwrap(); + assert!(record.resized(&path).is_empty(), "same length: only a hash tells"); + std::fs::write(&path, "{ }").unwrap(); + assert_eq!(record.resized(&path), ["a.geojson"]); + std::fs::remove_file(&path).unwrap(); + assert_eq!(record.resized(&path), ["a.geojson"]); + } + #[test] fn a_shapefile_set_records_its_companions_and_the_missing_ones() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index c152d493eb..28e4cadf22 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -247,8 +247,8 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { /// Whether `path` names a file on this computer a record is made for: a Shapefile or `.geojson` /// file, not an address, a database or a network share (`\\host\share`, `//host/share`, /// `/\host\share`, `\\?\UNC\host\share`). Windows' long form of a local path (`\\?\C:\…`) is one -/// on this computer. -fn local_file(path: &str) -> bool { +/// on this computer. The desktop's Layers pane marks the same layers this publishes. +pub fn local_file(path: &str) -> bool { let b = rustgis_map::paths::local_path(path).as_bytes(); let sep = |i: usize| b.get(i).is_some_and(|c| matches!(c, b'/' | b'\\')); // Windows takes any two leading separators, of either kind, for a network share, unless they diff --git a/crates/ui-egui/src/diskwatch.rs b/crates/ui-egui/src/diskwatch.rs index 30f21d7a3d..b860f095cf 100644 --- a/crates/ui-egui/src/diskwatch.rs +++ b/crates/ui-egui/src/diskwatch.rs @@ -10,6 +10,7 @@ use std::collections::{HashMap, HashSet}; use rustgis_engine::Session; +use rustgis_engine::data::sync::FileSync; use rustgis_engine::observe::Observation; use serde_json::{Value, json}; @@ -74,6 +75,10 @@ struct Source { /// that started after the write, what was there just after it (RustGIS's result), and the /// targets whose data the write left behind (another layer on the table it wrote). wrote: Option<(Seen, u64, Seen, HashSet)>, + /// The target that write left in step whatever its baseline: a publish compares the file's + /// contents with its record before writing, so a change of metadata alone before it (the same + /// bytes written again) doesn't keep the publishing layer marked. + verified: Option, } struct Tracked { @@ -83,6 +88,9 @@ struct Tracked { baseline: Option, /// Not Now: the observation it was given for. A newer one notifies again. dismissed: Option, + /// The first scan that may give it a baseline: after [`Monitor::rebaseline`], a scan that + /// started before (and may have seen the file before the change) doesn't. + from_scan: u64, } /// Change detection for every target, by observations alone (no clock, no files). @@ -104,7 +112,7 @@ impl Monitor { match self.targets.get_mut(&target) { Some(t) if t.key == key => {} _ => { - self.targets.insert(target.clone(), Tracked { key: key.clone(), baseline: None, dismissed: None }); + self.targets.insert(target.clone(), Tracked { key: key.clone(), baseline: None, dismissed: None, from_scan: 0 }); } } seen.insert(target); @@ -115,6 +123,25 @@ impl Monitor { self.targets.values().any(|t| t.baseline.is_none()) } + /// Forgets what `target` was read with, so the first readable scan numbered `next_scan` or + /// later is its baseline: its data was brought in step with its file some other way, and a + /// scan already under way may have looked before that. `true`, as a scan is then wanted. + /// An earlier publish's acknowledgment still pending for it no longer applies: what it wrote + /// was replaced. + pub(crate) fn rebaseline(&mut self, target: &Target, next_scan: u64) -> bool { + if let Some(t) = self.targets.get_mut(target) { + t.baseline = None; + t.dismissed = None; + t.from_scan = next_scan; + if let Some(source) = self.sources.get_mut(&t.key) + && source.verified.as_ref() == Some(target) + { + source.verified = None; + } + } + true + } + /// Whether a target reads `key`. pub(crate) fn watches(&self, key: &str) -> bool { self.targets.values().any(|t| t.key == key) @@ -151,17 +178,20 @@ impl Monitor { Some((before, after, result, stale)) if agreed && scan >= *after => Some((before.clone(), result.clone(), stale.clone())), _ => None, }; - if acknowledged.is_some() { + let verified = if acknowledged.is_some() { source.wrote = None; - } + source.verified.take() + } else { + None + }; for (target, t) in self.targets.iter_mut().filter(|(_, t)| t.key == key) { // A target added since: the first readable scan is what it was read with (a look // that couldn't read the metadata says nothing about what the layer holds). - if t.baseline.is_none() && !matches!(seen, Seen::Unreadable(_)) { + if t.baseline.is_none() && !matches!(seen, Seen::Unreadable(_)) && scan >= t.from_scan { t.baseline = Some(seen.clone()); } if let Some((before, result, stale)) = &acknowledged - && t.baseline.as_ref() == Some(before) + && (t.baseline.as_ref() == Some(before) || verified.as_ref() == Some(target)) && !stale.contains(target) { t.baseline = Some(result.clone()); @@ -249,6 +279,16 @@ impl Monitor { _ => (before, stale), }; source.wrote = Some((before, next_scan, result, stale)); + // Only a publish marks its writer verified, after this ([`Monitor::verified`]). + source.verified = None; + } + + /// The write just recorded ([`Monitor::wrote`]) checked `target`'s file against its record + /// before writing it, so `target` takes the result whatever it held before. + pub(crate) fn verified(&mut self, key: &str, target: &Target) { + if let Some(source) = self.sources.get_mut(key).filter(|s| s.wrote.is_some()) { + source.verified = Some(target.clone()); + } } } @@ -276,17 +316,35 @@ struct Found { /// Read in place: a RustGIS write into its GeoPackage table reads it again /// (`gpkg::reset_sources`), so that write leaves nothing stale on it. streamed: bool, + /// For a layer read whole from a Shapefile or GeoJSON file it can publish to: the record of + /// its files at the last sync. Reload Data can't read such a layer again, so it's marked in + /// the Layers pane and not offered in the notification. + record: Option, +} + +/// The file a layer read whole from a local Shapefile or GeoJSON publishes to, and its record +/// (`source.publish`): held in memory, not read in place. +fn published_file(l: &rustgis_engine::map::Layer) -> Option<(&str, &rustgis_engine::data::sync::Synced)> { + let synced = l.synced.as_ref()?; + // As `source.publish` takes it: a network share isn't a file on this computer. + let path = l.dataset.source.as_deref().filter(|p| rustgis_engine::cmd::publish::local_file(p))?; + (l.source.is_none() && l.gpkg.is_none()).then_some((path, synced)) +} + +/// Whether the layer publishes to a file the Layers pane marks ([`published_file`]). +pub(crate) fn publishes(l: &rustgis_engine::map::Layer) -> bool { + published_file(l).is_some() } -/// The linked local sources in every map: layers Reload Data reads and local archives. Remote, -/// embedded and in-memory data aren't targets; a converted (GDAL, DuckDB) layer is watched at the +/// The linked local sources in every map: layers Reload Data reads, layers that publish to their +/// file, and local archives. Remote, embedded and other in-memory data aren't targets; a converted (GDAL, DuckDB) layer is watched at the /// original file, not the conversion. fn targets(s: &Session) -> Vec { let mut out = Vec::new(); for m in &s.project.maps { for l in &m.layers { + let target = Target::Layer { map: m.instance_id(), layer: l.instance_id() }; if let Some((path, table, _)) = crate::reloaddata::source(l) { - let target = Target::Layer { map: m.instance_id(), layer: l.instance_id() }; let repairable = l.linked_file().is_some(); out.push(Found { target, @@ -297,6 +355,19 @@ fn targets(s: &Session) -> Vec { name: l.name.clone(), repairable, streamed: l.gpkg.is_none() && l.source.is_some(), + record: None, + }); + } else if let Some((path, synced)) = published_file(l) { + out.push(Found { + target, + key: key(path), + path: path.to_string(), + table: None, + map: m.name.clone(), + name: l.name.clone(), + repairable: false, + streamed: false, + record: Some(synced.files.clone()), }); } } @@ -316,6 +387,7 @@ fn targets(s: &Session) -> Vec { name: tile.name.clone(), repairable: true, streamed: false, + record: None, }); } } @@ -323,7 +395,9 @@ fn targets(s: &Session) -> Vec { out } -type Scanned = crate::Slot>; +/// One scan's observations by key, and each published layer's file parts that differ from its +/// record by length or presence ([`FileSync::resized`]), with the record compared. +type Scanned = crate::Slot<(Vec<(String, Seen)>, Vec<(Target, FileSync, Vec)>)>; /// The app's side of the monitor: its targets, its one scan at a time and when the next is due. #[derive(Default)] @@ -340,6 +414,9 @@ pub struct Watch { /// A scan couldn't start, or one hasn't answered for [`STALLED`]: said once each. said_failed: bool, said_stalled: bool, + /// Each published layer's file parts the latest scan found resized against the record it + /// compared with; it counts only while the layer still has that record. + resized: HashMap)>, } impl Watch { @@ -348,9 +425,23 @@ impl Watch { } } -/// What the Layers pane says of a layer of the active map. +/// What the Layers pane says of a layer of the active map. A layer that publishes to its file is +/// changed, too, when its files' lengths or presence differ from its record: that catches a change +/// made while the project was closed, which the first scan takes as the baseline. pub(crate) fn layer_status(app: &RustGisApp, layer: &rustgis_engine::map::Layer) -> Status { - app.diskwatch.status(&Target::Layer { map: app.session.map().instance_id(), layer: layer.instance_id() }) + let target = Target::Layer { map: app.session.map().instance_id(), layer: layer.instance_id() }; + let status = app.diskwatch.status(&target); + let resized = app + .diskwatch + .resized + .get(&target) + .is_some_and(|(record, parts)| !parts.is_empty() && layer.synced.as_ref().is_some_and(|s| s.files == *record)); + if status == Status::Unchanged && resized { Status::Changed } else { status } +} + +/// Whether Reload Data can read the layer again; a layer that publishes to its file can't. +pub(crate) fn reloadable(layer: &rustgis_engine::map::Layer) -> bool { + crate::reloaddata::source(layer).is_some() } /// The target of tile layer `i` of `map`: its archive and which of the map's archives with that @@ -410,15 +501,26 @@ pub(crate) fn tick(app: &mut RustGisApp, ctx: &egui::Context) { } if w.revision != Some(app.session.revision) { w.revision = Some(app.session.revision); + let before = std::mem::take(&mut w.found); w.found = targets(&app.session); w.wanted |= w.monitor.track(w.found.iter().map(|f| (f.target.clone(), f.key.clone()))); + // A published layer whose record changed was published, however the command reached the + // session (the assistant's relay and file bridge run it without this watcher's hooks): the + // file now holds what it does, so the next scan is its baseline again. + for f in &w.found { + if let Some(record) = &f.record + && before.iter().any(|b| b.target == f.target && b.record.as_ref().is_some_and(|old| old != record)) + { + w.wanted |= w.monitor.rebaseline(&f.target, w.scans + 1); + } + } } let focused = ctx.input(|i| i.focused); w.wanted |= focused && !w.focused; w.focused = focused; if let Some((scan, slot, _)) = &w.scan { let done = slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - if let Some(results) = done { + if let Some((results, resized)) = done { let scan = *scan; w.scan = None; // A scan answered: a later stall is a new episode, said again. @@ -428,6 +530,16 @@ pub(crate) fn tick(app: &mut RustGisApp, ctx: &egui::Context) { w.monitor.observed(&key, seen, scan); } } + // As for the observations: a file being written by RustGIS (a publish) is its write's, + // not a change, until it lands. + let found = &w.found; + w.resized = resized + .into_iter() + .filter(|(target, _, _)| { + exporting.is_none() || found.iter().find(|f| f.target == *target).map(|f| f.key.as_str()) != exporting.as_deref() + }) + .map(|(target, record, parts)| (target, (record, parts))) + .collect(); } } let now = web_time::Instant::now(); @@ -482,11 +594,20 @@ fn start(w: &mut Watch, now: web_time::Instant) -> Result<(), String> { for f in &w.found { paths.entry(f.key.clone()).or_insert_with(|| f.path.clone()); } + let records: Vec<(Target, String, FileSync)> = + w.found.iter().filter_map(|f| Some((f.target.clone(), f.path.clone(), f.record.clone()?))).collect(); let slot = Scanned::default(); let out = slot.clone(); let spawned = std::thread::Builder::new().name("Changed on disk".into()).spawn(move || { let results = paths.into_iter().map(|(key, path)| (key, look(&path))).collect(); - *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(results); + let resized = records + .into_iter() + .map(|(target, path, record)| { + let parts = record.resized(std::path::Path::new(&path)); + (target, record, parts) + }) + .collect(); + *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some((results, resized)); }); w.due = Some(now + EVERY); w.wanted = false; @@ -587,6 +708,12 @@ pub(crate) fn plan_write(app: &RustGisApp, command: &str, params: &Value) -> Opt let edit = l.gpkg.as_ref()?; (edit.path.clone(), edit.table.clone(), Some(Target::Layer { map: s.map().instance_id(), layer: l.instance_id() })) } + // Publishing replaces its layer's file: the publishing layer holds what it wrote. + "source.publish" => { + let l = s.map().layer(s.layer_id(params).ok()?)?; + let (path, _) = published_file(l)?; + (path.to_string(), String::new(), Some(Target::Layer { map: s.map().instance_id(), layer: l.instance_id() })) + } // Conversion replaces a whole file; it writes no container table and refreshes no reader. "data.convert" => (params.get("path").and_then(Value::as_str)?.to_string(), String::new(), None), // An existing GeoPackage written into; a new file isn't watched yet. @@ -615,10 +742,14 @@ pub(crate) fn plan_write(app: &RustGisApp, command: &str, params: &Value) -> Opt let stale = if command == "data.convert" { // The explicit format can differ from the suffix: even a .gpkg is replaced whole. app.diskwatch.found.iter().filter(|f| f.key == key).map(|f| f.target.clone()).collect() + } else if command == "source.publish" { + // Every other layer on the file keeps the rows it read. + app.diskwatch.found.iter().filter(|f| f.key == key && Some(&f.target) != writer.as_ref()).map(|f| f.target.clone()).collect() } else { stale_for(&app.diskwatch.found, &key, &path, &table, writer.as_ref()) }; - Some(Write { before: None, result: None, key, path, stale, writer, table }) + let verified = command == "source.publish"; + Some(Write { before: None, result: None, key, path, stale, writer, table, verified }) } /// [`plan_write`] for a save or export that writes here, on the UI thread: the file is written @@ -642,6 +773,8 @@ pub(crate) struct Write { writer: Option, /// The table written, as predicted, then as the engine named it. table: String, + /// A publish: it checked the file's contents against the writer's record first. + verified: bool, } impl Write { @@ -672,7 +805,7 @@ pub(crate) fn ran(app: &mut RustGisApp, command: &str, params: &Value, answer: & } // A save that wrote nothing (no edits) isn't a write. "gpkg.saveEdits" if answer.get("saved").and_then(Value::as_bool) == Some(false) => {} - "gpkg.saveEdits" | "share.exportLayer" | "share.exportTable" | "data.convert" => { + "gpkg.saveEdits" | "share.exportLayer" | "share.exportTable" | "data.convert" | "source.publish" => { if let Some(write) = before { wrote(app, write, answer); } @@ -722,6 +855,11 @@ pub(crate) fn wrote(app: &mut RustGisApp, mut write: Write, answer: &Value) { let w = &mut app.diskwatch; // A scan under way started before the write; the next one is the first after it. w.monitor.wrote(&write.key, before, result, write.stale, write.writer.as_ref(), w.scans + 1); + if write.verified + && let Some(writer) = &write.writer + { + w.monitor.verified(&write.key, writer); + } } /// Activates the target's map and returns the target's layer id or tile index there. @@ -777,7 +915,7 @@ fn act(app: &mut RustGisApp, target: &Target, repair: bool) { pub(crate) fn show(app: &mut RustGisApp, ctx: &egui::Context) { let w = &app.diskwatch; let mut groups: Vec<(&str, &str, Status, Vec<&Found>)> = Vec::new(); - for f in w.found.iter().filter(|f| w.monitor.notifies(&f.target)) { + for f in w.found.iter().filter(|f| f.record.is_none() && w.monitor.notifies(&f.target)) { match groups.iter_mut().find(|g| g.0 == f.key) { Some(g) => g.3.push(f), None => groups.push((&f.key, &f.path, w.status(&f.target), vec![f])), @@ -1136,6 +1274,37 @@ mod tests { assert!(!m.keys().contains("a")); } + #[test] + fn a_scan_that_looked_before_a_rebaseline_does_not_become_the_baseline() { + let [v0, v1] = observations(2).try_into().unwrap(); + let (mut m, scan) = monitor(&v0); + // Started (scan `scan`) while the file still held v0; a publish then wrote v1 and the + // layer's record changed before that scan landed. + m.rebaseline(&layer(3), scan + 1); + m.observed("b", v0.clone(), scan); + assert_eq!(m.status(&layer(3)), Status::Unchanged, "no baseline yet"); + for next in [scan + 1, scan + 2] { + m.observed("b", v1.clone(), next); + } + assert_eq!(m.status(&layer(3)), Status::Unchanged, "the scans after it are the baseline"); + assert_eq!(m.status(&layer(1)), Status::Unchanged, "another file's target is untouched"); + } + + #[test] + fn a_rebaseline_drops_an_earlier_publish_still_to_be_acknowledged() { + let [v0, v1, v2] = observations(3).try_into().unwrap(); + let (mut m, scan) = monitor(&v0); + // Published in the app (v0 to v1), then again through the session alone (v2) before any + // scan acknowledged the first. + m.wrote("b", v0.clone(), v1.clone(), HashSet::new(), Some(&layer(3)), scan); + m.verified("b", &layer(3)); + m.rebaseline(&layer(3), scan); + for next in [scan, scan + 1] { + m.observed("b", v2.clone(), next); + } + assert_eq!(m.status(&layer(3)), Status::Unchanged, "the second publish's file is the baseline"); + } + #[test] fn two_tile_layers_on_one_archive_are_separate_targets() { use rustgis_engine::map::{Map, TileKind, TileLayer}; @@ -1344,6 +1513,174 @@ mod tests { assert_eq!(app.diskwatch.status(&other), Status::Unchanged, "RustGIS's own export isn't an outside change for another table"); } + /// What the Layers pane says of layer `id`'s edits, once its hash has landed. + fn edited(app: &mut RustGisApp, ctx: &egui::Context, id: u32) -> bool { + let end = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let l = app.session.map().layer(id).unwrap().clone(); + if let Some(edited) = crate::syncmark::hashed(app, &l, ctx) { + return edited; + } + assert!(std::time::Instant::now() < end, "the hash never landed"); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + + fn status_of(app: &RustGisApp, id: u32) -> Status { + layer_status(app, app.session.map().layer(id).unwrap()) + } + + /// A GeoJSON read whole, as the desktop's Add Data reads it, so it can publish. + fn published_layer(dir: &std::path::Path) -> (RustGisApp, u32, std::path::PathBuf) { + let path = dir.join("parcels.geojson"); + std::fs::write( + &path, + r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"owner":"Smith"},"geometry":{"type":"Point","coordinates":[1,2]}}]}"#, + ) + .unwrap(); + let mut app = RustGisApp::new(Session::new(rustgis_engine::Project::default())); + let id = layer_id(app.run("layer.addData", json!({"path": path, "stream": false})).unwrap()); + assert!(app.session.map().layer(id).unwrap().synced.is_some()); + app.watch_files = true; + (app, id, path) + } + + #[test] + fn a_publish_clears_a_mark_left_by_the_same_bytes_written_again() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, path) = published_layer(dir.path()); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + // Another program writes the same bytes: only its metadata changed. + std::thread::sleep(std::time::Duration::from_millis(20)); + std::fs::write(&path, std::fs::read(&path).unwrap()).unwrap(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Changed); + app.run("edit.enable", json!({"on": true})).unwrap(); + app.run("edit.setValue", json!({"layer": id, "row": 0, "field": "owner", "value": "Jones"})).unwrap(); + app.run("source.publish", json!({"layer": id})).unwrap(); + scan(&mut app, &ctx, 3); + assert_eq!(status_of(&app, id), Status::Unchanged, "the publish checked the contents and wrote them"); + } + + #[test] + fn a_publish_through_the_session_alone_clears_the_mark_too() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, path) = published_layer(dir.path()); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + std::thread::sleep(std::time::Duration::from_millis(20)); + std::fs::write(&path, std::fs::read(&path).unwrap()).unwrap(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Changed); + // As the assistant's relay runs it: the session's own dispatch, without the app's hooks. + app.session.run("edit.enable", &json!({"on": true})).unwrap(); + app.session.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "owner", "value": "Jones"})).unwrap(); + app.session.run("source.publish", &json!({"layer": id})).unwrap(); + scan(&mut app, &ctx, 3); + assert_eq!(status_of(&app, id), Status::Unchanged, "the new record says the file was published"); + } + + #[test] + fn a_scan_left_unread_across_a_session_publish_does_not_become_the_baseline() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, _) = published_layer(dir.path()); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + // A scan looks at the file as it is, and its answer waits, untaken. + app.diskwatch.wanted = true; + tick(&mut app, &ctx); + let slot = app.diskwatch.scan.as_ref().map(|(_, slot, _)| slot.clone()).unwrap(); + let end = std::time::Instant::now() + std::time::Duration::from_secs(10); + while slot.lock().unwrap().is_none() { + assert!(std::time::Instant::now() < end, "the scan never answered"); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + // Published through the session alone, then that old answer lands with later scans. + app.session.run("edit.enable", &json!({"on": true})).unwrap(); + app.session.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "owner", "value": "Jones"})).unwrap(); + app.session.run("source.publish", &json!({"layer": id})).unwrap(); + scan(&mut app, &ctx, 3); + assert_eq!(status_of(&app, id), Status::Unchanged, "the answer from before the publish isn't the baseline"); + } + + #[test] + fn edits_on_a_missing_file_point_to_export_not_publish() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, path) = published_layer(dir.path()); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + app.run("edit.enable", json!({"on": true})).unwrap(); + app.run("edit.setValue", json!({"layer": id, "row": 0, "field": "owner", "value": "Jones"})).unwrap(); + assert!(edited(&mut app, &ctx, id)); + std::fs::remove_file(&path).unwrap(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Missing); + let l = app.session.map().layer(id).unwrap().clone(); + let (_, detail, _) = crate::panes::layer_note(&mut app, &l, &ctx).unwrap(); + assert!(detail.contains("Export Features"), "{detail}"); + assert!(!detail.contains("writes them to the file"), "Publish refuses a missing file: {detail}"); + } + + #[test] + fn a_layer_on_a_network_share_is_not_marked_as_publishable() { + let dir = tempfile::tempdir().unwrap(); + let (app, id, _) = published_layer(dir.path()); + assert!(publishes(app.session.map().layer(id).unwrap())); + // Publish to Source refuses a share, so the Layers pane doesn't promise it. + for share in [r"\\host\share\parcels.geojson", "//host/share/parcels.geojson"] { + let mut l = app.session.map().layer(id).unwrap().clone(); + l.dataset.source = Some(share.into()); + assert!(!publishes(&l), "{share}"); + } + } + + #[test] + fn a_published_file_resized_while_the_project_was_closed_is_marked() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, path) = published_layer(dir.path()); + // Changed before the first scan, which takes what is there as its baseline. + std::fs::write(&path, r#"{"type":"FeatureCollection","features":[]}"#).unwrap(); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Changed, "its length differs from the record"); + assert!(!edited(&mut app, &ctx, id), "the layer holds what it read"); + } + + #[test] + fn edits_are_marked_until_published_and_the_publish_is_not_an_outside_change() { + let dir = tempfile::tempdir().unwrap(); + let (mut app, id, path) = published_layer(dir.path()); + let ctx = egui::Context::default(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Unchanged); + assert!(!edited(&mut app, &ctx, id)); + app.run("edit.enable", json!({"on": true})).unwrap(); + app.run("edit.setValue", json!({"layer": id, "row": 0, "field": "owner", "value": "Jones"})).unwrap(); + assert!(edited(&mut app, &ctx, id), "an edit the file lacks"); + crate::exportjob::start(&mut app, "source.publish", json!({"layer": id})).unwrap(); + let end = std::time::Instant::now() + std::time::Duration::from_secs(30); + while crate::exportjob::land(&mut app) { + assert!(std::time::Instant::now() < end, "the publish never landed"); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(std::fs::read_to_string(&path).unwrap().contains("Jones")); + scan(&mut app, &ctx, 3); + assert_eq!(status_of(&app, id), Status::Unchanged, "RustGIS's own publish is not an outside change"); + assert!(!edited(&mut app, &ctx, id), "published"); + // Published where it is asked (a script), not on the export thread: the same. + app.run("edit.setValue", json!({"layer": id, "row": 0, "field": "owner", "value": "Lee"})).unwrap(); + app.run("source.publish", json!({"layer": id})).unwrap(); + scan(&mut app, &ctx, 3); + assert_eq!(status_of(&app, id), Status::Unchanged, "a publish run in place is RustGIS's own write too"); + assert!(!edited(&mut app, &ctx, id)); + // Another program writes the file; the notification leaves it to the Layers pane. + std::fs::write(&path, r#"{"type":"FeatureCollection","features":[]}"#).unwrap(); + scan(&mut app, &ctx, 2); + assert_eq!(status_of(&app, id), Status::Changed); + assert!(app.diskwatch.found.iter().all(|f| f.record.is_some()), "offered no Reload Data"); + } + #[test] fn a_queued_reload_that_is_refused_leaves_a_later_change_marked() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bb5f1f12fe..ea27e15cf5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -78,6 +78,8 @@ mod popupmedia; mod portalsui; #[cfg(not(target_arch = "wasm32"))] mod readjob; +#[cfg(not(target_arch = "wasm32"))] +mod syncmark; /// How a command asked for ended ([`readjob::run_then`] on the desktop; the browser runs it where /// it's asked): what it answered when it ran, or why it never ran (cancelled, or left), which is @@ -1203,6 +1205,9 @@ pub struct RustGisApp { pub watch_files: bool, #[cfg(not(target_arch = "wasm32"))] diskwatch: diskwatch::Watch, + /// Which layers hold edits their file lacks, for the Layers pane. + #[cfg(not(target_arch = "wasm32"))] + syncmark: syncmark::Marks, /// ModelBuilder's view: the model on its canvas. pub mb: modelbuilder::ModelBuilder, pub ws: workspacepanes::WorkspaceState, @@ -1473,6 +1478,8 @@ impl RustGisApp { watch_files: false, #[cfg(not(target_arch = "wasm32"))] diskwatch: diskwatch::Watch::default(), + #[cfg(not(target_arch = "wasm32"))] + syncmark: syncmark::Marks::default(), mb: modelbuilder::ModelBuilder::default(), ws: workspacepanes::WorkspaceState::default(), time: timebar::TimeBarState::default(), @@ -5866,6 +5873,8 @@ impl RustGisApp { if self.watch_files { diskwatch::tick(self, ctx); } + #[cfg(not(target_arch = "wasm32"))] + syncmark::tick(self); self.ensure_script_font(); // Fonts downloaded by the web app. let fonts: Vec<(String, Vec)> = match &self.services.font_inbox { diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 12220646e6..f5c2147f3c 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -1904,6 +1904,48 @@ fn disk_note(app: &RustGisApp, layer: Option<&Layer>, tile: Option) -> Op } } +/// The note after a layer's name, its details and tint, as [`disk_note`] gives them. A layer read +/// whole from a Shapefile or GeoJSON file it publishes to (`source.publish`) also says whether it +/// holds edits the file lacks (`syncmark`), and its file's changes point to Publish to Source, +/// which refuses to overwrite them, rather than Reload Data, which can't read it again. +pub(crate) fn layer_note(app: &mut RustGisApp, l: &Layer, ctx: &egui::Context) -> Option<(String, String, Option)> { + #[cfg(not(target_arch = "wasm32"))] + { + use crate::diskwatch::Status; + if !crate::diskwatch::publishes(l) || crate::diskwatch::reloadable(l) { + return disk_note(app, Some(l), None); + } + let edited = crate::syncmark::edited(app, l, ctx) == Some(true); + let status = if app.watch_files { crate::diskwatch::layer_status(app, l) } else { Status::Unchanged }; + let unpublished = tl!("This layer has edits its file doesn't have yet. Publish to Source writes them to the file."); + match (status, edited) { + (Status::Changed, true) => Some(( + tl!("(edits not published, changed on disk)").to_string(), + tl!("This layer has edits its file doesn't have, and the file changed on disk since it was read. Publish to Source refuses rather than overwrite that change; Export Features keeps the edits in a new file.").to_string(), + Some(false), + )), + (Status::Changed, false) => Some(( + tl!("(changed on disk)").to_string(), + tl!("Changed on disk: the file may have changed since it was read. Add it again to see the change; Source File State checks it.").to_string(), + Some(false), + )), + (Status::Unchanged | Status::Folder, true) => Some((tl!("(edits not published)").to_string(), unpublished.to_string(), None)), + // Missing or unreadable: Publish refuses a file unlike its record, so edits can only be + // exported until it's back. + (status, edited) => crate::diskwatch::row_note(&status, false).map(|(note, detail, tint)| { + let blocked = tl!("This layer has edits its file doesn't have. Publish to Source can't write them until the file is back as it was read; Export Features keeps them in a new file."); + let detail = if edited { crate::i18n::fmt("{detail}\n{blocked}", &[("detail", &detail), ("blocked", blocked)]) } else { detail }; + (note, detail, tint) + }), + } + } + #[cfg(target_arch = "wasm32")] + { + let _ = ctx; + disk_note(app, Some(l), None) + } +} + /// A layer's row and, expanded, its legend; returns the rectangle of both. With `reorder` off /// (the list is filtered or children are sorted by name) the row can't be dragged. fn layer_row(app: &mut RustGisApp, ui: &mut Ui, l: &Layer, scale: f64, reorder: bool, t: &Tokens) -> Rect { @@ -1918,7 +1960,7 @@ fn layer_row(app: &mut RustGisApp, ui: &mut Ui, l: &Layer, scale: f64, reorder: (text, w) }); let mut shown = if l.definition_query.is_some() { format!("{} {}", l.name, tl!("(filtered)")) } else { l.name.clone() }; - let disk = disk_note(app, Some(l), None); + let disk = layer_note(app, l, ui.ctx()); if let Some((note, _, _)) = disk.as_ref().filter(|(note, _, _)| !note.is_empty()) { shown = format!("{shown} {note}"); } diff --git a/crates/ui-egui/src/syncmark.rs b/crates/ui-egui/src/syncmark.rs new file mode 100644 index 0000000000..cfcda3778b --- /dev/null +++ b/crates/ui-egui/src/syncmark.rs @@ -0,0 +1,260 @@ +//! Edits not yet published (#1385): whether a layer read whole from a Shapefile or GeoJSON file +//! holds rows its file lacks, for the Layers pane. The layer's rows are hashed +//! (`rustgis_data::sync::rows_digest`) on a thread of their own and compared with the record of +//! the last sync; the answer is kept until the rows, fields, coordinate system or record change. +//! An edit copies the rows, so a changed layer is told by their allocation without a hash. The +//! rows hashed are held only while their hash runs. A hash whose layer changed or went away is +//! cancelled (`rows_digest` looks at Cancel between rows), and no more hashes run at once than +//! the thread setting allows (`rustgis_data::threads::max`), so opening a project of many large +//! layers doesn't start one each. Nothing is counted: the mark is a status, not a run with +//! progress. App state alone: nothing here dirties the project or adds an undo step. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Weak}; + +use rustgis_engine::data::{Feature, Field, sync::Synced}; +use rustgis_engine::map::Layer; + +/// A layer's answer, or the hash under way for it. +struct Entry { + rows: Weak>, + fields: Vec, + crs: Option, + synced: Synced, + /// The rows' hash once it lands; `Err` when it couldn't be made. + digest: Option>, + /// The one hash under way for the layer. One for an earlier state is cancelled and lands + /// before another starts, so edits in quick succession never hash a layer twice at once. + job: Option, + /// When to try again after the OS refused a thread. + retry: Option, + /// The answer for the layer's previous state, shown until this state's hash lands, so the + /// note doesn't drop out during each edit's hash. + shown: Option, +} + +/// A hash on its thread: where it lands, its Cancel, and whether it hashes the layer as its entry +/// has it. Dropped, it is cancelled. +struct Job { + slot: crate::Slot>, + stop: Arc, + current: bool, +} + +impl Drop for Job { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + } +} + +/// The hash thread's Cancel, as `rows_digest` looks at it. +struct Stop(Arc); + +impl rustgis_engine::data::progress::Probe for Stop { + fn cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + fn begin(&self, _: u64) {} + fn advance(&self, _: u64) {} +} + +impl Entry { + /// Takes in its hash if it has landed: the answer, when it hashed the layer as the entry has + /// it. Whether a hash landed, freeing its turn. + fn land(&mut self) -> bool { + let Some(job) = &self.job else { return false }; + let landed = job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let Some(digest) = landed else { return false }; + let current = job.current; + self.job = None; + if current { + self.digest = Some(digest); + } + true + } + + /// Whether the layer holds edits its file lacks: its own hash's answer once that lands, else + /// the previous state's. + fn answer(&self) -> Option { + match &self.digest { + Some(Ok(digest)) => Some(*digest != self.synced.rows), + // This state's hash failed: unknown, not the previous state's answer. + Some(Err(_)) => None, + None => self.shown, + } + } + + /// Whether it answers for `l` as it is now. + fn of(&self, l: &Layer) -> bool { + Weak::ptr_eq(&self.rows, &l.dataset.features.allocation_key()) + && self.fields == l.dataset.fields + && self.crs == l.dataset.crs + && l.synced.as_ref() == Some(&self.synced) + } +} + +/// How long after the OS refused a thread one is asked for again. +const RETRY: web_time::Duration = web_time::Duration::from_secs(5); + +/// Each layer's answer, by its map's instance and its own. +#[derive(Default)] +pub(crate) struct Marks { + entries: HashMap<(u64, u64), Entry>, + /// The project revision the entries were last pruned at. + revision: Option, + /// Hashes under way, as counted once a frame ([`tick`]) and as started since. + running: usize, + /// The project the entries belong to (`Session::opened`). + opened: u64, +} + +/// Whether `l`, a layer of the active map, holds edits its file lacks: `None` while that isn't +/// known yet (its hash is under way, or couldn't start) or for a layer with no record. Cheap on +/// the UI thread: a hash starts only when the layer changed, on a thread of its own, and the +/// pane is drawn again when it lands. +pub(crate) fn edited(app: &mut crate::RustGisApp, l: &Layer, ctx: &egui::Context) -> Option { + let synced = l.synced.as_ref()?; + let id = (app.session.map().instance_id(), l.instance_id()); + if !app.syncmark.entries.get(&id).is_some_and(|e| e.of(l)) { + // The earlier hash is stopped, and kept until it lands. + let replaced = app.syncmark.entries.remove(&id); + let shown = replaced.as_ref().and_then(Entry::answer); + let earlier = replaced.and_then(|mut e| e.job.take()).map(|mut job| { + job.stop.store(true, Ordering::Relaxed); + job.current = false; + job + }); + let entry = Entry { + rows: l.dataset.features.allocation_key(), + fields: l.dataset.fields.clone(), + crs: l.dataset.crs, + synced: synced.clone(), + digest: None, + job: earlier, + retry: None, + shown, + }; + app.syncmark.entries.insert(id, entry); + } + let marks = &mut app.syncmark; + let entry = marks.entries.get_mut(&id)?; + if entry.land() { + marks.running = marks.running.saturating_sub(1); + } + let running = marks.running; + let now = web_time::Instant::now(); + if entry.job.is_some() { + // Looked at again soon: either it lands, or the next hash starts once it has. + ctx.request_repaint_after(web_time::Duration::from_millis(100)); + } else if entry.digest.is_none() && entry.retry.is_none_or(|at| now >= at) { + if running >= rustgis_engine::data::threads::max() { + // Its turn comes when another hash lands. + ctx.request_repaint_after(web_time::Duration::from_millis(100)); + } else { + let (slot, stop) = (crate::Slot::default(), Arc::new(AtomicBool::new(false))); + let (out, probe) = (slot.clone(), Stop(stop.clone())); + let dataset = l.dataset.clone(); + let spawned = std::thread::Builder::new().name("Edits not published".into()).spawn(move || { + let _probe = rustgis_engine::data::progress::install(Some(Arc::new(probe))); + // A panic lands as an answer too, so it never holds the turn. Neither this nor a + // refused thread below can be brought about in a test: `rows_digest` has no panic + // path, and the OS refuses threads only under load. + let digest = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::data::sync::rows_digest(&dataset))) + .map_err(|_| "the hash stopped".to_string()) + .and_then(|d| d.map_err(|e| e.to_string())); + *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(digest); + }); + match spawned { + Ok(_) => { + entry.job = Some(Job { slot, stop, current: true }); + entry.retry = None; + marks.running += 1; + ctx.request_repaint_after(web_time::Duration::from_millis(100)); + } + // A thread the OS refuses costs time, not the mark: the layer stays unmarked (an + // unknown state isn't shown as edited) and it is tried again shortly. + Err(_) => { + entry.retry = Some(now + RETRY); + ctx.request_repaint_after(RETRY); + } + } + } + } + entry.answer() +} + +/// [`edited`] as this state's own hash answers it, `None` until that lands (not the previous +/// state's answer, which [`edited`] shows meanwhile). +#[cfg(test)] +pub(crate) fn hashed(app: &mut crate::RustGisApp, l: &Layer, ctx: &egui::Context) -> Option { + edited(app, l, ctx); + let e = app.syncmark.entries.get(&(app.session.map().instance_id(), l.instance_id()))?; + match e.digest.as_ref()? { + Ok(digest) => Some(*digest != e.synced.rows), + Err(_) => None, + } +} + +/// Once a frame: forgets another project's answers, takes in every finished hash, whether its row is drawn or not (another map, a +/// filtered or collapsed list), so one that landed never holds a turn another layer is waiting +/// for, and counts those still running. Once per project revision, forgets layers that are gone, +/// cancelling their hashes, so the answers kept follow the project. +pub(crate) fn tick(app: &mut crate::RustGisApp) { + let marks = &mut app.syncmark; + // Another project: nothing of the old one's applies, and its hashes are cancelled as their + // entries go. + if marks.opened != app.session.opened { + *marks = Marks { opened: app.session.opened, ..Marks::default() }; + } + for e in marks.entries.values_mut() { + e.land(); + } + marks.running = marks.entries.values().filter(|e| e.job.is_some()).count(); + if marks.revision == Some(app.session.revision) { + return; + } + app.syncmark.revision = Some(app.session.revision); + let live: std::collections::HashSet<(u64, u64)> = + app.session.project.maps.iter().flat_map(|m| m.layers.iter().map(move |l| (m.instance_id(), l.instance_id()))).collect(); + app.syncmark.entries.retain(|id, _| live.contains(id)); +} + +#[cfg(test)] +mod tests { + use super::*; + use rustgis_engine::data::sync::FileSync; + + /// An entry whose hash, for the layer as it has it or an earlier state, has landed. + fn landed(current: bool) -> Entry { + let slot = crate::Slot::default(); + *slot.lock().unwrap() = Some(Ok(7)); + Entry { + rows: Weak::new(), + fields: Vec::new(), + crs: None, + synced: Synced { files: FileSync { files: Vec::new() }, rows: 7 }, + digest: None, + job: Some(Job { slot, stop: Arc::new(AtomicBool::new(false)), current }), + retry: None, + shown: Some(true), + } + } + + #[test] + fn a_landed_hash_frees_its_turn_and_answers_only_for_the_state_it_hashed() { + let mut e = landed(true); + e.land(); + assert!(e.job.is_none(), "no longer counted as running"); + assert_eq!(e.digest, Some(Ok(7))); + let mut e = landed(false); + e.land(); + assert!(e.job.is_none()); + assert_eq!(e.digest, None, "an earlier state's hash doesn't answer"); + assert_eq!(e.answer(), Some(true), "the previous answer stays shown until this state's lands"); + let mut e = landed(true); + *e.job.as_ref().unwrap().slot.lock().unwrap() = Some(Err("the hash stopped".into())); + e.land(); + assert_eq!(e.answer(), None, "a failed hash is unknown, not the previous state's answer"); + } +} diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index b8514014af..8b57169b8d 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -397,6 +397,14 @@ can't check it. On the desktop, make the layer active and choose **Source File State** in Command Search; what it found shows in the status bar. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. +The desktop's Layers pane also marks such a layer as it goes: **(edits not published)** once it +holds rows its file lacks (its rows are hashed in the background while the pane shows the layer, +once after the project opens and again after each edit), and **(changed on disk)** when the file changes, by the same look at its +metadata as for linked files, never its data. A file whose length differs from the record, or +that gained or lost a part, is marked even when it changed while the project was closed; one +changed then without a change of length is found by Source File State or Publish. The row's +tooltip says what to do; no notification offers Reload Data, which can't read such a layer. + ### Publish to source `source.publish` writes a layer's edits back over the Shapefile or `.geojson` file it was read