From 1cbc5dd1285ed5868f68b67a52c872d1653a3ef5 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 19:07:09 -0600 Subject: [PATCH 1/7] feat(data): remember the file a layer was read from and say when they differ A layer read whole from a Shapefile or .geojson file records each part of the file (size, modification time, SHA-256) and a hash of its rows as read, taken around the read so a concurrent write leaves no record. The new source.syncState command reports inSync, edited, sourceChanged or both, with the parts that changed. First step of publishing edits back (#1385). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/data/Cargo.toml | 2 + crates/data/src/dbf.rs | 8 +- crates/data/src/lib.rs | 4 + crates/data/src/source/mod.rs | 1 + crates/data/src/sync.rs | 218 +++++++++++++++++++++++++++++++ crates/engine/src/cmd/items.rs | 1 + crates/engine/src/cmd/layer.rs | 39 +++++- crates/engine/src/cmd/layout.rs | 1 + crates/engine/src/cmd/mod.rs | 2 + crates/engine/src/cmd/publish.rs | 48 +++++++ crates/engine/src/tests.rs | 43 ++++++ crates/map/src/lib.rs | 7 + crates/map/src/source.rs | 33 ++++- docs/user-guide/index.md | 13 ++ 15 files changed, 411 insertions(+), 10 deletions(-) create mode 100644 crates/data/src/sync.rs create mode 100644 crates/engine/src/cmd/publish.rs diff --git a/Cargo.lock b/Cargo.lock index b45e0e7749..6db26f0570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4772,6 +4772,7 @@ dependencies = [ "ruzstd", "serde", "serde_json", + "sha2 0.11.1", "shapefile", "sysinfo", "tempfile", diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index c6fcfa2b8b..6ff9ebd2b9 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -57,6 +57,8 @@ tokio-util = { workspace = true } rustls = { workspace = true } webpki-roots = { workspace = true } # Native web clients verify servers with the system's trust (see `http`). +# Hashes a source's files, to tell whether they changed since a layer was read (`sync`). +sha2 = { workspace = true } ureq = { version = "3", default-features = false, features = ["rustls", "gzip", "platform-verifier"] } # Private temporary workspaces for downloaded JPEG 2000 conversion through GDAL. tempfile = "3" diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index f7960c94c3..1968c7a4cc 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -181,11 +181,13 @@ pub(crate) fn read_cpg(path: &std::path::Path, notes: &mut Vec) -> Optio } } +/// Every ASCII case of the `.cpg` extension, the conventional spellings first: tried in turn, +/// without scanning a potentially large directory, and the first there wins. +pub(crate) const CPG_SPELLINGS: [&str; 8] = ["cpg", "CPG", "Cpg", "cPg", "cpG", "CPg", "CpG", "cPG"]; + fn try_read_cpg(path: &std::path::Path) -> std::io::Result>> { use std::io::Read; - // Check every ASCII case combination without scanning a potentially large directory. - // Prefer the conventional spellings if multiple sidecars exist. - for ext in ["cpg", "CPG", "Cpg", "cPg", "cpG", "CPg", "CpG", "cPG"] { + for ext in CPG_SPELLINGS { match std::fs::File::open(path.with_extension(ext)) { Ok(file) => { let mut bytes = Vec::new(); diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 60d0de8915..1b4a96690a 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -44,6 +44,7 @@ pub mod sheets; pub mod source; pub mod source_layers; pub mod sqlite; +pub mod sync; pub mod threads; pub mod time; pub mod topojson; @@ -377,6 +378,9 @@ impl Serialize for Dataset { pub struct ReadReport { pub skipped: usize, pub messages: Vec, + /// The files read, as they were, when the reader recorded them ([`sync::FileSync::around`]): + /// what a layer kept in the project compares with its source before writing edits back. + pub synced: Option, } /// What a read says when it drops Z or M values (see [`Dataset::zm_dropped`]): from `features` of diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index 5ecd24b48c..939c5e2c31 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -562,6 +562,7 @@ fn materialize_within(src: &dyn FeatureSource, budget: u64) -> Result<(Dataset, let report = ReadReport { skipped: skipped + src.skipped_at_open(), messages: info.notes.iter().map(|n| format!("{}: {n}", info.name)).collect(), + synced: None, }; if ds.geometry_type == GeometryType::None && let Some(g) = ds.features.iter().find_map(|f| f.geometry.as_ref()) diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs new file mode 100644 index 0000000000..2d32d759aa --- /dev/null +++ b/crates/data/src/sync.rs @@ -0,0 +1,218 @@ +//! The files a layer was read from, as they were when it was read ([`FileSync`]): what a layer +//! kept in the project compares with its source before its edits are written back to it, so +//! that writing them never overwrites a change made to the file since (#1385). + +use serde::{Deserialize, Serialize}; + +/// The files of a source as they were at a moment: each one's name, size, modification time +/// and SHA-256 hash, and the files that weren't there. A Shapefile is its `.shp`, `.shx`, +/// `.dbf`, `.prj` and `.cpg` (the ones its reader opens); a GeoJSON file is itself. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileSync { + /// Each file by its name beside the source (`parcels.dbf`), with its stamp, or `None` for + /// one that wasn't there, so a `.prj` added since counts as a change too. + pub files: Vec<(String, Option)>, +} + +/// One file as it was. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileStamp { + pub len: u64, + /// When it was last written, in nanoseconds since 1970 (`None` where the system doesn't + /// say). Only a shortcut: the hash decides whether the file changed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modified: Option, + /// The SHA-256 of its bytes, in hexadecimal. + pub sha256: String, +} + +/// Whether `path` names a source whose files a [`FileSync`] records: a Shapefile or a +/// `.geojson` file. A `.json` file isn't one yet: it may hold a city model or a topology, which +/// can't be written back as they were read. +pub fn is_syncable(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + [".shp", ".geojson"].iter().any(|e| lower.ends_with(e)) +} + +#[cfg(not(target_arch = "wasm32"))] +mod native { + use super::{FileStamp, FileSync}; + use std::path::{Path, PathBuf}; + + /// The files of the source at `path`, each as a name beside it and its path: for a + /// Shapefile the companions its reader opens (the `.cpg` in whichever case is there, else + /// `.cpg`), otherwise the file alone. + fn parts(path: &Path) -> Vec<(String, PathBuf)> { + let mut paths = vec![path.to_path_buf()]; + if path.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("shp")) { + for ext in ["shx", "dbf", "prj"] { + paths.push(path.with_extension(ext)); + } + let cpg = crate::dbf::CPG_SPELLINGS + .iter() + .map(|e| path.with_extension(e)) + .find(|p| p.is_file()) + .unwrap_or_else(|| path.with_extension("cpg")); + paths.push(cpg); + } + paths.into_iter().map(|p| (p.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), p)).collect() + } + + /// How recent a modification time is too recent to trust: a file system that keeps times + /// to the second or two (FAT, some network shares) can give a write made just after it + /// the same time. + const FRESH: std::time::Duration = std::time::Duration::from_secs(3); + + /// The file as it is now, hashed, or `None` when it isn't there. Its time is left out + /// when it is too recent to tell a later write by ([`FRESH`]), so that the file is always + /// hashed when compared. + fn stamp(path: &Path) -> std::io::Result> { + let m = match std::fs::metadata(path) { + Ok(m) if m.is_file() => m, + Ok(_) => return Err(std::io::Error::other(format!("{} isn't a file", path.display()))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + let modified = m.modified().ok().filter(|t| t.elapsed().is_ok_and(|age| age >= FRESH)); + let modified = modified.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).and_then(|d| u64::try_from(d.as_nanos()).ok()); + Ok(Some(FileStamp { len: m.len(), modified, sha256: sha256(path)? })) + } + + /// The SHA-256 of the file's bytes, read a block at a time. + fn sha256(path: &Path) -> std::io::Result { + use sha2::Digest; + use std::io::Read; + let mut file = std::fs::File::open(path)?; + let mut hasher = sha2::Sha256::new(); + let mut buf = vec![0u8; 1 << 16]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + } + Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect()) + } + + impl FileSync { + /// The files of the source at `path` as they are now, each read whole to hash it. + pub fn read(path: &Path) -> std::io::Result { + let files = parts(path).into_iter().map(|(name, p)| Ok((name, stamp(&p)?))).collect::>()?; + Ok(FileSync { files }) + } + + /// Runs `read`, which reads the source at `path`, and returns with its result the files + /// as it read them: hashed before and after, so a write while it ran is caught whatever + /// the file times say. `None` when they can't be read or were written to: what was read + /// may then be a mix, and there is nothing to compare with later. + pub fn around(path: &Path, read: impl FnOnce() -> R) -> (R, Option) { + let before = FileSync::read(path).ok(); + let out = read(); + let content = + |s: &FileSync| s.files.iter().map(|(n, s)| (n.clone(), s.as_ref().map(|s| (s.len, s.sha256.clone())))).collect::>(); + let sync = before.and_then(|before| FileSync::read(path).ok().filter(|after| content(after) == content(&before))); + (out, sync) + } + + /// The names of the files that differ now from this record: changed, added, removed or + /// unreadable. Empty when the source is as it was. A file whose size and recorded + /// modification time are both unchanged is taken as unchanged without reading it; any + /// other of the same size is hashed, so a touch or a copy isn't counted as a change. + pub fn changed(&self, path: &Path) -> Vec { + let now = parts(path); + let mut changed = Vec::new(); + // Files the set has now that the record lacks (a `.cpg` in another case). + for (name, _) in &now { + if !self.files.iter().any(|(n, _)| n == name) { + changed.push(name.clone()); + } + } + for (name, was) in &self.files { + let Some((_, p)) = now.iter().find(|(n, _)| n == name) else { + // Out of the set now: changed if it held anything. + if was.is_some() { + changed.push(name.clone()); + } + continue; + }; + let same = match (was, std::fs::metadata(p)) { + (None, Err(e)) => e.kind() == std::io::ErrorKind::NotFound, + (Some(was), Ok(m)) if m.is_file() && m.len() == was.len => { + let modified = m + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .and_then(|d| u64::try_from(d.as_nanos()).ok()); + (was.modified.is_some() && modified == was.modified) || sha256(p).is_ok_and(|h| h == was.sha256) + } + _ => false, + }; + if !same { + changed.push(name.clone()); + } + } + changed + } + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn a_shapefile_set_records_its_companions_and_the_missing_ones() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + std::fs::write(d.join("a.cpg"), "UTF-8").unwrap(); + let path = d.join("a.shp"); + let sync = FileSync::read(&path).unwrap(); + let names: Vec<(&str, bool)> = sync.files.iter().map(|(n, s)| (n.as_str(), s.is_some())).collect(); + assert_eq!(names, [("a.shp", true), ("a.shx", true), ("a.dbf", true), ("a.prj", false), ("a.cpg", true)]); + assert!(sync.changed(&path).is_empty()); + } + + #[test] + fn changes_are_named_and_a_touch_is_not_one() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + let path = d.join("a.shp"); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + let sync = FileSync::read(&path).unwrap(); + // The same bytes written again (a new time): read and hashed, not a change. + std::thread::sleep(std::time::Duration::from_millis(20)); + std::fs::write(d.join("a.dbf"), "dbf").unwrap(); + assert!(sync.changed(&path).is_empty()); + // Other bytes of the same length, a file added and one removed. + std::fs::write(d.join("a.dbf"), "DBF").unwrap(); + std::fs::write(d.join("a.prj"), "GEOGCS").unwrap(); + std::fs::remove_file(d.join("a.shx")).unwrap(); + assert_eq!(sync.changed(&path), ["a.shx", "a.dbf", "a.prj"]); + } + + #[test] + fn a_write_while_reading_leaves_no_record() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + let path = d.join("a.geojson"); + std::fs::write(&path, "{}").unwrap(); + let ((), sync) = FileSync::around(&path, || std::fs::write(&path, "{ }").unwrap()); + assert_eq!(sync, None); + let ((), sync) = FileSync::around(&path, || ()); + assert_eq!(sync.map(|s| s.files.len()), Some(1)); + } + + #[test] + fn a_record_round_trips_without_an_unknown_time() { + let sync = FileSync { files: vec![("a.geojson".into(), Some(FileStamp { len: 2, modified: None, sha256: "ab".into() }))] }; + let text = serde_json::to_string(&sync).unwrap(); + assert_eq!(text, r#"{"files":[["a.geojson",{"len":2,"sha256":"ab"}]]}"#); + assert_eq!(serde_json::from_str::(&text).unwrap(), sync); + } +} diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 16b82d57a3..c4993fa232 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -659,6 +659,7 @@ fn template(p: &Project) -> (Project, TemplateLosses) { // A feature-less template has no loaded rows to save back to a source. l.postgis = None; l.gpkg = None; + l.synced = None; if let Some(rl) = &mut l.raster { rl.embedded = None; } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 27835cadc4..bbc13cba0d 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1401,8 +1401,11 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult { let src = rustgis_data::source::shp::ShpSource::open_path_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; return add_source_with(s, p, Origin::Path(path), Some(Box::new(src)), None); } - let mut read = rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; + let (read, synced) = + rustgis_data::sync::FileSync::around(Path::new(path), || rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs))); + let mut read = read.map_err(|e| e.to_string())?; read.0.source = Some(path.to_string()); + read.1 = unsynced_note(read.1, synced, path); return add_read(s, p, read, None); } // A large Shapefile is read in place, as a GeoPackage is; smaller ones are loaded whole. @@ -1580,10 +1583,34 @@ fn read_whole(p: &Value, path: &str) -> WholeRead { } std::fs::read(path).map_err(|e| format!("{path}: {e}")) })?; - match delimited { - Some(read) => Ok(read), - None => rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}")), + if let Some(read) = delimited { + return Ok(read); } + let read = || rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}")); + // A file its edits can be written back to: the files as read, to compare with before then. + #[cfg(not(target_arch = "wasm32"))] + if rustgis_data::sync::is_syncable(path) { + let (read, synced) = rustgis_data::sync::FileSync::around(Path::new(path), read); + return read.map(|(ds, report)| (ds, unsynced_note(report, synced, path))); + } + read() +} + +/// `report` with the files as read, or, when they couldn't be recorded, a note saying that the +/// layer can't be compared with its file later. +#[cfg(not(target_arch = "wasm32"))] +fn unsynced_note( + mut report: rustgis_data::ReadReport, + synced: Option, + path: &str, +) -> rustgis_data::ReadReport { + if synced.is_none() { + report.messages.push(format!( + "{path}: changed while it was read, or a part of it couldn't be read, so RustGIS can't tell later whether the file changed since; add it again to compare it" + )); + } + report.synced = synced; + report } /// Add Data's read of the file `p` names, made before it runs ([`super::reads`]): a raster read @@ -4731,7 +4758,11 @@ pub(crate) fn add_read( } let n = ds.features.len(); let empty_before = super::map_is_empty(s); + let synced = report.synced.map(|files| rustgis_map::source::Synced { files, rows: rustgis_map::source::rows_digest(&ds) }); let id = add_dataset(s, ds); + if let Some(l) = s.map_mut().layer_mut(id) { + l.synced = synced; + } if zoom.unwrap_or(empty_before) && let Some(e) = s.map().layer(id).map(|l| l.extent()) { diff --git a/crates/engine/src/cmd/layout.rs b/crates/engine/src/cmd/layout.rs index e6c9b80541..d8789110dc 100644 --- a/crates/engine/src/cmd/layout.rs +++ b/crates/engine/src/cmd/layout.rs @@ -1686,6 +1686,7 @@ fn publication(s: &mut Session, p: &Value) -> CmdResult { // Simplified sketches aren't the source's rows: saving them would overwrite it. layer.gpkg = None; layer.postgis = None; + layer.synced = None; layer.dataset.features = l .dataset .features diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 3b163efc13..1b51c6c684 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -36,6 +36,7 @@ pub mod plugins; pub mod portals; pub mod presentation; pub mod project; +pub mod publish; pub mod qgis; pub mod raster; pub mod reads; @@ -112,6 +113,7 @@ pub fn all() -> Vec { v.extend(sql::commands()); v.extend(pg::commands()); v.extend(gpkg::commands()); + v.extend(publish::commands()); v.extend(plugins::commands()); v.extend(sharing::commands()); v.extend(stac::commands()); diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs new file mode 100644 index 0000000000..7220a3b3ab --- /dev/null +++ b/crates/engine/src/cmd/publish.rs @@ -0,0 +1,48 @@ +//! A layer kept in the project and the file it was read from: whether its edits and the file are +//! in step (#1385). + +use crate::{CmdResult, CommandSpec as C, Session}; +use serde_json::Value; + +pub fn commands() -> Vec { + vec![ + C::new("source.syncState", "Source File State", "Data › Data Source", sync_state).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( + r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile or .geojson file"} — whether it has edits its file lacks ("edited"), the file changed since it was read ("sourceChanged"), both ("both") or neither ("inSync"), and which of the file's parts changed"#, + ), + ] +} + +/// Compares the layer with the files it was read from. Runs on the UI thread: it reads the +/// files' sizes and times, hashing only a file of the same size whose time changed or was too +/// recent to record (a copy, a touch, a file just written), and finds edits by hashing the +/// layer's rows, as `gpkg.pendingEdits` does, for no more rows than a layer read whole holds. +fn sync_state(s: &mut Session, p: &Value) -> CmdResult { + let id = s.layer_id(p)?; + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { + if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { + return Err(format!( + "{} has no record of its file as it was read (it was added before RustGIS kept one, the file changed while it was read, or an older RustGIS saved the project): add the file again to compare it", + l.name + )); + } + return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); + }; + let edited = rustgis_map::source::rows_digest(&l.dataset) != synced.rows; + #[cfg(target_arch = "wasm32")] + { + let _ = (edited, path); + Err("the web app can't read files on this computer".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + let changed = synced.files.changed(std::path::Path::new(path)); + let state = match (edited, !changed.is_empty()) { + (false, false) => "inSync", + (true, false) => "edited", + (false, true) => "sourceChanged", + (true, true) => "both", + }; + Ok(serde_json::json!({"layer": id, "path": path, "state": state, "edited": edited, "changed": changed})) + } +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a5cbe4c7e5..02513e8c68 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39253,3 +39253,46 @@ fn a_model_trained_within_the_work_limit_says_so() { .unwrap(); assert_eq!(r["epochs"], 3, "{r}"); } + +/// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read +/// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed +/// since, and neither once an edit is undone. The record is saved with the project. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { + let dir = std::env::temp_dir().join(format!("rustgis-engine-sync-state-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut s = world(); + for file in ["towns.shp", "towns.geojson"] { + let path = dir.join(file); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": path})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path, "name": file})).unwrap()["layer"].as_u64().unwrap(); + let state = |s: &mut Session| s.run("source.syncState", &json!({"layer": id})).unwrap(); + assert_eq!(state(&mut s)["state"], "inSync", "{file}"); + s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + assert_eq!(state(&mut s)["state"], "edited", "{file}"); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(state(&mut s)["state"], "inSync", "{file}"); + // Saved and opened again, it still knows the file as read. + let l = s.map().layer(id as u32).unwrap(); + let reopened: rustgis_map::Layer = serde_json::from_value(serde_json::to_value(l).unwrap()).unwrap(); + assert_eq!(reopened.synced, l.synced); + assert!(l.synced.is_some()); + } + // The same bytes written again is no change; other bytes are, named by file. + let dbf = dir.join("towns.dbf"); + let bytes = std::fs::read(&dbf).unwrap(); + std::fs::write(&dbf, &bytes).unwrap(); + assert_eq!(s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap()["state"], "inSync"); + let mut other = bytes.clone(); + *other.last_mut().unwrap() ^= 1; + std::fs::write(&dbf, other).unwrap(); + let state = s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + assert_eq!((state["state"].as_str(), state["changed"].clone()), (Some("sourceChanged"), json!(["towns.dbf"]))); + s.run("edit.setValue", &json!({"layer": "towns.shp", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + assert_eq!(s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap()["state"], "both"); + // A layer that wasn't read from such a file has nothing to compare with. + assert!(s.run("source.syncState", &json!({"layer": "Cities"})).is_err()); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 98e7b958fb..1e130cd7b6 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -229,6 +229,11 @@ pub struct Layer { /// The GeoPackage table this layer's edits save to. #[serde(default, skip_serializing_if = "Option::is_none")] pub gpkg: Option, + /// Set for a layer read whole from a Shapefile or GeoJSON file (its `dataset.source`): the + /// file and the layer as they were when last in step, to tell edits the file lacks and + /// changes made to the file since. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub synced: Option, /// The saved connection it was added from (its id, in the app's preferences): a layer whose /// source is read again logs in with that connection's login, and a project opened where /// the connection or its login is missing asks for one. Empty for a layer that came from no @@ -462,6 +467,7 @@ impl Layer { source: None, postgis: None, gpkg: None, + synced: None, connection: String::new(), anonymous: false, projected: Default::default(), @@ -2927,6 +2933,7 @@ impl Layer { source, postgis, gpkg, + synced: _, connection: _, anonymous: _, instance_id: _, diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 3e5e847fe8..70431fad9e 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -238,6 +238,14 @@ pub struct GpkgEdit { pub versions: Versions, } +/// FNV-1a's starting value. +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; + +/// FNV-1a over `bytes`, from `h`. +fn fnv1a(h: u64, bytes: &[u8]) -> u64 { + bytes.iter().fold(h, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)) +} + /// How a loaded row is told apart from the same row edited: a hash of its geometry and /// attributes (FNV-1a over their JSON, the same in every build). Non-finite attributes also /// contribute their position and native bits, since output JSON substitutes Null for them. @@ -247,7 +255,7 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { impl Fnv { /// Feeds bytes without buffering; also used for values JSON cannot distinguish. fn bytes(&mut self, buf: &[u8]) { - self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + self.0 = fnv1a(self.0, buf); } } impl std::io::Write for Fnv { @@ -259,10 +267,10 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { Ok(()) } } - let mut h = Fnv(0xcbf2_9ce4_8422_2325); + let mut h = Fnv(FNV_OFFSET); // As before, a row that can't be written hashes as no bytes. if serde_json::to_writer(&mut h, &(&f.geometry, &f.attrs)).is_err() { - return 0xcbf2_9ce4_8422_2325; + return FNV_OFFSET; } for (index, value) in f.attrs.iter().enumerate() { if let rustgis_data::Value::Float(number) = value @@ -276,6 +284,25 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { h.0 } +/// A layer kept in the project that was read whole from a file it can be written back to (a +/// Shapefile or GeoJSON file), as it and the file were at their last sync, when it was read: so +/// it can tell whether it has edits the file lacks and whether the file changed since (#1385). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Synced { + /// The files, as they were. + pub files: rustgis_data::sync::FileSync, + /// The layer's [`rows_digest`] then. + pub rows: u64, +} + +/// A hash of what a dataset holds: its fields, coordinate system, and each row's [`row_hash`] +/// in order. Not its name or where it came from, which are no edit to the file. +pub fn rows_digest(ds: &rustgis_data::Dataset) -> u64 { + let schema = serde_json::to_vec(&(&ds.fields, &ds.crs)).unwrap_or_default(); + let h = fnv1a(fnv1a(FNV_OFFSET, &schema), &(ds.features.len() as u64).to_le_bytes()); + ds.features.iter().fold(h, |h, f| fnv1a(h, &row_hash(f).to_le_bytes())) +} + /// A loaded row: its source version or fingerprint and [`row_hash`] as read. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RowVersion { diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 91d15a3179..48de67376b 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -302,6 +302,19 @@ the default and discarding edits defaults to false. Only an explicit `{"layer":42,"path":"/data/cities.fgb","table":null}`; `table` is a string for a selected table or raster subdataset. +### Source file state + +A layer read whole from a Shapefile or a `.geojson` file keeps its features in the project, so +edits change the project, not the file. It also remembers the file as it was read: each part's +size, modification time and SHA-256 hash, and a hash of the layer's rows. `source.syncState` +compares the two and answers `inSync`, `edited` (the layer has edits the file lacks), +`sourceChanged` (the file changed since it was read) or `both`, with the names of the parts +that changed. Writing the same bytes again isn't a change, and undoing an edit makes the layer +`inSync` again. A file written to while it was being read, a layer read in place and a `.json` +file have no record. The record is saved with the project; the web app can't check it. + +CLI and Python: `source.syncState` with `{"layer":"parcels"}`. + ## Keyboard shortcuts | Keys | Action | From b2c4ab0ece755bb719ff23e6da10a7b26007f883 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 20:11:58 -0600 Subject: [PATCH 2/7] fix(data): hash every same-size part and check a layer's file off the UI thread source.syncState now reads and hashes every part of the recorded size, so a rewrite that puts the size and time back is caught; the record keeps only sizes and hashes. On the desktop the hashing reads ahead on a thread of its own (as Reload Data and the Error Inspector do), with progress and Cancel. Also: a removed .CPG is named as spelled, the row digest hashes rows as a project saves them (NaN as null), and the test saves and opens a real project. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/sync.rs | 154 ++++++++++++++++----------- crates/engine/src/cmd/publish.rs | 51 ++++++++- crates/engine/src/cmd/reads.rs | 7 +- crates/engine/src/policy/dispatch.rs | 5 +- crates/engine/src/tests.rs | 24 ++++- crates/map/src/source.rs | 23 +++- crates/ui-egui/src/lib.rs | 4 + crates/ui-egui/src/readjob.rs | 2 +- docs/user-guide/index.md | 11 +- 9 files changed, 196 insertions(+), 85 deletions(-) diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index 2d32d759aa..e38b041008 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -4,9 +4,10 @@ use serde::{Deserialize, Serialize}; -/// The files of a source as they were at a moment: each one's name, size, modification time -/// and SHA-256 hash, and the files that weren't there. A Shapefile is its `.shp`, `.shx`, -/// `.dbf`, `.prj` and `.cpg` (the ones its reader opens); a GeoJSON file is itself. +/// The files of a source as they were at a moment: each one's name, size and SHA-256 hash, and +/// the files that weren't there. A Shapefile is its `.shp`, `.shx`, `.dbf`, `.prj` and `.cpg` +/// (the ones its reader opens); a GeoJSON file is itself. Only the contents count: a file +/// copied, touched or moved to another disk with the same bytes is the same file. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct FileSync { /// Each file by its name beside the source (`parcels.dbf`), with its stamp, or `None` for @@ -18,10 +19,6 @@ pub struct FileSync { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct FileStamp { pub len: u64, - /// When it was last written, in nanoseconds since 1970 (`None` where the system doesn't - /// say). Only a shortcut: the hash decides whether the file changed. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub modified: Option, /// The SHA-256 of its bytes, in hexadecimal. pub sha256: String, } @@ -37,6 +34,7 @@ pub fn is_syncable(path: &str) -> bool { #[cfg(not(target_arch = "wasm32"))] mod native { use super::{FileStamp, FileSync}; + use crate::{DataError, Result}; use std::path::{Path, PathBuf}; /// The files of the source at `path`, each as a name beside it and its path: for a @@ -58,28 +56,19 @@ mod native { paths.into_iter().map(|p| (p.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), p)).collect() } - /// How recent a modification time is too recent to trust: a file system that keeps times - /// to the second or two (FAT, some network shares) can give a write made just after it - /// the same time. - const FRESH: std::time::Duration = std::time::Duration::from_secs(3); - - /// The file as it is now, hashed, or `None` when it isn't there. Its time is left out - /// when it is too recent to tell a later write by ([`FRESH`]), so that the file is always - /// hashed when compared. - fn stamp(path: &Path) -> std::io::Result> { - let m = match std::fs::metadata(path) { - Ok(m) if m.is_file() => m, - Ok(_) => return Err(std::io::Error::other(format!("{} isn't a file", path.display()))), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(e), - }; - let modified = m.modified().ok().filter(|t| t.elapsed().is_ok_and(|age| age >= FRESH)); - let modified = modified.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).and_then(|d| u64::try_from(d.as_nanos()).ok()); - Ok(Some(FileStamp { len: m.len(), modified, sha256: sha256(path)? })) + /// The file's size, or `None` when it isn't there. + fn len(path: &Path) -> Result> { + match std::fs::metadata(path) { + Ok(m) if m.is_file() => Ok(Some(m.len())), + Ok(_) => Err(DataError::Format(format!("{} isn't a file", path.display()))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } } - /// The SHA-256 of the file's bytes, read a block at a time. - fn sha256(path: &Path) -> std::io::Result { + /// The SHA-256 of the file's bytes, read a block at a time, each counted as progress + /// ([`crate::progress`]) and stopping once the run is cancelled. + fn sha256(path: &Path) -> Result { use sha2::Digest; use std::io::Read; let mut file = std::fs::File::open(path)?; @@ -91,43 +80,60 @@ mod native { break; } hasher.update(buf.get(..n).unwrap_or_default()); + crate::progress::step(n)?; } Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect()) } + /// Bytes as a progress count. + fn count(bytes: u64) -> usize { + usize::try_from(bytes).unwrap_or(usize::MAX) + } + impl FileSync { /// The files of the source at `path` as they are now, each read whole to hash it. - pub fn read(path: &Path) -> std::io::Result { - let files = parts(path).into_iter().map(|(name, p)| Ok((name, stamp(&p)?))).collect::>()?; + pub fn read(path: &Path) -> Result { + let parts = parts(path); + let total = parts.iter().filter_map(|(_, p)| std::fs::metadata(p).ok()).map(|m| m.len()).sum(); + crate::progress::begin(count(total)); + let mut files = Vec::new(); + for (name, p) in parts { + let stamp = match len(&p)? { + None => None, + Some(len) => Some(FileStamp { len, sha256: sha256(&p)? }), + }; + files.push((name, stamp)); + } Ok(FileSync { files }) } /// Runs `read`, which reads the source at `path`, and returns with its result the files - /// as it read them: hashed before and after, so a write while it ran is caught whatever - /// the file times say. `None` when they can't be read or were written to: what was read - /// may then be a mix, and there is nothing to compare with later. + /// as it read them: hashed before and after, so a write while it ran is caught. `None` + /// when they can't be read or were written to: what was read may then be a mix, and + /// there is nothing to compare with later. pub fn around(path: &Path, read: impl FnOnce() -> R) -> (R, Option) { let before = FileSync::read(path).ok(); let out = read(); - let content = - |s: &FileSync| s.files.iter().map(|(n, s)| (n.clone(), s.as_ref().map(|s| (s.len, s.sha256.clone())))).collect::>(); - let sync = before.and_then(|before| FileSync::read(path).ok().filter(|after| content(after) == content(&before))); + let sync = before.and_then(|before| FileSync::read(path).ok().filter(|after| *after == before)); (out, sync) } /// The names of the files that differ now from this record: changed, added, removed or - /// unreadable. Empty when the source is as it was. A file whose size and recorded - /// modification time are both unchanged is taken as unchanged without reading it; any - /// other of the same size is hashed, so a touch or a copy isn't counted as a change. - pub fn changed(&self, path: &Path) -> Vec { + /// unreadable. Empty when the source is as it was. A file of the size recorded is read + /// whole and hashed, so its work grows with the file: a caller on the desktop's UI + /// thread does it on a thread of its own. Errs only when the run is cancelled. + pub fn changed(&self, path: &Path) -> Result> { let now = parts(path); let mut changed = Vec::new(); - // Files the set has now that the record lacks (a `.cpg` in another case). - for (name, _) in &now { - if !self.files.iter().any(|(n, _)| n == name) { + // Files there now that the record lacks (a `.cpg` added in another case); not the + // `.cpg` name `parts` falls back to when there is none. + for (name, p) in &now { + if !self.files.iter().any(|(n, _)| n == name) && p.exists() { changed.push(name.clone()); } } + let total = self.files.iter().filter_map(|(_, s)| s.as_ref()).map(|s| s.len).sum(); + crate::progress::begin(count(total)); for (name, was) in &self.files { let Some((_, p)) = now.iter().find(|(n, _)| n == name) else { // Out of the set now: changed if it held anything. @@ -136,23 +142,20 @@ mod native { } continue; }; - let same = match (was, std::fs::metadata(p)) { - (None, Err(e)) => e.kind() == std::io::ErrorKind::NotFound, - (Some(was), Ok(m)) if m.is_file() && m.len() == was.len => { - let modified = m - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .and_then(|d| u64::try_from(d.as_nanos()).ok()); - (was.modified.is_some() && modified == was.modified) || sha256(p).is_ok_and(|h| h == was.sha256) - } + let same = match (was, len(p)) { + (None, Ok(None)) => true, + (Some(was), Ok(Some(len))) if len == was.len => match sha256(p) { + Ok(hash) => hash == was.sha256, + Err(DataError::Cancelled) => return Err(DataError::Cancelled), + Err(_) => false, + }, _ => false, }; if !same { changed.push(name.clone()); } } - changed + Ok(changed) } } } @@ -173,7 +176,7 @@ mod tests { let sync = FileSync::read(&path).unwrap(); let names: Vec<(&str, bool)> = sync.files.iter().map(|(n, s)| (n.as_str(), s.is_some())).collect(); assert_eq!(names, [("a.shp", true), ("a.shx", true), ("a.dbf", true), ("a.prj", false), ("a.cpg", true)]); - assert!(sync.changed(&path).is_empty()); + assert!(sync.changed(&path).unwrap().is_empty()); } #[test] @@ -185,15 +188,44 @@ mod tests { std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); } let sync = FileSync::read(&path).unwrap(); - // The same bytes written again (a new time): read and hashed, not a change. - std::thread::sleep(std::time::Duration::from_millis(20)); + // The same bytes written again: not a change. std::fs::write(d.join("a.dbf"), "dbf").unwrap(); - assert!(sync.changed(&path).is_empty()); + assert!(sync.changed(&path).unwrap().is_empty()); // Other bytes of the same length, a file added and one removed. std::fs::write(d.join("a.dbf"), "DBF").unwrap(); std::fs::write(d.join("a.prj"), "GEOGCS").unwrap(); std::fs::remove_file(d.join("a.shx")).unwrap(); - assert_eq!(sync.changed(&path), ["a.shx", "a.dbf", "a.prj"]); + assert_eq!(sync.changed(&path).unwrap(), ["a.shx", "a.dbf", "a.prj"]); + } + + /// A copy that keeps times (`cp -p`, `rsync -t`) can put back the size and time a file had + /// with other bytes: the hash still tells. + #[test] + fn a_rewrite_that_puts_the_time_back_is_a_change() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a.geojson"); + std::fs::write(&path, "{}").unwrap(); + let sync = FileSync::read(&path).unwrap(); + let time = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, "[]").unwrap(); + std::fs::File::options().write(true).open(&path).unwrap().set_modified(time).unwrap(); + assert_eq!(sync.changed(&path).unwrap(), ["a.geojson"]); + } + + #[test] + fn a_removed_cpg_is_named_as_it_was_spelled() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + // Recorded under a spelling no file has now: as `a.CPG` is once removed on a + // case-sensitive file system (one that ignores case would find `a.cpg` there). + let path = d.join("a.shp"); + let mut sync = FileSync::read(&path).unwrap(); + let cpg = sync.files.iter_mut().find(|(n, _)| n == "a.cpg").unwrap(); + *cpg = ("a.CPG".into(), Some(FileStamp { len: 5, sha256: "00".into() })); + assert_eq!(sync.changed(&path).unwrap(), ["a.CPG"]); } #[test] @@ -209,10 +241,10 @@ mod tests { } #[test] - fn a_record_round_trips_without_an_unknown_time() { - let sync = FileSync { files: vec![("a.geojson".into(), Some(FileStamp { len: 2, modified: None, sha256: "ab".into() }))] }; + fn a_record_round_trips() { + let sync = FileSync { files: vec![("a.geojson".into(), Some(FileStamp { len: 2, sha256: "ab".into() })), ("a.prj".into(), None)] }; let text = serde_json::to_string(&sync).unwrap(); - assert_eq!(text, r#"{"files":[["a.geojson",{"len":2,"sha256":"ab"}]]}"#); + assert_eq!(text, r#"{"files":[["a.geojson",{"len":2,"sha256":"ab"}],["a.prj",null]]}"#); assert_eq!(serde_json::from_str::(&text).unwrap(), sync); } } diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 7220a3b3ab..e20293c28e 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -12,10 +12,45 @@ pub fn commands() -> Vec { ] } -/// Compares the layer with the files it was read from. Runs on the UI thread: it reads the -/// files' sizes and times, hashing only a file of the same size whose time changed or was too -/// recent to record (a copy, a touch, a file just written), and finds edits by hashing the -/// layer's rows, as `gpkg.pendingEdits` does, for no more rows than a layer read whole holds. +/// The key a check of layer `id`'s files made ahead is taken by (see [`super::reads`]). +#[cfg(not(target_arch = "wasm32"))] +fn ahead_key(id: u32) -> String { + format!("syncState:{id}") +} + +/// A check of a layer's files made ahead: the record and path checked, and what changed. +#[cfg(not(target_arch = "wasm32"))] +struct Checked { + files: rustgis_data::sync::FileSync, + path: String, + changed: Result, String>, +} + +/// The check `source.syncState` makes before it runs in the desktop app: every part of the +/// file read and hashed on a thread of its own, with progress and Cancel. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn sync_ahead(s: &Session, p: &Value) -> Option { + let id = s.layer_id(p).ok()?; + let l = s.map().layer(id)?; + let (files, path) = (l.synced.as_ref()?.files.clone(), l.dataset.source.clone()?); + Some(super::reads::ahead("Source File State", ahead_key(id), move || { + let changed = files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string()); + Checked { files, path, changed } + })) +} + +#[cfg(target_arch = "wasm32")] +pub(crate) fn sync_ahead(_: &Session, _: &Value) -> Option { + None +} + +/// Compares the layer with the files it was read from: each part of the size recorded is read +/// and hashed, so the work grows with the file. The desktop app checks them on a thread of its +/// own first ([`sync_ahead`]), with progress and **Cancel**, and the command takes that check as +/// it lands; when the layer's record changed meanwhile it says so rather than check again here, +/// which would hold the window. The assistant, scripts and the command line check where they +/// ask. Edits are found by hashing the layer's rows here, as `gpkg.pendingEdits` does: no more +/// rows than a layer read whole from a file of at most [`rustgis_data::io::MAX_BYTES`] holds. fn sync_state(s: &mut Session, p: &Value) -> CmdResult { let id = s.layer_id(p)?; let l = s.map().layer(id).ok_or("the layer no longer exists")?; @@ -36,7 +71,13 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - let changed = synced.files.changed(std::path::Path::new(path)); + let (files, path) = (synced.files.clone(), path.clone()); + let changed = match super::reads::take::(s, &ahead_key(id)) { + Some(checked) if checked.files == files && checked.path == path => checked.changed?, + // Checking again here would hold the window: the check made on its thread is dropped. + Some(_) => return Err("the layer's file or its record changed while it was checked: check again".into()), + None => files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?, + }; let state = match (edited, !changed.is_empty()) { (false, false) => "inSync", (true, false) => "edited", diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index 935028a3d1..0fbbe4d1c9 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -1,8 +1,8 @@ //! Reads the desktop app makes on a thread of its own before a command adds what they read //! (#1291 step 3): Add Data's whole-file reads (a raster, a raster GDAL converts, a GeoJSON, //! delimited text or spreadsheet read whole, a CityJSON or CityGML model's layers), Reload Data, Load All -//! Features, Convert To Annotation's read of a layer read in place, native models or scripts and the -//! Error Inspector's topology check (#1291). [`read_ahead`] +//! Features, Convert To Annotation's read of a layer read in place, native models or scripts, the +//! Error Inspector's topology check (#1291) and Source File State's hashing of a layer's file. [`read_ahead`] //! takes what the read needs of the session, [`ReadAhead::run`] reads on the thread, and [`read_land`] //! runs the admitted command under its initiating owner: the command takes //! it instead of reading again, so what it adds, says, logs and puts in Undo is the same as when @@ -139,7 +139,8 @@ pub(crate) fn prepare(s: &Session, op: Authorized, owner: Continuatio (name, work, None) } "topology.validate" => ("Error Inspector", super::design::topology_ahead(s, params), None), - _ => return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()), + "source.syncState" => ("Source File State", super::publish::sync_ahead(s, params), None), + _ => return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives, topology and Source File State prepare background work".into()), }; let name = work.as_ref().map_or(name, |work| work.name); Ok(ReadAhead { name, op, owner, source, work }) diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 8d21782204..993b19583b 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -151,11 +151,12 @@ impl Session { | "map.reloadTiles" | "map.repairTiles" | "topology.validate" + | "source.syncState" ) { - return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()); + return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives, topology and Source File State prepare background work".into()); } let mut params = params.as_object().cloned().ok_or("read parameters must be an object")?; - if matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") { + if matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert" | "source.syncState") { let id = self.layer_id(&Value::Object(params.clone()))?; params.insert("layer".into(), json!(id)); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 02513e8c68..07cced5d47 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39274,11 +39274,25 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { assert_eq!(state(&mut s)["state"], "edited", "{file}"); s.run("edit.undo", &Value::Null).unwrap(); assert_eq!(state(&mut s)["state"], "inSync", "{file}"); - // Saved and opened again, it still knows the file as read. - let l = s.map().layer(id as u32).unwrap(); - let reopened: rustgis_map::Layer = serde_json::from_value(serde_json::to_value(l).unwrap()).unwrap(); - assert_eq!(reopened.synced, l.synced); - assert!(l.synced.is_some()); + assert!(s.map().layer(id as u32).unwrap().synced.is_some()); + } + // Saved and opened again, both still know their files as read. + let project = dir.join("towns.rgis"); + s.run("project.saveAs", &json!({"path": project.display().to_string()})).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.open", &json!({"path": project.display().to_string()})).unwrap(); + for layer in ["towns.shp", "towns.geojson"] { + assert_eq!(t.run("source.syncState", &json!({"layer": layer})).unwrap()["state"], "inSync", "{layer}"); + } + // The desktop app hashes the file on a thread of its own; the command takes that check. + { + use crate::cmd::reads::{read_ahead, read_land}; + let here = t.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + let ahead = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + assert!(ahead.reads()); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + assert_eq!(read_land(&mut t, read).unwrap(), here); + assert!(t.read_ahead.is_none(), "the check was taken"); } // The same bytes written again is no change; other bytes are, named by file. let dbf = dir.join("towns.dbf"); diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 70431fad9e..4f923784df 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -295,12 +295,17 @@ pub struct Synced { pub rows: u64, } -/// A hash of what a dataset holds: its fields, coordinate system, and each row's [`row_hash`] -/// in order. Not its name or where it came from, which are no edit to the file. +/// A hash of what a dataset holds, as a project saves it: its fields, coordinate system, and +/// each row's geometry and attributes in order. Not its name or where it came from, which are +/// no edit to the file. A number that isn't finite hashes as the null a project saves it as, so +/// a layer is the same before and after its project is saved and opened again. pub fn rows_digest(ds: &rustgis_data::Dataset) -> u64 { let schema = serde_json::to_vec(&(&ds.fields, &ds.crs)).unwrap_or_default(); let h = fnv1a(fnv1a(FNV_OFFSET, &schema), &(ds.features.len() as u64).to_le_bytes()); - ds.features.iter().fold(h, |h, f| fnv1a(h, &row_hash(f).to_le_bytes())) + ds.features.iter().fold(h, |h, f| { + let row = serde_json::to_vec(&(&f.geometry, &f.attrs)).unwrap_or_default(); + fnv1a(fnv1a(h, &(row.len() as u64).to_le_bytes()), &row) + }) } /// A loaded row: its source version or fingerprint and [`row_hash`] as read. @@ -1726,6 +1731,18 @@ mod tests { } /// The hash is FNV-1a over the row's JSON, the same as the projects saved with it hold. + #[test] + fn a_rows_digest_is_the_same_after_a_project_round_trip() { + use rustgis_data::{Dataset, Feature, Field, FieldType, GeometryType, Value}; + let mut ds = Dataset::new("a".to_string(), GeometryType::None, vec![Field::new("v", FieldType::Double)]); + ds.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(f64::NAN)] }].into(); + let reopened: Dataset = serde_json::from_value(serde_json::to_value(&ds).unwrap()).unwrap(); + assert_eq!(super::rows_digest(&reopened), super::rows_digest(&ds)); + let mut edited = ds.clone(); + edited.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(1.0)] }].into(); + assert_ne!(super::rows_digest(&edited), super::rows_digest(&ds)); + } + #[test] fn row_hashes_are_fnv_over_the_rows_json() { use rustgis_data::{Coord, Feature, Geometry, Value}; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 756ad691f6..d5f04ba32b 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2261,6 +2261,10 @@ impl RustGisApp { let params = if params.is_null() { json!({}) } else { params }; return readjob::run(self, "topology.validate", params); } + // Comparing a layer with its file hashes the file: on a thread of its own. + if id == "source.syncState" { + return readjob::run(self, "source.syncState", params); + } // Converting the labels of a layer read in place reads all of its source first, on a // thread of its own, whoever asks (a click, the assistant). if id == "labels.convert" diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index a4ee355a4f..a31cf96e5b 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -78,7 +78,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: let r = app.run(command, p); return answered(app, then, r); } - let layer = matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") + let layer = matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert" | "source.syncState") .then(|| app.session.layer_id(&p).ok().and_then(|id| app.session.map().layer(id)).map(|l| (l.instance_id(), l.name.clone()))) .flatten(); let notice_mark = app.session.notice_mark(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 48de67376b..56b21139af 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -306,11 +306,12 @@ or raster subdataset. A layer read whole from a Shapefile or a `.geojson` file keeps its features in the project, so edits change the project, not the file. It also remembers the file as it was read: each part's -size, modification time and SHA-256 hash, and a hash of the layer's rows. `source.syncState` -compares the two and answers `inSync`, `edited` (the layer has edits the file lacks), -`sourceChanged` (the file changed since it was read) or `both`, with the names of the parts -that changed. Writing the same bytes again isn't a change, and undoing an edit makes the layer -`inSync` again. A file written to while it was being read, a layer read in place and a `.json` +size and SHA-256 hash, and a hash of the layer's rows. `source.syncState` compares the two and +answers `inSync`, `edited` (the layer has edits the file lacks), `sourceChanged` (the file +changed since it was read) or `both`, with the names of the parts that changed. Only the bytes +count: writing the same bytes again isn't a change, and a copy that puts back the file's size +and time is. Undoing an edit makes the layer `inSync` again. On the desktop the file is hashed +in the background, with progress and Cancel in the status bar. A file written to while it was being read, a layer read in place and a `.json` file have no record. The record is saved with the project; the web app can't check it. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. From 761b398926fe127f749e1564414b80d7becb78fb Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 21:54:11 -0600 Subject: [PATCH 3/7] fix(engine): keep Source File State's hashing off the UI thread and say what it found The row digest moves to rustgis_data::sync with the record, streams into the hash, counts rows and stops on Cancel. Add Data reads a small Shapefile ahead on its own thread, with or without a coordinate system, and makes the record with the read: the files hashed once just before it (a cancel there stops before reading), then the rows. source.syncState hashes file and rows ahead, refuses to land if the layer changed meanwhile, and on the desktop says its answer as a notice. The guide says each side is compared with its own record. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/lib.rs | 7 +-- crates/data/src/sync.rs | 92 ++++++++++++++++++++++++++------ crates/engine/src/cmd/layer.rs | 83 ++++++++++++++++++++-------- crates/engine/src/cmd/publish.rs | 67 +++++++++++++++-------- crates/engine/src/tests.rs | 24 +++++++++ crates/map/src/lib.rs | 2 +- crates/map/src/source.rs | 36 ------------- crates/ui-egui/src/lib.rs | 26 ++++++++- crates/ui-egui/src/readjob.rs | 17 ++++++ docs/user-guide/index.md | 22 +++++--- 10 files changed, 267 insertions(+), 109 deletions(-) diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 1b4a96690a..2edcfe4253 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -378,9 +378,10 @@ impl Serialize for Dataset { pub struct ReadReport { pub skipped: usize, pub messages: Vec, - /// The files read, as they were, when the reader recorded them ([`sync::FileSync::around`]): - /// what a layer kept in the project compares with its source before writing edits back. - pub synced: Option, + /// The files read and the rows read from them, when the read recorded them + /// ([`sync::FileSync::around`]): what a layer kept in the project compares with its source + /// before writing edits back. + pub synced: Option, } /// What a read says when it drops Z or M values (see [`Dataset::zm_dropped`]): from `features` of diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index e38b041008..456da0e412 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -23,6 +23,52 @@ pub struct FileStamp { pub sha256: String, } +/// A layer kept in the project that was read whole from a file it can be written back to (a +/// Shapefile or GeoJSON file), as it and the file were at their last sync, when it was read: so +/// it can tell whether it has edits the file lacks and whether the file changed since. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Synced { + /// The files, as they were. + pub files: FileSync, + /// The layer's [`rows_digest`] then. + pub rows: u64, +} + +/// A hash of what a dataset holds, as a project saves it: its fields, coordinate system, and +/// each row's geometry and attributes in order (FNV-1a over their JSON, written to the hash as +/// it is made). Not its name or where it came from, which are no edit to the file. A number that +/// isn't finite hashes as the null a project saves it as, so a layer is the same before and +/// after its project is saved and opened again. Its work grows with the rows: each is counted +/// as progress ([`crate::progress`]), and it stops once the run is cancelled; the desktop app +/// runs it off the UI thread. +pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { + /// FNV-1a over the bytes written to it. + struct Fnv(u64); + impl std::io::Write for Fnv { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + /// Writes `value`'s JSON to the hash (nothing, should it not write), then a byte JSON never + /// writes, so values can't run into each other. + fn put(h: &mut Fnv, value: &impl serde::Serialize) { + let _ = serde_json::to_writer(&mut *h, value); + h.0 = (h.0 ^ 0xff).wrapping_mul(0x0100_0000_01b3); + } + let mut h = Fnv(0xcbf2_9ce4_8422_2325); + put(&mut h, &(&ds.fields, &ds.crs)); + crate::progress::begin(ds.features.len()); + for f in ds.features.iter() { + put(&mut h, &(&f.geometry, &f.attrs)); + crate::progress::step(1)?; + } + Ok(h.0) +} + /// Whether `path` names a source whose files a [`FileSync`] records: a Shapefile or a /// `.geojson` file. A `.json` file isn't one yet: it may hold a city model or a topology, which /// can't be written back as they were read. @@ -108,14 +154,19 @@ mod native { } /// Runs `read`, which reads the source at `path`, and returns with its result the files - /// as it read them: hashed before and after, so a write while it ran is caught. `None` - /// when they can't be read or were written to: what was read may then be a mix, and - /// there is nothing to compare with later. - pub fn around(path: &Path, read: impl FnOnce() -> R) -> (R, Option) { - let before = FileSync::read(path).ok(); - let out = read(); - let sync = before.and_then(|before| FileSync::read(path).ok().filter(|after| *after == before)); - (out, sync) + /// as they were just before it ran (`None` when a part couldn't be read). Hashed before + /// the read, the record can only be older than what was read: a write while it ran, or + /// after, leaves the file unlike the record, so it shows as changed and is never written + /// over. That costs one more pass over the files than the read, about a second for a + /// part of the largest a reader takes (256 MiB) where SHA-256 runs at 250 MB/s. Errs, + /// without reading, when the run is cancelled while the files are hashed. + pub fn around(path: &Path, read: impl FnOnce() -> R) -> Result<(R, Option)> { + let before = match FileSync::read(path) { + Ok(sync) => Some(sync), + Err(DataError::Cancelled) => return Err(DataError::Cancelled), + Err(_) => None, + }; + Ok((read(), before)) } /// The names of the files that differ now from this record: changed, added, removed or @@ -229,15 +280,26 @@ mod tests { } #[test] - fn a_write_while_reading_leaves_no_record() { + fn a_write_while_reading_shows_as_a_change() { let dir = tempfile::tempdir().unwrap(); - let d = dir.path(); - let path = d.join("a.geojson"); + let path = dir.path().join("a.geojson"); std::fs::write(&path, "{}").unwrap(); - let ((), sync) = FileSync::around(&path, || std::fs::write(&path, "{ }").unwrap()); - assert_eq!(sync, None); - let ((), sync) = FileSync::around(&path, || ()); - assert_eq!(sync.map(|s| s.files.len()), Some(1)); + let ((), sync) = FileSync::around(&path, || std::fs::write(&path, "[]").unwrap()).unwrap(); + assert_eq!(sync.unwrap().changed(&path).unwrap(), ["a.geojson"]); + let ((), sync) = FileSync::around(&path, || ()).unwrap(); + assert!(sync.unwrap().changed(&path).unwrap().is_empty()); + } + + #[test] + fn a_rows_digest_is_the_same_after_a_project_round_trip() { + use crate::{Dataset, Feature, Field, FieldType, GeometryType, Value}; + let mut ds = Dataset::new("a".to_string(), GeometryType::None, vec![Field::new("v", FieldType::Double)]); + ds.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(f64::NAN)] }].into(); + let reopened: Dataset = serde_json::from_value(serde_json::to_value(&ds).unwrap()).unwrap(); + assert_eq!(rows_digest(&reopened).unwrap(), rows_digest(&ds).unwrap()); + let mut edited = ds.clone(); + edited.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(1.0)] }].into(); + assert_ne!(rows_digest(&edited).unwrap(), rows_digest(&ds).unwrap()); } #[test] diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index bbc13cba0d..d6406b7bb5 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1401,11 +1401,11 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult { let src = rustgis_data::source::shp::ShpSource::open_path_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; return add_source_with(s, p, Origin::Path(path), Some(Box::new(src)), None); } - let (read, synced) = - rustgis_data::sync::FileSync::around(Path::new(path), || rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs))); - let mut read = read.map_err(|e| e.to_string())?; - read.0.source = Some(path.to_string()); - read.1 = unsynced_note(read.1, synced, path); + // Read ahead in the desktop app ([`add_data_ahead`]). + let read = match super::reads::take(s, &whole_key(path)) { + Some(read) => read, + None => read_shapefile_with(path, name, crs), + }?; return add_read(s, p, read, None); } // A large Shapefile is read in place, as a GeoPackage is; smaller ones are loaded whole. @@ -1590,32 +1590,48 @@ fn read_whole(p: &Value, path: &str) -> WholeRead { // A file its edits can be written back to: the files as read, to compare with before then. #[cfg(not(target_arch = "wasm32"))] if rustgis_data::sync::is_syncable(path) { - let (read, synced) = rustgis_data::sync::FileSync::around(Path::new(path), read); - return read.map(|(ds, report)| (ds, unsynced_note(report, synced, path))); + return read_synced(path, read); } read() } -/// `report` with the files as read, or, when they couldn't be recorded, a note saying that the -/// layer can't be compared with its file later. +/// A Shapefile read whole in coordinate system `crs`, which replaces its `.prj`, with the record +/// of what was read ([`read_synced`]). #[cfg(not(target_arch = "wasm32"))] -fn unsynced_note( - mut report: rustgis_data::ReadReport, - synced: Option, - path: &str, -) -> rustgis_data::ReadReport { - if synced.is_none() { - report.messages.push(format!( - "{path}: changed while it was read, or a part of it couldn't be read, so RustGIS can't tell later whether the file changed since; add it again to compare it" - )); - } - report.synced = synced; - report +fn read_shapefile_with(path: &str, name: &str, crs: rustgis_data::raster::Crs) -> WholeRead { + read_synced(path, || { + let mut read = rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; + read.0.source = Some(path.to_string()); + Ok(read) + }) +} + +/// `read`, the read of the file at `path`, with the record of what it read: the files hashed +/// just before ([`rustgis_data::sync::FileSync::around`]) and the digest of the rows read. Its +/// work grows with the file, so the desktop app makes it off the UI thread ([`add_data_ahead`]). +/// When the files couldn't be hashed the layer has no record, and a note says it can't be +/// compared with its file later. +#[cfg(not(target_arch = "wasm32"))] +fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { + let (read, files) = rustgis_data::sync::FileSync::around(Path::new(path), read).map_err(|e| format!("{path}: {e}"))?; + let (ds, mut report) = read?; + report.synced = match files { + Some(files) => { + Some(rustgis_data::sync::Synced { files, rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))? }) + } + None => { + report.messages.push(format!( + "{path}: a part of it couldn't be read to record it, so RustGIS can't tell later whether the file changed since; add it again to compare it" + )); + None + } + }; + Ok((ds, report)) } /// Add Data's read of the file `p` names, made before it runs ([`super::reads`]): a raster read /// whole or converted by GDAL, a GeoJSON read whole, delimited text, a spreadsheet or a city model -/// (CityJSON or CityGML) read into its layers. `None` for +/// (CityJSON or CityGML) read into its layers, a Shapefile read whole. `None` for /// any other file, which [`add_data`] reads as it adds it. Each is picked only where `add_data` /// takes that read, so a read made is never wasted; a few it makes whole are left to it (a GeoJSON /// with `"stream"` given, a `.json` with a coordinate system or layer given, a large GeoJSON that @@ -1656,6 +1672,27 @@ pub(crate) fn add_data_ahead(p: &Value) -> Option { let (key, q) = (city_key(&path), p.clone()); return Some(ahead(NAME, key, move || read_city_file(&q, &path))); } + // A Shapefile read whole, in the coordinate system given if one is: as add_data reads it (one + // past the whole-file reader's cap, or with more records than it takes, is read in place). + if lower.ends_with(".shp") { + let q = past_cap_shapefile(&path, p).unwrap_or_else(|| p.clone()); + let stream = b(&q, "stream").unwrap_or_else(|| { + rustgis_data::source::shp::record_count(Path::new(&path)).is_some_and(|n| n > rustgis_map::source::STREAM_ABOVE) + }); + if stream { + return None; + } + let key = whole_key(&path); + return match ps(&q, "crs").map(str::trim).filter(|c| !c.is_empty()) { + Some(text) => { + // One add_data can't parse it refuses as it runs. + let crs = rustgis_data::raster::Crs::parse(text).ok()?; + let name = Path::new(&path).file_stem().and_then(|n| n.to_str()).unwrap_or("layer").to_string(); + Some(ahead(NAME, key, move || read_shapefile_with(&path, &name, crs))) + } + None => Some(ahead(NAME, key, move || read_whole(&q, &path))), + }; + } let (key, q) = (whole_key(&path), p.clone()); if rustgis_data::sheets::is_sheet_name(&path) { return Some(ahead(NAME, key, move || read_sheet_file(&q, &path))); @@ -4758,7 +4795,7 @@ pub(crate) fn add_read( } let n = ds.features.len(); let empty_before = super::map_is_empty(s); - let synced = report.synced.map(|files| rustgis_map::source::Synced { files, rows: rustgis_map::source::rows_digest(&ds) }); + let synced = report.synced; let id = add_dataset(s, ds); if let Some(l) = s.map_mut().layer_mut(id) { l.synced = synced; diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index e20293c28e..7e3565f66f 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -18,24 +18,42 @@ fn ahead_key(id: u32) -> String { format!("syncState:{id}") } -/// A check of a layer's files made ahead: the record and path checked, and what changed. +/// A check of a layer made ahead: the record, path and rows checked (the rows shared with the +/// layer, not copied), the rows' digest and what changed in the files. #[cfg(not(target_arch = "wasm32"))] struct Checked { - files: rustgis_data::sync::FileSync, + synced: rustgis_data::sync::Synced, path: String, + dataset: rustgis_data::Dataset, + rows: Result, changed: Result, String>, } +#[cfg(not(target_arch = "wasm32"))] +impl Checked { + /// Whether it checked layer `l` as it is: the same record and path, and the same rows (the + /// same allocation: an edit copies them), fields and coordinate system. + fn of(&self, l: &rustgis_map::Layer) -> bool { + l.synced.as_ref() == Some(&self.synced) + && l.dataset.source.as_deref() == Some(self.path.as_str()) + && l.dataset.features.shares(&self.dataset.features) + && l.dataset.fields == self.dataset.fields + && l.dataset.crs == self.dataset.crs + } +} + /// The check `source.syncState` makes before it runs in the desktop app: every part of the -/// file read and hashed on a thread of its own, with progress and Cancel. +/// file read and hashed, and the layer's rows hashed, on a thread of its own with progress and +/// Cancel. #[cfg(not(target_arch = "wasm32"))] pub(crate) fn sync_ahead(s: &Session, p: &Value) -> Option { let id = s.layer_id(p).ok()?; let l = s.map().layer(id)?; - let (files, path) = (l.synced.as_ref()?.files.clone(), l.dataset.source.clone()?); + let (synced, path, dataset) = (l.synced.clone()?, l.dataset.source.clone()?, l.dataset.clone()); Some(super::reads::ahead("Source File State", ahead_key(id), move || { - let changed = files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string()); - Checked { files, path, changed } + let changed = synced.files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string()); + let rows = rustgis_data::sync::rows_digest(&dataset).map_err(|e| e.to_string()); + Checked { synced, path, dataset, rows, changed } })) } @@ -44,40 +62,47 @@ pub(crate) fn sync_ahead(_: &Session, _: &Value) -> Option CmdResult { let id = s.layer_id(p)?; let l = s.map().layer(id).ok_or("the layer no longer exists")?; let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { return Err(format!( - "{} has no record of its file as it was read (it was added before RustGIS kept one, the file changed while it was read, or an older RustGIS saved the project): add the file again to compare it", + "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", l.name )); } return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); }; - let edited = rustgis_map::source::rows_digest(&l.dataset) != synced.rows; #[cfg(target_arch = "wasm32")] { - let _ = (edited, path); + let _ = (synced, path); Err("the web app can't read files on this computer".into()) } #[cfg(not(target_arch = "wasm32"))] { - let (files, path) = (synced.files.clone(), path.clone()); - let changed = match super::reads::take::(s, &ahead_key(id)) { - Some(checked) if checked.files == files && checked.path == path => checked.changed?, + let (was, files, path) = (synced.rows, synced.files.clone(), path.clone()); + let ahead = super::reads::take::(s, &ahead_key(id)); + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (rows, changed) = match ahead { + Some(checked) if checked.of(l) => (checked.rows?, checked.changed?), // Checking again here would hold the window: the check made on its thread is dropped. - Some(_) => return Err("the layer's file or its record changed while it was checked: check again".into()), - None => files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?, + Some(_) => return Err("the layer or its file's record changed while it was checked: check again".into()), + None => ( + rustgis_data::sync::rows_digest(&l.dataset).map_err(|e| e.to_string())?, + files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?, + ), }; + let edited = rows != was; let state = match (edited, !changed.is_empty()) { (false, false) => "inSync", (true, false) => "edited", diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 07cced5d47..25e77ca476 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39293,6 +39293,19 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); assert_eq!(read_land(&mut t, read).unwrap(), here); assert!(t.read_ahead.is_none(), "the check was taken"); + // The layer edited while it was checked: said, not checked again here. + let read = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})) + .unwrap() + .run(&rustgis_analysis::control::Control::stoppable()) + .unwrap(); + t.run("edit.setValue", &json!({"layer": "towns.shp", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + let e = read_land(&mut t, read).unwrap_err(); + assert!(e.contains("changed while it was checked"), "{e}"); + t.run("edit.undo", &Value::Null).unwrap(); + // Cancelled as it starts, the hashing looks and stops. + let control = rustgis_analysis::control::Control::cancelled_once_started(); + let e = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})).unwrap().run(&control).err().unwrap(); + assert!(e.to_lowercase().contains("cancel") && control.looked() > 0, "{e}"); } // The same bytes written again is no change; other bytes are, named by file. let dbf = dir.join("towns.dbf"); @@ -39306,6 +39319,17 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { assert_eq!((state["state"].as_str(), state["changed"].clone()), (Some("sourceChanged"), json!(["towns.dbf"]))); s.run("edit.setValue", &json!({"layer": "towns.shp", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); assert_eq!(s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap()["state"], "both"); + // Read ahead (the desktop app's Add Data), in a coordinate system given: the record comes + // with the read. + { + use crate::cmd::reads::{read_ahead, read_land}; + let p = json!({"path": dir.join("towns.shp"), "crs": "EPSG:4326", "name": "ahead"}); + let ahead = read_ahead(&mut s, "layer.addData", &p).unwrap(); + assert!(ahead.reads()); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + read_land(&mut s, read).unwrap(); + assert!(s.map().layer_by_name("ahead").unwrap().synced.is_some()); + } // A layer that wasn't read from such a file has nothing to compare with. assert!(s.run("source.syncState", &json!({"layer": "Cities"})).is_err()); std::fs::remove_dir_all(dir).unwrap(); diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 1e130cd7b6..5b8837fd2e 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -233,7 +233,7 @@ pub struct Layer { /// file and the layer as they were when last in step, to tell edits the file lacks and /// changes made to the file since. #[serde(default, skip_serializing_if = "Option::is_none")] - pub synced: Option, + pub synced: Option, /// The saved connection it was added from (its id, in the app's preferences): a layer whose /// source is read again logs in with that connection's login, and a project opened where /// the connection or its login is missing asks for one. Empty for a layer that came from no diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 4f923784df..85907a73e6 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -284,30 +284,6 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { h.0 } -/// A layer kept in the project that was read whole from a file it can be written back to (a -/// Shapefile or GeoJSON file), as it and the file were at their last sync, when it was read: so -/// it can tell whether it has edits the file lacks and whether the file changed since (#1385). -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Synced { - /// The files, as they were. - pub files: rustgis_data::sync::FileSync, - /// The layer's [`rows_digest`] then. - pub rows: u64, -} - -/// A hash of what a dataset holds, as a project saves it: its fields, coordinate system, and -/// each row's geometry and attributes in order. Not its name or where it came from, which are -/// no edit to the file. A number that isn't finite hashes as the null a project saves it as, so -/// a layer is the same before and after its project is saved and opened again. -pub fn rows_digest(ds: &rustgis_data::Dataset) -> u64 { - let schema = serde_json::to_vec(&(&ds.fields, &ds.crs)).unwrap_or_default(); - let h = fnv1a(fnv1a(FNV_OFFSET, &schema), &(ds.features.len() as u64).to_le_bytes()); - ds.features.iter().fold(h, |h, f| { - let row = serde_json::to_vec(&(&f.geometry, &f.attrs)).unwrap_or_default(); - fnv1a(fnv1a(h, &(row.len() as u64).to_le_bytes()), &row) - }) -} - /// A loaded row: its source version or fingerprint and [`row_hash`] as read. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RowVersion { @@ -1731,18 +1707,6 @@ mod tests { } /// The hash is FNV-1a over the row's JSON, the same as the projects saved with it hold. - #[test] - fn a_rows_digest_is_the_same_after_a_project_round_trip() { - use rustgis_data::{Dataset, Feature, Field, FieldType, GeometryType, Value}; - let mut ds = Dataset::new("a".to_string(), GeometryType::None, vec![Field::new("v", FieldType::Double)]); - ds.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(f64::NAN)] }].into(); - let reopened: Dataset = serde_json::from_value(serde_json::to_value(&ds).unwrap()).unwrap(); - assert_eq!(super::rows_digest(&reopened), super::rows_digest(&ds)); - let mut edited = ds.clone(); - edited.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(1.0)] }].into(); - assert_ne!(super::rows_digest(&edited), super::rows_digest(&ds)); - } - #[test] fn row_hashes_are_fnv_over_the_rows_json() { use rustgis_data::{Coord, Feature, Geometry, Value}; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d5f04ba32b..60afe05180 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1133,6 +1133,27 @@ pub struct RustGisApp { seen_project: Option, } +/// Says what `source.syncState` found, once its check lands: whether the layer and its file are +/// in step, and which parts of the file changed. A check that failed was said already. +#[cfg(not(target_arch = "wasm32"))] +fn sync_state_said(app: &mut RustGisApp, landed: Landed) { + let Landed::Ran(Ok(v)) = landed else { return }; + let layer = v.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()); + let layer = layer.and_then(|id| app.session.map().layer(id)).map(|l| l.name.clone()).unwrap_or_default(); + let path = v.get("path").and_then(Value::as_str).unwrap_or_default(); + let file = std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or(path).to_string(); + let parts: Vec<&str> = v.get("changed").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect(); + let parts = parts.join(", "); + let args = [("layer", layer.as_str()), ("file", file.as_str()), ("parts", parts.as_str())]; + let text = match v.get("state").and_then(Value::as_str) { + Some("inSync") => i18n::fmt("{layer} is in step with {file}", &args), + Some("edited") => i18n::fmt("{layer} has edits {file} doesn't have", &args), + Some("sourceChanged") => i18n::fmt("{file} changed since {layer} was read from it ({parts})", &args), + _ => i18n::fmt("{layer} has edits {file} doesn't have, and the file changed since it was read ({parts})", &args), + }; + app.session.notify(text, false); +} + impl RustGisApp { /// The layer to export: `layer` (an id or name, as commands take it; the dialog's is checked /// against its `instance`, so a reused id isn't taken for it), else the active layer. @@ -2261,9 +2282,10 @@ impl RustGisApp { let params = if params.is_null() { json!({}) } else { params }; return readjob::run(self, "topology.validate", params); } - // Comparing a layer with its file hashes the file: on a thread of its own. + // Comparing a layer with its file hashes the file: on a thread of its own, and what + // it found said once it lands (Command Search shows no answer). if id == "source.syncState" { - return readjob::run(self, "source.syncState", params); + return readjob::run_then(self, "source.syncState", params, Some(sync_state_said)); } // Converting the labels of a layer read in place reads all of its source first, on a // thread of its own, whoever asks (a click, the assistant). diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index a31cf96e5b..694b75fe7c 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -1097,6 +1097,23 @@ mod tests { assert!(l.is_streamed() && !l.labels.enabled); } + /// Source File State hashes the file on a thread of its own and says what it found once + /// it lands: Command Search shows no answer. + #[test] + fn a_source_file_state_check_runs_on_a_thread_and_says_what_it_found() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("places.geojson"); + publish_point(&path, 1.0); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let id = app.run("layer.addData", json!({"path": path})).unwrap()["layer"].as_u64().unwrap(); + app.reads_off_frame = true; + let mark = app.session.notice_mark(); + assert_eq!(app.run_by_user("source.syncState", json!({"layer": id})).unwrap()["started"], "Source File State"); + landed(&mut app, 60); + let said: Vec = app.session.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t == "places is in step with places.geojson"), "{said:?}"); + } + fn reload_fixture() -> (tempfile::TempDir, RustGisApp, u32, std::path::PathBuf) { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("streamed.geojson"); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 56b21139af..67da5e1a09 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -305,14 +305,20 @@ or raster subdataset. ### Source file state A layer read whole from a Shapefile or a `.geojson` file keeps its features in the project, so -edits change the project, not the file. It also remembers the file as it was read: each part's -size and SHA-256 hash, and a hash of the layer's rows. `source.syncState` compares the two and -answers `inSync`, `edited` (the layer has edits the file lacks), `sourceChanged` (the file -changed since it was read) or `both`, with the names of the parts that changed. Only the bytes -count: writing the same bytes again isn't a change, and a copy that puts back the file's size -and time is. Undoing an edit makes the layer `inSync` again. On the desktop the file is hashed -in the background, with progress and Cancel in the status bar. A file written to while it was being read, a layer read in place and a `.json` -file have no record. The record is saved with the project; the web app can't check it. +edits change the project, not the file. It also keeps a record of what was read: each part of +the file's size and SHA-256 hash, and a hash of the layer's rows. `source.syncState` compares +the layer with the rows read, and the file with the parts read, and answers `inSync`, `edited` +(the layer has edits since it was read), `sourceChanged` (the file changed since it was read) +or `both`, with the names of the parts that changed. It doesn't compare the layer with the file +as it is now: the same edit made to both shows as `both`. Only the bytes count: writing the +same bytes again isn't a change, and a copy that puts back the file's size and time is. +Undoing an edit makes the layer `inSync` again. On the desktop the file and the rows are hashed +in the background, with progress and Cancel in the status bar. + +The record is made just before the file is read, so a file written to while it was being read +shows as changed, unless the write was undone before RustGIS looked again. A layer read in +place and a `.json` file have no record. The record is saved with the project; the web app +can't check it. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. From 6aa5de81028420f88ffd22267b49d72c927fc1b2 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 10 Nov 2026 23:28:24 -0600 Subject: [PATCH 4/7] fix(engine): only a layer read from its file keeps its record, and only local files are checked Pasted layers and offline-area copies drop the record, so they never claim the file. source.syncState refuses a path that isn't a Shapefile or .geojson file on this computer, a network share included, without opening it. The edit notices say the layer was edited since it was read, the guide says how to start the check on the desktop, and the row digest is pinned by a test. The desktop's Import Shapefile and a GeoJSON added with "stream": false now read on a thread of their own too. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/sync.rs | 20 ++++++++++++++++++++ crates/engine/src/cmd/layer.rs | 14 ++++++++------ crates/engine/src/cmd/layer_copy.rs | 3 +++ crates/engine/src/cmd/publish.rs | 14 +++++++++++++- crates/engine/src/cmd/workspace.rs | 2 ++ crates/engine/src/tests.rs | 17 +++++++++++++++++ crates/ui-egui/src/lib.rs | 8 ++++++-- crates/ui-egui/src/readjob.rs | 24 ++++++++++++++++++++++++ docs/user-guide/index.md | 3 ++- 9 files changed, 95 insertions(+), 10 deletions(-) diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index 456da0e412..d70314a093 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -302,6 +302,26 @@ mod tests { assert_ne!(rows_digest(&edited).unwrap(), rows_digest(&ds).unwrap()); } + /// The digest is saved in projects with no version: a change in how fields, coordinate + /// systems, geometry or values serialize changes every saved one, and each layer of an older + /// project would then show as edited. This pins it; a change needs a way to read the old one. + #[test] + fn a_rows_digest_is_pinned() { + use crate::{Coord, Dataset, Feature, Field, FieldType, Geometry, GeometryType, Value}; + let mut ds = Dataset::new( + "a".to_string(), + GeometryType::Point, + vec![Field::new("name", FieldType::Text), Field::new("n", FieldType::Integer)], + ); + ds.features = vec![Feature { + id: 1, + geometry: Some(Geometry::Point(Coord { x: -90.05, y: 35.15 })), + attrs: vec![Value::Text("Main St".into()), Value::Int(4)], + }] + .into(); + assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092); + } + #[test] fn a_record_round_trips() { let sync = FileSync { files: vec![("a.geojson".into(), Some(FileStamp { len: 2, sha256: "ab".into() })), ("a.prj".into(), None)] }; diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index d6406b7bb5..d1c18bf557 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1633,9 +1633,8 @@ fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { /// whole or converted by GDAL, a GeoJSON read whole, delimited text, a spreadsheet or a city model /// (CityJSON or CityGML) read into its layers, a Shapefile read whole. `None` for /// any other file, which [`add_data`] reads as it adds it. Each is picked only where `add_data` -/// takes that read, so a read made is never wasted; a few it makes whole are left to it (a GeoJSON -/// with `"stream"` given, a `.json` with a coordinate system or layer given, a large GeoJSON that -/// isn't a FeatureCollection). +/// takes that read, so a read made is never wasted; a few it makes whole are left to it (a `.json` +/// with a coordinate system or layer given, a large GeoJSON that isn't a FeatureCollection). #[cfg(not(target_arch = "wasm32"))] pub(crate) fn add_data_ahead(p: &Value) -> Option { use super::reads::ahead; @@ -1659,13 +1658,16 @@ pub(crate) fn add_data_ahead(p: &Value) -> Option { } let given = |k: &str| ps(p, k).is_some_and(|v| !v.trim().is_empty()); let lower = path.to_ascii_lowercase(); - // A GeoJSON read whole: small, with no "stream" given (which add_data decides by), and not a + // A GeoJSON read whole: small with no "stream" given (which add_data decides by), or with + // `"stream": false`, and not a // CityJSON model (named so, or a .json with a coordinate system or a layer given). + // `"stream": false` reads one whole however large (within the reader's limit). let geojson = is_geojson(&path) && !rustgis_data::cityjson::is_cityjson_name(&path) - && b(p, "stream").is_none() + && b(p, "stream") != Some(true) && !(lower.ends_with(".json") && (given("crs") || given("layer"))) - && std::fs::metadata(&path).is_ok_and(|m| m.is_file() && m.len() <= rustgis_data::source::geojson::IN_PLACE_ABOVE); + && std::fs::metadata(&path) + .is_ok_and(|m| m.is_file() && (b(p, "stream") == Some(false) || m.len() <= rustgis_data::source::geojson::IN_PLACE_ABOVE)); // A city model: its layers, read whole. A model named `.json` (not `.city.json`) is a file of // several layers only with a coordinate system or a layer given, which Add Data reads as it runs. if rustgis_data::io::is_multi_layer_name(&path) && is_city_model_path(&path) { diff --git a/crates/engine/src/cmd/layer_copy.rs b/crates/engine/src/cmd/layer_copy.rs index 7e9983a3e8..0caf4d13ac 100644 --- a/crates/engine/src/cmd/layer_copy.rs +++ b/crates/engine/src/cmd/layer_copy.rs @@ -59,6 +59,9 @@ fn copy(s: &mut Session, p: &Value) -> CmdResult { .map(|l| { let mut l = l.clone(); l.selection.clear(); + // A copy isn't the layer read from its file: it can't say whether the file has its + // edits, nor write them back. + l.synced = None; l }) .collect(); diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 7e3565f66f..1489c58360 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -49,7 +49,7 @@ impl Checked { pub(crate) fn sync_ahead(s: &Session, p: &Value) -> Option { let id = s.layer_id(p).ok()?; let l = s.map().layer(id)?; - let (synced, path, dataset) = (l.synced.clone()?, l.dataset.source.clone()?, l.dataset.clone()); + let (synced, path, dataset) = (l.synced.clone()?, l.dataset.source.clone().filter(|p| local_file(p))?, l.dataset.clone()); Some(super::reads::ahead("Source File State", ahead_key(id), move || { let changed = synced.files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string()); let rows = rustgis_data::sync::rows_digest(&dataset).map_err(|e| e.to_string()); @@ -83,6 +83,11 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { } return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); }; + // A project from elsewhere names the path: only a local file of a kind a record is made for + // is opened, never a network share (opening one can send the user's login to its host). + if !local_file(path) { + return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); + } #[cfg(target_arch = "wasm32")] { let _ = (synced, path); @@ -112,3 +117,10 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { Ok(serde_json::json!({"layer": id, "path": path, "state": state, "edited": edited, "changed": changed})) } } + +/// Whether `path` names a file on this computer a record is made for: a Shapefile or `.geojson` +/// file, not an address, a database or a network share (`\\host\share`, `//host/share`). +fn local_file(path: &str) -> bool { + let local = rustgis_map::paths::local_path(path); + rustgis_map::paths::is_local(path) && rustgis_data::sync::is_syncable(path) && !local.starts_with("//") && !local.starts_with("\\\\") +} diff --git a/crates/engine/src/cmd/workspace.rs b/crates/engine/src/cmd/workspace.rs index cb1b8c2bc5..dce39da42a 100644 --- a/crates/engine/src/cmd/workspace.rs +++ b/crates/engine/src/cmd/workspace.rs @@ -604,6 +604,8 @@ fn snapshot(m: &mut rustgis_map::Map, e: &Extent) -> rustgis_map::Map { c.dataset.features = l.dataset.features.iter().filter(|f| f.geometry.as_ref().is_some_and(|g| g.extent().intersects(e))).cloned().collect(); c.selection.clear(); + // Only part of the rows read: no longer the layer its file's record describes. + c.synced = None; c.touch_geometry(); out.layers.push(c); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 25e77ca476..29608554db 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39329,6 +39329,23 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); read_land(&mut s, read).unwrap(); assert!(s.map().layer_by_name("ahead").unwrap().synced.is_some()); + assert_eq!(s.run("source.syncState", &json!({"layer": "ahead"})).unwrap()["state"], "inSync"); + } + // A copy (pasted, or an offline area's part of the rows) isn't the layer read from the file: + // it has no record. + s.run("layer.copy", &json!({"layer": "towns.shp"})).unwrap(); + s.run("layer.paste", &Value::Null).unwrap(); + let pasted = s.map().layers.iter().filter(|l| l.dataset.source.as_deref().is_some_and(|p| p.ends_with("towns.shp"))).count(); + assert!(pasted >= 2 && s.map().layers.iter().filter(|l| l.synced.is_some()).count() == 3, "only the layers read hold one"); + s.run("offline.download", &json!({"name": "Here", "extent": [-180.0, -90.0, 180.0, 90.0]})).unwrap(); + assert!(s.project.offline_areas[0].data.layers.iter().all(|l| l.synced.is_none())); + // A project from elsewhere names the path: a network share or a file no record is made for + // is refused without being opened. + for path in [r"\\host\share\towns.shp", "//host/share/towns.shp", "towns.json"] { + let id = s.map().layer_by_name("towns.shp").unwrap().id; + s.map_mut().layer_mut(id).unwrap().dataset.source = Some(path.into()); + let e = s.run("source.syncState", &json!({"layer": id})).unwrap_err(); + assert!(e.contains("isn't a Shapefile or .geojson file on this computer"), "{path}: {e}"); } // A layer that wasn't read from such a file has nothing to compare with. assert!(s.run("source.syncState", &json!({"layer": "Cities"})).is_err()); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 60afe05180..c6dcc2a214 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1147,9 +1147,9 @@ fn sync_state_said(app: &mut RustGisApp, landed: Landed) { let args = [("layer", layer.as_str()), ("file", file.as_str()), ("parts", parts.as_str())]; let text = match v.get("state").and_then(Value::as_str) { Some("inSync") => i18n::fmt("{layer} is in step with {file}", &args), - Some("edited") => i18n::fmt("{layer} has edits {file} doesn't have", &args), + Some("edited") => i18n::fmt("{layer} was edited since it was read from {file}", &args), Some("sourceChanged") => i18n::fmt("{file} changed since {layer} was read from it ({parts})", &args), - _ => i18n::fmt("{layer} has edits {file} doesn't have, and the file changed since it was read ({parts})", &args), + _ => i18n::fmt("{layer} was edited, and {file} changed, since the layer was read from it ({parts})", &args), }; app.session.notify(text, false); } @@ -3359,6 +3359,10 @@ impl RustGisApp { self.ui.pending_shapefile = Some(json!({"crs": text})); open("shapefile"); } else if let Some(path) = dialogs::pick_open(&["shp", "zip"], tl!("Shapefile")) { + // Read on a thread of its own, as Add Data's other files are. + #[cfg(not(target_arch = "wasm32"))] + return readjob::run(self, "layer.addData", json!({"path": path, "crs": text})); + #[cfg(target_arch = "wasm32")] return self.run("layer.addData", json!({"path": path, "crs": text})); } } diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 694b75fe7c..ece9c58e42 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -1114,6 +1114,30 @@ mod tests { assert!(said.iter().any(|t| t == "places is in step with places.geojson"), "{said:?}"); } + /// The desktop's Add Data reads a Shapefile (in a coordinate system given, as Import + /// Shapefile asks) and a GeoJSON with `"stream": false` on a thread of its own, recording the + /// file with the read. + #[test] + fn whole_files_are_read_and_recorded_on_a_thread() { + let dir = tempfile::tempdir().unwrap(); + let geojson = dir.path().join("places.geojson"); + publish_point(&geojson, 1.0); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("layer.addData", json!({"path": geojson, "name": "seed"})).unwrap(); + let shp = dir.path().join("towns.shp"); + app.run("share.exportLayer", json!({"layer": "seed", "path": shp})).unwrap(); + app.reads_off_frame = true; + for (p, name) in [ + (json!({"path": shp, "crs": "EPSG:4326", "name": "towns"}), "towns"), + (json!({"path": geojson, "stream": false, "name": "whole"}), "whole"), + ] { + assert_eq!(run(&mut app, "layer.addData", p).unwrap()["started"], "Add Data", "{name}"); + landed(&mut app, 60); + let id = app.session.map().layer_by_name(name).unwrap().id; + assert!(app.session.map().layer(id).unwrap().synced.is_some(), "{name}"); + } + } + fn reload_fixture() -> (tempfile::TempDir, RustGisApp, u32, std::path::PathBuf) { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("streamed.geojson"); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 67da5e1a09..07b8427566 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -320,7 +320,8 @@ shows as changed, unless the write was undone before RustGIS looked again. A lay place and a `.json` file have no record. The record is saved with the project; the web app can't check it. -CLI and Python: `source.syncState` with `{"layer":"parcels"}`. +On the desktop, make the layer active and choose **Source File State** in Command Search; what it +found shows in the status bar. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. ## Keyboard shortcuts From 0230eefd65d72656c66e314aba4b31760b6ca6f1 Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 04:05:26 -0600 Subject: [PATCH 5/7] fix(engine): refuse any network-share spelling and keep a recorded file's path whole Two leading separators of either kind (/\host, \/host, file:///\\host) are a network share on Windows: source.syncState refuses them all. A layer whose file is recorded keeps the file's absolute path, so a later check doesn't depend on the directory RustGIS runs in (a saved project still stores it relative to its folder). The notice for a file that can't be recorded is tested both ways, and a Shapefile read ahead in another system than its own is checked to be in step. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 16 +++++++++++----- crates/engine/src/cmd/publish.rs | 8 +++++--- crates/engine/src/tests.rs | 23 ++++++++++++++++++++++- crates/ui-egui/src/readjob.rs | 8 ++++++++ docs/user-guide/index.md | 3 ++- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index d1c18bf557..ccc7c735b1 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1614,7 +1614,14 @@ fn read_shapefile_with(path: &str, name: &str, crs: rustgis_data::raster::Crs) - #[cfg(not(target_arch = "wasm32"))] fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { let (read, files) = rustgis_data::sync::FileSync::around(Path::new(path), read).map_err(|e| format!("{path}: {e}"))?; - let (ds, mut report) = read?; + let (mut ds, mut report) = read?; + // Kept whole, as the record checks it later: a relative path would be taken from wherever + // RustGIS runs then, not where the file was read. + if files.is_some() + && let Ok(full) = std::path::absolute(path) + { + ds.source = Some(full.display().to_string()); + } report.synced = match files { Some(files) => { Some(rustgis_data::sync::Synced { files, rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))? }) @@ -1658,10 +1665,9 @@ pub(crate) fn add_data_ahead(p: &Value) -> Option { } let given = |k: &str| ps(p, k).is_some_and(|v| !v.trim().is_empty()); let lower = path.to_ascii_lowercase(); - // A GeoJSON read whole: small with no "stream" given (which add_data decides by), or with - // `"stream": false`, and not a - // CityJSON model (named so, or a .json with a coordinate system or a layer given). - // `"stream": false` reads one whole however large (within the reader's limit). + // A GeoJSON read whole: small with no "stream" given (which add_data decides by), or any size + // (within the reader's limit) with `"stream": false`; and not a CityJSON model (named so, or a + // .json with a coordinate system or a layer given). let geojson = is_geojson(&path) && !rustgis_data::cityjson::is_cityjson_name(&path) && b(p, "stream") != Some(true) diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 1489c58360..2ae6408e6a 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -119,8 +119,10 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { } /// Whether `path` names a file on this computer a record is made for: a Shapefile or `.geojson` -/// file, not an address, a database or a network share (`\\host\share`, `//host/share`). +/// file, not an address, a database or a network share (`\\host\share`, `//host/share`, +/// `/\host\share`). fn local_file(path: &str) -> bool { - let local = rustgis_map::paths::local_path(path); - rustgis_map::paths::is_local(path) && rustgis_data::sync::is_syncable(path) && !local.starts_with("//") && !local.starts_with("\\\\") + // Windows takes any two leading separators, of either kind, for a network share. + let share = rustgis_map::paths::local_path(path).as_bytes().get(..2).is_some_and(|p| p.iter().all(|&b| matches!(b, b'/' | b'\\'))); + rustgis_map::paths::is_local(path) && rustgis_data::sync::is_syncable(path) && !share } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 29608554db..1f58edc095 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39341,12 +39341,33 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { assert!(s.project.offline_areas[0].data.layers.iter().all(|l| l.synced.is_none())); // A project from elsewhere names the path: a network share or a file no record is made for // is refused without being opened. - for path in [r"\\host\share\towns.shp", "//host/share/towns.shp", "towns.json"] { + let shares = [r"\\host\share\towns.shp", "//host/share/towns.shp", r"/\host\share\towns.shp", r"\/host/share/towns.shp"]; + for path in shares.into_iter().chain([r"file:///\\host\share\towns.shp", "towns.json"]) { let id = s.map().layer_by_name("towns.shp").unwrap().id; s.map_mut().layer_mut(id).unwrap().dataset.source = Some(path.into()); let e = s.run("source.syncState", &json!({"layer": id})).unwrap_err(); assert!(e.contains("isn't a Shapefile or .geojson file on this computer"), "{path}: {e}"); + // Nor read on a thread of its own: the desktop's check makes no read, and refuses as it lands. + let ahead = crate::cmd::reads::read_ahead(&mut s, "source.syncState", &json!({"layer": id})).unwrap(); + assert!(!ahead.reads(), "{path}"); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + assert!(crate::cmd::reads::read_land(&mut s, read).is_err(), "{path}"); + } + // A part that can't be read to record it (a folder where the .prj would be): the layer is + // added without a record, and says so; one read cleanly says nothing of it. + let said = |s: &Session, mark: u64| s.notices_since(mark).iter().filter(|n| n.text.contains("couldn't be read to record it")).count(); + let odd = dir.join("odd"); + std::fs::create_dir_all(odd.join("odd.prj")).unwrap(); + for ext in ["shp", "shx", "dbf"] { + std::fs::copy(dir.join(format!("towns.{ext}")), odd.join(format!("odd.{ext}"))).unwrap(); } + let mark = s.notice_mark(); + let id = s.run("layer.addData", &json!({"path": odd.join("odd.shp")})).unwrap()["layer"].as_u64().unwrap(); + assert!(s.map().layer(id as u32).unwrap().synced.is_none()); + assert_eq!(said(&s, mark), 1); + let mark = s.notice_mark(); + s.run("layer.addData", &json!({"path": dir.join("towns.geojson"), "name": "clean"})).unwrap(); + assert_eq!(said(&s, mark), 0); // A layer that wasn't read from such a file has nothing to compare with. assert!(s.run("source.syncState", &json!({"layer": "Cities"})).is_err()); std::fs::remove_dir_all(dir).unwrap(); diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index ece9c58e42..021cbbb237 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -1129,12 +1129,20 @@ mod tests { app.reads_off_frame = true; for (p, name) in [ (json!({"path": shp, "crs": "EPSG:4326", "name": "towns"}), "towns"), + // Another system than the file's: the rows recorded are the rows as read in it. + (json!({"path": shp, "crs": "EPSG:3857", "name": "projected"}), "projected"), (json!({"path": geojson, "stream": false, "name": "whole"}), "whole"), ] { assert_eq!(run(&mut app, "layer.addData", p).unwrap()["started"], "Add Data", "{name}"); landed(&mut app, 60); let id = app.session.map().layer_by_name(name).unwrap().id; assert!(app.session.map().layer(id).unwrap().synced.is_some(), "{name}"); + if name == "projected" { + assert_eq!(app.session.map().layer(id).unwrap().dataset.crs, rustgis_engine::data::raster::Crs::parse("EPSG:3857").ok()); + } + app.reads_off_frame = false; + assert_eq!(app.run("source.syncState", json!({"layer": id})).unwrap()["state"], "inSync", "{name}"); + app.reads_off_frame = true; } } diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 07b8427566..c679f9c62e 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -311,7 +311,8 @@ the layer with the rows read, and the file with the parts read, and answers `inS (the layer has edits since it was read), `sourceChanged` (the file changed since it was read) or `both`, with the names of the parts that changed. It doesn't compare the layer with the file as it is now: the same edit made to both shows as `both`. Only the bytes count: writing the -same bytes again isn't a change, and a copy that puts back the file's size and time is. +same bytes again isn't a change, while a copy that puts back the file's size and time with other +bytes is one. Undoing an edit makes the layer `inSync` again. On the desktop the file and the rows are hashed in the background, with progress and Cancel in the status bar. From 203ced5bc3d9afa172352974f7ce89f743173720 Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 07:12:35 -0600 Subject: [PATCH 6/7] fix(data): tell a cleared non-finite number from the number, and skip hashing files too large to read The row digest tells a non-finite number from an empty cell (by position and bits), so clearing one is an edit; a layer whose project saved one as null now reads as edited, since it no longer holds what its file does. A file past the reader's limit is refused before it is hashed. Windows' long form of a local path (\\?\C:\...) is checked like any local file, while \\?\UNC\ and other shares are still refused. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/sync.rs | 41 ++++++++++++++++++++++---------- crates/engine/src/cmd/layer.rs | 5 ++++ crates/engine/src/cmd/publish.rs | 11 ++++++--- crates/engine/src/tests.rs | 8 ++++++- docs/user-guide/index.md | 3 ++- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index d70314a093..ff65602256 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -34,13 +34,14 @@ pub struct Synced { pub rows: u64, } -/// A hash of what a dataset holds, as a project saves it: its fields, coordinate system, and -/// each row's geometry and attributes in order (FNV-1a over their JSON, written to the hash as -/// it is made). Not its name or where it came from, which are no edit to the file. A number that -/// isn't finite hashes as the null a project saves it as, so a layer is the same before and -/// after its project is saved and opened again. Its work grows with the rows: each is counted -/// as progress ([`crate::progress`]), and it stops once the run is cancelled; the desktop app -/// runs it off the UI thread. +/// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and +/// attributes in order (FNV-1a over their JSON, written to the hash as it is made). Not its name +/// or where it came from, which are no edit to the file. A number that isn't finite, which JSON +/// writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit +/// rows are: clearing one is an edit. A project saves one as null (and says so), so a layer that +/// held one reads as edited once its project is opened again: it no longer holds what its file +/// does. Its work grows with the rows: each is counted as progress ([`crate::progress`]), and it +/// stops once the run is cancelled; the desktop app runs it off the UI thread. pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { /// FNV-1a over the bytes written to it. struct Fnv(u64); @@ -64,6 +65,13 @@ pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { crate::progress::begin(ds.features.len()); for f in ds.features.iter() { put(&mut h, &(&f.geometry, &f.attrs)); + for (index, value) in f.attrs.iter().enumerate() { + if let crate::Value::Float(number) = value + && !number.is_finite() + { + put(&mut h, &("non-finite", index, number.to_bits())); + } + } crate::progress::step(1)?; } Ok(h.0) @@ -291,15 +299,22 @@ mod tests { } #[test] - fn a_rows_digest_is_the_same_after_a_project_round_trip() { + fn a_rows_digest_tells_a_non_finite_number_from_an_empty_cell() { use crate::{Dataset, Feature, Field, FieldType, GeometryType, Value}; + let row = |v: Value| vec![Feature { id: 1, geometry: None, attrs: vec![v] }].into(); let mut ds = Dataset::new("a".to_string(), GeometryType::None, vec![Field::new("v", FieldType::Double)]); - ds.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(f64::NAN)] }].into(); + ds.features = row(Value::Float(f64::NAN)); + let digest = |v: Value| { + let mut d = ds.clone(); + d.features = row(v); + rows_digest(&d).unwrap() + }; + // Clearing it is an edit, and so is a project's saving it as null. let reopened: Dataset = serde_json::from_value(serde_json::to_value(&ds).unwrap()).unwrap(); - assert_eq!(rows_digest(&reopened).unwrap(), rows_digest(&ds).unwrap()); - let mut edited = ds.clone(); - edited.features = vec![Feature { id: 1, geometry: None, attrs: vec![Value::Float(1.0)] }].into(); - assert_ne!(rows_digest(&edited).unwrap(), rows_digest(&ds).unwrap()); + assert_ne!(rows_digest(&reopened).unwrap(), rows_digest(&ds).unwrap()); + assert_eq!(rows_digest(&reopened).unwrap(), digest(Value::Null)); + assert_ne!(digest(Value::Float(f64::INFINITY)), digest(Value::Float(f64::NAN))); + assert_eq!(digest(Value::Float(f64::NAN)), rows_digest(&ds).unwrap()); } /// The digest is saved in projects with no version: a change in how fields, coordinate diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index ccc7c735b1..1e721daa0b 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1613,6 +1613,11 @@ fn read_shapefile_with(path: &str, name: &str, crs: rustgis_data::raster::Crs) - /// compared with its file later. #[cfg(not(target_arch = "wasm32"))] fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { + // A file past what the readers take (its `.dbf` too) is refused by the read: not hashed first. + let over = |p: &Path| std::fs::metadata(p).is_ok_and(|m| m.len() > rustgis_data::io::MAX_BYTES); + if over(Path::new(path)) || over(&Path::new(path).with_extension("dbf")) { + return read(); + } let (read, files) = rustgis_data::sync::FileSync::around(Path::new(path), read).map_err(|e| format!("{path}: {e}"))?; let (mut ds, mut report) = read?; // Kept whole, as the record checks it later: a relative path would be taken from wherever diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 2ae6408e6a..97887904b8 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -120,9 +120,14 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { /// Whether `path` names a file on this computer a record is made for: a Shapefile or `.geojson` /// file, not an address, a database or a network share (`\\host\share`, `//host/share`, -/// `/\host\share`). +/// `/\host\share`, `\\?\UNC\host\share`). Windows' long form of a local path (`\\?\C:\…`) is one +/// on this computer. fn local_file(path: &str) -> bool { - // Windows takes any two leading separators, of either kind, for a network share. - let share = rustgis_map::paths::local_path(path).as_bytes().get(..2).is_some_and(|p| p.iter().all(|&b| matches!(b, b'/' | b'\\'))); + let b = rustgis_map::paths::local_path(path).as_bytes(); + let sep = |i: usize| b.get(i).is_some_and(|c| matches!(c, b'/' | b'\\')); + // Windows takes any two leading separators, of either kind, for a network share, unless they + // begin the long form of a path on a drive. + let drive = b.get(2) == Some(&b'?') && sep(3) && b.get(4).is_some_and(u8::is_ascii_alphabetic) && b.get(5) == Some(&b':') && sep(6); + let share = sep(0) && sep(1) && !drive; rustgis_map::paths::is_local(path) && rustgis_data::sync::is_syncable(path) && !share } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 1f58edc095..a7e2891f2d 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39342,7 +39342,8 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { // A project from elsewhere names the path: a network share or a file no record is made for // is refused without being opened. let shares = [r"\\host\share\towns.shp", "//host/share/towns.shp", r"/\host\share\towns.shp", r"\/host/share/towns.shp"]; - for path in shares.into_iter().chain([r"file:///\\host\share\towns.shp", "towns.json"]) { + let more = [r"file:///\\host\share\towns.shp", r"\\?\UNC\host\share\towns.shp", "towns.json"]; + for path in shares.into_iter().chain(more) { let id = s.map().layer_by_name("towns.shp").unwrap().id; s.map_mut().layer_mut(id).unwrap().dataset.source = Some(path.into()); let e = s.run("source.syncState", &json!({"layer": id})).unwrap_err(); @@ -39353,6 +39354,11 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); assert!(crate::cmd::reads::read_land(&mut s, read).is_err(), "{path}"); } + // Windows' long form of a path on a drive is a file on this computer: checked (and missing here). + let id = s.map().layer_by_name("towns.shp").unwrap().id; + s.map_mut().layer_mut(id).unwrap().dataset.source = Some(r"\\?\C:\data\towns.shp".into()); + let v = s.run("source.syncState", &json!({"layer": id})).unwrap(); + assert!(v["changed"].as_array().unwrap().iter().any(|c| c.as_str().is_some_and(|c| c.ends_with("towns.shp"))), "{v}"); // A part that can't be read to record it (a folder where the .prj would be): the layer is // added without a record, and says so; one read cleanly says nothing of it. let said = |s: &Session, mark: u64| s.notices_since(mark).iter().filter(|n| n.text.contains("couldn't be read to record it")).count(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index c679f9c62e..f1e17f9102 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -313,7 +313,8 @@ or `both`, with the names of the parts that changed. It doesn't compare the laye as it is now: the same edit made to both shows as `both`. Only the bytes count: writing the same bytes again isn't a change, while a copy that puts back the file's size and time with other bytes is one. -Undoing an edit makes the layer `inSync` again. On the desktop the file and the rows are hashed +Undoing an edit makes the layer `inSync` again. A number that isn't finite (NaN, infinity), which a +project saves as empty, shows the layer as `edited` once its project is opened again. On the desktop the file and the rows are hashed in the background, with progress and Cancel in the status bar. The record is made just before the file is read, so a file written to while it was being read From c06cb93aa0416c4db7dbd6a8e82b15d22aa461c3 Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 11 Nov 2026 09:52:26 -0600 Subject: [PATCH 7/7] fix(engine): check a .dbf's size only beside a Shapefile A .geojson with an unrelated large .dbf beside it no longer goes without a record. The long-path test on Windows uses a path in its own folder, and the guide says that undoing every edit reads inSync only when the file hasn't changed either. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 6 ++++-- crates/engine/src/tests.rs | 8 +++++++- docs/user-guide/index.md | 3 ++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 1e721daa0b..8ac1ebd6b9 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1613,9 +1613,11 @@ fn read_shapefile_with(path: &str, name: &str, crs: rustgis_data::raster::Crs) - /// compared with its file later. #[cfg(not(target_arch = "wasm32"))] fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { - // A file past what the readers take (its `.dbf` too) is refused by the read: not hashed first. + // A file past what the readers take (a Shapefile's `.dbf` too) is refused by the read: not + // hashed first. let over = |p: &Path| std::fs::metadata(p).is_ok_and(|m| m.len() > rustgis_data::io::MAX_BYTES); - if over(Path::new(path)) || over(&Path::new(path).with_extension("dbf")) { + let shp = path.to_ascii_lowercase().ends_with(".shp"); + if over(Path::new(path)) || (shp && over(&Path::new(path).with_extension("dbf"))) { return read(); } let (read, files) = rustgis_data::sync::FileSync::around(Path::new(path), read).map_err(|e| format!("{path}: {e}"))?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a7e2891f2d..cb54fa26df 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39356,7 +39356,13 @@ fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { } // Windows' long form of a path on a drive is a file on this computer: checked (and missing here). let id = s.map().layer_by_name("towns.shp").unwrap().id; - s.map_mut().layer_mut(id).unwrap().dataset.source = Some(r"\\?\C:\data\towns.shp".into()); + // In the test's own folder on Windows (a file there by chance can't match); elsewhere it's + // just a name no file has. + #[cfg(windows)] + let long = format!(r"\\?\{}", dir.join("missing").join("towns.shp").display()); + #[cfg(not(windows))] + let long = r"\\?\C:\missing\towns.shp".to_string(); + s.map_mut().layer_mut(id).unwrap().dataset.source = Some(long); let v = s.run("source.syncState", &json!({"layer": id})).unwrap(); assert!(v["changed"].as_array().unwrap().iter().any(|c| c.as_str().is_some_and(|c| c.ends_with("towns.shp"))), "{v}"); // A part that can't be read to record it (a folder where the .prj would be): the layer is diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index f1e17f9102..175fb1bcd5 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -313,7 +313,8 @@ or `both`, with the names of the parts that changed. It doesn't compare the laye as it is now: the same edit made to both shows as `both`. Only the bytes count: writing the same bytes again isn't a change, while a copy that puts back the file's size and time with other bytes is one. -Undoing an edit makes the layer `inSync` again. A number that isn't finite (NaN, infinity), which a +Undoing every edit makes the layer match its record again, so it reads `inSync` if the file +hasn't changed either. A number that isn't finite (NaN, infinity), which a project saves as empty, shows the layer as `edited` once its project is opened again. On the desktop the file and the rows are hashed in the background, with progress and Cancel in the status bar.