diff --git a/Cargo.lock b/Cargo.lock index b45e0e7749..6db26f0570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4772,6 +4772,7 @@ dependencies = [ "ruzstd", "serde", "serde_json", + "sha2 0.11.1", "shapefile", "sysinfo", "tempfile", diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index c6fcfa2b8b..6ff9ebd2b9 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -57,6 +57,8 @@ tokio-util = { workspace = true } rustls = { workspace = true } webpki-roots = { workspace = true } # Native web clients verify servers with the system's trust (see `http`). +# Hashes a source's files, to tell whether they changed since a layer was read (`sync`). +sha2 = { workspace = true } ureq = { version = "3", default-features = false, features = ["rustls", "gzip", "platform-verifier"] } # Private temporary workspaces for downloaded JPEG 2000 conversion through GDAL. tempfile = "3" diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index f7960c94c3..1968c7a4cc 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -181,11 +181,13 @@ pub(crate) fn read_cpg(path: &std::path::Path, notes: &mut Vec) -> Optio } } +/// Every ASCII case of the `.cpg` extension, the conventional spellings first: tried in turn, +/// without scanning a potentially large directory, and the first there wins. +pub(crate) const CPG_SPELLINGS: [&str; 8] = ["cpg", "CPG", "Cpg", "cPg", "cpG", "CPg", "CpG", "cPG"]; + fn try_read_cpg(path: &std::path::Path) -> std::io::Result>> { use std::io::Read; - // Check every ASCII case combination without scanning a potentially large directory. - // Prefer the conventional spellings if multiple sidecars exist. - for ext in ["cpg", "CPG", "Cpg", "cPg", "cpG", "CPg", "CpG", "cPG"] { + for ext in CPG_SPELLINGS { match std::fs::File::open(path.with_extension(ext)) { Ok(file) => { let mut bytes = Vec::new(); diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 60d0de8915..2edcfe4253 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -44,6 +44,7 @@ pub mod sheets; pub mod source; pub mod source_layers; pub mod sqlite; +pub mod sync; pub mod threads; pub mod time; pub mod topojson; @@ -377,6 +378,10 @@ impl Serialize for Dataset { pub struct ReadReport { pub skipped: usize, pub messages: Vec, + /// The files read and the rows read from them, when the read recorded them + /// ([`sync::FileSync::around`]): what a layer kept in the project compares with its source + /// before writing edits back. + pub synced: Option, } /// What a read says when it drops Z or M values (see [`Dataset::zm_dropped`]): from `features` of diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index 5ecd24b48c..939c5e2c31 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -562,6 +562,7 @@ fn materialize_within(src: &dyn FeatureSource, budget: u64) -> Result<(Dataset, let report = ReadReport { skipped: skipped + src.skipped_at_open(), messages: info.notes.iter().map(|n| format!("{}: {n}", info.name)).collect(), + synced: None, }; if ds.geometry_type == GeometryType::None && let Some(g) = ds.features.iter().find_map(|f| f.geometry.as_ref()) diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs new file mode 100644 index 0000000000..ff65602256 --- /dev/null +++ b/crates/data/src/sync.rs @@ -0,0 +1,347 @@ +//! The files a layer was read from, as they were when it was read ([`FileSync`]): what a layer +//! kept in the project compares with its source before its edits are written back to it, so +//! that writing them never overwrites a change made to the file since (#1385). + +use serde::{Deserialize, Serialize}; + +/// The files of a source as they were at a moment: each one's name, size and SHA-256 hash, and +/// the files that weren't there. A Shapefile is its `.shp`, `.shx`, `.dbf`, `.prj` and `.cpg` +/// (the ones its reader opens); a GeoJSON file is itself. Only the contents count: a file +/// copied, touched or moved to another disk with the same bytes is the same file. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileSync { + /// Each file by its name beside the source (`parcels.dbf`), with its stamp, or `None` for + /// one that wasn't there, so a `.prj` added since counts as a change too. + pub files: Vec<(String, Option)>, +} + +/// One file as it was. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileStamp { + pub len: u64, + /// The SHA-256 of its bytes, in hexadecimal. + pub sha256: String, +} + +/// A layer kept in the project that was read whole from a file it can be written back to (a +/// Shapefile or GeoJSON file), as it and the file were at their last sync, when it was read: so +/// it can tell whether it has edits the file lacks and whether the file changed since. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Synced { + /// The files, as they were. + pub files: FileSync, + /// The layer's [`rows_digest`] then. + pub rows: u64, +} + +/// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and +/// attributes in order (FNV-1a over their JSON, written to the hash as it is made). Not its name +/// or where it came from, which are no edit to the file. A number that isn't finite, which JSON +/// writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit +/// rows are: clearing one is an edit. A project saves one as null (and says so), so a layer that +/// held one reads as edited once its project is opened again: it no longer holds what its file +/// does. Its work grows with the rows: each is counted as progress ([`crate::progress`]), and it +/// stops once the run is cancelled; the desktop app runs it off the UI thread. +pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { + /// FNV-1a over the bytes written to it. + struct Fnv(u64); + impl std::io::Write for Fnv { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + /// Writes `value`'s JSON to the hash (nothing, should it not write), then a byte JSON never + /// writes, so values can't run into each other. + fn put(h: &mut Fnv, value: &impl serde::Serialize) { + let _ = serde_json::to_writer(&mut *h, value); + h.0 = (h.0 ^ 0xff).wrapping_mul(0x0100_0000_01b3); + } + let mut h = Fnv(0xcbf2_9ce4_8422_2325); + put(&mut h, &(&ds.fields, &ds.crs)); + crate::progress::begin(ds.features.len()); + for f in ds.features.iter() { + put(&mut h, &(&f.geometry, &f.attrs)); + for (index, value) in f.attrs.iter().enumerate() { + if let crate::Value::Float(number) = value + && !number.is_finite() + { + put(&mut h, &("non-finite", index, number.to_bits())); + } + } + crate::progress::step(1)?; + } + Ok(h.0) +} + +/// Whether `path` names a source whose files a [`FileSync`] records: a Shapefile or a +/// `.geojson` file. A `.json` file isn't one yet: it may hold a city model or a topology, which +/// can't be written back as they were read. +pub fn is_syncable(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + [".shp", ".geojson"].iter().any(|e| lower.ends_with(e)) +} + +#[cfg(not(target_arch = "wasm32"))] +mod native { + use super::{FileStamp, FileSync}; + use crate::{DataError, Result}; + use std::path::{Path, PathBuf}; + + /// The files of the source at `path`, each as a name beside it and its path: for a + /// Shapefile the companions its reader opens (the `.cpg` in whichever case is there, else + /// `.cpg`), otherwise the file alone. + fn parts(path: &Path) -> Vec<(String, PathBuf)> { + let mut paths = vec![path.to_path_buf()]; + if path.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("shp")) { + for ext in ["shx", "dbf", "prj"] { + paths.push(path.with_extension(ext)); + } + let cpg = crate::dbf::CPG_SPELLINGS + .iter() + .map(|e| path.with_extension(e)) + .find(|p| p.is_file()) + .unwrap_or_else(|| path.with_extension("cpg")); + paths.push(cpg); + } + paths.into_iter().map(|p| (p.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), p)).collect() + } + + /// The file's size, or `None` when it isn't there. + fn len(path: &Path) -> Result> { + match std::fs::metadata(path) { + Ok(m) if m.is_file() => Ok(Some(m.len())), + Ok(_) => Err(DataError::Format(format!("{} isn't a file", path.display()))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// The SHA-256 of the file's bytes, read a block at a time, each counted as progress + /// ([`crate::progress`]) and stopping once the run is cancelled. + fn sha256(path: &Path) -> Result { + use sha2::Digest; + use std::io::Read; + let mut file = std::fs::File::open(path)?; + let mut hasher = sha2::Sha256::new(); + let mut buf = vec![0u8; 1 << 16]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + crate::progress::step(n)?; + } + Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect()) + } + + /// Bytes as a progress count. + fn count(bytes: u64) -> usize { + usize::try_from(bytes).unwrap_or(usize::MAX) + } + + impl FileSync { + /// The files of the source at `path` as they are now, each read whole to hash it. + pub fn read(path: &Path) -> Result { + let parts = parts(path); + let total = parts.iter().filter_map(|(_, p)| std::fs::metadata(p).ok()).map(|m| m.len()).sum(); + crate::progress::begin(count(total)); + let mut files = Vec::new(); + for (name, p) in parts { + let stamp = match len(&p)? { + None => None, + Some(len) => Some(FileStamp { len, sha256: sha256(&p)? }), + }; + files.push((name, stamp)); + } + Ok(FileSync { files }) + } + + /// Runs `read`, which reads the source at `path`, and returns with its result the files + /// as they were just before it ran (`None` when a part couldn't be read). Hashed before + /// the read, the record can only be older than what was read: a write while it ran, or + /// after, leaves the file unlike the record, so it shows as changed and is never written + /// over. That costs one more pass over the files than the read, about a second for a + /// part of the largest a reader takes (256 MiB) where SHA-256 runs at 250 MB/s. Errs, + /// without reading, when the run is cancelled while the files are hashed. + pub fn around(path: &Path, read: impl FnOnce() -> R) -> Result<(R, Option)> { + let before = match FileSync::read(path) { + Ok(sync) => Some(sync), + Err(DataError::Cancelled) => return Err(DataError::Cancelled), + Err(_) => None, + }; + Ok((read(), before)) + } + + /// The names of the files that differ now from this record: changed, added, removed or + /// unreadable. Empty when the source is as it was. A file of the size recorded is read + /// whole and hashed, so its work grows with the file: a caller on the desktop's UI + /// thread does it on a thread of its own. Errs only when the run is cancelled. + pub fn changed(&self, path: &Path) -> Result> { + let now = parts(path); + let mut changed = Vec::new(); + // Files there now that the record lacks (a `.cpg` added in another case); not the + // `.cpg` name `parts` falls back to when there is none. + for (name, p) in &now { + if !self.files.iter().any(|(n, _)| n == name) && p.exists() { + changed.push(name.clone()); + } + } + let total = self.files.iter().filter_map(|(_, s)| s.as_ref()).map(|s| s.len).sum(); + crate::progress::begin(count(total)); + for (name, was) in &self.files { + let Some((_, p)) = now.iter().find(|(n, _)| n == name) else { + // Out of the set now: changed if it held anything. + if was.is_some() { + changed.push(name.clone()); + } + continue; + }; + let same = match (was, len(p)) { + (None, Ok(None)) => true, + (Some(was), Ok(Some(len))) if len == was.len => match sha256(p) { + Ok(hash) => hash == was.sha256, + Err(DataError::Cancelled) => return Err(DataError::Cancelled), + Err(_) => false, + }, + _ => false, + }; + if !same { + changed.push(name.clone()); + } + } + Ok(changed) + } + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn a_shapefile_set_records_its_companions_and_the_missing_ones() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + std::fs::write(d.join("a.cpg"), "UTF-8").unwrap(); + let path = d.join("a.shp"); + let sync = FileSync::read(&path).unwrap(); + let names: Vec<(&str, bool)> = sync.files.iter().map(|(n, s)| (n.as_str(), s.is_some())).collect(); + assert_eq!(names, [("a.shp", true), ("a.shx", true), ("a.dbf", true), ("a.prj", false), ("a.cpg", true)]); + assert!(sync.changed(&path).unwrap().is_empty()); + } + + #[test] + fn changes_are_named_and_a_touch_is_not_one() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + let path = d.join("a.shp"); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + let sync = FileSync::read(&path).unwrap(); + // The same bytes written again: not a change. + std::fs::write(d.join("a.dbf"), "dbf").unwrap(); + assert!(sync.changed(&path).unwrap().is_empty()); + // Other bytes of the same length, a file added and one removed. + std::fs::write(d.join("a.dbf"), "DBF").unwrap(); + std::fs::write(d.join("a.prj"), "GEOGCS").unwrap(); + std::fs::remove_file(d.join("a.shx")).unwrap(); + assert_eq!(sync.changed(&path).unwrap(), ["a.shx", "a.dbf", "a.prj"]); + } + + /// A copy that keeps times (`cp -p`, `rsync -t`) can put back the size and time a file had + /// with other bytes: the hash still tells. + #[test] + fn a_rewrite_that_puts_the_time_back_is_a_change() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a.geojson"); + std::fs::write(&path, "{}").unwrap(); + let sync = FileSync::read(&path).unwrap(); + let time = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, "[]").unwrap(); + std::fs::File::options().write(true).open(&path).unwrap().set_modified(time).unwrap(); + assert_eq!(sync.changed(&path).unwrap(), ["a.geojson"]); + } + + #[test] + fn a_removed_cpg_is_named_as_it_was_spelled() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + for ext in ["shp", "shx", "dbf"] { + std::fs::write(d.join(format!("a.{ext}")), ext).unwrap(); + } + // Recorded under a spelling no file has now: as `a.CPG` is once removed on a + // case-sensitive file system (one that ignores case would find `a.cpg` there). + let path = d.join("a.shp"); + let mut sync = FileSync::read(&path).unwrap(); + let cpg = sync.files.iter_mut().find(|(n, _)| n == "a.cpg").unwrap(); + *cpg = ("a.CPG".into(), Some(FileStamp { len: 5, sha256: "00".into() })); + assert_eq!(sync.changed(&path).unwrap(), ["a.CPG"]); + } + + #[test] + fn a_write_while_reading_shows_as_a_change() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a.geojson"); + std::fs::write(&path, "{}").unwrap(); + let ((), sync) = FileSync::around(&path, || std::fs::write(&path, "[]").unwrap()).unwrap(); + assert_eq!(sync.unwrap().changed(&path).unwrap(), ["a.geojson"]); + let ((), sync) = FileSync::around(&path, || ()).unwrap(); + assert!(sync.unwrap().changed(&path).unwrap().is_empty()); + } + + #[test] + fn a_rows_digest_tells_a_non_finite_number_from_an_empty_cell() { + use crate::{Dataset, Feature, Field, FieldType, GeometryType, Value}; + let row = |v: Value| vec![Feature { id: 1, geometry: None, attrs: vec![v] }].into(); + let mut ds = Dataset::new("a".to_string(), GeometryType::None, vec![Field::new("v", FieldType::Double)]); + ds.features = row(Value::Float(f64::NAN)); + let digest = |v: Value| { + let mut d = ds.clone(); + d.features = row(v); + rows_digest(&d).unwrap() + }; + // Clearing it is an edit, and so is a project's saving it as null. + let reopened: Dataset = serde_json::from_value(serde_json::to_value(&ds).unwrap()).unwrap(); + assert_ne!(rows_digest(&reopened).unwrap(), rows_digest(&ds).unwrap()); + assert_eq!(rows_digest(&reopened).unwrap(), digest(Value::Null)); + assert_ne!(digest(Value::Float(f64::INFINITY)), digest(Value::Float(f64::NAN))); + assert_eq!(digest(Value::Float(f64::NAN)), rows_digest(&ds).unwrap()); + } + + /// The digest is saved in projects with no version: a change in how fields, coordinate + /// systems, geometry or values serialize changes every saved one, and each layer of an older + /// project would then show as edited. This pins it; a change needs a way to read the old one. + #[test] + fn a_rows_digest_is_pinned() { + use crate::{Coord, Dataset, Feature, Field, FieldType, Geometry, GeometryType, Value}; + let mut ds = Dataset::new( + "a".to_string(), + GeometryType::Point, + vec![Field::new("name", FieldType::Text), Field::new("n", FieldType::Integer)], + ); + ds.features = vec![Feature { + id: 1, + geometry: Some(Geometry::Point(Coord { x: -90.05, y: 35.15 })), + attrs: vec![Value::Text("Main St".into()), Value::Int(4)], + }] + .into(); + assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092); + } + + #[test] + fn a_record_round_trips() { + let sync = FileSync { files: vec![("a.geojson".into(), Some(FileStamp { len: 2, sha256: "ab".into() })), ("a.prj".into(), None)] }; + let text = serde_json::to_string(&sync).unwrap(); + assert_eq!(text, r#"{"files":[["a.geojson",{"len":2,"sha256":"ab"}],["a.prj",null]]}"#); + assert_eq!(serde_json::from_str::(&text).unwrap(), sync); + } +} diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 16b82d57a3..c4993fa232 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -659,6 +659,7 @@ fn template(p: &Project) -> (Project, TemplateLosses) { // A feature-less template has no loaded rows to save back to a source. l.postgis = None; l.gpkg = None; + l.synced = None; if let Some(rl) = &mut l.raster { rl.embedded = None; } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 27835cadc4..8ac1ebd6b9 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1401,8 +1401,11 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult { let src = rustgis_data::source::shp::ShpSource::open_path_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; return add_source_with(s, p, Origin::Path(path), Some(Box::new(src)), None); } - let mut read = rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; - read.0.source = Some(path.to_string()); + // Read ahead in the desktop app ([`add_data_ahead`]). + let read = match super::reads::take(s, &whole_key(path)) { + Some(read) => read, + None => read_shapefile_with(path, name, crs), + }?; return add_read(s, p, read, None); } // A large Shapefile is read in place, as a GeoPackage is; smaller ones are loaded whole. @@ -1580,19 +1583,72 @@ fn read_whole(p: &Value, path: &str) -> WholeRead { } std::fs::read(path).map_err(|e| format!("{path}: {e}")) })?; - match delimited { - Some(read) => Ok(read), - None => rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}")), + if let Some(read) = delimited { + return Ok(read); + } + let read = || rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}")); + // A file its edits can be written back to: the files as read, to compare with before then. + #[cfg(not(target_arch = "wasm32"))] + if rustgis_data::sync::is_syncable(path) { + return read_synced(path, read); + } + read() +} + +/// A Shapefile read whole in coordinate system `crs`, which replaces its `.prj`, with the record +/// of what was read ([`read_synced`]). +#[cfg(not(target_arch = "wasm32"))] +fn read_shapefile_with(path: &str, name: &str, crs: rustgis_data::raster::Crs) -> WholeRead { + read_synced(path, || { + let mut read = rustgis_data::io::read_shapefile_in(Path::new(path), name, Some(crs)).map_err(|e| e.to_string())?; + read.0.source = Some(path.to_string()); + Ok(read) + }) +} + +/// `read`, the read of the file at `path`, with the record of what it read: the files hashed +/// just before ([`rustgis_data::sync::FileSync::around`]) and the digest of the rows read. Its +/// work grows with the file, so the desktop app makes it off the UI thread ([`add_data_ahead`]). +/// When the files couldn't be hashed the layer has no record, and a note says it can't be +/// compared with its file later. +#[cfg(not(target_arch = "wasm32"))] +fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { + // A file past what the readers take (a Shapefile's `.dbf` too) is refused by the read: not + // hashed first. + let over = |p: &Path| std::fs::metadata(p).is_ok_and(|m| m.len() > rustgis_data::io::MAX_BYTES); + let shp = path.to_ascii_lowercase().ends_with(".shp"); + if over(Path::new(path)) || (shp && over(&Path::new(path).with_extension("dbf"))) { + return read(); + } + let (read, files) = rustgis_data::sync::FileSync::around(Path::new(path), read).map_err(|e| format!("{path}: {e}"))?; + let (mut ds, mut report) = read?; + // Kept whole, as the record checks it later: a relative path would be taken from wherever + // RustGIS runs then, not where the file was read. + if files.is_some() + && let Ok(full) = std::path::absolute(path) + { + ds.source = Some(full.display().to_string()); } + report.synced = match files { + Some(files) => { + Some(rustgis_data::sync::Synced { files, rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))? }) + } + None => { + report.messages.push(format!( + "{path}: a part of it couldn't be read to record it, so RustGIS can't tell later whether the file changed since; add it again to compare it" + )); + None + } + }; + Ok((ds, report)) } /// Add Data's read of the file `p` names, made before it runs ([`super::reads`]): a raster read /// whole or converted by GDAL, a GeoJSON read whole, delimited text, a spreadsheet or a city model -/// (CityJSON or CityGML) read into its layers. `None` for +/// (CityJSON or CityGML) read into its layers, a Shapefile read whole. `None` for /// any other file, which [`add_data`] reads as it adds it. Each is picked only where `add_data` -/// takes that read, so a read made is never wasted; a few it makes whole are left to it (a GeoJSON -/// with `"stream"` given, a `.json` with a coordinate system or layer given, a large GeoJSON that -/// isn't a FeatureCollection). +/// takes that read, so a read made is never wasted; a few it makes whole are left to it (a `.json` +/// with a coordinate system or layer given, a large GeoJSON that isn't a FeatureCollection). #[cfg(not(target_arch = "wasm32"))] pub(crate) fn add_data_ahead(p: &Value) -> Option { use super::reads::ahead; @@ -1616,19 +1672,42 @@ pub(crate) fn add_data_ahead(p: &Value) -> Option { } let given = |k: &str| ps(p, k).is_some_and(|v| !v.trim().is_empty()); let lower = path.to_ascii_lowercase(); - // A GeoJSON read whole: small, with no "stream" given (which add_data decides by), and not a - // CityJSON model (named so, or a .json with a coordinate system or a layer given). + // A GeoJSON read whole: small with no "stream" given (which add_data decides by), or any size + // (within the reader's limit) with `"stream": false`; and not a CityJSON model (named so, or a + // .json with a coordinate system or a layer given). let geojson = is_geojson(&path) && !rustgis_data::cityjson::is_cityjson_name(&path) - && b(p, "stream").is_none() + && b(p, "stream") != Some(true) && !(lower.ends_with(".json") && (given("crs") || given("layer"))) - && std::fs::metadata(&path).is_ok_and(|m| m.is_file() && m.len() <= rustgis_data::source::geojson::IN_PLACE_ABOVE); + && std::fs::metadata(&path) + .is_ok_and(|m| m.is_file() && (b(p, "stream") == Some(false) || m.len() <= rustgis_data::source::geojson::IN_PLACE_ABOVE)); // A city model: its layers, read whole. A model named `.json` (not `.city.json`) is a file of // several layers only with a coordinate system or a layer given, which Add Data reads as it runs. if rustgis_data::io::is_multi_layer_name(&path) && is_city_model_path(&path) { let (key, q) = (city_key(&path), p.clone()); return Some(ahead(NAME, key, move || read_city_file(&q, &path))); } + // A Shapefile read whole, in the coordinate system given if one is: as add_data reads it (one + // past the whole-file reader's cap, or with more records than it takes, is read in place). + if lower.ends_with(".shp") { + let q = past_cap_shapefile(&path, p).unwrap_or_else(|| p.clone()); + let stream = b(&q, "stream").unwrap_or_else(|| { + rustgis_data::source::shp::record_count(Path::new(&path)).is_some_and(|n| n > rustgis_map::source::STREAM_ABOVE) + }); + if stream { + return None; + } + let key = whole_key(&path); + return match ps(&q, "crs").map(str::trim).filter(|c| !c.is_empty()) { + Some(text) => { + // One add_data can't parse it refuses as it runs. + let crs = rustgis_data::raster::Crs::parse(text).ok()?; + let name = Path::new(&path).file_stem().and_then(|n| n.to_str()).unwrap_or("layer").to_string(); + Some(ahead(NAME, key, move || read_shapefile_with(&path, &name, crs))) + } + None => Some(ahead(NAME, key, move || read_whole(&q, &path))), + }; + } let (key, q) = (whole_key(&path), p.clone()); if rustgis_data::sheets::is_sheet_name(&path) { return Some(ahead(NAME, key, move || read_sheet_file(&q, &path))); @@ -4731,7 +4810,11 @@ pub(crate) fn add_read( } let n = ds.features.len(); let empty_before = super::map_is_empty(s); + let synced = report.synced; let id = add_dataset(s, ds); + if let Some(l) = s.map_mut().layer_mut(id) { + l.synced = synced; + } if zoom.unwrap_or(empty_before) && let Some(e) = s.map().layer(id).map(|l| l.extent()) { diff --git a/crates/engine/src/cmd/layer_copy.rs b/crates/engine/src/cmd/layer_copy.rs index 7e9983a3e8..0caf4d13ac 100644 --- a/crates/engine/src/cmd/layer_copy.rs +++ b/crates/engine/src/cmd/layer_copy.rs @@ -59,6 +59,9 @@ fn copy(s: &mut Session, p: &Value) -> CmdResult { .map(|l| { let mut l = l.clone(); l.selection.clear(); + // A copy isn't the layer read from its file: it can't say whether the file has its + // edits, nor write them back. + l.synced = None; l }) .collect(); diff --git a/crates/engine/src/cmd/layout.rs b/crates/engine/src/cmd/layout.rs index e6c9b80541..d8789110dc 100644 --- a/crates/engine/src/cmd/layout.rs +++ b/crates/engine/src/cmd/layout.rs @@ -1686,6 +1686,7 @@ fn publication(s: &mut Session, p: &Value) -> CmdResult { // Simplified sketches aren't the source's rows: saving them would overwrite it. layer.gpkg = None; layer.postgis = None; + layer.synced = None; layer.dataset.features = l .dataset .features diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 3b163efc13..1b51c6c684 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -36,6 +36,7 @@ pub mod plugins; pub mod portals; pub mod presentation; pub mod project; +pub mod publish; pub mod qgis; pub mod raster; pub mod reads; @@ -112,6 +113,7 @@ pub fn all() -> Vec { v.extend(sql::commands()); v.extend(pg::commands()); v.extend(gpkg::commands()); + v.extend(publish::commands()); v.extend(plugins::commands()); v.extend(sharing::commands()); v.extend(stac::commands()); diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs new file mode 100644 index 0000000000..97887904b8 --- /dev/null +++ b/crates/engine/src/cmd/publish.rs @@ -0,0 +1,133 @@ +//! A layer kept in the project and the file it was read from: whether its edits and the file are +//! in step (#1385). + +use crate::{CmdResult, CommandSpec as C, Session}; +use serde_json::Value; + +pub fn commands() -> Vec { + vec![ + C::new("source.syncState", "Source File State", "Data › Data Source", sync_state).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( + r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile or .geojson file"} — whether it has edits its file lacks ("edited"), the file changed since it was read ("sourceChanged"), both ("both") or neither ("inSync"), and which of the file's parts changed"#, + ), + ] +} + +/// The key a check of layer `id`'s files made ahead is taken by (see [`super::reads`]). +#[cfg(not(target_arch = "wasm32"))] +fn ahead_key(id: u32) -> String { + format!("syncState:{id}") +} + +/// A check of a layer made ahead: the record, path and rows checked (the rows shared with the +/// layer, not copied), the rows' digest and what changed in the files. +#[cfg(not(target_arch = "wasm32"))] +struct Checked { + synced: rustgis_data::sync::Synced, + path: String, + dataset: rustgis_data::Dataset, + rows: Result, + changed: Result, String>, +} + +#[cfg(not(target_arch = "wasm32"))] +impl Checked { + /// Whether it checked layer `l` as it is: the same record and path, and the same rows (the + /// same allocation: an edit copies them), fields and coordinate system. + fn of(&self, l: &rustgis_map::Layer) -> bool { + l.synced.as_ref() == Some(&self.synced) + && l.dataset.source.as_deref() == Some(self.path.as_str()) + && l.dataset.features.shares(&self.dataset.features) + && l.dataset.fields == self.dataset.fields + && l.dataset.crs == self.dataset.crs + } +} + +/// The check `source.syncState` makes before it runs in the desktop app: every part of the +/// file read and hashed, and the layer's rows hashed, on a thread of its own with progress and +/// Cancel. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn sync_ahead(s: &Session, p: &Value) -> Option { + let id = s.layer_id(p).ok()?; + let l = s.map().layer(id)?; + let (synced, path, dataset) = (l.synced.clone()?, l.dataset.source.clone().filter(|p| local_file(p))?, l.dataset.clone()); + Some(super::reads::ahead("Source File State", ahead_key(id), move || { + let changed = synced.files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string()); + let rows = rustgis_data::sync::rows_digest(&dataset).map_err(|e| e.to_string()); + Checked { synced, path, dataset, rows, changed } + })) +} + +#[cfg(target_arch = "wasm32")] +pub(crate) fn sync_ahead(_: &Session, _: &Value) -> Option { + None +} + +/// Compares the layer and the file it was read from, each with the record of them made when it +/// was read: whether the layer's rows differ from the rows read (edits the file lacks), and which +/// parts of the file differ from the parts read. It doesn't compare the layer with the file as +/// it is now, so the same edit made to both shows as `both`. Each part of the size recorded is +/// read and hashed, and the rows are hashed, so the work grows with the file. The desktop app +/// does it on a thread of its own first ([`sync_ahead`]), with progress and **Cancel**, and the +/// command takes that check as it lands; when the layer changed meanwhile it says so rather than +/// check again here, which would hold the window. The assistant, scripts and the command line +/// check where they ask. +fn sync_state(s: &mut Session, p: &Value) -> CmdResult { + let id = s.layer_id(p)?; + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { + if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { + return Err(format!( + "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", + l.name + )); + } + return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); + }; + // A project from elsewhere names the path: only a local file of a kind a record is made for + // is opened, never a network share (opening one can send the user's login to its host). + if !local_file(path) { + return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); + } + #[cfg(target_arch = "wasm32")] + { + let _ = (synced, path); + Err("the web app can't read files on this computer".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + let (was, files, path) = (synced.rows, synced.files.clone(), path.clone()); + let ahead = super::reads::take::(s, &ahead_key(id)); + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (rows, changed) = match ahead { + Some(checked) if checked.of(l) => (checked.rows?, checked.changed?), + // Checking again here would hold the window: the check made on its thread is dropped. + Some(_) => return Err("the layer or its file's record changed while it was checked: check again".into()), + None => ( + rustgis_data::sync::rows_digest(&l.dataset).map_err(|e| e.to_string())?, + files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?, + ), + }; + let edited = rows != was; + let state = match (edited, !changed.is_empty()) { + (false, false) => "inSync", + (true, false) => "edited", + (false, true) => "sourceChanged", + (true, true) => "both", + }; + Ok(serde_json::json!({"layer": id, "path": path, "state": state, "edited": edited, "changed": changed})) + } +} + +/// Whether `path` names a file on this computer a record is made for: a Shapefile or `.geojson` +/// file, not an address, a database or a network share (`\\host\share`, `//host/share`, +/// `/\host\share`, `\\?\UNC\host\share`). Windows' long form of a local path (`\\?\C:\…`) is one +/// on this computer. +fn local_file(path: &str) -> bool { + let b = rustgis_map::paths::local_path(path).as_bytes(); + let sep = |i: usize| b.get(i).is_some_and(|c| matches!(c, b'/' | b'\\')); + // Windows takes any two leading separators, of either kind, for a network share, unless they + // begin the long form of a path on a drive. + let drive = b.get(2) == Some(&b'?') && sep(3) && b.get(4).is_some_and(u8::is_ascii_alphabetic) && b.get(5) == Some(&b':') && sep(6); + let share = sep(0) && sep(1) && !drive; + rustgis_map::paths::is_local(path) && rustgis_data::sync::is_syncable(path) && !share +} diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index 935028a3d1..0fbbe4d1c9 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -1,8 +1,8 @@ //! Reads the desktop app makes on a thread of its own before a command adds what they read //! (#1291 step 3): Add Data's whole-file reads (a raster, a raster GDAL converts, a GeoJSON, //! delimited text or spreadsheet read whole, a CityJSON or CityGML model's layers), Reload Data, Load All -//! Features, Convert To Annotation's read of a layer read in place, native models or scripts and the -//! Error Inspector's topology check (#1291). [`read_ahead`] +//! Features, Convert To Annotation's read of a layer read in place, native models or scripts, the +//! Error Inspector's topology check (#1291) and Source File State's hashing of a layer's file. [`read_ahead`] //! takes what the read needs of the session, [`ReadAhead::run`] reads on the thread, and [`read_land`] //! runs the admitted command under its initiating owner: the command takes //! it instead of reading again, so what it adds, says, logs and puts in Undo is the same as when @@ -139,7 +139,8 @@ pub(crate) fn prepare(s: &Session, op: Authorized, owner: Continuatio (name, work, None) } "topology.validate" => ("Error Inspector", super::design::topology_ahead(s, params), None), - _ => return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()), + "source.syncState" => ("Source File State", super::publish::sync_ahead(s, params), None), + _ => return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives, topology and Source File State prepare background work".into()), }; let name = work.as_ref().map_or(name, |work| work.name); Ok(ReadAhead { name, op, owner, source, work }) diff --git a/crates/engine/src/cmd/workspace.rs b/crates/engine/src/cmd/workspace.rs index cb1b8c2bc5..dce39da42a 100644 --- a/crates/engine/src/cmd/workspace.rs +++ b/crates/engine/src/cmd/workspace.rs @@ -604,6 +604,8 @@ fn snapshot(m: &mut rustgis_map::Map, e: &Extent) -> rustgis_map::Map { c.dataset.features = l.dataset.features.iter().filter(|f| f.geometry.as_ref().is_some_and(|g| g.extent().intersects(e))).cloned().collect(); c.selection.clear(); + // Only part of the rows read: no longer the layer its file's record describes. + c.synced = None; c.touch_geometry(); out.layers.push(c); } diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 8d21782204..993b19583b 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -151,11 +151,12 @@ impl Session { | "map.reloadTiles" | "map.repairTiles" | "topology.validate" + | "source.syncState" ) { - return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives and topology prepare background work".into()); + return Err("only Add Data, Load All, Reload Data, Convert To Annotation, models, scripts, tile archives, topology and Source File State prepare background work".into()); } let mut params = params.as_object().cloned().ok_or("read parameters must be an object")?; - if matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") { + if matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert" | "source.syncState") { let id = self.layer_id(&Value::Object(params.clone()))?; params.insert("layer".into(), json!(id)); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a5cbe4c7e5..cb54fa26df 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -39253,3 +39253,134 @@ fn a_model_trained_within_the_work_limit_says_so() { .unwrap(); assert_eq!(r["epochs"], 3, "{r}"); } + +/// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read +/// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed +/// since, and neither once an edit is undone. The record is saved with the project. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_layer_read_from_a_file_says_whether_it_and_the_file_are_in_step() { + let dir = std::env::temp_dir().join(format!("rustgis-engine-sync-state-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut s = world(); + for file in ["towns.shp", "towns.geojson"] { + let path = dir.join(file); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": path})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path, "name": file})).unwrap()["layer"].as_u64().unwrap(); + let state = |s: &mut Session| s.run("source.syncState", &json!({"layer": id})).unwrap(); + assert_eq!(state(&mut s)["state"], "inSync", "{file}"); + s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + assert_eq!(state(&mut s)["state"], "edited", "{file}"); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(state(&mut s)["state"], "inSync", "{file}"); + assert!(s.map().layer(id as u32).unwrap().synced.is_some()); + } + // Saved and opened again, both still know their files as read. + let project = dir.join("towns.rgis"); + s.run("project.saveAs", &json!({"path": project.display().to_string()})).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.open", &json!({"path": project.display().to_string()})).unwrap(); + for layer in ["towns.shp", "towns.geojson"] { + assert_eq!(t.run("source.syncState", &json!({"layer": layer})).unwrap()["state"], "inSync", "{layer}"); + } + // The desktop app hashes the file on a thread of its own; the command takes that check. + { + use crate::cmd::reads::{read_ahead, read_land}; + let here = t.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + let ahead = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + assert!(ahead.reads()); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + assert_eq!(read_land(&mut t, read).unwrap(), here); + assert!(t.read_ahead.is_none(), "the check was taken"); + // The layer edited while it was checked: said, not checked again here. + let read = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})) + .unwrap() + .run(&rustgis_analysis::control::Control::stoppable()) + .unwrap(); + t.run("edit.setValue", &json!({"layer": "towns.shp", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + let e = read_land(&mut t, read).unwrap_err(); + assert!(e.contains("changed while it was checked"), "{e}"); + t.run("edit.undo", &Value::Null).unwrap(); + // Cancelled as it starts, the hashing looks and stops. + let control = rustgis_analysis::control::Control::cancelled_once_started(); + let e = read_ahead(&mut t, "source.syncState", &json!({"layer": "towns.shp"})).unwrap().run(&control).err().unwrap(); + assert!(e.to_lowercase().contains("cancel") && control.looked() > 0, "{e}"); + } + // The same bytes written again is no change; other bytes are, named by file. + let dbf = dir.join("towns.dbf"); + let bytes = std::fs::read(&dbf).unwrap(); + std::fs::write(&dbf, &bytes).unwrap(); + assert_eq!(s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap()["state"], "inSync"); + let mut other = bytes.clone(); + *other.last_mut().unwrap() ^= 1; + std::fs::write(&dbf, other).unwrap(); + let state = s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap(); + assert_eq!((state["state"].as_str(), state["changed"].clone()), (Some("sourceChanged"), json!(["towns.dbf"]))); + s.run("edit.setValue", &json!({"layer": "towns.shp", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + assert_eq!(s.run("source.syncState", &json!({"layer": "towns.shp"})).unwrap()["state"], "both"); + // Read ahead (the desktop app's Add Data), in a coordinate system given: the record comes + // with the read. + { + use crate::cmd::reads::{read_ahead, read_land}; + let p = json!({"path": dir.join("towns.shp"), "crs": "EPSG:4326", "name": "ahead"}); + let ahead = read_ahead(&mut s, "layer.addData", &p).unwrap(); + assert!(ahead.reads()); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + read_land(&mut s, read).unwrap(); + assert!(s.map().layer_by_name("ahead").unwrap().synced.is_some()); + assert_eq!(s.run("source.syncState", &json!({"layer": "ahead"})).unwrap()["state"], "inSync"); + } + // A copy (pasted, or an offline area's part of the rows) isn't the layer read from the file: + // it has no record. + s.run("layer.copy", &json!({"layer": "towns.shp"})).unwrap(); + s.run("layer.paste", &Value::Null).unwrap(); + let pasted = s.map().layers.iter().filter(|l| l.dataset.source.as_deref().is_some_and(|p| p.ends_with("towns.shp"))).count(); + assert!(pasted >= 2 && s.map().layers.iter().filter(|l| l.synced.is_some()).count() == 3, "only the layers read hold one"); + s.run("offline.download", &json!({"name": "Here", "extent": [-180.0, -90.0, 180.0, 90.0]})).unwrap(); + assert!(s.project.offline_areas[0].data.layers.iter().all(|l| l.synced.is_none())); + // A project from elsewhere names the path: a network share or a file no record is made for + // is refused without being opened. + let shares = [r"\\host\share\towns.shp", "//host/share/towns.shp", r"/\host\share\towns.shp", r"\/host/share/towns.shp"]; + let more = [r"file:///\\host\share\towns.shp", r"\\?\UNC\host\share\towns.shp", "towns.json"]; + for path in shares.into_iter().chain(more) { + let id = s.map().layer_by_name("towns.shp").unwrap().id; + s.map_mut().layer_mut(id).unwrap().dataset.source = Some(path.into()); + let e = s.run("source.syncState", &json!({"layer": id})).unwrap_err(); + assert!(e.contains("isn't a Shapefile or .geojson file on this computer"), "{path}: {e}"); + // Nor read on a thread of its own: the desktop's check makes no read, and refuses as it lands. + let ahead = crate::cmd::reads::read_ahead(&mut s, "source.syncState", &json!({"layer": id})).unwrap(); + assert!(!ahead.reads(), "{path}"); + let read = ahead.run(&rustgis_analysis::control::Control::stoppable()).unwrap(); + assert!(crate::cmd::reads::read_land(&mut s, read).is_err(), "{path}"); + } + // Windows' long form of a path on a drive is a file on this computer: checked (and missing here). + let id = s.map().layer_by_name("towns.shp").unwrap().id; + // In the test's own folder on Windows (a file there by chance can't match); elsewhere it's + // just a name no file has. + #[cfg(windows)] + let long = format!(r"\\?\{}", dir.join("missing").join("towns.shp").display()); + #[cfg(not(windows))] + let long = r"\\?\C:\missing\towns.shp".to_string(); + s.map_mut().layer_mut(id).unwrap().dataset.source = Some(long); + let v = s.run("source.syncState", &json!({"layer": id})).unwrap(); + assert!(v["changed"].as_array().unwrap().iter().any(|c| c.as_str().is_some_and(|c| c.ends_with("towns.shp"))), "{v}"); + // A part that can't be read to record it (a folder where the .prj would be): the layer is + // added without a record, and says so; one read cleanly says nothing of it. + let said = |s: &Session, mark: u64| s.notices_since(mark).iter().filter(|n| n.text.contains("couldn't be read to record it")).count(); + let odd = dir.join("odd"); + std::fs::create_dir_all(odd.join("odd.prj")).unwrap(); + for ext in ["shp", "shx", "dbf"] { + std::fs::copy(dir.join(format!("towns.{ext}")), odd.join(format!("odd.{ext}"))).unwrap(); + } + let mark = s.notice_mark(); + let id = s.run("layer.addData", &json!({"path": odd.join("odd.shp")})).unwrap()["layer"].as_u64().unwrap(); + assert!(s.map().layer(id as u32).unwrap().synced.is_none()); + assert_eq!(said(&s, mark), 1); + let mark = s.notice_mark(); + s.run("layer.addData", &json!({"path": dir.join("towns.geojson"), "name": "clean"})).unwrap(); + assert_eq!(said(&s, mark), 0); + // A layer that wasn't read from such a file has nothing to compare with. + assert!(s.run("source.syncState", &json!({"layer": "Cities"})).is_err()); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 98e7b958fb..5b8837fd2e 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -229,6 +229,11 @@ pub struct Layer { /// The GeoPackage table this layer's edits save to. #[serde(default, skip_serializing_if = "Option::is_none")] pub gpkg: Option, + /// Set for a layer read whole from a Shapefile or GeoJSON file (its `dataset.source`): the + /// file and the layer as they were when last in step, to tell edits the file lacks and + /// changes made to the file since. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub synced: Option, /// The saved connection it was added from (its id, in the app's preferences): a layer whose /// source is read again logs in with that connection's login, and a project opened where /// the connection or its login is missing asks for one. Empty for a layer that came from no @@ -462,6 +467,7 @@ impl Layer { source: None, postgis: None, gpkg: None, + synced: None, connection: String::new(), anonymous: false, projected: Default::default(), @@ -2927,6 +2933,7 @@ impl Layer { source, postgis, gpkg, + synced: _, connection: _, anonymous: _, instance_id: _, diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 3e5e847fe8..85907a73e6 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -238,6 +238,14 @@ pub struct GpkgEdit { pub versions: Versions, } +/// FNV-1a's starting value. +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; + +/// FNV-1a over `bytes`, from `h`. +fn fnv1a(h: u64, bytes: &[u8]) -> u64 { + bytes.iter().fold(h, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)) +} + /// How a loaded row is told apart from the same row edited: a hash of its geometry and /// attributes (FNV-1a over their JSON, the same in every build). Non-finite attributes also /// contribute their position and native bits, since output JSON substitutes Null for them. @@ -247,7 +255,7 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { impl Fnv { /// Feeds bytes without buffering; also used for values JSON cannot distinguish. fn bytes(&mut self, buf: &[u8]) { - self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + self.0 = fnv1a(self.0, buf); } } impl std::io::Write for Fnv { @@ -259,10 +267,10 @@ pub fn row_hash(f: &rustgis_data::Feature) -> u64 { Ok(()) } } - let mut h = Fnv(0xcbf2_9ce4_8422_2325); + let mut h = Fnv(FNV_OFFSET); // As before, a row that can't be written hashes as no bytes. if serde_json::to_writer(&mut h, &(&f.geometry, &f.attrs)).is_err() { - return 0xcbf2_9ce4_8422_2325; + return FNV_OFFSET; } for (index, value) in f.attrs.iter().enumerate() { if let rustgis_data::Value::Float(number) = value diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 756ad691f6..c6dcc2a214 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1133,6 +1133,27 @@ pub struct RustGisApp { seen_project: Option, } +/// Says what `source.syncState` found, once its check lands: whether the layer and its file are +/// in step, and which parts of the file changed. A check that failed was said already. +#[cfg(not(target_arch = "wasm32"))] +fn sync_state_said(app: &mut RustGisApp, landed: Landed) { + let Landed::Ran(Ok(v)) = landed else { return }; + let layer = v.get("layer").and_then(Value::as_u64).and_then(|id| u32::try_from(id).ok()); + let layer = layer.and_then(|id| app.session.map().layer(id)).map(|l| l.name.clone()).unwrap_or_default(); + let path = v.get("path").and_then(Value::as_str).unwrap_or_default(); + let file = std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or(path).to_string(); + let parts: Vec<&str> = v.get("changed").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect(); + let parts = parts.join(", "); + let args = [("layer", layer.as_str()), ("file", file.as_str()), ("parts", parts.as_str())]; + let text = match v.get("state").and_then(Value::as_str) { + Some("inSync") => i18n::fmt("{layer} is in step with {file}", &args), + Some("edited") => i18n::fmt("{layer} was edited since it was read from {file}", &args), + Some("sourceChanged") => i18n::fmt("{file} changed since {layer} was read from it ({parts})", &args), + _ => i18n::fmt("{layer} was edited, and {file} changed, since the layer was read from it ({parts})", &args), + }; + app.session.notify(text, false); +} + impl RustGisApp { /// The layer to export: `layer` (an id or name, as commands take it; the dialog's is checked /// against its `instance`, so a reused id isn't taken for it), else the active layer. @@ -2261,6 +2282,11 @@ impl RustGisApp { let params = if params.is_null() { json!({}) } else { params }; return readjob::run(self, "topology.validate", params); } + // Comparing a layer with its file hashes the file: on a thread of its own, and what + // it found said once it lands (Command Search shows no answer). + if id == "source.syncState" { + return readjob::run_then(self, "source.syncState", params, Some(sync_state_said)); + } // Converting the labels of a layer read in place reads all of its source first, on a // thread of its own, whoever asks (a click, the assistant). if id == "labels.convert" @@ -3333,6 +3359,10 @@ impl RustGisApp { self.ui.pending_shapefile = Some(json!({"crs": text})); open("shapefile"); } else if let Some(path) = dialogs::pick_open(&["shp", "zip"], tl!("Shapefile")) { + // Read on a thread of its own, as Add Data's other files are. + #[cfg(not(target_arch = "wasm32"))] + return readjob::run(self, "layer.addData", json!({"path": path, "crs": text})); + #[cfg(target_arch = "wasm32")] return self.run("layer.addData", json!({"path": path, "crs": text})); } } diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index a4ee355a4f..021cbbb237 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -78,7 +78,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: let r = app.run(command, p); return answered(app, then, r); } - let layer = matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert") + let layer = matches!(command, "layer.loadAll" | "layer.reloadData" | "labels.convert" | "source.syncState") .then(|| app.session.layer_id(&p).ok().and_then(|id| app.session.map().layer(id)).map(|l| (l.instance_id(), l.name.clone()))) .flatten(); let notice_mark = app.session.notice_mark(); @@ -1097,6 +1097,55 @@ mod tests { assert!(l.is_streamed() && !l.labels.enabled); } + /// Source File State hashes the file on a thread of its own and says what it found once + /// it lands: Command Search shows no answer. + #[test] + fn a_source_file_state_check_runs_on_a_thread_and_says_what_it_found() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("places.geojson"); + publish_point(&path, 1.0); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let id = app.run("layer.addData", json!({"path": path})).unwrap()["layer"].as_u64().unwrap(); + app.reads_off_frame = true; + let mark = app.session.notice_mark(); + assert_eq!(app.run_by_user("source.syncState", json!({"layer": id})).unwrap()["started"], "Source File State"); + landed(&mut app, 60); + let said: Vec = app.session.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|t| t == "places is in step with places.geojson"), "{said:?}"); + } + + /// The desktop's Add Data reads a Shapefile (in a coordinate system given, as Import + /// Shapefile asks) and a GeoJSON with `"stream": false` on a thread of its own, recording the + /// file with the read. + #[test] + fn whole_files_are_read_and_recorded_on_a_thread() { + let dir = tempfile::tempdir().unwrap(); + let geojson = dir.path().join("places.geojson"); + publish_point(&geojson, 1.0); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("layer.addData", json!({"path": geojson, "name": "seed"})).unwrap(); + let shp = dir.path().join("towns.shp"); + app.run("share.exportLayer", json!({"layer": "seed", "path": shp})).unwrap(); + app.reads_off_frame = true; + for (p, name) in [ + (json!({"path": shp, "crs": "EPSG:4326", "name": "towns"}), "towns"), + // Another system than the file's: the rows recorded are the rows as read in it. + (json!({"path": shp, "crs": "EPSG:3857", "name": "projected"}), "projected"), + (json!({"path": geojson, "stream": false, "name": "whole"}), "whole"), + ] { + assert_eq!(run(&mut app, "layer.addData", p).unwrap()["started"], "Add Data", "{name}"); + landed(&mut app, 60); + let id = app.session.map().layer_by_name(name).unwrap().id; + assert!(app.session.map().layer(id).unwrap().synced.is_some(), "{name}"); + if name == "projected" { + assert_eq!(app.session.map().layer(id).unwrap().dataset.crs, rustgis_engine::data::raster::Crs::parse("EPSG:3857").ok()); + } + app.reads_off_frame = false; + assert_eq!(app.run("source.syncState", json!({"layer": id})).unwrap()["state"], "inSync", "{name}"); + app.reads_off_frame = true; + } + } + fn reload_fixture() -> (tempfile::TempDir, RustGisApp, u32, std::path::PathBuf) { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("streamed.geojson"); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 91d15a3179..175fb1bcd5 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -302,6 +302,30 @@ the default and discarding edits defaults to false. Only an explicit `{"layer":42,"path":"/data/cities.fgb","table":null}`; `table` is a string for a selected table or raster subdataset. +### Source file state + +A layer read whole from a Shapefile or a `.geojson` file keeps its features in the project, so +edits change the project, not the file. It also keeps a record of what was read: each part of +the file's size and SHA-256 hash, and a hash of the layer's rows. `source.syncState` compares +the layer with the rows read, and the file with the parts read, and answers `inSync`, `edited` +(the layer has edits since it was read), `sourceChanged` (the file changed since it was read) +or `both`, with the names of the parts that changed. It doesn't compare the layer with the file +as it is now: the same edit made to both shows as `both`. Only the bytes count: writing the +same bytes again isn't a change, while a copy that puts back the file's size and time with other +bytes is one. +Undoing every edit makes the layer match its record again, so it reads `inSync` if the file +hasn't changed either. A number that isn't finite (NaN, infinity), which a +project saves as empty, shows the layer as `edited` once its project is opened again. On the desktop the file and the rows are hashed +in the background, with progress and Cancel in the status bar. + +The record is made just before the file is read, so a file written to while it was being read +shows as changed, unless the write was undone before RustGIS looked again. A layer read in +place and a `.json` file have no record. The record is saved with the project; the web app +can't check it. + +On the desktop, make the layer active and choose **Source File State** in Command Search; what it +found shows in the status bar. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. + ## Keyboard shortcuts | Keys | Action |