diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 0b2e2eb1dc..061d93507a 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -52,7 +52,7 @@ pub fn defs() -> Vec { }, ToolDef { name: "run_command", - description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Each run is one undo step. Commands that read or write files, use the network or replace the project ask the user first.", + description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Project edits are undoable. Source publication and other external file writes cannot be undone. Commands that read or write files, use the network or replace the project ask the user first.", schema: json!({ "type": "object", "properties": { @@ -648,6 +648,7 @@ fn words(a: &Value) -> Result, String> { Ok(words) } +/// Ranks command descriptions and reports project changes independently of undo snapshots. fn search_commands(s: &Session, a: &Value) -> Result { let words = words(a)?; let limit = limit_arg(a, 12, 40); @@ -664,7 +665,7 @@ fn search_commands(s: &Session, a: &Value) -> Result { (c.id.to_lowercase(), c.label.to_lowercase(), c.location.to_lowercase(), c.params.to_lowercase()); let sc = score(&words, &[&id, &label, &loc, ¶ms]); (sc > 0).then(|| { - (sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.mutates})) + (sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.changes_project()})) }) }) .collect(); @@ -1090,6 +1091,17 @@ mod tests { assert_eq!(data["[redacted]"], "private field"); } + /// Command discovery distinguishes publishing's project change from a read-only query. + #[test] + fn publishing_reports_a_project_change_in_command_search() { + let s = Session::new(rustgis_engine::Project::default()); + for (id, changes) in [("source.publish", true), ("source.syncState", false)] { + let out = search_commands(&s, &json!({"query":id, "limit":40})).unwrap(); + let command = out["commands"].as_array().unwrap().iter().find(|c| c["id"] == id).unwrap(); + assert_eq!(command["changesProject"], changes, "{id}"); + } + } + #[test] fn wrappers_after_a_prefix_are_made_plain_too() { let origin = "https://assistant-login.example"; diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index 46e9491126..222495973c 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -203,6 +203,72 @@ fn retype_columns(dbf: &mut [u8], from: u8, to: u8) -> Vec { changed } +/// A column of a `.dbf` as its header describes it: its name's bytes (up to the first NUL), +/// type letter, length in bytes and decimals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Column { + pub name: Vec, + pub ty: u8, + pub len: u8, + pub decimals: u8, +} + +#[cfg(not(target_arch = "wasm32"))] +/// A `.dbf` read as its bytes, without decoding a value: its columns, and each record that isn't +/// marked deleted, as the bytes of each of its columns. `None` when the header doesn't describe +/// records that fit the file. +pub(crate) struct RawTable<'a> { + pub columns: Vec, + pub records: Vec>, + /// The records marked deleted, which aren't among `records`. + pub deleted: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +/// Reads `dbf` as a [`RawTable`]: what a rewrite is compared with, byte for byte, where a +/// decoded value would hide a difference (text with leading spaces, which a read trims; a code +/// page's bytes). +pub(crate) fn raw_table(dbf: &[u8]) -> Option> { + let le16 = |at: usize| Some(usize::from(u16::from_le_bytes([*dbf.get(at)?, *dbf.get(at + 1)?]))); + let count = usize::try_from(u32::from_le_bytes(dbf.get(4..8)?.try_into().ok()?)).ok()?; + let (header_len, record_len) = (le16(8)?, le16(10)?); + let mut columns = Vec::new(); + for start in (32..header_len.min(dbf.len())).step_by(32) { + let d = dbf.get(start..start + 32).filter(|d| d.first() != Some(&0x0D)); + let Some(d) = d else { break }; + let name = d.get(..11)?; + let name = name.get(..name.iter().position(|b| *b == 0).unwrap_or(11))?.to_vec(); + columns.push(Column { name, ty: *d.get(11)?, len: *d.get(16)?, decimals: *d.get(17)? }); + } + let width: usize = 1 + columns.iter().map(|c| usize::from(c.len)).sum::(); + if width != record_len || record_len == 0 { + return None; + } + let body = dbf.get(header_len..)?; + // Every byte must belong to a counted record or the optional EOF marker. A rewrite + // must not discard uncounted records or other data left past the declared body. + let trailing = body.get(count.checked_mul(record_len)?..)?; + if !trailing.is_empty() && trailing != [0x1a] { + return None; + } + let (mut records, mut deleted) = (Vec::new(), 0); + for record in body.chunks_exact(record_len).take(count) { + if record.first() == Some(&b'*') { + deleted += 1; + continue; + } + let mut at = 1; + let mut values = Vec::with_capacity(columns.len()); + for c in &columns { + let end = at + usize::from(c.len); + values.push(record.get(at..end)?); + at = end; + } + records.push(values); + } + Some(RawTable { columns, records, deleted }) +} + /// A date (`D`) field's value, `YYYYMMDD`: `Ok(None)` when it's blank or zeros, as writers leave /// a date empty, and `Err(())` when it isn't a date (`20260230`, `2026-3-5`). pub(crate) fn date(field: &[u8]) -> std::result::Result, ()> { @@ -280,17 +346,18 @@ mod tests { assert!(dates_as_text(&mut [0u8; 10]).is_empty()); } - /// Floating point columns are marked as numeric, so `dbase` reads them as `f64`; others are kept. + /// Float columns are marked as numeric columns; others, and what follows the header, are kept. #[test] fn float_columns_read_as_numeric() { - let mut dbf = vec![0u8; 32 + 3 * 32 + 1]; - dbf[8] = (dbf.len()) as u8; - for (k, ty) in b"NFf".iter().copied().enumerate() { + let mut dbf = vec![0u8; 32 + 3 * 32 + 2]; + dbf[8] = (32 + 3 * 32 + 1) as u8; + for (k, ty) in b"FfC".iter().copied().enumerate() { dbf[32 + k * 32 + 11] = ty; } dbf[32 + 3 * 32] = 0x0D; + dbf[32 + 3 * 32 + 1] = b'F'; floats_as_numeric(&mut dbf); - assert_eq!([dbf[43], dbf[75], dbf[107]], [b'N', b'N', b'N']); + assert_eq!([dbf[43], dbf[75], dbf[107], dbf[129]], [b'N', b'N', b'C', b'F']); floats_as_numeric(&mut [0u8; 10]); } diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index ee13dee547..27f0266636 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -603,78 +603,8 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R "shp" => { crate::progress::begin(ds.features.len()); let (parts, left_out) = shapefile_parts_in(ds, crs)?; - // Every part is written to a temporary file first and renamed into place only once all - // are written, the .shp last, so a failed write leaves the old Shapefile whole. - let mut written: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); - let staged = parts.iter().try_for_each(|(e, bytes)| { - // The .shp keeps the name given (`Roads.SHP` stays so); the others take its stem. - let target = if *e == "shp" { path.to_path_buf() } else { path.with_extension(e) }; - let tmp = temp_beside(&target)?; - let result = write_new(&tmp, bytes); - if result.is_ok() { - written.push((tmp, target)); - } - result - }); - // A cancel before the parts land leaves the old Shapefile as it was. - if let Err(e) = staged.map_err(DataError::from).and_then(|()| crate::progress::check()) { - for (tmp, _) in &written { - let _ = std::fs::remove_file(tmp); - } - return Err(e); - } - written.sort_by_key(|(_, target)| target == path); - // The old parts are moved aside first and put back if any new part fails to land, so - // the old Shapefile stays a matching set. - let mut backups: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); - let mut placed: Vec = Vec::new(); - let mut result = Ok(()); - for (tmp, target) in &written { - // A cancel before the last part lands rolls back like a failed rename. - if crate::progress::check().is_err() { - result = Err(std::io::Error::other(DataError::Cancelled)); - break; - } - if target.exists() { - // Any failure here rolls back like a failed rename. - let bak = match temp_beside(target) { - Ok(b) => b, - Err(e) => { - result = Err(e); - break; - } - }; - if let Err(e) = std::fs::rename(target, &bak) { - result = Err(e); - break; - } - backups.push((bak, target.clone())); - } - if let Err(e) = std::fs::rename(tmp, target) { - result = Err(e); - break; - } - placed.push(target.clone()); - } - if let Err(e) = result { - for p in &placed { - let _ = std::fs::remove_file(p); - } - for (bak, target) in &backups { - let _ = std::fs::rename(bak, target); - } - for (tmp, _) in &written { - let _ = std::fs::remove_file(tmp); - } - return Err(e.into()); - } - for (bak, _) in &backups { - let _ = std::fs::remove_file(bak); - } - // A spatial index left from an older file of the same name would no longer match. - for stale in ["qix", "sbn", "sbx"] { - let _ = std::fs::remove_file(path.with_extension(stale)); - } + let parts: Vec<(std::path::PathBuf, Vec)> = parts.into_iter().map(|(e, bytes)| (shapefile_part(path, e), bytes)).collect(); + place_shapefile(path, &stage_shapefile(&parts)?)?; Ok((left_out, None)) } _ if crs != Crs::LonLat && !PROJECTED_FORMATS.contains(&ext.as_str()) => { @@ -688,6 +618,121 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R } } +/// The path of the part with extension `ext` of the Shapefile at `path`: the `.shp` keeps the +/// name given (`Roads.SHP` stays so); the others take its stem. +pub(crate) fn shapefile_part(path: &Path, ext: &str) -> std::path::PathBuf { + if ext == "shp" { path.to_path_buf() } else { path.with_extension(ext) } +} + +/// Writes each of a Shapefile's parts (its path and bytes) to a temporary file beside it, synced +/// to the disk; returns each temporary file with the part it becomes. Nothing is left behind +/// when one fails or the run is cancelled (see [`place_shapefile`]). +pub(crate) fn stage_shapefile(parts: &[(std::path::PathBuf, Vec)]) -> Result> { + let mut written: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); + let staged = parts.iter().try_for_each(|(target, bytes)| { + let tmp = temp_beside(target)?; + let result = write_new_like(&tmp, target, bytes); + if result.is_ok() { + written.push((tmp, target.clone())); + } + result + }); + // A cancel before the parts land leaves the old Shapefile as it was. + if let Err(e) = staged.map_err(DataError::from).and_then(|()| crate::progress::check()) { + unstage(&written); + return Err(e); + } + Ok(written) +} + +/// Removes the temporary files [`stage_shapefile`] wrote. +pub(crate) fn unstage(staged: &[(std::path::PathBuf, std::path::PathBuf)]) { + for (tmp, _) in staged { + let _ = std::fs::remove_file(tmp); + } +} + +/// Renames each staged part of the Shapefile at `path` ([`stage_shapefile`]) into place, the +/// `.shp` last, so a failed write leaves the old Shapefile whole: each old part is moved aside +/// first and put back if any new part fails to land, and a Cancel before the last part lands +/// rolls back the same way. Once all have landed, the old parts and the spatial indexes that +/// would no longer match (`.qix`, `.sbn`, `.sbx`) are removed. +pub(crate) fn place_shapefile(path: &Path, staged: &[(std::path::PathBuf, std::path::PathBuf)]) -> Result<()> { + let mut written: Vec<&(std::path::PathBuf, std::path::PathBuf)> = staged.iter().collect(); + written.sort_by_key(|(_, target)| target == path); + let mut backups: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); + let mut placed: Vec = Vec::new(); + let mut result = Ok(()); + for (tmp, target) in written { + // A cancel before the last part lands rolls back like a failed rename. + if crate::progress::check().is_err() { + result = Err(std::io::Error::other(DataError::Cancelled)); + break; + } + if target.exists() { + // Any failure here rolls back like a failed rename. + let bak = match temp_beside(target) { + Ok(b) => b, + Err(e) => { + result = Err(e); + break; + } + }; + // Reserve this name before moving the old part: a previous crash's backup + // (or a symlink) must never be replaced when a process id is reused. + if let Err(e) = std::fs::OpenOptions::new().write(true).create_new(true).open(&bak).map(drop) { + result = Err(e); + break; + } + if let Err(e) = std::fs::rename(target, &bak) { + let _ = std::fs::remove_file(&bak); + result = Err(e); + break; + } + backups.push((bak, target.clone())); + } + if let Err(e) = std::fs::rename(tmp, target) { + result = Err(e); + break; + } + placed.push(target.clone()); + } + if let Err(e) = result { + for p in &placed { + let _ = std::fs::remove_file(p); + } + // An old part that can't be put back is named, so it isn't lost under its temporary name. + let mut kept = Vec::new(); + for (bak, target) in &backups { + if std::fs::rename(bak, target).is_err() { + kept.push(format!("{} is kept as {}", target.display(), bak.display())); + } + } + unstage(staged); + let e = DataError::from(e); + if kept.is_empty() { + return Err(e); + } + return Err(DataError::Format(format!("{e}; putting the old file back failed: {}", kept.join(", ")))); + } + for (bak, _) in &backups { + let _ = std::fs::remove_file(bak); + } + // A spatial index left from an older file of the same name would no longer match. + for stale in ["qix", "sbn", "sbx"] { + // Sidecars may use any letter case, even beside a lowercase `.shp`. + for capitals in 0..8 { + let extension: String = stale + .bytes() + .enumerate() + .map(|(i, b)| char::from(if capitals & (1 << i) == 0 { b } else { b.to_ascii_uppercase() })) + .collect(); + let _ = std::fs::remove_file(path.with_extension(extension)); + } + } + Ok(()) +} + /// A new temporary name beside `path`, unique per call, so concurrent writes to the same target /// never share one. pub(crate) fn temp_beside(path: &Path) -> std::io::Result { @@ -704,17 +749,48 @@ pub(crate) fn temp_beside(path: &Path) -> std::io::Result { /// Writes and syncs a new file (`create_new`, so an existing file or symlink is never followed); /// removes it on failure. +#[cfg(test)] fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { - write_new_with(path, |w| w.write_all(bytes)) + write_new_with(path, |_, w| w.write_all(bytes)) +} + +/// [`write_new`], the new file taking the mode of the file at `like` (on Unix, when there is one) +/// before any content is written: a private part (`0600`) stays private once the new file +/// replaces it, and its content is never readable by others meanwhile. A mode that can't be set +/// is an error, rather than a part made readable. +fn write_new_like(path: &Path, like: &Path, bytes: &[u8]) -> std::io::Result<()> { + #[cfg(unix)] + let mode = match std::fs::symlink_metadata(like) { + Ok(old) if !old.file_type().is_symlink() => Some(old.permissions()), + Ok(_) => None, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + #[cfg(not(unix))] + let _ = like; + write_new_with(path, |file, w| { + // Through the handle opened, not the path, which could name another file by now. + #[cfg(unix)] + if let Some(mode) = mode { + use std::os::unix::fs::PermissionsExt; + if file.metadata()?.permissions().mode() != mode.mode() { + file.set_permissions(mode)?; + } + } + #[cfg(not(unix))] + let _ = file; + w.write_all(bytes) + }) } -/// [`write_new`] with the content written by `write`, through a buffer, as it goes. -fn write_new_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { +/// [`write_new`] with the content written by `write`, through a buffer, as it goes; `write` is +/// also given the file opened, before anything is written to it. +fn write_new_with(path: &Path, write: impl FnOnce(&std::fs::File, &mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { use std::io::Write; let f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?; let res = (|| { let mut w = std::io::BufWriter::new(&f); - write(&mut w)?; + write(&f, &mut w)?; w.flush()?; drop(w); f.sync_all() @@ -2320,17 +2396,17 @@ pub fn read_zip_in(file_name: &str, bytes: &[u8], crs: Option) -> Result, - shx: Option>, - dbf: Vec, - prj: Option, - cpg: Option>, +pub(crate) struct ShapefileParts { + pub shp: Vec, + pub shx: Option>, + pub dbf: Vec, + pub prj: Option, + pub cpg: Option>, } /// Decodes attributes and shapes together, tracking geometry lost during CRS conversion so /// a later projection definition cannot silently repair an incomplete layer. -fn read_shapefile_parts(mut parts: ShapefileParts, name: &str, crs: Option) -> Result<(Dataset, ReadReport)> { +pub(crate) fn read_shapefile_parts(mut parts: ShapefileParts, name: &str, crs: Option) -> Result<(Dataset, ReadReport)> { use shapefile::Shape; use shapefile::dbase::FieldValue; use std::io::Cursor; @@ -2564,7 +2640,31 @@ fn integral_numeric_to_integer(ds: &mut Dataset) { } /// WGS 84 as ESRI WKT, for a Shapefile's `.prj`. -const WGS84_PRJ: &str = "GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"; +pub(crate) const WGS84_PRJ: &str = "GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"; + +/// The longest prefix of `s` whose text takes at most `max` bytes in `encoding`, cut at a +/// character boundary: what a `.dbf` in that code page holds of it. +fn cut_encoded<'a>(s: &'a str, max: usize, encoding: &crate::dbf::DbfEncoding) -> &'a str { + use shapefile::dbase::encoding::Encoding; + let encoder = encoding.decoder(); + let fits = |t: &str| encoder.encode(t).map_or(t.len(), |b| b.len()) <= max; + if fits(s) { + return s; + } + // The longest prefix that fits: a character takes at least one byte, so it is within `max` + // characters, and a shorter prefix never takes more bytes. + let ends: Vec = s.char_indices().map(|(i, _)| i).skip(1).take(max).chain([s.len()]).collect(); + let (mut lo, mut hi) = (0usize, ends.len()); + while lo < hi { + let mid = (lo + hi).div_ceil(2); + if ends.get(mid - 1).is_some_and(|&e| fits(s.get(..e).unwrap_or(""))) { + lo = mid; + } else { + hi = mid - 1; + } + } + lo.checked_sub(1).and_then(|i| ends.get(i)).and_then(|&e| s.get(..e)).unwrap_or("") +} /// The longest prefix of `s` that fits in `max` bytes, cut at a character boundary. pub(crate) fn cut(s: &str, max: usize) -> &str { @@ -2807,7 +2907,7 @@ pub fn shapefile_values_notes(ds: &Dataset, target: &str) -> Vec { FieldType::Date if !value.asks_null() && dbf_date(value).is_none() => loss.empty += 1, FieldType::Integer | FieldType::Double => { let d = decimals.get(k).copied().unwrap_or(0); - match dbf_number(value, field.ty, d) { + match dbf_number(value, field.ty, d, if field.ty == FieldType::Integer { 18 } else { 24 }) { None => loss.empty += 1, Some(number) => { // The writer uses fixed decimal text, including ties rounded to even. @@ -2854,6 +2954,13 @@ pub fn shapefile_values_notes(ds: &Dataset, target: &str) -> Vec { notes } +/// The names of the `.dbf` columns a Shapefile written from `fields` has, in order +/// ([`dbf_names`]). +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn shapefile_column_names(fields: &[Field]) -> Vec { + dbf_names(fields) +} + /// dBase field names (at most 10 bytes, unique without regard to case) for the fields. fn dbf_names(fields: &[Field]) -> Vec { let mut out: Vec = Vec::with_capacity(fields.len()); @@ -2892,15 +2999,15 @@ fn dbf_decimals(ds: &Dataset, k: usize) -> usize { } /// The number a `.dbf` numeric field of type `ty` with `d` decimals holds for `v`: none for a -/// value that isn't a number, or one rejected by the writer's size and precision checks. +/// value that isn't a number, or one rejected by the selected column's `width` and precision. /// Rejected numbers are left empty rather than cut short. -fn dbf_number(v: &Value, ty: FieldType, d: usize) -> Option { +fn dbf_number(v: &Value, ty: FieldType, d: usize, width: usize) -> Option { let x = v.as_f64().filter(|x| { // A whole number past 2^53 would be held only rounded (dBase's decimal text goes through // f64), and one too small for the field's decimals would read 0. let exact = !(matches!(v, Value::Int(i) if i.unsigned_abs() > 1 << 53) || ty == FieldType::Double && *x != 0.0 && x.abs() < 0.5 * 10f64.powi(-(d as i32))); - exact && format!("{x:.d$}").len() <= if ty == FieldType::Integer { 18 } else { 24 } + exact && format!("{x:.d$}").len() <= width })?; Some(if ty == FieldType::Integer { x.round() } else { x }) } @@ -3224,48 +3331,159 @@ pub fn shapefile_parts(ds: &Dataset) -> Result<(Vec<(&'static str, Vec)>, us /// As [`shapefile_parts`], for a dataset whose geometry is in `crs`'s coordinates: the `.prj` /// says `crs`, and is left out for a system WKT can't describe. pub fn shapefile_parts_in(ds: &Dataset, crs: Crs) -> Result<(Vec<(&'static str, Vec)>, usize)> { + shapefile_parts_styled(ds, crs, None).map(|(parts, left_out, _)| (parts, left_out)) +} + +/// What a Shapefile written over the one a layer was read from keeps of that file's `.dbf` +/// (#1385): its code page (the encoding its text is written in, and the language driver byte +/// its header names), and each column's width, decimals and type (`F` stays `F`) where the +/// layer's field of that name still fits it. +pub(crate) struct DbfStyle { + pub encoding: crate::dbf::DbfEncoding, + pub ldid: u8, + pub columns: Vec, +} + +/// The `.dbf` column a field is written to: its type letter, width and decimals. +#[derive(Clone, Copy, PartialEq, Eq)] +struct Spec { + ty: u8, + len: u8, + decimals: u8, +} + +/// The column field `k` of `ds` is written to as a new file has it: numbers in 18 digits +/// (integers) or 24 characters with as many decimals as fit, text as wide as its longest value. +fn new_column(ds: &Dataset, k: usize) -> Spec { + let Some(f) = ds.fields.get(k) else { return Spec { ty: b'C', len: 1, decimals: 0 } }; + match f.ty { + // 18 digits: GDAL reads a wider field as a real number. + FieldType::Integer => Spec { ty: b'N', len: 18, decimals: 0 }, + FieldType::Double => Spec { ty: b'N', len: 24, decimals: u8::try_from(dbf_decimals(ds, k)).unwrap_or(15) }, + FieldType::Boolean => Spec { ty: b'L', len: 1, decimals: 0 }, + FieldType::Date => Spec { ty: b'D', len: 8, decimals: 0 }, + // A `.dbf` has no date-time field that other programs read, so they're ISO text. + FieldType::Text | FieldType::DateTime => { + let longest = ds.features.iter().filter_map(|r| r.attrs.get(k)).map(|v| v.to_string().len()).max().unwrap_or(0); + Spec { ty: b'C', len: u8::try_from(longest.clamp(1, 254)).unwrap_or(254), decimals: 0 } + } + } +} + +impl std::fmt::Display for Spec { + /// As a `.dbf` column is described: its type letter, width and, for a number, decimals + /// (`N(10,2)`, `C(24)`). + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ty = char::from(self.ty); + if matches!(self.ty, b'N' | b'F') { write!(f, "{ty}({},{})", self.len, self.decimals) } else { write!(f, "{ty}({})", self.len) } + } +} + +/// The column field `k` (written as `name`) is written to over a file whose `.dbf` `style` +/// describes: the file's column of that name where the field's kind and every value still fit +/// it, so the file keeps its widths and decimals; otherwise a new one ([`new_column`]), at least +/// as wide as the file's, and, when a column of that name changes kind or can't +/// hold the values, what changed (`'ACRES' N(10,2) to N(24,9)`). +fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, Option) { + let new = new_column(ds, k); + let (Some(field), Some(was)) = (ds.fields.get(k), style.columns.iter().find(|c| c.name == name.as_bytes())) else { + return (new, None); + }; + let values = || ds.features.iter().filter_map(|r| r.attrs.get(k)).filter(|v| !v.is_null()); + let ty = was.ty.to_ascii_uppercase(); + let kept = Spec { ty, len: was.len, decimals: was.decimals }; + let encoding = style.encoding.decoder(); + // Bytes of a value's text in the file's code page. + let encoded_len = |v: &Value| { + use shapefile::dbase::encoding::Encoding; + let text = v.to_string(); + encoding.encode(&text).map_or(text.len(), |b| b.len()) + }; + let fits = match (field.ty, ty) { + (FieldType::Integer | FieldType::Double, b'N' | b'F') => values().all(|v| { + let d = usize::from(was.decimals); + dbf_number(v, field.ty, d, usize::from(was.len)).is_some_and(|x| { + let text = format!("{x:.d$}"); + text.len() <= usize::from(was.len) && text.parse::().ok() == v.as_f64() + }) + }), + (FieldType::Boolean, b'L') | (FieldType::Date, b'D') => true, + (FieldType::Text | FieldType::DateTime, b'C') => values().all(|v| encoded_len(v) <= usize::from(was.len)), + // Another kind of field now: say how its physical column changes too. + _ => return (new, Some(format!("'{}' {kept} to {new}", dbf_notice_name(&field.name)))), + }; + if fits { + return (kept, None); + } + let spec = if new.ty == b'C' { + // Text wider than the file's column: as wide as its longest value, in the file's code + // page, up to the 254 bytes written (a longer value is cut, which publishing refuses). + let longest = values().map(encoded_len).max().unwrap_or(0).min(254); + Spec { len: u8::try_from(longest).unwrap_or(254), ..new } + } else { + // A number with more digits or decimals: never narrower than the file's column. + Spec { len: new.len.max(was.len), ..new } + }; + (spec, Some(format!("'{}' {kept} to {spec}", dbf_notice_name(&field.name)))) +} + +/// As [`shapefile_parts_in`], the `.dbf` written as `style` says when it's given (in the file's +/// code page, its columns kept where the values fit): also returns, for each field whose column +/// couldn't hold its values, how its column changed. +pub(crate) fn shapefile_parts_styled( + ds: &Dataset, + crs: Crs, + style: Option<&DbfStyle>, +) -> Result<(Vec<(&'static str, Vec)>, usize, Vec)> { use shapefile::dbase::{FieldName, FieldValue, Record, TableWriterBuilder}; use std::io::Cursor; let fmt = |e: shapefile::Error| DataError::Format(format!("Shapefile: {e}")); let names = dbf_names(&ds.fields); let dbf_name = |n: &str| FieldName::try_from(n).map_err(|e| DataError::Format(format!("Shapefile: field {n}: {e}"))); - let mut table = TableWriterBuilder::new(); - // Per field: decimals of a Double field (numbers too large for 24 characters become null). - let mut decimals = vec![0usize; ds.fields.len()]; - for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { - table = match f.ty { - // 18 digits: GDAL reads a wider field as a real number. - FieldType::Integer => table.add_numeric_field(dbf_name(n)?, 18, 0), - FieldType::Double => { - let d = dbf_decimals(ds, k); - if let Some(slot) = decimals.get_mut(k) { - *slot = d; - } - table.add_numeric_field(dbf_name(n)?, 24, u8::try_from(d).unwrap_or(15)) - } - FieldType::Boolean => table.add_logical_field(dbf_name(n)?), - FieldType::Date => table.add_date_field(dbf_name(n)?), - // A `.dbf` has no date-time field that other programs read, so they're ISO text. - FieldType::Text | FieldType::DateTime => { - let longest = ds.features.iter().filter_map(|r| r.attrs.get(k)).map(|v| v.to_string().len()).max().unwrap_or(0); - table.add_character_field(dbf_name(n)?, u8::try_from(longest.clamp(1, 254)).unwrap_or(254)) + let mut table = match style { + Some(style) => TableWriterBuilder::with_encoding(style.encoding.decoder()), + None => TableWriterBuilder::new(), + }; + // Per field: its column (numbers too large for it become null), and how the columns that + // couldn't hold their values changed. + let mut specs = Vec::with_capacity(ds.fields.len()); + let mut widened = Vec::new(); + for (k, n) in names.iter().enumerate() { + let spec = match style { + Some(style) => { + let (spec, changed) = kept_column(ds, k, n, style); + widened.extend(changed); + spec } + None => new_column(ds, k), + }; + table = match spec.ty { + // A float column is written as a numeric one (both hold the number as text) and + // marked `F` again below: `dbase` writes an `F` column's number as an `f32`. + b'N' | b'F' => table.add_numeric_field(dbf_name(n)?, spec.len, spec.decimals), + b'L' => table.add_logical_field(dbf_name(n)?), + b'D' => table.add_date_field(dbf_name(n)?), + _ => table.add_character_field(dbf_name(n)?, spec.len), }; + specs.push(spec); } let record = |r: &crate::Feature| -> Record { let mut rec = Record::default(); for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { let v = r.attrs.get(k).unwrap_or(&Value::Null); + let decimals = specs.get(k).map_or(0, |s| usize::from(s.decimals)); + let width = specs.get(k).map_or(0, |s| usize::from(s.len)); let value = match f.ty { - FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals.get(k).copied().unwrap_or(0))), + FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals, width)), FieldType::Boolean => FieldValue::Logical(dbf_boolean(v)), FieldType::Date => FieldValue::Date(dbf_date(v).and_then(|d| { let (y, m, d) = d.ymd(); shapefile::dbase::Date::new(u32::try_from(d).ok()?, u32::try_from(m).ok()?, u32::try_from(y).ok()?).ok() })), - FieldType::Text | FieldType::DateTime => { - FieldValue::Character((!v.is_null()).then(|| cut(&v.to_string(), 254).to_string())) - } + FieldType::Text | FieldType::DateTime => FieldValue::Character((!v.is_null()).then(|| match style { + Some(style) => cut_encoded(&v.to_string(), 254, &style.encoding).to_string(), + None => cut(&v.to_string(), 254).to_string(), + })), }; rec.insert(n.clone(), value); } @@ -3341,10 +3559,25 @@ pub fn shapefile_parts_in(ds: &Dataset, crs: Crs) -> Result<(Vec<(&'static str, } } let prj = if crs == Crs::LonLat { Some(WGS84_PRJ.to_string()) } else { crate::raster::wkt::to_wkt(crs) }; - let mut parts = vec![("shp", shp.into_inner()), ("shx", shx.into_inner()), ("dbf", dbf.into_inner())]; + let mut dbf = dbf.into_inner(); + if let Some(style) = style { + // The file's language driver, whatever `dbase` takes the encoding for, and its float + // columns marked `F` again. + if let Some(b) = dbf.get_mut(29) { + *b = style.ldid; + } + for (k, spec) in specs.iter().enumerate() { + if spec.ty == b'F' + && let Some(b) = dbf.get_mut(32 + k * 32 + 11) + { + *b = b'F'; + } + } + } + let mut parts = vec![("shp", shp.into_inner()), ("shx", shx.into_inner()), ("dbf", dbf)]; parts.extend(prj.map(|p| ("prj", p.into_bytes()))); parts.push(("cpg", b"UTF-8".to_vec())); - Ok((parts, left_out)) + Ok((parts, left_out, widened)) } fn is_outer

(r: &shapefile::PolygonRing

) -> bool { @@ -3368,6 +3601,33 @@ mod tests { use super::*; use std::io::Write; + /// Text is cut to what its code page holds in the bytes given, at a character boundary. + #[test] + fn text_is_cut_to_its_code_page_bytes() { + let cp1252 = crate::dbf::DbfEncoding::Standard(encoding_rs::WINDOWS_1252); + let e = "é".repeat(300); + assert_eq!(cut_encoded(&e, 254, &cp1252).chars().count(), 254); + assert_eq!(cut_encoded(&e, 254, &crate::dbf::DbfEncoding::Utf8).chars().count(), 127); + assert_eq!(cut_encoded("short", 254, &cp1252), "short"); + assert_eq!(cut_encoded("ab", 0, &cp1252), ""); + } + + /// A text column as wide as a `.dbf` allows (255 bytes, a width the writer never uses) with a + /// longer value gets a column of the 254 bytes written, not a crash. + #[test] + fn a_full_width_text_column_with_a_longer_value_is_written_at_254() { + let mut ds = Dataset::new("a", GeometryType::Point, vec![Field::new("NOTE", FieldType::Text)]); + ds.push(Some(Geometry::Point(Coord::new(0.0, 0.0))), vec![Value::Text("x".repeat(300))]); + let style = DbfStyle { + encoding: crate::dbf::DbfEncoding::Utf8, + ldid: 0, + columns: vec![crate::dbf::Column { name: b"NOTE".to_vec(), ty: b'C', len: 255, decimals: 0 }], + }; + let (spec, changed) = kept_column(&ds, 0, "NOTE", &style); + assert_eq!(spec.len, 254); + assert_eq!(changed.as_deref(), Some("'NOTE' C(255) to C(254)")); + } + /// A Shapefile export of a polygon layer holding a point and a feature without geometry writes /// the polygon and counts both left out, the point for its kind; a layer of polygons loses none. #[cfg(not(target_arch = "wasm32"))] @@ -3580,7 +3840,7 @@ mod tests { Ok(()) } - /// Exercises caller cleanup through the public Shapefile writer, one fresh process per collision. + /// Exercises staging and backup cleanup through the public writer, one fresh process per collision. #[cfg(not(target_arch = "wasm32"))] #[test] fn shapefile_staging_collisions_preserve_existing_entries() -> std::io::Result<()> { @@ -3591,32 +3851,35 @@ mod tests { const CHILD: &str = "RUSTGIS_SHAPEFILE_STAGING_COLLISION"; let Ok(case) = std::env::var(CHILD) else { let kinds: &[&str] = if cfg!(unix) { &["file", "live", "dangling"] } else { &["file"] }; - for index in 0..parts.len() { - for kind in kinds { - let output = std::process::Command::new(std::env::current_exe()?) - .args([ - "--exact", - "io::tests::shapefile_staging_collisions_preserve_existing_entries", - "--nocapture", - "--test-threads=1", - ]) - .env(CHILD, format!("{index}:{kind}")) - .output()?; - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - output.status.success() - && stdout.contains("test result: ok. 1 passed; 0 failed; 0 ignored;") - && stdout.contains(&format!("collision case completed: {index}:{kind}")), - "part={index} kind={kind}: {}{}", - stdout, - String::from_utf8_lossy(&output.stderr) - ); + for phase in ["stage", "backup"] { + for index in 0..parts.len() { + for kind in kinds { + let output = std::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + "io::tests::shapefile_staging_collisions_preserve_existing_entries", + "--nocapture", + "--test-threads=1", + ]) + .env(CHILD, format!("{phase}:{index}:{kind}")) + .output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("test result: ok. 1 passed; 0 failed; 0 ignored;") + && stdout.contains(&format!("collision case completed: {phase}:{index}:{kind}")), + "phase={phase} part={index} kind={kind}: {}{}", + stdout, + String::from_utf8_lossy(&output.stderr) + ); + } } } - println!("collision scenarios: {}", parts.len() * kinds.len()); + println!("collision scenarios: {}", 2 * parts.len() * kinds.len()); return Ok(()); }; - let (index, kind) = case.split_once(':').ok_or_else(|| std::io::Error::other("missing collision case"))?; + let (phase, case_parts) = case.split_once(':').ok_or_else(|| std::io::Error::other("missing collision phase"))?; + let (index, kind) = case_parts.split_once(':').ok_or_else(|| std::io::Error::other("missing collision case"))?; let index = index.parse::().map_err(std::io::Error::other)?; if !matches!(kind, "file" | "live" | "dangling") { return Err(std::io::Error::other("unknown collision kind")); @@ -3628,12 +3891,17 @@ mod tests { for (extension, bytes) in &parts { std::fs::write(destination(extension), bytes)?; } - let (extension, _) = parts.get(index).ok_or_else(|| std::io::Error::other("collision part out of range"))?; + let mut ordered: Vec<_> = parts.iter().collect(); + if phase == "backup" { + ordered.sort_by_key(|(extension, _)| *extension == "shp"); + } + let (extension, _) = ordered.get(index).ok_or_else(|| std::io::Error::other("collision part out of range"))?; let target = destination(extension); - // This child runs only this test: write_path's first staging counter is zero. + // This child starts the counter at zero; backups follow all staged parts. let mut name = std::ffi::OsString::from("."); name.push(target.file_name().ok_or_else(|| std::io::Error::other("missing part name"))?); - name.push(format!(".{}.{index}.tmp", std::process::id())); + let counter = if phase == "backup" { parts.len() + index } else { index }; + name.push(format!(".{}.{counter}.tmp", std::process::id())); let collision = target.with_file_name(name); #[cfg(unix)] let link_target = root.join("link-target"); diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index ecc593c471..b9428a5b60 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -37,6 +37,8 @@ pub mod ogc; pub mod osm; pub mod photo; pub mod progress; +#[cfg(not(target_arch = "wasm32"))] +pub mod publish; pub mod query; pub mod raster; pub mod s3_address; diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs new file mode 100644 index 0000000000..f0945c819d --- /dev/null +++ b/crates/data/src/publish.rs @@ -0,0 +1,1469 @@ +//! Writing a layer kept in the project back over the Shapefile it was read from (#1385), so that +//! the file shows its edits and keeps everything else it held. +//! +//! A Shapefile can't change one row in place: the whole file is written again. That is only +//! safe when nothing but the edits changes, so [`shapefile`] proves it before it touches the +//! file, and refuses, saying why, when it can't: +//! +//! 1. The file is as it was when the layer was read ([`FileSync::changed`]): a change made since +//! would be overwritten. And it reads now as the layer was read: rows that weren't edited are +//! written as the file holds them, not as an older read held them. +//! 2. The file as it was read, written again the same way, holds what the file holds: the same +//! shapes (no Z or M values dropped, none left out), the same columns and the same values, +//! compared as the bytes the `.dbf` holds, not as read (text a read trims, a code page's +//! characters). Whatever a read and write of the file loses shows here, before the edits. +//! 3. The layer, written, reads back as the layer: every value and shape the edits made fits. +//! +//! A shape that wasn't edited is written with the coordinates the file holds, not converted to +//! longitude/latitude and back, so a projected file's shapes stay as they were to the bit. +//! +//! Only then are the new parts written beside the old ones, checked again against the bytes +//! meant, and renamed into place, the `.shp` last; any failure puts the old parts back +//! ([`crate::io::place_shapefile`]). The file keeps its code page, column widths, decimals, +//! `.prj` and `.cpg` where the edits allow; sidecars RustGIS doesn't write (`.shp.xml`) are left. + +use crate::dbf::{Column, RawTable}; +use crate::io::{DbfStyle, ShapefileParts}; +use crate::raster::Crs; +use crate::sync::{FileSync, Synced}; +use crate::{DataError, Dataset, Feature, FieldType, Geometry, Result, Value}; +use std::path::{Path, PathBuf}; + +/// How far an edited shape may read back from where the layer has it, relative to the size of its +/// coordinates, in a file in another system than longitude/latitude: the layer's shape is +/// projected to the file's system and read back through the inverse, which isn't exact to the +/// last bit (a few parts in 10¹³ measured, in a state plane system in feet). One part in 10¹¹ is +/// a tenth of a millimetre at ten thousand kilometres; anything more is refused. +const PROJECTED_TOLERANCE: f64 = 1e-11; + +/// A layer written back over its Shapefile: the files as they were written, and the layer as the +/// file now reads (`read`: its rows in the same order, as a Shapefile holds them: an empty text as +/// no value, a column of whole numbers as integers, date-times as text, an edited shape in another +/// system back through its projection), to record as the layer's new sync point; the features +/// written, and what it says (fields given other columns or left out, projection round-off, +/// a changed `.prj`). +pub struct Published { + pub files: FileSync, + pub read: Dataset, + /// Digest of `read`, computed before placement so cancellation cannot interrupt completion. + pub rows: u64, + pub features: usize, + pub notes: Vec, +} + +/// A refusal: nothing was written. +fn refused(path: &Path, why: impl std::fmt::Display) -> DataError { + DataError::Format(format!("{}: {why}; nothing was written", path.display())) +} + +/// Placement replaces a directory entry, so following a linked part while reading is unsafe. +fn refuse_link(source: &Path, part: &Path) -> Result<()> { + match std::fs::symlink_metadata(part) { + Ok(m) if m.file_type().is_symlink() => Err(refused( + source, + format!( + "{} is a symbolic link; add the actual source files instead so publishing keeps the link and its target unchanged", + part.display() + ), + )), + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } +} + +/// A publisher's lock, released before its descriptor closes so duplicates inherited during +/// process creation cannot extend its lifetime. +struct SourceLock(std::fs::File); + +impl Drop for SourceLock { + /// Unlock the shared file description, including any duplicate descriptors. + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +/// Holds an exclusive OS lock on a persistent sidecar through publication. The sidecar is +/// never unlinked: deleting it could let a third process lock a different inode while another +/// still holds the old one. Dropping the guard explicitly unlocks before closing its handle; +/// contention refuses immediately. This coordinates RustGIS publishers, not other programs. +fn source_lock(path: &Path) -> Result { + refuse_link(path, path)?; + let target = path.canonicalize()?; + let mut name = std::ffi::OsString::from("."); + name.push(target.file_name().ok_or_else(|| refused(path, "it has no file name"))?); + name.push(".rustgis-lock"); + let lock_path = target.with_file_name(name); + if std::fs::symlink_metadata(&lock_path).is_ok_and(|m| !m.file_type().is_file()) { + return Err(refused(path, format!("{} isn't a regular lock file", lock_path.display()))); + } + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32); + } + let opened = match options.open(&lock_path) { + Ok(file) => { + // The empty sidecar contains no source data. Accounts sharing the directory need + // read/write access regardless of source groups, ACLs or a private umask: NFS + // exclusive locks require a writable handle. Set modes through the open handle. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o666))?; + } + Ok(file) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + options.create_new(false); + options.open(&lock_path) + } + Err(e) => Err(e), + }; + let file = opened.map_err(|e| refused(path, format!("couldn't open {}: {e}", lock_path.display())))?; + if !file.metadata()?.is_file() { + return Err(refused(path, "its lock isn't a regular file")); + } + file.try_lock().map_err(|e| match e { + std::fs::TryLockError::WouldBlock => refused(path, "another RustGIS publisher holds this source; try again after it finishes"), + std::fs::TryLockError::Error(e) => refused(path, format!("couldn't lock {}: {e}", lock_path.display())), + })?; + Ok(SourceLock(file)) +} + +/// The Shapefile at `path`'s parts as they are, read into memory, with its `.cpg`'s path. +struct Original { + shp: Vec, + shx: Option>, + dbf: Vec, + prj: Option>, + cpg: Option<(PathBuf, Vec)>, +} + +/// Reads the file at `p`, `None` when it isn't there; refuses one past what a reader takes. +fn read_part(p: &Path) -> Result>> { + match std::fs::metadata(p) { + Ok(m) if m.len() > crate::io::MAX_BYTES => Err(DataError::Format(format!("{} is too large to read", p.display()))), + Ok(_) => Ok(Some(std::fs::read(p)?)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } +} + +impl Original { + /// Reads the parts of the Shapefile at `path`: its `.shp` and `.dbf` must be there; its + /// `.shx`, `.prj` and `.cpg` (in whichever case it is spelled) may be. + fn read(path: &Path) -> Result { + let missing = |ext: &str| refused(path, format!("its .{ext} isn't there")); + let shp = read_part(path)?.ok_or_else(|| missing("shp"))?; + let dbf = read_part(&path.with_extension("dbf"))?.ok_or_else(|| missing("dbf"))?; + let shx = read_part(&path.with_extension("shx"))?; + let prj = read_part(&path.with_extension("prj"))?; + let mut cpg = None; + for ext in crate::dbf::CPG_SPELLINGS { + let p = path.with_extension(ext); + if let Some(bytes) = read_part(&p)? { + cpg = Some((p, bytes)); + break; + } + } + Ok(Original { shp, shx, dbf, prj, cpg }) + } + + /// Its parts as the reader takes them. + fn parts(&self) -> ShapefileParts { + ShapefileParts { + shp: self.shp.clone(), + shx: self.shx.clone(), + dbf: self.dbf.clone(), + prj: self.prj.as_ref().and_then(|p| String::from_utf8(p.clone()).ok()), + cpg: self.cpg.as_ref().map(|(_, b)| b.clone()), + } + } + + /// The coordinate system its `.prj` names, as the reader takes it (`None` for none, one it + /// can't read). + fn crs(&self) -> Option { + let text = std::str::from_utf8(self.prj.as_deref()?).ok()?; + crate::raster::wkt::read_wkt(text).ok().map(|(c, _)| c) + } +} + +/// Writes `ds`, read whole from the Shapefile at `path` when it and its files were as `synced` +/// records, back over that Shapefile: see the module's documentation for what it checks first +/// and what it keeps. Its work grows with the file: it is hashed twice, read twice, written in +/// memory twice and read back, the hashing and writing counting progress, and it stops once the +/// run is cancelled (before the last part is placed, which puts the old file back). Reading the +/// file into memory and back, and comparing the shapes, table and rows (one pass each, without +/// a check), are steps that run to their end before Cancel is seen. Hashing the prepared bytes +/// and writing/syncing each staged part likewise run as single steps. +pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result { + let _lock = source_lock(path)?; + let unchanged = |when: &str| -> Result<()> { + refuse_link(path, path)?; + for ext in ["shx", "dbf", "prj"].into_iter().chain(crate::dbf::CPG_SPELLINGS) { + refuse_link(path, &path.with_extension(ext))?; + } + let changed = synced.files.changed(path)?; + if changed.is_empty() { + return Ok(()); + } + Err(refused(path, format!("{} changed {when}, and writing the layer would overwrite that", changed.join(", ")))) + }; + unchanged("since the layer was read")?; + let original = Original::read(path)?; + let crs = ds.crs.filter(|c| *c != Crs::LonLat); + let written_crs = crs.unwrap_or(Crs::LonLat); + let mut notes = Vec::new(); + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + + let Some(table) = crate::dbf::raw_table(&original.dbf) else { + return Err(refused(path, "its .dbf's header doesn't describe its records")); + }; + // A record marked deleted still has its shape in the `.shp`; a rewrite would drop both. + if table.deleted > 0 { + return Err(refused( + path, + format!("{} of its records are marked deleted, and writing it again would remove them from the file", table.deleted), + )); + } + let mut encoding_notes = Vec::new(); + let ldid = original.dbf.get(29).copied().unwrap_or(0); + let style = DbfStyle { + encoding: crate::dbf::encoding(original.cpg.as_ref().map(|(_, b)| b.as_slice()), ldid, &mut encoding_notes), + ldid, + columns: table.columns.clone(), + }; + // A code page RustGIS can't write (one it reads as UTF-8 for want of a decoder): edited text + // would be written in UTF-8 under the file's language driver. + if let Some(note) = encoding_notes.first() { + return Err(refused(path, format!("{note}, so its text can't be written in its own code page"))); + } + + // 1. It reads as the layer was read: in the layer's system, and in the file's own + // coordinates (the system taken for longitude/latitude, which converts nothing). + let (as_read, _) = crate::io::read_shapefile_parts(original.parts(), &name, ds.crs)?; + if crate::sync::rows_digest(&as_read)? != synced.rows { + // Define Projection changes the layer's CRS, which is part of its digest. The + // unedited file must still reproduce the saved baseline in its own CRS. Keep the + // current-CRS read above for imports with an override and native-coordinate reuse. + let (baseline, _) = crate::io::read_shapefile_parts(original.parts(), &name, None)?; + if crate::sync::rows_digest(&baseline)? != synced.rows { + return Err(refused( + path, + "it reads otherwise now than at the saved sync point, even in the file's own coordinate system; the import CRS may differ or RustGIS may read it more fully now: restore the original import CRS or add the file again with the intended CRS", + )); + } + } + let raw = match crs { + Some(_) => Some(crate::io::read_shapefile_parts(original.parts(), &name, Some(Crs::LonLat))?.0), + None => None, + }; + let as_held = raw.as_ref().map(|raw| (&as_read, raw)); + + // 2. The file as read, written again, is the file. + crate::progress::begin(as_read.features.len()); + let (again, _, _) = crate::io::shapefile_parts_styled(&*in_file(&as_read, crs, as_held)?, written_crs, Some(&style))?; + let part = |parts: &[(&str, Vec)], ext: &str| parts.iter().find(|(e, _)| *e == ext).map(|(_, b)| b.clone()).unwrap_or_default(); + if let Some(why) = same_shapes(&original.shp, &part(&again, "shp"), 0.0) { + return Err(refused(path, format!("written again, its shapes wouldn't be the same: {why}"))); + } + if let Some(why) = same_table(&table, &part(&again, "dbf")) { + return Err(refused(path, format!("written again, its table wouldn't be the same: {why}"))); + } + + // 3. The layer, written, reads back as the layer. + if let Some(note) = crate::io::shapefile_names_note(&ds.fields, &name) { + return Err(refused(path, note)); + } + crate::progress::begin(ds.features.len()); + let (mut parts, left_out, widened) = crate::io::shapefile_parts_styled(&*in_file(ds, crs, as_held)?, written_crs, Some(&style))?; + if left_out > 0 { + return Err(refused( + path, + format!("{left_out} of the layer's features have no shape a Shapefile of its kind holds (none, or one of another kind)"), + )); + } + let back = ShapefileParts { + shp: part(&parts, "shp"), + shx: Some(part(&parts, "shx")), + dbf: part(&parts, "dbf"), + prj: None, + cpg: original.cpg.as_ref().map(|(_, b)| b.clone()), + }; + let (read_back, _) = crate::io::read_shapefile_parts(back, &name, crs)?; + if let Some(why) = same_rows(ds, &read_back, if crs.is_some() { PROJECTED_TOLERANCE } else { 0.0 }) { + return Err(refused(path, format!("the file couldn't hold the layer as it is: {why}"))); + } + + if crs.is_some() + && let Some(note) = projection_note(ds, &read_back)? + { + notes.push(format!("{}: {note}", path.display())); + } + + // The file's columns no field of the layer has now (deleted or renamed): left out, and said. + let kept_names = crate::io::shapefile_column_names(&ds.fields); + let gone: Vec = + table.columns.iter().filter(|c| !kept_names.iter().any(|n| n.as_bytes() == c.name.as_slice())).map(column).collect(); + if !gone.is_empty() { + let shown: Vec<&str> = gone.iter().take(5).map(String::as_str).collect(); + let more = if gone.len() > 5 { format!(" and {} more", gone.len() - 5) } else { String::new() }; + notes.push(format!( + "{}: {} left out, as no field of the layer has its name now: {}{more}", + path.display(), + if gone.len() == 1 { "1 column of the file was".to_string() } else { format!("{} columns of the file were", gone.len()) }, + shown.join(", ") + )); + } + if let Some(note) = crate::io::dates_as_text_note(&ds.fields, &path.display().to_string()) { + notes.push(note); + } + // The reader infers integers from whole numeric values, even in a decimal DBF column. + let integral: Vec = ds + .fields + .iter() + .zip(&read_back.fields) + .filter(|(a, b)| a.ty == FieldType::Double && b.ty == FieldType::Integer) + .map(|(field, _)| field.name.clone()) + .collect(); + if !integral.is_empty() { + notes.push(format!( + "{}: {} double field{} read back as integer because all non-null values are whole numbers: {}", + path.display(), + integral.len(), + if integral.len() == 1 { "" } else { "s" }, + names_list(&integral) + )); + } + if !widened.is_empty() { + let shown: Vec<&str> = widened.iter().take(5).map(String::as_str).collect(); + let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; + notes.push(format!( + "{}: {} columns other than the file's, to hold their values: {}{more}", + path.display(), + if widened.len() == 1 { "1 field was given".to_string() } else { format!("{} fields were given", widened.len()) }, + shown.join(", ") + )); + } + + // The `.prj` and `.cpg` as the file had them, unless the layer is in another system. + parts.retain(|(e, _)| !matches!(*e, "prj" | "cpg")); + if Some(written_crs) == original.crs() || (ds.crs.is_none() && original.crs().is_none()) { + if let Some(prj) = &original.prj { + parts.push(("prj", prj.clone())); + } + } else { + let Some(prj) = (match crs { + None => Some(crate::io::WGS84_PRJ.to_string()), + Some(c) => crate::raster::wkt::to_wkt(c), + }) else { + return Err(refused(path, format!("the layer is in {}, which a .prj can't describe", written_crs.label()))); + }; + notes.push(format!("{}: its .prj now says {}, the system the layer is in", path.display(), written_crs.label())); + parts.push(("prj", prj.into_bytes())); + } + let mut targets: Vec<(PathBuf, Vec)> = parts.into_iter().map(|(e, bytes)| (crate::io::shapefile_part(path, e), bytes)).collect(); + if let Some((cpg, bytes)) = &original.cpg { + targets.push((cpg.clone(), bytes.clone())); + } + let rows = crate::sync::rows_digest(&read_back)?; + // The record of the files as they will be, made before any is placed: once they are, the + // publish has happened, whatever comes after. + let files = FileSync::of_written(path, &targets)?; + + // Written beside the old parts, checked, and placed. + let staged = crate::io::stage_shapefile(&targets)?; + let checked = (|| -> Result<()> { + for ((tmp, _), (_, bytes)) in staged.iter().zip(&targets) { + if std::fs::read(tmp)? != *bytes { + return Err(refused(path, format!("{} didn't read back as written", tmp.display()))); + } + } + unchanged("while the layer was written") + })(); + if let Err(e) = checked { + crate::io::unstage(&staged); + return Err(e); + } + crate::io::place_shapefile(path, &staged)?; + Ok(Published { files, read: read_back, rows, features: ds.features.len(), notes }) +} + +/// `ds` in the file's coordinate system `crs` (`None`: longitude/latitude, as it is held). Given +/// the file as read and in its own coordinates (`as_held`, the same rows), a feature whose shape +/// is the one read takes the file's coordinates as they are; any other is projected. A native +/// shape that failed conversion is refused: missing geometry cannot identify its source shape. +fn in_file<'a>(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Dataset)>) -> Result> { + let Some(c) = crs else { return Ok(std::borrow::Cow::Borrowed(ds)) }; + let t = c.transformer().map_err(DataError::Format)?; + let (read, raw): (Vec<&Feature>, Vec<&Feature>) = match as_held { + Some((read, raw)) => (read.features.iter().collect(), raw.features.iter().collect()), + None => (Vec::new(), Vec::new()), + }; + // File reads number rows anew; persistent layer ids can have gaps after a deletion. + // Exact geometry keys retain correspondence without trusting those regenerated ids. + let key = |f: &Feature| serde_json::to_vec(&f.geometry).map_err(|e| DataError::Format(e.to_string())); + let mut at: std::collections::HashMap, (usize, bool)> = std::collections::HashMap::new(); + for (i, f) in read.iter().enumerate() { + crate::progress::check()?; + if f.geometry.is_none() && raw.get(i).is_some_and(|f| f.geometry.is_some()) { + return Err(DataError::Format(format!( + "source shape {} couldn't be converted to longitude/latitude, so its native geometry can't be matched safely; check the source projection and add the file again; nothing was written", + i + 1 + ))); + } + at.entry(key(f)?) + .and_modify(|(first, ambiguous)| *ambiguous |= raw.get(*first).zip(raw.get(i)).is_some_and(|(a, b)| a.geometry != b.geometry)) + .or_insert((i, false)); + } + let mut rows = Vec::with_capacity(ds.features.len()); + for f in ds.features.iter() { + // A shape taken as the file holds it is quick, but a layer of many is not. + crate::progress::check()?; + let matched = at.get(&key(f)?); + if matched.is_some_and(|(_, ambiguous)| *ambiguous) { + return Err(DataError::Format("several source shapes read as the same geometry but have different native coordinates; their correspondence is ambiguous, so nothing was written".into())); + } + let held = matched.and_then(|&(i, _)| raw.get(i)); + let mut row = f.clone(); + match held { + Some(h) => row.geometry = h.geometry.clone(), + // Looks at Cancel without counting: the write that follows counts the features. + None => crate::io::project_each(std::slice::from_mut(&mut row), &t, c, false)?, + } + rows.push(row); + } + let mut out = ds.clone(); + out.features = rows.into(); + Ok(std::borrow::Cow::Owned(out)) +} + +/// The first few names, quoted, and how many more. +fn names_list(names: &[String]) -> String { + let shown: Vec = names.iter().take(5).map(|n| format!("'{}'", crate::io::dbf_notice_name(n))).collect(); + let more = if names.len() > 5 { format!(" and {} more", names.len() - 5) } else { String::new() }; + format!("{}{more}", shown.join(", ")) +} + +/// A value's bytes for a notice: decoded loosely, quoted, cut after 40 bytes, control characters as +/// spaces. +fn shown(bytes: &[u8]) -> String { + format!("'{}'", crate::io::dbf_notice_name(&String::from_utf8_lossy(bytes))) +} + +/// A shape as the file holds it: its kind, and its coordinates part by part. +fn shape_parts(shape: &shapefile::Shape) -> (&'static str, Vec>) { + use shapefile::Shape; + let xy = |ps: &[shapefile::Point]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + let xym = |ps: &[shapefile::PointM]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + let xyz = |ps: &[shapefile::PointZ]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + match shape { + Shape::NullShape => ("no shape", Vec::new()), + Shape::Point(p) => ("Point", vec![vec![(p.x, p.y)]]), + Shape::PointM(p) => ("PointM", vec![vec![(p.x, p.y)]]), + Shape::PointZ(p) => ("PointZ", vec![vec![(p.x, p.y)]]), + Shape::Multipoint(m) => ("Multipoint", vec![xy(m.points())]), + Shape::MultipointM(m) => ("MultipointM", vec![xym(m.points())]), + Shape::MultipointZ(m) => ("MultipointZ", vec![xyz(m.points())]), + Shape::Polyline(l) => ("Polyline", l.parts().iter().map(|p| xy(p)).collect()), + Shape::PolylineM(l) => ("PolylineM", l.parts().iter().map(|p| xym(p)).collect()), + Shape::PolylineZ(l) => ("PolylineZ", l.parts().iter().map(|p| xyz(p)).collect()), + Shape::Polygon(p) => ("Polygon", p.rings().iter().map(|r| xy(r.points())).collect()), + Shape::PolygonM(p) => ("PolygonM", p.rings().iter().map(|r| xym(r.points())).collect()), + Shape::PolygonZ(p) => ("PolygonZ", p.rings().iter().map(|r| xyz(r.points())).collect()), + Shape::Multipatch(_) => ("Multipatch", Vec::new()), + } +} + +/// Whether `a` and `b` are within `tolerance` of each other, relative to their size. +fn near(a: f64, b: f64, tolerance: f64) -> bool { + a == b || (a - b).abs() <= tolerance * a.abs().max(b.abs()).max(1.0) +} + +/// Whether two lists of parts hold the same coordinates, within `tolerance`. +fn same_coords(a: &[Vec<(f64, f64)>], b: &[Vec<(f64, f64)>], tolerance: f64) -> bool { + a.len() == b.len() + && a.iter() + .zip(b) + .all(|(p, q)| p.len() == q.len() && p.iter().zip(q).all(|(s, t)| near(s.0, t.0, tolerance) && near(s.1, t.1, tolerance))) +} + +/// Why the shapes of `.shp` file `b` aren't those of `a`, or `None` when they are: one shape for +/// each, of the same kind, with the same coordinates within `tolerance`. Counts each kind of +/// difference and names the first shape of it. +fn same_shapes(a: &[u8], b: &[u8], tolerance: f64) -> Option { + let read = |bytes: &[u8]| -> std::result::Result, String> { + let mut reader = shapefile::ShapeReader::new(std::io::Cursor::new(bytes)).map_err(|e| e.to_string())?; + reader.iter_shapes().collect::, _>>().map_err(|e| e.to_string()) + }; + let (a, b) = match (read(a), read(b)) { + (Ok(a), Ok(b)) => (a, b), + (Err(e), _) | (_, Err(e)) => return Some(format!("its shapes couldn't be read ({e})")), + }; + let (a, b): (Vec<_>, Vec<_>) = (a.iter().map(shape_parts).collect(), b.iter().map(shape_parts).collect()); + // Each kind of shape a read leaves out, with the first record holding one. + let mut out = Vec::new(); + for kind in ["no shape", "Multipatch"] { + let held: Vec = a.iter().enumerate().filter(|(_, (k, _))| *k == kind).map(|(i, _)| i + 1).collect(); + if let Some(first) = held.first() { + let what = if kind == "no shape" { "have no shape" } else { "are multipatches" }; + out.push(format!("{} records {what}, which RustGIS leaves out (the first, record {first})", held.len())); + } + } + if !out.is_empty() { + return Some(out.join("; ")); + } + if a.len() != b.len() { + return Some(format!("{} shapes would be {}", a.len(), b.len())); + } + let kinds: Vec = a.iter().zip(&b).enumerate().filter(|(_, (s, t))| s.0 != t.0).map(|(i, _)| i).collect(); + if let Some(&first) = kinds.first() { + let (was, now) = (a.get(first).map_or("", |s| s.0), b.get(first).map_or("", |s| s.0)); + let zm = if was.ends_with('Z') || was.ends_with('M') { ", as RustGIS doesn't keep Z or M values" } else { "" }; + return Some(format!("{} shapes would change kind{zm} (the first, record {}: {was} to {now})", kinds.len(), first + 1)); + } + let moved: Vec = + a.iter().zip(&b).enumerate().filter(|(_, (s, t))| !same_coords(&s.1, &t.1, tolerance)).map(|(i, _)| i).collect(); + moved.first().map(|first| format!("{} shapes would change (the first, record {})", moved.len(), first + 1)) +} + +/// A `.dbf` value as a value, for comparing what two files hold: text without the spaces or NULs +/// that pad it, a number as the decimal it says exactly (no `f64`, which can't tell +/// 9007199254740993 from 9007199254740992), a date or yes/no without the forms of empty. +#[derive(PartialEq)] +enum Cell<'a> { + Bytes(&'a [u8]), + /// Negative, the whole digits without leading zeros, the decimals without trailing zeros. + Number(bool, &'a [u8], &'a [u8]), + Empty, +} + +/// A number's text as its sign, whole digits and decimals with the zeros that don't count left +/// off (`-003.50` is `-3.5`; `-0` is `0`), or `None` when it isn't plain decimal digits. +fn decimal(text: &[u8]) -> Option<(bool, &[u8], &[u8])> { + let (negative, digits) = match text.split_first() { + Some((b'-', rest)) => (true, rest), + Some((b'+', rest)) => (false, rest), + _ => (false, text), + }; + let (whole, decimals) = match digits.iter().position(|b| *b == b'.') { + Some(at) => (digits.get(..at)?, digits.get(at + 1..)?), + None => (digits, &b""[..]), + }; + if whole.is_empty() && decimals.is_empty() || !whole.iter().chain(decimals).all(u8::is_ascii_digit) { + return None; + } + let whole = whole.get(whole.iter().position(|b| *b != b'0').unwrap_or(whole.len())..)?; + let decimals = decimals.get(..decimals.iter().rposition(|b| *b != b'0').map_or(0, |i| i + 1))?; + Some((negative && !(whole.is_empty() && decimals.is_empty()), whole, decimals)) +} + +/// What value `bytes` of a column of type `ty` holds ([`Cell`]). +fn cell(ty: u8, bytes: &[u8]) -> Cell<'_> { + let padding = |b: &u8| *b == b' ' || *b == 0; + let end = bytes.iter().rposition(|b| !padding(b)).map_or(0, |i| i + 1); + let right = bytes.get(..end).unwrap_or_default(); + let trimmed = right.get(right.iter().position(|b| !padding(b)).unwrap_or(right.len())..).unwrap_or_default(); + match ty.to_ascii_uppercase() { + b'C' => { + if right.is_empty() { + Cell::Empty + } else { + Cell::Bytes(right) + } + } + b'N' | b'F' => { + if trimmed.is_empty() { + return Cell::Empty; + } + match decimal(trimmed) { + Some((negative, whole, decimals)) => Cell::Number(negative, whole, decimals), + None => Cell::Bytes(trimmed), + } + } + b'L' => match trimmed.first().map(u8::to_ascii_uppercase) { + Some(b'T' | b'Y' | b'1') => Cell::Bytes(b"T"), + Some(b'F' | b'N' | b'0') => Cell::Bytes(b"F"), + _ => Cell::Empty, + }, + b'D' if trimmed.is_empty() || trimmed == b"00000000" => Cell::Empty, + _ => Cell::Bytes(trimmed), + } +} + +/// A column described for a notice: its name, type, width and decimals (`N(10,2)`). +fn column(c: &Column) -> String { + let name = String::from_utf8_lossy(&c.name); + let ty = char::from(c.ty); + if matches!(c.ty.to_ascii_uppercase(), b'N' | b'F') { + format!("'{}' {ty}({},{})", crate::io::dbf_notice_name(&name), c.len, c.decimals) + } else { + format!("'{}' {ty}({})", crate::io::dbf_notice_name(&name), c.len) + } +} + +/// Why the `.dbf` `b` doesn't hold what `a` holds, or `None` when it does: the same columns +/// (names, types, widths, decimals) and, record by record, the same values ([`cell`]). Counts the +/// values that differ and names the first. +fn same_table(a: &RawTable, b: &[u8]) -> Option { + let Some(b) = crate::dbf::raw_table(b) else { return Some("its .dbf couldn't be written".into()) }; + if a.columns.len() != b.columns.len() { + return Some(format!("it has {} columns and would have {}", a.columns.len(), b.columns.len())); + } + let changed: Vec = a + .columns + .iter() + .zip(&b.columns) + .filter(|(s, t)| s.name != t.name || !s.ty.eq_ignore_ascii_case(&t.ty) || s.len != t.len || s.decimals != t.decimals) + .map(|(s, t)| format!("{} would be {}", column(s), column(t))) + .collect(); + if !changed.is_empty() { + return Some(format!("its columns would change: {}", changed.join(", "))); + } + if a.records.len() != b.records.len() { + return Some(format!("{} records would be {}", a.records.len(), b.records.len())); + } + let mut first = None; + let mut count = 0usize; + for (row, (r, s)) in a.records.iter().zip(&b.records).enumerate() { + for (c, (x, y)) in a.columns.iter().zip(r.iter().zip(s)) { + if cell(c.ty, x) != cell(c.ty, y) { + count += 1; + first.get_or_insert((row, c, *x, *y)); + } + } + } + first.map(|(row, c, x, y)| { + format!( + "{count} values would change (the first, record {} of '{}': {} would be {})", + row + 1, + crate::io::dbf_notice_name(&String::from_utf8_lossy(&c.name)), + shown(x), + shown(y) + ) + }) +} + +/// Whether two values of a field of type `ty` are the same once written to a `.dbf` and read: +/// numbers by their value (a whole number reads back as an integer), empty text and no value +/// alike (a `.dbf` can't tell them apart), a date-time by its text. +fn same_value(ty: FieldType, a: &Value, b: &Value) -> bool { + let text = |v: &Value| match v { + Value::Null => String::new(), + v => v.to_string(), + }; + match (a, b) { + _ if a == b => true, + (Value::Int(_) | Value::Float(_), Value::Int(_) | Value::Float(_)) => a.as_f64() == b.as_f64(), + _ if matches!(ty, FieldType::Text | FieldType::DateTime) => text(a) == text(b), + _ => false, + } +} + +/// A geometry as its parts of coordinates: a point is a part of one, each polygon ring closed +/// and turned to run the same way whichever way it was drawn (a Shapefile orders a ring's +/// vertices by whether it is a hole). +fn geometry_parts(g: Option<&Geometry>) -> Vec> { + let xy = |cs: &[crate::Coord]| cs.iter().map(|c| (c.x, c.y)).collect::>(); + match g { + None => Vec::new(), + Some(Geometry::Point(c)) => vec![vec![(c.x, c.y)]], + Some(Geometry::MultiPoint(cs)) => cs.iter().map(|c| vec![(c.x, c.y)]).collect(), + Some(Geometry::Line(parts)) => parts.iter().map(|p| xy(p)).collect(), + Some(Geometry::Polygon(polys)) => polys + .iter() + .flatten() + .map(|r| { + let mut ring = xy(r); + if ring.first() != ring.last() { + ring.extend(ring.first().copied()); + } + // Twice the signed area: positive runs anticlockwise. + let area: f64 = ring.windows(2).map(|w| w.first().zip(w.get(1)).map_or(0.0, |(p, q)| p.0 * q.1 - q.0 * p.1)).sum(); + if area < 0.0 { + ring.reverse(); + } + ring + }) + .collect(), + } +} + +/// Projection round-off actually present in the checked read-back: a count, the first feature +/// and the largest coordinate difference in degrees. Exact round-trips need no notice. +fn projection_note(ds: &Dataset, back: &Dataset) -> Result> { + let (mut count, mut first, mut largest) = (0usize, 0usize, 0.0_f64); + for (k, (f, g)) in ds.features.iter().zip(back.features.iter()).enumerate() { + crate::progress::check()?; + let (p, q) = (geometry_parts(f.geometry.as_ref()), geometry_parts(g.geometry.as_ref())); + if same_coords(&p, &q, 0.0) { + continue; + } + if count == 0 { + first = k + 1; + } + count += 1; + for (a, b) in p.iter().flatten().zip(q.iter().flatten()) { + largest = largest.max((a.0 - b.0).abs()).max((a.1 - b.1).abs()); + } + } + Ok((count > 0).then(|| { + format!( + "shapes with projection round-off: {count} (the first, feature {first}); the largest coordinate difference is {largest:e} degrees" + ) + })) +} + +/// Why `back`, the layer `ds` written to a Shapefile and read again, isn't `ds`, or `None` when +/// it is: the same fields, features, values ([`same_value`]) and shapes within `tolerance`. +/// Counts the values and shapes that differ and names the first of each. +fn same_rows(ds: &Dataset, back: &Dataset, tolerance: f64) -> Option { + let names = |d: &Dataset| d.fields.iter().map(|f| f.name.clone()).collect::>(); + if names(ds) != names(back) { + return Some(format!("its fields would read back as {}", names_list(&names(back)))); + } + if ds.features.len() != back.features.len() { + return Some(format!("its {} features would read back as {}", ds.features.len(), back.features.len())); + } + let (mut values, mut first_value, mut shapes, mut first_shape) = (0usize, None, 0usize, None); + for (k, (f, g)) in ds.features.iter().zip(back.features.iter()).enumerate() { + for (field, (a, b)) in ds.fields.iter().zip(f.attrs.iter().zip(&g.attrs)) { + if !same_value(field.ty, a, b) { + values += 1; + first_value.get_or_insert((k, field.name.clone(), a.clone(), b.clone())); + } + } + // Points are compared as parts of one point each, so a point and a multipoint of one agree. + let (p, q) = (geometry_parts(f.geometry.as_ref()), geometry_parts(g.geometry.as_ref())); + let grouping = |g: Option<&Geometry>| match g { + Some(Geometry::Polygon(polys)) => Some(polys.iter().map(Vec::len).collect::>()), + _ => None, + }; + if grouping(f.geometry.as_ref()) != grouping(g.geometry.as_ref()) || !same_coords(&p, &q, tolerance) { + shapes += 1; + first_shape.get_or_insert(k); + } + } + let mut out = Vec::new(); + if let Some((k, name, a, b)) = first_value { + let show = |v: &Value| if v.is_null() { "empty".to_string() } else { shown(v.to_string().as_bytes()) }; + out.push(format!( + "{values} values would read back otherwise (the first, feature {} of '{}': {} as {})", + k + 1, + crate::io::dbf_notice_name(&name), + show(&a), + show(&b) + )); + } + if let Some(k) = first_shape { + out.push(format!("{shapes} shapes would read back otherwise (the first, feature {})", k + 1)); + } + (!out.is_empty()).then(|| out.join("; ")) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{Coord, Field, GeometryType}; + + /// A small polygon layer of parcels with an owner, an area with decimals and a count. + fn parcels() -> Dataset { + let mut ds = Dataset::new( + "parcels", + GeometryType::Polygon, + vec![Field::new("OWNER", FieldType::Text), Field::new("ACRES", FieldType::Double), Field::new("UNITS", FieldType::Integer)], + ); + let square = |x: f64| { + Geometry::Polygon(vec![vec![vec![ + Coord::new(x, 35.0), + Coord::new(x, 35.1), + Coord::new(x + 0.1, 35.1), + Coord::new(x + 0.1, 35.0), + Coord::new(x, 35.0), + ]]]) + }; + ds.push(Some(square(-90.0)), vec![Value::Text("Smith".into()), Value::Float(1.25), Value::Int(2)]); + ds.push(Some(square(-89.8)), vec![Value::Text("Jones".into()), Value::Float(0.5), Value::Int(1)]); + ds + } + + /// Writes `ds` as a Shapefile at `path` and reads it back with its sync record. + fn written(ds: &Dataset, path: &Path) -> (Dataset, Synced) { + crate::io::write_path(ds, path).unwrap(); + read(path) + } + + /// Reads the Shapefile at `path` with its sync record, as Add Data does. + fn read(path: &Path) -> (Dataset, Synced) { + let (read, files) = FileSync::around(path, || crate::io::read_path(path).unwrap()).unwrap(); + let rows = crate::sync::rows_digest(&read.0).unwrap(); + (read.0, Synced { files: files.unwrap(), rows }) + } + + /// An edit is written back: the file reads as the layer, keeps its columns and `.prj`, and + /// its new record matches the files as they are. + #[test] + fn an_edit_is_written_back_and_the_file_keeps_its_columns() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let prj = std::fs::read(path.with_extension("prj")).unwrap(); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{:?}", published.notes); + let (back, _) = crate::io::read_path(&path).unwrap(); + assert_eq!(back.features.iter().next().unwrap().attrs[0], Value::Text("Smyth".into())); + assert_eq!(std::fs::read(path.with_extension("prj")).unwrap(), prj); + assert_eq!(crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns, columns); + assert!(published.files.changed(&path).unwrap().is_empty()); + // Only the reusable lock sidecar remains; no staged or backup parts. + let names: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert!(names.iter().all(|n| !n.starts_with('.') || n == ".parcels.shp.rustgis-lock"), "{names:?}"); + } + + /// Reads follow symlinks, but replacement would sever them and leave the target stale. + #[cfg(unix)] + #[test] + fn linked_shapefile_parts_are_refused_without_replacing_the_link_or_target() { + for ext in ["shp", "shx", "dbf", "prj", "CPG"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + if ext == "CPG" { + std::fs::write(path.with_extension(ext), b"UTF-8").unwrap(); + } + let part = path.with_extension(ext); + let target = dir.path().join(format!("original.{ext}")); + std::fs::rename(&part, &target).unwrap(); + std::os::unix::fs::symlink(&target, &part).unwrap(); + let before = std::fs::read(&target).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().expect("linked part must be refused").to_string(); + assert!(e.contains("symbolic link") && e.contains("nothing was written"), "{ext}: {e}"); + assert_eq!(std::fs::read_link(&part).unwrap(), target); + assert_eq!(std::fs::read(&target).unwrap(), before); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + } + + /// An independent handle represents a concurrent publisher. Its lock refuses a publish + /// without touching the source, and is reusable after closing, with no stale-lock cleanup. + #[test] + fn a_concurrent_publisher_is_refused_until_the_lock_is_released() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let held = source_lock(&path).unwrap(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("another RustGIS publisher") && e.contains("nothing was written"), "{e}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + drop(held); + shapefile(&ds, &path, &sync).unwrap(); + // A second publisher that loaded the old baseline now sees the committed change. + assert!(shapefile(&ds, &path, &sync).err().unwrap().to_string().contains("changed since the layer was read")); + } + + /// A duplicated descriptor must not keep a finished publisher's lock alive. + #[test] + fn a_source_lock_releases_while_a_duplicate_descriptor_stays_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + let held = source_lock(&path).unwrap(); + let duplicate = held.0.try_clone().unwrap(); + drop(held); + let next = source_lock(&path).expect("a finished publisher must release its lock despite a duplicated descriptor"); + drop(duplicate); + assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); + drop(next); + } + + /// A shared source's empty lock is writable across accounts, even under a private umask. + /// Reopening it keeps a writable handle (required by NFS) and excludes a second publisher. + #[cfg(unix)] + #[test] + fn shared_source_locks_are_writable_and_reusable() { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).unwrap(); + let held = source_lock(&path).unwrap(); + let lock_path = dir.path().join(".parcels.shp.rustgis-lock"); + assert_eq!(std::fs::metadata(&lock_path).unwrap().permissions().mode() & 0o777, 0o666); + drop(held); + let mut held = source_lock(&path).unwrap(); + // A zero-byte write checks the descriptor's access mode without storing source data. + assert_eq!(held.0.write(&[]).unwrap(), 0); + assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); + drop(held); + source_lock(&path).unwrap(); + } + + /// A collapsed hole is inferred as an outer ring on read: identical flat coordinates + /// must not let that topology change overwrite the source. + #[test] + fn a_polygon_hole_that_reads_as_an_outer_ring_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[0].geometry { + polys[0].push(vec![Coord::new(-89.98, 35.02), Coord::new(-89.96, 35.02), Coord::new(-89.98, 35.02)]); + } + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().expect("changed ring grouping must be refused").to_string(); + assert!(e.contains("shapes would read back otherwise") && e.contains("nothing was written"), "{e}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + + /// Geometry edits remove obsolete indexes regardless of their extension's letter case. + #[test] + fn publishing_removes_case_variant_indexes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let indexes = ["qix", "QIX", "qIx", "sbn", "SBN", "SBX"]; + for extension in indexes { + std::fs::write(path.with_extension(extension), b"old index").unwrap(); + } + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[0].geometry { + polys[0][0][1].x += 0.005; + } + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + for extension in indexes { + assert!(!path.with_extension(extension).exists(), "stale index {extension} survived"); + } + } + + /// A DBF record beyond its declared count must not disappear during an unrelated edit. + #[test] + fn publishing_refuses_unaccounted_dbf_records() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let width = usize::from(u16::from_le_bytes([dbf[10], dbf[11]])); + let count = u32::from_le_bytes(dbf[4..8].try_into().unwrap()) as usize; + let extra = dbf[header..header + width].to_vec(); + dbf.truncate(header + count * width); + dbf.extend(extra); + dbf.push(0x1a); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.len(), 2); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let error = shapefile(&ds, &path, &sync).err().expect("uncounted DBF records must be refused").to_string(); + assert!(error.contains("header") && error.contains("records") && error.contains("nothing was written"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + + /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a + /// project that couldn't hold a value) is refused: its unedited rows aren't the file's. + #[test] + fn a_layer_read_otherwise_than_the_file_reads_now_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (ds, mut sync) = written(&parcels(), &path); + // As an older read had it: one value otherwise. + let mut older = ds.clone(); + let mut rows: Vec<_> = older.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(1.2); + older.features = rows.into(); + sync.rows = crate::sync::rows_digest(&older).unwrap(); + let e = shapefile(&older, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("reads otherwise now") && e.contains("nothing was written"), "{e}"); + } + + /// Defining a missing or incorrect source CRS publishes the definition without moving + /// its native coordinates, and records a sync point that permits the next publish. + #[test] + fn a_defined_projection_publishes_without_moving_native_coordinates() { + for (source, crs) in [(None, Crs::WebMercator), (None, Crs::LonLat), (Some(Crs::WebMercator), Crs::LonLat)] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + match source { + Some(c) => std::fs::write(path.with_extension("prj"), crate::raster::wkt::to_wkt(c).unwrap()).unwrap(), + None => std::fs::remove_file(path.with_extension("prj")).unwrap(), + } + let before = std::fs::read(&path).unwrap(); + let (ds, sync) = read(&path); + let defined = crate::io::define_projection(&ds, crs).unwrap(); + let published = shapefile(&defined, &path, &sync).unwrap(); + if source.is_none() { + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none()); + } + assert!(published.notes.iter().any(|n| n.contains(".prj now says")), "{:?}", published.notes); + let (back, next) = read(&path); + assert_eq!(Original::read(&path).unwrap().crs(), Some(crs)); + assert_eq!(back.crs, (crs != Crs::LonLat).then_some(crs)); + assert!(same_rows(&defined, &back, PROJECTED_TOLERANCE).is_none()); + assert_eq!(next.rows, published.rows); + shapefile(&back, &path, &next).unwrap(); + } + } + + /// An unedited numeric overflow marker must not silently become an empty cell. + #[test] + fn an_overflow_cell_is_refused_without_changing_the_source() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&dbf).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let at = header + 1 + usize::from(table.columns[0].len) + usize::from(table.columns[1].len); + dbf[at..at + 18].fill(b'*'); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.first().unwrap().attrs[2], Value::Null); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + let error = shapefile(&ds, &path, &sync).err().expect("overflow marker must be refused").to_string(); + assert!(error.contains("table wouldn't be the same") && error.contains("UNITS") && error.contains("***"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + + /// A number past what an `f64` holds exactly, in a column that wasn't edited, is refused + /// rather than written back rounded. + #[test] + fn a_number_an_f64_rounds_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // UNITS, an N(18,0) column, holding 2⁵³ + 1 in the first record. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&dbf).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let at = header + 1 + usize::from(table.columns[0].len) + usize::from(table.columns[1].len); + dbf[at..at + 18].copy_from_slice(b" 9007199254740993"); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("table wouldn't be the same") && e.contains("9007199254740993"), "{e}"); + // Written the same in other forms, a number is the same. + assert!(cell(b'N', b" 003.50") == cell(b'N', b"3.5")); + assert!(cell(b'N', b"-0.00") == cell(b'N', b"0")); + assert!(cell(b'N', b"9007199254740993") != cell(b'N', b"9007199254740992")); + } + + /// A code page RustGIS reads as UTF-8 for want of a decoder (DOS French Canadian, by its + /// language driver) is refused: edited text would be written in another code page. + #[test] + fn a_code_page_rustgis_cant_write_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + std::fs::remove_file(path.with_extension("cpg")).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + dbf[29] = 0x6C; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("code page 863") && e.contains("nothing was written"), "{e}"); + } + + /// Once published, the layer as the file reads it (an empty text as no value) is its record: + /// it publishes again. + #[test] + fn a_layer_the_file_reads_otherwise_publishes_again() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text(String::new()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.read.features.iter().next().unwrap().attrs[0], Value::Null); + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&published.read).unwrap() }; + let mut ds = published.read; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + shapefile(&ds, &path, &synced).unwrap(); + } + + /// A field deleted from the layer leaves the file's column out, and says so. + #[test] + fn a_deleted_field_leaves_its_column_out_and_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + ds.fields.remove(2); + let rows: Vec<_> = ds + .features + .iter() + .cloned() + .map(|mut f| { + f.attrs.remove(2); + f + }) + .collect(); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!( + published.notes.iter().any(|n| n.contains("1 column of the file was left out") && n.contains("'UNITS' N(18,0)")), + "{:?}", + published.notes + ); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + assert_eq!(columns.len(), 2); + } + + /// A file with records marked deleted is refused: writing it again would drop them, shapes and + /// all. + #[test] + fn a_file_with_deleted_records_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + dbf[header] = b'*'; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("1 of its records are marked deleted") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), dbf); + } + + /// A file changed since it was read is never written over. + #[test] + fn a_changed_file_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (ds, sync) = written(&parcels(), &path); + let other = Dataset { features: ds.features.iter().take(1).cloned().collect::>().into(), ..ds.clone() }; + crate::io::write_path(&other, &path).unwrap(); + let before = std::fs::read(path.with_extension("dbf")).unwrap(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("changed since the layer was read") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), before); + } + + /// Text in a code page is written back in it, with the file's language driver; a character + /// the code page can't hold is refused rather than written as something else. + #[test] + fn a_code_page_is_kept_and_what_it_cant_hold_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // The same file in Windows-1252: its `.cpg` says so. + std::fs::write(path.with_extension("cpg"), "1252").unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Muñoz".into()); + ds.features = rows.clone().into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + // The record of what was published, as the layer takes it. + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&published.read).unwrap() }; + let dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + assert!(dbf.windows(5).any(|w| w == b"Mu\xf1oz"), "written in Windows-1252"); + assert_eq!(std::fs::read(path.with_extension("cpg")).unwrap(), b"1252"); + assert_eq!(crate::io::read_path(&path).unwrap().0.features.iter().next().unwrap().attrs[0], Value::Text("Muñoz".into())); + // A character Windows-1252 doesn't have. + rows[1].attrs[0] = Value::Text("Łukasz".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &synced).err().unwrap().to_string(); + assert!(e.contains("couldn't hold the layer") && e.contains("Łukasz"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), dbf); + } + + /// What a read loses is refused before the edits are: text a read trims, a feature without a + /// shape, a number too precise for the column. + #[test] + fn what_a_read_loses_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // The first owner with a space before it, as another program may have written it. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let at = dbf.windows(5).position(|w| w == b"Smith").unwrap(); + dbf.splice(at..at + 5, *b" Smit"); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("table wouldn't be the same") && e.contains("' Smit'"), "{e}"); + } + + /// A float (`F`) column reads every digit its text holds, not the seven of an `f32`, and is + /// written back as an `F` column with its width and decimals. + #[test] + fn a_float_column_keeps_its_digits_and_its_type() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut ds = parcels(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(123456.12345678); + ds.features = rows.into(); + crate::io::write_path(&ds, &path).unwrap(); + // ACRES, the second column, made a float column as other programs write one. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + dbf[32 + 32 + 11] = b'F'; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.iter().next().unwrap().attrs[1], Value::Float(123456.12345678)); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + assert_eq!(columns[1].ty, b'F'); + assert_eq!(crate::io::read_path(&path).unwrap().0.features.iter().next().unwrap().attrs[1], Value::Float(123456.12345678)); + } + + /// Converting a text field to a date changes its physical column, which publishing says. + #[test] + fn a_converted_field_names_its_old_and_new_column_types() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut source = parcels(); + let mut rows: Vec<_> = source.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Text("2025-01-02".into()); + } + source.features = rows.into(); + let (mut ds, sync) = written(&source, &path); + ds.fields[0].ty = FieldType::Date; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Date(crate::Date::parse("2025-01-02").unwrap()); + } + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + assert!(published.notes[0].contains("'OWNER' C(10) to D(8)"), "{:?}", published.notes); + let bytes = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&bytes).unwrap(); + assert_eq!(table.columns[0].ty, b'D'); + } + + /// A date-time still fits its text column, but losing the field type must be said once. + #[test] + fn a_date_time_field_written_as_text_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let stamp = "2025-01-02T12:30:00Z"; + let mut source = parcels(); + let mut rows: Vec<_> = source.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Text(stamp.into()); + } + source.features = rows.into(); + let (mut ds, sync) = written(&source, &path); + ds.fields[0].ty = FieldType::DateTime; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::DateTime(crate::DateTime::parse(stamp).unwrap()); + } + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!(note.contains("1 date-time field was written as text (ISO 8601)") && note.contains("'OWNER'"), "{note}"); + assert!(note.contains(&path.display().to_string()), "{note}"); + assert_eq!(published.read.fields[0].ty, FieldType::Text); + assert_eq!(published.read.features.first().unwrap().attrs[0], Value::Text(stamp.into())); + } + + /// Editing the last fractional values changes the read-back field kind, which must be said. + #[test] + fn a_double_field_read_back_as_integer_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + assert_eq!(ds.fields[1].ty, FieldType::Double); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(100.0); + rows[1].attrs[1] = Value::Float(200.0); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.read.fields[1].ty, FieldType::Integer); + assert_eq!(published.read.features.first().unwrap().attrs[1], Value::Int(100)); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!(note.contains("1 double field read back as integer") && note.contains("'ACRES'"), "{note}"); + assert!(note.contains(&path.display().to_string()), "{note}"); + let again = shapefile(&published.read, &path, &Synced { files: published.files, rows: published.rows }).unwrap(); + assert!(again.notes.is_empty(), "{:?}", again.notes); + } + + /// A whole-valued double reads as Integer, but its existing wide decimal column still fits. + #[test] + fn an_integral_double_keeps_its_wide_decimal_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut ds = parcels(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(100.0); + rows[1].attrs[1] = Value::Float(200.0); + ds.features = rows.into(); + let (mut ds, sync) = written(&ds, &path); + assert_eq!(ds.fields[1].ty, FieldType::Integer); + let before = std::fs::read(path.with_extension("dbf")).unwrap(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{:?}", published.notes); + let after = std::fs::read(path.with_extension("dbf")).unwrap(); + let (a, b) = (crate::dbf::raw_table(&before).unwrap(), crate::dbf::raw_table(&after).unwrap()); + assert_eq!(a.columns, b.columns); + for (a, b) in a.records.iter().zip(&b.records) { + assert_eq!(a[1], b[1]); + } + } + + /// An unconverted source shape cannot supply geometry to a new geometry-less feature. + #[test] + fn an_unconverted_shape_cannot_be_copied_to_a_geometryless_feature() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut native = Dataset::new("parcels", GeometryType::Point, vec![Field::new("OWNER", FieldType::Text)]); + native.push(Some(Geometry::Point(Coord::new(1e30, 1e30))), vec![Value::Text("old".into())]); + native.push(Some(Geometry::Point(Coord::new(500_000.0, 0.0))), vec![Value::Text("kept".into())]); + crate::io::write_path(&native, &path).unwrap(); + let crs = Crs::parse("EPSG:32616").unwrap(); + std::fs::write(path.with_extension("prj"), crate::raster::wkt::to_wkt(crs).unwrap()).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.iter().filter(|f| f.geometry.is_none()).count(), 1); + ds.push(None, vec![Value::Text("new".into())]); + let error = shapefile(&ds, &path, &sync).err().expect("unconverted shapes must not be assigned to new features").to_string(); + assert!(error.contains("source shape 1") && error.contains("converted") && error.contains("nothing was written"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + + /// After deletion, persistent layer ids differ from file row numbers. An attribute-only + /// second publish must still copy the untouched projected coordinates to the bit. + #[test] + fn repeated_publication_keeps_native_shapes_after_deleting_a_feature() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let crs = Crs::parse("EPSG:2274").unwrap(); + crate::io::write_path_in(&parcels(), &path, crs).unwrap(); + let (mut ds, sync) = read(&path); + let survivor = ds.features.get(1).unwrap().clone(); + let id = survivor.id; + ds.features = vec![survivor].into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + ds = published.read; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].id = id; + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let sync = Synced { files: published.files, rows: published.rows }; + let before = std::fs::read(&path).unwrap(); + shapefile(&ds, &path, &sync).unwrap(); + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none()); + } + + /// The writer uses UTF-8 name bytes even with a legacy code page: a rewrite is refused + /// before the differing name or width can reach the source. + #[test] + fn a_legacy_non_ascii_column_name_refuses_the_rewrite_without_changing_the_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + std::fs::write(path.with_extension("cpg"), "1252").unwrap(); + let mut bytes = std::fs::read(path.with_extension("dbf")).unwrap(); + bytes[32..43].fill(0); + bytes[32..37].copy_from_slice(b"Se\xf1or"); + std::fs::write(path.with_extension("dbf"), &bytes).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.fields[0].name, "Señor"); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("its columns would change") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), bytes); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + + /// A widened column is said; a layer in another system gets a `.prj` for it, said too. + #[test] + fn a_wider_value_widens_its_column_and_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smith and Jones Holdings".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + assert!(published.notes[0].contains("1 field was given columns other than the file's"), "{:?}", published.notes); + assert!(published.notes[0].contains("'OWNER' C(5) to C(24)"), "{:?}", published.notes); + let (back, _) = crate::io::read_path(&path).unwrap(); + assert_eq!(back.features.iter().next().unwrap().attrs[0], Value::Text("Smith and Jones Holdings".into())); + } + + /// A projected file's unedited shapes stay where they were, to the tolerance. + #[test] + fn a_projected_file_keeps_its_unedited_shapes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + // UTM in metres, and a state plane system in US feet. + for code in ["EPSG:32616", "EPSG:2274"] { + let crs = Crs::parse(code).unwrap(); + let mut ds = parcels(); + ds.crs = Some(crs); + crate::io::write_path_in(&ds, &path, crs).unwrap(); + let before = std::fs::read(&path).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[2] = Value::Int(3); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{code}: {:?}", published.notes); + // Every shape as it was, to the bit: none was edited. + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none(), "{code}"); + // A shape moved is projected; the others stay as they were. + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[1].geometry { + polys[0][0][1].y += 0.01; + } + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + if code == "EPSG:2274" { + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!( + note.contains("shapes with projection round-off: 1") && note.contains("feature 2") && note.contains("degrees"), + "{note}" + ); + } + let shapes = |b: &[u8]| -> Vec<_> { + let mut r = shapefile::ShapeReader::new(std::io::Cursor::new(b.to_vec())).unwrap(); + r.iter_shapes().map(|s| shape_parts(&s.unwrap()).1).collect() + }; + let (was, now) = (shapes(&before), shapes(&std::fs::read(&path).unwrap())); + assert_eq!(was[0], now[0], "{code}"); + assert_ne!(was[1], now[1], "{code}"); + } + } +} diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index ff65602256..3e626f844f 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -30,14 +30,14 @@ pub struct FileStamp { pub struct Synced { /// The files, as they were. pub files: FileSync, - /// The layer's [`rows_digest`] then. + /// The layer's [`rows_digest`] then: the file's as it read then too. pub rows: u64, } /// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and /// attributes in order (FNV-1a over their JSON, written to the hash as it is made). Not its name -/// or where it came from, which are no edit to the file. A number that isn't finite, which JSON -/// writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit +/// or where it came from, or display aliases, which are no edit to the file. A number that isn't +/// finite, which JSON writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit /// rows are: clearing one is an edit. A project saves one as null (and says so), so a layer that /// held one reads as edited once its project is opened again: it no longer holds what its file /// does. Its work grows with the rows: each is counted as progress ([`crate::progress`]), and it @@ -61,7 +61,13 @@ pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { h.0 = (h.0 ^ 0xff).wrapping_mul(0x0100_0000_01b3); } let mut h = Fnv(0xcbf2_9ce4_8422_2325); - put(&mut h, &(&ds.fields, &ds.crs)); + // Whole Shapefile/GeoJSON reads record fields with no aliases. Clear display-only aliases + // when comparing a layer, keeping the serialized form and existing saved digests unchanged. + let mut fields = ds.fields.clone(); + for field in &mut fields { + field.alias = None; + } + put(&mut h, &(&fields, &ds.crs)); crate::progress::begin(ds.features.len()); for f in ds.features.iter() { put(&mut h, &(&f.geometry, &f.attrs)); @@ -161,6 +167,28 @@ mod native { Ok(FileSync { files }) } + /// The files of the source at `path` as just written: each of `written` (a path and + /// the bytes written to it) by those bytes, so a write made since by someone else isn't + /// taken for this one; any other part as it is on the disk. + pub fn of_written(path: &Path, written: &[(PathBuf, Vec)]) -> Result { + use sha2::Digest; + let mut files = Vec::new(); + for (name, p) in parts(path) { + let stamp = match written.iter().find(|(w, _)| *w == p) { + Some((_, bytes)) => Some(FileStamp { + len: bytes.len() as u64, + sha256: sha2::Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect(), + }), + None => match len(&p)? { + None => None, + Some(len) => Some(FileStamp { len, sha256: sha256(&p)? }), + }, + }; + files.push((name, stamp)); + } + Ok(FileSync { files }) + } + /// Runs `read`, which reads the source at `path`, and returns with its result the files /// as they were just before it ran (`None` when a part couldn't be read). Hashed before /// the read, the record can only be older than what was read: a write while it ran, or @@ -335,6 +363,8 @@ mod tests { }] .into(); assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092); + ds.fields[0].alias = Some("Street name".into()); + assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092, "an alias changes display, not file contents"); } #[test] diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index 4485de10d4..7e4013a293 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -44,6 +44,7 @@ pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ ("share.exportMap", "Export Map", super::share::map_work), ("layout.export", "Export Layout", super::layout::export_work), ("report.export", "Export Report", super::report::export_work), + ("source.publish", "Publish to Source", super::publish::publish_work), ]; /// `command` and what it is called, when the app writes its file on another thread (see @@ -65,7 +66,7 @@ pub(crate) fn write_here(s: &mut Session, p: &Value, command: &str) -> CmdResult let work = start(s, p)?; written_under(name, &control, work) })?; - s.check_continuation(&owner).map_err(|why| completion_refused(wrote.written_path.as_deref(), why))?; + s.check_continuation(&owner).map_err(|why| completion_with_notes(command, wrote.written_path.as_deref(), &wrote.notes, why))?; said(s, wrote) } @@ -77,6 +78,24 @@ pub(crate) fn completion_refused(written_path: Option<&str>, why: String) -> Str } } +/// A committed publication's format notices remain part of its refusal diagnostic; its +/// finalizer stays blocked, so reporting the write cannot update a revoked layer's record. +fn completion_with_notes(command: &str, written_path: Option<&str>, notes: &[String], why: String) -> String { + if command == "source.publish" + && let Some(path) = written_path + { + let mut message = + format!("Published {path}; completion was refused: {why}; the layer's record wasn't updated; add the file again to compare it"); + for note in notes { + message.push('\n'); + message.push_str(note); + } + message + } else { + completion_refused(written_path, why) + } +} + /// `work` under `control`, as [`rustgis_analysis::run_under`] runs it, except that a file once /// placed is written: a Cancel that comes after the last step doesn't turn it into a failure. fn written_under(name: &str, control: &std::sync::Arc, work: Work) -> Result { @@ -284,8 +303,9 @@ pub fn export_land(s: &mut Session, started: Started, done: Result Vec { vec![ C::new("source.syncState", "Source File State", "Data › Data Source", sync_state).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile or .geojson file"} — whether it has edits its file lacks ("edited"), the file changed since it was read ("sourceChanged"), both ("both") or neither ("inSync"), and which of the file's parts changed"#, ), + C::new("source.publish", "Publish to Source", "Data › Data Source", |s, p| super::exports::write_here(s, p, "source.publish")).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files).with(crate::policy::Effect::Edits)).view().params( + r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile"} — writes the layer's edits back over the Shapefile it was read from, keeping its code page, columns and .prj; refused, with nothing written, when the file changed since it was read or can't hold the layer as it is (Z or M values, features without shapes, text its code page lacks)"#, + ), ] } +/// The layer's record of the file it was read from and that file's path, or why it has none +/// (it wasn't read whole from a Shapefile or `.geojson` file, or its record was lost), or why +/// the file isn't opened (it isn't on this computer). +fn record(l: &rustgis_map::Layer) -> Result<(&rustgis_data::sync::Synced, &str), String> { + let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { + if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { + return Err(format!( + "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", + l.name + )); + } + return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); + }; + // A project from elsewhere names the path: only a local file of a kind a record is made for + // is opened, never a network share (opening one can send the user's login to its host). + if !local_file(path) { + return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); + } + Ok((synced, path.as_str())) +} + +/// `source.publish`'s work, its inputs taken from the session ([`super::exports`]): writes layer +/// `p` names back over the Shapefile it was read from ([`rustgis_data::publish::shapefile`], +/// which says what it checks first), when it has edits the file lacks. Its work grows with the +/// file (hashed, read, written twice in memory and read back): the desktop app runs it on a +/// thread of its own with progress and Cancel. Once written, the layer holds its rows as the file +/// reads them (the same, but for what a Shapefile can't tell apart: an empty text and no value), +/// and its record is of the files written and those rows, so it reads `inSync`; the project then +/// has a change to save. +pub(crate) fn publish_work(s: &Session, p: &Value) -> Result { + let id = s.layer_id(p)?; + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (synced, path) = record(l)?; + if !path.to_ascii_lowercase().ends_with(".shp") { + return Err(format!("{}: only a layer read from a Shapefile can be published to its file so far; {path} isn't one", l.name)); + } + #[cfg(target_arch = "wasm32")] + { + let _ = synced; + Err("the web app can't write files on this computer".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use super::exports::{Then, Wrote}; + let (synced, path, dataset, instance, name) = + (synced.clone(), path.to_string(), l.dataset.clone(), l.instance_id(), l.name.clone()); + Ok(Box::new(move || { + let rows = rustgis_data::sync::rows_digest(&dataset).map_err(|e| e.to_string())?; + let features = dataset.features.len(); + if rows == synced.rows { + let changed = synced.files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?; + if !changed.is_empty() { + return Err(format!( + "{path}: {} changed since {name} was read, and {name} has no edits to write: add the file again to read it as it is", + changed.join(", ") + )); + } + return Ok(Wrote { + answer: serde_json::json!({"layer": id, "path": path, "published": false, "features": features}), + notes: vec![format!("{name} has no edits its file lacks, so {path} wasn't written")], + written_path: None, + then: None, + owner: None, + }); + } + let published = rustgis_data::publish::shapefile(&dataset, std::path::Path::new(&path), &synced).map_err(|e| e.to_string())?; + // The layer as the file now reads it, its features keeping their ids: what the layer + // holds once published, and its record (the file reads so next time too). + let mut held = dataset.clone(); + let ids: Vec = dataset.features.iter().map(|f| f.id).collect(); + held.features = published + .read + .features + .iter() + .zip(&ids) + .map(|(f, id)| rustgis_data::Feature { id: *id, ..f.clone() }) + .collect::>() + .into(); + held.fields = published.read.fields.clone(); + for field in &mut held.fields { + if let Some(was) = dataset.fields.iter().find(|was| was.name == field.name) { + field.alias = was.alias.clone(); + } + } + held.crs = published.read.crs; + // Already hashed before the files were placed: Cancel must not leave a successful + // write without its sync record. Feature ids do not contribute to the digest. + let read = published.rows; + // Unchanged when the file reads as the layer is. + let held = (read != rows).then_some(held); + let new = rustgis_data::sync::Synced { files: published.files, rows: read }; + let (said, written_path) = (path.clone(), Some(path.clone())); + let then: Then = Box::new(move |s: &mut Session, _: &mut Value| { + s.remember_source_write(instance, said.clone(), new.clone()); + // The record follows the file: the layer read from it takes the record of what + // was written, unless it was read again or its record changed meanwhile. + let live = s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).find(|l| l.instance_id() == instance); + match live { + Some(l) if l.synced.as_ref() == Some(&synced) && l.dataset.source.as_deref() == Some(said.as_str()) => { + l.synced = Some(new); + // Not edited meanwhile: it takes what the file holds (an empty text as no + // value, whole numbers as integers). Edited, it keeps its edits, which + // the file then lacks. + if let Some(held) = held + && l.dataset.features.shares(&dataset.features) + && l.dataset.fields == dataset.fields + && l.dataset.crs == dataset.crs + { + l.dataset.features = held.features; + l.dataset.fields = held.fields; + l.dataset.crs = held.crs; + l.touch_geometry(); + s.revision = s.revision.wrapping_add(1); + } + s.mark_unsaved(); + } + _ => s.warn(format!("{said} was written, but {name} changed while it was: add the file again to compare them")), + } + s.notify(format!("Published {name} to {said}"), false); + Ok(()) + }); + Ok(Wrote { + answer: serde_json::json!({"layer": id, "path": path, "published": true, "features": published.features}), + notes: published.notes, + written_path, + then: Some(then), + owner: None, + }) + })) + } +} + /// The key a check of layer `id`'s files made ahead is taken by (see [`super::reads`]). #[cfg(not(target_arch = "wasm32"))] fn ahead_key(id: u32) -> String { @@ -74,20 +211,7 @@ pub(crate) fn sync_ahead(_: &Session, _: &Value) -> Option CmdResult { let id = s.layer_id(p)?; let l = s.map().layer(id).ok_or("the layer no longer exists")?; - let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { - if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { - return Err(format!( - "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", - l.name - )); - } - return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); - }; - // A project from elsewhere names the path: only a local file of a kind a record is made for - // is opened, never a network share (opening one can send the user's login to its host). - if !local_file(path) { - return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); - } + let (synced, path) = record(l)?; #[cfg(target_arch = "wasm32")] { let _ = (synced, path); @@ -95,7 +219,7 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - let (was, files, path) = (synced.rows, synced.files.clone(), path.clone()); + let (was, files, path) = (synced.rows, synced.files.clone(), path.to_string()); let ahead = super::reads::take::(s, &ahead_key(id)); let l = s.map().layer(id).ok_or("the layer no longer exists")?; let (rows, changed) = match ahead { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index af28904915..8b1c6fd86f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -86,7 +86,8 @@ pub struct CommandSpec { pub shortcut: &'static str, /// Parameter description for scripts and agents. pub params: &'static str, - /// Whether it changes the project (and so takes an undo snapshot). + /// Whether it takes an automatic project-edit transaction (undo snapshot and dirty marking). + /// Changes committed without that transaction declare `Effect::Edits` explicitly. pub mutates: bool, /// `None` = enabled; `Some(reason)` = disabled. pub enabled: fn(&Session) -> Option<&'static str>, @@ -111,6 +112,12 @@ impl CommandSpec { param_keys(self.params) } + /// Whether it can change the project, including external commits that update metadata + /// without an automatic undo snapshot. Logging and command discovery use this contract. + pub const fn changes_project(&self) -> bool { + self.mutates || self.effects.contains(policy::Effect::Edits) + } + /// Whether the parameter description includes a required top-level input. pub fn needs_input(&self) -> bool { needs_input(self.params) @@ -163,7 +170,8 @@ impl CommandSpec { self.params = p; self } - /// Doesn't change the project (no undo snapshot, not dirtying). + /// Suppresses automatic undo snapshots and dirty marking. Usually read-only; a command + /// committing external changes can update metadata itself and declares `Effect::Edits`. pub const fn view(mut self) -> Self { self.mutates = false; self @@ -818,6 +826,9 @@ pub struct Session { /// How many of the commands running now restore the project if they fail: a replacement is /// put aside only under one of them, as only then can it be rolled back. restorable: u32, + /// File writes committed inside the outer command, retained through layer removal or a + /// project replacement so its rollback can restore the current source-file record. + published_syncs: Vec<(u64, String, rustgis_data::sync::Synced)>, /// Who runs commands now: `None` is the user; the MCP relay names an attached agent around /// its calls. Each undo step remembers who made it ([`Session::undo_actor`]). pub actor: Option, @@ -1197,6 +1208,7 @@ impl Session { depth: 0, diagnostic_source: None, restorable: 0, + published_syncs: Vec::new(), actor: None, back: Vec::new(), forward: Vec::new(), @@ -1747,6 +1759,10 @@ impl Session { // session's own flag says so, not the answer, which a plugin's command could forge. (Ok(_), Some(_)) if std::mem::take(&mut self.request_only) => {} (Ok(_), Some(mut snap)) => { + // A successful script may publish and then remove the layer. Its undo + // snapshot still needs the committed file record, even without a live layer + // from which undo could carry that record forward. + self.keep_published_syncs(&mut snap); fresh_source_ids(&snap, &mut self.project); forget_label_state_of_removed_features(&snap, &mut self.project); // An edit may change the dates the time slider's window filters on: bound layers @@ -1800,7 +1816,8 @@ impl Session { let failed_history = if self.opened == opened { std::mem::take(&mut self.project.gp_history) } else { Vec::new() }; // The project as it was before the command, so the layout open then, whatever // the command opened on the way (a task's steps). - self.project = snap; + let committed = std::mem::replace(&mut self.project, snap); + let sync_changed = self.keep_synced(&committed); (self.trusted_tasks, self.trusted_models, self.trusted_notebooks) = trust; self.active_layout = open_layout; // Replaced during the command: what it held beside itself (its undo history, its @@ -1833,7 +1850,11 @@ impl Session { self.push_gp_record(record); recorded_failure = true; } - if recorded_failure { + if sync_changed { + self.mark_unsaved(); + self.warn("Source-file records were kept: rolling back a command does not undo its file writes"); + } + if recorded_failure || sync_changed { if self.depth == 0 && self.project.gp_history.len() > 500 { let excess = self.project.gp_history.len() - 500; self.project.gp_history.drain(..excess); @@ -1858,6 +1879,7 @@ impl Session { // The command, and those it ran, are done: a replaced project stays replaced. if self.depth == 0 { self.put_aside = None; + self.published_syncs.clear(); } self.fix_active_layer(); self.revision = self.revision.wrapping_add(1); @@ -2494,6 +2516,7 @@ impl Session { self.keep_time_position(&cur); self.keep_tile_pick(&cur); self.keep_headers_said(&cur); + self.keep_synced(&cur); self.redo.push((cur, was, label, actor, std::mem::replace(&mut self.state, state), files, old_trust)); self.touch_all_geometry(); true @@ -2519,6 +2542,7 @@ impl Session { self.keep_time_position(&cur); self.keep_tile_pick(&cur); self.keep_headers_said(&cur); + self.keep_synced(&cur); self.undo.push((cur, was, label, actor, std::mem::replace(&mut self.state, state), files, old_trust)); self.touch_all_geometry(); true @@ -2537,6 +2561,51 @@ impl Session { cmd::zarr::follow_time(self); } + /// Remembers a physical source write only while a command can still roll its project back. + /// The outer guard clears these records on success or failure; they never go in a project. + #[cfg(not(target_arch = "wasm32"))] + fn remember_source_write(&mut self, instance: u64, path: String, synced: rustgis_data::sync::Synced) { + if self.restorable > 0 { + self.published_syncs.retain(|(i, p, _)| *i != instance || *p != path); + self.published_syncs.push((instance, path, synced)); + } + } + + /// Gives a command's undo snapshot the records of files it physically wrote, including + /// layers the command removed after publishing. The records belong to these instances only. + fn keep_published_syncs(&self, project: &mut Project) { + for l in project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()) { + if let Some((_, _, synced)) = + self.published_syncs.iter().rev().find(|(i, p, _)| *i == l.instance_id() && l.dataset.source.as_deref() == Some(p.as_str())) + { + l.synced = Some(synced.clone()); + } + } + } + + /// After undo, redo or failed-command rollback swaps the project: each layer keeps its + /// file record (`synced`), as restoring the project doesn't change the file. A published + /// layer keeps the record of what was written, so restored edits compare with that file. + /// Returns whether the restored project took a different record and needs saving. + fn keep_synced(&mut self, before: &Project) -> bool { + let mut changed = false; + for l in self.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()) { + let old = before.maps.iter().flat_map(|m| &m.layers).find(|o| o.instance_id() == l.instance_id()); + let synced = old.filter(|o| o.dataset.source == l.dataset.source).map(|o| o.synced.clone()).or_else(|| { + self.published_syncs + .iter() + .rev() + .find(|(i, p, _)| *i == l.instance_id() && l.dataset.source.as_deref() == Some(p.as_str())) + .map(|(_, _, synced)| Some(synced.clone())) + }); + if let Some(synced) = synced { + changed |= l.synced != synced; + l.synced = synced; + } + } + changed + } + /// After undo or redo swaps the project: the tile layer picked in the Layers pane stays the one /// picked (picking isn't an undo step), found again on the map of its name by its address, /// kind, role and name (without the name when only one layer has the rest: a rename undone); diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index f81e00644f..88ec45b8e4 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -35,8 +35,8 @@ pub(crate) fn now_ms() -> u64 { } /// Whether command `id`, run by itself with `params`, goes into the log: one that changes the -/// project (all but those built with `.view()`; a plugin's when its manifest asks for the Edit -/// permission), undo and redo, a save, the log's own clearing and switching, and a command whose +/// project (automatic edits or an explicit `Edits` effect; a plugin's when its manifest asks for +/// the Edit permission), undo and redo, a save, the log's own clearing and switching, and a command whose /// parameters name a file, folder, address or connection (an export), as the plugin permission /// gate reads them. Reading and looking around (listing, describing, moving the view) is left /// out: agents, scripts and plugins call those hundreds of times, and they would push the changes @@ -60,7 +60,7 @@ pub(crate) fn is_logged(s: &Session, id: &str, params: &Value) -> bool { rustgis_plugins::command(id).is_some_and(|c| c.mutates()) } else { match s.registry.get(id) { - Some(spec) => spec.mutates, + Some(spec) => spec.changes_project(), None => return false, } }; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 5a7c08462d..0cd7c5381f 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -44271,6 +44271,10 @@ fn long_commands_stop_when_cancelled_while_they_run() { s.run("layer.addDataText", &json!({"fileName": "training.geojson", "text": training.to_string()})).unwrap(); s.run("layout.new", &json!({"page": "a4", "orientation": "landscape", "template": "blank"})).unwrap(); s.run("layout.addFrame", &json!({"x": 20, "y": 20, "w": 150, "h": 150})).unwrap(); + // A layer read from a Shapefile, edited, to publish. + s.run("share.exportLayer", &json!({"layer": "Cities", "path": out("towns.shp")})).unwrap(); + s.run("layer.addData", &json!({"path": out("towns.shp"), "name": "towns"})).unwrap(); + s.run("edit.setValue", &json!({"layer": "towns", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); // What didn't stop, and how often it looked. let mut kept_going = Vec::new(); @@ -44327,6 +44331,7 @@ fn long_commands_stop_when_cancelled_while_they_run() { ("share.exportMap", json!({"path": out("map.png"), "width": 320, "height": 200})), ("layout.export", json!({"path": out("layout.pdf"), "dpi": 72})), ("report.export", json!({"path": out("report.pdf"), "dpi": 72})), + ("source.publish", json!({"layer": "towns"})), ]; s.run("report.new", &json!({"layer": "Cities"})).unwrap(); let registered = crate::cmd::exports::IN_BACKGROUND.iter().map(|(c, ..)| *c).collect::>(); @@ -44984,6 +44989,168 @@ fn change_detection_leaves_out_see_through_cells() { ); } +/// A layer read from a Shapefile publishes its edits to the file (#1385): the file then holds +/// them, the layer reads `inSync`, and undoing an edit made before shows it as an edit again, +/// as the file keeps it. A layer without edits writes nothing; a file changed since it was read, +/// or one read from a GeoJSON file, is refused with nothing written. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_layer_publishes_its_edits_to_its_shapefile() { + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + impl Dir { + fn path(&self) -> &std::path::Path { + &self.0 + } + } + let dir = Dir(std::env::temp_dir().join(format!("rustgis-engine-publish-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(dir.path()); + std::fs::create_dir_all(dir.path()).unwrap(); + let path = dir.path().join("towns.shp"); + let mut s = world(); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": path})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path, "name": "towns"})).unwrap()["layer"].as_u64().unwrap(); + let state = |s: &mut Session| s.run("source.syncState", &json!({"layer": id})).unwrap()["state"].clone(); + let dbf = || std::fs::read(dir.path().join("towns.dbf")).unwrap(); + let before = dbf(); + let out = s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(out["published"], false); + assert_eq!(dbf(), before, "nothing to write"); + s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + s.mark_saved(); + let undo_before_publish = s.undo_depth(); + let out = s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(s.undo_depth(), undo_before_publish, "publishing cannot be undone separately from the edits"); + assert!(s.log.entries().any(|e| e.command == "source.publish" && e.ok)); + assert_eq!(out["published"], true); + assert!(s.dirty(), "the layer's new record is a change to save"); + let (read, _) = rustgis_data::io::read_path(&path).unwrap(); + let name = read.fields.iter().position(|f| f.name == "name").unwrap(); + assert_eq!(read.features.iter().next().unwrap().attrs[name], rustgis_data::Value::Text("Renamed".into())); + assert_eq!(state(&mut s), "inSync"); + // Undo doesn't change the file: the layer now lacks an edit the file has. + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(state(&mut s), "edited"); + s.run("edit.redo", &Value::Null).unwrap(); + assert_eq!(state(&mut s), "inSync"); + // An empty text, which the file holds as no value: the layer takes it as the file reads it, + // and is in step with it. + { + let l = s.map_mut().layer_mut(id as u32).unwrap(); + l.dataset.fields.iter_mut().find(|f| f.name == "name").unwrap().alias = Some("Town name".into()); + } + s.run("edit.setValue", &json!({"layer": id, "row": 2, "field": "name", "value": ""})).unwrap(); + s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(state(&mut s), "inSync"); + let l = s.map().layer(id as u32).unwrap(); + let name = l.dataset.fields.iter().position(|f| f.name == "name").unwrap(); + assert!(l.dataset.features.get(2).unwrap().attrs[name].is_null()); + assert_eq!(l.dataset.fields[name].alias.as_deref(), Some("Town name")); + // Cancel arriving after the final .shp rename cannot turn a committed publish into an + // error or leave its old sync record. The probe observes the real file, not an internal hook. + struct CancelAfterPlacement { + path: std::path::PathBuf, + before: Vec, + } + impl rustgis_data::progress::Probe for CancelAfterPlacement { + /// Changes only after the last .shp rename, the publication commit point. + fn cancelled(&self) -> bool { + std::fs::read(&self.path).is_ok_and(|bytes| bytes != self.before) + } + fn begin(&self, _: u64) {} + fn advance(&self, _: u64) {} + } + s.run("edit.setGeometry", &json!({"layer": id, "row": 0, "geometry": {"type": "Point", "coordinates": [1, 2]}})).unwrap(); + { + let probe = std::sync::Arc::new(CancelAfterPlacement { path: path.clone(), before: std::fs::read(&path).unwrap() }); + let _installed = rustgis_data::progress::install(Some(probe.clone())); + // Run the real worker directly: synchronous Session dispatch installs its own probe. + let work = crate::cmd::publish::publish_work(&s, &json!({"layer": id})).unwrap(); + let mut wrote = work().unwrap(); + assert_eq!(wrote.answer["published"], true); + wrote.then.take().unwrap()(&mut s, &mut wrote.answer).unwrap(); + assert!(rustgis_data::progress::Probe::cancelled(probe.as_ref()), "the probe saw the committed shape"); + } + assert_eq!(state(&mut s), "inSync"); + // Define Projection makes the geographic CRS explicit. Publishing normalizes it as + // the reader does, so the live layer and its new sync point agree after completion. + s.run("layer.defineProjection", &json!({"layer": id, "crs": "EPSG:4326"})).unwrap(); + assert_eq!(state(&mut s), "edited"); + assert_eq!(s.run("source.publish", &json!({"layer": id})).unwrap()["published"], true); + assert_eq!(state(&mut s), "inSync"); + assert_eq!(s.run("source.publish", &json!({"layer": id})).unwrap()["published"], false); + // A later script failure rolls project edits back, but not its committed file or baseline, + // including when the script removed the published layer before failing. + for (n, (edited_inside, removed)) in [(false, false), (true, false), (true, true)].into_iter().enumerate() { + if !edited_inside { + s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Committed"})).unwrap(); + } + s.mark_saved(); + let undo = s.undo_depth(); + let mut lines = Vec::new(); + let value = if edited_inside { format!("In script {n}") } else { "Committed".into() }; + if edited_inside { + lines.push(format!("edit.setValue {}", json!({"layer":id,"row":1,"field":"name","value":value}))); + } + lines.push(format!("source.publish {}", json!({"layer":id}))); + if removed { + lines.push(format!("layer.remove {}", json!({"layer":id}))); + } + lines.push("layer.remove {\"layer\":\"missing-layer\"}".into()); + assert!(s.run("script.run", &json!({"script":lines.join("\n")})).is_err()); + let checked = s.run("source.syncState", &json!({"layer":id})).unwrap(); + assert_eq!(checked["changed"], json!([]), "the committed file retains its current baseline"); + assert_eq!(checked["state"], if edited_inside { "edited" } else { "inSync" }); + assert!(s.dirty(), "the committed baseline still needs saving after rollback"); + assert_eq!(s.undo_depth(), undo, "a failed script adds no undo step"); + let (back, _) = rustgis_data::io::read_path(&path).unwrap(); + assert_eq!(back.features.get(1).unwrap().attrs[name], rustgis_data::Value::Text(value)); + } + s.run("source.publish", &json!({"layer":id})).unwrap(); + assert_eq!(state(&mut s), "inSync"); + // A successful script can remove a layer after publication too. Undo restores its old + // rows against the committed file baseline; redo and a second undo keep that baseline. + let script = [ + format!("edit.setValue {}", json!({"layer":id,"row":1,"field":"name","value":"Published and removed"})), + format!("source.publish {}", json!({"layer":id})), + format!("layer.remove {}", json!({"layer":id})), + ] + .join("\n"); + s.run("script.run", &json!({"script":script})).unwrap(); + assert!(s.map().layer(id as u32).is_none()); + for again in [false, true] { + if again { + s.run("edit.redo", &Value::Null).unwrap(); + assert!(s.map().layer(id as u32).is_none()); + } + s.run("edit.undo", &Value::Null).unwrap(); + let checked = s.run("source.syncState", &json!({"layer":id})).unwrap(); + assert_eq!(checked["changed"], json!([])); + assert_eq!(checked["state"], "edited"); + let (back, _) = rustgis_data::io::read_path(&path).unwrap(); + assert_eq!(back.features.get(1).unwrap().attrs[name], rustgis_data::Value::Text("Published and removed".into())); + } + // Changed since: refused, the file left as it was. + s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Mine"})).unwrap(); + let mut theirs = dbf(); + *theirs.last_mut().unwrap() ^= 1; + std::fs::write(dir.path().join("towns.dbf"), &theirs).unwrap(); + let e = s.run("source.publish", &json!({"layer": id})).unwrap_err(); + assert!(e.contains("towns.dbf changed since the layer was read") && e.contains("nothing was written"), "{e}"); + assert_eq!(dbf(), theirs); + // GeoJSON isn't published yet. + let geojson = dir.path().join("towns.geojson"); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": geojson})).unwrap(); + s.run("layer.addData", &json!({"path": geojson, "name": "towns.geojson"})).unwrap(); + let e = s.run("source.publish", &json!({"layer": "towns.geojson"})).unwrap_err(); + assert!(e.contains("only a layer read from a Shapefile"), "{e}"); +} + /// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read /// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed /// since, and neither once an edit is undone. The record is saved with the project. diff --git a/crates/engine/src/tests/permission_policy.rs b/crates/engine/src/tests/permission_policy.rs index e0cabd1f75..48c6d12423 100644 --- a/crates/engine/src/tests/permission_policy.rs +++ b/crates/engine/src/tests/permission_policy.rs @@ -643,6 +643,31 @@ fn permission_policy_export_worker_keeps_its_initiating_owner() { let a = export_start(&mut s, "share.exportLayer", ¶ms("owner-a.geojson")).unwrap(); let b = export_start(&mut s, "share.exportLayer", ¶ms("owner-b.geojson")).unwrap(); assert!(export_land(&mut s, b.started(), a.run(&Control::new())).unwrap_err().contains("another run")); + // A committed publish still reports its format changes when its owner is revoked before delivery. + let shp = path("published.shp"); + s.run("share.exportLayer", &json!({"layer":"Cities", "path":shp})).unwrap(); + let id = s.run("layer.addData", &json!({"path":shp, "name":"published"})).unwrap()["layer"].as_u64().unwrap() as u32; + let long_name = "x".repeat(200); + s.run("edit.setValue", &json!({"layer":id,"row":0,"field":"name","value":long_name})).unwrap(); + let synced = s.map().layer(id).unwrap().synced.clone(); + live.0.store(true, Ordering::SeqCst); + let scope = Scope::agent(AgentAccess::Allow, false, "publish-owner", None, live.clone()); + let run = s.with_scope(scope, |s| export_start(s, "source.publish", &json!({"layer":id}))).unwrap().unwrap(); + let started = run.started(); + let wrote = run.run(&Control::new()); + let notes = wrote.as_ref().unwrap().notes.clone(); + assert!(!notes.is_empty(), "the widened text column has a notice"); + live.0.store(false, Ordering::SeqCst); + let mark = s.notice_mark(); + let error = export_land(&mut s, started, wrote).unwrap_err(); + assert!(error.contains("Published ") && error.contains("record wasn't updated"), "{error}"); + for note in notes { + assert!(s.notices_since(mark).iter().any(|n| n.text.contains(¬e)), "missing {note}: {error}"); + } + assert_eq!(s.map().layer(id).unwrap().synced, synced, "the revoked finalizer must not update the layer"); + let (disk, _) = rustgis_data::io::read_path(&shp).unwrap(); + let name = disk.field_index("name").unwrap(); + assert_eq!(disk.features.first().unwrap().attrs[name], rustgis_data::Value::Text(long_name)); std::fs::remove_dir_all(dir).unwrap(); } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 27a904d0b4..e44005e258 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2516,6 +2516,15 @@ impl RustGisApp { if id == "source.syncState" { return readjob::run_then(self, "source.syncState", params, Some(sync_state_said)); } + // Publishing hashes, reads and writes the whole file: on a thread of its own, with + // progress and Cancel, as an export is. + if id == "source.publish" { + let params = if params.is_null() { json!({}) } else { params }; + // Refused before it starts (another export running, no record): said, as Command + // Search shows no answer. + let mark = self.session.notice_mark(); + return exportjob::start(self, "source.publish", params).inspect_err(|e| self.command_failed(id, e, mark)); + } // Converting the labels of a layer read in place reads all of its source first, on a // thread of its own, whoever asks (a click, the assistant). if id == "labels.convert" diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 18be8f69f0..7288e47955 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -397,6 +397,75 @@ can't check it. On the desktop, make the layer active and choose **Source File State** in Command Search; what it found shows in the status bar. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. +### Publish to source + +`source.publish` writes a layer's edits back over the Shapefile it was read from, so the file +that other people and programs read holds them. A Shapefile can't change one row in place, so +the whole file is written again, and RustGIS first makes sure that only the edits change. It +refuses, with nothing written, when: + +- the file changed since the layer was read (`source.syncState` would say `sourceChanged` or + `both`): writing would overwrite that change; +- the layer was read otherwise than the file reads now (by an older RustGIS), so rows that + weren't edited could change; +- the file, read and written again, wouldn't be the same file: shapes with Z or M values, records + without a shape, multipatches, records marked deleted, text with spaces before it, numbers with + more digits than RustGIS holds exactly, columns of a type RustGIS doesn't write (memo, binary), + text in a code page RustGIS can't write. The message names what would change and where; +- the file can't hold the layer as it is: a feature without a shape, a field name longer than 10 + bytes, text in characters the file's code page lacks. + +The file keeps its code page (its `.cpg` and the language driver in its `.dbf`), each column's +width, decimals and type where the values still fit, and its `.prj`. A column that can't hold +an edited value gets one that can, and a notice names it with its old and new form; a field +deleted or renamed in the layer leaves its column out of the file, and a notice names it too. A layer read in another +coordinate system than its `.prj` names, including one assigned with Define Projection, gets a +`.prj` for that system, said too. A shape that wasn't edited is +written with the coordinates the file holds, so a projected file's untouched shapes stay as they +were to the bit. An edited shape in a projected system is converted from longitude/latitude, as +RustGIS holds it, and read back: publishing refuses one that would move by more than one part in +10¹¹ of its coordinates (a tenth of a millimetre at ten thousand kilometres). A notice +names the file, counts shapes with projection round-off and identifies the first feature and +the largest coordinate difference in degrees; exact round trips give no such notice. + +Publishing refuses symbolic links in any source part: add the actual source files instead. +This keeps each link and its target unchanged. + +The new parts are written beside the old ones, with the old parts' file permissions, checked, +and then renamed into place, the `.shp` last. An OS lock prevents two RustGIS processes from +publishing the same source at once; a competing publish refuses immediately. The empty +`.filename.shp.rustgis-lock` sidecar stays beside the source for reuse; the OS releases its lock +when the publisher finishes or exits. On Unix, this empty file permits read/write access across +accounts so users sharing the source directory can reopen it for locking, including on NFS. +Other programs do not follow this locking protocol: changes they make after RustGIS's final +source check can be overwritten by the publish. Stop other writers before publishing. +If one part fails to land, the old parts are put back; +should putting one back fail too, the message names the temporary file that holds it, beside the +Shapefile, so it can be renamed back by hand. Sidecars RustGIS doesn't write, such as `.shp.xml` +metadata, are left alone. The spatial indexes that would no longer match (`.qix`, `.sbn`, `.sbx`) +are removed where they can be; one that can't be (held open by another program) is left, and +should be deleted before that program uses the file. + +This rollback needs RustGIS to keep running. Replacing several Shapefile parts is not atomic: +a process crash or power loss between renames can leave a mixed or incomplete set. Keep an +independent backup before publishing. After such an interruption, preserve the whole directory, +including hidden `....tmp` files (for example, +`.parcels.dbf.1234.5.tmp`); these can contain both staged new parts and displaced old parts. +Restore a complete, consistent Shapefile from your backup before opening or publishing it again. +Do not restore an arbitrary temporary file or mix parts from different versions. Temporary +files may help with manual recovery, but are not a crash-safe backup. + +Once published, the layer holds its rows as the file reads them, which differs only where a +Shapefile can't tell values apart (an empty text reads as no value, a column of whole numbers as +integers, date-time fields as ISO 8601 text). A double field becoming integer or a date-time field +becoming text produces a notice. The layer reads `inSync`; the project has a change to save. Undo doesn't change the file: +undoing an edit made before publishing shows the layer as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't +be published yet. + +On the desktop, make the layer active and choose **Publish to Source** in Command Search; it runs +in the background, with progress and Cancel in the status bar. CLI and Python: `source.publish` +with `{"layer":"parcels"}`. + ## Keyboard shortcuts | Keys | Action |