From 401709db2c778c47abf02f9f82879f5b1c3d1c56 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 12:20:21 -0600 Subject: [PATCH 01/15] feat(data): publish a layer's edits back to its Shapefile (#1385) source.publish writes an in-memory layer read from a Shapefile back over that file, after proving nothing but the edits changes: the file is as it was read and reads as the layer was read, the file read and written again is byte-for-byte the same table and the same shapes, and the layer written reads back as the layer. The new parts are staged beside the old, checked, and swapped in with the .shp last, rolling back on failure. The rewrite keeps the file's code page and language driver, column widths, decimals and F types where values fit, its .prj and .cpg, and the exact coordinates of every unedited shape. It runs as a background export with progress and Cancel; undo and redo keep each layer's sync record. Also reads dBase F columns as f64: dbase parsed them into an f32. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/dbf.rs | 90 ++++ crates/data/src/io.rs | 345 ++++++++++---- crates/data/src/lib.rs | 2 + crates/data/src/publish.rs | 758 +++++++++++++++++++++++++++++++ crates/data/src/sync.rs | 22 + crates/engine/src/cmd/exports.rs | 1 + crates/engine/src/cmd/publish.rs | 113 ++++- crates/engine/src/lib.rs | 14 + crates/engine/src/tests.rs | 67 +++ crates/ui-egui/src/lib.rs | 6 + docs/user-guide/index.md | 35 ++ 11 files changed, 1336 insertions(+), 117 deletions(-) create mode 100644 crates/data/src/publish.rs diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index 95c8356eb1..034d81041a 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -190,6 +190,82 @@ pub(crate) fn dates_as_text(dbf: &mut [u8]) -> Vec { dates } +/// Marks a `.dbf`'s float (`F`) columns as numeric (`N`) columns in its header: both hold a +/// number as decimal text, and `dbase` reads an `F` column's into an `f32`, which keeps only +/// about seven digits (`123456.12345678` would read `123456.125`). Read as `N`, they keep the +/// `f64` the text gives. +pub(crate) fn floats_as_numeric(dbf: &mut [u8]) { + let header_len = usize::from(u16::from_le_bytes([dbf.get(8).copied().unwrap_or(0), dbf.get(9).copied().unwrap_or(0)])); + for start in (32..header_len.min(dbf.len())).step_by(32) { + if dbf.get(start).is_none_or(|b| *b == 0x0D) { + break; + } + if let Some(ty) = dbf.get_mut(start + 11) + && ty.eq_ignore_ascii_case(&b'F') + { + *ty = b'N'; + } + } +} + +/// A column of a `.dbf` as its header describes it: its name's bytes (up to the first NUL), +/// type letter, length in bytes and decimals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Column { + pub name: Vec, + pub ty: u8, + pub len: u8, + pub decimals: u8, +} + +#[cfg(not(target_arch = "wasm32"))] +/// A `.dbf` read as its bytes, without decoding a value: its columns, and each record that isn't +/// marked deleted, as the bytes of each of its columns. `None` when the header doesn't describe +/// records that fit the file. +pub(crate) struct RawTable<'a> { + pub columns: Vec, + pub records: Vec>, +} + +#[cfg(not(target_arch = "wasm32"))] +/// Reads `dbf` as a [`RawTable`]: what a rewrite is compared with, byte for byte, where a +/// decoded value would hide a difference (text with leading spaces, which a read trims; a code +/// page's bytes). +pub(crate) fn raw_table(dbf: &[u8]) -> Option> { + let le16 = |at: usize| Some(usize::from(u16::from_le_bytes([*dbf.get(at)?, *dbf.get(at + 1)?]))); + let count = usize::try_from(u32::from_le_bytes(dbf.get(4..8)?.try_into().ok()?)).ok()?; + let (header_len, record_len) = (le16(8)?, le16(10)?); + let mut columns = Vec::new(); + for start in (32..header_len.min(dbf.len())).step_by(32) { + let d = dbf.get(start..start + 32).filter(|d| d.first() != Some(&0x0D)); + let Some(d) = d else { break }; + let name = d.get(..11)?; + let name = name.get(..name.iter().position(|b| *b == 0).unwrap_or(11))?.to_vec(); + columns.push(Column { name, ty: *d.get(11)?, len: *d.get(16)?, decimals: *d.get(17)? }); + } + let width: usize = 1 + columns.iter().map(|c| usize::from(c.len)).sum::(); + if width != record_len || record_len == 0 { + return None; + } + let body = dbf.get(header_len..)?; + let mut records = Vec::new(); + for record in body.chunks_exact(record_len).take(count) { + // A record marked deleted (`*`) isn't read, so it isn't written either. + if record.first() == Some(&b'*') { + continue; + } + let mut at = 1; + let mut values = Vec::with_capacity(columns.len()); + for c in &columns { + let end = at + usize::from(c.len); + values.push(record.get(at..end)?); + at = end; + } + records.push(values); + } + Some(RawTable { columns, records }) +} + /// A date (`D`) field's value, `YYYYMMDD`: `Ok(None)` when it's blank or zeros, as writers leave /// a date empty, and `Err(())` when it isn't a date (`20260230`, `2026-3-5`). pub(crate) fn date(field: &[u8]) -> std::result::Result, ()> { @@ -267,6 +343,20 @@ mod tests { assert!(dates_as_text(&mut [0u8; 10]).is_empty()); } + /// Float columns are marked as numeric columns; others, and what follows the header, are kept. + #[test] + fn float_columns_read_as_numeric() { + let mut dbf = vec![0u8; 32 + 3 * 32 + 2]; + dbf[8] = (32 + 3 * 32 + 1) as u8; + for (k, ty) in b"FfC".iter().copied().enumerate() { + dbf[32 + k * 32 + 11] = ty; + } + dbf[32 + 3 * 32] = 0x0D; + dbf[32 + 3 * 32 + 1] = b'F'; + floats_as_numeric(&mut dbf); + assert_eq!([dbf[43], dbf[75], dbf[107], dbf[129]], [b'N', b'N', b'C', b'F']); + } + /// A header date that isn't one gets a placeholder; a real one, and a header cut too short /// to have one, are left as they are. #[test] diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 19a69cf320..5cb86ab482 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -603,78 +603,8 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R "shp" => { crate::progress::begin(ds.features.len()); let (parts, left_out) = shapefile_parts_in(ds, crs)?; - // Every part is written to a temporary file first and renamed into place only once all - // are written, the .shp last, so a failed write leaves the old Shapefile whole. - let mut written: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); - let staged = parts.iter().try_for_each(|(e, bytes)| { - // The .shp keeps the name given (`Roads.SHP` stays so); the others take its stem. - let target = if *e == "shp" { path.to_path_buf() } else { path.with_extension(e) }; - let tmp = temp_beside(&target)?; - let result = write_new(&tmp, bytes); - if result.is_ok() { - written.push((tmp, target)); - } - result - }); - // A cancel before the parts land leaves the old Shapefile as it was. - if let Err(e) = staged.map_err(DataError::from).and_then(|()| crate::progress::check()) { - for (tmp, _) in &written { - let _ = std::fs::remove_file(tmp); - } - return Err(e); - } - written.sort_by_key(|(_, target)| target == path); - // The old parts are moved aside first and put back if any new part fails to land, so - // the old Shapefile stays a matching set. - let mut backups: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); - let mut placed: Vec = Vec::new(); - let mut result = Ok(()); - for (tmp, target) in &written { - // A cancel before the last part lands rolls back like a failed rename. - if crate::progress::check().is_err() { - result = Err(std::io::Error::other(DataError::Cancelled)); - break; - } - if target.exists() { - // Any failure here rolls back like a failed rename. - let bak = match temp_beside(target) { - Ok(b) => b, - Err(e) => { - result = Err(e); - break; - } - }; - if let Err(e) = std::fs::rename(target, &bak) { - result = Err(e); - break; - } - backups.push((bak, target.clone())); - } - if let Err(e) = std::fs::rename(tmp, target) { - result = Err(e); - break; - } - placed.push(target.clone()); - } - if let Err(e) = result { - for p in &placed { - let _ = std::fs::remove_file(p); - } - for (bak, target) in &backups { - let _ = std::fs::rename(bak, target); - } - for (tmp, _) in &written { - let _ = std::fs::remove_file(tmp); - } - return Err(e.into()); - } - for (bak, _) in &backups { - let _ = std::fs::remove_file(bak); - } - // A spatial index left from an older file of the same name would no longer match. - for stale in ["qix", "sbn", "sbx"] { - let _ = std::fs::remove_file(path.with_extension(stale)); - } + let parts: Vec<(std::path::PathBuf, Vec)> = parts.into_iter().map(|(e, bytes)| (shapefile_part(path, e), bytes)).collect(); + place_shapefile(path, &stage_shapefile(&parts)?)?; Ok((left_out, None)) } _ if crs != Crs::LonLat && !PROJECTED_FORMATS.contains(&ext.as_str()) => { @@ -688,6 +618,106 @@ pub fn write_path_to(ds: &Dataset, path: &Path, crs: Crs, gpkg: &GpkgTable) -> R } } +/// The path of the part with extension `ext` of the Shapefile at `path`: the `.shp` keeps the +/// name given (`Roads.SHP` stays so); the others take its stem. +pub(crate) fn shapefile_part(path: &Path, ext: &str) -> std::path::PathBuf { + if ext == "shp" { path.to_path_buf() } else { path.with_extension(ext) } +} + +/// Writes each of a Shapefile's parts (its path and bytes) to a temporary file beside it, synced +/// to the disk; returns each temporary file with the part it becomes. Nothing is left behind +/// when one fails or the run is cancelled (see [`place_shapefile`]). +pub(crate) fn stage_shapefile(parts: &[(std::path::PathBuf, Vec)]) -> Result> { + let mut written: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); + let staged = parts.iter().try_for_each(|(target, bytes)| { + let tmp = temp_beside(target)?; + let result = write_new(&tmp, bytes); + if result.is_ok() { + written.push((tmp, target.clone())); + } + result + }); + // A cancel before the parts land leaves the old Shapefile as it was. + if let Err(e) = staged.map_err(DataError::from).and_then(|()| crate::progress::check()) { + unstage(&written); + return Err(e); + } + Ok(written) +} + +/// Removes the temporary files [`stage_shapefile`] wrote. +pub(crate) fn unstage(staged: &[(std::path::PathBuf, std::path::PathBuf)]) { + for (tmp, _) in staged { + let _ = std::fs::remove_file(tmp); + } +} + +/// Renames each staged part of the Shapefile at `path` ([`stage_shapefile`]) into place, the +/// `.shp` last, so a failed write leaves the old Shapefile whole: each old part is moved aside +/// first and put back if any new part fails to land, and a Cancel before the last part lands +/// rolls back the same way. Once all have landed, the old parts and the spatial indexes that +/// would no longer match (`.qix`, `.sbn`, `.sbx`) are removed. +pub(crate) fn place_shapefile(path: &Path, staged: &[(std::path::PathBuf, std::path::PathBuf)]) -> Result<()> { + let mut written: Vec<&(std::path::PathBuf, std::path::PathBuf)> = staged.iter().collect(); + written.sort_by_key(|(_, target)| target == path); + let mut backups: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); + let mut placed: Vec = Vec::new(); + let mut result = Ok(()); + for (tmp, target) in written { + // A cancel before the last part lands rolls back like a failed rename. + if crate::progress::check().is_err() { + result = Err(std::io::Error::other(DataError::Cancelled)); + break; + } + if target.exists() { + // Any failure here rolls back like a failed rename. + let bak = match temp_beside(target) { + Ok(b) => b, + Err(e) => { + result = Err(e); + break; + } + }; + if let Err(e) = std::fs::rename(target, &bak) { + result = Err(e); + break; + } + backups.push((bak, target.clone())); + } + if let Err(e) = std::fs::rename(tmp, target) { + result = Err(e); + break; + } + placed.push(target.clone()); + } + if let Err(e) = result { + for p in &placed { + let _ = std::fs::remove_file(p); + } + // An old part that can't be put back is named, so it isn't lost under its temporary name. + let mut kept = Vec::new(); + for (bak, target) in &backups { + if std::fs::rename(bak, target).is_err() { + kept.push(format!("{} is kept as {}", target.display(), bak.display())); + } + } + unstage(staged); + let e = DataError::from(e); + if kept.is_empty() { + return Err(e); + } + return Err(DataError::Format(format!("{e}; putting the old file back failed: {}", kept.join(", ")))); + } + for (bak, _) in &backups { + let _ = std::fs::remove_file(bak); + } + // A spatial index left from an older file of the same name would no longer match. + for stale in ["qix", "sbn", "sbx"] { + let _ = std::fs::remove_file(path.with_extension(stale)); + } + Ok(()) +} + /// A new temporary name beside `path`, unique per call, so concurrent writes to the same target /// never share one. pub(crate) fn temp_beside(path: &Path) -> std::io::Result { @@ -2320,17 +2350,17 @@ pub fn read_zip_in(file_name: &str, bytes: &[u8], crs: Option) -> Result, - shx: Option>, - dbf: Vec, - prj: Option, - cpg: Option>, +pub(crate) struct ShapefileParts { + pub shp: Vec, + pub shx: Option>, + pub dbf: Vec, + pub prj: Option, + pub cpg: Option>, } /// Decodes attributes and shapes together, tracking geometry lost during CRS conversion so /// a later projection definition cannot silently repair an incomplete layer. -fn read_shapefile_parts(mut parts: ShapefileParts, name: &str, crs: Option) -> Result<(Dataset, ReadReport)> { +pub(crate) fn read_shapefile_parts(mut parts: ShapefileParts, name: &str, crs: Option) -> Result<(Dataset, ReadReport)> { use shapefile::Shape; use shapefile::dbase::FieldValue; use std::io::Cursor; @@ -2339,6 +2369,7 @@ fn read_shapefile_parts(mut parts: ShapefileParts, name: &str, crs: Option) let encoding = crate::dbf::encoding(parts.cpg.as_deref(), parts.dbf.get(29).copied().unwrap_or(0), &mut encoding_notes); crate::dbf::readable_header_date(&mut parts.dbf); let date_columns = crate::dbf::dates_as_text(&mut parts.dbf); + crate::dbf::floats_as_numeric(&mut parts.dbf); let dbf = shapefile::dbase::ReaderBuilder::new() .with_encoding(encoding.decoder()) .build(Cursor::new(parts.dbf)) @@ -2549,7 +2580,7 @@ fn integral_numeric_to_integer(ds: &mut Dataset) { } /// WGS 84 as ESRI WKT, for a Shapefile's `.prj`. -const WGS84_PRJ: &str = "GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"; +pub(crate) const WGS84_PRJ: &str = "GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"; /// The longest prefix of `s` that fits in `max` bytes, cut at a character boundary. pub(crate) fn cut(s: &str, max: usize) -> &str { @@ -3209,40 +3240,137 @@ pub fn shapefile_parts(ds: &Dataset) -> Result<(Vec<(&'static str, Vec)>, us /// As [`shapefile_parts`], for a dataset whose geometry is in `crs`'s coordinates: the `.prj` /// says `crs`, and is left out for a system WKT can't describe. pub fn shapefile_parts_in(ds: &Dataset, crs: Crs) -> Result<(Vec<(&'static str, Vec)>, usize)> { + shapefile_parts_styled(ds, crs, None).map(|(parts, left_out, _)| (parts, left_out)) +} + +/// What a Shapefile written over the one a layer was read from keeps of that file's `.dbf` +/// (#1385): its code page (the encoding its text is written in, and the language driver byte +/// its header names), and each column's width, decimals and type (`F` stays `F`) where the +/// layer's field of that name still fits it. +pub(crate) struct DbfStyle { + pub encoding: crate::dbf::DbfEncoding, + pub ldid: u8, + pub columns: Vec, +} + +/// The `.dbf` column a field is written to: its type letter, width and decimals. +#[derive(Clone, Copy, PartialEq, Eq)] +struct Spec { + ty: u8, + len: u8, + decimals: u8, +} + +/// The column field `k` of `ds` is written to as a new file has it: numbers in 18 digits +/// (integers) or 24 characters with as many decimals as fit, text as wide as its longest value. +fn new_column(ds: &Dataset, k: usize) -> Spec { + let Some(f) = ds.fields.get(k) else { return Spec { ty: b'C', len: 1, decimals: 0 } }; + match f.ty { + // 18 digits: GDAL reads a wider field as a real number. + FieldType::Integer => Spec { ty: b'N', len: 18, decimals: 0 }, + FieldType::Double => Spec { ty: b'N', len: 24, decimals: u8::try_from(dbf_decimals(ds, k)).unwrap_or(15) }, + FieldType::Boolean => Spec { ty: b'L', len: 1, decimals: 0 }, + FieldType::Date => Spec { ty: b'D', len: 8, decimals: 0 }, + // A `.dbf` has no date-time field that other programs read, so they're ISO text. + FieldType::Text | FieldType::DateTime => { + let longest = ds.features.iter().filter_map(|r| r.attrs.get(k)).map(|v| v.to_string().len()).max().unwrap_or(0); + Spec { ty: b'C', len: u8::try_from(longest.clamp(1, 254)).unwrap_or(254), decimals: 0 } + } + } +} + +/// The column field `k` (written as `name`) is written to over a file whose `.dbf` `style` +/// describes: the file's column of that name where the field's kind and every value still fit +/// it, so the file keeps its widths and decimals; otherwise a new one ([`new_column`]), and +/// `true` when the file had a column of that name and kind that was too small for its values. +fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, bool) { + let new = new_column(ds, k); + let (Some(field), Some(was)) = (ds.fields.get(k), style.columns.iter().find(|c| c.name == name.as_bytes())) else { + return (new, false); + }; + let values = || ds.features.iter().filter_map(|r| r.attrs.get(k)).filter(|v| !v.is_null()); + let ty = was.ty.to_ascii_uppercase(); + let kept = Spec { ty, len: was.len, decimals: was.decimals }; + let encoding = style.encoding.decoder(); + // Bytes of a value's text in the file's code page. + let encoded_len = |v: &Value| { + use shapefile::dbase::encoding::Encoding; + let text = v.to_string(); + encoding.encode(&text).map_or(text.len(), |b| b.len()) + }; + let fits = match (field.ty, ty) { + (FieldType::Integer | FieldType::Double, b'N' | b'F') => values().all(|v| { + let d = usize::from(was.decimals); + dbf_number(v, field.ty, d).is_some_and(|x| { + let text = format!("{x:.d$}"); + text.len() <= usize::from(was.len) && text.parse::().ok() == v.as_f64() + }) + }), + (FieldType::Boolean, b'L') | (FieldType::Date, b'D') => true, + (FieldType::Text | FieldType::DateTime, b'C') => values().all(|v| encoded_len(v) <= usize::from(was.len)), + // Another kind of field now: a new column, and nothing of the old one to keep. + _ => return (new, false), + }; + if fits { + return (kept, false); + } + // Text wider than the file's column: as wide as its longest value, in the file's code page. + if new.ty == b'C' { + let longest = values().map(encoded_len).max().unwrap_or(0); + return (Spec { len: u8::try_from(longest.clamp(usize::from(was.len), 254)).unwrap_or(254), ..new }, true); + } + (new, true) +} + +/// As [`shapefile_parts_in`], the `.dbf` written as `style` says when it's given (in the file's +/// code page, its columns kept where the values fit): also returns the names of the fields whose +/// column was too small for their values and was made wider. +pub(crate) fn shapefile_parts_styled( + ds: &Dataset, + crs: Crs, + style: Option<&DbfStyle>, +) -> Result<(Vec<(&'static str, Vec)>, usize, Vec)> { use shapefile::dbase::{FieldName, FieldValue, Record, TableWriterBuilder}; use std::io::Cursor; let fmt = |e: shapefile::Error| DataError::Format(format!("Shapefile: {e}")); let names = dbf_names(&ds.fields); let dbf_name = |n: &str| FieldName::try_from(n).map_err(|e| DataError::Format(format!("Shapefile: field {n}: {e}"))); - let mut table = TableWriterBuilder::new(); - // Per field: decimals of a Double field (numbers too large for 24 characters become null). - let mut decimals = vec![0usize; ds.fields.len()]; + let mut table = match style { + Some(style) => TableWriterBuilder::with_encoding(style.encoding.decoder()), + None => TableWriterBuilder::new(), + }; + // Per field: its column (numbers too large for it become null), and the fields whose column + // was made wider than the file's. + let mut specs = Vec::with_capacity(ds.fields.len()); + let mut widened = Vec::new(); for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { - table = match f.ty { - // 18 digits: GDAL reads a wider field as a real number. - FieldType::Integer => table.add_numeric_field(dbf_name(n)?, 18, 0), - FieldType::Double => { - let d = dbf_decimals(ds, k); - if let Some(slot) = decimals.get_mut(k) { - *slot = d; + let spec = match style { + Some(style) => { + let (spec, wider) = kept_column(ds, k, n, style); + if wider { + widened.push(f.name.clone()); } - table.add_numeric_field(dbf_name(n)?, 24, u8::try_from(d).unwrap_or(15)) - } - FieldType::Boolean => table.add_logical_field(dbf_name(n)?), - FieldType::Date => table.add_date_field(dbf_name(n)?), - // A `.dbf` has no date-time field that other programs read, so they're ISO text. - FieldType::Text | FieldType::DateTime => { - let longest = ds.features.iter().filter_map(|r| r.attrs.get(k)).map(|v| v.to_string().len()).max().unwrap_or(0); - table.add_character_field(dbf_name(n)?, u8::try_from(longest.clamp(1, 254)).unwrap_or(254)) + spec } + None => new_column(ds, k), }; + table = match spec.ty { + // A float column is written as a numeric one (both hold the number as text) and + // marked `F` again below: `dbase` writes an `F` column's number as an `f32`. + b'N' | b'F' => table.add_numeric_field(dbf_name(n)?, spec.len, spec.decimals), + b'L' => table.add_logical_field(dbf_name(n)?), + b'D' => table.add_date_field(dbf_name(n)?), + _ => table.add_character_field(dbf_name(n)?, spec.len), + }; + specs.push(spec); } let record = |r: &crate::Feature| -> Record { let mut rec = Record::default(); for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { let v = r.attrs.get(k).unwrap_or(&Value::Null); + let decimals = specs.get(k).map_or(0, |s| usize::from(s.decimals)); let value = match f.ty { - FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals.get(k).copied().unwrap_or(0))), + FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals)), FieldType::Boolean => FieldValue::Logical(dbf_boolean(v)), FieldType::Date => FieldValue::Date(dbf_date(v).and_then(|d| { let (y, m, d) = d.ymd(); @@ -3326,10 +3454,25 @@ pub fn shapefile_parts_in(ds: &Dataset, crs: Crs) -> Result<(Vec<(&'static str, } } let prj = if crs == Crs::LonLat { Some(WGS84_PRJ.to_string()) } else { crate::raster::wkt::to_wkt(crs) }; - let mut parts = vec![("shp", shp.into_inner()), ("shx", shx.into_inner()), ("dbf", dbf.into_inner())]; + let mut dbf = dbf.into_inner(); + if let Some(style) = style { + // The file's language driver, whatever `dbase` takes the encoding for, and its float + // columns marked `F` again. + if let Some(b) = dbf.get_mut(29) { + *b = style.ldid; + } + for (k, spec) in specs.iter().enumerate() { + if spec.ty == b'F' + && let Some(b) = dbf.get_mut(32 + k * 32 + 11) + { + *b = b'F'; + } + } + } + let mut parts = vec![("shp", shp.into_inner()), ("shx", shx.into_inner()), ("dbf", dbf)]; parts.extend(prj.map(|p| ("prj", p.into_bytes()))); parts.push(("cpg", b"UTF-8".to_vec())); - Ok((parts, left_out)) + Ok((parts, left_out, widened)) } fn is_outer

(r: &shapefile::PolygonRing

) -> bool { diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index ecc593c471..b9428a5b60 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -37,6 +37,8 @@ pub mod ogc; pub mod osm; pub mod photo; pub mod progress; +#[cfg(not(target_arch = "wasm32"))] +pub mod publish; pub mod query; pub mod raster; pub mod s3_address; diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs new file mode 100644 index 0000000000..793caed29c --- /dev/null +++ b/crates/data/src/publish.rs @@ -0,0 +1,758 @@ +//! Writing a layer kept in the project back over the Shapefile it was read from (#1385), so that +//! the file shows its edits and keeps everything else it held. +//! +//! A Shapefile can't change one row in place: the whole file is written again. That is only +//! safe when nothing but the edits changes, so [`shapefile`] proves it before it touches the +//! file, and refuses, saying why, when it can't: +//! +//! 1. The file is as it was when the layer was read ([`FileSync::changed`]): a change made since +//! would be overwritten. And it reads now as the layer was read: rows that weren't edited are +//! written as the file holds them, not as an older read held them. +//! 2. The file as it was read, written again the same way, holds what the file holds: the same +//! shapes (no Z or M values dropped, none left out), the same columns and the same values, +//! compared as the bytes the `.dbf` holds, not as read (text a read trims, a code page's +//! characters). Whatever a read and write of the file loses shows here, before the edits. +//! 3. The layer, written, reads back as the layer: every value and shape the edits made fits. +//! +//! A shape that wasn't edited is written with the coordinates the file holds, not converted to +//! longitude/latitude and back, so a projected file's shapes stay as they were to the bit. +//! +//! Only then are the new parts written beside the old ones, checked again against the bytes +//! meant, and renamed into place, the `.shp` last; any failure puts the old parts back +//! ([`crate::io::place_shapefile`]). The file keeps its code page, column widths, decimals, +//! `.prj` and `.cpg` where the edits allow; sidecars RustGIS doesn't write (`.shp.xml`) are left. + +use crate::dbf::{Column, RawTable}; +use crate::io::{DbfStyle, ShapefileParts}; +use crate::raster::Crs; +use crate::sync::{FileSync, Synced}; +use crate::{DataError, Dataset, Feature, FieldType, Geometry, Result, Value}; +use std::path::{Path, PathBuf}; + +/// How far an edited shape may read back from where the layer has it, relative to the size of its +/// coordinates, in a file in another system than longitude/latitude: the layer's shape is +/// projected to the file's system and read back through the inverse, which isn't exact to the +/// last bit (a few parts in 10¹³ measured, in a state plane system in feet). One part in 10¹¹ is +/// a tenth of a millimetre at ten thousand kilometres; anything more is refused. +const PROJECTED_TOLERANCE: f64 = 1e-11; + +/// A layer written back over its Shapefile: the files as they were written, to record as the +/// layer's new sync point, the features written, and what it says (fields given wider columns, +/// a `.prj` that now says another system). +pub struct Published { + pub files: FileSync, + pub features: usize, + pub notes: Vec, +} + +/// A refusal: nothing was written. +fn refused(path: &Path, why: impl std::fmt::Display) -> DataError { + DataError::Format(format!("{}: {why}; nothing was written", path.display())) +} + +/// The Shapefile at `path`'s parts as they are, read into memory, with its `.cpg`'s path. +struct Original { + shp: Vec, + shx: Option>, + dbf: Vec, + prj: Option>, + cpg: Option<(PathBuf, Vec)>, +} + +/// Reads the file at `p`, `None` when it isn't there; refuses one past what a reader takes. +fn read_part(p: &Path) -> Result>> { + match std::fs::metadata(p) { + Ok(m) if m.len() > crate::io::MAX_BYTES => Err(DataError::Format(format!("{} is too large to read", p.display()))), + Ok(_) => Ok(Some(std::fs::read(p)?)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } +} + +impl Original { + /// Reads the parts of the Shapefile at `path`: its `.shp` and `.dbf` must be there; its + /// `.shx`, `.prj` and `.cpg` (in whichever case it is spelled) may be. + fn read(path: &Path) -> Result { + let missing = |ext: &str| refused(path, format!("its .{ext} isn't there")); + let shp = read_part(path)?.ok_or_else(|| missing("shp"))?; + let dbf = read_part(&path.with_extension("dbf"))?.ok_or_else(|| missing("dbf"))?; + let shx = read_part(&path.with_extension("shx"))?; + let prj = read_part(&path.with_extension("prj"))?; + let mut cpg = None; + for ext in crate::dbf::CPG_SPELLINGS { + let p = path.with_extension(ext); + if let Some(bytes) = read_part(&p)? { + cpg = Some((p, bytes)); + break; + } + } + Ok(Original { shp, shx, dbf, prj, cpg }) + } + + /// Its parts as the reader takes them. + fn parts(&self) -> ShapefileParts { + ShapefileParts { + shp: self.shp.clone(), + shx: self.shx.clone(), + dbf: self.dbf.clone(), + prj: self.prj.as_ref().and_then(|p| String::from_utf8(p.clone()).ok()), + cpg: self.cpg.as_ref().map(|(_, b)| b.clone()), + } + } + + /// The coordinate system its `.prj` names, as the reader takes it (`None` for none, one it + /// can't read, or longitude/latitude). + fn crs(&self) -> Option { + let text = std::str::from_utf8(self.prj.as_deref()?).ok()?; + crate::raster::wkt::read_wkt(text).ok().map(|(c, _)| c).filter(|c| *c != Crs::LonLat) + } +} + +/// Writes `ds`, read whole from the Shapefile at `path` when it and its files were as `synced` +/// records, back over that Shapefile: see the module's documentation for what it checks first +/// and what it keeps. Its work grows with the file: it is hashed twice, read twice, written in +/// memory twice and read back, the hashing and writing counting progress, and it stops once the +/// run is cancelled (before the last part is placed, which puts the old file back). Reading the +/// file into memory and back are single steps without a check. +pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result { + let unchanged = |when: &str| -> Result<()> { + let changed = synced.files.changed(path)?; + if changed.is_empty() { + return Ok(()); + } + Err(refused(path, format!("{} changed {when}, and writing the layer would overwrite that", changed.join(", ")))) + }; + unchanged("since the layer was read")?; + let original = Original::read(path)?; + let crs = ds.crs.filter(|c| *c != Crs::LonLat); + let written_crs = crs.unwrap_or(Crs::LonLat); + let mut notes = Vec::new(); + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + + let Some(table) = crate::dbf::raw_table(&original.dbf) else { + return Err(refused(path, "its .dbf's header doesn't describe its records")); + }; + let mut encoding_notes = Vec::new(); + let ldid = original.dbf.get(29).copied().unwrap_or(0); + let style = DbfStyle { + encoding: crate::dbf::encoding(original.cpg.as_ref().map(|(_, b)| b.as_slice()), ldid, &mut encoding_notes), + ldid, + columns: table.columns.clone(), + }; + + // 1. It reads as the layer was read: in the layer's system, and in the file's own + // coordinates (the system taken for longitude/latitude, which converts nothing). + let (as_read, _) = crate::io::read_shapefile_parts(original.parts(), &name, crs)?; + if crate::sync::rows_digest(&as_read)? != synced.rows { + return Err(refused( + path, + "it reads otherwise now than when the layer was read (RustGIS reads it more fully since), so rows that weren't edited could change: add the file again to read it as it is", + )); + } + let raw = match crs { + Some(_) => Some(crate::io::read_shapefile_parts(original.parts(), &name, Some(Crs::LonLat))?.0), + None => None, + }; + let as_held = raw.as_ref().map(|raw| (&as_read, raw)); + + // 2. The file as read, written again, is the file. + let (again, _, _) = crate::io::shapefile_parts_styled(&*in_file(&as_read, crs, as_held)?, written_crs, Some(&style))?; + let part = |parts: &[(&str, Vec)], ext: &str| parts.iter().find(|(e, _)| *e == ext).map(|(_, b)| b.clone()).unwrap_or_default(); + if let Some(why) = same_shapes(&original.shp, &part(&again, "shp"), 0.0) { + return Err(refused(path, format!("written again, its shapes wouldn't be the same: {why}"))); + } + if let Some(why) = same_table(&table, &part(&again, "dbf")) { + return Err(refused(path, format!("written again, its table wouldn't be the same: {why}"))); + } + + // 3. The layer, written, reads back as the layer. + if let Some(note) = crate::io::shapefile_names_note(&ds.fields, &name) { + return Err(refused(path, note)); + } + crate::progress::begin(ds.features.len()); + let (mut parts, left_out, widened) = crate::io::shapefile_parts_styled(&*in_file(ds, crs, as_held)?, written_crs, Some(&style))?; + if left_out > 0 { + return Err(refused( + path, + format!("{left_out} of the layer's features have no shape a Shapefile of its kind holds (none, or one of another kind)"), + )); + } + let back = ShapefileParts { + shp: part(&parts, "shp"), + shx: Some(part(&parts, "shx")), + dbf: part(&parts, "dbf"), + prj: None, + cpg: original.cpg.as_ref().map(|(_, b)| b.clone()), + }; + let (read_back, _) = crate::io::read_shapefile_parts(back, &name, crs)?; + if let Some(why) = same_rows(ds, &read_back, if crs.is_some() { PROJECTED_TOLERANCE } else { 0.0 }) { + return Err(refused(path, format!("the file couldn't hold the layer as it is: {why}"))); + } + if !widened.is_empty() { + notes.push(format!( + "{}: {} given wider columns than the file had, to hold their values: {}", + path.display(), + if widened.len() == 1 { "1 field was".to_string() } else { format!("{} fields were", widened.len()) }, + names_list(&widened) + )); + } + + // The `.prj` and `.cpg` as the file had them, unless the layer is in another system. + parts.retain(|(e, _)| !matches!(*e, "prj" | "cpg")); + if crs == original.crs() { + if let Some(prj) = &original.prj { + parts.push(("prj", prj.clone())); + } + } else { + let Some(prj) = (match crs { + None => Some(crate::io::WGS84_PRJ.to_string()), + Some(c) => crate::raster::wkt::to_wkt(c), + }) else { + return Err(refused(path, format!("the layer is in {}, which a .prj can't describe", written_crs.label()))); + }; + notes.push(format!("{}: its .prj now says {}, the system the layer is in", path.display(), written_crs.label())); + parts.push(("prj", prj.into_bytes())); + } + let mut targets: Vec<(PathBuf, Vec)> = parts.into_iter().map(|(e, bytes)| (crate::io::shapefile_part(path, e), bytes)).collect(); + if let Some((cpg, bytes)) = &original.cpg { + targets.push((cpg.clone(), bytes.clone())); + } + // The record of the files as they will be, made before any is placed: once they are, the + // publish has happened, whatever comes after. + let files = FileSync::of_written(path, &targets)?; + + // Written beside the old parts, checked, and placed. + let staged = crate::io::stage_shapefile(&targets)?; + let checked = (|| -> Result<()> { + for ((tmp, _), (_, bytes)) in staged.iter().zip(&targets) { + if std::fs::read(tmp)? != *bytes { + return Err(refused(path, format!("{} didn't read back as written", tmp.display()))); + } + } + unchanged("while the layer was written") + })(); + if let Err(e) = checked { + crate::io::unstage(&staged); + return Err(e); + } + crate::io::place_shapefile(path, &staged)?; + Ok(Published { files, features: ds.features.len(), notes }) +} + +/// `ds` in the file's coordinate system `crs` (`None`: longitude/latitude, as it is held). Given +/// the file as read and in its own coordinates (`as_held`, the same rows), a feature whose shape +/// is the one read takes the file's coordinates as they are; any other is projected. +fn in_file<'a>(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Dataset)>) -> Result> { + let Some(c) = crs else { return Ok(std::borrow::Cow::Borrowed(ds)) }; + let t = c.transformer().map_err(DataError::Format)?; + let (read, raw): (Vec<&Feature>, Vec<&Feature>) = match as_held { + Some((read, raw)) => (read.features.iter().collect(), raw.features.iter().collect()), + None => (Vec::new(), Vec::new()), + }; + let at: std::collections::HashMap = read.iter().enumerate().map(|(i, f)| (f.id, i)).collect(); + let mut rows = Vec::with_capacity(ds.features.len()); + for f in ds.features.iter() { + let held = at.get(&f.id).and_then(|&i| read.get(i).filter(|r| r.geometry == f.geometry).and(raw.get(i))); + let mut row = f.clone(); + match held { + Some(h) => row.geometry = h.geometry.clone(), + None => crate::io::project_features(std::slice::from_mut(&mut row), &t, c)?, + } + rows.push(row); + } + let mut out = ds.clone(); + out.features = rows.into(); + Ok(std::borrow::Cow::Owned(out)) +} + +/// The first few names, quoted, and how many more. +fn names_list(names: &[String]) -> String { + let shown: Vec = names.iter().take(5).map(|n| format!("'{}'", crate::io::dbf_notice_name(n))).collect(); + let more = if names.len() > 5 { format!(" and {} more", names.len() - 5) } else { String::new() }; + format!("{}{more}", shown.join(", ")) +} + +/// A value's bytes for a notice: decoded loosely, quoted, cut after 40 bytes, control characters as +/// spaces. +fn shown(bytes: &[u8]) -> String { + format!("'{}'", crate::io::dbf_notice_name(&String::from_utf8_lossy(bytes))) +} + +/// A shape as the file holds it: its kind, and its coordinates part by part. +fn shape_parts(shape: &shapefile::Shape) -> (&'static str, Vec>) { + use shapefile::Shape; + let xy = |ps: &[shapefile::Point]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + let xym = |ps: &[shapefile::PointM]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + let xyz = |ps: &[shapefile::PointZ]| ps.iter().map(|p| (p.x, p.y)).collect::>(); + match shape { + Shape::NullShape => ("no shape", Vec::new()), + Shape::Point(p) => ("Point", vec![vec![(p.x, p.y)]]), + Shape::PointM(p) => ("PointM", vec![vec![(p.x, p.y)]]), + Shape::PointZ(p) => ("PointZ", vec![vec![(p.x, p.y)]]), + Shape::Multipoint(m) => ("Multipoint", vec![xy(m.points())]), + Shape::MultipointM(m) => ("MultipointM", vec![xym(m.points())]), + Shape::MultipointZ(m) => ("MultipointZ", vec![xyz(m.points())]), + Shape::Polyline(l) => ("Polyline", l.parts().iter().map(|p| xy(p)).collect()), + Shape::PolylineM(l) => ("PolylineM", l.parts().iter().map(|p| xym(p)).collect()), + Shape::PolylineZ(l) => ("PolylineZ", l.parts().iter().map(|p| xyz(p)).collect()), + Shape::Polygon(p) => ("Polygon", p.rings().iter().map(|r| xy(r.points())).collect()), + Shape::PolygonM(p) => ("PolygonM", p.rings().iter().map(|r| xym(r.points())).collect()), + Shape::PolygonZ(p) => ("PolygonZ", p.rings().iter().map(|r| xyz(r.points())).collect()), + Shape::Multipatch(_) => ("Multipatch", Vec::new()), + } +} + +/// Whether `a` and `b` are within `tolerance` of each other, relative to their size. +fn near(a: f64, b: f64, tolerance: f64) -> bool { + a == b || (a - b).abs() <= tolerance * a.abs().max(b.abs()).max(1.0) +} + +/// Whether two lists of parts hold the same coordinates, within `tolerance`. +fn same_coords(a: &[Vec<(f64, f64)>], b: &[Vec<(f64, f64)>], tolerance: f64) -> bool { + a.len() == b.len() + && a.iter() + .zip(b) + .all(|(p, q)| p.len() == q.len() && p.iter().zip(q).all(|(s, t)| near(s.0, t.0, tolerance) && near(s.1, t.1, tolerance))) +} + +/// Why the shapes of `.shp` file `b` aren't those of `a`, or `None` when they are: one shape for +/// each, of the same kind, with the same coordinates within `tolerance`. Counts each kind of +/// difference and names the first shape of it. +fn same_shapes(a: &[u8], b: &[u8], tolerance: f64) -> Option { + let read = |bytes: &[u8]| -> std::result::Result, String> { + let mut reader = shapefile::ShapeReader::new(std::io::Cursor::new(bytes)).map_err(|e| e.to_string())?; + reader.iter_shapes().collect::, _>>().map_err(|e| e.to_string()) + }; + let (a, b) = match (read(a), read(b)) { + (Ok(a), Ok(b)) => (a, b), + (Err(e), _) | (_, Err(e)) => return Some(format!("its shapes couldn't be read ({e})")), + }; + let (a, b): (Vec<_>, Vec<_>) = (a.iter().map(shape_parts).collect(), b.iter().map(shape_parts).collect()); + // Each kind of shape a read leaves out, with the first record holding one. + let mut out = Vec::new(); + for kind in ["no shape", "Multipatch"] { + let held: Vec = a.iter().enumerate().filter(|(_, (k, _))| *k == kind).map(|(i, _)| i + 1).collect(); + if let Some(first) = held.first() { + let what = if kind == "no shape" { "have no shape" } else { "are multipatches" }; + out.push(format!("{} records {what}, which RustGIS leaves out (the first, record {first})", held.len())); + } + } + if !out.is_empty() { + return Some(out.join("; ")); + } + if a.len() != b.len() { + return Some(format!("{} shapes would be {}", a.len(), b.len())); + } + let kinds: Vec = a.iter().zip(&b).enumerate().filter(|(_, (s, t))| s.0 != t.0).map(|(i, _)| i).collect(); + if let Some(&first) = kinds.first() { + let (was, now) = (a.get(first).map_or("", |s| s.0), b.get(first).map_or("", |s| s.0)); + let zm = if was.ends_with('Z') || was.ends_with('M') { ", as RustGIS doesn't keep Z or M values" } else { "" }; + return Some(format!("{} shapes would change kind{zm} (the first, record {}: {was} to {now})", kinds.len(), first + 1)); + } + let moved: Vec = + a.iter().zip(&b).enumerate().filter(|(_, (s, t))| !same_coords(&s.1, &t.1, tolerance)).map(|(i, _)| i).collect(); + moved.first().map(|first| format!("{} shapes would change (the first, record {})", moved.len(), first + 1)) +} + +/// A `.dbf` value as a value, for comparing what two files hold: text without the spaces or NULs +/// that pad it, a number as the number it says, a date or yes/no without the forms of empty. +#[derive(PartialEq)] +enum Cell<'a> { + Bytes(&'a [u8]), + Number(f64), + Empty, +} + +/// What value `bytes` of a column of type `ty` holds ([`Cell`]). +fn cell(ty: u8, bytes: &[u8]) -> Cell<'_> { + let padding = |b: &u8| *b == b' ' || *b == 0; + let end = bytes.iter().rposition(|b| !padding(b)).map_or(0, |i| i + 1); + let right = bytes.get(..end).unwrap_or_default(); + let trimmed = right.get(right.iter().position(|b| !padding(b)).unwrap_or(right.len())..).unwrap_or_default(); + match ty.to_ascii_uppercase() { + b'C' => { + if right.is_empty() { + Cell::Empty + } else { + Cell::Bytes(right) + } + } + b'N' | b'F' => { + if trimmed.is_empty() || trimmed.iter().all(|b| *b == b'*') { + return Cell::Empty; + } + match std::str::from_utf8(trimmed).ok().and_then(|t| t.parse::().ok()) { + Some(x) => Cell::Number(x), + None => Cell::Bytes(trimmed), + } + } + b'L' => match trimmed.first().map(u8::to_ascii_uppercase) { + Some(b'T' | b'Y' | b'1') => Cell::Bytes(b"T"), + Some(b'F' | b'N' | b'0') => Cell::Bytes(b"F"), + _ => Cell::Empty, + }, + b'D' if trimmed.is_empty() || trimmed == b"00000000" => Cell::Empty, + _ => Cell::Bytes(trimmed), + } +} + +/// A column described for a notice: its name, type, width and decimals (`N(10,2)`). +fn column(c: &Column) -> String { + let name = String::from_utf8_lossy(&c.name); + let ty = char::from(c.ty); + if matches!(c.ty.to_ascii_uppercase(), b'N' | b'F') { + format!("'{}' {ty}({},{})", crate::io::dbf_notice_name(&name), c.len, c.decimals) + } else { + format!("'{}' {ty}({})", crate::io::dbf_notice_name(&name), c.len) + } +} + +/// Why the `.dbf` `b` doesn't hold what `a` holds, or `None` when it does: the same columns +/// (names, types, widths, decimals) and, record by record, the same values ([`cell`]). Counts the +/// values that differ and names the first. +fn same_table(a: &RawTable, b: &[u8]) -> Option { + let Some(b) = crate::dbf::raw_table(b) else { return Some("its .dbf couldn't be written".into()) }; + if a.columns.len() != b.columns.len() { + return Some(format!("it has {} columns and would have {}", a.columns.len(), b.columns.len())); + } + let changed: Vec = a + .columns + .iter() + .zip(&b.columns) + .filter(|(s, t)| s.name != t.name || !s.ty.eq_ignore_ascii_case(&t.ty) || s.len != t.len || s.decimals != t.decimals) + .map(|(s, t)| format!("{} would be {}", column(s), column(t))) + .collect(); + if !changed.is_empty() { + return Some(format!("its columns would change: {}", changed.join(", "))); + } + if a.records.len() != b.records.len() { + return Some(format!("{} records would be {}", a.records.len(), b.records.len())); + } + let mut first = None; + let mut count = 0usize; + for (row, (r, s)) in a.records.iter().zip(&b.records).enumerate() { + for (c, (x, y)) in a.columns.iter().zip(r.iter().zip(s)) { + if cell(c.ty, x) != cell(c.ty, y) { + count += 1; + first.get_or_insert((row, c, *x, *y)); + } + } + } + first.map(|(row, c, x, y)| { + format!( + "{count} values would change (the first, record {} of '{}': {} would be {})", + row + 1, + crate::io::dbf_notice_name(&String::from_utf8_lossy(&c.name)), + shown(x), + shown(y) + ) + }) +} + +/// Whether two values of a field of type `ty` are the same once written to a `.dbf` and read: +/// numbers by their value (a whole number reads back as an integer), empty text and no value +/// alike (a `.dbf` can't tell them apart), a date-time by its text. +fn same_value(ty: FieldType, a: &Value, b: &Value) -> bool { + let text = |v: &Value| match v { + Value::Null => String::new(), + v => v.to_string(), + }; + match (a, b) { + _ if a == b => true, + (Value::Int(_) | Value::Float(_), Value::Int(_) | Value::Float(_)) => a.as_f64() == b.as_f64(), + _ if matches!(ty, FieldType::Text | FieldType::DateTime) => text(a) == text(b), + _ => false, + } +} + +/// A geometry as its parts of coordinates: a point is a part of one, each polygon ring closed +/// and turned to run the same way whichever way it was drawn (a Shapefile orders a ring's +/// vertices by whether it is a hole). +fn geometry_parts(g: Option<&Geometry>) -> Vec> { + let xy = |cs: &[crate::Coord]| cs.iter().map(|c| (c.x, c.y)).collect::>(); + match g { + None => Vec::new(), + Some(Geometry::Point(c)) => vec![vec![(c.x, c.y)]], + Some(Geometry::MultiPoint(cs)) => cs.iter().map(|c| vec![(c.x, c.y)]).collect(), + Some(Geometry::Line(parts)) => parts.iter().map(|p| xy(p)).collect(), + Some(Geometry::Polygon(polys)) => polys + .iter() + .flatten() + .map(|r| { + let mut ring = xy(r); + if ring.first() != ring.last() { + ring.extend(ring.first().copied()); + } + // Twice the signed area: positive runs anticlockwise. + let area: f64 = ring.windows(2).map(|w| w.first().zip(w.get(1)).map_or(0.0, |(p, q)| p.0 * q.1 - q.0 * p.1)).sum(); + if area < 0.0 { + ring.reverse(); + } + ring + }) + .collect(), + } +} + +/// Why `back`, the layer `ds` written to a Shapefile and read again, isn't `ds`, or `None` when +/// it is: the same fields, features, values ([`same_value`]) and shapes within `tolerance`. +/// Counts the values and shapes that differ and names the first of each. +fn same_rows(ds: &Dataset, back: &Dataset, tolerance: f64) -> Option { + let names = |d: &Dataset| d.fields.iter().map(|f| f.name.clone()).collect::>(); + if names(ds) != names(back) { + return Some(format!("its fields would read back as {}", names_list(&names(back)))); + } + if ds.features.len() != back.features.len() { + return Some(format!("its {} features would read back as {}", ds.features.len(), back.features.len())); + } + let (mut values, mut first_value, mut shapes, mut first_shape) = (0usize, None, 0usize, None); + for (k, (f, g)) in ds.features.iter().zip(back.features.iter()).enumerate() { + for (field, (a, b)) in ds.fields.iter().zip(f.attrs.iter().zip(&g.attrs)) { + if !same_value(field.ty, a, b) { + values += 1; + first_value.get_or_insert((k, field.name.clone(), a.clone(), b.clone())); + } + } + // Points are compared as parts of one point each, so a point and a multipoint of one agree. + let (p, q) = (geometry_parts(f.geometry.as_ref()), geometry_parts(g.geometry.as_ref())); + if !same_coords(&p, &q, tolerance) { + shapes += 1; + first_shape.get_or_insert(k); + } + } + let mut out = Vec::new(); + if let Some((k, name, a, b)) = first_value { + let show = |v: &Value| if v.is_null() { "empty".to_string() } else { shown(v.to_string().as_bytes()) }; + out.push(format!( + "{values} values would read back otherwise (the first, feature {} of '{}': {} as {})", + k + 1, + crate::io::dbf_notice_name(&name), + show(&a), + show(&b) + )); + } + if let Some(k) = first_shape { + out.push(format!("{shapes} shapes would read back otherwise (the first, feature {})", k + 1)); + } + (!out.is_empty()).then(|| out.join("; ")) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{Coord, Field, GeometryType}; + + /// A small polygon layer of parcels with an owner, an area with decimals and a count. + fn parcels() -> Dataset { + let mut ds = Dataset::new( + "parcels", + GeometryType::Polygon, + vec![Field::new("OWNER", FieldType::Text), Field::new("ACRES", FieldType::Double), Field::new("UNITS", FieldType::Integer)], + ); + let square = |x: f64| { + Geometry::Polygon(vec![vec![vec![ + Coord::new(x, 35.0), + Coord::new(x, 35.1), + Coord::new(x + 0.1, 35.1), + Coord::new(x + 0.1, 35.0), + Coord::new(x, 35.0), + ]]]) + }; + ds.push(Some(square(-90.0)), vec![Value::Text("Smith".into()), Value::Float(1.25), Value::Int(2)]); + ds.push(Some(square(-89.8)), vec![Value::Text("Jones".into()), Value::Float(0.5), Value::Int(1)]); + ds + } + + /// Writes `ds` as a Shapefile at `path` and reads it back with its sync record. + fn written(ds: &Dataset, path: &Path) -> (Dataset, Synced) { + crate::io::write_path(ds, path).unwrap(); + read(path) + } + + /// Reads the Shapefile at `path` with its sync record, as Add Data does. + fn read(path: &Path) -> (Dataset, Synced) { + let (read, files) = FileSync::around(path, || crate::io::read_path(path).unwrap()).unwrap(); + let rows = crate::sync::rows_digest(&read.0).unwrap(); + (read.0, Synced { files: files.unwrap(), rows }) + } + + /// An edit is written back: the file reads as the layer, keeps its columns and `.prj`, and + /// its new record matches the files as they are. + #[test] + fn an_edit_is_written_back_and_the_file_keeps_its_columns() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let prj = std::fs::read(path.with_extension("prj")).unwrap(); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{:?}", published.notes); + let (back, _) = crate::io::read_path(&path).unwrap(); + assert_eq!(back.features.iter().next().unwrap().attrs[0], Value::Text("Smyth".into())); + assert_eq!(std::fs::read(path.with_extension("prj")).unwrap(), prj); + assert_eq!(crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns, columns); + assert!(published.files.changed(&path).unwrap().is_empty()); + // Nothing left beside it. + let names: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert!(names.iter().all(|n| !n.starts_with('.')), "{names:?}"); + } + + /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a + /// project that couldn't hold a value) is refused: its unedited rows aren't the file's. + #[test] + fn a_layer_read_otherwise_than_the_file_reads_now_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (ds, mut sync) = written(&parcels(), &path); + // As an older read had it: one value otherwise. + let mut older = ds.clone(); + let mut rows: Vec<_> = older.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(1.2); + older.features = rows.into(); + sync.rows = crate::sync::rows_digest(&older).unwrap(); + let e = shapefile(&older, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("reads otherwise now") && e.contains("nothing was written"), "{e}"); + } + + /// A file changed since it was read is never written over. + #[test] + fn a_changed_file_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (ds, sync) = written(&parcels(), &path); + let other = Dataset { features: ds.features.iter().take(1).cloned().collect::>().into(), ..ds.clone() }; + crate::io::write_path(&other, &path).unwrap(); + let before = std::fs::read(path.with_extension("dbf")).unwrap(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("changed since the layer was read") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), before); + } + + /// Text in a code page is written back in it, with the file's language driver; a character + /// the code page can't hold is refused rather than written as something else. + #[test] + fn a_code_page_is_kept_and_what_it_cant_hold_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // The same file in Windows-1252: its `.cpg` says so. + std::fs::write(path.with_extension("cpg"), "1252").unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Muñoz".into()); + ds.features = rows.clone().into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + // The record of what was published, as the layer takes it. + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&ds).unwrap() }; + let dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + assert!(dbf.windows(5).any(|w| w == b"Mu\xf1oz"), "written in Windows-1252"); + assert_eq!(std::fs::read(path.with_extension("cpg")).unwrap(), b"1252"); + assert_eq!(crate::io::read_path(&path).unwrap().0.features.iter().next().unwrap().attrs[0], Value::Text("Muñoz".into())); + // A character Windows-1252 doesn't have. + rows[1].attrs[0] = Value::Text("Łukasz".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &synced).err().unwrap().to_string(); + assert!(e.contains("couldn't hold the layer") && e.contains("Łukasz"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), dbf); + } + + /// What a read loses is refused before the edits are: text a read trims, a feature without a + /// shape, a number too precise for the column. + #[test] + fn what_a_read_loses_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // The first owner with a space before it, as another program may have written it. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let at = dbf.windows(5).position(|w| w == b"Smith").unwrap(); + dbf.splice(at..at + 5, *b" Smit"); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("table wouldn't be the same") && e.contains("' Smit'"), "{e}"); + } + + /// A float (`F`) column reads every digit its text holds, not the seven of an `f32`, and is + /// written back as an `F` column with its width and decimals. + #[test] + fn a_float_column_keeps_its_digits_and_its_type() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut ds = parcels(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(123456.12345678); + ds.features = rows.into(); + crate::io::write_path(&ds, &path).unwrap(); + // ACRES, the second column, made a float column as other programs write one. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + dbf[32 + 32 + 11] = b'F'; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.iter().next().unwrap().attrs[1], Value::Float(123456.12345678)); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + assert_eq!(columns[1].ty, b'F'); + assert_eq!(crate::io::read_path(&path).unwrap().0.features.iter().next().unwrap().attrs[1], Value::Float(123456.12345678)); + } + + /// A widened column is said; a layer in another system gets a `.prj` for it, said too. + #[test] + fn a_wider_value_widens_its_column_and_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smith and Jones Holdings".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + assert!(published.notes[0].contains("1 field was given wider columns") && published.notes[0].contains("'OWNER'")); + let (back, _) = crate::io::read_path(&path).unwrap(); + assert_eq!(back.features.iter().next().unwrap().attrs[0], Value::Text("Smith and Jones Holdings".into())); + } + + /// A projected file's unedited shapes stay where they were, to the tolerance. + #[test] + fn a_projected_file_keeps_its_unedited_shapes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + // UTM in metres, and a state plane system in US feet. + for code in ["EPSG:32616", "EPSG:2274"] { + let crs = Crs::parse(code).unwrap(); + let mut ds = parcels(); + ds.crs = Some(crs); + crate::io::write_path_in(&ds, &path, crs).unwrap(); + let before = std::fs::read(&path).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[2] = Value::Int(3); + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + // Every shape as it was, to the bit: none was edited. + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none(), "{code}"); + // A shape moved is projected; the others stay as they were. + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[1].geometry { + polys[0][0][1].y += 0.01; + } + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + let shapes = |b: &[u8]| -> Vec<_> { + let mut r = shapefile::ShapeReader::new(std::io::Cursor::new(b.to_vec())).unwrap(); + r.iter_shapes().map(|s| shape_parts(&s.unwrap()).1).collect() + }; + let (was, now) = (shapes(&before), shapes(&std::fs::read(&path).unwrap())); + assert_eq!(was[0], now[0], "{code}"); + assert_ne!(was[1], now[1], "{code}"); + } + } +} diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index ff65602256..4fd3b41cce 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -161,6 +161,28 @@ mod native { Ok(FileSync { files }) } + /// The files of the source at `path` as just written: each of `written` (a path and + /// the bytes written to it) by those bytes, so a write made since by someone else isn't + /// taken for this one; any other part as it is on the disk. + pub fn of_written(path: &Path, written: &[(PathBuf, Vec)]) -> Result { + use sha2::Digest; + let mut files = Vec::new(); + for (name, p) in parts(path) { + let stamp = match written.iter().find(|(w, _)| *w == p) { + Some((_, bytes)) => Some(FileStamp { + len: bytes.len() as u64, + sha256: sha2::Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect(), + }), + None => match len(&p)? { + None => None, + Some(len) => Some(FileStamp { len, sha256: sha256(&p)? }), + }, + }; + files.push((name, stamp)); + } + Ok(FileSync { files }) + } + /// Runs `read`, which reads the source at `path`, and returns with its result the files /// as they were just before it ran (`None` when a part couldn't be read). Hashed before /// the read, the record can only be older than what was read: a write while it ran, or diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index 4485de10d4..e84fdb1fc8 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -44,6 +44,7 @@ pub(crate) const IN_BACKGROUND: &[(&str, &str, Start)] = &[ ("share.exportMap", "Export Map", super::share::map_work), ("layout.export", "Export Layout", super::layout::export_work), ("report.export", "Export Report", super::report::export_work), + ("source.publish", "Publish to Source", super::publish::publish_work), ]; /// `command` and what it is called, when the app writes its file on another thread (see diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 97887904b8..475b16a0c6 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -1,5 +1,5 @@ //! A layer kept in the project and the file it was read from: whether its edits and the file are -//! in step (#1385). +//! in step, and writing its edits back to the file (#1385). use crate::{CmdResult, CommandSpec as C, Session}; use serde_json::Value; @@ -9,9 +9,103 @@ pub fn commands() -> Vec { C::new("source.syncState", "Source File State", "Data › Data Source", sync_state).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile or .geojson file"} — whether it has edits its file lacks ("edited"), the file changed since it was read ("sourceChanged"), both ("both") or neither ("inSync"), and which of the file's parts changed"#, ), + C::new("source.publish", "Publish to Source", "Data › Data Source", |s, p| super::exports::write_here(s, p, "source.publish")).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( + r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile"} — writes the layer's edits back over the Shapefile it was read from, keeping its code page, columns and .prj; refused, with nothing written, when the file changed since it was read or can't hold the layer as it is (Z or M values, features without shapes, text its code page lacks)"#, + ), ] } +/// The layer's record of the file it was read from and that file's path, or why it has none +/// (it wasn't read whole from a Shapefile or `.geojson` file, or its record was lost), or why +/// the file isn't opened (it isn't on this computer). +fn record(l: &rustgis_map::Layer) -> Result<(&rustgis_data::sync::Synced, &str), String> { + let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { + if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { + return Err(format!( + "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", + l.name + )); + } + return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); + }; + // A project from elsewhere names the path: only a local file of a kind a record is made for + // is opened, never a network share (opening one can send the user's login to its host). + if !local_file(path) { + return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); + } + Ok((synced, path.as_str())) +} + +/// `source.publish`'s work, its inputs taken from the session ([`super::exports`]): writes layer +/// `p` names back over the Shapefile it was read from ([`rustgis_data::publish::shapefile`], +/// which says what it checks first), when it has edits the file lacks. Its work grows with the +/// file (hashed, read, written twice in memory and read back): the desktop app runs it on a +/// thread of its own with progress and Cancel. Once written, the layer's record is of the files +/// written and the rows published, so it reads `inSync`; the project then has a change to save. +pub(crate) fn publish_work(s: &Session, p: &Value) -> Result { + let id = s.layer_id(p)?; + let l = s.map().layer(id).ok_or("the layer no longer exists")?; + let (synced, path) = record(l)?; + if !path.to_ascii_lowercase().ends_with(".shp") { + return Err(format!("{}: only a layer read from a Shapefile can be published to its file so far; {path} isn't one", l.name)); + } + #[cfg(target_arch = "wasm32")] + { + let _ = synced; + Err("the web app can't write files on this computer".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use super::exports::{Then, Wrote}; + let (synced, path, dataset, instance, name) = + (synced.clone(), path.to_string(), l.dataset.clone(), l.instance_id(), l.name.clone()); + Ok(Box::new(move || { + let rows = rustgis_data::sync::rows_digest(&dataset).map_err(|e| e.to_string())?; + let features = dataset.features.len(); + if rows == synced.rows { + let changed = synced.files.changed(std::path::Path::new(&path)).map_err(|e| e.to_string())?; + if !changed.is_empty() { + return Err(format!( + "{path}: {} changed since {name} was read, and {name} has no edits to write: add the file again to read it as it is", + changed.join(", ") + )); + } + return Ok(Wrote { + answer: serde_json::json!({"layer": id, "path": path, "published": false, "features": features}), + notes: vec![format!("{name} has no edits its file lacks, so {path} wasn't written")], + written_path: None, + then: None, + owner: None, + }); + } + let published = rustgis_data::publish::shapefile(&dataset, std::path::Path::new(&path), &synced).map_err(|e| e.to_string())?; + let new = rustgis_data::sync::Synced { files: published.files, rows }; + let (said, written_path) = (path.clone(), Some(path.clone())); + let then: Then = Box::new(move |s: &mut Session, _: &mut Value| { + // The record follows the file: the layer read from it takes the record of what + // was written, unless it was read again or its record changed meanwhile. + let live = s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).find(|l| l.instance_id() == instance); + match live { + Some(l) if l.synced.as_ref() == Some(&synced) && l.dataset.source.as_deref() == Some(said.as_str()) => { + l.synced = Some(new); + s.mark_unsaved(); + } + _ => s.warn(format!("{said} was written, but {name} changed while it was: add the file again to compare them")), + } + s.notify(format!("Published {name} to {said}"), false); + Ok(()) + }); + Ok(Wrote { + answer: serde_json::json!({"layer": id, "path": path, "published": true, "features": published.features}), + notes: published.notes, + written_path, + then: Some(then), + owner: None, + }) + })) + } +} + /// The key a check of layer `id`'s files made ahead is taken by (see [`super::reads`]). #[cfg(not(target_arch = "wasm32"))] fn ahead_key(id: u32) -> String { @@ -74,20 +168,7 @@ pub(crate) fn sync_ahead(_: &Session, _: &Value) -> Option CmdResult { let id = s.layer_id(p)?; let l = s.map().layer(id).ok_or("the layer no longer exists")?; - let (Some(synced), Some(path)) = (&l.synced, &l.dataset.source) else { - if l.dataset.source.as_deref().is_some_and(rustgis_data::sync::is_syncable) && !l.is_streamed() { - return Err(format!( - "{} has no record of its file as it was read (it was added before RustGIS kept one, a part of the file couldn't be read, or an older RustGIS saved the project): add the file again to compare it", - l.name - )); - } - return Err(format!("{} wasn't read whole from a Shapefile or .geojson file, so there's no file to compare it with", l.name)); - }; - // A project from elsewhere names the path: only a local file of a kind a record is made for - // is opened, never a network share (opening one can send the user's login to its host). - if !local_file(path) { - return Err(format!("{}: {path} isn't a Shapefile or .geojson file on this computer, so it isn't checked", l.name)); - } + let (synced, path) = record(l)?; #[cfg(target_arch = "wasm32")] { let _ = (synced, path); @@ -95,7 +176,7 @@ fn sync_state(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - let (was, files, path) = (synced.rows, synced.files.clone(), path.clone()); + let (was, files, path) = (synced.rows, synced.files.clone(), path.to_string()); let ahead = super::reads::take::(s, &ahead_key(id)); let l = s.map().layer(id).ok_or("the layer no longer exists")?; let (rows, changed) = match ahead { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b3e06dc903..6685d914a9 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -2433,6 +2433,7 @@ impl Session { self.reopen_layout(open, saved); self.keep_time_position(&cur); self.keep_tile_pick(&cur); + self.keep_synced(&cur); self.redo.push((cur, was, label, actor, std::mem::replace(&mut self.state, state), files, old_trust)); self.touch_all_geometry(); true @@ -2457,6 +2458,7 @@ impl Session { self.reopen_layout(open, saved); self.keep_time_position(&cur); self.keep_tile_pick(&cur); + self.keep_synced(&cur); self.undo.push((cur, was, label, actor, std::mem::replace(&mut self.state, state), files, old_trust)); self.touch_all_geometry(); true @@ -2475,6 +2477,18 @@ impl Session { cmd::zarr::follow_time(self); } + /// After undo or redo swaps the project: each layer keeps the record of its file it had + /// (`synced`), as undo doesn't change the file. A layer published since keeps the record of + /// what was written, so undoing an edit made before shows it as an edit the file holds. + fn keep_synced(&mut self, before: &Project) { + for l in self.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()) { + let old = before.maps.iter().flat_map(|m| &m.layers).find(|o| o.instance_id() == l.instance_id()); + if let Some(old) = old.filter(|o| o.dataset.source == l.dataset.source) { + l.synced = old.synced.clone(); + } + } + } + /// After undo or redo swaps the project: the tile layer picked in the Layers pane stays the one /// picked (picking isn't an undo step), found again on the map of its name by its address, /// kind, role and name (without the name when only one layer has the rest: a rename undone); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 1b129b123d..83b0b7fc37 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -43716,6 +43716,10 @@ fn long_commands_stop_when_cancelled_while_they_run() { s.run("layer.addDataText", &json!({"fileName": "training.geojson", "text": training.to_string()})).unwrap(); s.run("layout.new", &json!({"page": "a4", "orientation": "landscape", "template": "blank"})).unwrap(); s.run("layout.addFrame", &json!({"x": 20, "y": 20, "w": 150, "h": 150})).unwrap(); + // A layer read from a Shapefile, edited, to publish. + s.run("share.exportLayer", &json!({"layer": "Cities", "path": out("towns.shp")})).unwrap(); + s.run("layer.addData", &json!({"path": out("towns.shp"), "name": "towns"})).unwrap(); + s.run("edit.setValue", &json!({"layer": "towns", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); // What didn't stop, and how often it looked. let mut kept_going = Vec::new(); @@ -43772,6 +43776,7 @@ fn long_commands_stop_when_cancelled_while_they_run() { ("share.exportMap", json!({"path": out("map.png"), "width": 320, "height": 200})), ("layout.export", json!({"path": out("layout.pdf"), "dpi": 72})), ("report.export", json!({"path": out("report.pdf"), "dpi": 72})), + ("source.publish", json!({"layer": "towns"})), ]; s.run("report.new", &json!({"layer": "Cities"})).unwrap(); let registered = crate::cmd::exports::IN_BACKGROUND.iter().map(|(c, ..)| *c).collect::>(); @@ -44429,6 +44434,68 @@ fn change_detection_leaves_out_see_through_cells() { ); } +/// A layer read from a Shapefile publishes its edits to the file (#1385): the file then holds +/// them, the layer reads `inSync`, and undoing an edit made before shows it as an edit again, +/// as the file keeps it. A layer without edits writes nothing; a file changed since it was read, +/// or one read from a GeoJSON file, is refused with nothing written. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_layer_publishes_its_edits_to_its_shapefile() { + // Removed however the test ends. + struct Dir(std::path::PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + impl Dir { + fn path(&self) -> &std::path::Path { + &self.0 + } + } + let dir = Dir(std::env::temp_dir().join(format!("rustgis-engine-publish-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(dir.path()); + std::fs::create_dir_all(dir.path()).unwrap(); + let path = dir.path().join("towns.shp"); + let mut s = world(); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": path})).unwrap(); + let id = s.run("layer.addData", &json!({"path": path, "name": "towns"})).unwrap()["layer"].as_u64().unwrap(); + let state = |s: &mut Session| s.run("source.syncState", &json!({"layer": id})).unwrap()["state"].clone(); + let dbf = || std::fs::read(dir.path().join("towns.dbf")).unwrap(); + let before = dbf(); + let out = s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(out["published"], false); + assert_eq!(dbf(), before, "nothing to write"); + s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + s.mark_saved(); + let out = s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(out["published"], true); + assert!(s.dirty(), "the layer's new record is a change to save"); + let (read, _) = rustgis_data::io::read_path(&path).unwrap(); + let name = read.fields.iter().position(|f| f.name == "name").unwrap(); + assert_eq!(read.features.iter().next().unwrap().attrs[name], rustgis_data::Value::Text("Renamed".into())); + assert_eq!(state(&mut s), "inSync"); + // Undo doesn't change the file: the layer now lacks an edit the file has. + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(state(&mut s), "edited"); + s.run("edit.redo", &Value::Null).unwrap(); + assert_eq!(state(&mut s), "inSync"); + // Changed since: refused, the file left as it was. + s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Mine"})).unwrap(); + let mut theirs = dbf(); + *theirs.last_mut().unwrap() ^= 1; + std::fs::write(dir.path().join("towns.dbf"), &theirs).unwrap(); + let e = s.run("source.publish", &json!({"layer": id})).unwrap_err(); + assert!(e.contains("towns.dbf changed since the layer was read") && e.contains("nothing was written"), "{e}"); + assert_eq!(dbf(), theirs); + // GeoJSON isn't published yet. + let geojson = dir.path().join("towns.geojson"); + s.run("share.exportLayer", &json!({"layer": "Cities", "path": geojson})).unwrap(); + s.run("layer.addData", &json!({"path": geojson, "name": "towns.geojson"})).unwrap(); + let e = s.run("source.publish", &json!({"layer": "towns.geojson"})).unwrap_err(); + assert!(e.contains("only a layer read from a Shapefile"), "{e}"); +} + /// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read /// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed /// since, and neither once an edit is undone. The record is saved with the project. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a838a8d341..8ed6294210 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2462,6 +2462,12 @@ impl RustGisApp { if id == "source.syncState" { return readjob::run_then(self, "source.syncState", params, Some(sync_state_said)); } + // Publishing hashes, reads and writes the whole file: on a thread of its own, with + // progress and Cancel, as an export is. + if id == "source.publish" { + let params = if params.is_null() { json!({}) } else { params }; + return exportjob::start(self, "source.publish", params); + } // Converting the labels of a layer read in place reads all of its source first, on a // thread of its own, whoever asks (a click, the assistant). if id == "labels.convert" diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 18be8f69f0..68bd196230 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -397,6 +397,41 @@ can't check it. On the desktop, make the layer active and choose **Source File State** in Command Search; what it found shows in the status bar. CLI and Python: `source.syncState` with `{"layer":"parcels"}`. +### Publish to source + +`source.publish` writes a layer's edits back over the Shapefile it was read from, so the file +that other people and programs read holds them. A Shapefile can't change one row in place, so +the whole file is written again, and RustGIS first makes sure that only the edits change. It +refuses, with nothing written, when: + +- the file changed since the layer was read (`source.syncState` would say `sourceChanged` or + `both`): writing would overwrite that change; +- the file, read and written again, wouldn't be the same file: shapes with Z or M values, records + without a shape, multipatches, text with spaces before it, columns of a type RustGIS doesn't + write (memo, binary). The message names what would change and where; +- the file can't hold the layer as it is: a feature without a shape, a field name longer than 10 + bytes, text in characters the file's code page lacks. + +The file keeps its code page (its `.cpg` and the language driver in its `.dbf`), each column's +width, decimals and type where the values still fit, and its `.prj`. A column too narrow for an +edited value is made wider, and a notice says which. A layer read in another coordinate system +than its `.prj` names gets a `.prj` for that system, said too. The new parts are written beside +the old ones, checked, and then renamed into place, the `.shp` last; if any fails, the old parts +are put back. Sidecars RustGIS doesn't write, such as `.shp.xml` metadata, are left alone; the +spatial indexes that would no longer match (`.qix`, `.sbn`, `.sbx`) are removed. In a projected +coordinate system, RustGIS holds the shapes in longitude/latitude, so unedited shapes go through +that conversion and back: they may move by one part in 10¹¹ of their coordinates at most (a tenth +of a millimetre at ten thousand kilometres), and anything more is refused. + +Once published, the layer reads `inSync`, and the project has a change to save (its record of +the file). Undo doesn't change the file: undoing an edit made before publishing shows the layer +as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't +be published yet. + +On the desktop, make the layer active and choose **Publish to Source** in Command Search; it runs +in the background, with progress and Cancel in the status bar. CLI and Python: `source.publish` +with `{"layer":"parcels"}`. + ## Keyboard shortcuts | Keys | Action | From 9203f1b197bcf5742e2dd3fc0f61fadcb1e555f7 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 12:58:15 -0600 Subject: [PATCH 02/15] fix(data): publish refuses deleted records and names each column it changes A text column 255 bytes wide (the most a .dbf allows) with a longer value crashed in clamp; it now gets the 254 bytes written. A column that couldn't hold its values is named with its old and new form ('ACRES' N(10,2) to N(24,9)), as a number's column may change decimals rather than widen, and never gets narrower than the file's. A file with records marked deleted is refused, as writing it again would drop them. Projecting edited shapes no longer resets the progress total. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/dbf.rs | 8 +++-- crates/data/src/io.rs | 70 ++++++++++++++++++++++++++------------ crates/data/src/publish.rs | 39 ++++++++++++++++++--- 3 files changed, 88 insertions(+), 29 deletions(-) diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index 034d81041a..3105637735 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -225,6 +225,8 @@ pub(crate) struct Column { pub(crate) struct RawTable<'a> { pub columns: Vec, pub records: Vec>, + /// The records marked deleted, which aren't among `records`. + pub deleted: usize, } #[cfg(not(target_arch = "wasm32"))] @@ -248,10 +250,10 @@ pub(crate) fn raw_table(dbf: &[u8]) -> Option> { return None; } let body = dbf.get(header_len..)?; - let mut records = Vec::new(); + let (mut records, mut deleted) = (Vec::new(), 0); for record in body.chunks_exact(record_len).take(count) { - // A record marked deleted (`*`) isn't read, so it isn't written either. if record.first() == Some(&b'*') { + deleted += 1; continue; } let mut at = 1; @@ -263,7 +265,7 @@ pub(crate) fn raw_table(dbf: &[u8]) -> Option> { } records.push(values); } - Some(RawTable { columns, records }) + Some(RawTable { columns, records, deleted }) } /// A date (`D`) field's value, `YYYYMMDD`: `Ok(None)` when it's blank or zeros, as writers leave diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 5cb86ab482..5cfd0f6b2e 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -3279,14 +3279,24 @@ fn new_column(ds: &Dataset, k: usize) -> Spec { } } +impl std::fmt::Display for Spec { + /// As a `.dbf` column is described: its type letter, width and, for a number, decimals + /// (`N(10,2)`, `C(24)`). + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ty = char::from(self.ty); + if matches!(self.ty, b'N' | b'F') { write!(f, "{ty}({},{})", self.len, self.decimals) } else { write!(f, "{ty}({})", self.len) } + } +} + /// The column field `k` (written as `name`) is written to over a file whose `.dbf` `style` /// describes: the file's column of that name where the field's kind and every value still fit -/// it, so the file keeps its widths and decimals; otherwise a new one ([`new_column`]), and -/// `true` when the file had a column of that name and kind that was too small for its values. -fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, bool) { +/// it, so the file keeps its widths and decimals; otherwise a new one ([`new_column`]), at least +/// as wide as the file's, and, when the file had a column of that name and kind that couldn't +/// hold the values, what changed (`'ACRES' N(10,2) to N(24,9)`). +fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, Option) { let new = new_column(ds, k); let (Some(field), Some(was)) = (ds.fields.get(k), style.columns.iter().find(|c| c.name == name.as_bytes())) else { - return (new, false); + return (new, None); }; let values = || ds.features.iter().filter_map(|r| r.attrs.get(k)).filter(|v| !v.is_null()); let ty = was.ty.to_ascii_uppercase(); @@ -3309,22 +3319,26 @@ fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, b (FieldType::Boolean, b'L') | (FieldType::Date, b'D') => true, (FieldType::Text | FieldType::DateTime, b'C') => values().all(|v| encoded_len(v) <= usize::from(was.len)), // Another kind of field now: a new column, and nothing of the old one to keep. - _ => return (new, false), + _ => return (new, None), }; if fits { - return (kept, false); + return (kept, None); } - // Text wider than the file's column: as wide as its longest value, in the file's code page. - if new.ty == b'C' { - let longest = values().map(encoded_len).max().unwrap_or(0); - return (Spec { len: u8::try_from(longest.clamp(usize::from(was.len), 254)).unwrap_or(254), ..new }, true); - } - (new, true) + let spec = if new.ty == b'C' { + // Text wider than the file's column: as wide as its longest value, in the file's code + // page, up to the 254 bytes written (a longer value is cut, which publishing refuses). + let longest = values().map(encoded_len).max().unwrap_or(0).min(254); + Spec { len: u8::try_from(longest).unwrap_or(254), ..new } + } else { + // A number with more digits or decimals: never narrower than the file's column. + Spec { len: new.len.max(was.len), ..new } + }; + (spec, Some(format!("'{}' {kept} to {spec}", dbf_notice_name(&field.name)))) } /// As [`shapefile_parts_in`], the `.dbf` written as `style` says when it's given (in the file's -/// code page, its columns kept where the values fit): also returns the names of the fields whose -/// column was too small for their values and was made wider. +/// code page, its columns kept where the values fit): also returns, for each field whose column +/// couldn't hold its values, how its column changed. pub(crate) fn shapefile_parts_styled( ds: &Dataset, crs: Crs, @@ -3339,17 +3353,15 @@ pub(crate) fn shapefile_parts_styled( Some(style) => TableWriterBuilder::with_encoding(style.encoding.decoder()), None => TableWriterBuilder::new(), }; - // Per field: its column (numbers too large for it become null), and the fields whose column - // was made wider than the file's. + // Per field: its column (numbers too large for it become null), and how the columns that + // couldn't hold their values changed. let mut specs = Vec::with_capacity(ds.fields.len()); let mut widened = Vec::new(); - for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { + for (k, n) in names.iter().enumerate() { let spec = match style { Some(style) => { - let (spec, wider) = kept_column(ds, k, n, style); - if wider { - widened.push(f.name.clone()); - } + let (spec, changed) = kept_column(ds, k, n, style); + widened.extend(changed); spec } None => new_column(ds, k), @@ -3496,6 +3508,22 @@ mod tests { use super::*; use std::io::Write; + /// A text column as wide as a `.dbf` allows (255 bytes, a width the writer never uses) with a + /// longer value gets a column of the 254 bytes written, not a crash. + #[test] + fn a_full_width_text_column_with_a_longer_value_is_written_at_254() { + let mut ds = Dataset::new("a", GeometryType::Point, vec![Field::new("NOTE", FieldType::Text)]); + ds.push(Some(Geometry::Point(Coord::new(0.0, 0.0))), vec![Value::Text("x".repeat(300))]); + let style = DbfStyle { + encoding: crate::dbf::DbfEncoding::Utf8, + ldid: 0, + columns: vec![crate::dbf::Column { name: b"NOTE".to_vec(), ty: b'C', len: 255, decimals: 0 }], + }; + let (spec, changed) = kept_column(&ds, 0, "NOTE", &style); + assert_eq!(spec.len, 254); + assert_eq!(changed.as_deref(), Some("'NOTE' C(255) to C(254)")); + } + /// A Shapefile export of a polygon layer holding a point and a feature without geometry writes /// the polygon and counts both left out, the point for its kind; a layer of polygons loses none. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 793caed29c..d2b4768492 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -132,6 +132,13 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result 0 { + return Err(refused( + path, + format!("{} of its records are marked deleted, and writing it again would remove them from the file", table.deleted), + )); + } let mut encoding_notes = Vec::new(); let ldid = original.dbf.get(29).copied().unwrap_or(0); let style = DbfStyle { @@ -156,6 +163,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result)], ext: &str| parts.iter().find(|(e, _)| *e == ext).map(|(_, b)| b.clone()).unwrap_or_default(); if let Some(why) = same_shapes(&original.shp, &part(&again, "shp"), 0.0) { @@ -189,11 +197,13 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result = widened.iter().take(5).map(String::as_str).collect(); + let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; notes.push(format!( - "{}: {} given wider columns than the file had, to hold their values: {}", + "{}: {} columns other than the file's, to hold their values: {}{more}", path.display(), - if widened.len() == 1 { "1 field was".to_string() } else { format!("{} fields were", widened.len()) }, - names_list(&widened) + if widened.len() == 1 { "1 field was given".to_string() } else { format!("{} fields were given", widened.len()) }, + shown.join(", ") )); } @@ -256,7 +266,8 @@ fn in_file<'a>(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Da let mut row = f.clone(); match held { Some(h) => row.geometry = h.geometry.clone(), - None => crate::io::project_features(std::slice::from_mut(&mut row), &t, c)?, + // Looks at Cancel without counting: the write that follows counts the features. + None => crate::io::project_each(std::slice::from_mut(&mut row), &t, c, false)?, } rows.push(row); } @@ -618,6 +629,23 @@ mod tests { assert!(e.contains("reads otherwise now") && e.contains("nothing was written"), "{e}"); } + /// A file with records marked deleted is refused: writing it again would drop them, shapes and + /// all. + #[test] + fn a_file_with_deleted_records_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + dbf[header] = b'*'; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("1 of its records are marked deleted") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), dbf); + } + /// A file changed since it was read is never written over. #[test] fn a_changed_file_is_refused() { @@ -714,7 +742,8 @@ mod tests { ds.features = rows.into(); let published = shapefile(&ds, &path, &sync).unwrap(); assert_eq!(published.notes.len(), 1, "{:?}", published.notes); - assert!(published.notes[0].contains("1 field was given wider columns") && published.notes[0].contains("'OWNER'")); + assert!(published.notes[0].contains("1 field was given columns other than the file's"), "{:?}", published.notes); + assert!(published.notes[0].contains("'OWNER' C(5) to C(24)"), "{:?}", published.notes); let (back, _) = crate::io::read_path(&path).unwrap(); assert_eq!(back.features.iter().next().unwrap().attrs[0], Value::Text("Smith and Jones Holdings".into())); } From 71e513845d22dd831e06eb1f396c110abe10d1b3 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 13:21:32 -0600 Subject: [PATCH 03/15] fix(data): publish compares numbers exactly and records the file as it reads A number is compared as the decimal its text says, not as an f64, so a value past 2^53 that a read rounds is refused rather than written back rounded. The record a publish leaves holds the digest of the file as it reads (Synced.read, saved only when it differs from the layer's), so a file that holds the layer as a Shapefile can (an empty text as no value) publishes again. A code page RustGIS reads as UTF-8 for want of a decoder is refused. Staged parts take the old parts' permissions before any content is written, and text is cut to 254 bytes in the file's code page. A refused start says so in the desktop app. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/io.rs | 71 +++++++++++++++++-- crates/data/src/publish.rs | 115 ++++++++++++++++++++++++++++--- crates/data/src/sync.rs | 14 ++++ crates/engine/src/cmd/layer.rs | 8 ++- crates/engine/src/cmd/publish.rs | 3 +- crates/ui-egui/src/lib.rs | 5 +- docs/user-guide/index.md | 25 ++++--- 7 files changed, 212 insertions(+), 29 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 5cfd0f6b2e..be0c205930 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -631,7 +631,7 @@ pub(crate) fn stage_shapefile(parts: &[(std::path::PathBuf, Vec)]) -> Result let mut written: Vec<(std::path::PathBuf, std::path::PathBuf)> = Vec::new(); let staged = parts.iter().try_for_each(|(target, bytes)| { let tmp = temp_beside(target)?; - let result = write_new(&tmp, bytes); + let result = write_new_like(&tmp, target, bytes); if result.is_ok() { written.push((tmp, target.clone())); } @@ -734,10 +734,37 @@ pub(crate) fn temp_beside(path: &Path) -> std::io::Result { /// Writes and syncs a new file (`create_new`, so an existing file or symlink is never followed); /// removes it on failure. +#[cfg(test)] fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { write_new_with(path, |w| w.write_all(bytes)) } +/// [`write_new`], the new file taking the mode of the file at `like` (on Unix, when there is one) +/// before any content is written: a private part (`0600`) stays private once the new file +/// replaces it, and its content is never readable by others meanwhile. A mode that can't be set +/// is an error, rather than a part made readable. +fn write_new_like(path: &Path, like: &Path, bytes: &[u8]) -> std::io::Result<()> { + #[cfg(unix)] + let mode = match std::fs::symlink_metadata(like) { + Ok(old) if !old.file_type().is_symlink() => Some(old.permissions()), + Ok(_) => None, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + #[cfg(not(unix))] + let _ = like; + write_new_with(path, |w| { + #[cfg(unix)] + if let Some(mode) = mode { + use std::os::unix::fs::PermissionsExt; + if std::fs::metadata(path)?.permissions().mode() != mode.mode() { + std::fs::set_permissions(path, mode)?; + } + } + w.write_all(bytes) + }) +} + /// [`write_new`] with the content written by `write`, through a buffer, as it goes. fn write_new_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { use std::io::Write; @@ -2582,6 +2609,30 @@ fn integral_numeric_to_integer(ds: &mut Dataset) { /// WGS 84 as ESRI WKT, for a Shapefile's `.prj`. pub(crate) const WGS84_PRJ: &str = "GEOGCS[\"GCS_WGS_1984\",DATUM[\"D_WGS_1984\",SPHEROID[\"WGS_1984\",6378137.0,298.257223563]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"; +/// The longest prefix of `s` whose text takes at most `max` bytes in `encoding`, cut at a +/// character boundary: what a `.dbf` in that code page holds of it. +fn cut_encoded<'a>(s: &'a str, max: usize, encoding: &crate::dbf::DbfEncoding) -> &'a str { + use shapefile::dbase::encoding::Encoding; + let encoder = encoding.decoder(); + let fits = |t: &str| encoder.encode(t).map_or(t.len(), |b| b.len()) <= max; + if fits(s) { + return s; + } + // The longest prefix that fits: a character takes at least one byte, so it is within `max` + // characters, and a shorter prefix never takes more bytes. + let ends: Vec = s.char_indices().map(|(i, _)| i).skip(1).take(max).chain([s.len()]).collect(); + let (mut lo, mut hi) = (0usize, ends.len()); + while lo < hi { + let mid = (lo + hi).div_ceil(2); + if ends.get(mid - 1).is_some_and(|&e| fits(s.get(..e).unwrap_or(""))) { + lo = mid; + } else { + hi = mid - 1; + } + } + lo.checked_sub(1).and_then(|i| ends.get(i)).and_then(|&e| s.get(..e)).unwrap_or("") +} + /// The longest prefix of `s` that fits in `max` bytes, cut at a character boundary. pub(crate) fn cut(s: &str, max: usize) -> &str { let mut end = s.len().min(max); @@ -3388,9 +3439,10 @@ pub(crate) fn shapefile_parts_styled( let (y, m, d) = d.ymd(); shapefile::dbase::Date::new(u32::try_from(d).ok()?, u32::try_from(m).ok()?, u32::try_from(y).ok()?).ok() })), - FieldType::Text | FieldType::DateTime => { - FieldValue::Character((!v.is_null()).then(|| cut(&v.to_string(), 254).to_string())) - } + FieldType::Text | FieldType::DateTime => FieldValue::Character((!v.is_null()).then(|| match style { + Some(style) => cut_encoded(&v.to_string(), 254, &style.encoding).to_string(), + None => cut(&v.to_string(), 254).to_string(), + })), }; rec.insert(n.clone(), value); } @@ -3508,6 +3560,17 @@ mod tests { use super::*; use std::io::Write; + /// Text is cut to what its code page holds in the bytes given, at a character boundary. + #[test] + fn text_is_cut_to_its_code_page_bytes() { + let cp1252 = crate::dbf::DbfEncoding::Standard(encoding_rs::WINDOWS_1252); + let e = "é".repeat(300); + assert_eq!(cut_encoded(&e, 254, &cp1252).chars().count(), 254); + assert_eq!(cut_encoded(&e, 254, &crate::dbf::DbfEncoding::Utf8).chars().count(), 127); + assert_eq!(cut_encoded("short", 254, &cp1252), "short"); + assert_eq!(cut_encoded("ab", 0, &cp1252), ""); + } + /// A text column as wide as a `.dbf` allows (255 bytes, a width the writer never uses) with a /// longer value gets a column of the 254 bytes written, not a crash. #[test] diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index d2b4768492..fff8e9dfc9 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -36,11 +36,13 @@ use std::path::{Path, PathBuf}; /// a tenth of a millimetre at ten thousand kilometres; anything more is refused. const PROJECTED_TOLERANCE: f64 = 1e-11; -/// A layer written back over its Shapefile: the files as they were written, to record as the -/// layer's new sync point, the features written, and what it says (fields given wider columns, -/// a `.prj` that now says another system). +/// A layer written back over its Shapefile: the files as they were written and the +/// [`crate::sync::rows_digest`] of the file as it reads now, to record as the layer's new sync +/// point, the features written, and what it says (fields given other columns, a `.prj` that now +/// says another system). pub struct Published { pub files: FileSync, + pub rows: u64, pub features: usize, pub notes: Vec, } @@ -146,11 +148,16 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Result = widened.iter().take(5).map(String::as_str).collect(); let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; @@ -246,7 +255,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Option { } /// A `.dbf` value as a value, for comparing what two files hold: text without the spaces or NULs -/// that pad it, a number as the number it says, a date or yes/no without the forms of empty. +/// that pad it, a number as the decimal it says exactly (no `f64`, which can't tell +/// 9007199254740993 from 9007199254740992), a date or yes/no without the forms of empty. #[derive(PartialEq)] enum Cell<'a> { Bytes(&'a [u8]), - Number(f64), + /// Negative, the whole digits without leading zeros, the decimals without trailing zeros. + Number(bool, &'a [u8], &'a [u8]), Empty, } +/// A number's text as its sign, whole digits and decimals with the zeros that don't count left +/// off (`-003.50` is `-3.5`; `-0` is `0`), or `None` when it isn't plain decimal digits. +fn decimal(text: &[u8]) -> Option<(bool, &[u8], &[u8])> { + let (negative, digits) = match text.split_first() { + Some((b'-', rest)) => (true, rest), + Some((b'+', rest)) => (false, rest), + _ => (false, text), + }; + let (whole, decimals) = match digits.iter().position(|b| *b == b'.') { + Some(at) => (digits.get(..at)?, digits.get(at + 1..)?), + None => (digits, &b""[..]), + }; + if whole.is_empty() && decimals.is_empty() || !whole.iter().chain(decimals).all(u8::is_ascii_digit) { + return None; + } + let whole = whole.get(whole.iter().position(|b| *b != b'0').unwrap_or(whole.len())..)?; + let decimals = decimals.get(..decimals.iter().rposition(|b| *b != b'0').map_or(0, |i| i + 1))?; + Some((negative && !(whole.is_empty() && decimals.is_empty()), whole, decimals)) +} + /// What value `bytes` of a column of type `ty` holds ([`Cell`]). fn cell(ty: u8, bytes: &[u8]) -> Cell<'_> { let padding = |b: &u8| *b == b' ' || *b == 0; @@ -392,8 +423,8 @@ fn cell(ty: u8, bytes: &[u8]) -> Cell<'_> { if trimmed.is_empty() || trimmed.iter().all(|b| *b == b'*') { return Cell::Empty; } - match std::str::from_utf8(trimmed).ok().and_then(|t| t.parse::().ok()) { - Some(x) => Cell::Number(x), + match decimal(trimmed) { + Some((negative, whole, decimals)) => Cell::Number(negative, whole, decimals), None => Cell::Bytes(trimmed), } } @@ -585,7 +616,7 @@ mod tests { fn read(path: &Path) -> (Dataset, Synced) { let (read, files) = FileSync::around(path, || crate::io::read_path(path).unwrap()).unwrap(); let rows = crate::sync::rows_digest(&read.0).unwrap(); - (read.0, Synced { files: files.unwrap(), rows }) + (read.0, Synced { files: files.unwrap(), rows, read: None }) } /// An edit is written back: the file reads as the layer, keeps its columns and `.prj`, and @@ -629,6 +660,68 @@ mod tests { assert!(e.contains("reads otherwise now") && e.contains("nothing was written"), "{e}"); } + /// A number past what an `f64` holds exactly, in a column that wasn't edited, is refused + /// rather than written back rounded. + #[test] + fn a_number_an_f64_rounds_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + // UNITS, an N(18,0) column, holding 2⁵³ + 1 in the first record. + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&dbf).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let at = header + 1 + usize::from(table.columns[0].len) + usize::from(table.columns[1].len); + dbf[at..at + 18].copy_from_slice(b" 9007199254740993"); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("table wouldn't be the same") && e.contains("9007199254740993"), "{e}"); + // Written the same in other forms, a number is the same. + assert!(cell(b'N', b" 003.50") == cell(b'N', b"3.5")); + assert!(cell(b'N', b"-0.00") == cell(b'N', b"0")); + assert!(cell(b'N', b"9007199254740993") != cell(b'N', b"9007199254740992")); + } + + /// A code page RustGIS reads as UTF-8 for want of a decoder (DOS French Canadian, by its + /// language driver) is refused: edited text would be written in another code page. + #[test] + fn a_code_page_rustgis_cant_write_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + std::fs::remove_file(path.with_extension("cpg")).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + dbf[29] = 0x6C; + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (ds, sync) = read(&path); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("code page 863") && e.contains("nothing was written"), "{e}"); + } + + /// Once published, a layer the file holds otherwise than the layer does (an empty text reads + /// back as no value) publishes again: its record holds the file as it reads. + #[test] + fn a_layer_the_file_reads_otherwise_publishes_again() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text(String::new()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + let rows_now = crate::sync::rows_digest(&ds).unwrap(); + assert_ne!(published.rows, rows_now, "the file reads the empty text as no value"); + let synced = Synced { files: published.files, rows: rows_now, read: Some(published.rows) }; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + shapefile(&ds, &path, &synced).unwrap(); + } + /// A file with records marked deleted is refused: writing it again would drop them, shapes and /// all. #[test] @@ -675,7 +768,7 @@ mod tests { ds.features = rows.clone().into(); let published = shapefile(&ds, &path, &sync).unwrap(); // The record of what was published, as the layer takes it. - let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&ds).unwrap() }; + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&ds).unwrap(), read: Some(published.rows) }; let dbf = std::fs::read(path.with_extension("dbf")).unwrap(); assert!(dbf.windows(5).any(|w| w == b"Mu\xf1oz"), "written in Windows-1252"); assert_eq!(std::fs::read(path.with_extension("cpg")).unwrap(), b"1252"); diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index 4fd3b41cce..2e0f36ebe1 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -32,6 +32,20 @@ pub struct Synced { pub files: FileSync, /// The layer's [`rows_digest`] then. pub rows: u64, + /// The [`rows_digest`] of the file as it reads then, where that isn't `rows`: after a + /// publish, the file holds the layer as a Shapefile can (an empty text reads as no value, a + /// column of whole numbers as integers, an edited shape in another system back through its + /// projection). Not saved when it is `rows`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub read: Option, +} + +impl Synced { + /// The [`rows_digest`] of the file as it read at the sync: what reading it now must give + /// for its rows that weren't edited to be the file's. + pub fn file_rows(&self) -> u64 { + self.read.unwrap_or(self.rows) + } } /// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 940bcac1e4..674202942e 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1926,9 +1926,11 @@ fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { ds.source = Some(full.display().to_string()); } report.synced = match files { - Some(files) => { - Some(rustgis_data::sync::Synced { files, rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))? }) - } + Some(files) => Some(rustgis_data::sync::Synced { + files, + rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))?, + read: None, + }), None => { report.messages.push(format!( "{path}: a part of it couldn't be read to record it, so RustGIS can't tell later whether the file changed since; add it again to compare it" diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 475b16a0c6..2090154fc1 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -79,7 +79,8 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result Date: Fri, 13 Nov 2026 15:22:30 -0600 Subject: [PATCH 04/15] fix(data): finish Shapefile publish fidelity checks --- crates/data/src/dbf.rs | 4 + crates/data/src/io.rs | 25 ++++-- crates/data/src/publish.rs | 136 +++++++++++++++++++++++++++---- crates/data/src/sync.rs | 16 +--- crates/engine/src/cmd/layer.rs | 8 +- crates/engine/src/cmd/publish.rs | 40 ++++++++- crates/engine/src/tests.rs | 34 ++++++++ docs/user-guide/index.md | 21 +++-- 8 files changed, 228 insertions(+), 56 deletions(-) diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index 3105637735..7ccffc7489 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -250,6 +250,10 @@ pub(crate) fn raw_table(dbf: &[u8]) -> Option> { return None; } let body = dbf.get(header_len..)?; + // Fewer records than the header counts: not a table this can compare. + if body.len() < count.checked_mul(record_len)? { + return None; + } let (mut records, mut deleted) = (Vec::new(), 0); for record in body.chunks_exact(record_len).take(count) { if record.first() == Some(&b'*') { diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index be0c205930..736ab2aaec 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -736,7 +736,7 @@ pub(crate) fn temp_beside(path: &Path) -> std::io::Result { /// removes it on failure. #[cfg(test)] fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { - write_new_with(path, |w| w.write_all(bytes)) + write_new_with(path, |_, w| w.write_all(bytes)) } /// [`write_new`], the new file taking the mode of the file at `like` (on Unix, when there is one) @@ -753,25 +753,29 @@ fn write_new_like(path: &Path, like: &Path, bytes: &[u8]) -> std::io::Result<()> }; #[cfg(not(unix))] let _ = like; - write_new_with(path, |w| { + write_new_with(path, |file, w| { + // Through the handle opened, not the path, which could name another file by now. #[cfg(unix)] if let Some(mode) = mode { use std::os::unix::fs::PermissionsExt; - if std::fs::metadata(path)?.permissions().mode() != mode.mode() { - std::fs::set_permissions(path, mode)?; + if file.metadata()?.permissions().mode() != mode.mode() { + file.set_permissions(mode)?; } } + #[cfg(not(unix))] + let _ = file; w.write_all(bytes) }) } -/// [`write_new`] with the content written by `write`, through a buffer, as it goes. -fn write_new_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { +/// [`write_new`] with the content written by `write`, through a buffer, as it goes; `write` is +/// also given the file opened, before anything is written to it. +fn write_new_with(path: &Path, write: impl FnOnce(&std::fs::File, &mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { use std::io::Write; let f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?; let res = (|| { let mut w = std::io::BufWriter::new(&f); - write(&mut w)?; + write(&f, &mut w)?; w.flush()?; drop(w); f.sync_all() @@ -2921,6 +2925,13 @@ pub fn shapefile_values_notes(ds: &Dataset, target: &str) -> Vec { notes } +/// The names of the `.dbf` columns a Shapefile written from `fields` has, in order +/// ([`dbf_names`]). +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn shapefile_column_names(fields: &[Field]) -> Vec { + dbf_names(fields) +} + /// dBase field names (at most 10 bytes, unique without regard to case) for the fields. fn dbf_names(fields: &[Field]) -> Vec { let mut out: Vec = Vec::with_capacity(fields.len()); diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index fff8e9dfc9..f0289fd68e 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -36,12 +36,15 @@ use std::path::{Path, PathBuf}; /// a tenth of a millimetre at ten thousand kilometres; anything more is refused. const PROJECTED_TOLERANCE: f64 = 1e-11; -/// A layer written back over its Shapefile: the files as they were written and the -/// [`crate::sync::rows_digest`] of the file as it reads now, to record as the layer's new sync -/// point, the features written, and what it says (fields given other columns, a `.prj` that now -/// says another system). +/// A layer written back over its Shapefile: the files as they were written, and the layer as the +/// file now reads (`read`: its rows in the same order, as a Shapefile holds them: an empty text as +/// no value, a column of whole numbers as integers, an edited shape in another system back through +/// its projection), to record as the layer's new sync point; the features written, and what it +/// says (fields given other columns or left out, a `.prj` that now says another system). pub struct Published { pub files: FileSync, + pub read: Dataset, + /// Digest of `read`, computed before placement so cancellation cannot interrupt completion. pub rows: u64, pub features: usize, pub notes: Vec, @@ -52,6 +55,37 @@ fn refused(path: &Path, why: impl std::fmt::Display) -> DataError { DataError::Format(format!("{}: {why}; nothing was written", path.display())) } +/// Holds an exclusive OS lock on a persistent sidecar through publication. The sidecar is +/// never unlinked: deleting it could let a third process lock a different inode while another +/// still holds the old one. Dropping the handle (including on process exit) releases the lock; +/// contention refuses immediately. This coordinates RustGIS publishers, not other programs. +fn source_lock(path: &Path) -> Result { + let target = path.canonicalize()?; + let mut name = std::ffi::OsString::from("."); + name.push(target.file_name().ok_or_else(|| refused(path, "it has no file name"))?); + name.push(".rustgis-lock"); + let lock_path = target.with_file_name(name); + if std::fs::symlink_metadata(&lock_path).is_ok_and(|m| !m.file_type().is_file()) { + return Err(refused(path, format!("{} isn't a regular lock file", lock_path.display()))); + } + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32); + } + let file = options.open(&lock_path).map_err(|e| refused(path, format!("couldn't open {}: {e}", lock_path.display())))?; + if !file.metadata()?.is_file() { + return Err(refused(path, "its lock isn't a regular file")); + } + file.try_lock().map_err(|e| match e { + std::fs::TryLockError::WouldBlock => refused(path, "another RustGIS publisher holds this source; try again after it finishes"), + std::fs::TryLockError::Error(e) => refused(path, format!("couldn't lock {}: {e}", lock_path.display())), + })?; + Ok(file) +} + /// The Shapefile at `path`'s parts as they are, read into memory, with its `.cpg`'s path. struct Original { shp: Vec, @@ -115,8 +149,11 @@ impl Original { /// and what it keeps. Its work grows with the file: it is hashed twice, read twice, written in /// memory twice and read back, the hashing and writing counting progress, and it stops once the /// run is cancelled (before the last part is placed, which puts the old file back). Reading the -/// file into memory and back are single steps without a check. +/// file into memory and back, and comparing the shapes, table and rows (one pass each, without +/// a check), are steps that run to their end before Cancel is seen. Hashing the prepared bytes +/// and writing/syncing each staged part likewise run as single steps. pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result { + let _lock = source_lock(path)?; let unchanged = |when: &str| -> Result<()> { let changed = synced.files.changed(path)?; if changed.is_empty() { @@ -157,7 +194,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Result = + table.columns.iter().filter(|c| !kept_names.iter().any(|n| n.as_bytes() == c.name.as_slice())).map(column).collect(); + if !gone.is_empty() { + let shown: Vec<&str> = gone.iter().take(5).map(String::as_str).collect(); + let more = if gone.len() > 5 { format!(" and {} more", gone.len() - 5) } else { String::new() }; + notes.push(format!( + "{}: {} left out, as no field of the layer has its name now: {}{more}", + path.display(), + if gone.len() == 1 { "1 column of the file was".to_string() } else { format!("{} columns of the file were", gone.len()) }, + shown.join(", ") + )); + } if !widened.is_empty() { let shown: Vec<&str> = widened.iter().take(5).map(String::as_str).collect(); let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; @@ -236,6 +286,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Result(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Da let at: std::collections::HashMap = read.iter().enumerate().map(|(i, f)| (f.id, i)).collect(); let mut rows = Vec::with_capacity(ds.features.len()); for f in ds.features.iter() { + // A shape taken as the file holds it is quick, but a layer of many is not. + crate::progress::check()?; let held = at.get(&f.id).and_then(|&i| read.get(i).filter(|r| r.geometry == f.geometry).and(raw.get(i))); let mut row = f.clone(); match held { @@ -616,7 +669,7 @@ mod tests { fn read(path: &Path) -> (Dataset, Synced) { let (read, files) = FileSync::around(path, || crate::io::read_path(path).unwrap()).unwrap(); let rows = crate::sync::rows_digest(&read.0).unwrap(); - (read.0, Synced { files: files.unwrap(), rows, read: None }) + (read.0, Synced { files: files.unwrap(), rows }) } /// An edit is written back: the file reads as the layer, keeps its columns and `.prj`, and @@ -638,9 +691,29 @@ mod tests { assert_eq!(std::fs::read(path.with_extension("prj")).unwrap(), prj); assert_eq!(crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns, columns); assert!(published.files.changed(&path).unwrap().is_empty()); - // Nothing left beside it. + // Only the reusable lock sidecar remains; no staged or backup parts. let names: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); - assert!(names.iter().all(|n| !n.starts_with('.')), "{names:?}"); + assert!(names.iter().all(|n| !n.starts_with('.') || n == ".parcels.shp.rustgis-lock"), "{names:?}"); + } + + /// An independent handle represents a concurrent publisher. Its lock refuses a publish + /// without touching the source, and is reusable after closing, with no stale-lock cleanup. + #[test] + fn a_concurrent_publisher_is_refused_until_the_lock_is_released() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let held = source_lock(&path).unwrap(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("another RustGIS publisher") && e.contains("nothing was written"), "{e}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + drop(held); + shapefile(&ds, &path, &sync).unwrap(); + // A second publisher that loaded the old baseline now sees the committed change. + assert!(shapefile(&ds, &path, &sync).err().unwrap().to_string().contains("changed since the layer was read")); } /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a @@ -702,8 +775,8 @@ mod tests { assert!(e.contains("code page 863") && e.contains("nothing was written"), "{e}"); } - /// Once published, a layer the file holds otherwise than the layer does (an empty text reads - /// back as no value) publishes again: its record holds the file as it reads. + /// Once published, the layer as the file reads it (an empty text as no value) is its record: + /// it publishes again. #[test] fn a_layer_the_file_reads_otherwise_publishes_again() { let dir = tempfile::tempdir().unwrap(); @@ -713,15 +786,42 @@ mod tests { rows[0].attrs[0] = Value::Text(String::new()); ds.features = rows.into(); let published = shapefile(&ds, &path, &sync).unwrap(); - let rows_now = crate::sync::rows_digest(&ds).unwrap(); - assert_ne!(published.rows, rows_now, "the file reads the empty text as no value"); - let synced = Synced { files: published.files, rows: rows_now, read: Some(published.rows) }; + assert_eq!(published.read.features.iter().next().unwrap().attrs[0], Value::Null); + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&published.read).unwrap() }; + let mut ds = published.read; let mut rows: Vec<_> = ds.features.iter().cloned().collect(); rows[1].attrs[0] = Value::Text("Jonas".into()); ds.features = rows.into(); shapefile(&ds, &path, &synced).unwrap(); } + /// A field deleted from the layer leaves the file's column out, and says so. + #[test] + fn a_deleted_field_leaves_its_column_out_and_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + ds.fields.remove(2); + let rows: Vec<_> = ds + .features + .iter() + .cloned() + .map(|mut f| { + f.attrs.remove(2); + f + }) + .collect(); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!( + published.notes.iter().any(|n| n.contains("1 column of the file was left out") && n.contains("'UNITS' N(18,0)")), + "{:?}", + published.notes + ); + let columns = crate::dbf::raw_table(&std::fs::read(path.with_extension("dbf")).unwrap()).unwrap().columns; + assert_eq!(columns.len(), 2); + } + /// A file with records marked deleted is refused: writing it again would drop them, shapes and /// all. #[test] @@ -768,7 +868,7 @@ mod tests { ds.features = rows.clone().into(); let published = shapefile(&ds, &path, &sync).unwrap(); // The record of what was published, as the layer takes it. - let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&ds).unwrap(), read: Some(published.rows) }; + let synced = Synced { files: published.files, rows: crate::sync::rows_digest(&published.read).unwrap() }; let dbf = std::fs::read(path.with_extension("dbf")).unwrap(); assert!(dbf.windows(5).any(|w| w == b"Mu\xf1oz"), "written in Windows-1252"); assert_eq!(std::fs::read(path.with_extension("cpg")).unwrap(), b"1252"); diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index 2e0f36ebe1..d588382abc 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -30,22 +30,8 @@ pub struct FileStamp { pub struct Synced { /// The files, as they were. pub files: FileSync, - /// The layer's [`rows_digest`] then. + /// The layer's [`rows_digest`] then: the file's as it read then too. pub rows: u64, - /// The [`rows_digest`] of the file as it reads then, where that isn't `rows`: after a - /// publish, the file holds the layer as a Shapefile can (an empty text reads as no value, a - /// column of whole numbers as integers, an edited shape in another system back through its - /// projection). Not saved when it is `rows`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub read: Option, -} - -impl Synced { - /// The [`rows_digest`] of the file as it read at the sync: what reading it now must give - /// for its rows that weren't edited to be the file's. - pub fn file_rows(&self) -> u64 { - self.read.unwrap_or(self.rows) - } } /// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 674202942e..940bcac1e4 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1926,11 +1926,9 @@ fn read_synced(path: &str, read: impl FnOnce() -> WholeRead) -> WholeRead { ds.source = Some(full.display().to_string()); } report.synced = match files { - Some(files) => Some(rustgis_data::sync::Synced { - files, - rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))?, - read: None, - }), + Some(files) => { + Some(rustgis_data::sync::Synced { files, rows: rustgis_data::sync::rows_digest(&ds).map_err(|e| format!("{path}: {e}"))? }) + } None => { report.messages.push(format!( "{path}: a part of it couldn't be read to record it, so RustGIS can't tell later whether the file changed since; add it again to compare it" diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 2090154fc1..74c6e956e1 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -40,8 +40,10 @@ fn record(l: &rustgis_map::Layer) -> Result<(&rustgis_data::sync::Synced, &str), /// `p` names back over the Shapefile it was read from ([`rustgis_data::publish::shapefile`], /// which says what it checks first), when it has edits the file lacks. Its work grows with the /// file (hashed, read, written twice in memory and read back): the desktop app runs it on a -/// thread of its own with progress and Cancel. Once written, the layer's record is of the files -/// written and the rows published, so it reads `inSync`; the project then has a change to save. +/// thread of its own with progress and Cancel. Once written, the layer holds its rows as the file +/// reads them (the same, but for what a Shapefile can't tell apart: an empty text and no value), +/// and its record is of the files written and those rows, so it reads `inSync`; the project then +/// has a change to save. pub(crate) fn publish_work(s: &Session, p: &Value) -> Result { let id = s.layer_id(p)?; let l = s.map().layer(id).ok_or("the layer no longer exists")?; @@ -79,8 +81,25 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result = dataset.features.iter().map(|f| f.id).collect(); + held.features = published + .read + .features + .iter() + .zip(&ids) + .map(|(f, id)| rustgis_data::Feature { id: *id, ..f.clone() }) + .collect::>() + .into(); + held.fields = published.read.fields.clone(); + // Already hashed before the files were placed: Cancel must not leave a successful + // write without its sync record. Feature ids do not contribute to the digest. + let read = published.rows; + // Unchanged when the file reads as the layer is. + let held = (read != rows).then_some(held); + let new = rustgis_data::sync::Synced { files: published.files, rows: read }; let (said, written_path) = (path.clone(), Some(path.clone())); let then: Then = Box::new(move |s: &mut Session, _: &mut Value| { // The record follows the file: the layer read from it takes the record of what @@ -89,6 +108,19 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result { l.synced = Some(new); + // Not edited meanwhile: it takes what the file holds (an empty text as no + // value, whole numbers as integers). Edited, it keeps its edits, which + // the file then lacks. + if let Some(held) = held + && l.dataset.features.shares(&dataset.features) + && l.dataset.fields == dataset.fields + && l.dataset.crs == dataset.crs + { + l.dataset.features = held.features; + l.dataset.fields = held.fields; + l.touch_geometry(); + s.revision = s.revision.wrapping_add(1); + } s.mark_unsaved(); } _ => s.warn(format!("{said} was written, but {name} changed while it was: add the file again to compare them")), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ce43066456..fdc909b816 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -44755,6 +44755,40 @@ fn a_layer_publishes_its_edits_to_its_shapefile() { assert_eq!(state(&mut s), "edited"); s.run("edit.redo", &Value::Null).unwrap(); assert_eq!(state(&mut s), "inSync"); + // An empty text, which the file holds as no value: the layer takes it as the file reads it, + // and is in step with it. + s.run("edit.setValue", &json!({"layer": id, "row": 2, "field": "name", "value": ""})).unwrap(); + s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(state(&mut s), "inSync"); + let l = s.map().layer(id as u32).unwrap(); + let name = l.dataset.fields.iter().position(|f| f.name == "name").unwrap(); + assert!(l.dataset.features.get(2).unwrap().attrs[name].is_null()); + // Cancel arriving after the final .shp rename cannot turn a committed publish into an + // error or leave its old sync record. The probe observes the real file, not an internal hook. + struct CancelAfterPlacement { + path: std::path::PathBuf, + before: Vec, + } + impl rustgis_data::progress::Probe for CancelAfterPlacement { + /// Changes only after the last .shp rename, the publication commit point. + fn cancelled(&self) -> bool { + std::fs::read(&self.path).is_ok_and(|bytes| bytes != self.before) + } + fn begin(&self, _: u64) {} + fn advance(&self, _: u64) {} + } + s.run("edit.setGeometry", &json!({"layer": id, "row": 0, "geometry": {"type": "Point", "coordinates": [1, 2]}})).unwrap(); + { + let probe = std::sync::Arc::new(CancelAfterPlacement { path: path.clone(), before: std::fs::read(&path).unwrap() }); + let _installed = rustgis_data::progress::install(Some(probe.clone())); + // Run the real worker directly: synchronous Session dispatch installs its own probe. + let work = crate::cmd::publish::publish_work(&s, &json!({"layer": id})).unwrap(); + let mut wrote = work().unwrap(); + assert_eq!(wrote.answer["published"], true); + wrote.then.take().unwrap()(&mut s, &mut wrote.answer).unwrap(); + assert!(rustgis_data::progress::Probe::cancelled(probe.as_ref()), "the probe saw the committed shape"); + } + assert_eq!(state(&mut s), "inSync"); // Changed since: refused, the file left as it was. s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Mine"})).unwrap(); let mut theirs = dbf(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index f1f03246da..40a037ab5d 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -406,6 +406,8 @@ refuses, with nothing written, when: - the file changed since the layer was read (`source.syncState` would say `sourceChanged` or `both`): writing would overwrite that change; +- the layer was read otherwise than the file reads now (by an older RustGIS), so rows that + weren't edited could change; - the file, read and written again, wouldn't be the same file: shapes with Z or M values, records without a shape, multipatches, records marked deleted, text with spaces before it, numbers with more digits than RustGIS holds exactly, columns of a type RustGIS doesn't write (memo, binary), @@ -414,25 +416,30 @@ refuses, with nothing written, when: bytes, text in characters the file's code page lacks. The file keeps its code page (its `.cpg` and the language driver in its `.dbf`), each column's -width, decimals and type where the values still fit, and its `.prj`. A column too narrow for an -edited value is made wider, and a notice says which. A layer read in another coordinate system -than its `.prj` names gets a `.prj` for that system, said too. A shape that wasn't edited is +width, decimals and type where the values still fit, and its `.prj`. A column that can't hold +an edited value gets one that can, and a notice names it with its old and new form; a field +deleted or renamed in the layer leaves its column out of the file, and a notice names it too. A layer read in another +coordinate system than its `.prj` names gets a `.prj` for that system, said too. A shape that wasn't edited is written with the coordinates the file holds, so a projected file's untouched shapes stay as they were to the bit. An edited shape in a projected system is converted from longitude/latitude, as RustGIS holds it, and read back: publishing refuses one that would move by more than one part in 10¹¹ of its coordinates (a tenth of a millimetre at ten thousand kilometres). The new parts are written beside the old ones, with the old parts' file permissions, checked, -and then renamed into place, the `.shp` last. If one fails to land, the old parts are put back; +and then renamed into place, the `.shp` last. An OS lock prevents two RustGIS processes from +publishing the same source at once; a competing publish refuses immediately. The empty +`.filename.shp.rustgis-lock` sidecar stays beside the source for reuse; the OS releases its lock +when the publisher finishes or exits. Other programs do not follow this locking protocol. If one fails to land, the old parts are put back; should putting one back fail too, the message names the temporary file that holds it, beside the Shapefile, so it can be renamed back by hand. Sidecars RustGIS doesn't write, such as `.shp.xml` metadata, are left alone. The spatial indexes that would no longer match (`.qix`, `.sbn`, `.sbx`) are removed where they can be; one that can't be (held open by another program) is left, and should be deleted before that program uses the file. -Once published, the layer reads `inSync`, and the project has a change to save (its record of -the file). Undo doesn't change the file: undoing an edit made before publishing shows the layer -as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't +Once published, the layer holds its rows as the file reads them, which differs only where a +Shapefile can't tell values apart (an empty text reads as no value, a column of whole numbers as +integers), so it reads `inSync`; the project has a change to save. Undo doesn't change the file: +undoing an edit made before publishing shows the layer as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't be published yet. On the desktop, make the layer active and choose **Publish to Source** in Command Search; it runs From a23b1460373b9117d861bb94a1662d766d8bf164 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 16:07:22 -0600 Subject: [PATCH 05/15] fix(data): preserve linked sources and shared publisher locks --- crates/data/src/publish.rs | 101 ++++++++++++++++++++++++++++++++++++- docs/user-guide/index.md | 7 ++- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index f0289fd68e..b080756e4e 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -55,11 +55,28 @@ fn refused(path: &Path, why: impl std::fmt::Display) -> DataError { DataError::Format(format!("{}: {why}; nothing was written", path.display())) } +/// Placement replaces a directory entry, so following a linked part while reading is unsafe. +fn refuse_link(source: &Path, part: &Path) -> Result<()> { + match std::fs::symlink_metadata(part) { + Ok(m) if m.file_type().is_symlink() => Err(refused( + source, + format!( + "{} is a symbolic link; add the actual source files instead so publishing keeps the link and its target unchanged", + part.display() + ), + )), + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } +} + /// Holds an exclusive OS lock on a persistent sidecar through publication. The sidecar is /// never unlinked: deleting it could let a third process lock a different inode while another /// still holds the old one. Dropping the handle (including on process exit) releases the lock; /// contention refuses immediately. This coordinates RustGIS publishers, not other programs. fn source_lock(path: &Path) -> Result { + refuse_link(path, path)?; let target = path.canonicalize()?; let mut name = std::ffi::OsString::from("."); name.push(target.file_name().ok_or_else(|| refused(path, "it has no file name"))?); @@ -69,13 +86,41 @@ fn source_lock(path: &Path) -> Result { return Err(refused(path, format!("{} isn't a regular lock file", lock_path.display()))); } let mut options = std::fs::OpenOptions::new(); - options.read(true).write(true).create(true).truncate(false); + options.read(true).write(true).create_new(true); #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.mode(0o600).custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32); } - let file = options.open(&lock_path).map_err(|e| refused(path, format!("couldn't open {}: {e}", lock_path.display())))?; + let opened = match options.open(&lock_path) { + Ok(file) => { + // The empty sidecar contains no source data. Any account sharing the directory + // must be able to lock it, regardless of source groups, ACLs or a private umask. + // Unix exclusive locks need only read access; set modes through the open handle. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o444))?; + } + Ok(file) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + options.create_new(false); + let existing = options.open(&lock_path); + #[cfg(unix)] + let existing = match existing { + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + // Unix advisory exclusive locks also work on read-only handles. + options.write(false); + options.open(&lock_path) + } + result => result, + }; + existing + } + Err(e) => Err(e), + }; + let file = opened.map_err(|e| refused(path, format!("couldn't open {}: {e}", lock_path.display())))?; if !file.metadata()?.is_file() { return Err(refused(path, "its lock isn't a regular file")); } @@ -155,6 +200,10 @@ impl Original { pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result { let _lock = source_lock(path)?; let unchanged = |when: &str| -> Result<()> { + refuse_link(path, path)?; + for ext in ["shx", "dbf", "prj"].into_iter().chain(crate::dbf::CPG_SPELLINGS) { + refuse_link(path, &path.with_extension(ext))?; + } let changed = synced.files.changed(path)?; if changed.is_empty() { return Ok(()); @@ -696,6 +745,34 @@ mod tests { assert!(names.iter().all(|n| !n.starts_with('.') || n == ".parcels.shp.rustgis-lock"), "{names:?}"); } + /// Reads follow symlinks, but replacement would sever them and leave the target stale. + #[cfg(unix)] + #[test] + fn linked_shapefile_parts_are_refused_without_replacing_the_link_or_target() { + for ext in ["shp", "shx", "dbf", "prj", "CPG"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + if ext == "CPG" { + std::fs::write(path.with_extension(ext), b"UTF-8").unwrap(); + } + let part = path.with_extension(ext); + let target = dir.path().join(format!("original.{ext}")); + std::fs::rename(&part, &target).unwrap(); + std::os::unix::fs::symlink(&target, &part).unwrap(); + let before = std::fs::read(&target).unwrap(); + let (mut ds, sync) = read(&path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().expect("linked part must be refused").to_string(); + assert!(e.contains("symbolic link") && e.contains("nothing was written"), "{ext}: {e}"); + assert_eq!(std::fs::read_link(&part).unwrap(), target); + assert_eq!(std::fs::read(&target).unwrap(), before); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + } + /// An independent handle represents a concurrent publisher. Its lock refuses a publish /// without touching the source, and is reusable after closing, with no stale-lock cleanup. #[test] @@ -716,6 +793,26 @@ mod tests { assert!(shapefile(&ds, &path, &sync).err().unwrap().to_string().contains("changed since the layer was read")); } + /// A shared source's empty lock is readable across accounts, even under a private umask. + /// Its read-only handle still excludes a second publisher on Unix. + #[cfg(unix)] + #[test] + fn shared_source_locks_are_readable_and_can_be_locked_read_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).unwrap(); + let held = source_lock(&path).unwrap(); + let lock_path = dir.path().join(".parcels.shp.rustgis-lock"); + assert_eq!(std::fs::metadata(&lock_path).unwrap().permissions().mode() & 0o777, 0o444); + drop(held); + let held = source_lock(&path).unwrap(); + assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); + drop(held); + source_lock(&path).unwrap(); + } + /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a /// project that couldn't hold a value) is refused: its unedited rows aren't the file's. #[test] diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 40a037ab5d..fdfe454d2c 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -425,11 +425,16 @@ were to the bit. An edited shape in a projected system is converted from longitu RustGIS holds it, and read back: publishing refuses one that would move by more than one part in 10¹¹ of its coordinates (a tenth of a millimetre at ten thousand kilometres). +Publishing refuses symbolic links in any source part: add the actual source files instead. +This keeps each link and its target unchanged. + The new parts are written beside the old ones, with the old parts' file permissions, checked, and then renamed into place, the `.shp` last. An OS lock prevents two RustGIS processes from publishing the same source at once; a competing publish refuses immediately. The empty `.filename.shp.rustgis-lock` sidecar stays beside the source for reuse; the OS releases its lock -when the publisher finishes or exits. Other programs do not follow this locking protocol. If one fails to land, the old parts are put back; +when the publisher finishes or exits. On Unix, this empty file is readable across accounts, +so users sharing the source can lock it without write access to the sidecar. Other programs do +not follow this locking protocol. If one fails to land, the old parts are put back; should putting one back fail too, the message names the temporary file that holds it, beside the Shapefile, so it can be renamed back by hand. Sidecars RustGIS doesn't write, such as `.shp.xml` metadata, are left alone. The spatial indexes that would no longer match (`.qix`, `.sbn`, `.sbx`) From e99726fd8ccd754e59a3507a61ac86c4ae247ff2 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 17:03:34 -0600 Subject: [PATCH 06/15] fix(publish): report column changes and preserve writable locks --- crates/assistant/src/tools.rs | 14 +++++- crates/data/src/io.rs | 6 +-- crates/data/src/publish.rs | 82 ++++++++++++++++++++++++-------- crates/engine/src/cmd/publish.rs | 4 +- crates/engine/src/lib.rs | 12 ++++- crates/engine/src/project_log.rs | 6 +-- crates/engine/src/tests.rs | 3 ++ docs/user-guide/index.md | 8 ++-- 8 files changed, 102 insertions(+), 33 deletions(-) diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 0b2e2eb1dc..b7d0741d40 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -648,6 +648,7 @@ fn words(a: &Value) -> Result, String> { Ok(words) } +/// Ranks command descriptions and reports project changes independently of undo snapshots. fn search_commands(s: &Session, a: &Value) -> Result { let words = words(a)?; let limit = limit_arg(a, 12, 40); @@ -664,7 +665,7 @@ fn search_commands(s: &Session, a: &Value) -> Result { (c.id.to_lowercase(), c.label.to_lowercase(), c.location.to_lowercase(), c.params.to_lowercase()); let sc = score(&words, &[&id, &label, &loc, ¶ms]); (sc > 0).then(|| { - (sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.mutates})) + (sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.changes_project()})) }) }) .collect(); @@ -1090,6 +1091,17 @@ mod tests { assert_eq!(data["[redacted]"], "private field"); } + /// Command discovery distinguishes publishing's project change from a read-only query. + #[test] + fn publishing_reports_a_project_change_in_command_search() { + let s = Session::new(rustgis_engine::Project::default()); + for (id, changes) in [("source.publish", true), ("source.syncState", false)] { + let out = search_commands(&s, &json!({"query":id, "limit":40})).unwrap(); + let command = out["commands"].as_array().unwrap().iter().find(|c| c["id"] == id).unwrap(); + assert_eq!(command["changesProject"], changes, "{id}"); + } + } + #[test] fn wrappers_after_a_prefix_are_made_plain_too() { let origin = "https://assistant-login.example"; diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 736ab2aaec..c5a9a043b6 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -3353,7 +3353,7 @@ impl std::fmt::Display for Spec { /// The column field `k` (written as `name`) is written to over a file whose `.dbf` `style` /// describes: the file's column of that name where the field's kind and every value still fit /// it, so the file keeps its widths and decimals; otherwise a new one ([`new_column`]), at least -/// as wide as the file's, and, when the file had a column of that name and kind that couldn't +/// as wide as the file's, and, when a column of that name changes kind or can't /// hold the values, what changed (`'ACRES' N(10,2) to N(24,9)`). fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, Option) { let new = new_column(ds, k); @@ -3380,8 +3380,8 @@ fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, O }), (FieldType::Boolean, b'L') | (FieldType::Date, b'D') => true, (FieldType::Text | FieldType::DateTime, b'C') => values().all(|v| encoded_len(v) <= usize::from(was.len)), - // Another kind of field now: a new column, and nothing of the old one to keep. - _ => return (new, None), + // Another kind of field now: say how its physical column changes too. + _ => return (new, Some(format!("'{}' {kept} to {new}", dbf_notice_name(&field.name)))), }; if fits { return (kept, None); diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index b080756e4e..c2fa65e23f 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -94,29 +94,19 @@ fn source_lock(path: &Path) -> Result { } let opened = match options.open(&lock_path) { Ok(file) => { - // The empty sidecar contains no source data. Any account sharing the directory - // must be able to lock it, regardless of source groups, ACLs or a private umask. - // Unix exclusive locks need only read access; set modes through the open handle. + // The empty sidecar contains no source data. Accounts sharing the directory need + // read/write access regardless of source groups, ACLs or a private umask: NFS + // exclusive locks require a writable handle. Set modes through the open handle. #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - file.set_permissions(std::fs::Permissions::from_mode(0o444))?; + file.set_permissions(std::fs::Permissions::from_mode(0o666))?; } Ok(file) } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { options.create_new(false); - let existing = options.open(&lock_path); - #[cfg(unix)] - let existing = match existing { - Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { - // Unix advisory exclusive locks also work on read-only handles. - options.write(false); - options.open(&lock_path) - } - result => result, - }; - existing + options.open(&lock_path) } Err(e) => Err(e), }; @@ -793,11 +783,12 @@ mod tests { assert!(shapefile(&ds, &path, &sync).err().unwrap().to_string().contains("changed since the layer was read")); } - /// A shared source's empty lock is readable across accounts, even under a private umask. - /// Its read-only handle still excludes a second publisher on Unix. + /// A shared source's empty lock is writable across accounts, even under a private umask. + /// Reopening it keeps a writable handle (required by NFS) and excludes a second publisher. #[cfg(unix)] #[test] - fn shared_source_locks_are_readable_and_can_be_locked_read_only() { + fn shared_source_locks_are_writable_and_reusable() { + use std::io::Write; use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("parcels.shp"); @@ -805,9 +796,11 @@ mod tests { std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).unwrap(); let held = source_lock(&path).unwrap(); let lock_path = dir.path().join(".parcels.shp.rustgis-lock"); - assert_eq!(std::fs::metadata(&lock_path).unwrap().permissions().mode() & 0o777, 0o444); + assert_eq!(std::fs::metadata(&lock_path).unwrap().permissions().mode() & 0o777, 0o666); drop(held); - let held = source_lock(&path).unwrap(); + let mut held = source_lock(&path).unwrap(); + // A zero-byte write checks the descriptor's access mode without storing source data. + assert_eq!(held.write(&[]).unwrap(), 0); assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); drop(held); source_lock(&path).unwrap(); @@ -1021,6 +1014,55 @@ mod tests { assert_eq!(crate::io::read_path(&path).unwrap().0.features.iter().next().unwrap().attrs[1], Value::Float(123456.12345678)); } + /// Converting a text field to a date changes its physical column, which publishing says. + #[test] + fn a_converted_field_names_its_old_and_new_column_types() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut source = parcels(); + let mut rows: Vec<_> = source.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Text("2025-01-02".into()); + } + source.features = rows.into(); + let (mut ds, sync) = written(&source, &path); + ds.fields[0].ty = FieldType::Date; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Date(crate::Date::parse("2025-01-02").unwrap()); + } + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + assert!(published.notes[0].contains("'OWNER' C(10) to D(8)"), "{:?}", published.notes); + let bytes = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&bytes).unwrap(); + assert_eq!(table.columns[0].ty, b'D'); + } + + /// The writer uses UTF-8 name bytes even with a legacy code page: a rewrite is refused + /// before the differing name or width can reach the source. + #[test] + fn a_legacy_non_ascii_column_name_refuses_the_rewrite_without_changing_the_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + std::fs::write(path.with_extension("cpg"), "1252").unwrap(); + let mut bytes = std::fs::read(path.with_extension("dbf")).unwrap(); + bytes[32..43].fill(0); + bytes[32..37].copy_from_slice(b"Se\xf1or"); + std::fs::write(path.with_extension("dbf"), &bytes).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.fields[0].name, "Señor"); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().unwrap().to_string(); + assert!(e.contains("its columns would change") && e.contains("nothing was written"), "{e}"); + assert_eq!(std::fs::read(path.with_extension("dbf")).unwrap(), bytes); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + /// A widened column is said; a layer in another system gets a `.prj` for it, said too. #[test] fn a_wider_value_widens_its_column_and_says_so() { diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 74c6e956e1..a08cc96de9 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -4,12 +4,14 @@ use crate::{CmdResult, CommandSpec as C, Session}; use serde_json::Value; +/// File inspection, and publishing with an explicit edit effect but no automatic undo step: +/// undo cannot reverse the external write, so its committed sync record follows the file. pub fn commands() -> Vec { vec![ C::new("source.syncState", "Source File State", "Data › Data Source", sync_state).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile or .geojson file"} — whether it has edits its file lacks ("edited"), the file changed since it was read ("sourceChanged"), both ("both") or neither ("inSync"), and which of the file's parts changed"#, ), - C::new("source.publish", "Publish to Source", "Data › Data Source", |s, p| super::exports::write_here(s, p, "source.publish")).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).view().params( + C::new("source.publish", "Publish to Source", "Data › Data Source", |s, p| super::exports::write_here(s, p, "source.publish")).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files).with(crate::policy::Effect::Edits)).view().params( r#"{"layer"?: "name|id (default: the active layer): one read whole from a Shapefile"} — writes the layer's edits back over the Shapefile it was read from, keeping its code page, columns and .prj; refused, with nothing written, when the file changed since it was read or can't hold the layer as it is (Z or M values, features without shapes, text its code page lacks)"#, ), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a92301683a..71465a53cd 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -86,7 +86,8 @@ pub struct CommandSpec { pub shortcut: &'static str, /// Parameter description for scripts and agents. pub params: &'static str, - /// Whether it changes the project (and so takes an undo snapshot). + /// Whether it takes an automatic project-edit transaction (undo snapshot and dirty marking). + /// Changes committed without that transaction declare `Effect::Edits` explicitly. pub mutates: bool, /// `None` = enabled; `Some(reason)` = disabled. pub enabled: fn(&Session) -> Option<&'static str>, @@ -111,6 +112,12 @@ impl CommandSpec { param_keys(self.params) } + /// Whether it can change the project, including external commits that update metadata + /// without an automatic undo snapshot. Logging and command discovery use this contract. + pub const fn changes_project(&self) -> bool { + self.mutates || self.effects.contains(policy::Effect::Edits) + } + /// Whether the parameter description includes a required top-level input. pub fn needs_input(&self) -> bool { needs_input(self.params) @@ -163,7 +170,8 @@ impl CommandSpec { self.params = p; self } - /// Doesn't change the project (no undo snapshot, not dirtying). + /// Suppresses automatic undo snapshots and dirty marking. Usually read-only; a command + /// committing external changes can update metadata itself and declares `Effect::Edits`. pub const fn view(mut self) -> Self { self.mutates = false; self diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index f81e00644f..88ec45b8e4 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -35,8 +35,8 @@ pub(crate) fn now_ms() -> u64 { } /// Whether command `id`, run by itself with `params`, goes into the log: one that changes the -/// project (all but those built with `.view()`; a plugin's when its manifest asks for the Edit -/// permission), undo and redo, a save, the log's own clearing and switching, and a command whose +/// project (automatic edits or an explicit `Edits` effect; a plugin's when its manifest asks for +/// the Edit permission), undo and redo, a save, the log's own clearing and switching, and a command whose /// parameters name a file, folder, address or connection (an export), as the plugin permission /// gate reads them. Reading and looking around (listing, describing, moving the view) is left /// out: agents, scripts and plugins call those hundreds of times, and they would push the changes @@ -60,7 +60,7 @@ pub(crate) fn is_logged(s: &Session, id: &str, params: &Value) -> bool { rustgis_plugins::command(id).is_some_and(|c| c.mutates()) } else { match s.registry.get(id) { - Some(spec) => spec.mutates, + Some(spec) => spec.changes_project(), None => return false, } }; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index fdc909b816..a1b17919fd 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -44743,7 +44743,10 @@ fn a_layer_publishes_its_edits_to_its_shapefile() { assert_eq!(dbf(), before, "nothing to write"); s.run("edit.setValue", &json!({"layer": id, "row": 0, "field": "name", "value": "Renamed"})).unwrap(); s.mark_saved(); + let undo_before_publish = s.undo_depth(); let out = s.run("source.publish", &json!({"layer": id})).unwrap(); + assert_eq!(s.undo_depth(), undo_before_publish, "publishing cannot be undone separately from the edits"); + assert!(s.log.entries().any(|e| e.command == "source.publish" && e.ok)); assert_eq!(out["published"], true); assert!(s.dirty(), "the layer's new record is a change to save"); let (read, _) = rustgis_data::io::read_path(&path).unwrap(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index fdfe454d2c..1a83de5b94 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -432,9 +432,11 @@ The new parts are written beside the old ones, with the old parts' file permissi and then renamed into place, the `.shp` last. An OS lock prevents two RustGIS processes from publishing the same source at once; a competing publish refuses immediately. The empty `.filename.shp.rustgis-lock` sidecar stays beside the source for reuse; the OS releases its lock -when the publisher finishes or exits. On Unix, this empty file is readable across accounts, -so users sharing the source can lock it without write access to the sidecar. Other programs do -not follow this locking protocol. If one fails to land, the old parts are put back; +when the publisher finishes or exits. On Unix, this empty file permits read/write access across +accounts so users sharing the source directory can reopen it for locking, including on NFS. +Other programs do not follow this locking protocol: changes they make after RustGIS's final +source check can be overwritten by the publish. Stop other writers before publishing. +If one part fails to land, the old parts are put back; should putting one back fail too, the message names the temporary file that holds it, beside the Shapefile, so it can be renamed back by hand. Sidecars RustGIS doesn't write, such as `.shp.xml` metadata, are left alone. The spatial indexes that would no longer match (`.qix`, `.sbn`, `.sbx`) From 350820b51d1fb767f34b696b35a612048ee67db9 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 17:48:42 -0600 Subject: [PATCH 07/15] fix(data): publish a defined source projection --- crates/data/src/publish.rs | 44 ++++++++++++++++++++++++++++++++------ docs/user-guide/index.md | 3 ++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index c2fa65e23f..4ab2adc3fe 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -172,10 +172,10 @@ impl Original { } /// The coordinate system its `.prj` names, as the reader takes it (`None` for none, one it - /// can't read, or longitude/latitude). + /// can't read). fn crs(&self) -> Option { let text = std::str::from_utf8(self.prj.as_deref()?).ok()?; - crate::raster::wkt::read_wkt(text).ok().map(|(c, _)| c).filter(|c| *c != Crs::LonLat) + crate::raster::wkt::read_wkt(text).ok().map(|(c, _)| c) } } @@ -234,10 +234,16 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Some(crate::io::read_shapefile_parts(original.parts(), &name, Some(Crs::LonLat))?.0), @@ -307,7 +313,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result Date: Fri, 13 Nov 2026 18:27:53 -0600 Subject: [PATCH 08/15] fix(data): complete defined projection publishing --- crates/data/src/publish.rs | 15 ++++++++++----- crates/engine/src/cmd/publish.rs | 2 ++ crates/engine/src/tests.rs | 7 +++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 4ab2adc3fe..fb2389075c 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -232,7 +232,7 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result std::fs::write(path.with_extension("prj"), crate::raster::wkt::to_wkt(c).unwrap()).unwrap(), + None => std::fs::remove_file(path.with_extension("prj")).unwrap(), + } let before = std::fs::read(&path).unwrap(); let (ds, sync) = read(&path); let defined = crate::io::define_projection(&ds, crs).unwrap(); let published = shapefile(&defined, &path, &sync).unwrap(); - assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none()); + if source.is_none() { + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none()); + } assert!(published.notes.iter().any(|n| n.contains(".prj now says")), "{:?}", published.notes); let (back, next) = read(&path); assert_eq!(Original::read(&path).unwrap().crs(), Some(crs)); diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index a08cc96de9..7d287f189f 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -96,6 +96,7 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result>() .into(); held.fields = published.read.fields.clone(); + held.crs = published.read.crs; // Already hashed before the files were placed: Cancel must not leave a successful // write without its sync record. Feature ids do not contribute to the digest. let read = published.rows; @@ -120,6 +121,7 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result Date: Fri, 13 Nov 2026 19:41:57 -0600 Subject: [PATCH 09/15] fix(data): preserve source state across repeated publication --- crates/assistant/src/tools.rs | 2 +- crates/data/src/io.rs | 13 +-- crates/data/src/publish.rs | 136 +++++++++++++++++++++++++++++-- crates/data/src/sync.rs | 14 +++- crates/engine/src/cmd/publish.rs | 6 ++ crates/engine/src/lib.rs | 47 +++++++++-- crates/engine/src/tests.rs | 34 ++++++++ docs/user-guide/index.md | 4 +- 8 files changed, 231 insertions(+), 25 deletions(-) diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index b7d0741d40..061d93507a 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -52,7 +52,7 @@ pub fn defs() -> Vec { }, ToolDef { name: "run_command", - description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Each run is one undo step. Commands that read or write files, use the network or replace the project ask the user first.", + description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Project edits are undoable. Source publication and other external file writes cannot be undone. Commands that read or write files, use the network or replace the project ask the user first.", schema: json!({ "type": "object", "properties": { diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index c5a9a043b6..ff884c7732 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -2878,7 +2878,7 @@ pub fn shapefile_values_notes(ds: &Dataset, target: &str) -> Vec { FieldType::Date if !value.asks_null() && dbf_date(value).is_none() => loss.empty += 1, FieldType::Integer | FieldType::Double => { let d = decimals.get(k).copied().unwrap_or(0); - match dbf_number(value, field.ty, d) { + match dbf_number(value, field.ty, d, if field.ty == FieldType::Integer { 18 } else { 24 }) { None => loss.empty += 1, Some(number) => { // The writer uses fixed decimal text, including ties rounded to even. @@ -2970,15 +2970,15 @@ fn dbf_decimals(ds: &Dataset, k: usize) -> usize { } /// The number a `.dbf` numeric field of type `ty` with `d` decimals holds for `v`: none for a -/// value that isn't a number, or one rejected by the writer's size and precision checks. +/// value that isn't a number, or one rejected by the selected column's `width` and precision. /// Rejected numbers are left empty rather than cut short. -fn dbf_number(v: &Value, ty: FieldType, d: usize) -> Option { +fn dbf_number(v: &Value, ty: FieldType, d: usize, width: usize) -> Option { let x = v.as_f64().filter(|x| { // A whole number past 2^53 would be held only rounded (dBase's decimal text goes through // f64), and one too small for the field's decimals would read 0. let exact = !(matches!(v, Value::Int(i) if i.unsigned_abs() > 1 << 53) || ty == FieldType::Double && *x != 0.0 && x.abs() < 0.5 * 10f64.powi(-(d as i32))); - exact && format!("{x:.d$}").len() <= if ty == FieldType::Integer { 18 } else { 24 } + exact && format!("{x:.d$}").len() <= width })?; Some(if ty == FieldType::Integer { x.round() } else { x }) } @@ -3373,7 +3373,7 @@ fn kept_column(ds: &Dataset, k: usize, name: &str, style: &DbfStyle) -> (Spec, O let fits = match (field.ty, ty) { (FieldType::Integer | FieldType::Double, b'N' | b'F') => values().all(|v| { let d = usize::from(was.decimals); - dbf_number(v, field.ty, d).is_some_and(|x| { + dbf_number(v, field.ty, d, usize::from(was.len)).is_some_and(|x| { let text = format!("{x:.d$}"); text.len() <= usize::from(was.len) && text.parse::().ok() == v.as_f64() }) @@ -3443,8 +3443,9 @@ pub(crate) fn shapefile_parts_styled( for (k, (f, n)) in ds.fields.iter().zip(&names).enumerate() { let v = r.attrs.get(k).unwrap_or(&Value::Null); let decimals = specs.get(k).map_or(0, |s| usize::from(s.decimals)); + let width = specs.get(k).map_or(0, |s| usize::from(s.len)); let value = match f.ty { - FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals)), + FieldType::Integer | FieldType::Double => FieldValue::Numeric(dbf_number(v, f.ty, decimals, width)), FieldType::Boolean => FieldValue::Logical(dbf_boolean(v)), FieldType::Date => FieldValue::Date(dbf_date(v).and_then(|d| { let (y, m, d) = d.ymd(); diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index fb2389075c..5aef8744ca 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -40,7 +40,7 @@ const PROJECTED_TOLERANCE: f64 = 1e-11; /// file now reads (`read`: its rows in the same order, as a Shapefile holds them: an empty text as /// no value, a column of whole numbers as integers, an edited shape in another system back through /// its projection), to record as the layer's new sync point; the features written, and what it -/// says (fields given other columns or left out, a `.prj` that now says another system). +/// says (fields given other columns or left out, projection round-off, a changed `.prj`). pub struct Published { pub files: FileSync, pub read: Dataset, @@ -286,6 +286,12 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result = @@ -364,12 +370,25 @@ fn in_file<'a>(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Da Some((read, raw)) => (read.features.iter().collect(), raw.features.iter().collect()), None => (Vec::new(), Vec::new()), }; - let at: std::collections::HashMap = read.iter().enumerate().map(|(i, f)| (f.id, i)).collect(); + // File reads number rows anew; persistent layer ids can have gaps after a deletion. + // Exact geometry keys retain correspondence without trusting those regenerated ids. + let key = |f: &Feature| serde_json::to_vec(&f.geometry).map_err(|e| DataError::Format(e.to_string())); + let mut at: std::collections::HashMap, (usize, bool)> = std::collections::HashMap::new(); + for (i, f) in read.iter().enumerate() { + crate::progress::check()?; + at.entry(key(f)?) + .and_modify(|(first, ambiguous)| *ambiguous |= raw.get(*first).zip(raw.get(i)).is_some_and(|(a, b)| a.geometry != b.geometry)) + .or_insert((i, false)); + } let mut rows = Vec::with_capacity(ds.features.len()); for f in ds.features.iter() { // A shape taken as the file holds it is quick, but a layer of many is not. crate::progress::check()?; - let held = at.get(&f.id).and_then(|&i| read.get(i).filter(|r| r.geometry == f.geometry).and(raw.get(i))); + let matched = at.get(&key(f)?); + if matched.is_some_and(|(_, ambiguous)| *ambiguous) { + return Err(DataError::Format("several source shapes read as the same geometry but have different native coordinates; their correspondence is ambiguous, so nothing was written".into())); + } + let held = matched.and_then(|&(i, _)| raw.get(i)); let mut row = f.clone(); match held { Some(h) => row.geometry = h.geometry.clone(), @@ -634,6 +653,31 @@ fn geometry_parts(g: Option<&Geometry>) -> Vec> { } } +/// Projection round-off actually present in the checked read-back: a count, the first feature +/// and the largest coordinate difference in degrees. Exact round-trips need no notice. +fn projection_note(ds: &Dataset, back: &Dataset) -> Result> { + let (mut count, mut first, mut largest) = (0usize, 0usize, 0.0_f64); + for (k, (f, g)) in ds.features.iter().zip(back.features.iter()).enumerate() { + crate::progress::check()?; + let (p, q) = (geometry_parts(f.geometry.as_ref()), geometry_parts(g.geometry.as_ref())); + if same_coords(&p, &q, 0.0) { + continue; + } + if count == 0 { + first = k + 1; + } + count += 1; + for (a, b) in p.iter().flatten().zip(q.iter().flatten()) { + largest = largest.max((a.0 - b.0).abs()).max((a.1 - b.1).abs()); + } + } + Ok((count > 0).then(|| { + format!( + "shapes with projection round-off: {count} (the first, feature {first}); the largest coordinate difference is {largest:e} degrees" + ) + })) +} + /// Why `back`, the layer `ds` written to a Shapefile and read again, isn't `ds`, or `None` when /// it is: the same fields, features, values ([`same_value`]) and shapes within `tolerance`. /// Counts the values and shapes that differ and names the first of each. @@ -655,7 +699,11 @@ fn same_rows(ds: &Dataset, back: &Dataset, tolerance: f64) -> Option { } // Points are compared as parts of one point each, so a point and a multipoint of one agree. let (p, q) = (geometry_parts(f.geometry.as_ref()), geometry_parts(g.geometry.as_ref())); - if !same_coords(&p, &q, tolerance) { + let grouping = |g: Option<&Geometry>| match g { + Some(Geometry::Polygon(polys)) => Some(polys.iter().map(Vec::len).collect::>()), + _ => None, + }; + if grouping(f.geometry.as_ref()) != grouping(g.geometry.as_ref()) || !same_coords(&p, &q, tolerance) { shapes += 1; first_shape.get_or_insert(k); } @@ -812,6 +860,23 @@ mod tests { source_lock(&path).unwrap(); } + /// A collapsed hole is inferred as an outer ring on read: identical flat coordinates + /// must not let that topology change overwrite the source. + #[test] + fn a_polygon_hole_that_reads_as_an_outer_ring_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[0].geometry { + polys[0].push(vec![Coord::new(-89.98, 35.02), Coord::new(-89.96, 35.02), Coord::new(-89.98, 35.02)]); + } + ds.features = rows.into(); + let e = shapefile(&ds, &path, &sync).err().expect("changed ring grouping must be refused").to_string(); + assert!(e.contains("shapes would read back otherwise") && e.contains("nothing was written"), "{e}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a /// project that couldn't hold a value) is refused: its unedited rows aren't the file's. #[test] @@ -1075,6 +1140,56 @@ mod tests { assert_eq!(table.columns[0].ty, b'D'); } + /// A whole-valued double reads as Integer, but its existing wide decimal column still fits. + #[test] + fn an_integral_double_keeps_its_wide_decimal_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut ds = parcels(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(100.0); + rows[1].attrs[1] = Value::Float(200.0); + ds.features = rows.into(); + let (mut ds, sync) = written(&ds, &path); + assert_eq!(ds.fields[1].ty, FieldType::Integer); + let before = std::fs::read(path.with_extension("dbf")).unwrap(); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{:?}", published.notes); + let after = std::fs::read(path.with_extension("dbf")).unwrap(); + let (a, b) = (crate::dbf::raw_table(&before).unwrap(), crate::dbf::raw_table(&after).unwrap()); + assert_eq!(a.columns, b.columns); + for (a, b) in a.records.iter().zip(&b.records) { + assert_eq!(a[1], b[1]); + } + } + + /// After deletion, persistent layer ids differ from file row numbers. An attribute-only + /// second publish must still copy the untouched projected coordinates to the bit. + #[test] + fn repeated_publication_keeps_native_shapes_after_deleting_a_feature() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let crs = Crs::parse("EPSG:2274").unwrap(); + crate::io::write_path_in(&parcels(), &path, crs).unwrap(); + let (mut ds, sync) = read(&path); + let survivor = ds.features.get(1).unwrap().clone(); + let id = survivor.id; + ds.features = vec![survivor].into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + ds = published.read; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].id = id; + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let sync = Synced { files: published.files, rows: published.rows }; + let before = std::fs::read(&path).unwrap(); + shapefile(&ds, &path, &sync).unwrap(); + assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none()); + } + /// The writer uses UTF-8 name bytes even with a legacy code page: a rewrite is refused /// before the differing name or width can reach the source. #[test] @@ -1131,7 +1246,8 @@ mod tests { let mut rows: Vec<_> = ds.features.iter().cloned().collect(); rows[1].attrs[2] = Value::Int(3); ds.features = rows.into(); - shapefile(&ds, &path, &sync).unwrap(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert!(published.notes.is_empty(), "{code}: {:?}", published.notes); // Every shape as it was, to the bit: none was edited. assert!(same_shapes(&before, &std::fs::read(&path).unwrap(), 0.0).is_none(), "{code}"); // A shape moved is projected; the others stay as they were. @@ -1141,7 +1257,15 @@ mod tests { polys[0][0][1].y += 0.01; } ds.features = rows.into(); - shapefile(&ds, &path, &sync).unwrap(); + let published = shapefile(&ds, &path, &sync).unwrap(); + if code == "EPSG:2274" { + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!( + note.contains("shapes with projection round-off: 1") && note.contains("feature 2") && note.contains("degrees"), + "{note}" + ); + } let shapes = |b: &[u8]| -> Vec<_> { let mut r = shapefile::ShapeReader::new(std::io::Cursor::new(b.to_vec())).unwrap(); r.iter_shapes().map(|s| shape_parts(&s.unwrap()).1).collect() diff --git a/crates/data/src/sync.rs b/crates/data/src/sync.rs index d588382abc..3e626f844f 100644 --- a/crates/data/src/sync.rs +++ b/crates/data/src/sync.rs @@ -36,8 +36,8 @@ pub struct Synced { /// A hash of what a dataset holds: its fields, coordinate system, and each row's geometry and /// attributes in order (FNV-1a over their JSON, written to the hash as it is made). Not its name -/// or where it came from, which are no edit to the file. A number that isn't finite, which JSON -/// writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit +/// or where it came from, or display aliases, which are no edit to the file. A number that isn't +/// finite, which JSON writes as null, is told from an empty cell by its position and bits, as the GeoPackage edit /// rows are: clearing one is an edit. A project saves one as null (and says so), so a layer that /// held one reads as edited once its project is opened again: it no longer holds what its file /// does. Its work grows with the rows: each is counted as progress ([`crate::progress`]), and it @@ -61,7 +61,13 @@ pub fn rows_digest(ds: &crate::Dataset) -> crate::Result { h.0 = (h.0 ^ 0xff).wrapping_mul(0x0100_0000_01b3); } let mut h = Fnv(0xcbf2_9ce4_8422_2325); - put(&mut h, &(&ds.fields, &ds.crs)); + // Whole Shapefile/GeoJSON reads record fields with no aliases. Clear display-only aliases + // when comparing a layer, keeping the serialized form and existing saved digests unchanged. + let mut fields = ds.fields.clone(); + for field in &mut fields { + field.alias = None; + } + put(&mut h, &(&fields, &ds.crs)); crate::progress::begin(ds.features.len()); for f in ds.features.iter() { put(&mut h, &(&f.geometry, &f.attrs)); @@ -357,6 +363,8 @@ mod tests { }] .into(); assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092); + ds.fields[0].alias = Some("Street name".into()); + assert_eq!(rows_digest(&ds).unwrap(), 1_724_049_670_388_433_092, "an alias changes display, not file contents"); } #[test] diff --git a/crates/engine/src/cmd/publish.rs b/crates/engine/src/cmd/publish.rs index 7d287f189f..6b3222eb07 100644 --- a/crates/engine/src/cmd/publish.rs +++ b/crates/engine/src/cmd/publish.rs @@ -96,6 +96,11 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result>() .into(); held.fields = published.read.fields.clone(); + for field in &mut held.fields { + if let Some(was) = dataset.fields.iter().find(|was| was.name == field.name) { + field.alias = was.alias.clone(); + } + } held.crs = published.read.crs; // Already hashed before the files were placed: Cancel must not leave a successful // write without its sync record. Feature ids do not contribute to the digest. @@ -105,6 +110,7 @@ pub(crate) fn publish_work(s: &Session, p: &Value) -> Result, /// Who runs commands now: `None` is the user; the MCP relay names an attached agent around /// its calls. Each undo step remembers who made it ([`Session::undo_actor`]). pub actor: Option, @@ -1205,6 +1208,7 @@ impl Session { depth: 0, diagnostic_source: None, restorable: 0, + published_syncs: Vec::new(), actor: None, back: Vec::new(), forward: Vec::new(), @@ -1808,7 +1812,8 @@ impl Session { let failed_history = if self.opened == opened { std::mem::take(&mut self.project.gp_history) } else { Vec::new() }; // The project as it was before the command, so the layout open then, whatever // the command opened on the way (a task's steps). - self.project = snap; + let committed = std::mem::replace(&mut self.project, snap); + let sync_changed = self.keep_synced(&committed); (self.trusted_tasks, self.trusted_models, self.trusted_notebooks) = trust; self.active_layout = open_layout; // Replaced during the command: what it held beside itself (its undo history, its @@ -1841,7 +1846,11 @@ impl Session { self.push_gp_record(record); recorded_failure = true; } - if recorded_failure { + if sync_changed { + self.mark_unsaved(); + self.warn("Source-file records were kept: rolling back a command does not undo its file writes"); + } + if recorded_failure || sync_changed { if self.depth == 0 && self.project.gp_history.len() > 500 { let excess = self.project.gp_history.len() - 500; self.project.gp_history.drain(..excess); @@ -1866,6 +1875,7 @@ impl Session { // The command, and those it ran, are done: a replaced project stays replaced. if self.depth == 0 { self.put_aside = None; + self.published_syncs.clear(); } self.fix_active_layer(); self.revision = self.revision.wrapping_add(1); @@ -2547,16 +2557,37 @@ impl Session { cmd::zarr::follow_time(self); } - /// After undo or redo swaps the project: each layer keeps the record of its file it had - /// (`synced`), as undo doesn't change the file. A layer published since keeps the record of - /// what was written, so undoing an edit made before shows it as an edit the file holds. - fn keep_synced(&mut self, before: &Project) { + /// Remembers a physical source write only while a command can still roll its project back. + /// The outer guard clears these records on success or failure; they never go in a project. + #[cfg(not(target_arch = "wasm32"))] + fn remember_source_write(&mut self, instance: u64, path: String, synced: rustgis_data::sync::Synced) { + if self.restorable > 0 { + self.published_syncs.retain(|(i, p, _)| *i != instance || *p != path); + self.published_syncs.push((instance, path, synced)); + } + } + + /// After undo, redo or failed-command rollback swaps the project: each layer keeps its + /// file record (`synced`), as restoring the project doesn't change the file. A published + /// layer keeps the record of what was written, so restored edits compare with that file. + /// Returns whether the restored project took a different record and needs saving. + fn keep_synced(&mut self, before: &Project) -> bool { + let mut changed = false; for l in self.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()) { let old = before.maps.iter().flat_map(|m| &m.layers).find(|o| o.instance_id() == l.instance_id()); - if let Some(old) = old.filter(|o| o.dataset.source == l.dataset.source) { - l.synced = old.synced.clone(); + let synced = old.filter(|o| o.dataset.source == l.dataset.source).map(|o| o.synced.clone()).or_else(|| { + self.published_syncs + .iter() + .rev() + .find(|(i, p, _)| *i == l.instance_id() && l.dataset.source.as_deref() == Some(p.as_str())) + .map(|(_, _, synced)| Some(synced.clone())) + }); + if let Some(synced) = synced { + changed |= l.synced != synced; + l.synced = synced; } } + changed } /// After undo or redo swaps the project: the tile layer picked in the Layers pane stays the one diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 34db5eb223..02cf37aa32 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -44760,12 +44760,17 @@ fn a_layer_publishes_its_edits_to_its_shapefile() { assert_eq!(state(&mut s), "inSync"); // An empty text, which the file holds as no value: the layer takes it as the file reads it, // and is in step with it. + { + let l = s.map_mut().layer_mut(id as u32).unwrap(); + l.dataset.fields.iter_mut().find(|f| f.name == "name").unwrap().alias = Some("Town name".into()); + } s.run("edit.setValue", &json!({"layer": id, "row": 2, "field": "name", "value": ""})).unwrap(); s.run("source.publish", &json!({"layer": id})).unwrap(); assert_eq!(state(&mut s), "inSync"); let l = s.map().layer(id as u32).unwrap(); let name = l.dataset.fields.iter().position(|f| f.name == "name").unwrap(); assert!(l.dataset.features.get(2).unwrap().attrs[name].is_null()); + assert_eq!(l.dataset.fields[name].alias.as_deref(), Some("Town name")); // Cancel arriving after the final .shp rename cannot turn a committed publish into an // error or leave its old sync record. The probe observes the real file, not an internal hook. struct CancelAfterPlacement { @@ -44799,6 +44804,35 @@ fn a_layer_publishes_its_edits_to_its_shapefile() { assert_eq!(s.run("source.publish", &json!({"layer": id})).unwrap()["published"], true); assert_eq!(state(&mut s), "inSync"); assert_eq!(s.run("source.publish", &json!({"layer": id})).unwrap()["published"], false); + // A later script failure rolls project edits back, but not its committed file or baseline, + // including when the script removed the published layer before failing. + for (n, (edited_inside, removed)) in [(false, false), (true, false), (true, true)].into_iter().enumerate() { + if !edited_inside { + s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Committed"})).unwrap(); + } + s.mark_saved(); + let undo = s.undo_depth(); + let mut lines = Vec::new(); + let value = if edited_inside { format!("In script {n}") } else { "Committed".into() }; + if edited_inside { + lines.push(format!("edit.setValue {}", json!({"layer":id,"row":1,"field":"name","value":value}))); + } + lines.push(format!("source.publish {}", json!({"layer":id}))); + if removed { + lines.push(format!("layer.remove {}", json!({"layer":id}))); + } + lines.push("layer.remove {\"layer\":\"missing-layer\"}".into()); + assert!(s.run("script.run", &json!({"script":lines.join("\n")})).is_err()); + let checked = s.run("source.syncState", &json!({"layer":id})).unwrap(); + assert_eq!(checked["changed"], json!([]), "the committed file retains its current baseline"); + assert_eq!(checked["state"], if edited_inside { "edited" } else { "inSync" }); + assert!(s.dirty(), "the committed baseline still needs saving after rollback"); + assert_eq!(s.undo_depth(), undo, "a failed script adds no undo step"); + let (back, _) = rustgis_data::io::read_path(&path).unwrap(); + assert_eq!(back.features.get(1).unwrap().attrs[name], rustgis_data::Value::Text(value)); + } + s.run("source.publish", &json!({"layer":id})).unwrap(); + assert_eq!(state(&mut s), "inSync"); // Changed since: refused, the file left as it was. s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Mine"})).unwrap(); let mut theirs = dbf(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index f87ddc70d2..731050fce3 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -424,7 +424,9 @@ coordinate system than its `.prj` names, including one assigned with Define Proj written with the coordinates the file holds, so a projected file's untouched shapes stay as they were to the bit. An edited shape in a projected system is converted from longitude/latitude, as RustGIS holds it, and read back: publishing refuses one that would move by more than one part in -10¹¹ of its coordinates (a tenth of a millimetre at ten thousand kilometres). +10¹¹ of its coordinates (a tenth of a millimetre at ten thousand kilometres). A notice +names the file, counts shapes with projection round-off and identifies the first feature and +the largest coordinate difference in degrees; exact round trips give no such notice. Publishing refuses symbolic links in any source part: add the actual source files instead. This keeps each link and its target unchanged. From f3f79659fde8da56a3c03482a9bdc8446693895c Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 21:32:28 -0600 Subject: [PATCH 10/15] fix(data): keep publication notices and undo records --- crates/data/src/publish.rs | 38 +++++++++++++++++++++++++++++++++++--- crates/engine/src/lib.rs | 16 ++++++++++++++++ crates/engine/src/tests.rs | 22 ++++++++++++++++++++++ docs/user-guide/index.md | 3 ++- 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 5aef8744ca..1e145ff0c1 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -38,9 +38,10 @@ const PROJECTED_TOLERANCE: f64 = 1e-11; /// A layer written back over its Shapefile: the files as they were written, and the layer as the /// file now reads (`read`: its rows in the same order, as a Shapefile holds them: an empty text as -/// no value, a column of whole numbers as integers, an edited shape in another system back through -/// its projection), to record as the layer's new sync point; the features written, and what it -/// says (fields given other columns or left out, projection round-off, a changed `.prj`). +/// no value, a column of whole numbers as integers, date-times as text, an edited shape in another +/// system back through its projection), to record as the layer's new sync point; the features +/// written, and what it says (fields given other columns or left out, projection round-off, +/// a changed `.prj`). pub struct Published { pub files: FileSync, pub read: Dataset, @@ -306,6 +307,9 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result = widened.iter().take(5).map(String::as_str).collect(); let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; @@ -1140,6 +1144,34 @@ mod tests { assert_eq!(table.columns[0].ty, b'D'); } + /// A date-time still fits its text column, but losing the field type must be said once. + #[test] + fn a_date_time_field_written_as_text_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let stamp = "2025-01-02T12:30:00Z"; + let mut source = parcels(); + let mut rows: Vec<_> = source.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::Text(stamp.into()); + } + source.features = rows.into(); + let (mut ds, sync) = written(&source, &path); + ds.fields[0].ty = FieldType::DateTime; + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + for row in &mut rows { + row.attrs[0] = Value::DateTime(crate::DateTime::parse(stamp).unwrap()); + } + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!(note.contains("1 date-time field was written as text (ISO 8601)") && note.contains("'OWNER'"), "{note}"); + assert!(note.contains(&path.display().to_string()), "{note}"); + assert_eq!(published.read.fields[0].ty, FieldType::Text); + assert_eq!(published.read.features.first().unwrap().attrs[0], Value::Text(stamp.into())); + } + /// A whole-valued double reads as Integer, but its existing wide decimal column still fits. #[test] fn an_integral_double_keeps_its_wide_decimal_column() { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index bdb39c64b8..8b1c6fd86f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1759,6 +1759,10 @@ impl Session { // session's own flag says so, not the answer, which a plugin's command could forge. (Ok(_), Some(_)) if std::mem::take(&mut self.request_only) => {} (Ok(_), Some(mut snap)) => { + // A successful script may publish and then remove the layer. Its undo + // snapshot still needs the committed file record, even without a live layer + // from which undo could carry that record forward. + self.keep_published_syncs(&mut snap); fresh_source_ids(&snap, &mut self.project); forget_label_state_of_removed_features(&snap, &mut self.project); // An edit may change the dates the time slider's window filters on: bound layers @@ -2567,6 +2571,18 @@ impl Session { } } + /// Gives a command's undo snapshot the records of files it physically wrote, including + /// layers the command removed after publishing. The records belong to these instances only. + fn keep_published_syncs(&self, project: &mut Project) { + for l in project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()) { + if let Some((_, _, synced)) = + self.published_syncs.iter().rev().find(|(i, p, _)| *i == l.instance_id() && l.dataset.source.as_deref() == Some(p.as_str())) + { + l.synced = Some(synced.clone()); + } + } + } + /// After undo, redo or failed-command rollback swaps the project: each layer keeps its /// file record (`synced`), as restoring the project doesn't change the file. A published /// layer keeps the record of what was written, so restored edits compare with that file. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 02cf37aa32..695e38b140 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -44833,6 +44833,28 @@ fn a_layer_publishes_its_edits_to_its_shapefile() { } s.run("source.publish", &json!({"layer":id})).unwrap(); assert_eq!(state(&mut s), "inSync"); + // A successful script can remove a layer after publication too. Undo restores its old + // rows against the committed file baseline; redo and a second undo keep that baseline. + let script = [ + format!("edit.setValue {}", json!({"layer":id,"row":1,"field":"name","value":"Published and removed"})), + format!("source.publish {}", json!({"layer":id})), + format!("layer.remove {}", json!({"layer":id})), + ] + .join("\n"); + s.run("script.run", &json!({"script":script})).unwrap(); + assert!(s.map().layer(id as u32).is_none()); + for again in [false, true] { + if again { + s.run("edit.redo", &Value::Null).unwrap(); + assert!(s.map().layer(id as u32).is_none()); + } + s.run("edit.undo", &Value::Null).unwrap(); + let checked = s.run("source.syncState", &json!({"layer":id})).unwrap(); + assert_eq!(checked["changed"], json!([])); + assert_eq!(checked["state"], "edited"); + let (back, _) = rustgis_data::io::read_path(&path).unwrap(); + assert_eq!(back.features.get(1).unwrap().attrs[name], rustgis_data::Value::Text("Published and removed".into())); + } // Changed since: refused, the file left as it was. s.run("edit.setValue", &json!({"layer": id, "row": 1, "field": "name", "value": "Mine"})).unwrap(); let mut theirs = dbf(); diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 731050fce3..302c119201 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -448,7 +448,8 @@ should be deleted before that program uses the file. Once published, the layer holds its rows as the file reads them, which differs only where a Shapefile can't tell values apart (an empty text reads as no value, a column of whole numbers as -integers), so it reads `inSync`; the project has a change to save. Undo doesn't change the file: +integers, date-time fields as ISO 8601 text, with a notice), so it reads `inSync`; the project has +a change to save. Undo doesn't change the file: undoing an edit made before publishing shows the layer as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't be published yet. From 2df455604127bfc27a77c47bb950f96d5ba95bfb Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 22:25:56 -0600 Subject: [PATCH 11/15] docs(data): explain Shapefile crash recovery limits --- docs/user-guide/index.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 302c119201..4a3a9175f3 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -446,6 +446,15 @@ metadata, are left alone. The spatial indexes that would no longer match (`.qix` are removed where they can be; one that can't be (held open by another program) is left, and should be deleted before that program uses the file. +This rollback needs RustGIS to keep running. Replacing several Shapefile parts is not atomic: +a process crash or power loss between renames can leave a mixed or incomplete set. Keep an +independent backup before publishing. After such an interruption, preserve the whole directory, +including hidden `....tmp` files (for example, +`.parcels.dbf.1234.5.tmp`); these can contain both staged new parts and displaced old parts. +Restore a complete, consistent Shapefile from your backup before opening or publishing it again. +Do not restore an arbitrary temporary file or mix parts from different versions. Temporary +files may help with manual recovery, but are not a crash-safe backup. + Once published, the layer holds its rows as the file reads them, which differs only where a Shapefile can't tell values apart (an empty text reads as no value, a column of whole numbers as integers, date-time fields as ISO 8601 text, with a notice), so it reads `inSync`; the project has From b784f911f8d13e284ad0e8ea3911deddb1017130 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 13 Nov 2026 23:05:09 -0600 Subject: [PATCH 12/15] fix(data): report integer field inference when publishing --- crates/data/src/publish.rs | 39 ++++++++++++++++++++++++++++++++++++++ docs/user-guide/index.md | 4 ++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 1e145ff0c1..8d5d11e326 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -310,6 +310,23 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result = ds + .fields + .iter() + .zip(&read_back.fields) + .filter(|(a, b)| a.ty == FieldType::Double && b.ty == FieldType::Integer) + .map(|(field, _)| field.name.clone()) + .collect(); + if !integral.is_empty() { + notes.push(format!( + "{}: {} double field{} read back as integer because all non-null values are whole numbers: {}", + path.display(), + integral.len(), + if integral.len() == 1 { "" } else { "s" }, + names_list(&integral) + )); + } if !widened.is_empty() { let shown: Vec<&str> = widened.iter().take(5).map(String::as_str).collect(); let more = if widened.len() > 5 { format!(" and {} more", widened.len() - 5) } else { String::new() }; @@ -1172,6 +1189,28 @@ mod tests { assert_eq!(published.read.features.first().unwrap().attrs[0], Value::Text(stamp.into())); } + /// Editing the last fractional values changes the read-back field kind, which must be said. + #[test] + fn a_double_field_read_back_as_integer_says_so() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + assert_eq!(ds.fields[1].ty, FieldType::Double); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[1] = Value::Float(100.0); + rows[1].attrs[1] = Value::Float(200.0); + ds.features = rows.into(); + let published = shapefile(&ds, &path, &sync).unwrap(); + assert_eq!(published.read.fields[1].ty, FieldType::Integer); + assert_eq!(published.read.features.first().unwrap().attrs[1], Value::Int(100)); + assert_eq!(published.notes.len(), 1, "{:?}", published.notes); + let note = &published.notes[0]; + assert!(note.contains("1 double field read back as integer") && note.contains("'ACRES'"), "{note}"); + assert!(note.contains(&path.display().to_string()), "{note}"); + let again = shapefile(&published.read, &path, &Synced { files: published.files, rows: published.rows }).unwrap(); + assert!(again.notes.is_empty(), "{:?}", again.notes); + } + /// A whole-valued double reads as Integer, but its existing wide decimal column still fits. #[test] fn an_integral_double_keeps_its_wide_decimal_column() { diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 4a3a9175f3..7288e47955 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -457,8 +457,8 @@ files may help with manual recovery, but are not a crash-safe backup. Once published, the layer holds its rows as the file reads them, which differs only where a Shapefile can't tell values apart (an empty text reads as no value, a column of whole numbers as -integers, date-time fields as ISO 8601 text, with a notice), so it reads `inSync`; the project has -a change to save. Undo doesn't change the file: +integers, date-time fields as ISO 8601 text). A double field becoming integer or a date-time field +becoming text produces a notice. The layer reads `inSync`; the project has a change to save. Undo doesn't change the file: undoing an edit made before publishing shows the layer as `edited` again. A layer with no edits writes nothing. GeoJSON and layers read in place can't be published yet. From 49b1f04d5f9799fe7972f02df0a4df19c8f140c5 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 00:40:42 -0600 Subject: [PATCH 13/15] fix(data): preserve recovery files and publication notices --- crates/data/src/io.rs | 67 ++++++++++++-------- crates/data/src/publish.rs | 24 ++++++- crates/engine/src/cmd/exports.rs | 23 ++++++- crates/engine/src/tests/permission_policy.rs | 25 ++++++++ 4 files changed, 110 insertions(+), 29 deletions(-) diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 77b189a51e..b01ca9c4aa 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -678,7 +678,14 @@ pub(crate) fn place_shapefile(path: &Path, staged: &[(std::path::PathBuf, std::p break; } }; + // Reserve this name before moving the old part: a previous crash's backup + // (or a symlink) must never be replaced when a process id is reused. + if let Err(e) = std::fs::OpenOptions::new().write(true).create_new(true).open(&bak).map(drop) { + result = Err(e); + break; + } if let Err(e) = std::fs::rename(target, &bak) { + let _ = std::fs::remove_file(&bak); result = Err(e); break; } @@ -3825,7 +3832,7 @@ mod tests { Ok(()) } - /// Exercises caller cleanup through the public Shapefile writer, one fresh process per collision. + /// Exercises staging and backup cleanup through the public writer, one fresh process per collision. #[cfg(not(target_arch = "wasm32"))] #[test] fn shapefile_staging_collisions_preserve_existing_entries() -> std::io::Result<()> { @@ -3836,32 +3843,35 @@ mod tests { const CHILD: &str = "RUSTGIS_SHAPEFILE_STAGING_COLLISION"; let Ok(case) = std::env::var(CHILD) else { let kinds: &[&str] = if cfg!(unix) { &["file", "live", "dangling"] } else { &["file"] }; - for index in 0..parts.len() { - for kind in kinds { - let output = std::process::Command::new(std::env::current_exe()?) - .args([ - "--exact", - "io::tests::shapefile_staging_collisions_preserve_existing_entries", - "--nocapture", - "--test-threads=1", - ]) - .env(CHILD, format!("{index}:{kind}")) - .output()?; - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - output.status.success() - && stdout.contains("test result: ok. 1 passed; 0 failed; 0 ignored;") - && stdout.contains(&format!("collision case completed: {index}:{kind}")), - "part={index} kind={kind}: {}{}", - stdout, - String::from_utf8_lossy(&output.stderr) - ); + for phase in ["stage", "backup"] { + for index in 0..parts.len() { + for kind in kinds { + let output = std::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + "io::tests::shapefile_staging_collisions_preserve_existing_entries", + "--nocapture", + "--test-threads=1", + ]) + .env(CHILD, format!("{phase}:{index}:{kind}")) + .output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("test result: ok. 1 passed; 0 failed; 0 ignored;") + && stdout.contains(&format!("collision case completed: {phase}:{index}:{kind}")), + "phase={phase} part={index} kind={kind}: {}{}", + stdout, + String::from_utf8_lossy(&output.stderr) + ); + } } } - println!("collision scenarios: {}", parts.len() * kinds.len()); + println!("collision scenarios: {}", 2 * parts.len() * kinds.len()); return Ok(()); }; - let (index, kind) = case.split_once(':').ok_or_else(|| std::io::Error::other("missing collision case"))?; + let (phase, case_parts) = case.split_once(':').ok_or_else(|| std::io::Error::other("missing collision phase"))?; + let (index, kind) = case_parts.split_once(':').ok_or_else(|| std::io::Error::other("missing collision case"))?; let index = index.parse::().map_err(std::io::Error::other)?; if !matches!(kind, "file" | "live" | "dangling") { return Err(std::io::Error::other("unknown collision kind")); @@ -3873,12 +3883,17 @@ mod tests { for (extension, bytes) in &parts { std::fs::write(destination(extension), bytes)?; } - let (extension, _) = parts.get(index).ok_or_else(|| std::io::Error::other("collision part out of range"))?; + let mut ordered: Vec<_> = parts.iter().collect(); + if phase == "backup" { + ordered.sort_by_key(|(extension, _)| *extension == "shp"); + } + let (extension, _) = ordered.get(index).ok_or_else(|| std::io::Error::other("collision part out of range"))?; let target = destination(extension); - // This child runs only this test: write_path's first staging counter is zero. + // This child starts the counter at zero; backups follow all staged parts. let mut name = std::ffi::OsString::from("."); name.push(target.file_name().ok_or_else(|| std::io::Error::other("missing part name"))?); - name.push(format!(".{}.{index}.tmp", std::process::id())); + let counter = if phase == "backup" { parts.len() + index } else { index }; + name.push(format!(".{}.{counter}.tmp", std::process::id())); let collision = target.with_file_name(name); #[cfg(unix)] let link_target = root.join("link-target"); diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 8d5d11e326..f6e45ccde7 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -558,7 +558,7 @@ fn cell(ty: u8, bytes: &[u8]) -> Cell<'_> { } } b'N' | b'F' => { - if trimmed.is_empty() || trimmed.iter().all(|b| *b == b'*') { + if trimmed.is_empty() { return Cell::Empty; } match decimal(trimmed) { @@ -944,6 +944,28 @@ mod tests { } } + /// An unedited numeric overflow marker must not silently become an empty cell. + #[test] + fn an_overflow_cell_is_refused_without_changing_the_source() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let table = crate::dbf::raw_table(&dbf).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let at = header + 1 + usize::from(table.columns[0].len) + usize::from(table.columns[1].len); + dbf[at..at + 18].fill(b'*'); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.first().unwrap().attrs[2], Value::Null); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[1].attrs[0] = Value::Text("Jonas".into()); + ds.features = rows.into(); + let error = shapefile(&ds, &path, &sync).err().expect("overflow marker must be refused").to_string(); + assert!(error.contains("table wouldn't be the same") && error.contains("UNITS") && error.contains("***"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + /// A number past what an `f64` holds exactly, in a column that wasn't edited, is refused /// rather than written back rounded. #[test] diff --git a/crates/engine/src/cmd/exports.rs b/crates/engine/src/cmd/exports.rs index e84fdb1fc8..7e4013a293 100644 --- a/crates/engine/src/cmd/exports.rs +++ b/crates/engine/src/cmd/exports.rs @@ -66,7 +66,7 @@ pub(crate) fn write_here(s: &mut Session, p: &Value, command: &str) -> CmdResult let work = start(s, p)?; written_under(name, &control, work) })?; - s.check_continuation(&owner).map_err(|why| completion_refused(wrote.written_path.as_deref(), why))?; + s.check_continuation(&owner).map_err(|why| completion_with_notes(command, wrote.written_path.as_deref(), &wrote.notes, why))?; said(s, wrote) } @@ -78,6 +78,24 @@ pub(crate) fn completion_refused(written_path: Option<&str>, why: String) -> Str } } +/// A committed publication's format notices remain part of its refusal diagnostic; its +/// finalizer stays blocked, so reporting the write cannot update a revoked layer's record. +fn completion_with_notes(command: &str, written_path: Option<&str>, notes: &[String], why: String) -> String { + if command == "source.publish" + && let Some(path) = written_path + { + let mut message = + format!("Published {path}; completion was refused: {why}; the layer's record wasn't updated; add the file again to compare it"); + for note in notes { + message.push('\n'); + message.push_str(note); + } + message + } else { + completion_refused(written_path, why) + } +} + /// `work` under `control`, as [`rustgis_analysis::run_under`] runs it, except that a file once /// placed is written: a Cancel that comes after the last step doesn't turn it into a failure. fn written_under(name: &str, control: &std::sync::Arc, work: Work) -> Result { @@ -285,8 +303,9 @@ pub fn export_land(s: &mut Session, started: Started, done: Result Date: Sat, 14 Nov 2026 01:26:08 -0600 Subject: [PATCH 14/15] fix(data): refuse native reuse of unconverted shapes --- crates/data/src/publish.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index f6e45ccde7..95a50750d8 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -383,7 +383,8 @@ pub fn shapefile(ds: &Dataset, path: &Path, synced: &Synced) -> Result(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Dataset)>) -> Result> { let Some(c) = crs else { return Ok(std::borrow::Cow::Borrowed(ds)) }; let t = c.transformer().map_err(DataError::Format)?; @@ -397,6 +398,12 @@ fn in_file<'a>(ds: &'a Dataset, crs: Option, as_held: Option<(&Dataset, &Da let mut at: std::collections::HashMap, (usize, bool)> = std::collections::HashMap::new(); for (i, f) in read.iter().enumerate() { crate::progress::check()?; + if f.geometry.is_none() && raw.get(i).is_some_and(|f| f.geometry.is_some()) { + return Err(DataError::Format(format!( + "source shape {} couldn't be converted to longitude/latitude, so its native geometry can't be matched safely; check the source projection and add the file again; nothing was written", + i + 1 + ))); + } at.entry(key(f)?) .and_modify(|(first, ambiguous)| *ambiguous |= raw.get(*first).zip(raw.get(i)).is_some_and(|(a, b)| a.geometry != b.geometry)) .or_insert((i, false)); @@ -1259,6 +1266,25 @@ mod tests { } } + /// An unconverted source shape cannot supply geometry to a new geometry-less feature. + #[test] + fn an_unconverted_shape_cannot_be_copied_to_a_geometryless_feature() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let mut native = Dataset::new("parcels", GeometryType::Point, vec![Field::new("OWNER", FieldType::Text)]); + native.push(Some(Geometry::Point(Coord::new(1e30, 1e30))), vec![Value::Text("old".into())]); + native.push(Some(Geometry::Point(Coord::new(500_000.0, 0.0))), vec![Value::Text("kept".into())]); + crate::io::write_path(&native, &path).unwrap(); + let crs = Crs::parse("EPSG:32616").unwrap(); + std::fs::write(path.with_extension("prj"), crate::raster::wkt::to_wkt(crs).unwrap()).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.iter().filter(|f| f.geometry.is_none()).count(), 1); + ds.push(None, vec![Value::Text("new".into())]); + let error = shapefile(&ds, &path, &sync).err().expect("unconverted shapes must not be assigned to new features").to_string(); + assert!(error.contains("source shape 1") && error.contains("converted") && error.contains("nothing was written"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + /// After deletion, persistent layer ids differ from file row numbers. An attribute-only /// second publish must still copy the untouched projected coordinates to the bit. #[test] From f3c12cc6b6d055a6cb3f77d14e5980e2324a66da Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 14 Nov 2026 02:19:11 -0600 Subject: [PATCH 15/15] fix(data): handle DBF tails, index case and source lock release --- crates/data/src/dbf.rs | 6 ++- crates/data/src/io.rs | 10 ++++- crates/data/src/publish.rs | 80 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index d199ba5096..222495973c 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -245,8 +245,10 @@ pub(crate) fn raw_table(dbf: &[u8]) -> Option> { return None; } let body = dbf.get(header_len..)?; - // Fewer records than the header counts: not a table this can compare. - if body.len() < count.checked_mul(record_len)? { + // Every byte must belong to a counted record or the optional EOF marker. A rewrite + // must not discard uncounted records or other data left past the declared body. + let trailing = body.get(count.checked_mul(record_len)?..)?; + if !trailing.is_empty() && trailing != [0x1a] { return None; } let (mut records, mut deleted) = (Vec::new(), 0); diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index b01ca9c4aa..27f0266636 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -720,7 +720,15 @@ pub(crate) fn place_shapefile(path: &Path, staged: &[(std::path::PathBuf, std::p } // A spatial index left from an older file of the same name would no longer match. for stale in ["qix", "sbn", "sbx"] { - let _ = std::fs::remove_file(path.with_extension(stale)); + // Sidecars may use any letter case, even beside a lowercase `.shp`. + for capitals in 0..8 { + let extension: String = stale + .bytes() + .enumerate() + .map(|(i, b)| char::from(if capitals & (1 << i) == 0 { b } else { b.to_ascii_uppercase() })) + .collect(); + let _ = std::fs::remove_file(path.with_extension(extension)); + } } Ok(()) } diff --git a/crates/data/src/publish.rs b/crates/data/src/publish.rs index 95a50750d8..f0945c819d 100644 --- a/crates/data/src/publish.rs +++ b/crates/data/src/publish.rs @@ -72,11 +72,22 @@ fn refuse_link(source: &Path, part: &Path) -> Result<()> { } } +/// A publisher's lock, released before its descriptor closes so duplicates inherited during +/// process creation cannot extend its lifetime. +struct SourceLock(std::fs::File); + +impl Drop for SourceLock { + /// Unlock the shared file description, including any duplicate descriptors. + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + /// Holds an exclusive OS lock on a persistent sidecar through publication. The sidecar is /// never unlinked: deleting it could let a third process lock a different inode while another -/// still holds the old one. Dropping the handle (including on process exit) releases the lock; +/// still holds the old one. Dropping the guard explicitly unlocks before closing its handle; /// contention refuses immediately. This coordinates RustGIS publishers, not other programs. -fn source_lock(path: &Path) -> Result { +fn source_lock(path: &Path) -> Result { refuse_link(path, path)?; let target = path.canonicalize()?; let mut name = std::ffi::OsString::from("."); @@ -119,7 +130,7 @@ fn source_lock(path: &Path) -> Result { std::fs::TryLockError::WouldBlock => refused(path, "another RustGIS publisher holds this source; try again after it finishes"), std::fs::TryLockError::Error(e) => refused(path, format!("couldn't lock {}: {e}", lock_path.display())), })?; - Ok(file) + Ok(SourceLock(file)) } /// The Shapefile at `path`'s parts as they are, read into memory, with its `.cpg`'s path. @@ -865,6 +876,21 @@ mod tests { assert!(shapefile(&ds, &path, &sync).err().unwrap().to_string().contains("changed since the layer was read")); } + /// A duplicated descriptor must not keep a finished publisher's lock alive. + #[test] + fn a_source_lock_releases_while_a_duplicate_descriptor_stays_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + written(&parcels(), &path); + let held = source_lock(&path).unwrap(); + let duplicate = held.0.try_clone().unwrap(); + drop(held); + let next = source_lock(&path).expect("a finished publisher must release its lock despite a duplicated descriptor"); + drop(duplicate); + assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); + drop(next); + } + /// A shared source's empty lock is writable across accounts, even under a private umask. /// Reopening it keeps a writable handle (required by NFS) and excludes a second publisher. #[cfg(unix)] @@ -882,7 +908,7 @@ mod tests { drop(held); let mut held = source_lock(&path).unwrap(); // A zero-byte write checks the descriptor's access mode without storing source data. - assert_eq!(held.write(&[]).unwrap(), 0); + assert_eq!(held.0.write(&[]).unwrap(), 0); assert!(source_lock(&path).err().unwrap().to_string().contains("another RustGIS publisher")); drop(held); source_lock(&path).unwrap(); @@ -905,6 +931,52 @@ mod tests { assert!(sync.files.changed(&path).unwrap().is_empty()); } + /// Geometry edits remove obsolete indexes regardless of their extension's letter case. + #[test] + fn publishing_removes_case_variant_indexes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + let (mut ds, sync) = written(&parcels(), &path); + let indexes = ["qix", "QIX", "qIx", "sbn", "SBN", "SBX"]; + for extension in indexes { + std::fs::write(path.with_extension(extension), b"old index").unwrap(); + } + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + if let Some(Geometry::Polygon(polys)) = &mut rows[0].geometry { + polys[0][0][1].x += 0.005; + } + ds.features = rows.into(); + shapefile(&ds, &path, &sync).unwrap(); + for extension in indexes { + assert!(!path.with_extension(extension).exists(), "stale index {extension} survived"); + } + } + + /// A DBF record beyond its declared count must not disappear during an unrelated edit. + #[test] + fn publishing_refuses_unaccounted_dbf_records() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("parcels.shp"); + crate::io::write_path(&parcels(), &path).unwrap(); + let mut dbf = std::fs::read(path.with_extension("dbf")).unwrap(); + let header = usize::from(u16::from_le_bytes([dbf[8], dbf[9]])); + let width = usize::from(u16::from_le_bytes([dbf[10], dbf[11]])); + let count = u32::from_le_bytes(dbf[4..8].try_into().unwrap()) as usize; + let extra = dbf[header..header + width].to_vec(); + dbf.truncate(header + count * width); + dbf.extend(extra); + dbf.push(0x1a); + std::fs::write(path.with_extension("dbf"), &dbf).unwrap(); + let (mut ds, sync) = read(&path); + assert_eq!(ds.features.len(), 2); + let mut rows: Vec<_> = ds.features.iter().cloned().collect(); + rows[0].attrs[0] = Value::Text("Smyth".into()); + ds.features = rows.into(); + let error = shapefile(&ds, &path, &sync).err().expect("uncounted DBF records must be refused").to_string(); + assert!(error.contains("header") && error.contains("records") && error.contains("nothing was written"), "{error}"); + assert!(sync.files.changed(&path).unwrap().is_empty()); + } + /// A layer read otherwise than the file reads now (by an older RustGIS, or saved in a /// project that couldn't hold a value) is refused: its unedited rows aren't the file's. #[test]