Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
401709d
feat(data): publish a layer's edits back to its Shapefile (#1385)
Mavrick-F Nov 13, 2026
6558b39
Merge upstream main into feat/publish-shapefile
Mavrick-F Nov 13, 2026
9203f1b
fix(data): publish refuses deleted records and names each column it c…
Mavrick-F Nov 13, 2026
71e5138
fix(data): publish compares numbers exactly and records the file as i…
Mavrick-F Nov 13, 2026
f84669b
fix(data): finish Shapefile publish fidelity checks
Mavrick-F Nov 13, 2026
a23b146
fix(data): preserve linked sources and shared publisher locks
Mavrick-F Nov 13, 2026
e99726f
fix(publish): report column changes and preserve writable locks
Mavrick-F Nov 13, 2026
350820b
fix(data): publish a defined source projection
Mavrick-F Nov 13, 2026
a3eb74c
fix(data): complete defined projection publishing
Mavrick-F Nov 14, 2026
c266fe6
fix(data): preserve source state across repeated publication
Mavrick-F Nov 14, 2026
f3f7965
fix(data): keep publication notices and undo records
Mavrick-F Nov 14, 2026
2df4556
docs(data): explain Shapefile crash recovery limits
Mavrick-F Nov 14, 2026
b784f91
fix(data): report integer field inference when publishing
Mavrick-F Nov 14, 2026
3bdcd35
Merge main and share the Shapefile numeric reader fix
Mavrick-F Nov 14, 2026
49b1f04
fix(data): preserve recovery files and publication notices
Mavrick-F Nov 14, 2026
4a35e97
fix(data): refuse native reuse of unconverted shapes
Mavrick-F Nov 14, 2026
f3c12cc
fix(data): handle DBF tails, index case and source lock release
Mavrick-F Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions crates/assistant/src/tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ pub fn defs() -> Vec<ToolDef> {
},
ToolDef {
name: "run_command",
description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Each run is one undo step. Commands that read or write files, use the network or replace the project ask the user first.",
description: "Run a RustGIS command by id with JSON parameters (see search_commands for ids and parameters). Project edits are undoable. Source publication and other external file writes cannot be undone. Commands that read or write files, use the network or replace the project ask the user first.",
schema: json!({
"type": "object",
"properties": {
Expand Down Expand Up @@ -648,6 +648,7 @@ fn words(a: &Value) -> Result<Vec<String>, String> {
Ok(words)
}

/// Ranks command descriptions and reports project changes independently of undo snapshots.
fn search_commands(s: &Session, a: &Value) -> Result<Value, String> {
let words = words(a)?;
let limit = limit_arg(a, 12, 40);
Expand All @@ -664,7 +665,7 @@ fn search_commands(s: &Session, a: &Value) -> Result<Value, String> {
(c.id.to_lowercase(), c.label.to_lowercase(), c.location.to_lowercase(), c.params.to_lowercase());
let sc = score(&words, &[&id, &label, &loc, &params]);
(sc > 0).then(|| {
(sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.mutates}))
(sc, json!({"id": c.id, "label": c.label, "location": c.location, "params": c.params, "changesProject": c.changes_project()}))
})
})
.collect();
Expand Down Expand Up @@ -1090,6 +1091,17 @@ mod tests {
assert_eq!(data["[redacted]"], "private field");
}

/// Command discovery distinguishes publishing's project change from a read-only query.
#[test]
fn publishing_reports_a_project_change_in_command_search() {
let s = Session::new(rustgis_engine::Project::default());
for (id, changes) in [("source.publish", true), ("source.syncState", false)] {
let out = search_commands(&s, &json!({"query":id, "limit":40})).unwrap();
let command = out["commands"].as_array().unwrap().iter().find(|c| c["id"] == id).unwrap();
assert_eq!(command["changesProject"], changes, "{id}");
}
}

#[test]
fn wrappers_after_a_prefix_are_made_plain_too() {
let origin = "https://assistant-login.example";
Expand Down
77 changes: 72 additions & 5 deletions crates/data/src/dbf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,72 @@ fn retype_columns(dbf: &mut [u8], from: u8, to: u8) -> Vec<usize> {
changed
}

/// A column of a `.dbf` as its header describes it: its name's bytes (up to the first NUL),
/// type letter, length in bytes and decimals.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Column {
pub name: Vec<u8>,
pub ty: u8,
pub len: u8,
pub decimals: u8,
}

#[cfg(not(target_arch = "wasm32"))]
/// A `.dbf` read as its bytes, without decoding a value: its columns, and each record that isn't
/// marked deleted, as the bytes of each of its columns. `None` when the header doesn't describe
/// records that fit the file.
pub(crate) struct RawTable<'a> {
pub columns: Vec<Column>,
pub records: Vec<Vec<&'a [u8]>>,
/// The records marked deleted, which aren't among `records`.
pub deleted: usize,
}

#[cfg(not(target_arch = "wasm32"))]
/// Reads `dbf` as a [`RawTable`]: what a rewrite is compared with, byte for byte, where a
/// decoded value would hide a difference (text with leading spaces, which a read trims; a code
/// page's bytes).
pub(crate) fn raw_table(dbf: &[u8]) -> Option<RawTable<'_>> {
let le16 = |at: usize| Some(usize::from(u16::from_le_bytes([*dbf.get(at)?, *dbf.get(at + 1)?])));
let count = usize::try_from(u32::from_le_bytes(dbf.get(4..8)?.try_into().ok()?)).ok()?;
let (header_len, record_len) = (le16(8)?, le16(10)?);
let mut columns = Vec::new();
for start in (32..header_len.min(dbf.len())).step_by(32) {
let d = dbf.get(start..start + 32).filter(|d| d.first() != Some(&0x0D));
let Some(d) = d else { break };
let name = d.get(..11)?;
let name = name.get(..name.iter().position(|b| *b == 0).unwrap_or(11))?.to_vec();
columns.push(Column { name, ty: *d.get(11)?, len: *d.get(16)?, decimals: *d.get(17)? });
}
let width: usize = 1 + columns.iter().map(|c| usize::from(c.len)).sum::<usize>();
if width != record_len || record_len == 0 {
return None;
}
let body = dbf.get(header_len..)?;
// Every byte must belong to a counted record or the optional EOF marker. A rewrite
// must not discard uncounted records or other data left past the declared body.
let trailing = body.get(count.checked_mul(record_len)?..)?;
if !trailing.is_empty() && trailing != [0x1a] {
return None;
}
let (mut records, mut deleted) = (Vec::new(), 0);
for record in body.chunks_exact(record_len).take(count) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Records past the header count are dropped without a notice (Never silent, low confidence, about 20%). take(count) ignores whole records after the header's record count. same_table compares only the counted records, so a rewrite that doesn't carry them passes step 2. The .shp check catches the case where those rows have shapes, since the shape counts differ. A row with no shape gets through silently. The end-of-file byte 0x1A is the only normal extra byte, so publish could refuse when more than one record_len of bytes lies past count * record_len, saying how many records are unaccounted for.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reproduced and fixed in f3c12cc6. The raw-table check now refuses any bytes beyond the declared records except the optional single EOF marker. It therefore refuses uncounted records before replacement rather than dropping them. The regression appends a complete DBF record without changing the declared count or Shapefile, edits an attribute and publishes. It previously succeeded; now it reports the malformed record layout and verifies every source part is unchanged.

All 1,194 data tests (11 ignored), the engine publication regression, Clippy, wasm, hooks and full fork CI pass.

if record.first() == Some(&b'*') {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Possible silent loss (Quality, low confidence). Records marked deleted (*) are skipped here, so same_table never compares them. The rewrite then drops them from the .dbf with no notice, unless the shape-count check in same_shapes refuses the file first. I could not confirm how the shapefile crate pairs deleted records with their shapes, so I can't say which of those happens. Please check that a file with a soft-deleted row is refused with a message that names the deleted rows, or that the publish notes them. AGENTS.md requires a notice for dropped records.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 9203f1b: raw_table counts records marked deleted, and publish refuses such a file, saying how many, since writing it again would remove them (shapes and all). Test: a_file_with_deleted_records_is_refused.

deleted += 1;
continue;
}
let mut at = 1;
let mut values = Vec::with_capacity(columns.len());
for c in &columns {
let end = at + usize::from(c.len);
values.push(record.get(at..end)?);
at = end;
}
records.push(values);
}
Some(RawTable { columns, records, deleted })
}

/// A date (`D`) field's value, `YYYYMMDD`: `Ok(None)` when it's blank or zeros, as writers leave
/// a date empty, and `Err(())` when it isn't a date (`20260230`, `2026-3-5`).
pub(crate) fn date(field: &[u8]) -> std::result::Result<Option<crate::date::Date>, ()> {
Expand Down Expand Up @@ -280,17 +346,18 @@ mod tests {
assert!(dates_as_text(&mut [0u8; 10]).is_empty());
}

/// Floating point columns are marked as numeric, so `dbase` reads them as `f64`; others are kept.
/// Float columns are marked as numeric columns; others, and what follows the header, are kept.
#[test]
fn float_columns_read_as_numeric() {
let mut dbf = vec![0u8; 32 + 3 * 32 + 1];
dbf[8] = (dbf.len()) as u8;
for (k, ty) in b"NFf".iter().copied().enumerate() {
let mut dbf = vec![0u8; 32 + 3 * 32 + 2];
dbf[8] = (32 + 3 * 32 + 1) as u8;
for (k, ty) in b"FfC".iter().copied().enumerate() {
dbf[32 + k * 32 + 11] = ty;
}
dbf[32 + 3 * 32] = 0x0D;
dbf[32 + 3 * 32 + 1] = b'F';
floats_as_numeric(&mut dbf);
assert_eq!([dbf[43], dbf[75], dbf[107]], [b'N', b'N', b'N']);
assert_eq!([dbf[43], dbf[75], dbf[107], dbf[129]], [b'N', b'N', b'C', b'F']);
floats_as_numeric(&mut [0u8; 10]);
}

Expand Down
Loading
Loading